@geravant/sinain 1.11.0 → 1.12.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@geravant/sinain",
-  "version": "1.11.0",
+  "version": "1.12.0",
   "description": "Ambient intelligence that sees what you see, hears what you hear, and acts on your behalf",
   "type": "module",
   "bin": {

package/sinain-core/src/escalation/escalator.ts

@@ -237,6 +237,7 @@ export class Escalator {
       escalationReason,
       undefined,
       this.pendingUserCommand ?? undefined,
+      this.deps.wsHandler.getState().responseSize ?? "medium",
     );
 
     // Clear user command after building the message (consumed once)

package/sinain-core/src/escalation/message-builder.ts

@@ -1,4 +1,4 @@
-import type { ContextWindow, AgentEntry, EscalationMode, FeedbackRecord, UserCommand } from "../types.js";
+import type { ContextWindow, AgentEntry, EscalationMode, FeedbackRecord, UserCommand, ResponseSize } from "../types.js";
 import { normalizeAppName } from "../agent/context-window.js";
 import { levelFor, applyLevel } from "../privacy/index.js";
 
@@ -67,11 +67,18 @@ export function isCodingContext(context: ContextWindow): CodingContextResult {
   };
 }
 
-function getInstructions(mode: EscalationMode, context: ContextWindow): string {
+function sizeInstruction(size: ResponseSize): string {
+  switch (size) {
+    case "small": return "1-2 sentences";
+    case "large": return "3-5 sentences";
+    default: return "2-3 sentences";
+  }
+}
+
+function getInstructions(context: ContextWindow): string {
   const { coding, needsSolution } = isCodingContext(context);
 
   if (needsSolution) {
-    // Coding challenge/problem - be very action-oriented
     return `The user is working on a coding problem. Be PROACTIVE and SOLVE IT:
 
 1. Provide a solution approach and working code based on what you can see
@@ -92,13 +99,10 @@ Response should be actionable: working code with brief explanation.`;
 - If it's a non-code file (config, markdown, email): share a relevant insight, action item, or connection to their current project
 - If context is minimal: tell a short clever joke (tech humor — never repeat recent ones)
 
-NEVER just describe what the user is doing. Every response must teach, suggest, or connect dots.
-(2-5 sentences, or more + code if there's an error or code question).`;
+NEVER just describe what the user is doing. Every response must teach, suggest, or connect dots.`;
   }
 
-  // Non-coding context — proactive insights instead of activity descriptions
-  if (mode === "focus" || mode === "rich") {
-    return `Based on the above, ALWAYS provide a useful response for the user's HUD.
+  return `Based on the above, ALWAYS provide a useful response for the user's HUD.
 Important: Do NOT respond with NO_REPLY — a response is always required.
 
 - If there's an error: investigate and suggest a fix
@@ -109,40 +113,25 @@ Important: Do NOT respond with NO_REPLY — a response is always required.
 
 NEVER just describe what the user is doing — they can see their own screen.
 NEVER respond with "standing by", "monitoring", or similar filler.
-Every response must teach something, suggest something, or connect dots the user hasn't noticed.
-(2-5 sentences). Be specific and actionable.`;
-  }
-
-  return `Based on the above, proactively help the user:
-- If there's an error: investigate and suggest a fix
-- If they seem stuck: offer guidance
-- If they're coding: provide relevant insights
-- Keep your response concise and actionable (2-5 sentences)`;
+Every response must teach something, suggest something, or connect dots the user hasn't noticed.`;
 }
 
 /**
- * Build a structured escalation message with richness proportional to the context window preset.
- *
- * Expected message sizes:
- *   lean (selective):  ~7 KB  / ~1,700 tokens
- *   standard (focus):  ~25 KB / ~6,000 tokens
- *   rich:              ~111 KB / ~28,000 tokens
+ * Build a structured escalation message with full context (rich mode).
  *
- * All fit within the 256 KB HTTP hooks limit and 200K+ model context.
- *
- * In selective mode, sections are prioritized by relevance:
- * - Error escalations prioritize error sections
- * - Question escalations prioritize audio sections
- * - App context is always included
+ * Always includes all sections (screen, audio, errors).
+ * Response length is controlled by the `responseSize` parameter (small/medium/large)
+ * which is set by the user via the HUD overlay slider.
  */
 export function buildEscalationMessage(
   digest: string,
   context: ContextWindow,
   entry: AgentEntry,
-  mode: EscalationMode,
+  _mode: EscalationMode,
   escalationReason?: string,
   recentFeedback?: FeedbackRecord[],
   userCommand?: UserCommand,
+  responseSize: ResponseSize = "medium",
 ): string {
   const sections: string[] = [];
 
@@ -167,7 +156,6 @@ export function buildEscalationMessage(
   // Errors — extracted from OCR, full stack traces in rich mode
   const errors = context.screen.filter(e => hasErrorPattern(e.ocr));
   const hasErrors = errors.length > 0;
-  const hasQuestion = escalationReason?.startsWith("question:");
 
   // Privacy levels for agent_gateway destination
   let ocrLevel: import("../types.js").PrivacyLevel = "full";
@@ -183,99 +171,35 @@ export function buildEscalationMessage(
   const applyAudio = (text: string) => applyLevel(text.slice(0, context.preset.maxTranscriptChars), audioLevel, "audio");
   const applyTitle = (title: string | undefined) => title ? applyLevel(title, titlesLevel, "titles") : "";
 
-  // In selective mode, prioritize sections based on escalation reason
-  // In focus/rich modes, include everything
-  if (mode === "selective") {
-    // Error-triggered: prioritize errors, then screen
-    if (hasErrors) {
-      sections.push("## Errors (high priority)");
-      for (const e of errors) {
-        sections.push(`\`\`\`\n${applyOcr(e.ocr)}\n\`\`\``);
-      }
-      // Include screen context (reduced)
-      if (context.screen.length > 0) {
-        sections.push("## Screen (recent OCR)");
-        for (const e of context.screen.slice(0, 5)) { // Limit in selective mode
-          const ago = Math.round((Date.now() - e.ts) / 1000);
-          const app = normalizeAppName(e.meta.app);
-          const title = applyTitle(e.meta.windowTitle);
-          const titlePart = title ? ` [${title}]` : "";
-          sections.push(`- [${ago}s ago] [${app}]${titlePart} ${applyOcr(e.ocr)}`);
-        }
-      }
-    }
-    // Question-triggered: prioritize audio, then screen
-    else if (hasQuestion) {
-      if (context.audio.length > 0) {
-        sections.push("## Audio (recent transcripts)");
-        for (const e of context.audio) {
-          const ago = Math.round((Date.now() - e.ts) / 1000);
-          sections.push(`- [${ago}s ago] "${applyAudio(e.text)}"`);
-        }
-      }
-      // Include screen context (reduced)
-      if (context.screen.length > 0) {
-        sections.push("## Screen (recent OCR)");
-        for (const e of context.screen.slice(0, 5)) {
-          const ago = Math.round((Date.now() - e.ts) / 1000);
-          const app = normalizeAppName(e.meta.app);
-          const title = applyTitle(e.meta.windowTitle);
-          const titlePart = title ? ` [${title}]` : "";
-          sections.push(`- [${ago}s ago] [${app}]${titlePart} ${applyOcr(e.ocr)}`);
-        }
-      }
-    }
-    // Other triggers: balanced sections
-    else {
-      if (context.screen.length > 0) {
-        sections.push("## Screen (recent OCR)");
-        for (const e of context.screen) {
-          const ago = Math.round((Date.now() - e.ts) / 1000);
-          const app = normalizeAppName(e.meta.app);
-          const title = applyTitle(e.meta.windowTitle);
-          const titlePart = title ? ` [${title}]` : "";
-          sections.push(`- [${ago}s ago] [${app}]${titlePart} ${applyOcr(e.ocr)}`);
-        }
-      }
-      if (context.audio.length > 0) {
-        sections.push("## Audio (recent transcripts)");
-        for (const e of context.audio) {
-          const ago = Math.round((Date.now() - e.ts) / 1000);
-          sections.push(`- [${ago}s ago] "${applyAudio(e.text)}"`);
-        }
-      }
-    }
-  } else {
-    // Focus/rich mode: include all sections
-    if (hasErrors) {
-      sections.push("## Errors (high priority)");
-      for (const e of errors) {
-        sections.push(`\`\`\`\n${applyOcr(e.ocr)}\n\`\`\``);
-      }
+  // Always include all sections (rich mode)
+  if (hasErrors) {
+    sections.push("## Errors (high priority)");
+    for (const e of errors) {
+      sections.push(`\`\`\`\n${applyOcr(e.ocr)}\n\`\`\``);
     }
+  }
 
-    if (context.screen.length > 0) {
-      sections.push("## Screen (recent OCR)");
-      for (const e of context.screen) {
-        const ago = Math.round((Date.now() - e.ts) / 1000);
-        const app = normalizeAppName(e.meta.app);
-        const title = applyTitle(e.meta.windowTitle);
-        const titlePart = title ? ` [${title}]` : "";
-        sections.push(`- [${ago}s ago] [${app}]${titlePart} ${applyOcr(e.ocr)}`);
-      }
+  if (context.screen.length > 0) {
+    sections.push("## Screen (recent OCR)");
+    for (const e of context.screen) {
+      const ago = Math.round((Date.now() - e.ts) / 1000);
+      const app = normalizeAppName(e.meta.app);
+      const title = applyTitle(e.meta.windowTitle);
+      const titlePart = title ? ` [${title}]` : "";
+      sections.push(`- [${ago}s ago] [${app}]${titlePart} ${applyOcr(e.ocr)}`);
     }
+  }
 
-    if (context.audio.length > 0) {
-      sections.push("## Audio (recent transcripts)");
-      for (const e of context.audio) {
-        const ago = Math.round((Date.now() - e.ts) / 1000);
-        sections.push(`- [${ago}s ago] "${applyAudio(e.text)}"`);
-      }
+  if (context.audio.length > 0) {
+    sections.push("## Audio (recent transcripts)");
+    for (const e of context.audio) {
+      const ago = Math.round((Date.now() - e.ts) / 1000);
+      sections.push(`- [${ago}s ago] "${applyAudio(e.text)}"`);
     }
   }
 
-  // Mode-specific instructions (now context-aware)
-  sections.push(getInstructions(mode, context));
+  // Context-aware instructions (no size — that's in the response length section below)
+  sections.push(getInstructions(context));
 
   // Stale escalation hint — forces a proactive response after prolonged silence
   if (escalationReason === "stale") {
@@ -293,7 +217,10 @@ the local analyzer reported idle/no-change. Provide a PROACTIVE response:
     sections.push(formatInlineFeedback(recentFeedback));
   }
 
-  sections.push("Respond naturally — this will appear on the user's HUD overlay.");
+  // Response length — single authoritative size instruction, placed last for salience
+  const limit = sizeInstruction(responseSize);
+  sections.push(`## Response Length
+Your response MUST be ${limit}. This appears on the user's HUD overlay — be specific and actionable.`);
 
   return sections.join("\n\n");
 }

package/sinain-core/src/overlay/commands.ts

@@ -1,5 +1,5 @@
 import { execFile } from "node:child_process";
-import type { InboundMessage } from "../types.js";
+import type { InboundMessage, ResponseSize } from "../types.js";
 import type { WsHandler } from "./ws-handler.js";
 import type { AudioPipeline } from "../audio/pipeline.js";
 import type { CoreConfig } from "../types.js";
@@ -101,7 +101,7 @@ export function setupCommands(deps: CommandDeps): void {
         break;
       }
       case "command": {
-        handleCommand(msg.action, deps);
+        handleCommand(msg, deps);
         log(TAG, `command processed: ${msg.action}`);
         break;
       }
@@ -109,8 +109,11 @@ export function setupCommands(deps: CommandDeps): void {
   });
 }
 
-function handleCommand(action: string, deps: CommandDeps): void {
+const VALID_RESPONSE_SIZES = new Set<ResponseSize>(["small", "medium", "large"]);
+
+function handleCommand(msg: InboundMessage & { action: string }, deps: CommandDeps): void {
   const { wsHandler, systemAudioPipeline, micPipeline } = deps;
+  const action = msg.action;
 
   switch (action) {
     case "toggle_audio": {
@@ -173,6 +176,16 @@ function handleCommand(action: string, deps: CommandDeps): void {
       log(TAG, `escalation toggled ${nowActive ? "ON" : "OFF"}`);
       break;
     }
+    case "set_response_size": {
+      const size = (msg as any).responseSize as string;
+      if (VALID_RESPONSE_SIZES.has(size as ResponseSize)) {
+        wsHandler.updateState({ responseSize: size as ResponseSize });
+        log(TAG, `response size set to ${size}`);
+      } else {
+        log(TAG, `invalid response size: ${size}`);
+      }
+      break;
+    }
     case "open_settings": {
       const envPath = loadedEnvPath || `${process.env.HOME || process.env.USERPROFILE}/.sinain/.env`;
       const cmd = process.platform === "win32" ? "notepad" : "open";

package/sinain-core/src/overlay/ws-handler.ts

@@ -39,6 +39,7 @@ export class WsHandler {
     screen: "off",
     escalation: "active",
     connection: "disconnected",
+    responseSize: "medium",
   };
   private replayBuffer: FeedMessage[] = [];
   private spawnTaskBuffer: Map<string, SpawnTaskMessage> = new Map();
@@ -75,6 +76,7 @@ export class WsHandler {
       screen: this.state.screen,
       escalation: this.state.escalation,
       connection: this.state.connection,
+      responseSize: this.state.responseSize,
     });
 
     // Replay recent feed messages for late-joining clients
@@ -151,13 +153,14 @@ export class WsHandler {
 
   /** Send a status update to all connected overlays. */
   broadcastStatus(): void {
-    const msg: StatusMessage & { envPath?: string; escalation?: string } = {
+    const msg: StatusMessage & { envPath?: string; escalation?: string; responseSize?: string } = {
       type: "status",
       audio: this.state.audio,
       mic: this.state.mic,
       screen: this.state.screen,
       escalation: this.state.escalation,
       connection: this.state.connection,
+      responseSize: this.state.responseSize,
     };
     if (loadedEnvPath) msg.envPath = loadedEnvPath;
     this.broadcastMessage(msg);

package/sinain-core/src/types.ts

@@ -20,6 +20,7 @@ export interface StatusMessage {
   screen: string;
   escalation?: string;
   connection: string;
+  responseSize?: string;
 }
 
 /** sinain-core → Overlay: heartbeat ping */
@@ -244,6 +245,7 @@ export interface StopResult {
 
 export type EscalationMode = "off" | "selective" | "focus" | "rich";
 export type ContextRichness = "lean" | "standard" | "rich";
+export type ResponseSize = "small" | "medium" | "large";
 
 export type AnalysisProvider = "openrouter" | "ollama";
 
@@ -393,6 +395,7 @@ export interface BridgeState {
   screen: "active" | "off";
   escalation: "active" | "paused";
   connection: "connected" | "disconnected" | "connecting";
+  responseSize: ResponseSize;
 }
 
 // ── Learning / feedback types ──

package/sinain-memory/eval/benchmarks/base_adapter.py

@@ -0,0 +1,43 @@
+"""Base adapter and data classes for benchmark evaluation."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+
+
+@dataclass
+class BenchmarkQuestion:
+    id: str
+    text: str
+    gold_answer: str
+    category: str  # single-session, multi-session, temporal, etc.
+    evidence_session_ids: list[str] = field(default_factory=list)
+    metadata: dict = field(default_factory=dict)
+
+
+@dataclass
+class BenchmarkInstance:
+    """A set of conversations + questions that share the same context."""
+    id: str
+    sessions: list[list[dict]]  # list of sessions, each a list of feed items {source, text, ts}
+    questions: list[BenchmarkQuestion] = field(default_factory=list)
+    raw_sessions: list[dict] = field(default_factory=list)  # original benchmark format (for full-context condition)
+    metadata: dict = field(default_factory=dict)
+
+
+class BenchmarkAdapter(ABC):
+    """Abstract adapter: converts a published benchmark into sinain's format."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Benchmark name (e.g. 'longmemeval', 'locomo')."""
+
+    @abstractmethod
+    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
+        """Download (if needed) and parse the benchmark dataset."""
+
+    @abstractmethod
+    def format_full_context(self, instance: BenchmarkInstance) -> str:
+        """Render the full conversation history as a text string for the baseline condition."""

package/sinain-memory/eval/benchmarks/config.py

@@ -0,0 +1,23 @@
+"""Benchmark configuration — models, paths, thresholds."""
+
+from pathlib import Path
+
+BENCHMARKS_DIR = Path(__file__).resolve().parent
+DATA_DIR = BENCHMARKS_DIR / "data"
+RESULTS_DIR = BENCHMARKS_DIR / "results"
+
+# LLM models (via OpenRouter)
+QA_MODEL = "google/gemini-2.5-flash"
+JUDGE_MODEL = "openai/gpt-4o"
+
+# Retrieval
+K_VALUES = [1, 3, 5, 10]
+MAX_FACTS_PER_QUERY = 10
+
+# Ingestion
+DISTILLER_TIMEOUT_S = 30
+INTEGRATOR_TIMEOUT_S = 60
+
+# Dataset URLs
+LONGMEMEVAL_HF = "xiaowu0162/longmemeval-cleaned"
+LOCOMO_GITHUB = "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json"

package/sinain-memory/eval/benchmarks/evaluate.py

@@ -0,0 +1,146 @@
+"""Evaluation pipeline — score answers and compute aggregate metrics.
+
+Combines:
+- LLM-as-Judge (QA scoring, 1-5 scale)
+- Retrieval metrics (Recall@k, NDCG@k)
+- Token F1 overlap (mechanical, free)
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import defaultdict
+
+from .base_adapter import BenchmarkQuestion
+from .config import K_VALUES
+
+
+# ── Token F1 (mechanical, no LLM needed) ─────────────────────────────────────
+
+def _tokenize(text: str) -> list[str]:
+    """Simple whitespace + punctuation tokenizer."""
+    return re.findall(r"\w+", text.lower())
+
+
+def token_f1(predicted: str, gold: str | int) -> float:
+    """Compute token-level F1 between predicted and gold answers."""
+    pred_tokens = set(_tokenize(str(predicted)))
+    gold_tokens = set(_tokenize(str(gold)))
+    if not gold_tokens or not pred_tokens:
+        return 0.0
+    overlap = pred_tokens & gold_tokens
+    if not overlap:
+        return 0.0
+    precision = len(overlap) / len(pred_tokens)
+    recall = len(overlap) / len(gold_tokens)
+    return 2 * precision * recall / (precision + recall)
+
+
+# ── Retrieval metrics (reuse logic from retrieval_evaluator.py) ───────────────
+
+def dcg_at_k(relevant_positions: list[int], k: int) -> float:
+    """Discounted Cumulative Gain at k."""
+    score = 0.0
+    for pos in relevant_positions:
+        if pos < k:
+            score += 1.0 / math.log2(pos + 2)
+    return score
+
+
+def ndcg_at_k(relevant_positions: list[int], num_relevant: int, k: int) -> float:
+    """Normalized DCG at k."""
+    dcg = dcg_at_k(relevant_positions, k)
+    ideal_positions = list(range(min(num_relevant, k)))
+    idcg = dcg_at_k(ideal_positions, k)
+    return dcg / idcg if idcg > 0 else 0.0
+
+
+def compute_retrieval_metrics(
+    retrieved_ids: list[str],
+    expected_ids: list[str],
+    k_values: list[int] | None = None,
+) -> dict:
+    """Compute Recall@k and NDCG@k for a single question."""
+    ks = k_values or K_VALUES
+    expected_set = set(expected_ids)
+    relevant_positions = [i for i, rid in enumerate(retrieved_ids) if rid in expected_set]
+
+    result = {}
+    for k in ks:
+        hit = any(pos < k for pos in relevant_positions)
+        result[f"recall@{k}"] = 1.0 if hit else 0.0
+        result[f"ndcg@{k}"] = ndcg_at_k(relevant_positions, len(expected_set), k)
+    return result
+
+
+# ── Aggregate metrics ─────────────────────────────────────────────────────────
+
+def aggregate_results(per_question: list[dict]) -> dict:
+    """Compute aggregate metrics from per-question results.
+
+    Each per_question entry has:
+      {id, category, retrieval: {recall@k, ndcg@k}, answers: {condition: {score, f1}}}
+    """
+    if not per_question:
+        return {"error": "no results"}
+
+    # Per-condition scores
+    condition_scores: dict[str, list[float]] = defaultdict(list)
+    condition_f1s: dict[str, list[float]] = defaultdict(list)
+    # Per-category per-condition
+    cat_scores: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
+    # Retrieval
+    retrieval_metrics: dict[str, list[float]] = defaultdict(list)
+
+    for q in per_question:
+        cat = q.get("category", "unknown")
+
+        for cond, data in q.get("answers", {}).items():
+            if data.get("score") is not None:
+                condition_scores[cond].append(data["score"])
+                cat_scores[cat][cond].append(data["score"])
+            if data.get("f1") is not None:
+                condition_f1s[cond].append(data["f1"])
+
+        for metric, val in q.get("retrieval", {}).items():
+            if isinstance(val, (int, float)):
+                retrieval_metrics[metric].append(val)
+
+    def _mean(lst: list[float]) -> float:
+        return round(sum(lst) / len(lst), 4) if lst else 0.0
+
+    # Build summary
+    conditions = {}
+    for cond in sorted(condition_scores):
+        conditions[cond] = {
+            "mean_score": _mean(condition_scores[cond]),
+            "mean_f1": _mean(condition_f1s.get(cond, [])),
+            "n": len(condition_scores[cond]),
+        }
+
+    # IPR: sinain-memory vs full-context
+    sm_scores = condition_scores.get("sinain-memory", [])
+    fc_scores = condition_scores.get("full-context", [])
+    ipr = _mean(sm_scores) / _mean(fc_scores) if fc_scores and _mean(fc_scores) > 0 else None
+
+    # Category breakdown
+    categories = {}
+    for cat in sorted(cat_scores):
+        categories[cat] = {}
+        for cond in sorted(cat_scores[cat]):
+            categories[cat][cond] = {
+                "mean_score": _mean(cat_scores[cat][cond]),
+                "n": len(cat_scores[cat][cond]),
+            }
+
+    # Retrieval summary
+    retrieval = {k: _mean(v) for k, v in sorted(retrieval_metrics.items())}
+
+    return {
+        "total_questions": len(per_question),
+        "conditions": conditions,
+        "ipr": round(ipr, 4) if ipr else None,
+        "categories": categories,
+        "retrieval": retrieval,
+    }