@geravant/sinain 1.10.1 → 1.12.0

package/sinain-core/src/types.ts

@@ -18,7 +18,9 @@ export interface StatusMessage {
   audio: string;
   mic: string;
   screen: string;
+  escalation?: string;
   connection: string;
+  responseSize?: string;
 }
 
 /** sinain-core → Overlay: heartbeat ping */
@@ -28,7 +30,7 @@ export interface PingMessage {
 }
 
 /** sinain-core → Overlay: spawn task lifecycle update */
-export type SpawnTaskStatus = "spawned" | "polling" | "completed" | "failed" | "timeout";
+export type SpawnTaskStatus = "spawned" | "polling" | "completed" | "failed" | "timeout" | "awaiting_input" | "awaiting_permission";
 
 export interface SpawnTaskMessage {
   type: "spawn_task";
@@ -38,6 +40,10 @@ export interface SpawnTaskMessage {
   startedAt: number;
   completedAt?: number;
   resultPreview?: string;
+  /** Question the spawn is asking the user (status=awaiting_input) */
+  question?: string;
+  /** Tool permission request (status=awaiting_permission) */
+  permission?: { tool: string; input: Record<string, unknown> };
 }
 
 /** Overlay → sinain-core: user typed a message */
@@ -78,6 +84,20 @@ export interface SpawnCommandMessage {
   text: string;
 }
 
+/** Overlay → sinain-core: reply to a spawn question */
+export interface SpawnReplyMessage {
+  type: "spawn_reply";
+  taskId: string;
+  text: string;
+}
+
+/** Overlay → sinain-core: reply to a spawn permission request */
+export interface SpawnPermissionReplyMessage {
+  type: "spawn_permission_reply";
+  taskId: string;
+  decision: "allow" | "deny";
+}
+
 /** Cost update broadcast to overlay. */
 export interface CostMessage {
   type: "cost";
@@ -108,7 +128,7 @@ export interface CostSnapshot {
 }
 
 export type OutboundMessage = FeedMessage | StatusMessage | PingMessage | SpawnTaskMessage | CostMessage;
-export type InboundMessage = UserMessage | CommandMessage | PongMessage | ProfilingMessage | UserCommandMessage | SpawnCommandMessage;
+export type InboundMessage = UserMessage | CommandMessage | PongMessage | ProfilingMessage | UserCommandMessage | SpawnCommandMessage | SpawnReplyMessage | SpawnPermissionReplyMessage;
 
 /** Abstraction for user commands (text now, voice later). */
 export interface UserCommand {
@@ -163,27 +183,6 @@ export interface AudioPipelineConfig {
   gainDb: number;
 }
 
-export interface TraitConfig {
-  enabled: boolean;
-  configPath: string;   // path to ~/.sinain/traits.json
-  entropyHigh: boolean; // Phase 2: boosts entropy roll to 15%
-  logDir: string;       // path to ~/.sinain-core/traits/
-}
-
-export interface TraitLogEntry {
-  ts: string;
-  tickId: number;
-  enabled: boolean;
-  voice: string;
-  voice_stat: number;
-  voice_confidence: number;
-  activation_scores: Record<string, number>;
-  context_app: string;
-  hud_length: number;
-  synthesis: boolean;
-}
-
-
 export interface AudioChunk {
   buffer: Buffer;
   source: string;
@@ -246,6 +245,7 @@ export interface StopResult {
 
 export type EscalationMode = "off" | "selective" | "focus" | "rich";
 export type ContextRichness = "lean" | "standard" | "rich";
+export type ResponseSize = "small" | "medium" | "large";
 
 export type AnalysisProvider = "openrouter" | "ollama";
 
@@ -282,9 +282,6 @@ export interface AgentResult {
   tokensOut: number;
   model: string;
   parsedOk: boolean;
-  voice?: string;
-  voice_stat?: number;
-  voice_confidence?: number;
   /** Actual USD cost returned by OpenRouter (undefined if not available). */
   cost?: number;
 }
@@ -396,8 +393,9 @@ export interface BridgeState {
   audio: "active" | "muted";
   mic: "active" | "muted";
   screen: "active" | "off";
-  traits?: "active" | "off";
+  escalation: "active" | "paused";
   connection: "connected" | "disconnected" | "connecting";
+  responseSize: ResponseSize;
 }
 
 // ── Learning / feedback types ──
@@ -479,6 +477,5 @@ export interface CoreConfig {
   costDisplayEnabled: boolean;
   traceDir: string;
   learningConfig: LearningConfig;
-  traitConfig: TraitConfig;
   privacyConfig: PrivacyConfig;
 }

package/sinain-mcp-server/index.ts

@@ -451,6 +451,34 @@ server.tool(
   },
 );
 
+// 15. sinain_ask_user — blocking question to the user via overlay
+server.tool(
+  "sinain_ask_user",
+  "Ask the user a question and wait for their reply. Use when you need clarification, confirmation, or a decision. The question appears on the user's HUD overlay and blocks until they respond.",
+  {
+    question: z.string().describe("The question to ask the user"),
+  },
+  async ({ question }) => {
+    // Use the spawn task ID from the environment if available
+    const taskId = process.env.SINAIN_SPAWN_TASK_ID || `ask-${Date.now()}`;
+    try {
+      const resp = await fetch(`${SINAIN_CORE_URL}/spawn/ask`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ taskId, question }),
+        signal: AbortSignal.timeout(6 * 60_000), // 6 min (server times out at 5)
+      });
+      const data = await resp.json() as { ok: boolean; answer?: string };
+      if (data.ok && data.answer) {
+        return textResult(`User replied: ${data.answer}`);
+      }
+      return textResult("User did not reply.");
+    } catch (err: any) {
+      return textResult(`Failed to ask user: ${err.message}`);
+    }
+  },
+);
+
 // ---------------------------------------------------------------------------
 // Startup
 // ---------------------------------------------------------------------------

package/sinain-memory/eval/assertions.py

@@ -206,27 +206,6 @@ def assert_playbook_header_footer_intact(playbook_text: str) -> dict:
                    f"missing playbook comments: {', '.join(missing)}")
 
 
-# ---------------------------------------------------------------------------
-# Trait voice assertions (sinain-core wiring verification)
-# ---------------------------------------------------------------------------
-
-def assert_situation_has_active_voice(
-    situation_content: str, expected_trait: str | None = None
-) -> dict:
-    """Check SITUATION.md contains an Active Voice section (after trait wiring).
-
-    Called by tick_evaluator.py when processing live ticks that have SITUATION.md
-    content and a trait was selected for that tick.
-    """
-    has_section = "## Active Voice" in situation_content
-    if not has_section:
-        return _result("situation_has_active_voice", False, "no '## Active Voice' section")
-    if expected_trait and expected_trait not in situation_content:
-        return _result("situation_has_active_voice", False,
-                       f"section present but '{expected_trait}' not found")
-    return _result("situation_has_active_voice", True, "Active Voice section present")
-
-
 # ---------------------------------------------------------------------------
 # Runner: execute all applicable assertions for a tick
 # ---------------------------------------------------------------------------

package/sinain-memory/eval/benchmarks/base_adapter.py

@@ -0,0 +1,43 @@
+"""Base adapter and data classes for benchmark evaluation."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+
+
+@dataclass
+class BenchmarkQuestion:
+    id: str
+    text: str
+    gold_answer: str
+    category: str  # single-session, multi-session, temporal, etc.
+    evidence_session_ids: list[str] = field(default_factory=list)
+    metadata: dict = field(default_factory=dict)
+
+
+@dataclass
+class BenchmarkInstance:
+    """A set of conversations + questions that share the same context."""
+    id: str
+    sessions: list[list[dict]]  # list of sessions, each a list of feed items {source, text, ts}
+    questions: list[BenchmarkQuestion] = field(default_factory=list)
+    raw_sessions: list[dict] = field(default_factory=list)  # original benchmark format (for full-context condition)
+    metadata: dict = field(default_factory=dict)
+
+
+class BenchmarkAdapter(ABC):
+    """Abstract adapter: converts a published benchmark into sinain's format."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Benchmark name (e.g. 'longmemeval', 'locomo')."""
+
+    @abstractmethod
+    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
+        """Download (if needed) and parse the benchmark dataset."""
+
+    @abstractmethod
+    def format_full_context(self, instance: BenchmarkInstance) -> str:
+        """Render the full conversation history as a text string for the baseline condition."""

package/sinain-memory/eval/benchmarks/config.py

@@ -0,0 +1,23 @@
+"""Benchmark configuration — models, paths, thresholds."""
+
+from pathlib import Path
+
+BENCHMARKS_DIR = Path(__file__).resolve().parent
+DATA_DIR = BENCHMARKS_DIR / "data"
+RESULTS_DIR = BENCHMARKS_DIR / "results"
+
+# LLM models (via OpenRouter)
+QA_MODEL = "google/gemini-2.5-flash"
+JUDGE_MODEL = "openai/gpt-4o"
+
+# Retrieval
+K_VALUES = [1, 3, 5, 10]
+MAX_FACTS_PER_QUERY = 10
+
+# Ingestion
+DISTILLER_TIMEOUT_S = 30
+INTEGRATOR_TIMEOUT_S = 60
+
+# Dataset URLs
+LONGMEMEVAL_HF = "xiaowu0162/longmemeval-cleaned"
+LOCOMO_GITHUB = "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json"

package/sinain-memory/eval/benchmarks/evaluate.py

@@ -0,0 +1,146 @@
+"""Evaluation pipeline — score answers and compute aggregate metrics.
+
+Combines:
+- LLM-as-Judge (QA scoring, 1-5 scale)
+- Retrieval metrics (Recall@k, NDCG@k)
+- Token F1 overlap (mechanical, free)
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import defaultdict
+
+from .base_adapter import BenchmarkQuestion
+from .config import K_VALUES
+
+
+# ── Token F1 (mechanical, no LLM needed) ─────────────────────────────────────
+
+def _tokenize(text: str) -> list[str]:
+    """Simple whitespace + punctuation tokenizer."""
+    return re.findall(r"\w+", text.lower())
+
+
+def token_f1(predicted: str, gold: str | int) -> float:
+    """Compute token-level F1 between predicted and gold answers."""
+    pred_tokens = set(_tokenize(str(predicted)))
+    gold_tokens = set(_tokenize(str(gold)))
+    if not gold_tokens or not pred_tokens:
+        return 0.0
+    overlap = pred_tokens & gold_tokens
+    if not overlap:
+        return 0.0
+    precision = len(overlap) / len(pred_tokens)
+    recall = len(overlap) / len(gold_tokens)
+    return 2 * precision * recall / (precision + recall)
+
+
+# ── Retrieval metrics (reuse logic from retrieval_evaluator.py) ───────────────
+
+def dcg_at_k(relevant_positions: list[int], k: int) -> float:
+    """Discounted Cumulative Gain at k."""
+    score = 0.0
+    for pos in relevant_positions:
+        if pos < k:
+            score += 1.0 / math.log2(pos + 2)
+    return score
+
+
+def ndcg_at_k(relevant_positions: list[int], num_relevant: int, k: int) -> float:
+    """Normalized DCG at k."""
+    dcg = dcg_at_k(relevant_positions, k)
+    ideal_positions = list(range(min(num_relevant, k)))
+    idcg = dcg_at_k(ideal_positions, k)
+    return dcg / idcg if idcg > 0 else 0.0
+
+
+def compute_retrieval_metrics(
+    retrieved_ids: list[str],
+    expected_ids: list[str],
+    k_values: list[int] | None = None,
+) -> dict:
+    """Compute Recall@k and NDCG@k for a single question."""
+    ks = k_values or K_VALUES
+    expected_set = set(expected_ids)
+    relevant_positions = [i for i, rid in enumerate(retrieved_ids) if rid in expected_set]
+
+    result = {}
+    for k in ks:
+        hit = any(pos < k for pos in relevant_positions)
+        result[f"recall@{k}"] = 1.0 if hit else 0.0
+        result[f"ndcg@{k}"] = ndcg_at_k(relevant_positions, len(expected_set), k)
+    return result
+
+
+# ── Aggregate metrics ─────────────────────────────────────────────────────────
+
+def aggregate_results(per_question: list[dict]) -> dict:
+    """Compute aggregate metrics from per-question results.
+
+    Each per_question entry has:
+      {id, category, retrieval: {recall@k, ndcg@k}, answers: {condition: {score, f1}}}
+    """
+    if not per_question:
+        return {"error": "no results"}
+
+    # Per-condition scores
+    condition_scores: dict[str, list[float]] = defaultdict(list)
+    condition_f1s: dict[str, list[float]] = defaultdict(list)
+    # Per-category per-condition
+    cat_scores: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
+    # Retrieval
+    retrieval_metrics: dict[str, list[float]] = defaultdict(list)
+
+    for q in per_question:
+        cat = q.get("category", "unknown")
+
+        for cond, data in q.get("answers", {}).items():
+            if data.get("score") is not None:
+                condition_scores[cond].append(data["score"])
+                cat_scores[cat][cond].append(data["score"])
+            if data.get("f1") is not None:
+                condition_f1s[cond].append(data["f1"])
+
+        for metric, val in q.get("retrieval", {}).items():
+            if isinstance(val, (int, float)):
+                retrieval_metrics[metric].append(val)
+
+    def _mean(lst: list[float]) -> float:
+        return round(sum(lst) / len(lst), 4) if lst else 0.0
+
+    # Build summary
+    conditions = {}
+    for cond in sorted(condition_scores):
+        conditions[cond] = {
+            "mean_score": _mean(condition_scores[cond]),
+            "mean_f1": _mean(condition_f1s.get(cond, [])),
+            "n": len(condition_scores[cond]),
+        }
+
+    # IPR: sinain-memory vs full-context
+    sm_scores = condition_scores.get("sinain-memory", [])
+    fc_scores = condition_scores.get("full-context", [])
+    ipr = _mean(sm_scores) / _mean(fc_scores) if fc_scores and _mean(fc_scores) > 0 else None
+
+    # Category breakdown
+    categories = {}
+    for cat in sorted(cat_scores):
+        categories[cat] = {}
+        for cond in sorted(cat_scores[cat]):
+            categories[cat][cond] = {
+                "mean_score": _mean(cat_scores[cat][cond]),
+                "n": len(cat_scores[cat][cond]),
+            }
+
+    # Retrieval summary
+    retrieval = {k: _mean(v) for k, v in sorted(retrieval_metrics.items())}
+
+    return {
+        "total_questions": len(per_question),
+        "conditions": conditions,
+        "ipr": round(ipr, 4) if ipr else None,
+        "categories": categories,
+        "retrieval": retrieval,
+    }

package/sinain-memory/eval/benchmarks/ingest.py

@@ -0,0 +1,152 @@
+"""Ingestion pipeline — benchmark conversations → sinain triplestore.
+
+Runs session_distiller.py + knowledge_integrator.py via subprocess (exact production path).
+Caches results aggressively to avoid repeated LLM calls.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import tempfile
+from pathlib import Path
+from subprocess import run, PIPE, TimeoutExpired
+
+from .base_adapter import BenchmarkInstance
+from .config import DISTILLER_TIMEOUT_S, INTEGRATOR_TIMEOUT_S
+
+
+def _scripts_dir() -> Path:
+    """Locate sinain-memory scripts directory."""
+    return Path(__file__).resolve().parent.parent.parent
+
+
+def _content_hash(sessions: list[list[dict]]) -> str:
+    """Hash session content for caching."""
+    raw = json.dumps(sessions, sort_keys=True, ensure_ascii=False)
+    return hashlib.sha256(raw.encode()).hexdigest()[:16]
+
+
+def _run_script(script_name: str, args: list[str], timeout: int) -> str | None:
+    """Run a Python script from sinain-memory, return stdout or None on failure."""
+    script_path = _scripts_dir() / script_name
+    if not script_path.exists():
+        print(f"[ingest] {script_name} not found at {script_path}")
+        return None
+
+    env = {**os.environ, "PYTHONPATH": str(_scripts_dir())}
+    # Ensure a working model is available (common.py defaults may reference unreleased models)
+    if "SINAIN_BENCH_MODEL" in os.environ:
+        env["SINAIN_FAST_MODEL"] = os.environ["SINAIN_BENCH_MODEL"]
+    try:
+        result = run(
+            ["python3", str(script_path)] + args,
+            capture_output=True, text=True, timeout=timeout, env=env,
+        )
+        if result.returncode != 0:
+            print(f"[ingest] {script_name} failed: {result.stderr[:200]}")
+            return None
+        return result.stdout.strip()
+    except TimeoutExpired:
+        print(f"[ingest] {script_name} timed out ({timeout}s)")
+        return None
+
+
+def ingest_instance(
+    instance: BenchmarkInstance,
+    cache_dir: Path,
+) -> Path | None:
+    """Ingest a benchmark instance into a triplestore. Returns db_path or None.
+
+    Uses caching: if the same haystack was already ingested, returns the cached DB.
+    """
+    ch = _content_hash(instance.sessions)
+    cache_path = cache_dir / "stores" / f"{ch}.db"
+
+    if cache_path.exists():
+        return cache_path
+
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Create temp memory directory
+    tmp = tempfile.mkdtemp(prefix="sinain-bench-")
+    mem_dir = Path(tmp) / "memory"
+    for subdir in ["", "playbook-logs", "playbook-archive"]:
+        (mem_dir / subdir).mkdir(parents=True, exist_ok=True)
+
+    # Write a minimal playbook so integrator doesn't fail
+    (mem_dir / "sinain-playbook.md").write_text("# Sinain Playbook\n\n(benchmark run)\n")
+
+    success = False
+    try:
+        # Batch sessions into chunks of ~10 for fewer LLM calls.
+        # Each chunk becomes one distiller call with a combined transcript.
+        BATCH_SIZE = 10
+        num_sessions = len(instance.sessions)
+        batch_idx = 0
+
+        for start in range(0, num_sessions, BATCH_SIZE):
+            batch = instance.sessions[start:start + BATCH_SIZE]
+            # Flatten batch into one transcript
+            combined: list[dict] = []
+            for session in batch:
+                combined.extend(session)
+            if len(combined) < 3:
+                continue
+
+            first_ts = combined[0].get("ts", "2025-01-01T10:00:00Z")
+            meta = json.dumps({
+                "ts": first_ts,
+                "sessionKey": f"benchmark-batch-{batch_idx}",
+                "durationMs": len(combined) * 30000,
+            })
+            batch_idx += 1
+
+            # Step 1: Distill the batch
+            digest_json = _run_script("session_distiller.py", [
+                "--memory-dir", str(mem_dir),
+                "--transcript", json.dumps(combined),
+                "--session-meta", meta,
+            ], DISTILLER_TIMEOUT_S)
+
+            if not digest_json:
+                continue
+
+            try:
+                digest = json.loads(digest_json)
+            except json.JSONDecodeError:
+                continue
+
+            if digest.get("isEmpty") or digest.get("error"):
+                continue
+
+            # Step 2: Integrate into knowledge graph
+            _run_script("knowledge_integrator.py", [
+                "--memory-dir", str(mem_dir),
+                "--digest", json.dumps(digest),
+            ], INTEGRATOR_TIMEOUT_S)
+
+        # Copy the resulting DB to cache
+        db_path = mem_dir / "knowledge-graph.db"
+        if db_path.exists() and db_path.stat().st_size > 0:
+            shutil.copy2(db_path, cache_path)
+            success = True
+
+    finally:
+        shutil.rmtree(tmp, ignore_errors=True)
+
+    return cache_path if success else None
+
+
+def get_knowledge_doc(db_path: Path) -> str:
+    """Render a sinain-knowledge.md style document from a triplestore."""
+    import sys
+    sys.path.insert(0, str(_scripts_dir()))
+    from graph_query import query_top_facts, format_facts_text
+
+    facts = query_top_facts(str(db_path), limit=30)
+    if not facts:
+        return "(no knowledge available)"
+    return format_facts_text(facts, max_chars=6000)

package/sinain-memory/eval/benchmarks/judges/qa_judge.py

@@ -0,0 +1,81 @@
+"""LLM-as-Judge: QA answer quality evaluator (LongMemEval-compatible, 1-5 scale).
+
+Uses GPT-4o via OpenRouter for comparability with published results.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+# Add sinain-memory to path for common imports
+_koog_dir = str(Path(__file__).resolve().parent.parent.parent.parent)
+if _koog_dir not in sys.path:
+    sys.path.insert(0, _koog_dir)
+
+from common import LLMError, call_llm, extract_json  # noqa: E402
+
+SYSTEM_PROMPT = """\
+You are evaluating whether a predicted answer correctly answers a question.
+The gold (reference) answer is provided.
+
+Score on a scale of 1-5:
+  5: Perfect — captures all key information from the gold answer, no errors
+  4: Mostly correct — minor omissions or imprecision, main point is right
+  3: Partially correct — captures some key points but misses important details
+  2: Related but mostly wrong — touches the topic but answer is largely incorrect
+  1: Completely wrong, contradicts the gold answer, or says "I don't know" when the answer exists
+
+Special cases:
+- If the gold answer indicates abstention is correct (e.g. "I don't know" or "not mentioned"),
+  then a predicted "I don't know" scores 5.
+- Numeric answers within 10% of gold = full credit.
+- Getting the gist right but missing specifics = 3-4 depending on importance.
+
+Respond with ONLY a JSON object: {"score": <1-5>, "reasoning": "brief explanation"}"""
+
+
+def judge_qa(
+    question: str,
+    gold_answer: str,
+    predicted_answer: str,
+    *,
+    condition: str = "",
+    model: str | None = None,
+) -> dict | None:
+    """Score a QA answer. Returns {"score": 1-5, "reasoning": str} or None on failure."""
+    user_parts = [
+        f"## Question\n{question}",
+        f"\n## Gold Answer\n{gold_answer}",
+        f"\n## Predicted Answer\n{predicted_answer}",
+    ]
+    if condition:
+        user_parts.append(f"\n## Context Condition: {condition}")
+
+    try:
+        kwargs: dict = {
+            "system_prompt": SYSTEM_PROMPT,
+            "user_prompt": "\n".join(user_parts),
+            "max_tokens": 200,
+            "json_mode": True,
+        }
+        if model:
+            kwargs["model"] = model
+        else:
+            kwargs["script"] = "meeting_benchmark"
+
+        raw = call_llm(**kwargs)
+        result = extract_json(raw)
+
+        score = result.get("score")
+        reasoning = result.get("reasoning", "")
+
+        if not isinstance(score, (int, float)) or not (1 <= score <= 5):
+            print(f"[warn] qa_judge returned invalid score: {score}", file=sys.stderr)
+            return None
+
+        return {"score": int(score), "reasoning": str(reasoning)[:300]}
+
+    except (ValueError, LLMError, KeyError) as e:
+        print(f"[warn] qa_judge call failed: {e}", file=sys.stderr)
+        return None