@geravant/sinain 1.11.0 → 1.13.0

package/sinain-core/src/overlay/ws-handler.ts

@@ -39,6 +39,7 @@ export class WsHandler {
     screen: "off",
     escalation: "active",
     connection: "disconnected",
+    responseSize: "medium",
   };
   private replayBuffer: FeedMessage[] = [];
   private spawnTaskBuffer: Map<string, SpawnTaskMessage> = new Map();
@@ -75,6 +76,7 @@ export class WsHandler {
       screen: this.state.screen,
       escalation: this.state.escalation,
       connection: this.state.connection,
+      responseSize: this.state.responseSize,
     });
 
     // Replay recent feed messages for late-joining clients
@@ -151,13 +153,14 @@ export class WsHandler {
 
   /** Send a status update to all connected overlays. */
   broadcastStatus(): void {
-    const msg: StatusMessage & { envPath?: string; escalation?: string } = {
+    const msg: StatusMessage & { envPath?: string; escalation?: string; responseSize?: string } = {
       type: "status",
       audio: this.state.audio,
       mic: this.state.mic,
       screen: this.state.screen,
       escalation: this.state.escalation,
       connection: this.state.connection,
+      responseSize: this.state.responseSize,
     };
     if (loadedEnvPath) msg.envPath = loadedEnvPath;
     this.broadcastMessage(msg);

package/sinain-core/src/server.ts

@@ -184,6 +184,8 @@ export interface ServerDeps {
   onSpawnCommand?: (text: string) => void;
   getSpawnPending?: () => { id: string; task: string; label: string; ts: number } | null;
   respondSpawn?: (id: string, result: string) => { ok: boolean; error?: string };
+  embedTexts?: (texts: string[]) => Promise<Float32Array[]>;
+  isEmbeddingReady?: () => boolean;
 }
 
 function readBody(req: IncomingMessage, maxBytes: number): Promise<string> {
@@ -519,6 +521,35 @@ export function createAppServer(deps: ServerDeps) {
         return;
       }
 
+      // ── /embed ── (used by knowledge_integrator.py and graph_query.py)
+      if (req.method === "POST" && url.pathname === "/embed") {
+        if (!deps.embedTexts || !deps.isEmbeddingReady?.()) {
+          res.writeHead(503);
+          res.end(JSON.stringify({ error: "embedding model loading" }));
+          return;
+        }
+        let body = "";
+        req.on("data", (c: Buffer) => { body += c; });
+        req.on("end", async () => {
+          try {
+            const { texts } = JSON.parse(body);
+            if (!Array.isArray(texts) || texts.length === 0) {
+              res.writeHead(400);
+              res.end(JSON.stringify({ error: "texts array required" }));
+              return;
+            }
+            const embeddings = await deps.embedTexts!(texts);
+            // Return as base64-encoded float32 arrays for efficiency
+            const encoded = embeddings.map(e => Buffer.from(e.buffer).toString("base64"));
+            res.end(JSON.stringify({ embeddings: encoded, dims: 384 }));
+          } catch (err: any) {
+            res.writeHead(500);
+            res.end(JSON.stringify({ error: err.message?.slice(0, 200) }));
+          }
+        });
+        return;
+      }
+
       // ── /health ──
       if (req.method === "GET" && url.pathname === "/health") {
         res.end(JSON.stringify({

package/sinain-core/src/types.ts

@@ -20,6 +20,7 @@ export interface StatusMessage {
   screen: string;
   escalation?: string;
   connection: string;
+  responseSize?: string;
 }
 
 /** sinain-core → Overlay: heartbeat ping */
@@ -244,6 +245,7 @@ export interface StopResult {
 
 export type EscalationMode = "off" | "selective" | "focus" | "rich";
 export type ContextRichness = "lean" | "standard" | "rich";
+export type ResponseSize = "small" | "medium" | "large";
 
 export type AnalysisProvider = "openrouter" | "ollama";
 
@@ -393,6 +395,7 @@ export interface BridgeState {
   screen: "active" | "off";
   escalation: "active" | "paused";
   connection: "connected" | "disconnected" | "connecting";
+  responseSize: ResponseSize;
 }
 
 // ── Learning / feedback types ──

package/sinain-memory/README.md

@@ -0,0 +1,105 @@
+# sinain-memory
+
+Local-first knowledge pipeline for SinainHUD. Captures what the user sees and hears, distills it into a knowledge graph, and makes it retrievable for the agent's context.
+
+## Architecture
+
+Two-step pipeline: **LLM extraction** (what to remember) + **deterministic integration** (how to store it).
+
+```
+Audio transcripts + Screen OCR
+        |
+  session_distiller.py (LLM)
+  Extracts: facts[], entities[], decisions[]
+        |
+  knowledge_integrator.py (code — no LLM)
+  - Converts facts to graph assertions (deterministic)
+  - Creates entity:* nodes with freeform types
+  - Links facts to entities via ref edges
+  - Infers cross-entity relationships from fact content
+  - Deduplicates via embedding similarity (cosine 0.78)
+  - Auto-curates playbook (tag overlap, no LLM)
+        |
+  triplestore.py (SQLite EAV)
+  - 4 covering indexes: EAVT, AEVT, VAET, AVET
+  - FTS5 full-text search on fact values
+  - Confidence decay (60-day half-life)
+  - Touched-entities tracking for cache invalidation
+```
+
+## Key Design Decisions
+
+**Deterministic integration.** The integrator does NOT use an LLM. Early experiments showed that LLM-based integration produced 0-20 facts per run depending on model mood, token truncation, and format errors. The deterministic approach converts every distiller fact to a graph assertion — consistent, fast, and reliable.
+
+**Two-layer entity model.** `fact:*` entities store individual claims (searchable via FTS5 and tags). `entity:*` nodes represent real-world entities connected by typed ref edges. The VAET index enables backref traversal: "find all facts about Citibank" is an O(log n) index lookup.
+
+**Incremental distillation.** Long sessions (>8 min) trigger distillation when the feed buffer reaches capacity, before items are lost to the ring buffer. The `onFull` callback fires when 50% new items have accumulated since the last pass.
+
+**Embedding-based dedup.** sinain-core hosts an in-process all-MiniLM-L6-v2 model (384 dims, 2-4ms per embedding). The `/embed` endpoint is used for both write-time dedup (prevent storing semantic duplicates) and read-time re-ranking (surface most relevant facts for a query).
+
+## Triplestore
+
+SQLite-backed EAV store inspired by Datomic/RhizomeDB with 4 covering indexes:
+
+| Index | Query Pattern | Example |
+|-------|-------------|---------|
+| **EAVT** | What does entity X look like? | `store.entity("entity:citibank")` |
+| **AEVT** | Which entities have attribute Y? | `store.entities_with_attr("type")` |
+| **VAET** | What references entity Z? (backrefs) | `store.backrefs("entity:citibank")` |
+| **AVET** | Find entity by attribute+value | `store.lookup("type", "person")` |
+
+Additional features:
+- **FTS5** full-text search on fact values with auto-sync triggers
+- **Confidence decay**: exponential half-life (60 days) — facts lose relevance without reinforcement
+- **Temporal queries**: `entity_as_of(id, date)` for point-in-time knowledge
+- **Touched-entities index**: O(1) "was entity X modified since tx Y?" for cache invalidation
+- **Soft retraction**: facts are marked retracted, not deleted — preserves history
+
+## Retrieval
+
+Hybrid retrieval with Reciprocal Rank Fusion (RRF):
+
+1. **FTS5** keyword search on fact values
+2. **Tag-based** entity matching via AVET index
+3. **Top-confidence** facts as baseline
+4. **Entity graph boost**: facts linked to query-mentioned entities via backrefs get an RRF score bonus
+5. **Embedding re-ranking** (when sinain-core is running): semantic similarity between query and facts
+6. **Confidence decay** applied as tiebreaker
+
+Results are grouped by entity for cross-fact reasoning.
+
+## Distiller Output Schema
+
+```json
+{
+  "whatHappened": "2-3 sentence summary",
+  "facts": ["self-contained factual sentence", ...],
+  "decisions": ["who decided what, with deadline", ...],
+  "entities": [{"name": "entity-slug", "type": "freeform-type"}, ...],
+  "patterns": ["reusable technique or workflow", ...],
+  "preferences": ["user preference or habit", ...],
+  "isEmpty": false
+}
+```
+
+Facts are guided by 5 diversity dimensions: **WHO** (people, roles), **WHAT** (properties, descriptions), **HOW MUCH** (numbers, dates), **WHAT CHANGED** (decisions, agreements), **WHAT'S NEXT** (commitments, plans).
+
+## Files
+
+| File | Role |
+|------|------|
+| `session_distiller.py` | LLM extraction: transcript to structured digest |
+| `knowledge_integrator.py` | Deterministic storage: digest to graph ops + playbook |
+| `triplestore.py` | SQLite EAV with 4 indexes + FTS5 + temporal |
+| `graph_query.py` | Hybrid retrieval with RRF fusion |
+| `embed_client.py` | Python client for sinain-core `/embed` endpoint |
+| `common.py` | Shared LLM call utilities |
+| `memory-config.json` | Model selection, token limits, timeouts |
+
+## Configuration
+
+| Env Var | Default | Description |
+|---------|---------|-------------|
+| `SINAIN_MEMORY_DIR` | `~/.sinain/memory` | Knowledge graph directory |
+| `LEARNING_ENABLED` | `true` | Enable/disable distillation pipeline |
+| `AGENT_ENABLED` | `true` | Set `false` for capture-only mode |

package/sinain-memory/embed_client.py

@@ -0,0 +1,117 @@
+"""Embedding client — calls sinain-core's /embed endpoint for vector operations.
+
+Provides semantic similarity for:
+- Write path: dedup before asserting facts (knowledge_integrator.py)
+- Read path: semantic retrieval (graph_query.py)
+
+Falls back gracefully if sinain-core is not running or model not loaded.
+"""
+
+import base64
+import json
+import struct
+import urllib.request
+from functools import lru_cache
+
+SINAIN_CORE_URL = "http://localhost:9500"
+EMBED_TIMEOUT_S = 5
+SIMILARITY_THRESHOLD = 0.78  # calibrated: catches rephrased facts, rejects different facts
+
+
+def embed(texts: list[str]) -> list[list[float]] | None:
+    """Embed texts via sinain-core /embed endpoint. Returns None if unavailable."""
+    try:
+        data = json.dumps({"texts": texts}).encode()
+        req = urllib.request.Request(
+            f"{SINAIN_CORE_URL}/embed",
+            data=data,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=EMBED_TIMEOUT_S) as resp:
+            result = json.loads(resp.read())
+            # Decode base64 float32 arrays
+            embeddings = []
+            for b64 in result["embeddings"]:
+                raw = base64.b64decode(b64)
+                floats = list(struct.unpack(f"{len(raw)//4}f", raw))
+                embeddings.append(floats)
+            return embeddings
+    except Exception:
+        return None
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    """Cosine similarity between two vectors."""
+    dot = sum(x * y for x, y in zip(a, b))
+    return dot  # vectors are pre-normalized by the model
+
+
+def find_duplicates_batch(
+    new_texts: list[str],
+    existing_texts: list[str],
+    threshold: float = SIMILARITY_THRESHOLD,
+) -> dict[int, int]:
+    """Find duplicates for multiple new texts against existing texts in one batch.
+
+    Returns {new_index: existing_index} for texts with similarity >= threshold.
+    Single HTTP call for all texts — avoids per-fact round trips.
+    """
+    if not existing_texts or not new_texts:
+        return {}
+
+    all_texts = new_texts + existing_texts
+    embeddings = embed(all_texts)
+    if embeddings is None:
+        return {}
+
+    n_new = len(new_texts)
+    result = {}
+
+    for i in range(n_new):
+        best_idx = None
+        best_sim = threshold
+        for j in range(n_new, len(embeddings)):
+            sim = cosine(embeddings[i], embeddings[j])
+            if sim > best_sim:
+                best_sim = sim
+                best_idx = j - n_new
+        if best_idx is not None:
+            result[i] = best_idx
+
+    return result
+
+
+def find_duplicate(
+    new_text: str,
+    existing_texts: list[str],
+    threshold: float = SIMILARITY_THRESHOLD,
+) -> int | None:
+    """Find the index of the most similar existing text, or None if no match."""
+    result = find_duplicates_batch([new_text], existing_texts, threshold)
+    return result.get(0)
+
+
+def rank_by_similarity(
+    query: str,
+    texts: list[str],
+) -> list[tuple[int, float]] | None:
+    """Rank texts by semantic similarity to query. Returns [(index, score), ...] descending.
+
+    Returns None if embedding service unavailable (caller should fall back to keyword).
+    """
+    if not texts:
+        return []
+
+    all_texts = [query] + texts
+    embeddings = embed(all_texts)
+    if embeddings is None:
+        return None
+
+    query_emb = embeddings[0]
+    scored = []
+    for i, emb in enumerate(embeddings[1:]):
+        scored.append((i, cosine(query_emb, emb)))
+
+    scored.sort(key=lambda x: x[1], reverse=True)
+    return scored

package/sinain-memory/eval/benchmarks/base_adapter.py

@@ -0,0 +1,43 @@
+"""Base adapter and data classes for benchmark evaluation."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+
+
+@dataclass
+class BenchmarkQuestion:
+    id: str
+    text: str
+    gold_answer: str
+    category: str  # single-session, multi-session, temporal, etc.
+    evidence_session_ids: list[str] = field(default_factory=list)
+    metadata: dict = field(default_factory=dict)
+
+
+@dataclass
+class BenchmarkInstance:
+    """A set of conversations + questions that share the same context."""
+    id: str
+    sessions: list[list[dict]]  # list of sessions, each a list of feed items {source, text, ts}
+    questions: list[BenchmarkQuestion] = field(default_factory=list)
+    raw_sessions: list[dict] = field(default_factory=list)  # original benchmark format (for full-context condition)
+    metadata: dict = field(default_factory=dict)
+
+
+class BenchmarkAdapter(ABC):
+    """Abstract adapter: converts a published benchmark into sinain's format."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Benchmark name (e.g. 'longmemeval', 'locomo')."""
+
+    @abstractmethod
+    def load_dataset(self, data_dir: str) -> list[BenchmarkInstance]:
+        """Download (if needed) and parse the benchmark dataset."""
+
+    @abstractmethod
+    def format_full_context(self, instance: BenchmarkInstance) -> str:
+        """Render the full conversation history as a text string for the baseline condition."""

package/sinain-memory/eval/benchmarks/config.py

@@ -0,0 +1,23 @@
+"""Benchmark configuration — models, paths, thresholds."""
+
+from pathlib import Path
+
+BENCHMARKS_DIR = Path(__file__).resolve().parent
+DATA_DIR = BENCHMARKS_DIR / "data"
+RESULTS_DIR = BENCHMARKS_DIR / "results"
+
+# LLM models (via OpenRouter)
+QA_MODEL = "google/gemini-2.5-flash"
+JUDGE_MODEL = "openai/gpt-4o"
+
+# Retrieval
+K_VALUES = [1, 3, 5, 10]
+MAX_FACTS_PER_QUERY = 10
+
+# Ingestion
+DISTILLER_TIMEOUT_S = 30
+INTEGRATOR_TIMEOUT_S = 60
+
+# Dataset URLs
+LONGMEMEVAL_HF = "xiaowu0162/longmemeval-cleaned"
+LOCOMO_GITHUB = "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json"

package/sinain-memory/eval/benchmarks/evaluate.py

@@ -0,0 +1,146 @@
+"""Evaluation pipeline — score answers and compute aggregate metrics.
+
+Combines:
+- LLM-as-Judge (QA scoring, 1-5 scale)
+- Retrieval metrics (Recall@k, NDCG@k)
+- Token F1 overlap (mechanical, free)
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections import defaultdict
+
+from .base_adapter import BenchmarkQuestion
+from .config import K_VALUES
+
+
+# ── Token F1 (mechanical, no LLM needed) ─────────────────────────────────────
+
+def _tokenize(text: str) -> list[str]:
+    """Simple whitespace + punctuation tokenizer."""
+    return re.findall(r"\w+", text.lower())
+
+
+def token_f1(predicted: str, gold: str | int) -> float:
+    """Compute token-level F1 between predicted and gold answers."""
+    pred_tokens = set(_tokenize(str(predicted)))
+    gold_tokens = set(_tokenize(str(gold)))
+    if not gold_tokens or not pred_tokens:
+        return 0.0
+    overlap = pred_tokens & gold_tokens
+    if not overlap:
+        return 0.0
+    precision = len(overlap) / len(pred_tokens)
+    recall = len(overlap) / len(gold_tokens)
+    return 2 * precision * recall / (precision + recall)
+
+
+# ── Retrieval metrics (reuse logic from retrieval_evaluator.py) ───────────────
+
+def dcg_at_k(relevant_positions: list[int], k: int) -> float:
+    """Discounted Cumulative Gain at k."""
+    score = 0.0
+    for pos in relevant_positions:
+        if pos < k:
+            score += 1.0 / math.log2(pos + 2)
+    return score
+
+
+def ndcg_at_k(relevant_positions: list[int], num_relevant: int, k: int) -> float:
+    """Normalized DCG at k."""
+    dcg = dcg_at_k(relevant_positions, k)
+    ideal_positions = list(range(min(num_relevant, k)))
+    idcg = dcg_at_k(ideal_positions, k)
+    return dcg / idcg if idcg > 0 else 0.0
+
+
+def compute_retrieval_metrics(
+    retrieved_ids: list[str],
+    expected_ids: list[str],
+    k_values: list[int] | None = None,
+) -> dict:
+    """Compute Recall@k and NDCG@k for a single question."""
+    ks = k_values or K_VALUES
+    expected_set = set(expected_ids)
+    relevant_positions = [i for i, rid in enumerate(retrieved_ids) if rid in expected_set]
+
+    result = {}
+    for k in ks:
+        hit = any(pos < k for pos in relevant_positions)
+        result[f"recall@{k}"] = 1.0 if hit else 0.0
+        result[f"ndcg@{k}"] = ndcg_at_k(relevant_positions, len(expected_set), k)
+    return result
+
+
+# ── Aggregate metrics ─────────────────────────────────────────────────────────
+
+def aggregate_results(per_question: list[dict]) -> dict:
+    """Compute aggregate metrics from per-question results.
+
+    Each per_question entry has:
+      {id, category, retrieval: {recall@k, ndcg@k}, answers: {condition: {score, f1}}}
+    """
+    if not per_question:
+        return {"error": "no results"}
+
+    # Per-condition scores
+    condition_scores: dict[str, list[float]] = defaultdict(list)
+    condition_f1s: dict[str, list[float]] = defaultdict(list)
+    # Per-category per-condition
+    cat_scores: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
+    # Retrieval
+    retrieval_metrics: dict[str, list[float]] = defaultdict(list)
+
+    for q in per_question:
+        cat = q.get("category", "unknown")
+
+        for cond, data in q.get("answers", {}).items():
+            if data.get("score") is not None:
+                condition_scores[cond].append(data["score"])
+                cat_scores[cat][cond].append(data["score"])
+            if data.get("f1") is not None:
+                condition_f1s[cond].append(data["f1"])
+
+        for metric, val in q.get("retrieval", {}).items():
+            if isinstance(val, (int, float)):
+                retrieval_metrics[metric].append(val)
+
+    def _mean(lst: list[float]) -> float:
+        return round(sum(lst) / len(lst), 4) if lst else 0.0
+
+    # Build summary
+    conditions = {}
+    for cond in sorted(condition_scores):
+        conditions[cond] = {
+            "mean_score": _mean(condition_scores[cond]),
+            "mean_f1": _mean(condition_f1s.get(cond, [])),
+            "n": len(condition_scores[cond]),
+        }
+
+    # IPR: sinain-memory vs full-context
+    sm_scores = condition_scores.get("sinain-memory", [])
+    fc_scores = condition_scores.get("full-context", [])
+    ipr = _mean(sm_scores) / _mean(fc_scores) if fc_scores and _mean(fc_scores) > 0 else None
+
+    # Category breakdown
+    categories = {}
+    for cat in sorted(cat_scores):
+        categories[cat] = {}
+        for cond in sorted(cat_scores[cat]):
+            categories[cat][cond] = {
+                "mean_score": _mean(cat_scores[cat][cond]),
+                "n": len(cat_scores[cat][cond]),
+            }
+
+    # Retrieval summary
+    retrieval = {k: _mean(v) for k, v in sorted(retrieval_metrics.items())}
+
+    return {
+        "total_questions": len(per_question),
+        "conditions": conditions,
+        "ipr": round(ipr, 4) if ipr else None,
+        "categories": categories,
+        "retrieval": retrieval,
+    }