@geravant/sinain 1.12.0 → 1.13.0

package/sinain-core/package.json

@@ -12,6 +12,7 @@
     "eval:quick": "tsx eval/harness.ts --scenarios eval/scenarios/ --runs 1 --fast --report /dev/stdout"
   },
   "dependencies": {
+    "@huggingface/transformers": "^4.0.1",
     "@types/node": "^22.19.7",
     "@types/ws": "^8.18.1",
     "tsx": "^4.21.0",

package/sinain-core/src/buffers/feed-buffer.ts

@@ -10,11 +10,31 @@ export class FeedBuffer {
   private _version = 0;
   private maxSize: number;
   private _hwm = 0;
+  private _onFullCb: ((items: FeedItem[]) => void) | null = null;
+  private _onFullArmed = true;
+  private _onFullVersion = 0; // version at last re-arm
 
   constructor(maxSize = 100) {
     this.maxSize = maxSize;
   }
 
+  /**
+   * Register a callback that fires when the buffer reaches capacity AND
+   * at least half the buffer has been replaced with new items since the
+   * last distillation. This prevents rapid-fire triggers on the same content.
+   */
+  onFull(cb: (items: FeedItem[]) => void): void {
+    this._onFullCb = cb;
+    this._onFullArmed = true;
+    this._onFullVersion = 0;
+  }
+
+  /** Re-arm the onFull callback (call after incremental distillation completes). */
+  rearmOnFull(): void {
+    this._onFullVersion = this._version;
+    this._onFullArmed = true;
+  }
+
   /** Push a new feed item. Returns the created item. */
   push(text: string, priority: Priority, source: FeedItem["source"], channel: FeedChannel = "stream"): FeedItem {
     const item: FeedItem = {
@@ -27,6 +47,18 @@ export class FeedBuffer {
     };
     this.items.push(item);
     if (this.items.length > this._hwm) this._hwm = this.items.length;
+
+    // Fire onFull when buffer is at capacity AND enough new items have arrived
+    // since the last distillation (at least half the buffer replaced)
+    const newSinceRearm = this._version - this._onFullVersion;
+    if (this.items.length >= this.maxSize
+        && this._onFullCb && this._onFullArmed
+        && newSinceRearm >= Math.floor(this.maxSize / 2)) {
+      this._onFullArmed = false;
+      const snapshot = [...this.items];
+      queueMicrotask(() => this._onFullCb!(snapshot));
+    }
+
     if (this.items.length > this.maxSize) {
       this.items.shift();
     }

package/sinain-core/src/embedding/service.ts

@@ -0,0 +1,66 @@
+/**
+ * EmbeddingService — in-process sentence embeddings for knowledge dedup + retrieval.
+ *
+ * Loads all-MiniLM-L6-v2 via @huggingface/transformers (ONNX runtime, no Python).
+ * Model loads async at startup (~9s), embeddings are 2-4ms per text after that.
+ *
+ * Used by:
+ * - knowledge_integrator.py (via POST /embed) for dedup before asserting facts
+ * - graph_query.py (via POST /embed) for semantic retrieval
+ */
+
+import { log, warn } from "../log.js";
+
+const TAG = "embedding";
+const MODEL_ID = "Xenova/all-MiniLM-L6-v2";
+
+type Pipeline = (texts: string | string[], options: { pooling: string; normalize: boolean }) => Promise<{ data: Float32Array; dims: number[] }>;
+
+export class EmbeddingService {
+  private pipeline: Pipeline | null = null;
+  private loading = false;
+  private _ready = false;
+
+  get ready(): boolean {
+    return this._ready;
+  }
+
+  /** Load the model in the background. Non-blocking — returns immediately. */
+  loadAsync(): void {
+    if (this.loading || this._ready) return;
+    this.loading = true;
+
+    const start = Date.now();
+    log(TAG, `loading ${MODEL_ID} (background)...`);
+
+    import("@huggingface/transformers").then(async ({ pipeline }) => {
+      this.pipeline = await pipeline("feature-extraction", MODEL_ID) as unknown as Pipeline;
+      this._ready = true;
+      log(TAG, `model ready in ${Date.now() - start}ms (384 dims)`);
+    }).catch((err) => {
+      warn(TAG, `failed to load model: ${err.message?.slice(0, 100)}`);
+      this.loading = false;
+    });
+  }
+
+  /** Embed one or more texts. Returns array of float32 arrays (384 dims each). */
+  async embed(texts: string[]): Promise<Float32Array[]> {
+    if (!this.pipeline) {
+      throw new Error("Embedding model not loaded yet");
+    }
+
+    const results: Float32Array[] = [];
+    for (const text of texts) {
+      const output = await this.pipeline(text, { pooling: "mean", normalize: true });
+      results.push(new Float32Array(output.data));
+    }
+    return results;
+  }
+
+  /** Compute cosine similarity between two embeddings. */
+  static cosine(a: Float32Array, b: Float32Array): number {
+    let dot = 0;
+    for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+    return dot;
+  }
+}

package/sinain-core/src/index.ts

@@ -16,6 +16,7 @@ import { TraceStore } from "./trace/trace-store.js";
 import { FeedbackStore } from "./learning/feedback-store.js";
 import { SignalCollector } from "./learning/signal-collector.js";
 import { LocalCurationService } from "./learning/local-curation.js";
+import { EmbeddingService } from "./embedding/service.js";
 import { createAppServer } from "./server.js";
 import { Profiler } from "./profiler.js";
 import { CostTracker } from "./cost/tracker.js";
@@ -70,7 +71,7 @@ async function queryKnowledgeFactsMulti(entities: string[], maxFacts: number): P
   for (const dbPath of dbPaths) {
     if (!existsSync(dbPath)) continue;
     try {
-      const args = [scriptPath, "--db", dbPath, "--max-facts", String(maxFacts), "--format", "text"];
+      const args = [scriptPath, "--db", dbPath, "--max-facts", String(maxFacts), "--format", "compact"];
       if (entities.length > 0) args.push("--entities", JSON.stringify(entities));
       const out = execFileSync("python3", args, { timeout: 5000, encoding: "utf-8" }).trim();
       if (out) results.push(out);
@@ -338,11 +339,25 @@ async function main() {
     ? new FeedbackStore(config.learningConfig.feedbackDir, config.learningConfig.retentionDays)
     : null;
 
+  // ── Initialize embedding service (non-blocking) ──
+  const embeddingService = new EmbeddingService();
+  embeddingService.loadAsync(); // ~9s background load, server starts immediately
+
   // ── Initialize local knowledge pipeline ──
   const localCuration = new LocalCurationService();
-  localCuration.distillPendingSession(); // Recover any session saved before a force-kill
+  // Distill pending session in background — don't block server startup
+  setImmediate(() => {
+    localCuration.distillPendingSession();
+  });
   localCuration.startPeriodicCuration();
 
+  // Wire incremental distillation: when feed buffer fills, distill before items are lost
+  localCuration.setSenseBuffer(senseBuffer);
+  localCuration.setRearmCallback(() => feedBuffer.rearmOnFull());
+  feedBuffer.onFull((items) => {
+    localCuration.distillIncremental(items);
+  });
+
   // ── Initialize escalation ──
   const escalator = new Escalator({
     feedBuffer,
@@ -668,6 +683,8 @@ async function main() {
     },
     getSpawnPending: () => escalator.getSpawnPending(),
     respondSpawn: (id: string, result: string) => escalator.respondSpawn(id, result),
+    embedTexts: (texts: string[]) => embeddingService.embed(texts),
+    isEmbeddingReady: () => embeddingService.ready,
   });
 
   // ── Wire overlay profiling ──

package/sinain-core/src/learning/local-curation.ts

@@ -15,6 +15,7 @@ import { existsSync, mkdirSync, writeFileSync, readFileSync, unlinkSync, appendF
 import { resolve, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import type { FeedItem } from "../types.js";
+import type { SenseBuffer } from "../buffers/sense-buffer.js";
 import { log, warn, error } from "../log.js";
 
 const TAG = "local-curation";
@@ -55,6 +56,10 @@ export class LocalCurationService {
   private scriptsDir: string;
   private sessionStartTs: number;
   private curationTimer: ReturnType<typeof setInterval> | null = null;
+  private _lastDistilledTs = 0; // timestamp of last incremental distillation
+  private _incrementalRunning = false;
+  private _rearmCb: (() => void) | null = null; // callback to re-arm feed buffer onFull
+  private _senseBuffer: SenseBuffer | null = null;
 
   constructor() {
     this.memoryDir = resolveMemoryDir();
@@ -90,6 +95,100 @@ export class LocalCurationService {
     }
   }
 
+  /** Timestamp of last incremental distillation (items before this are already distilled). */
+  get lastDistilledTs(): number {
+    return this._lastDistilledTs;
+  }
+
+  /** Set the callback to re-arm the feed buffer's onFull trigger after distillation. */
+  setRearmCallback(cb: () => void): void {
+    this._rearmCb = cb;
+  }
+
+  /** Attach sense buffer for screen context in distillation. */
+  setSenseBuffer(sb: SenseBuffer): void {
+    this._senseBuffer = sb;
+  }
+
+  /** Extract screen context from sense buffer as feed-item-compatible entries. */
+  private getSenseContext(): Array<{ text: string; ts: number; source: string; channel: string }> {
+    if (!this._senseBuffer) return [];
+    const events = this._senseBuffer.queryByTime(this._lastDistilledTs || (Date.now() - 30 * 60 * 1000));
+    const items: Array<{ text: string; ts: number; source: string; channel: string }> = [];
+    for (const evt of events) {
+      // Include OCR text (what's visible on screen)
+      if (evt.ocr && evt.ocr.length > 20) {
+        const app = evt.semantic?.context?.app || "unknown";
+        items.push({
+          text: `[screen: ${app}] ${evt.ocr}`,
+          ts: evt.ts,
+          source: "sense",
+          channel: "screen",
+        });
+      }
+      // Include vision summaries (AI description of screen content)
+      if (evt.semantic?.visible?.summary) {
+        items.push({
+          text: `[screen-context] ${evt.semantic.visible.summary}`,
+          ts: evt.ts,
+          source: "sense",
+          channel: "screen",
+        });
+      }
+    }
+    return items;
+  }
+
+  /**
+   * Incremental distillation — called when the feed buffer reaches capacity.
+   * Distills the current buffer contents before they fall off the ring buffer.
+   * Runs async so it doesn't block new items from arriving.
+   */
+  async distillIncremental(feedItems: FeedItem[]): Promise<void> {
+    if (this._incrementalRunning) {
+      log(TAG, "incremental distillation already running — skipping");
+      return;
+    }
+    this._incrementalRunning = true;
+
+    try {
+      const itemCount = feedItems.length;
+      log(TAG, `incremental distillation: ${itemCount} items (buffer full)`);
+
+      const sessionMeta = {
+        ts: new Date().toISOString(),
+        sessionKey: "local-incremental",
+        durationMs: Date.now() - this.sessionStartTs,
+      };
+
+      const audioItems = feedItems.map(item => ({
+        text: item.text,
+        ts: item.ts,
+        source: item.source || "unknown",
+        channel: item.channel || "agent",
+      }));
+
+      // Merge screen context from sense buffer (OCR + vision summaries)
+      const senseItems = this.getSenseContext();
+      const transcript = [...audioItems, ...senseItems].sort((a, b) => a.ts - b.ts);
+
+      if (senseItems.length > 0) {
+        log(TAG, `including ${senseItems.length} screen context items in distillation`);
+      }
+
+      if (this.runDistillation(transcript, sessionMeta)) {
+        this._lastDistilledTs = Date.now();
+        log(TAG, `incremental distillation complete — ${itemCount} audio + ${senseItems.length} screen items processed`);
+      }
+    } catch (err: any) {
+      warn(TAG, `incremental distillation failed: ${err.message?.slice(0, 100)}`);
+    } finally {
+      this._incrementalRunning = false;
+      // Re-arm the buffer callback so next fill triggers another distillation
+      this._rearmCb?.();
+    }
+  }
+
   /**
    * Save feed items to disk for deferred distillation.
    * Called during shutdown — instant (no LLM), survives tsx force-kill.
@@ -160,13 +259,20 @@ export class LocalCurationService {
    * picked up on next startup via distillPendingSession().
    */
   async distillSession(feedItems: FeedItem[]): Promise<void> {
-    if (feedItems.length < 1) {
-      log(TAG, `skipping distillation — only ${feedItems.length} feed items`);
+    // Filter to only items not yet covered by incremental distillation
+    const items = this._lastDistilledTs > 0
+      ? feedItems.filter(i => i.ts > this._lastDistilledTs)
+      : feedItems;
+
+    if (items.length < 1) {
+      log(TAG, `skipping shutdown distillation — all ${feedItems.length} items already distilled incrementally`);
       return;
     }
 
+    log(TAG, `shutdown distillation: ${items.length} items (${feedItems.length - items.length} already distilled incrementally)`);
+
     // Step 0: Save to disk FIRST — survives force-kill
-    this.savePendingSession(feedItems);
+    this.savePendingSession(items);
 
     const sessionMeta = {
       ts: new Date().toISOString(),
@@ -174,7 +280,7 @@ export class LocalCurationService {
       durationMs: Date.now() - this.sessionStartTs,
     };
 
-    const transcript = feedItems.map(item => ({
+    const transcript = items.map(item => ({
       text: item.text,
       ts: item.ts,
       source: item.source || "unknown",
@@ -204,13 +310,37 @@ export class LocalCurationService {
     log(TAG, `distilling session: ${transcript.length} items, ${Math.round(sessionMeta.durationMs / 60000)} min`);
 
     try {
+      // Step 0.5: Retrieve existing entities for context (Mem0 retrieve-before-extract pattern)
+      let existingEntities = "";
+      const dbPath = resolve(this.memoryDir, "knowledge-graph.db");
+      if (existsSync(dbPath)) {
+        try {
+          existingEntities = execFileSync("python3", [
+            resolve(this.scriptsDir, "graph_query.py"),
+            "--db", dbPath,
+            "--top", "20",
+            "--format", "compact",
+          ], {
+            timeout: 5_000,
+            encoding: "utf-8",
+            env: { ...process.env, PYTHONPATH: this.scriptsDir },
+          }).trim();
+        } catch {
+          // Non-fatal — distillation works without existing entities
+        }
+      }
+
       // Step 1: Distill session into a SessionDigest
-      const digestJson = execFileSync("python3", [
+      const distillerArgs = [
         resolve(this.scriptsDir, "session_distiller.py"),
         "--memory-dir", this.memoryDir,
         "--transcript", JSON.stringify(transcript),
         "--session-meta", JSON.stringify(sessionMeta),
-      ], {
+      ];
+      if (existingEntities) {
+        distillerArgs.push("--existing-entities", existingEntities);
+      }
+      const digestJson = execFileSync("python3", distillerArgs, {
         timeout: 30_000,
         encoding: "utf-8",
         env: { ...process.env, PYTHONPATH: this.scriptsDir },
@@ -236,7 +366,7 @@ export class LocalCurationService {
           "--memory-dir", this.memoryDir,
           "--digest", JSON.stringify(digest),
         ], {
-          timeout: 30_000,
+          timeout: 60_000, // 60s: LLM call (~10s) + embedding dedup (~5s) + graph ops
           encoding: "utf-8",
           env: { ...process.env, PYTHONPATH: this.scriptsDir },
         });

package/sinain-core/src/server.ts

@@ -184,6 +184,8 @@ export interface ServerDeps {
   onSpawnCommand?: (text: string) => void;
   getSpawnPending?: () => { id: string; task: string; label: string; ts: number } | null;
   respondSpawn?: (id: string, result: string) => { ok: boolean; error?: string };
+  embedTexts?: (texts: string[]) => Promise<Float32Array[]>;
+  isEmbeddingReady?: () => boolean;
 }
 
 function readBody(req: IncomingMessage, maxBytes: number): Promise<string> {
@@ -519,6 +521,35 @@ export function createAppServer(deps: ServerDeps) {
         return;
       }
 
+      // ── /embed ── (used by knowledge_integrator.py and graph_query.py)
+      if (req.method === "POST" && url.pathname === "/embed") {
+        if (!deps.embedTexts || !deps.isEmbeddingReady?.()) {
+          res.writeHead(503);
+          res.end(JSON.stringify({ error: "embedding model loading" }));
+          return;
+        }
+        let body = "";
+        req.on("data", (c: Buffer) => { body += c; });
+        req.on("end", async () => {
+          try {
+            const { texts } = JSON.parse(body);
+            if (!Array.isArray(texts) || texts.length === 0) {
+              res.writeHead(400);
+              res.end(JSON.stringify({ error: "texts array required" }));
+              return;
+            }
+            const embeddings = await deps.embedTexts!(texts);
+            // Return as base64-encoded float32 arrays for efficiency
+            const encoded = embeddings.map(e => Buffer.from(e.buffer).toString("base64"));
+            res.end(JSON.stringify({ embeddings: encoded, dims: 384 }));
+          } catch (err: any) {
+            res.writeHead(500);
+            res.end(JSON.stringify({ error: err.message?.slice(0, 200) }));
+          }
+        });
+        return;
+      }
+
       // ── /health ──
       if (req.method === "GET" && url.pathname === "/health") {
         res.end(JSON.stringify({

package/sinain-memory/README.md

@@ -0,0 +1,105 @@
+# sinain-memory
+
+Local-first knowledge pipeline for SinainHUD. Captures what the user sees and hears, distills it into a knowledge graph, and makes it retrievable for the agent's context.
+
+## Architecture
+
+Two-step pipeline: **LLM extraction** (what to remember) + **deterministic integration** (how to store it).
+
+```
+Audio transcripts + Screen OCR
+        |
+  session_distiller.py (LLM)
+  Extracts: facts[], entities[], decisions[]
+        |
+  knowledge_integrator.py (code — no LLM)
+  - Converts facts to graph assertions (deterministic)
+  - Creates entity:* nodes with freeform types
+  - Links facts to entities via ref edges
+  - Infers cross-entity relationships from fact content
+  - Deduplicates via embedding similarity (cosine 0.78)
+  - Auto-curates playbook (tag overlap, no LLM)
+        |
+  triplestore.py (SQLite EAV)
+  - 4 covering indexes: EAVT, AEVT, VAET, AVET
+  - FTS5 full-text search on fact values
+  - Confidence decay (60-day half-life)
+  - Touched-entities tracking for cache invalidation
+```
+
+## Key Design Decisions
+
+**Deterministic integration.** The integrator does NOT use an LLM. Early experiments showed that LLM-based integration produced 0-20 facts per run depending on model mood, token truncation, and format errors. The deterministic approach converts every distiller fact to a graph assertion — consistent, fast, and reliable.
+
+**Two-layer entity model.** `fact:*` entities store individual claims (searchable via FTS5 and tags). `entity:*` nodes represent real-world entities connected by typed ref edges. The VAET index enables backref traversal: "find all facts about Citibank" is an O(log n) index lookup.
+
+**Incremental distillation.** Long sessions (>8 min) trigger distillation when the feed buffer reaches capacity, before items are lost to the ring buffer. The `onFull` callback fires when 50% new items have accumulated since the last pass.
+
+**Embedding-based dedup.** sinain-core hosts an in-process all-MiniLM-L6-v2 model (384 dims, 2-4ms per embedding). The `/embed` endpoint is used for both write-time dedup (prevent storing semantic duplicates) and read-time re-ranking (surface most relevant facts for a query).
+
+## Triplestore
+
+SQLite-backed EAV store inspired by Datomic/RhizomeDB with 4 covering indexes:
+
+| Index | Query Pattern | Example |
+|-------|-------------|---------|
+| **EAVT** | What does entity X look like? | `store.entity("entity:citibank")` |
+| **AEVT** | Which entities have attribute Y? | `store.entities_with_attr("type")` |
+| **VAET** | What references entity Z? (backrefs) | `store.backrefs("entity:citibank")` |
+| **AVET** | Find entity by attribute+value | `store.lookup("type", "person")` |
+
+Additional features:
+- **FTS5** full-text search on fact values with auto-sync triggers
+- **Confidence decay**: exponential half-life (60 days) — facts lose relevance without reinforcement
+- **Temporal queries**: `entity_as_of(id, date)` for point-in-time knowledge
+- **Touched-entities index**: O(1) "was entity X modified since tx Y?" for cache invalidation
+- **Soft retraction**: facts are marked retracted, not deleted — preserves history
+
+## Retrieval
+
+Hybrid retrieval with Reciprocal Rank Fusion (RRF):
+
+1. **FTS5** keyword search on fact values
+2. **Tag-based** entity matching via AVET index
+3. **Top-confidence** facts as baseline
+4. **Entity graph boost**: facts linked to query-mentioned entities via backrefs get an RRF score bonus
+5. **Embedding re-ranking** (when sinain-core is running): semantic similarity between query and facts
+6. **Confidence decay** applied as tiebreaker
+
+Results are grouped by entity for cross-fact reasoning.
+
+## Distiller Output Schema
+
+```json
+{
+  "whatHappened": "2-3 sentence summary",
+  "facts": ["self-contained factual sentence", ...],
+  "decisions": ["who decided what, with deadline", ...],
+  "entities": [{"name": "entity-slug", "type": "freeform-type"}, ...],
+  "patterns": ["reusable technique or workflow", ...],
+  "preferences": ["user preference or habit", ...],
+  "isEmpty": false
+}
+```
+
+Facts are guided by 5 diversity dimensions: **WHO** (people, roles), **WHAT** (properties, descriptions), **HOW MUCH** (numbers, dates), **WHAT CHANGED** (decisions, agreements), **WHAT'S NEXT** (commitments, plans).
+
+## Files
+
+| File | Role |
+|------|------|
+| `session_distiller.py` | LLM extraction: transcript to structured digest |
+| `knowledge_integrator.py` | Deterministic storage: digest to graph ops + playbook |
+| `triplestore.py` | SQLite EAV with 4 indexes + FTS5 + temporal |
+| `graph_query.py` | Hybrid retrieval with RRF fusion |
+| `embed_client.py` | Python client for sinain-core `/embed` endpoint |
+| `common.py` | Shared LLM call utilities |
+| `memory-config.json` | Model selection, token limits, timeouts |
+
+## Configuration
+
+| Env Var | Default | Description |
+|---------|---------|-------------|
+| `SINAIN_MEMORY_DIR` | `~/.sinain/memory` | Knowledge graph directory |
+| `LEARNING_ENABLED` | `true` | Enable/disable distillation pipeline |
+| `AGENT_ENABLED` | `true` | Set `false` for capture-only mode |

package/sinain-memory/embed_client.py

@@ -0,0 +1,117 @@
+"""Embedding client — calls sinain-core's /embed endpoint for vector operations.
+
+Provides semantic similarity for:
+- Write path: dedup before asserting facts (knowledge_integrator.py)
+- Read path: semantic retrieval (graph_query.py)
+
+Falls back gracefully if sinain-core is not running or model not loaded.
+"""
+
+import base64
+import json
+import struct
+import urllib.request
+from functools import lru_cache
+
+SINAIN_CORE_URL = "http://localhost:9500"
+EMBED_TIMEOUT_S = 5
+SIMILARITY_THRESHOLD = 0.78  # calibrated: catches rephrased facts, rejects different facts
+
+
+def embed(texts: list[str]) -> list[list[float]] | None:
+    """Embed texts via sinain-core /embed endpoint. Returns None if unavailable."""
+    try:
+        data = json.dumps({"texts": texts}).encode()
+        req = urllib.request.Request(
+            f"{SINAIN_CORE_URL}/embed",
+            data=data,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        with urllib.request.urlopen(req, timeout=EMBED_TIMEOUT_S) as resp:
+            result = json.loads(resp.read())
+            # Decode base64 float32 arrays
+            embeddings = []
+            for b64 in result["embeddings"]:
+                raw = base64.b64decode(b64)
+                floats = list(struct.unpack(f"{len(raw)//4}f", raw))
+                embeddings.append(floats)
+            return embeddings
+    except Exception:
+        return None
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    """Cosine similarity between two vectors."""
+    dot = sum(x * y for x, y in zip(a, b))
+    return dot  # vectors are pre-normalized by the model
+
+
+def find_duplicates_batch(
+    new_texts: list[str],
+    existing_texts: list[str],
+    threshold: float = SIMILARITY_THRESHOLD,
+) -> dict[int, int]:
+    """Find duplicates for multiple new texts against existing texts in one batch.
+
+    Returns {new_index: existing_index} for texts with similarity >= threshold.
+    Single HTTP call for all texts — avoids per-fact round trips.
+    """
+    if not existing_texts or not new_texts:
+        return {}
+
+    all_texts = new_texts + existing_texts
+    embeddings = embed(all_texts)
+    if embeddings is None:
+        return {}
+
+    n_new = len(new_texts)
+    result = {}
+
+    for i in range(n_new):
+        best_idx = None
+        best_sim = threshold
+        for j in range(n_new, len(embeddings)):
+            sim = cosine(embeddings[i], embeddings[j])
+            if sim > best_sim:
+                best_sim = sim
+                best_idx = j - n_new
+        if best_idx is not None:
+            result[i] = best_idx
+
+    return result
+
+
+def find_duplicate(
+    new_text: str,
+    existing_texts: list[str],
+    threshold: float = SIMILARITY_THRESHOLD,
+) -> int | None:
+    """Find the index of the most similar existing text, or None if no match."""
+    result = find_duplicates_batch([new_text], existing_texts, threshold)
+    return result.get(0)
+
+
+def rank_by_similarity(
+    query: str,
+    texts: list[str],
+) -> list[tuple[int, float]] | None:
+    """Rank texts by semantic similarity to query. Returns [(index, score), ...] descending.
+
+    Returns None if embedding service unavailable (caller should fall back to keyword).
+    """
+    if not texts:
+        return []
+
+    all_texts = [query] + texts
+    embeddings = embed(all_texts)
+    if embeddings is None:
+        return None
+
+    query_emb = embeddings[0]
+    scored = []
+    for i, emb in enumerate(embeddings[1:]):
+        scored.append((i, cosine(query_emb, emb)))
+
+    scored.sort(key=lambda x: x[1], reverse=True)
+    return scored