futura-scion 0.2.1 → 0.2.2

package/README.md

@@ -53,6 +53,63 @@ scion run "fix var in src/legacy.js" --intent fix    # fix + gate-verify + certi
 scion brain --all          # fleet-brain memory inventory
 ```
 
+## The brief organ — context in, verified contract out
+
+FS's rungs act on commands, but real work arrives as CONTEXT — tickets,
+specs, conversations. The brief organ (`scion brief`, `docs/brief.md`)
+ingests a context document and produces a structured problem model:
+entities, business rules (`MUST`/`never` sentences), acceptance criteria
+(Gherkin `Given/When/Then`, bullets). Each requirement is classified
+**checkable** (compiles to a Gate verifier command — the same oracle that
+judges code judges the business rules) or **judgment** (kept visible for
+the clarify/review loop, never dropped). A failing requirement **vetoes
+completeness**, so FS cannot declare work done while the contract is
+broken.
+
+## Temporal memory: forgetting, supersession, semantic recall (cortex lineage)
+
+The brain never grows stale or noisy — two deterministic organs (ported from
+agentic-cortex, LLM-free throughout) keep it sharp:
+
+- **Temporal expiry** — a memory whose text carries a deadline (`2026-12-25`,
+  `in 3 days`, `tomorrow`, or an explicit `ttlDays`) gets that lifespan on
+  save; `scion brain --sweep` soft-deletes everything whose time has passed.
+- **Supersession** — saving a statement-like fact with a near-identical title
+  (`deploy is Monday` over `deploy is Friday`) marks the old fact
+  `superseded_by` the new one and excludes it from retrieval. Gate-verified
+  memories are append-only evidence and never auto-supersede.
+- **Semantic recall** — every memory carries a deterministic 512-dim
+  hashed-feature vector (no model, no download, byte-identical on every
+  machine). Hybrid search blends FTS keyword rank with vector similarity;
+  when FTS cannot match at all (morphological near-misses like `pools
+  exhausting` vs `pool exhaustion`), retrieval falls back to vector-only.
+  `scion brain` reports the forgetting ledger (`superseded` / `expired`).
+
+## FS Desktop (the UI)
+
+The web UI ships inside the npm package — one command after install:
+
+```bash
+scion ui                   # boots the kernel + opens FS Desktop in the browser
+# or: scion serve, then open http://127.0.0.1:5107/ui
+```
+
+### Windows installer (two-tier)
+
+A native Windows app is also built from this repo:
+
+```bash
+cd desktop && npm install && npm run dist   # → dist-desktop/FS-Desktop-Setup-<ver>.exe
+```
+
+The shell is a thin Electron host (~78 MB installer) that **detects the npm kernel at
+launch**: if `futura-scion` is installed globally (or `SCION_BIN` is set) it spawns
+`scion serve` itself and opens the UI window; otherwise it shows a guided setup screen
+with the one-line install command and a Retry button. The kernel always runs under the
+user's own Node — updates flow through `npm update -g futura-scion` without
+re-installing the shell. Silent install: `FS-Desktop-Setup-<ver>.exe /S`.
+
+
 `better-sqlite3` ships prebuilt binaries for common platforms (Linux/macOS/Windows,
 x64/arm64); exotic platforms need build tools. FS keeps its brain per project
 (`.scion/`), reads config from `config/scion.config.yaml` (or shipped defaults),

package/bin/scion.js

@@ -16,8 +16,9 @@
 'use strict';
 
 import { createInterface } from 'node:readline';
-import { existsSync } from 'node:fs';
+import { existsSync, readFileSync } from 'node:fs';
 import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { spawnSync } from 'node:child_process';
 import * as trail from '../src/kernel/trail.js';
 import {
@@ -91,7 +92,7 @@ async function oneShot(text, opts = {}, depth = 0) {
   // persisted scopes (brain config). interpret() strips the @tokens itself.
   const projectScopes = opts.noProjectScopes ? [] : getProjectScopes();
   const mergedScopes = resolveScopes(text, { flag: opts.scope ?? null, projectScopes });
-  const interp = interpret(text, { existsSync: (p) => existsSync(p), intentOverride: opts.intentOverride, scopes: mergedScopes.length ? mergedScopes : undefined });
+  let interp = interpret(text, { existsSync: (p) => existsSync(p), intentOverride: opts.intentOverride, scopes: mergedScopes.length ? mergedScopes : undefined });
   trail.journal('nlu.interpret', {
     utterance: String(text).slice(0, 200),
     intent: interp.intent,
@@ -103,6 +104,26 @@ async function oneShot(text, opts = {}, depth = 0) {
     override: opts.intentOverride ?? null,
   });
 
+  // Tier 3 — the System-1 language-parse rung: ONLY when the deterministic
+  // interpreter found no evidence at all, and an llm provider is configured
+  // AND budgeted (daily_tokens > 0 — off by default). The model parses the
+  // utterance into intent JSON; the SAME schema check validates it; an
+  // abstention falls through to the clarify loop. Never a guess.
+  if (interp.intentEvidence.length === 0 && interp.confidence === 0 && !interp.question && !opts.intentOverride) {
+    const cfg = loadConfig();
+    if (cfg.llm?.daily_tokens > 0 && cfg.llm?.baseUrl && cfg.llm?.model) {
+      const { llmInterpret } = await import('../src/mind/nlu.js');
+      const parsed = await llmInterpret(text, { llmCfg: cfg.llm });
+      trail.journal('nlu.llm-parse', {
+        utterance: String(text).slice(0, 200),
+        accepted: !!parsed,
+        intent: parsed?.intent ?? null,
+        confidence: parsed?.confidence ?? null,
+      });
+      if (parsed) interp = parsed;
+    }
+  }
+
   // Route pure service intents to their kernel surface — a question about
   // state should not become a queue task.
   if (interp.intent === 'show') {
@@ -159,7 +180,7 @@ async function oneShot(text, opts = {}, depth = 0) {
         // Unresolved answer: fall through and surface the question.
       }
     }
-    return { ok: false, intent: 'clarify', prompt: cl.prompt, options: cl.options, open: cl.open, gap_id: gap.id };
+    return { ok: false, intent: 'clarify', prompt: cl.prompt, options: cl.options, open: cl.open, gap_id: gap.id, lang_notice: interp.lang_notice ?? null };
   }
 
   // Ordinary task — enriched with extracted slots.
@@ -311,17 +332,130 @@ switch (cmd || '') {
     const db = getDb();
     const scope = restArgs[0] === '--all' ? 'all'
       : restArgs[0] === '--project' ? resolve(restArgs[1] || '.').replace(/\\/g, '/') : null;
+    // Temporal forgetting (cortex lineage): --sweep expires every memory
+    // whose lifespan has passed; --sweep --dry-run previews it.
+    if (arg === '--sweep' || restArgs.includes('--sweep')) {
+      const { sweepExpires } = await import('../src/brain/forgetting.js');
+      console.log(JSON.stringify(sweepExpires({ dryRun: arg === '--dry-run' || restArgs.includes('--dry-run') }), null, 2));
+      break;
+    }
     const perProject = db.prepare(
       'SELECT project, COUNT(*) as memories, SUM(is_active) as active FROM memories GROUP BY project ORDER BY memories DESC'
     ).all();
+    const forgetting = db.prepare(
+      `SELECT SUM(CASE WHEN is_active = 0 AND superseded_by IS NOT NULL THEN 1 ELSE 0 END) as superseded,
+              SUM(CASE WHEN is_active = 0 AND expires_at IS NOT NULL THEN 1 ELSE 0 END) as expired,
+              SUM(CASE WHEN is_active = 1 AND expires_at IS NOT NULL THEN 1 ELSE 0 END) as expiring_future
+       FROM memories`
+    ).get();
+    // Claims (YOINK lineage): open bets, settled record, the overconfidence
+    // gap, the Brier score, and the dead-memory census.
+    const { calibration, deadMemories } = await import('../src/mind/claims.js');
+    const claims = db.prepare(
+      'SELECT COUNT(*) as total, SUM(CASE WHEN outcome IS NULL THEN 1 ELSE 0 END) as open, SUM(CASE WHEN outcome IS NOT NULL THEN 1 ELSE 0 END) as settled FROM claims'
+    ).get();
     console.log(JSON.stringify({
       ...brainStats(),
       ...(scope ? { scope } : {}),
       projects: perProject,
+      forgetting,
+      claims: { ...claims, record: calibration() },
+      dead_memories: deadMemories().length,
       replay_conflicts: listConflicts(20),
     }, null, 2));
     break;
   }
+  case 'brief': {
+    // The problem-understanding stage: ingest a context document (file or
+    // inline text), compile its requirements to Gate checks, optionally
+    // verify now. scion brief <file> [--verify] | scion brief show <id>
+    const briefs = await import('../src/mind/brief.js');
+    if (arg === 'show') {
+      try { console.log(JSON.stringify(briefs.getBrief(Number(restArgs[0])), null, 2)); }
+      catch (e) { console.error('✗ ' + e.message); process.exitCode = 1; }
+      break;
+    }
+    if (arg === 'list') {
+      console.log(JSON.stringify(briefs.listBriefs(), null, 2));
+      break;
+    }
+    try {
+      const { readFileSync } = await import('node:fs');
+      const text = arg && arg !== '-' && existsSync(arg) ? readFileSync(arg, 'utf8') : (arg || '');
+      if (!text.trim()) { console.error('✗ brief: provide a file path or inline text'); process.exitCode = 1; break; }
+      const b = briefs.ingest({ text, title: restArgs.find(x => !x.startsWith('--')), source: 'cli' });
+      const c = briefs.compileBrief(b.brief.id);
+      const out = { brief: b.brief, entities: b.entities.length, requirements: b.requirements.length, compiled: c.compiled, judgment: c.judgment };
+      if (process.argv.includes('--verify')) {
+        const v = await briefs.verifyBrief(b.brief.id);
+        out.verification = { passed: v.passed, failed: v.failed, coverage: v.coverage, uncheckable: v.uncheckable };
+        out.results = v.results;
+      }
+      console.log(JSON.stringify(out, null, 2));
+    } catch (e) { console.error('✗ ' + e.message); process.exitCode = 1; }
+    break;
+  }
+  case 'lang': {
+    // The multilingual layer: list configured languages, or set the active
+    // one (persisted per brain, like project scopes).
+    const langs = await import('../src/mind/langs.js');
+    if (!arg || arg === 'list') {
+      const packs = langs.loadPacks();
+      console.log(JSON.stringify({
+        active: langs.activeLanguage(),
+        languages: ['en', ...langs.configuredLanguages()],
+        packs: Object.values(packs).map(p => ({ lang: p.lang, name: p.name, script: p.script, entries: p.entries.length })),
+      }, null, 2));
+    } else if (arg === 'set') {
+      try { console.log(JSON.stringify(langs.setActiveLanguage(restArgs[0]), null, 2)); }
+      catch (e) { console.error('✗ ' + e.message); process.exitCode = 1; }
+    } else {
+      try { console.log(JSON.stringify(langs.setActiveLanguage(arg), null, 2)); }
+      catch (e) { console.error('✗ ' + e.message); process.exitCode = 1; }
+    }
+    break;
+  }
+  case 'claim': {
+    // YOINK lineage: turn a memory into a bet with a date on it.
+    // scion claim <memoryId> --claim "..." --settles 2027-01-01 --reads file:...#json.path --test "gte 10" --confidence 0.7
+    const { declare } = await import('../src/mind/claims.js');
+    const flag = (name) => {
+      const i = restArgs.indexOf('--' + name);
+      return i >= 0 ? restArgs[i + 1] : undefined;
+    };
+    try {
+      const r = declare({
+        memoryId: Number(arg),
+        claim: flag('claim'),
+        settles: flag('settles'),
+        reads: flag('reads'),
+        test: flag('test'),
+        confidence: Number(flag('confidence')),
+      });
+      console.log(JSON.stringify(r, null, 2));
+    } catch (e) {
+      console.error('✗ ' + e.message); process.exitCode = 1;
+    }
+    break;
+  }
+  case 'settle': {
+    // Settle everything due: read the sources, apply the tests, write the
+    // outcomes back beside the thoughts. Unreadable sources stay open.
+    const { settleAll } = await import('../src/mind/claims.js');
+    const r = await settleAll();
+    console.log(JSON.stringify(r, null, 2));
+    break;
+  }
+  case 'calibrate': {
+    // The record: said vs right — the gap and the Brier score.
+    const { calibration, deadMemories } = await import('../src/mind/claims.js');
+    console.log(JSON.stringify({
+      record: calibration(),
+      dead: deadMemories({ minAgeDays: Number(restArgs[0]) || 14 }),
+      trail_chain: (await import('../src/kernel/trail.js')).verifyChain(),
+    }, null, 2));
+    break;
+  }
   case 'conflicts': {
     const { listConflicts, clearConflicts } = await import('../src/mind/replay.js');
     if (arg === '--clear') {
@@ -732,7 +866,7 @@ switch (cmd || '') {
       opener.unref();
     }
     const sweep = setInterval(() => {
-      try { dailyMaintenance(); } catch { /* maintenance never kills the server */ }
+      dailyMaintenance().catch(() => { /* maintenance never kills the server */ });
     }, 60_000);
     for (const sig of ['SIGINT', 'SIGTERM']) {
       process.on(sig, async () => {
@@ -876,6 +1010,15 @@ switch (cmd || '') {
     }
     break;
   }
+  case '--version':
+  case '-v': {
+    // Self-reported version — the FS Desktop shell probes this to detect the
+    // kernel and warn on shell/kernel version skew.
+    let kernelPkg = { version: 'dev' };
+    try { kernelPkg = JSON.parse(readFileSync(resolve(fileURLToPath(import.meta.url), '../../package.json'), 'utf8')); } catch { /* dev checkout without package.json */ }
+    console.log(kernelPkg.version);
+    break;
+  }
   default:
     console.error(`unknown command: ${cmd}\nusage: scion [run "<task>" | swarm [n] | plan <target> [--out m.json] [--run] | workflow <manifest.json> | analyze [dir] | fix <file> | watch [dir] [--auto-fix] [--concurrency N] | economy | brain | remember "<c>" | search "<q>" | reason "<topic>" | forge [--dry-run] | review [-i] | resolve <id> approve "<fix>"|reject|defer | gaps [teach <id> <intent> | forget <id> | --all] | scopes [list|add <name>|rm <name>] | ui | serve [port] [--follower --lease-ms N --leader-poll-ms N --primary-url URL] | remote --url URL --workerId ID | leader | recipes [doctor | show <name>]`);
     process.exitCode = 1;

package/knowledge/lexicon.de.yaml

@@ -0,0 +1,24 @@
+# knowledge/lexicon.de.yaml — German language pack (data, not code).
+# Loaded by mind/langs.js; consulted by interpret() as static evidence
+# BEFORE learned terms. Adding a language = adding a file.
+lang: de
+name: German
+script: latin
+description: German command vocabulary — surface forms map to core intents
+entries:
+  - intent: fix
+    words: [beheben, reparieren, korrigieren, flicken]
+  - intent: run
+    words: [ausführen, starten, laufen, abschicken]
+  - intent: analyze
+    words: [analysieren, prüfen, scannen, untersuchen]
+  - intent: show
+    words: [zeigen, anzeigen, auflisten, status, bericht]
+  - intent: search
+    words: [suchen, finden, nachschlagen]
+  - intent: remember
+    words: [merken, speichern, notieren, aufzeichnen]
+  - intent: explain
+    words: [erklären, warum, wie, beschreiben]
+  - intent: plan
+    words: [planen, plan, terminieren]

package/knowledge/lexicon.ja.yaml

@@ -0,0 +1,25 @@
+# knowledge/lexicon.ja.yaml — Japanese language pack (data, not code).
+# CJK script: phrases have no word boundaries, so matching is Tier-2
+# closed-set longest-match over the raw text — enough for command
+# vocabularies, which are small fixed lists.
+lang: ja
+name: Japanese
+script: cjk
+description: Japanese command vocabulary — closed-set phrase matching
+entries:
+  - intent: fix
+    words: [修正, 修復, 直す, 改修]
+  - intent: run
+    words: [実行, 起動, 走らせる]
+  - intent: analyze
+    words: [解析, 分析, 調査]
+  - intent: show
+    words: [表示, 一覧, 状態, 報告]
+  - intent: search
+    words: [検索, 探す, 見つける]
+  - intent: remember
+    words: [記憶, 保存, 覚える, 記録]
+  - intent: explain
+    words: [説明, なぜ, どうして]
+  - intent: plan
+    words: [計画, 計画する, 見積]

package/knowledge/lexicon.tr.yaml

@@ -0,0 +1,24 @@
+# knowledge/lexicon.tr.yaml — Turkish language pack (data, not code).
+# Loaded by mind/langs.js; consulted by interpret() as static evidence
+# BEFORE learned terms. Adding a language = adding a file.
+lang: tr
+name: Turkish
+script: latin
+description: Turkish command vocabulary — surface forms map to core intents
+entries:
+  - intent: fix
+    words: [düzelt, düzeltme, onar, tamir, hallet]
+  - intent: run
+    words: [çalıştır, koş, yürüt, başlat]
+  - intent: analyze
+    words: [analiz, incele, tara, denetle]
+  - intent: show
+    words: [göster, listele, durum, rapor]
+  - intent: search
+    words: [ara, bul, hatırla]
+  - intent: remember
+    words: [hatırla, not al, kaydet, kayıt]
+  - intent: explain
+    words: [açıkla, neden, nasıl, anlat]
+  - intent: plan
+    words: [planla, plan, çizelgele]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "futura-scion",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "The fused scion of cortex-os-agent + persona: one zero-LLM-dependent agent stack — Mind proposes, Muscle executes, Gate disposes.",
   "type": "module",
   "main": "src/index.js",

package/src/brain/db.js

@@ -93,13 +93,35 @@ export function ensureSchema(db) {
       layer INTEGER DEFAULT 1,
       access_count INTEGER DEFAULT 0,
       last_accessed_at TEXT,
+      expires_at TEXT,
+      superseded_by INTEGER,
       created_at TEXT NOT NULL DEFAULT (datetime('now'))
     );
     CREATE INDEX IF NOT EXISTS idx_memories_project ON memories(project);
     CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type);
     CREATE INDEX IF NOT EXISTS idx_memories_active ON memories(is_active);
+    CREATE INDEX IF NOT EXISTS idx_memories_expires ON memories(is_active, expires_at);
   `);
 
+  // Semantic vectors (cortex embedding.js lineage): one BLOB per memory —
+  // a deterministic 256-dim hashed-feature vector (brain/embedding.js).
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS memory_vectors (
+      id INTEGER PRIMARY KEY,
+      vec BLOB NOT NULL
+    );
+  `);
+
+  // Temporal forgetting migration (cortex forgetting.js lineage): existing
+  // DBs gain expires_at + superseded_by. Idempotent via PRAGMA inspection.
+  const memoryCols = db.prepare(`PRAGMA table_info(memories)`).all().map(c => c.name);
+  if (!memoryCols.includes('expires_at')) {
+    db.exec(`ALTER TABLE memories ADD COLUMN expires_at TEXT`);
+  }
+  if (!memoryCols.includes('superseded_by')) {
+    db.exec(`ALTER TABLE memories ADD COLUMN superseded_by INTEGER`);
+  }
+
   // FTS5 virtual table + sync triggers for keyword search
   try {
     db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(title, content, tags, content=memories, content_rowid=id)`);
@@ -171,11 +193,69 @@ export function ensureSchema(db) {
       seq INTEGER PRIMARY KEY AUTOINCREMENT,
       ts INTEGER NOT NULL,
       kind TEXT NOT NULL,
-      data TEXT NOT NULL
+      data TEXT NOT NULL,
+      prev_hash TEXT,
+      hash TEXT
     );
     CREATE INDEX IF NOT EXISTS idx_trail_kind ON trail_events(kind);
   `);
 
+  // Hash-chain migration (YOINK ledger lineage): existing trails gain the
+  // linkage columns. Old rows chain from NULL backward — the chain verifier
+  // treats pre-migration rows as the genesis segment.
+  const trailCols = db.prepare(`PRAGMA table_info(trail_events)`).all().map(c => c.name);
+  if (!trailCols.includes('prev_hash')) db.exec(`ALTER TABLE trail_events ADD COLUMN prev_hash TEXT`);
+  if (!trailCols.includes('hash')) db.exec(`ALTER TABLE trail_events ADD COLUMN hash TEXT`);
+
+  // ── Mind: claims (YOINK lineage) — falsifiable forecasts with a settle
+  // date, a named source, and a confidence that gets graded. ──
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS claims (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      memory_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
+      claim TEXT NOT NULL,
+      settles TEXT NOT NULL,
+      source TEXT NOT NULL,
+      test TEXT NOT NULL,
+      confidence REAL NOT NULL,
+      outcome INTEGER,
+      reading TEXT,
+      settled_at TEXT,
+      read_from TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+    CREATE INDEX IF NOT EXISTS idx_claims_due ON claims(outcome, settles);
+  `);
+
+  // ── Mind: briefs (the problem-understanding stage) — a body of human
+  // context (ticket/spec/conversation) becomes a STRUCTURED problem model:
+  // entities, business rules, acceptance criteria. Requirements carry their
+  // own checkability class; compiled checks are the Gate's handles. ──
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS briefs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      title TEXT,
+      source TEXT,
+      text TEXT NOT NULL,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+    CREATE TABLE IF NOT EXISTS requirements (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      brief_id INTEGER NOT NULL REFERENCES briefs(id) ON DELETE CASCADE,
+      kind TEXT NOT NULL,
+      text TEXT NOT NULL,
+      subject TEXT,
+      checkable INTEGER NOT NULL DEFAULT 0,
+      check_spec TEXT,
+      compiled_cmd TEXT,
+      status TEXT NOT NULL DEFAULT 'open',
+      last_verified_at TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+    CREATE INDEX IF NOT EXISTS idx_req_brief ON requirements(brief_id);
+    CREATE INDEX IF NOT EXISTS idx_req_status ON requirements(status);
+  `);
+
   // ── Mind: replay index (Tier-0 learned decisions, now persistent) ──
   db.exec(`
     CREATE TABLE IF NOT EXISTS replay_index (

package/src/brain/embedding.js

@@ -0,0 +1,120 @@
+/**
+ * brain/embedding.js — deterministic semantic vectors (the embeddings organ,
+ * ported from agentic-cortex's embedding.js with a doctrine-conscious swap).
+ *
+ * Cortex embeds with @xenova/transformers (BGE-base, ~400 MB download, model
+ * inference). FS's doctrine is zero-dependency and least-LLM, so the same
+ * CAPABILITY arrives as a MECHANISM instead: hashed feature embeddings.
+ *
+ *   embed(text) → a 256-dim L2-normalized vector built from word unigrams,
+ *   bigrams, and character trigrams, each feature hashed (FNV-1a) into the
+ *   dimension space with a sign. Fully deterministic: the same text always
+ *   yields the same vector, on every machine, with no model, no download,
+ *   no network — and the quality floor is exactly the token-overlap signal
+ *   the memory graph already uses, plus SUB-WORD and WORD-ORDER tolerance
+ *   that bag-of-tokens misses ("authfailure" ≈ "auth failure", "deploy
+ *   failed" ≈ "failure on deploy").
+ *
+ * Honest limits (vs a neural encoder): no synonymy ("car" ≉ "automobile"),
+ * no cross-lingual transfer. Synonymy is already handled separately by the
+ * NLU layer's synonym expansion at query time, which is why this floor is
+ * acceptable. A neural encoder can replace `embed()` later without touching
+ * any call site — the vector is opaque to consumers.
+ *
+ * @module brain/embedding
+ */
+
+'use strict';
+
+export const DIMS = 512;
+
+/** FNV-1a 32-bit — fast, deterministic, well-distributed for short strings. */
+function fnv1a(str) {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i);
+    h = (h + (h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24)) >>> 0;
+  }
+  return h;
+}
+
+/** Tokenize into words (letters/digits), lowercased. */
+function words(text) {
+  return String(text || '').toLowerCase().match(/[a-z0-9_]+/g) || [];
+}
+
+/** Pure-function stopwords: contribute no signal to vector space. */
+export const STOPWORDS = new Set([
+  'the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'be',
+  'been', 'being', 'to', 'of', 'in', 'on', 'at', 'for', 'with', 'by', 'from',
+  'that', 'this', 'it', 'its', 'as', 'when', 'then', 'than', 'so', 'if',
+  'we', 'you', 'i', 'they', 'he', 'she', 'do', 'does', 'did', 'not', 'no',
+]);
+
+/** Is this token a pure-function stopword (contentless for retrieval)? */
+export function isStopword(w) {
+  return STOPWORDS.has(String(w || '').toLowerCase());
+}
+
+/**
+ * Embed text into a fixed-dim L2-normalized vector.
+ * Features: word unigrams (weight 1.0), adjacent bigrams (weight 1.5 —
+ * word-order signal), character trigrams of each word (weight 0.5 —
+ * sub-word/morphology tolerance: "login" ≈ "logins" ≈ "logging-in").
+ *
+ * @param {string} text
+ * @returns {Float64Array} DIMS-length normalized vector
+ */
+export function embed(text) {
+  const v = new Float64Array(DIMS);
+  const add = (feature, weight) => {
+    const h = fnv1a(feature);
+    const idx = h % DIMS;
+    const sign = (h & 0x80000000) ? -1 : 1;
+    v[idx] += sign * weight;
+  };
+
+  const ws = words(text);
+  const content = ws.filter(w => !STOPWORDS.has(w));
+  for (let i = 0; i < ws.length; i++) {
+    if (!STOPWORDS.has(ws[i])) {
+      add(ws[i], 1.5);
+      const w = ws[i];
+      if (w.length >= 3) {
+        for (let j = 0; j + 3 <= w.length; j++) add('§' + w.slice(j, j + 3), 1.0);
+      } else {
+        add('§' + w, 1.0);
+      }
+    }
+    // Bigrams bridge stopwords — "pool of connections" keeps the pair.
+    if (i + 1 < ws.length) add(ws[i] + '_' + ws[i + 1], 1.5);
+  }
+
+  let norm = 0;
+  for (let i = 0; i < DIMS; i++) norm += v[i] * v[i];
+  norm = Math.sqrt(norm);
+  if (norm > 0) for (let i = 0; i < DIMS; i++) v[i] /= norm;
+  return v;
+}
+
+/** Cosine similarity (inputs expected L2-normalized; safe for zero vectors). */
+export function cosine(a, b) {
+  let dot = 0;
+  for (let i = 0; i < DIMS; i++) dot += a[i] * b[i];
+  return Math.max(-1, Math.min(1, dot));
+}
+
+/** Convenience: similarity of two texts, 0..1. */
+export function similarity(textA, textB) {
+  return cosine(embed(textA), embed(textB));
+}
+
+/** Serialize a vector for SQLite BLOB storage. */
+export function pack(vec) {
+  return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
+}
+
+/** Deserialize a vector from a SQLite BLOB. */
+export function unpack(buf) {
+  return new Float64Array(buf.buffer, buf.byteOffset, buf.byteLength / 8);
+}