open-memex 0.7.3 → 0.8.0-alpha.2

package/scripts/bench-semantic.ts

@@ -0,0 +1,198 @@
+/**
+ * Semantic-ranking experiment (the D77 embedding gate, Stone 2026-10-07).
+ *
+ * Question: does a local multilingual embedding + RRF hybrid beat the
+ * lexical ceiling? Target: pooled LongMemEval-S R@5 40.6% / MRR 0.284.
+ * Also measured on the synthetic fixture, esp. the 5 cross-lingual
+ * vocabulary-mismatch queries (the slice Stone cares about).
+ *
+ * Model: paraphrase-multilingual-MiniLM-L12-v2 via the `transformers` npm
+ * package (local, no API). Chosen multilingual because the mismatch slice
+ * is EN<->ZH. Embeddings are cached to disk; cosine is brute-forced in JS
+ * (22k x 384 is milliseconds).
+ *
+ * Modes:
+ *   --pooled --data <longmemeval_s_cleaned.json>   22k sessions, 470 questions
+ *   --fixture                                      51 synthetic memories, 41 queries
+ *
+ * Usage:
+ *   HF_HUB_CACHE=~/workspace/bench-data/hf-cache node --experimental-strip-types \
+ *     scripts/bench-semantic.ts --pooled --data ~/workspace/bench-data/longmemeval_s_cleaned.json
+ */
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const args = process.argv.slice(2);
+const POOLED = args.includes("--pooled");
+const dataIdx = args.indexOf("--data");
+const dataPath = dataIdx === -1 ? "" : args[dataIdx + 1];
+const limitIdx = args.indexOf("--limit");
+const LIMIT = limitIdx === -1 ? Infinity : Number(args[limitIdx + 1]);
+const MODEL = (() => {
+  const i = args.indexOf("--model");
+  return i === -1 ? "Xenova/paraphrase-multilingual-MiniLM-L12-v2" : args[i + 1];
+})();
+const K = 10;
+const DEPTH = 50;
+
+const HOME = fs.mkdtempSync(path.join(os.tmpdir(), "sem-bench-"));
+process.env.OPEN_MEMEX_HOME = HOME;
+
+import { pipeline } from "@huggingface/transformers";
+import { serialize, normalizeFrontmatter } from "../src/store/markdown.ts";
+import { syncScope } from "../src/store/sync.ts";
+import { search } from "../src/retrieve/search.ts";
+import { memoriesDirPath } from "../src/paths.ts";
+
+interface Doc {
+  id: string;
+  text: string;
+}
+interface Q {
+  text: string;
+  gold: Set<string>;
+  note: string;
+}
+
+const docs: Doc[] = [];
+const questions: Q[] = [];
+
+if (POOLED) {
+  if (!dataPath) throw new Error("missing --data");
+  const items = JSON.parse(fs.readFileSync(dataPath, "utf8"));
+  let qi = 0;
+  for (const item of items) {
+    if (item.question_id.endsWith("_abs")) continue;
+    if (qi >= LIMIT) break;
+    const sids: string[] = item.haystack_session_ids;
+    const gold = new Set<string>();
+    for (const g of item.answer_session_ids) {
+      const i = sids.indexOf(g);
+      if (i !== -1) gold.add(`q${qi}-s${i}`);
+    }
+    if (gold.size === 0) continue;
+    item.haystack_sessions.forEach((turns: Array<{ role: string; content: string }>, i: number) => {
+      docs.push({
+        id: `q${qi}-s${i}`,
+        text: turns.map((t) => `${t.role}: ${t.content}`).join("\n").slice(0, 2000),
+      });
+    });
+    questions.push({ text: item.question, gold, note: item.question_type });
+    qi++;
+  }
+} else {
+  const { MEMORIES, QUERIES } = await import("./retrieval-eval/fixture.ts");
+  for (const m of MEMORIES) {
+    docs.push({
+      id: m.id,
+      text: [m.body, ...(m.aliases ?? []), ...(m.tags ?? [])].join("\n").slice(0, 2000),
+    });
+  }
+  for (const q of QUERIES.slice(0, LIMIT)) {
+    questions.push({ text: q.query, gold: new Set(q.gold), note: q.note });
+  }
+}
+
+console.log(`docs: ${docs.length}, questions: ${questions.length}, model: ${MODEL}`);
+
+// Seed the FTS index (both modes) so the BM25 arm runs on the same corpus.
+{
+  const dir = memoriesDirPath("personal");
+  fs.mkdirSync(dir, { recursive: true });
+  const B = 2000;
+  for (let i = 0; i < docs.length; i += B) {
+    for (const d of docs.slice(i, i + B)) {
+      const fm = normalizeFrontmatter({ id: d.id, scope: "personal", type: "fact", tags: [] });
+      fs.writeFileSync(path.join(dir, `${d.id}.md`), serialize(fm, d.text), "utf8");
+    }
+    console.log(`  seeded ${Math.min(i + B, docs.length)}/${docs.length}`);
+  }
+  syncScope("personal");
+}
+
+// ---- embeddings (cached) ----
+const cachePath = POOLED
+  ? `${os.homedir()}/workspace/bench-data/emb-pooled.json`
+  : `${os.homedir()}/workspace/bench-data/emb-fixture.json`;
+let emb: number[][];
+if (fs.existsSync(cachePath)) {
+  console.log("loading cached embeddings…");
+  emb = JSON.parse(fs.readFileSync(cachePath, "utf8"));
+} else {
+  console.log("loading model… (first run downloads ~470MB)");
+  const extractor: any = await pipeline("feature-extraction", MODEL);
+  emb = [];
+  const B = 32;
+  for (let i = 0; i < docs.length; i += B) {
+    const batch = docs.map((d) => d.text).slice(i, i + B);
+    const out: any = await extractor(batch, { pooling: "mean", normalize: true });
+    const arr = out.tolist();
+    for (const v of arr) emb.push(v);
+    if (i % 320 === 0) console.log(`  embedded ${i}/${docs.length}`);
+  }
+  fs.writeFileSync(cachePath, JSON.stringify(emb));
+  console.log("embeddings cached");
+}
+const DIM = emb[0].length;
+console.log(`embedding dim: ${DIM}`);
+
+const extractor: any = await pipeline("feature-extraction", MODEL);
+
+function topVector(queryVec: number[], depth: number): string[] {
+  const scores: Array<{ id: string; s: number }> = [];
+  for (let i = 0; i < emb.length; i++) {
+    const v = emb[i];
+    let s = 0;
+    for (let d = 0; d < DIM; d++) s += queryVec[d] * v[d];
+    scores.push({ id: docs[i].id, s });
+  }
+  scores.sort((a, b) => b.s - a.s);
+  return scores.slice(0, depth).map((x) => x.id);
+}
+
+function rrf(lists: string[][], k = 60): string[] {
+  const scores = new Map<string, number>();
+  for (const list of lists) {
+    list.forEach((id, rank) => {
+      scores.set(id, (scores.get(id) ?? 0) + 1 / (k + rank + 1));
+    });
+  }
+  return [...scores.entries()].sort((a, b) => b[1] - a[1]).map(([id]) => id);
+}
+
+let bR5 = 0, bMRR = 0, vR5 = 0, vMRR = 0, hR5 = 0, hMRR = 0;
+const missDetail: string[] = [];
+let qi = 0;
+for (const q of questions) {
+  const bm25 = search(q.text, { scopeKeys: ["personal"], limit: DEPTH, synonyms: false }).map(
+    (h) => h.id,
+  );
+  const qv: number[] = (await extractor(q.text, { pooling: "mean", normalize: true })).tolist()[0];
+  const vec = topVector(qv, DEPTH);
+  const hyb = rrf([bm25, vec]).slice(0, K);
+
+  const rankIn = (top: string[]): number => {
+    for (let r = 0; r < top.slice(0, K).length; r++) if (q.gold.has(top[r])) return r;
+    return -1;
+  };
+  const rb = rankIn(bm25), rv = rankIn(vec), rh = rankIn(hyb);
+  if (rb !== -1 && rb < 5) bR5++;
+  if (rb !== -1) bMRR += 1 / (rb + 1);
+  if (rv !== -1 && rv < 5) vR5++;
+  if (rv !== -1) vMRR += 1 / (rv + 1);
+  if (rh !== -1 && rh < 5) hR5++;
+  if (rh !== -1) hMRR += 1 / (rh + 1);
+  if (!POOLED && rh === -1) {
+    missDetail.push(`  MISS(hybrid) query="${q.text}" gold=${[...q.gold].join(",")} [${q.note}] bm25rank=${rb} vecrank=${rv}`);
+  }
+  qi++;
+  if (qi % 100 === 0) console.log(`  ... ${qi}/${questions.length}`);
+}
+
+const n = questions.length;
+console.log(`\n==== semantic experiment (${POOLED ? "pooled 22k" : "fixture"}) ====`);
+console.log(`BM25-only : R@5 ${(bR5 / n * 100).toFixed(1)}%  MRR ${(bMRR / n).toFixed(3)}`);
+console.log(`vec-only  : R@5 ${(vR5 / n * 100).toFixed(1)}%  MRR ${(vMRR / n).toFixed(3)}`);
+console.log(`hybrid RRF: R@5 ${(hR5 / n * 100).toFixed(1)}%  MRR ${(hMRR / n).toFixed(3)}`);
+if (missDetail.length) console.log("hybrid misses:\n" + missDetail.join("\n"));

package/scripts/retrieval-eval/fixture.ts

@@ -0,0 +1,167 @@
+/**
+ * Synthetic retrieval-eval fixture (D77).
+ *
+ * 48 personal-scope memories + 36 queries with gold ids. Written BEFORE
+ * running the eval — the fixture is the test, not tuned to the results.
+ * All memories share one scope/tier so the measured ranking is pure BM25.
+ * Mixed EN/ZH; includes paraphrase and synonym queries (the hard cases
+ * from the retrieval-robustness question) and near-duplicate distractors.
+ */
+
+export interface FixtureMemory {
+  id: string;
+  type: string;
+  tags: string[];
+  aliases?: string[];
+  body: string;
+}
+
+export interface FixtureQuery {
+  query: string;
+  gold: string[];
+  note: string;
+}
+
+export const MEMORIES: FixtureMemory[] = [
+  // ---- deploy & CI ----
+  { id: "mem-001", type: "decision", tags: ["deploy", "ci"], body: "Deploy to production only from the release branch. Merging to main triggers staging; the release branch is cut from main every Thursday and production deploys only from release/*." },
+  { id: "mem-002", type: "gotcha", tags: ["ci", "github-actions"], body: "GitHub Actions macOS runners are 10x the per-minute cost of linux. The e2e job was moved to ubuntu-latest and macOS is kept only for the notarization step." },
+  { id: "mem-003", type: "howto", tags: ["deploy", "rollback"], body: "To roll back a production deploy: revert the merge commit on the release branch, push, and the pipeline redeploys the previous artifact. Do not redeploy old artifacts manually from the Actions UI." },
+  { id: "mem-004", type: "fact", tags: ["ci"], body: "CI runs lint, typecheck, unit tests, and the smoke suite on every PR. The full e2e suite runs only on the release branch to save minutes." },
+  // distractor: shares deploy vocabulary, different topic
+  { id: "mem-005", type: "fact", tags: ["deploy"], body: "The deploy dashboard is at deploys.internal.example.com. It shows who deployed what and when, but rollbacks must still go through the release branch revert flow." },
+
+  // ---- testing ----
+  { id: "mem-006", type: "preference", tags: ["testing"], body: "Prefer table-driven tests over copy-pasted cases. One test function, a slice of inputs and expected outputs — much easier to extend than five near-identical test blocks." },
+  { id: "mem-007", type: "gotcha", tags: ["testing", "flaky"], body: "The checkout flow test is flaky on CI because it depends on wall-clock timing. It was quarantined with t.Parallel disabled and a retry wrapper; do not unquarantine without fixing the clock dependency." },
+  { id: "mem-008", type: "decision", tags: ["testing", "coverage"], body: "Coverage gate is 80% on new code, measured by the CI coverage job. Legacy modules are exempt until they are touched — then the touched lines must be covered." },
+
+  // ---- editor & tools ----
+  { id: "mem-009", type: "preference", tags: ["editor", "vscode"], body: "VS Code: format on save with the workspace prettier config, never the global one. The workspace config lives in .vscode/settings.json and wins over user settings." },
+  { id: "mem-010", type: "howto", tags: ["git"], aliases: ["how to undo last commit", "git revert commit"], body: "To undo the last local commit but keep the changes: git reset --soft HEAD~1. To discard everything including working tree: git reset --hard HEAD~1 (destructive, no undo)." },
+  { id: "mem-011", type: "fact", tags: ["node"], body: "The repo pins Node 22.14.0 in .nvmrc. CI and the Dockerfile both use it; running 20.x locally causes the better-sqlite3 native binding to fail to load." },
+
+  // ---- database ----
+  { id: "mem-012", type: "gotcha", tags: ["postgres", "migrations"], body: "Postgres migrations must be backward compatible: never drop a column in the same release that stops writing to it. Expand first, migrate, then contract in the next release." },
+  { id: "mem-013", type: "decision", tags: ["postgres", "index"], body: "Added a partial index on orders(status) WHERE status = 'pending' after the pending-orders query started sequential-scanning 40M rows. Query time went from 9s to 40ms." },
+  { id: "mem-014", type: "howto", tags: ["sqlite"], aliases: ["sqlite busy timeout", "database is locked fix"], body: "SQLite 'database is locked' under concurrent writers: enable WAL mode and set a busy timeout (PRAGMA busy_timeout=5000). Readers never block writers in WAL." },
+
+  // ---- API design ----
+  { id: "mem-015", type: "decision", tags: ["api", "rest"], body: "Public API uses cursor pagination (opaque cursor param), never offset/limit. Offset breaks when rows are inserted concurrently; the cursor is the last seen id." },
+  { id: "mem-016", type: "lesson", tags: ["api", "versioning"], body: "Lesson from the v2 migration: versioning the whole API at once was a mistake. Additive changes ship unversioned; only breaking changes get a new version prefix." },
+
+  // ---- security ----
+  { id: "mem-017", type: "constraint", tags: ["security", "secrets"], body: "Never commit secrets. The pre-commit hook runs gitleaks; if a secret lands in history it must be rotated immediately — rewriting history does not unexpose it." },
+  { id: "mem-018", type: "howto", tags: ["security", "jwt"], aliases: ["refresh token rotation"], body: "JWT access tokens live 15 minutes; refresh tokens rotate on every use and are bound to the device fingerprint. A reused refresh token invalidates the whole token family." },
+
+  // ---- performance ----
+  { id: "mem-019", type: "gotcha", tags: ["performance", "n+1"], body: "The N+1 in the invoice endpoint came from lazy-loading line items per invoice in a loop. Fixed with a single JOIN fetch; p95 dropped from 2.1s to 180ms." },
+  { id: "mem-020", type: "decision", tags: ["performance", "caching"], aliases: ["cache invalidation strategy"], body: "Cache invalidation uses write-through for the product catalog: writes update both DB and cache synchronously. Stale reads were worse than the write latency cost." },
+
+  // ---- Chinese memories ----
+  { id: "mem-021", type: "decision", tags: ["部署", "规范"], body: "生产环境只允许周四发布，紧急 hotfix 需要 TL 在发布群里 @所有人 说明原因。周五不发布，避免周末 oncall。" },
+  { id: "mem-022", type: "gotcha", tags: ["微信", "小程序"], body: "微信小程序的 request 并发上限是 10，超过会直接失败不排队。批量上传图片必须自己做并发控制，分批 8 个一组发。" },
+  { id: "mem-023", type: "preference", tags: ["代码风格"], body: "中文注释写在代码上方，不要写在行尾。行尾注释在窄屏和 diff 里经常被截断，上方注释可读性更好。" },
+  { id: "mem-024", type: "howto", tags: ["git", "中文"], aliases: ["git 中文乱码", "commit message 乱码"], body: "Windows 上 git log 中文乱码：设置 git config --global core.quotepath false，同时把终端编码调成 UTF-8。" },
+  { id: "mem-025", type: "fact", tags: ["考勤"], body: "每月 5 号之前提交上月考勤，逾期系统自动锁定找 HR 解锁。弹性工作制但 10 点前要到岗。" },
+  // distractor sharing vocab
+  { id: "mem-026", type: "fact", tags: ["发布"], body: "发布群是企业微信里的「发版通知」群，CI 成功后机器人会自动推送版本号和 changelog 链接。" },
+
+  // ---- incidents ----
+  { id: "mem-027", type: "lesson", tags: ["incident", "postgres"], body: "Incident 2026-08: the primary ran out of disk because WAL archiving stalled silently. Lesson: alert on archive_command failures, not just disk usage — disk was the symptom." },
+  { id: "mem-028", type: "lesson", tags: ["incident", "dns"], body: "Incident 2026-09: DNS TTL of 24h turned a 5-minute failover into a day-long outage. Critical records now use 300s TTL; the long TTL only ever saved pennies." },
+
+  // ---- architecture decisions ----
+  { id: "mem-029", type: "decision", tags: ["architecture", "monorepo"], body: "Monorepo with pnpm workspaces won over polyrepo: cross-package refactors were the dominant pain and CI can already scope builds per changed package." },
+  { id: "mem-030", type: "decision", tags: ["architecture", "queue"], body: "Chose SQS over RabbitMQ for the notification pipeline: at-least-once delivery is fine there, and we did not want to operate another stateful service." },
+  { id: "mem-031", type: "fact", tags: ["architecture"], body: "The system has three services: api-gateway (edge), core (business logic), worker (async jobs). The worker scales independently on queue depth." },
+
+  // ---- frontend ----
+  { id: "mem-032", type: "gotcha", tags: ["react", "hooks"], body: "React useEffect with an empty dep array still re-runs in StrictMode dev double-invoke. Effects that POST must be idempotent or guarded by a ref — production is fine, dev double-fires." },
+  { id: "mem-033", type: "preference", tags: ["css"], body: "Use CSS modules for component styles, global stylesheet only for resets and tokens. The one time we put component CSS global it leaked into the admin panel." },
+
+  // ---- personal productivity ----
+  { id: "mem-034", type: "preference", tags: ["productivity"], body: "Deep work block is 9–11am, no meetings. The calendar auto-declines invites in that window with a note pointing to the team agreement." },
+  { id: "mem-035", type: "fact", tags: ["productivity", "standup"], body: "Standup is async in Slack #standup by 10am. Format: yesterday / today / blockers, one line each. No video call unless someone flags a blocker." },
+
+  // ---- more distractors & variety ----
+  { id: "mem-036", type: "fact", tags: ["ci"], body: "The CI badge in the README reflects the main branch only. PR builds show as checks on the PR itself, not on the badge." },
+  { id: "mem-037", type: "howto", tags: ["docker"], body: "Multi-stage Docker build: first stage compiles with dev dependencies, final stage copies only dist/ and production node_modules. Image went from 1.2GB to 180MB." },
+  { id: "mem-038", type: "gotcha", tags: ["docker", "arm"], body: "Docker builds on Apple Silicon default to arm64; the prod cluster is amd64. Always build with --platform linux/amd64 or the container crashes on deploy with exec format error." },
+  { id: "mem-039", type: "decision", tags: ["logging"], body: "Structured JSON logs everywhere, no free-text log lines. The log aggregator parses fields; free text broke the error-rate dashboard twice." },
+  { id: "mem-040", type: "preference", tags: ["code-review"], body: "Code review SLA: first review within 4 business hours. If you are blocked on review, ping in #dev — do not merge your own PR except for docs." },
+
+  // ---- synonyms/paraphrase targets ----
+  { id: "mem-041", type: "howto", tags: ["k8s"], aliases: ["kubernetes pod crashloop", "pod 一直重启"], body: "Kubernetes CrashLoopBackOff: check kubectl describe pod for the exit code first, then logs --previous. Nine times out of ten it is a missing env var, not the image." },
+  { id: "mem-042", type: "fact", tags: ["oncall"], body: "Oncall rotation is weekly, Monday to Monday. The handoff note must list: active incidents, risky deploys this week, and anything with a TODO owner on vacation." },
+  { id: "mem-043", type: "decision", tags: ["auth"], body: "SSO via OIDC is mandatory for all internal tools. Password-only auth was removed after the credential-stuffing scare in March; service accounts use mTLS instead." },
+  { id: "mem-044", type: "gotcha", tags: ["timezone"], body: "Store all timestamps in UTC, convert at the display edge. The reporting bug that mixed server-local time with UTC took a week to untangle because both looked plausible." },
+  { id: "mem-045", type: "howto", tags: ["ssh"], body: "SSH into prod goes through the bastion with agent forwarding disabled. Copy the key to the bastion instead — forwarding once let a compromised dev box pivot further." },
+  { id: "mem-046", type: "preference", tags: ["meetings"], body: "No-meeting Wednesdays for the eng team. Recurring meetings that land on Wednesday get moved, not exempted." },
+  { id: "mem-047", type: "fact", tags: ["vpn"], body: "The company VPN auto-disconnects after 12 hours. Long-running remote jobs should run on the dev server inside the network, not over a laptop VPN session." },
+  { id: "mem-048", type: "lesson", tags: ["estimation"], body: "Estimates are given as ranges (3–5 days), never single numbers. The single-number estimates were wrong 80% of the time; ranges forced the uncertainty conversation upfront." },
+  { id: "mem-049", type: "fact", tags: ["备份", "数据库"], body: "数据库备份每天凌晨两点执行，全量保留三十天，恢复演练每个季度做一次。" },
+  { id: "mem-050", type: "decision", tags: ["发布", "测试"], body: "上线前必须跑完全量回归测试，测试报告要贴到发布群里，TL 确认后才能发。" },
+  { id: "mem-051", type: "gotcha", tags: ["staging"], body: "The staging environment wipes itself every night at midnight. Never leave test data there overnight — it will be gone by morning." },
+];
+
+export const QUERIES: FixtureQuery[] = [
+  // ---- exact-term queries ----
+  { query: "production deploy release branch", gold: ["mem-001"], note: "exact terms" },
+  { query: "GitHub Actions macOS runner cost", gold: ["mem-002"], note: "exact terms" },
+  { query: "rollback production deploy revert", gold: ["mem-003"], note: "exact terms" },
+  { query: "table driven tests", gold: ["mem-006"], note: "exact terms" },
+  { query: "partial index pending orders", gold: ["mem-013"], note: "exact terms" },
+  { query: "cursor pagination offset", gold: ["mem-015"], note: "exact terms" },
+  { query: "WAL archiving disk full", gold: ["mem-027"], note: "exact terms" },
+  { query: "JWT refresh token rotation", gold: ["mem-018"], note: "exact terms" },
+
+  // ---- paraphrase queries (same meaning, different words) ----
+  { query: "how do I undo my last commit", gold: ["mem-010"], note: "paraphrase; alias covers it" },
+  { query: "sqlite locked concurrent writers", gold: ["mem-014"], note: "paraphrase; alias covers it" },
+  { query: "k8s pod keeps restarting", gold: ["mem-041"], note: "paraphrase; alias covers it" },
+  { query: "when can we ship to prod", gold: ["mem-001"], note: "paraphrase of deploy rule" },
+  { query: "tests that fail randomly on CI", gold: ["mem-007"], note: "paraphrase: flaky" },
+  { query: "why did the API versioning go wrong", gold: ["mem-016"], note: "paraphrase of lesson" },
+  { query: "slow invoice endpoint fix", gold: ["mem-019"], note: "paraphrase: N+1" },
+
+  // ---- synonym queries (no shared content words) ----
+  { query: "shipping code to production safely", gold: ["mem-001"], note: "synonym: shipping/deploy" },
+  { query: "unreliable test quarantine", gold: ["mem-007"], note: "synonym: unreliable/flaky" },
+  { query: "database locking issue", gold: ["mem-014"], note: "synonym" },
+  { query: "stale data problem cache", gold: ["mem-020"], note: "synonym: stale reads" },
+
+  // ---- Chinese queries ----
+  { query: "周四发布 hotfix 流程", gold: ["mem-021"], note: "CJK exact" },
+  { query: "小程序并发限制", gold: ["mem-022"], note: "CJK exact" },
+  { query: "git 中文显示乱码", gold: ["mem-024"], note: "CJK; alias covers it" },
+  { query: "注释写在哪里比较好", gold: ["mem-023"], note: "CJK paraphrase" },
+  { query: "考勤什么时候交", gold: ["mem-025"], note: "CJK paraphrase" },
+
+  // ---- cross-language (expected hard) ----
+  { query: "production release thursday rule", gold: ["mem-021"], note: "EN query, ZH memory — hard" },
+  { query: "wechat miniprogram request limit", gold: ["mem-022"], note: "EN query, ZH memory — hard" },
+
+  // ---- vocabulary-mismatch (translation pairs; the real-world slice LongMemEval-S lacks) ----
+  // Written 2026-10-07 BEFORE any index-time expansion or embedding layer exists:
+  // each query shares ~zero content words with its gold memory.
+  { query: "database backup retention policy", gold: ["mem-049"], note: "EN query, ZH memory — 备份/保留" },
+  { query: "full regression test before production release", gold: ["mem-050"], note: "EN query, ZH memory — 回归/上线" },
+  { query: "预发布环境数据为什么会丢", gold: ["mem-051"], note: "ZH query, EN memory — staging/wipe" },
+  { query: "单点登录强制要求", gold: ["mem-043"], note: "ZH query, EN memory — SSO/mandatory" },
+
+  // ---- distractor resistance ----
+  { query: "where is the deploy dashboard", gold: ["mem-005"], note: "must prefer mem-005 over mem-001/003" },
+  { query: "CI badge meaning", gold: ["mem-036"], note: "must prefer mem-036 over mem-004" },
+  { query: "docker image too big", gold: ["mem-037"], note: "must prefer mem-037 over mem-038" },
+  { query: "container crashes on deploy", gold: ["mem-038"], note: "must prefer mem-038 over mem-037" },
+  { query: "standup format", gold: ["mem-035"], note: "exact" },
+  { query: "oncall handoff", gold: ["mem-042"], note: "exact" },
+  { query: "monorepo or polyrepo", gold: ["mem-029"], note: "exact" },
+  { query: "timestamps UTC or local", gold: ["mem-044"], note: "paraphrase" },
+
+  // ---- synonym-only queries (round 1 sees none of these words; round 2 must catch them) ----
+  { query: "how do we ship", gold: ["mem-001"], note: "synonym-only: ship→deploy/release" },
+  { query: "pager schedule", gold: ["mem-042"], note: "synonym-only: pager→oncall" },
+  { query: "my token got exposed", gold: ["mem-017"], note: "synonym-only: token→secret" },
+];

package/scripts/retrieval-eval.ts

@@ -0,0 +1,81 @@
+/**
+ * Retrieval eval (D77, layer 1): offline FTS5 baseline on a synthetic fixture.
+ *
+ * Usage: node --experimental-strip-types scripts/retrieval-eval.ts
+ *
+ * Seeds a scratch OPEN_MEMEX_HOME with the fixture memories (all personal
+ * scope, so every memory sits in the same review tier and the measured
+ * ranking is pure BM25), runs each fixture query through search(), and
+ * reports recall@1 / recall@5 / MRR plus a per-query miss table.
+ *
+ * The fixture (scripts/retrieval-eval/fixture.ts) was written BEFORE the
+ * first run. Do not tune the fixture to the results — a miss is a finding,
+ * not a bug in the fixture.
+ */
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const HOME = fs.mkdtempSync(path.join(os.tmpdir(), "retrieval-eval-"));
+process.env.OPEN_MEMEX_HOME = HOME;
+
+import { MEMORIES, QUERIES } from "./retrieval-eval/fixture.ts";
+import { serialize, normalizeFrontmatter } from "../src/store/markdown.ts";
+import { syncScope } from "../src/store/sync.ts";
+import { search } from "../src/retrieve/search.ts";
+import { memoriesDirPath } from "../src/paths.ts";
+
+const K = 5;
+
+// --- seed ---
+const dir = memoriesDirPath("personal");
+fs.mkdirSync(dir, { recursive: true });
+for (const m of MEMORIES) {
+  const fm = normalizeFrontmatter({
+    id: m.id,
+    scope: "personal",
+    type: m.type,
+    tags: m.tags,
+    aliases: m.aliases,
+    source: "retrieval-eval fixture",
+  });
+  fs.writeFileSync(path.join(dir, `${m.id}.md`), serialize(fm, m.body), "utf8");
+}
+const stats = syncScope("personal");
+console.log(`seeded ${MEMORIES.length} memories (indexed: ${stats.added})\n`);
+
+// --- run ---
+let r1 = 0;
+let r5 = 0;
+let mrr = 0;
+const misses: string[] = [];
+
+for (const q of QUERIES) {
+  const stats = { ftsQuery: "", candidates: 0, hiddenSuperseded: 0, hiddenExcluded: 0 };
+  const hits = search(q.query, { scopeKeys: ["personal"], limit: 50, stats });
+  const ids = hits.map((h) => h.id);
+  let rank = -1;
+  for (const g of q.gold) {
+    const i = ids.indexOf(g);
+    if (i !== -1 && (rank === -1 || i < rank)) rank = i;
+  }
+  if (rank === 0) r1++;
+  if (rank !== -1 && rank < K) r5++;
+  if (rank !== -1) mrr += 1 / (rank + 1);
+  else misses.push(`  MISS  rank=-  query="${q.query}" gold=${q.gold.join(",")} [${q.note}]`);
+  if (rank > 0) {
+    console.log(`  rank=${rank + 1}  query="${q.query}" gold=${q.gold.join(",")} [${q.note}]`);
+  }
+}
+
+const n = QUERIES.length;
+console.log(`\n==== retrieval-eval: FTS5 baseline ====`);
+console.log(`corpus : ${MEMORIES.length} memories (personal scope, single tier)`);
+console.log(`queries: ${n}`);
+console.log(`recall@1: ${(r1 / n).toFixed(3)} (${r1}/${n})`);
+console.log(`recall@5: ${(r5 / n).toFixed(3)} (${r5}/${n})`);
+console.log(`MRR     : ${(mrr / n).toFixed(3)}`);
+if (misses.length > 0) {
+  console.log(`\nmisses (${misses.length}):`);
+  for (const m of misses) console.log(m);
+}

package/scripts/smoke-pure.ts

@@ -8,6 +8,7 @@ import { DEFAULT_CONFIG } from "../src/config.ts";
 import { resolveProjectScope, resolveCwdScope, pickScopeRoot, PERSONAL_SCOPE } from "../src/scope.ts";
 import { cjkIndexText, cjkQueryExpr, hasCjk } from "../src/retrieve/cjk.ts";
 import { toFtsQuery } from "../src/retrieve/query.ts";
+import { expansionVariantsForDoc, MAX_AUTO_VARIANTS } from "../src/retrieve/synonyms.ts";
 import { contentHash, similarity, NEAR_DUP_THRESHOLD } from "../src/store/lifecycle.ts";
 import {
   MIN_NODE_VERSION,
@@ -867,5 +868,31 @@ ok("cursor keeps skill", shouldInstallSkill("cursor", false) === true);
   ok("truncation note speaks only when cut", truncationNote(20, 54)?.includes("34 more") === true && truncationNote(20, 20) === null);
 }
 
+console.log("== D80 index-time expansion ==");
+{
+  // EN term expands to its group siblings, minus what's already in the text.
+  const v1 = expansionVariantsForDoc("Deploy to production only from the release branch.");
+  ok("ship expands from deploy", v1.includes("ship"), v1.join(","));
+  ok("no dupes of present terms", !v1.includes("deploy") && !v1.includes("release"), v1.join(","));
+  ok("function words are not keywords", !v1.includes("to") && !v1.includes("the"), v1.join(","));
+  // ZH substring expands both directions: doc has 数据库 → database indexed.
+  const v2 = expansionVariantsForDoc("数据库备份每天凌晨两点执行，全量保留三十天。");
+  ok("database indexed from 数据库", v2.includes("database"), v2.join(","));
+  ok("backup indexed from 备份", v2.includes("backup"), v2.join(","));
+  ok("retention indexed from 保留", v2.includes("retention"), v2.join(","));
+  // EN doc gets ZH variants: staging → 预发布环境.
+  const v3 = expansionVariantsForDoc("The staging environment wipes itself every night at midnight.");
+  ok("预发布环境 indexed from staging", v3.includes("预发布环境"), v3.join(","));
+  ok("wipes inflects to wipe group", v3.includes("清空"), v3.join(","));
+  // Multi-word Latin phrase: regression test.
+  const v4 = expansionVariantsForDoc("上线前必须跑完全量回归测试。");
+  ok("regression test indexed from 回归测试", v4.includes("regression test"), v4.join(","));
+  // Cap bounds index growth.
+  const v5 = expansionVariantsForDoc("test deploy bug cache queue api docker config log mock review commit merge branch");
+  ok("variant cap respected", v5.length <= MAX_AUTO_VARIANTS, String(v5.length));
+  // Empty / no-match input.
+  ok("no curated terms → no variants", expansionVariantsForDoc("hello world").length === 0);
+}
+
 console.log(fails === 0 ? "\nALL PASS" : `\n${fails} FAILURES`);
 process.exit(fails === 0 ? 0 : 1);

package/src/retrieve/query.ts

@@ -14,7 +14,7 @@
 import { cjkQueryExpr, hasCjk } from "./cjk.ts";
 
 /** English function words that carry no retrieval signal in questions. */
-const LATIN_STOPWORDS = new Set([
+export const LATIN_STOPWORDS = new Set([
   "the", "a", "an", "and", "or", "of", "to", "in", "on", "for", "with",
   "without", "is", "are", "was", "were", "be", "been", "do", "does", "did",
   "how", "what", "why", "when", "where", "which", "who", "whom", "we", "i",
@@ -32,9 +32,11 @@ const CJK_QUESTION_BIGRAMS = new Set([
 
 const MAX_TERMS = 24;
 
-function latinTerms(q: string): string[] {
+function latinTerms(q: string, useStopwords = true): string[] {
   const raw = q.toLowerCase().match(/[a-z0-9_.\-]+/g) ?? [];
-  const kept = raw.filter((t) => t.length > 1 && !LATIN_STOPWORDS.has(t));
+  const kept = useStopwords
+    ? raw.filter((t) => t.length > 1 && !LATIN_STOPWORDS.has(t))
+    : raw.filter((t) => t.length > 1);
   const chosen = kept.length > 0 ? kept : raw;
   return [...new Set(chosen)].slice(0, MAX_TERMS);
 }
@@ -44,15 +46,27 @@ function latinTerms(q: string): string[] {
  * Latin tokens keep the old behavior (prefix match on content/tags/type).
  * CJK runs become an OR of bigrams against the `cjk` column (see cjk.ts).
  * Mixed queries OR the two parts together.
+ *
+ * Flags are ablation toggles (all default to the shipped behavior):
+ * - stopwords: false  → skip function-word filtering
+ * - orJoin: false     → AND the terms instead of OR
+ * - prefix: false     → exact term match instead of `term*`
  */
-export function toFtsQuery(q: string): string {
-  const latin = latinTerms(q)
-    .map((t) => `"${t.replace(/"/g, '""')}"*`)
-    .join(" OR ");
+export function toFtsQuery(
+  q: string,
+  flags: { stopwords?: boolean; orJoin?: boolean; prefix?: boolean } = {},
+): string {
+  const useStopwords = flags.stopwords !== false;
+  const useOr = flags.orJoin !== false;
+  const usePrefix = flags.prefix !== false;
+  const joiner = useOr ? " OR " : " AND ";
+  const latin = latinTerms(q, useStopwords)
+    .map((t) => (usePrefix ? `"${t.replace(/"/g, '""')}"*` : `"${t.replace(/"/g, '""')}"`))
+    .join(joiner);
   const cjk = hasCjk(q)
-    ? cjkQueryExpr(q, { dropBigrams: CJK_QUESTION_BIGRAMS, maxTerms: MAX_TERMS })
+    ? cjkQueryExpr(q, { dropBigrams: useStopwords ? CJK_QUESTION_BIGRAMS : undefined, maxTerms: MAX_TERMS })
     : "";
-  if (latin && cjk) return `(${latin}) OR {cjk}:(${cjk})`;
+  if (latin && cjk) return `(${latin}) ${useOr ? "OR" : "AND"} {cjk}:(${cjk})`;
   if (cjk) return `{cjk}:(${cjk})`;
   return latin;
 }

package/src/retrieve/search.ts

@@ -158,9 +158,19 @@ function resolveVisible(rows: RawRow[], limit: number, stats?: SearchStats): Sea
  */
 export function search(
   query: string,
-  opts: { scopeKeys?: string[]; limit?: number; type?: string; stats?: SearchStats } = {},
+  opts: {
+    scopeKeys?: string[];
+    limit?: number;
+    type?: string;
+    stats?: SearchStats;
+    /** Ablation toggles for query construction (default = shipped behavior). */
+    stopwords?: boolean;
+    orJoin?: boolean;
+    prefix?: boolean;
+  } = {},
 ): SearchHit[] {
-  const q = toFtsQuery(query);
+  const qflags = { stopwords: opts.stopwords, orJoin: opts.orJoin, prefix: opts.prefix };
+  const q = toFtsQuery(query, qflags);
   if (opts.stats) {
     opts.stats.ftsQuery = q;
     opts.stats.candidates = 0;
@@ -190,14 +200,7 @@ export function search(
     ORDER BY score ASC
     LIMIT ?
   `;
-  const params: unknown[] = [q];
-  if (opts.scopeKeys && opts.scopeKeys.length > 0) params.push(...opts.scopeKeys);
-  if (opts.type) params.push(opts.type);
-  params.push(fetchLimit);
-
-  const rows = db()
-    .prepare(sql)
-    .all(...(params as any[])) as Array<{
+  type Row = {
     id: string;
     scope_key: string;
     project_name: string;
@@ -211,10 +214,24 @@ export function search(
     created_at: number;
     snippet: string;
     score: number;
-  }>;
+  };
+  const fetchFts = (matchExpr: string): RawRow[] => {
+    const params: unknown[] = [matchExpr];
+    if (opts.scopeKeys && opts.scopeKeys.length > 0) params.push(...opts.scopeKeys);
+    if (opts.type) params.push(opts.type);
+    params.push(fetchLimit);
+    const rows = db()
+      .prepare(sql)
+      .all(...(params as any[])) as Row[];
+    // FTS5 bm25: lower = better; invert for intuition.
+    return rows.map((r) => ({ ...r, score: -r.score }));
+  };
+
+  const raw: RawRow[] = fetchFts(q);
 
-  // FTS5 bm25: lower = better; invert for intuition.
-  const raw: RawRow[] = rows.map((r) => ({ ...r, score: -r.score }));
+  // D80: vocabulary mismatch is solved at index time (synonyms.ts expands
+  // curated variants into the aliases column at sync time), so the query
+  // side is a single FTS round — no trigger, no second chance.
   if (opts.stats) opts.stats.candidates = raw.length;
   return resolveVisible(raw, limit, opts.stats);
 }