@modusensus/dsh-mneme 0.12.0 → 0.14.0-beta.1

package/test/bm25-corpus-cache.test.js

@@ -0,0 +1,145 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { createStore } from "../src/store.js";
+import { createService } from "../src/service.js";
+
+// #617 审计 O2 回归：BM25 语料索引按 mirror_state.generation 缓存。
+//
+// 断言观测点选 store.list({ limit: 500 }) 的调用次数，不选耗时——耗时在 CI 上
+// 抖动大，而「这一轮到底有没有重新取语料重建索引」是确定性事实。
+//
+// 一正一反两把尺子：正向锁「缓存真的命中了」，反向（写入 / 归档 / 开关）锁
+// 「该失效的时候真的失效了」——只有正向会让「永不失效」的实现也全绿。
+
+function setup(over = {}) {
+  const store = createStore(":memory:");
+  const config = { entitySearchEnabled: false, bm25SearchEnabled: true, ...over };
+  const service = createService({ store, mirror: null, config, logger: null });
+  // 只数语料取数：searchMemories 里还有别的 list 调用，按 limit 精确匹配。
+  let corpusLoads = 0;
+  const origList = store.list.bind(store);
+  store.list = (opts) => {
+    if (opts?.limit === 500) corpusLoads += 1;
+    return origList(opts);
+  };
+  return { store, service, loads: () => corpusLoads };
+}
+
+// auto 且不装 embedder：向量路不启动，词法结果按 blend 的 auto 分支领位后由
+// BM25 补位——这条路径下 BM25 命中真的会进返回值，断言才钉在用户可见的行为上，
+// 而不是「索引建了但没人用」。
+const recall = (service, q) => service.searchMemories(q, { mode: "auto", topK: 5, useRerank: false, recordRecall: false });
+
+test("#617-O2: 同一水位下多条查询共用一份语料索引，不重复重建", async () => {
+  const { store, service, loads } = setup();
+  store.save({ type: "project", title: "异步并发模式", content: "async runtime 选用 tokio", importance: 3 });
+
+  const first = await recall(service, "rust 异步");
+  assert.equal(loads(), 1, "首查要建一次索引");
+  assert.ok(first.length > 0, "夹具前提：这条查询本来就有命中");
+
+  // 换一条查询：索引是按语料缓存的，不该因为 query 变了就重建。
+  await recall(service, "tokio");
+  await recall(service, "完全不相干的词");
+  assert.equal(loads(), 1, "语料没变，三次查询只该取一次语料");
+});
+
+test("#617-O2: 写入新记忆后索引失效，新行立刻可召回", async () => {
+  const { store, service, loads } = setup();
+  store.save({ type: "project", title: "夜间 ETL 脚本", content: "离线批处理", importance: 3 });
+  await recall(service, "离线批处理");
+  assert.equal(loads(), 1);
+
+  const fresh = store.save({ type: "project", title: "令牌桶限流", content: "ratelimit 令牌桶", importance: 3 });
+  const ids = (await recall(service, "令牌桶")).map((r) => r.id);
+  assert.equal(loads(), 2, "写了新行，语料变了，必须重建");
+  assert.ok(ids.includes(fresh.id), "重建后新行必须能被词法召回（陈旧索引会漏掉它）");
+});
+
+test("#617-O2: 归档后索引失效，归档行退出词法召回", async () => {
+  const { store, service, loads } = setup();
+  const m = store.save({ type: "project", title: "令牌桶限流", content: "ratelimit 令牌桶", importance: 3 });
+  assert.ok((await recall(service, "令牌桶")).some((r) => r.id === m.id), "夹具前提：归档前能召回");
+
+  store.setArchived(m.id, true);
+  const ids = (await recall(service, "令牌桶")).map((r) => r.id);
+  assert.equal(loads(), 2, "可见性变了要重建");
+  assert.ok(!ids.includes(m.id), "归档行不得留在索引里");
+});
+
+test("#617-O2: 触达与重嵌不改语料，不该让索引失效", async () => {
+  const { store, service, loads } = setup();
+  const m = store.save({ type: "project", title: "异步并发模式", content: "async runtime 选用 tokio", importance: 3 });
+  await recall(service, "rust 异步");
+  assert.equal(loads(), 1);
+
+  // 这两类写入刻意不 bump generation（store.js：reads must not mark the mirror
+  // dirty），也不改 title+content。缓存若跟着它们失效，就等于每轮注入都白重建
+  // ——正是 O2 要消除的成本。
+  store.touchLastAccess(m.id);
+  store.setEmbedding(m.id, [0.1, 0.2, 0.3]);
+  await recall(service, "rust 异步");
+  assert.equal(loads(), 1, "触达 / 重嵌不构成语料变化");
+});
+
+test("#617-O2: bm25SearchEnabled=false 时根本不取语料", async () => {
+  const { store, service, loads } = setup({ bm25SearchEnabled: false });
+  store.save({ type: "project", title: "异步并发模式", content: "async runtime 选用 tokio", importance: 3 });
+  const rows = await recall(service, "rust 异步");
+  assert.equal(loads(), 0, "开关关着连语料都不该取");
+  assert.ok(!rows.some((r) => r.source === "bm25"));
+});
+
+test("#617-O2: 空库不缓存空索引，后续写入仍可召回", async () => {
+  const { store, service, loads } = setup();
+  assert.deepEqual(await recall(service, "令牌桶"), [], "夹具前提：空库没有命中");
+
+  const m = store.save({ type: "project", title: "令牌桶限流", content: "ratelimit 令牌桶", importance: 3 });
+  const ids = (await recall(service, "令牌桶")).map((r) => r.id);
+  assert.ok(ids.includes(m.id), "空库那一轮不得留下「空索引」这种缓存");
+  assert.equal(loads(), 2);
+});
+
+test("#617-O2: 水位偶发读不到时，不拿旧索引冒充当前语料", async () => {
+  const { store, service, loads } = setup();
+  store.save({ type: "project", title: "夜间 ETL 脚本", content: "离线批处理", importance: 3 });
+  await recall(service, "离线批处理");
+  assert.equal(loads(), 1, "夹具前提：正常路径已经缓存了一份索引");
+
+  // 模拟 mirror_state 读不出来（库锁 / 迁移窗口）。此时无法证明缓存新鲜，必须
+  // 退回重建，而不是把 generation 当成 null 去和缓存里的旧值比对后命中。
+  const realGet = store.getMirrorState.bind(store);
+  store.getMirrorState = () => { throw new Error("mirror_state unavailable"); };
+  const fresh = store.save({ type: "project", title: "令牌桶限流", content: "ratelimit 令牌桶", importance: 3 });
+  const ids = (await recall(service, "令牌桶")).map((r) => r.id);
+  assert.ok(ids.includes(fresh.id), "水位不可读时仍要看到新写入的行");
+  assert.equal(loads(), 2);
+
+  // 恢复可读：期间确实写过，水位已经变了，于是再重建一次拿到的才是与水位对齐
+  // 的索引——宁可多建一次，也不能让缓存条目声称自己代表某个它没见过的版本。
+  store.getMirrorState = realGet;
+  await recall(service, "令牌桶");
+  assert.equal(loads(), 3, "水位已前进，恢复后须重建一次把条目对齐到新水位");
+
+  // 对齐之后再查就不该再重建了。
+  await recall(service, "离线批处理");
+  assert.equal(loads(), 3, "对齐后回到正常缓存路径");
+});
+
+test("#617-O2: 水位读不到但期间没有写入时，恢复后直接命中旧索引", async () => {
+  const { store, service, loads } = setup();
+  store.save({ type: "project", title: "夜间 ETL 脚本", content: "离线批处理", importance: 3 });
+  await recall(service, "离线批处理");
+  assert.equal(loads(), 1);
+
+  // 读失败但语料没变：此时丢掉缓存纯属白干。保留上一份「已知有效 generation」
+  // 的条目，恢复后水位一致就直接命中。
+  const realGet = store.getMirrorState.bind(store);
+  store.getMirrorState = () => { throw new Error("mirror_state unavailable"); };
+  await recall(service, "离线批处理");
+  assert.equal(loads(), 2, "读不到水位只能重建（无法证明缓存新鲜）");
+
+  store.getMirrorState = realGet;
+  await recall(service, "离线批处理");
+  assert.equal(loads(), 2, "期间无写入，恢复后不该再白建一次");
+});

package/test/capacity-warning.test.js

@@ -0,0 +1,262 @@
+// #275 容量预警（空间维度）回归：store 行数口径 → service 判级与 fail-safe →
+// /health 的 capacity 挂载。三层各锁各的：
+//   store.capacityStats —— 活跃/归档两格，setArchived 翻位即反映；
+//   getCapacityHealth —— 阈值判级（>= 过线、0 = 关）、:memory: 的缺席语义、
+//     fail-safe（读坏显式 unknown，绝不伪装 ok）；
+//   /health —— capacity 与 mirror 同面但互相独立，且不带库文件路径（半张对外面
+//     不泄露本机布局）。
+// 面板卡在 lib/client.js（无 src 对应物），由 test/client.test.js 的字符串锁兜住。
+// 阈值配置一律在 createService 时传入：createService 闭包持有 config，事后改
+// service.config 属性够不着判级读的那份（闭包变量），改了也是假绿。
+// 判级用例用裸 config（{}）：memoryQualityFilter 解析默认开，短正文样例会被静默
+// 归档，行数断言就全歪了——裸 config 下该闸不参与，这正是这些用例要的写入面。
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { EventEmitter } from "node:events";
+import { createStore } from "../src/store.js";
+import { createService } from "../src/service.js";
+import { createApi } from "../src/api.js";
+import { createSettings } from "../src/settings.js";
+import { Config, CAPACITY_WARNING_DEFAULTS } from "../src/config.js";
+
+function seed(service, n) {
+  const ids = [];
+  for (let i = 0; i < n; i++) {
+    const res = service.saveWithDedupe({
+      type: "project", title: `容量样例 ${i}`, content: `容量样例正文 ${i}，长度足够不被过滤`, importance: 3
+    });
+    ids.push(res.memory.id);
+  }
+  return ids;
+}
+
+// ── store.capacityStats ───────────────────────────────────────────────────────
+
+test("capacityStats 分活跃/归档两格；归档与还原都即时反映", () => {
+  const store = createStore(":memory:");
+  const service = createService({ store, mirror: null, config: {} });
+  try {
+    const ids = seed(service, 3);
+    assert.deepEqual(store.capacityStats(), { active: 3, archived: 0 });
+    store.setArchived(ids[0], true);
+    store.setArchived(ids[1], true);
+    assert.deepEqual(store.capacityStats(), { active: 1, archived: 2 });
+    // 还原（memory_archive 的出口）回到活跃：占比口径要跟着翻回去
+    store.setArchived(ids[0], false);
+    assert.deepEqual(store.capacityStats(), { active: 2, archived: 1 });
+  } finally {
+    store.close();
+  }
+});
+
+// ── schema：阈值默认值与边界 ──────────────────────────────────────────────────
+
+test("capacityWarning schema：默认值锚定议题形态区间，边界与部分覆盖语义钉死", () => {
+  const cfg = Config({});
+  assert.deepEqual(
+    { ...cfg.capacityWarning },
+    { dbSizeWarnMb: 256, archivedRowsWarn: 5000, archivedRatioWarn: 0.95 },
+    "解析默认必须与 CAPACITY_WARNING_DEFAULTS 一致（service 的裸 config 回退也用这份）"
+  );
+  assert.deepEqual(
+    { ...CAPACITY_WARNING_DEFAULTS },
+    { dbSizeWarnMb: 256, archivedRowsWarn: 5000, archivedRatioWarn: 0.95 }
+  );
+  // 部分覆盖时其余键落默认（.default({}) 的内层默认语义）
+  assert.equal(Config({ capacityWarning: { dbSizeWarnMb: 0 } }).capacityWarning.archivedRowsWarn, 5000);
+  // 越界一律拒（负数、浮点 MB、占比 > 1）
+  assert.throws(() => Config({ capacityWarning: { dbSizeWarnMb: -1 } }));
+  assert.throws(() => Config({ capacityWarning: { dbSizeWarnMb: 1.5 } }));
+  assert.throws(() => Config({ capacityWarning: { archivedRowsWarn: -3 } }));
+  assert.throws(() => Config({ capacityWarning: { archivedRatioWarn: 1.5 } }));
+});
+
+// ── service.getCapacityHealth ────────────────────────────────────────────────
+
+test("getCapacityHealth ok 态：检查齐全、占比归零；:memory: 库无体积检查（缺席≠伪装 ok）", () => {
+  const store = createStore(":memory:");
+  const service = createService({ store, mirror: null, config: {} });
+  try {
+    seed(service, 1);
+    const cap = service.getCapacityHealth();
+    assert.equal(cap.status, "ok");
+    // :memory: 没有磁盘足迹：file_bytes 为 null，db_size 检查**缺席**而不是 ok
+    assert.equal(cap.file_bytes, null);
+    assert.equal(cap.checks.some((c) => c.key === "db_size"), false);
+    // 行数与占比检查在（total > 0）
+    assert.equal(cap.active_rows, 1);
+    assert.equal(cap.archived_rows, 0);
+    assert.equal(cap.archived_ratio, 0);
+    const rows = cap.checks.find((c) => c.key === "archived_rows");
+    assert.equal(rows.status, "ok");
+    assert.equal(rows.value, 0);
+    assert.equal(rows.threshold, 5000, "裸 config（未解析）也要落到同一份默认阈值");
+    const ratio = cap.checks.find((c) => c.key === "archived_ratio");
+    assert.equal(ratio.status, "ok");
+    // freelist 是页数 × 页大小：:memory: 也有页，给数字而不是 null
+    assert.equal(typeof cap.freelist_bytes, "number");
+    assert.ok(cap.freelist_bytes >= 0);
+  } finally {
+    store.close();
+  }
+});
+
+test("getCapacityHealth 判级：archived_rows 到阈值 → warn，值/阈值能对上；低一格回 ok", () => {
+  const store = createStore(":memory:");
+  try {
+    const mk = (threshold) => createService({
+      store, mirror: null,
+      config: { capacityWarning: { dbSizeWarnMb: 0, archivedRowsWarn: threshold, archivedRatioWarn: 0 } }
+    });
+    const ids = seed(mk(2), 3);
+    store.setArchived(ids[0], true);
+    store.setArchived(ids[1], true);
+    // 0 = 显式关掉另外两条，只剩条数检查——每条检查能被独立开关是判级的前提
+    const cap = mk(2).getCapacityHealth();
+    assert.equal(cap.status, "warn");
+    assert.deepEqual(cap.checks.map((c) => c.key), ["archived_rows"]);
+    assert.equal(cap.checks[0].status, "warn");
+    assert.equal(cap.checks[0].value, 2);
+    assert.equal(cap.checks[0].threshold, 2);
+    // 阈值之下一格就回到 ok
+    assert.equal(mk(3).getCapacityHealth().status, "ok");
+  } finally {
+    store.close();
+  }
+});
+
+test("getCapacityHealth 判级：archived_ratio 过线 → warn，值按千分位取整", () => {
+  const store = createStore(":memory:");
+  const service = createService({
+    store, mirror: null,
+    config: { capacityWarning: { dbSizeWarnMb: 0, archivedRowsWarn: 0, archivedRatioWarn: 0.5 } }
+  });
+  try {
+    const ids = seed(service, 3);
+    store.setArchived(ids[0], true);
+    store.setArchived(ids[1], true);
+    const cap = service.getCapacityHealth();
+    assert.equal(cap.status, "warn");
+    const ratio = cap.checks.find((c) => c.key === "archived_ratio");
+    assert.equal(ratio.status, "warn");
+    assert.equal(ratio.value, 0.667, "2/3 取整到千分位");
+    assert.equal(ratio.threshold, 0.5);
+  } finally {
+    store.close();
+  }
+});
+
+test("getCapacityHealth 判级：db_size 过线 → warn（文件库，足迹含主文件 + WAL）", () => {
+  const dir = mkdtempSync(join(tmpdir(), "dsh-mneme-cap-"));
+  try {
+    const store = createStore(join(dir, "memory.db"));
+    const service = createService({
+      store, mirror: null,
+      config: { capacityWarning: { dbSizeWarnMb: 1, archivedRowsWarn: 0, archivedRatioWarn: 0 } }
+    });
+    // 单条 2MB 正文把库顶过 1 MB 预警线——dbSizeWarnMb 是整数 MB，比它小的只有关 (0)
+    service.saveWithDedupe({ type: "project", title: "大块头", content: "x".repeat(2 * 1048576), importance: 3 });
+    const cap = service.getCapacityHealth();
+    assert.equal(cap.status, "warn");
+    const size = cap.checks.find((c) => c.key === "db_size");
+    assert.equal(size.status, "warn");
+    assert.ok(size.value_bytes >= size.threshold_bytes, "file_bytes 含主文件 + WAL，必须真的过线");
+    assert.equal(size.threshold_bytes, 1048576);
+    store.close();
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test("getCapacityHealth fail-safe：store 关掉后不抛、显式 unknown、绝不伪装 ok", () => {
+  const store = createStore(":memory:");
+  const warns = [];
+  const service = createService({ store, mirror: null, config: {}, logger: { warn: (...a) => warns.push(a) } });
+  seed(service, 1);
+  store.close();
+  const cap = service.getCapacityHealth();
+  assert.equal(cap.status, "unknown");
+  assert.equal(cap.file_bytes, null);
+  assert.equal(cap.archived_rows, null);
+  assert.deepEqual(cap.checks, []);
+  assert.ok(warns.some((w) => String(w[0]).includes("getCapacityHealth failed")), "失败必须走 logger.warn");
+});
+
+// ── /health 挂载 ─────────────────────────────────────────────────────────────
+
+class FakeRes extends EventEmitter {
+  constructor() { super(); this.statusCode = 200; this.body = ""; }
+  writeHead(code, headers) { this.statusCode = code; this.headers = headers; return this; }
+  end(text) { this.body = text ?? ""; this.emit("end"); return this; }
+}
+
+function makeHealthRoute(store, service) {
+  const settings = createSettings(store.db);
+  const commands = {
+    add: (def) => settings.addCommand(def),
+    remove: (id) => settings.removeCommand(id),
+    list: () => settings.listCommands()
+  };
+  const routes = [];
+  const ctx = { webServer: { register(route) { routes.push(route); return () => {}; } } };
+  createApi(ctx, service, settings, commands, undefined, undefined, "");
+  const route = routes.find((r) => r.path === "/api/dsh-mneme/health");
+  assert.ok(route, "必须注册 /health 路由");
+  return route;
+}
+
+test("/health 挂 capacity：与 mirror 同面、数字可读、不带库文件路径", async () => {
+  const dir = mkdtempSync(join(tmpdir(), "dsh-mneme-cap-"));
+  try {
+    const store = createStore(join(dir, "memory.db"));
+    const service = createService({ store, mirror: null, config: {} });
+    seed(service, 1);
+    const route = makeHealthRoute(store, service);
+    const res = new FakeRes();
+    await route.handler({ url: "/api/dsh-mneme/health", method: "GET", headers: {} }, res);
+    assert.equal(res.statusCode, 200);
+    const body = JSON.parse(res.body);
+    assert.ok(body.mirror, "镜像块不动");
+    assert.ok(body.capacity, "capacity 块必须在");
+    assert.equal(body.capacity.status, "ok");
+    assert.equal(typeof body.capacity.file_bytes, "number", "文件库要给磁盘足迹");
+    assert.equal(body.capacity.active_rows, 1);
+    assert.ok(Array.isArray(body.capacity.checks) && body.capacity.checks.length === 3, "文件库三道检查齐全");
+    // /health 是半张对外面：数字可以给，本机文件系统布局不能给
+    assert.ok(!res.body.includes(dir), "响应不得携带临时目录路径");
+    assert.ok(!res.body.includes("memory.db"), "响应不得携带库文件名");
+    // 库关掉后两个半张各自 fail-closed：mirror 既有语义不动，capacity 显式 unknown
+    store.close();
+    const resBroken = new FakeRes();
+    await route.handler({ url: "/api/dsh-mneme/health", method: "GET", headers: {} }, resBroken);
+    const broken = JSON.parse(resBroken.body);
+    assert.equal(broken.mirror.status, "unknown");
+    assert.equal(broken.capacity.status, "unknown");
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+test("/health：没接线 getCapacityHealth 的 service → capacity 显式 null，不编造形状", async () => {
+  const store = createStore(":memory:");
+  const service = createService({ store, mirror: null, config: {} });
+  try {
+    seed(service, 1);
+    const route = makeHealthRoute(store, service);
+    const res = new FakeRes();
+    await route.handler({ url: "/api/dsh-mneme/health", method: "GET", headers: {} }, res);
+    assert.equal(JSON.parse(res.body).capacity.status, "ok");
+    // 老 mock / 旧版本 service 没有这个方法：handler 不得炸，capacity 落 null
+    // （面板渲染成「未知」，不冒充正常）
+    delete service.getCapacityHealth;
+    const resOld = new FakeRes();
+    await route.handler({ url: "/api/dsh-mneme/health", method: "GET", headers: {} }, resOld);
+    assert.equal(resOld.statusCode, 200);
+    assert.equal(JSON.parse(resOld.body).capacity, null);
+  } finally {
+    store.close();
+  }
+});