@modusensus/dsh-mneme 0.4.6 → 0.5.0

package/test/graph-api.test.js

@@ -0,0 +1,175 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { EventEmitter } from "node:events";
+import { createStore } from "../src/store.js";
+import { createService } from "../src/service.js";
+import { createApi } from "../src/api.js";
+import { createSettings } from "../src/settings.js";
+
+// Ego-graph read API (graph panel P1). The routes under test:
+//   GET /api/dsh-mneme/semantic/graph/ego?entity=&depth=&limit=
+//   GET /api/dsh-mneme/semantic/graph/entity-attrs?entity=
+// Both are read-only: they must stay reachable without an apiToken, exactly
+// like list/search/semantic.
+
+class FakeRes extends EventEmitter {
+  constructor() { super(); this.statusCode = 200; this.body = ""; }
+  writeHead(code, headers) { this.statusCode = code; this.headers = headers; return this; }
+  end(text) { this.body = text ?? ""; this.emit("end"); return this; }
+}
+
+function req(path) {
+  const r = new EventEmitter();
+  r.url = path;
+  r.method = "GET";
+  r.headers = {};
+  return r;
+}
+
+function setup(apiToken = "") {
+  const store = createStore(":memory:");
+  const service = createService({ store, mirror: null, config: {} });
+  const settings = createSettings(store.db);
+  const commands = { add: () => {}, remove: () => {}, list: () => [] };
+  const routes = [];
+  const ctx = { webServer: { register(route) { routes.push(route); return () => {}; } } };
+  createApi(ctx, service, settings, commands, null, undefined, apiToken);
+  return { store, service, routes };
+}
+
+function graphRoute(routes, path) {
+  return routes.find((r) => r.path === path);
+}
+
+function buildChain(service, names, linkType = "related_to") {
+  // names → entities; each consecutive pair gets one relation. Returns a
+  // name→entity map so tests can assert on ids.
+  const map = new Map();
+  for (const n of names) map.set(n, service.createEntity({ name: n, type: "concept" }));
+  for (let i = 0; i < names.length - 1; i++) {
+    service.saveRelation({
+      from_entity: map.get(names[i]).id,
+      to_entity: map.get(names[i + 1]).id,
+      relation_type: linkType
+    });
+  }
+  return map;
+}
+
+test("ego route is registered and open without apiToken", async () => {
+  const { routes } = setup("secret-token");
+  const ego = graphRoute(routes, "/api/dsh-mneme/semantic/graph/ego");
+  const attrs = graphRoute(routes, "/api/dsh-mneme/semantic/graph/entity-attrs");
+  assert.ok(ego, "ego route must be registered");
+  assert.ok(attrs, "entity-attrs route must be registered");
+  // read-only: no 401 even with a token configured
+  const res = new FakeRes();
+  await ego.handler(req("/api/dsh-mneme/semantic/graph/ego"), res);
+  assert.equal(res.statusCode, 400, "missing param is a 400, not a 401");
+});
+
+test("ego depth 1 returns root + direct neighbors with distances", async () => {
+  const { routes, service } = setup();
+  const map = buildChain(service, ["A", "B", "C"]);
+  const route = graphRoute(routes, "/api/dsh-mneme/semantic/graph/ego");
+  const res = new FakeRes();
+  await route.handler(req("/api/dsh-mneme/semantic/graph/ego?entity=A&depth=1"), res);
+  assert.equal(res.statusCode, 200);
+  const data = JSON.parse(res.body);
+  assert.equal(data.root.name, "A");
+  assert.equal(data.nodes.length, 2, "root + B only (C is 2 hops)");
+  assert.equal(data.edges.length, 1);
+  const b = data.nodes.find((n) => n.name === "B");
+  assert.equal(b.distance, 1);
+  assert.equal(data.nodes.find((n) => n.name === "A").distance, 0);
+  // edge shape: ids + relation_type, memory_id nullable
+  const edge = data.edges[0];
+  assert.ok(edge.id && edge.from && edge.to);
+  assert.equal(edge.relation_type, "related_to");
+  assert.equal(edge.memory_id, null);
+});
+
+test("ego depth 2 includes 2-hop nodes, not 3-hop", async () => {
+  const { routes, service } = setup();
+  buildChain(service, ["A", "B", "C", "D"]);
+  const route = graphRoute(routes, "/api/dsh-mneme/semantic/graph/ego");
+  const res = new FakeRes();
+  await route.handler(req("/api/dsh-mneme/semantic/graph/ego?entity=A&depth=2"), res);
+  const data = JSON.parse(res.body);
+  const names = data.nodes.map((n) => n.name).sort();
+  assert.deepEqual(names, ["A", "B", "C"], "D is 3 hops away and must be excluded");
+  assert.equal(data.nodes.find((n) => n.name === "C").distance, 2);
+  assert.equal(data.edges.length, 2);
+});
+
+test("ego respects limit and keeps only in-graph edges", async () => {
+  const { routes, service } = setup();
+  const map = buildChain(service, ["A", "B", "C"]);
+  const route = graphRoute(routes, "/api/dsh-mneme/semantic/graph/ego");
+  const res = new FakeRes();
+  await route.handler(req("/api/dsh-mneme/semantic/graph/ego?entity=A&depth=2&limit=2"), res);
+  const data = JSON.parse(res.body);
+  assert.equal(data.nodes.length, 2, "limit caps node count");
+  assert.equal(data.edges.length, 1, "edges to cut nodes are dropped");
+});
+
+test("ego reports 404 for unknown entity and 400 without entity", async () => {
+  const { routes } = setup();
+  const route = graphRoute(routes, "/api/dsh-mneme/semantic/graph/ego");
+  let res = new FakeRes();
+  await route.handler(req("/api/dsh-mneme/semantic/graph/ego?entity=ghost"), res);
+  assert.equal(res.statusCode, 404);
+  assert.equal(JSON.parse(res.body).error, "entity-not-found");
+  res = new FakeRes();
+  await route.handler(req("/api/dsh-mneme/semantic/graph/ego"), res);
+  assert.equal(res.statusCode, 400);
+  assert.equal(JSON.parse(res.body).error, "missing-entity");
+});
+
+test("entity-attrs returns current valid attrs only", async () => {
+  const { routes, service } = setup();
+  const mem = service.saveWithDedupe({ type: "project", title: "t", content: "c" });
+  const entity = service.createEntity({ name: "X", type: "technology" });
+  service.saveAttr({ entity_id: entity.id, attr_key: "version", attr_value: "1", memory_id: mem.id, confidence: 0.9 });
+  // a second save of the same key invalidates the first: only one row stays valid
+  service.saveAttr({ entity_id: entity.id, attr_key: "version", attr_value: "2", memory_id: mem.id, confidence: 0.8 });
+  const route = graphRoute(routes, "/api/dsh-mneme/semantic/graph/entity-attrs");
+  const res = new FakeRes();
+  await route.handler(req("/api/dsh-mneme/semantic/graph/entity-attrs?entity=X"), res);
+  assert.equal(res.statusCode, 200);
+  const data = JSON.parse(res.body);
+  assert.equal(data.entity.name, "X");
+  assert.equal(data.attrs.length, 1);
+  assert.equal(data.attrs[0].value, "2", "latest valid attr wins");
+  assert.equal(data.attrs[0].confidence, 0.8);
+  // unknown / missing entity
+  let r2 = new FakeRes();
+  await route.handler(req("/api/dsh-mneme/semantic/graph/entity-attrs?entity=nope"), r2);
+  assert.equal(r2.statusCode, 404);
+  r2 = new FakeRes();
+  await route.handler(req("/api/dsh-mneme/semantic/graph/entity-attrs"), r2);
+  assert.equal(r2.statusCode, 400);
+});
+
+test("ego 2-hop over 100+ nodes stays under 50ms (spec §7)", async () => {
+  const { routes, service } = setup();
+  // star-of-stars: hub → 20 spokes, each spoke → 5 leaves = 121 nodes, 120 edges
+  const hub = service.createEntity({ name: "hub", type: "project" });
+  for (let i = 0; i < 20; i++) {
+    const spoke = service.createEntity({ name: `spoke-${i}`, type: "technology" });
+    service.saveRelation({ from_entity: hub.id, to_entity: spoke.id, relation_type: "uses" });
+    for (let j = 0; j < 5; j++) {
+      const leaf = service.createEntity({ name: `leaf-${i}-${j}`, type: "concept" });
+      service.saveRelation({ from_entity: spoke.id, to_entity: leaf.id, relation_type: "related_to" });
+    }
+  }
+  const route = graphRoute(routes, "/api/dsh-mneme/semantic/graph/ego");
+  const res = new FakeRes();
+  const start = process.hrtime.bigint();
+  await route.handler(req("/api/dsh-mneme/semantic/graph/ego?entity=hub&depth=2&limit=100"), res);
+  const ms = Number(process.hrtime.bigint() - start) / 1e6;
+  const data = JSON.parse(res.body);
+  assert.equal(data.nodes.length, 100, "limit caps the traversal");
+  assert.ok(data.edges.length > 0);
+  assert.ok(ms < 50, `2-hop query took ${ms.toFixed(1)}ms, spec budget is 50ms`);
+});

package/test/hot-memory.test.js

@@ -0,0 +1,145 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { createHotMemory, estimateTokens } from "../src/hot-memory.js";
+import { createStore } from "../src/store.js";
+import { createService } from "../src/service.js";
+import { createVectorIndex } from "../src/vector-index.js";
+
+// --- hot memory buffer ---
+
+test("hot memory keeps the latest rounds within maxRounds", () => {
+  const hot = createHotMemory({ maxRounds: 2, maxTokens: 10000 });
+  hot.add({ query: "第一轮", response: "答一" });
+  hot.add({ query: "第二轮", response: "答二" });
+  hot.add({ query: "第三轮", response: "答三" });
+  assert.equal(hot.rounds().length, 2);
+  assert.ok(hot.getContext().includes("第三轮"));
+  assert.ok(!hot.getContext().includes("第一轮"));
+});
+
+test("hot memory enforces the token budget", () => {
+  const hot = createHotMemory({ maxRounds: 10, maxTokens: 30 });
+  hot.add({ query: "很长的第一轮问题".repeat(10), response: "很长的回答".repeat(10) });
+  hot.add({ query: "第二轮", response: "答二" });
+  // The first round alone blows the budget; the newest round survives and
+  // the buffer never empties completely.
+  const rounds = hot.rounds();
+  assert.ok(rounds.length >= 1);
+  assert.equal(rounds[rounds.length - 1].query, "第二轮");
+});
+
+test("hot memory getContext uses the Q/A round format", () => {
+  const hot = createHotMemory({ maxRounds: 5, maxTokens: 10000 });
+  hot.add({ query: "Q1", response: "A1" });
+  hot.add({ query: "Q2", response: "A2" });
+  assert.equal(hot.getContext(), "Q: Q1\nA: A1\n\nQ: Q2\nA: A2");
+});
+
+test("hot memory ignores empty rounds and clears", () => {
+  const hot = createHotMemory({ maxRounds: 5, maxTokens: 10000 });
+  hot.add({ query: "", response: "x" });
+  hot.add(null);
+  assert.equal(hot.rounds().length, 0);
+  hot.add({ query: "q" });
+  hot.clear();
+  assert.equal(hot.getContext(), "");
+});
+
+test("estimateTokens counts CJK heavier than ASCII", () => {
+  assert.ok(estimateTokens("中文内容") > estimateTokens("abcd"));
+});
+
+// --- service-level: BM25 fusion + semantic dedup + selective injection ---
+
+function toyVec(text) {
+  const v = new Array(64).fill(0);
+  for (const t of String(text).toLowerCase().split(/[^\p{L}\p{N}]+/u).filter(Boolean)) {
+    let h = 0;
+    for (const ch of t) h = (h * 31 + ch.codePointAt(0)) >>> 0;
+    v[h % 64] += 1;
+  }
+  const norm = Math.sqrt(v.reduce((s, x) => s + x * x, 0)) || 1;
+  return v.map((x) => x / norm);
+}
+
+function setup(overrides = {}) {
+  const store = createStore(":memory:");
+  const config = {
+    entitySearchEnabled: false,
+    bm25SearchEnabled: true,
+    adaptiveThresholdEnabled: false,
+    searchSemanticDedup: true,
+    selectiveInjectEnabled: true,
+    ...overrides
+  };
+  const service = createService({ store, mirror: null, config, logger: null });
+  service.setVectorIndex(createVectorIndex({ store, logger: null }));
+  service.setEmbedder({ embedSingle: async (t) => toyVec(t) });
+  return { store, service };
+}
+
+test("searchMemories fuses BM25: scattered-term queries recall both rows", async () => {
+  const { store, service } = setup();
+  const a = store.save({ type: "project", title: "异步并发模式", content: "async runtime 选用 tokio", importance: 3 });
+  const b = store.save({ type: "decision", title: "语言迁移", content: "编译模块迁移到 Rust", importance: 3 });
+  store.save({ type: "project", title: "无关", content: "夜间 ETL 脚本", importance: 3 });
+
+  // "rust 异步" is not a substring of either row — LIKE misses both; BM25
+  // must surface both rows in the merged result.
+  const results = await service.searchMemories("rust 异步", { mode: "auto", topK: 5 });
+  const ids = results.map((r) => r.id);
+  assert.ok(ids.includes(a.id), "async row must be recalled via BM25");
+  assert.ok(ids.includes(b.id), "rust row must be recalled via BM25");
+  assert.equal(results.find((r) => r.id === a.id)?.source, "bm25");
+});
+
+test("bm25SearchEnabled=false restores the two-path behavior", async () => {
+  const { store, service } = setup({ bm25SearchEnabled: false });
+  const a = store.save({ type: "project", title: "异步并发模式", content: "async runtime 选用 tokio", importance: 3 });
+  const results = await service.searchMemories("rust 异步", { mode: "auto", topK: 5 });
+  assert.ok(!results.some((r) => r.id === a.id), "no BM25 → scattered-term miss is back");
+});
+
+test("search-time semantic dedup drops near-identical embeddings", async () => {
+  const { store, service } = setup();
+  const a = store.save({ type: "project", title: "偏好 A", content: "用户喜欢深色主题编辑器", importance: 3 });
+  const b = store.save({ type: "project", title: "偏好 A 备份", content: "用户喜欢深色主题编辑器（备份）", importance: 3 });
+  store.setEmbedding(a.id, toyVec("用户喜欢深色主题编辑器"));
+  store.setEmbedding(b.id, toyVec("用户喜欢深色主题编辑器"));
+
+  // auto (not keyword): keyword mode is the documented text-only path and is
+  // exempt from dedup by design; auto exercises the dedup the way production
+  // searches run.
+  const results = await service.searchMemories("深色主题", { mode: "auto", topK: 5 });
+  const ids = results.map((r) => r.id);
+  assert.ok(ids.includes(a.id) !== ids.includes(b.id), "one of the near-duplicate pair is dropped");
+});
+
+test("searchSemanticDedup=false keeps duplicate embeddings", async () => {
+  const { store, service } = setup({ searchSemanticDedup: false });
+  const a = store.save({ type: "project", title: "偏好 A", content: "用户喜欢深色主题编辑器", importance: 3 });
+  const b = store.save({ type: "project", title: "偏好 A 备份", content: "用户喜欢深色主题编辑器（备份）", importance: 3 });
+  store.setEmbedding(a.id, toyVec("用户喜欢深色主题编辑器"));
+  store.setEmbedding(b.id, toyVec("用户喜欢深色主题编辑器"));
+  const results = await service.searchMemories("深色主题", { mode: "keyword", topK: 5 });
+  assert.equal(results.length, 2);
+});
+
+test("selective injection re-orders candidates by query similarity", () => {
+  const { store, service } = setup();
+  const thesis = store.save({ type: "project", title: "湿地论文", content: "毕业论文研究城市湿地公园", importance: 5 });
+  const plugin = store.save({ type: "project", title: "插件项目", content: "dsh-mneme 记忆插件开发", importance: 5 });
+  store.setEmbedding(thesis.id, toyVec("毕业论文研究城市湿地公园"));
+  store.setEmbedding(plugin.id, toyVec("dsh-mneme 记忆插件开发"));
+
+  // Rule-based order would put both at equal importance; the query vector is
+  // about the thesis, so topic ranking must put the thesis memory first.
+  const picked = service.injectCandidates({
+    query: "论文写作",
+    queryVector: toyVec("毕业论文研究城市湿地公园"),
+    maxItems: 2,
+    threshold: 3
+  });
+  assert.equal(picked[0].id, thesis.id);
+  assert.ok(picked.some((m) => m.id === plugin.id));
+});

package/test/reasoning-effort.test.js

@@ -25,7 +25,7 @@ const embedder = {
 // ---------------------------------------------------------------- config schema
 
 test("issue#9: dreamMaxTokens accepts the widened 131072 cap and defaults to 4096", () => {
-  assert.equal(Config({}).dreamMaxTokens, 4096, "default unchanged");
+  assert.equal(Config({}).dreamMaxTokens, 8192, "default unchanged");
   assert.equal(Config({ dreamMaxTokens: 131072 }).dreamMaxTokens, 131072, "new upper bound accepted");
   assert.equal(Config({ dreamMaxTokens: 65536 }).dreamMaxTokens, 65536, "intermediate value accepted");
 });

package/test/recall-layer.test.js

@@ -17,9 +17,9 @@ const embedder = {
 
 // Real store + service wired with the mock embedder and a vector index over the
 // same store (the service's vector path prefers vectorIndex when set).
-function setup() {
+function setup(config = {}) {
   const store = createStore(":memory:");
-  const service = createService({ store, mirror: null, config: {} });
+  const service = createService({ store, mirror: null, config });
   const vectorIndex = createVectorIndex({ store });
   service.setEmbedder(embedder);
   service.setVectorIndex(vectorIndex);

package/test/search-fusion.test.js

@@ -0,0 +1,90 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { tokenize, createBM25Index } from "../src/search/bm25.js";
+import { adaptiveThreshold } from "../src/search/adaptive.js";
+
+// --- tokenizer ---
+
+test("tokenize keeps ASCII identifiers whole", () => {
+  assert.deepEqual(
+    tokenize("ZFS-4421 dsh_mneme v2"),
+    ["zfs", "4421", "dsh_mneme", "v2"]
+  );
+});
+
+test("tokenize splits CJK runs into bigrams", () => {
+  assert.deepEqual(tokenize("异步编程"), ["异步", "步编", "编程"]);
+  assert.deepEqual(tokenize("图"), ["图"]);
+});
+
+// --- BM25 index ---
+
+const DOCS = [
+  { id: "a", title: "异步并发模式", content: "async runtime 选用 tokio，任务 spawn 管理" },
+  { id: "b", title: "语言迁移", content: "编译模块从 Go 迁移到 Rust，内存安全" },
+  { id: "c", title: "无关条目", content: "夜间 ETL 用 Python 编写" }
+];
+
+test("BM25 recalls rows whose query terms are scattered", () => {
+  // The LIKE path cannot match "rust 异步" as a substring of either doc;
+  // BM25 must surface both term-bearing rows above the unrelated one.
+  const idx = createBM25Index(DOCS);
+  const hits = idx.search("rust 异步", { limit: 3 });
+  const ids = hits.map((h) => h.id);
+  assert.ok(ids.includes("a"), "the async doc must be recalled");
+  assert.ok(ids.includes("b"), "the rust doc must be recalled");
+  assert.ok(!ids.includes("c"), "the unrelated doc must not lead");
+  assert.ok(ids.indexOf("c") === -1 || hits.find((h) => h.id === "c").score < hits[0].score);
+});
+
+test("BM25 search scores are normalized to [0,1] with the max on top", () => {
+  const idx = createBM25Index(DOCS);
+  const hits = idx.search("tokio spawn", { limit: 3 });
+  assert.ok(hits.length >= 1);
+  assert.equal(hits[0].id, "a");
+  for (const h of hits) {
+    assert.ok(h.score >= 0 && h.score <= 1, `score ${h.score} out of [0,1]`);
+  }
+  assert.equal(hits[0].score, 1);
+});
+
+test("BM25 score(query, doc) matches search on single-doc corpora", () => {
+  const idx = createBM25Index(DOCS);
+  const doc = DOCS[0];
+  assert.ok(idx.score("tokio", doc) > 0);
+  assert.equal(idx.score("完全不相关词汇xyzzy", doc), 0);
+});
+
+test("BM25 drops untouched rows entirely", () => {
+  const idx = createBM25Index(DOCS);
+  const hits = idx.search("ETL Python", { limit: 3 });
+  assert.deepEqual(hits.map((h) => h.id), ["c"]);
+});
+
+// --- adaptive threshold ---
+
+test("adaptive threshold: entity/attr prefixes loosen to 0.5", () => {
+  assert.equal(adaptiveThreshold("entity:某个实体"), 0.5);
+  assert.equal(adaptiveThreshold("attr:lang=Rust"), 0.5);
+});
+
+test("adaptive threshold: short queries tighten, long queries loosen", () => {
+  assert.equal(adaptiveThreshold("go"), 0.7);
+  const long = "这是一个非常长的查询".repeat(8);
+  assert.equal(adaptiveThreshold(long), 0.6);
+});
+
+test("adaptive threshold: decisive head gap loosens to 0.5", () => {
+  const candidates = [
+    { _score: 0.9 }, { _score: 0.55 }, { _score: 0.54 }, { _score: 0.53 }, { _score: 0.52 }
+  ];
+  assert.equal(adaptiveThreshold("普通长度的查询词组", candidates), 0.5);
+});
+
+test("adaptive threshold: flat distribution keeps the 0.65 default", () => {
+  const candidates = [
+    { _score: 0.7 }, { _score: 0.68 }, { _score: 0.66 }, { _score: 0.64 }, { _score: 0.62 }
+  ];
+  assert.equal(adaptiveThreshold("普通长度的查询词组", candidates), 0.65);
+  assert.equal(adaptiveThreshold("普通长度的查询词组"), 0.65);
+});

package/test/service-search.test.js

@@ -25,9 +25,13 @@ const reranker = {
 
 // Real store + service, wired with the mock embedder and a vector index over
 // the same store (the service's vector path prefers vectorIndex when set).
-function setup({ withEmbedder = true } = {}) {
+// The v0.5.0 recall fusion extras (BM25 third path, search-time semantic
+// dedup) are disabled per-test when the assertion targets the legacy blend
+// mechanics — the toy embedder pins every vector hit to the same [1,0,0],
+// which semantic dedup would legitimately collapse.
+function setup({ withEmbedder = true, config = {} } = {}) {
   const store = createStore(":memory:");
-  const service = createService({ store, mirror: null, config: {} });
+  const service = createService({ store, mirror: null, config });
   const vectorIndex = createVectorIndex({ store });
   if (withEmbedder) {
     service.setEmbedder(embedder);