@modusensus/dsh-mneme 0.8.5 → 0.8.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.js CHANGED
@@ -43,7 +43,15 @@ export { Config };
43
43
  // unit-testable; the extractor only ever sees a callLLM(messages, options)
44
44
  // => Promise<string>. The route always carries a real provider/model (dsh-llm
45
45
  // GenerateOptions requires both) — never a bare stream.
46
- export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
46
+ //
47
+ // Issue #250: it also accounts for itself. Entity extraction is the background
48
+ // LLM path that runs on every memory write, yet it never reached
49
+ // llm_audit_logs — the adapter only ever held an llm handle, so it had no way
50
+ // to call service.saveLlmAudit. It now takes service (+ config for the shared
51
+ // llmAudit.enabled gate) and writes one row per stream attempt, matching the
52
+ // contract of runAuditedLlm in dream.js: a rejected effort attempt records its
53
+ // own error row and the retry records its own success row.
54
+ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger, service, config }) {
47
55
  return async function streamEntityText(messages, options = {}) {
48
56
  let route = {};
49
57
  if (options.provider) route.provider = options.provider;
@@ -58,8 +66,38 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
58
66
  } catch { /* fall through to whatever route we already have */ }
59
67
  }
60
68
  const effort = options.reasoningEffort;
69
+ // 没有解析出 provider/model 就没有可归属的模型——与 dream 的 resolveRoute
70
+ // 无路由早退同口径,那种情况不写审计行。
71
+ const modelId = route.provider && route.model ? `${route.provider}:${route.model}` : "";
61
72
  const tryStream = (withEffort) => {
62
73
  let text = "";
74
+ let inputTokens = 0;
75
+ let outputTokens = 0;
76
+ const startedAt = Date.now();
77
+ const timestamp = new Date(startedAt).toISOString();
78
+ // 记账是 best-effort:写审计行失败只 warn,绝不反噬抽取本身(CONTRIBUTING
79
+ // 的 fail-safe 硬约定)。
80
+ const writeAudit = (status, errorMessage) => {
81
+ if (config?.llmAudit?.enabled === false || !modelId || typeof service?.saveLlmAudit !== "function") return;
82
+ try {
83
+ service.saveLlmAudit({
84
+ timestamp,
85
+ trigger_source: "entityExtract",
86
+ operation_type: "entity_extract",
87
+ model_id: modelId,
88
+ input_tokens: inputTokens,
89
+ output_tokens: outputTokens,
90
+ total_tokens: inputTokens + outputTokens,
91
+ cost_usd: 0,
92
+ duration_ms: Date.now() - startedAt,
93
+ status,
94
+ error_message: errorMessage,
95
+ related_memory_ids: []
96
+ });
97
+ } catch (auditError) {
98
+ logger?.warn?.(`dsh-mneme: entity extraction llm audit write failed: ${String(auditError)}`);
99
+ }
100
+ };
63
101
  return (async () => {
64
102
  for await (const chunk of llm.stream({
65
103
  ...route,
@@ -68,11 +106,26 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
68
106
  messages
69
107
  })) {
70
108
  if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
71
- if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) return undefined;
109
+ // DSH 的 StreamChunk 契约把用量嵌在 chunk.usage(TokenUsage);兼容直接
110
+ // 平铺在 chunk 上的旧形态。归一放在这里,下面只需认平铺形状——与
111
+ // dream.js 的 streamText 同一处理。
112
+ if (chunk.type === "usage") {
113
+ const u = chunk.usage ?? chunk;
114
+ const i = u.input_tokens ?? u.inputTokens ?? u.prompt_tokens ?? u.promptTokens;
115
+ const o = u.output_tokens ?? u.outputTokens ?? u.completion_tokens ?? u.completionTokens;
116
+ if (Number.isFinite(i)) inputTokens = i;
117
+ if (Number.isFinite(o)) outputTokens = o;
118
+ }
119
+ if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
120
+ writeAudit("error", "llm stream aborted or errored");
121
+ return undefined;
122
+ }
72
123
  }
124
+ writeAudit("success", null);
73
125
  return text;
74
126
  })().catch((err) => {
75
127
  logger?.warn?.(`[dsh-mneme] entity extraction llm stream failed: ${String(err)}`);
128
+ writeAudit("error", String(err?.message ?? err));
76
129
  return undefined;
77
130
  });
78
131
  };
@@ -273,6 +326,9 @@ export const apply = (ctx, config) => {
273
326
  dimension: cfg.localEmbedDimension,
274
327
  device: cfg.localEmbedDevice,
275
328
  batchSize: cfg.localEmbedBatchSize,
329
+ // 池化方式必须与模型的训练口径一致(BGE 系 = CLS)。它既进 embed() 的调用,
330
+ // 也进 modelHash —— 池化改了就是换向量空间,既有索引会被判失配并重建。
331
+ pooling: cfg.localEmbedPooling,
276
332
  cacheDir: cfg.embedModelCacheDir,
277
333
  runtimeDir: cfg.runtimeDir,
278
334
  // #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
@@ -403,6 +459,7 @@ export const apply = (ctx, config) => {
403
459
  minIntervalMs: (cfg.dreamMinIntervalMinutes ?? 0) * 60000,
404
460
  logger: ctx.logger,
405
461
  semantic: { embedder, vectorIndex },
462
+ lastRunAtSeed: store.lastDreamRunAt("auto"),
406
463
  onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
407
464
  });
408
465
  service.setDreamHook(() => dream.maybeSchedule(service));
@@ -419,6 +476,7 @@ export const apply = (ctx, config) => {
419
476
  service,
420
477
  config: cfg,
421
478
  logger: ctx.logger,
479
+ lastRunAtSeed: store.lastDreamRunAt("sleep"),
422
480
  onRun: (signal) => (sleep ? runSleep(ctx, service, cfg, ctx.logger, { embedder, vectorIndex }, signal) : Promise.resolve({ ok: true, skipped: true }))
423
481
  });
424
482
  service.setSleepHook(() => sleep.noteWrite());
@@ -442,7 +500,11 @@ export const apply = (ctx, config) => {
442
500
  const streamEntityText = createEntityStreamAdapter({
443
501
  llm: ctx.llm,
444
502
  agentDefaultModel: ctx.agentDefaultModel,
445
- logger: ctx.logger
503
+ logger: ctx.logger,
504
+ // Issue #250: the adapter needs service to write its llm_audit_logs row
505
+ // (it never had it) and config for the shared llmAudit.enabled gate.
506
+ service,
507
+ config: cfg
446
508
  });
447
509
  service.setEntityExtractor((memory) =>
448
510
  extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
@@ -21,6 +21,32 @@ function modelHash(model) {
21
21
  return `${model}#${hashString(model)}`;
22
22
  }
23
23
 
24
+ /**
25
+ * Model families whose embedding is the CLS token rather than a mean over tokens.
26
+ *
27
+ * BGE (BAAI/bge-*) is trained that way: the model's own `1_Pooling/config.json`
28
+ * declares `pooling_mode_cls_token: true` / `pooling_mode_mean_tokens: false`, and
29
+ * the official README says "select the last hidden state of the first token" plus
30
+ * L2 normalization. transformers.js's `feature-extraction` defaults to `mean`, so
31
+ * feeding it a BGE model silently produces slightly-off vectors — no error, just a
32
+ * worse ranking. This table is the extension point: a family goes in only once its
33
+ * pooling is confirmed against the model's own `1_Pooling` (or equivalent) config.
34
+ */
35
+ const CLS_POOLED_MODEL_PATTERNS = [/bge/i];
36
+
37
+ /**
38
+ * Resolve the pooling to embed with.
39
+ *
40
+ * `auto` (default) picks by model family — BGE → `cls`, everything else → `mean`,
41
+ * so models that were never affected keep their exact previous behavior. An
42
+ * explicit `cls` / `mean` wins over the family table.
43
+ */
44
+ export function resolvePooling(pooling, model) {
45
+ const mode = String(pooling ?? "auto").trim().toLowerCase();
46
+ if (mode === "cls" || mode === "mean") return mode;
47
+ return CLS_POOLED_MODEL_PATTERNS.some((re) => re.test(String(model ?? ""))) ? "cls" : "mean";
48
+ }
49
+
24
50
  /**
25
51
  * Lazy default loader: dynamic import keeps module load cheap.
26
52
  *
@@ -71,8 +97,9 @@ function tensorToRows(tensor) {
71
97
 
72
98
  /**
73
99
  * ONNX text embedder backed by transformers.js (onnxruntime-node underneath).
74
- * Runs fully offline with mean pooling + L2 normalization for BERT-style
75
- * models like bge-small-zh. `engineFactory` is injectable for tests.
100
+ * Runs fully offline; pooling is resolved per model family (BGE → CLS, else mean —
101
+ * see resolvePooling) and vectors are L2-normalized. `engineFactory` is injectable
102
+ * for tests.
76
103
  */
77
104
  export class LocalEmbedder {
78
105
  constructor(opts = {}) {
@@ -87,6 +114,10 @@ export class LocalEmbedder {
87
114
  // (含测试)保持裸 fetch 的现状,配置默认值由 index.js 传进来。
88
115
  this.resilientModelDownload = opts.resilientModelDownload === true;
89
116
  this.useDtype = opts.useDtype || "q8";
117
+ // 池化:配置值(auto/cls/mean)→ 生效值(cls/mean)。解析结果既进 embed(),
118
+ // 也进 modelHash —— 换池化就是换向量空间,旧索引必须被判失配后重建。
119
+ this.poolingMode = String(opts.pooling ?? "auto").trim().toLowerCase();
120
+ this.pooling = resolvePooling(this.poolingMode, this.model);
90
121
  // 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
91
122
  this.remoteHost = String(opts.remoteHost ?? "").trim();
92
123
  this.logger = opts.logger ?? null;
@@ -114,19 +145,19 @@ export class LocalEmbedder {
114
145
  this.extractor = await this.engineFactory("feature-extraction", this.model, options);
115
146
  this.ready = true; // service reads this to flush queued re-embeds
116
147
  this.logger?.info?.(
117
- `[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device})`
148
+ `[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device}, pooling=${this.pooling})`
118
149
  );
119
150
  return this;
120
151
  }
121
152
 
122
- /** Embed many texts with mean pooling; chunks at batchSize. */
153
+ /** Embed many texts with the model's pooling; chunks at batchSize. */
123
154
  async embed(texts) {
124
155
  if (!Array.isArray(texts)) throw new TypeError("embed expects an array of strings");
125
156
  if (!this.extractor) throw new Error("LocalEmbedder not initialized");
126
157
  const out = [];
127
158
  for (let i = 0; i < texts.length; i += this.batchSize) {
128
159
  const chunk = texts.slice(i, i + this.batchSize);
129
- const tensor = await this.extractor(chunk, { pooling: "mean", normalize: true });
160
+ const tensor = await this.extractor(chunk, { pooling: this.pooling, normalize: true });
130
161
  out.push(...tensorToRows(tensor));
131
162
  }
132
163
  return out;
@@ -142,7 +173,14 @@ export class LocalEmbedder {
142
173
  }
143
174
 
144
175
  get modelHash() {
145
- return modelHash(this.model);
176
+ // Pooling is part of the vector space: a CLS index cannot be queried with mean
177
+ // vectors (or vice versa). Keep the historical `model#hash` shape for `mean` so
178
+ // indexes that were never affected are not invalidated needlessly; every other
179
+ // pooling gets a fingerprint of its own, which makes the index-consistency gate
180
+ // flag the stale mean-space index and rebuild it.
181
+ return this.pooling === "mean"
182
+ ? modelHash(this.model)
183
+ : modelHash(`${this.model}@${this.pooling}`);
146
184
  }
147
185
 
148
186
  dispose() {
package/src/settings.js CHANGED
@@ -172,8 +172,10 @@ const FEATURE_FLAG_ENUMS = {
172
172
  summarizeDedupeMode: ["off", "title", "vector"],
173
173
  // Issue #126:sleep 冲突阶段的动作集(conflict 默认 = 现状;full = 六分支)。
174
174
  sleepActionSet: ["conflict", "full"],
175
- // Issue #125:dream 候选集构造方式(window 默认 = 现状;hybrid 并入向量组)。
176
- dreamCandidateMode: ["window", "hybrid"]
175
+ // Issue #125:dream 候选集构造方式(window 默认 = 现状;hybrid 并入向量组)。
176
+ dreamCandidateMode: ["window", "hybrid"],
177
+ // 本地嵌入池化:auto = 按模型族判定(BGE → cls),可显式覆盖为 cls / mean。
178
+ localEmbedPooling: ["auto", "cls", "mean"]
177
179
  };
178
180
  const FEATURE_FLAG_STRING_MAX = 200;
179
181
 
package/src/store.js CHANGED
@@ -1532,6 +1532,22 @@ export function createStore(path) {
1532
1532
  return rows.map(toDreamRun);
1533
1533
  }
1534
1534
 
1535
+ // Issue #89:上次实际开跑时刻(epoch ms),从审计表恢复——调度器的 lastRunAt
1536
+ // 只活在内存里,进程重启即归零,闸门对新实例放行 → 重启后立刻连发。审计表
1537
+ // 本来就逐 run 落库(failed/degraded 也算 run),直接读它就是事实源,零迁移。
1538
+ function lastDreamRunAt(runType = null) {
1539
+ const row = runType
1540
+ ? db.prepare(
1541
+ "SELECT created_at FROM dream_runs WHERE run_type = ? ORDER BY created_at DESC, id LIMIT 1"
1542
+ ).get(runType)
1543
+ : db.prepare(
1544
+ "SELECT created_at FROM dream_runs ORDER BY created_at DESC, id LIMIT 1"
1545
+ ).get();
1546
+ if (!row?.created_at) return 0;
1547
+ const ms = Date.parse(row.created_at);
1548
+ return Number.isFinite(ms) ? ms : 0;
1549
+ }
1550
+
1535
1551
  /**
1536
1552
  * Latest ruling-rule version seen on the audit trail. policy_epoch is a config
1537
1553
  * value stamped onto each run by the caller; reading the newest row's epoch
@@ -2486,6 +2502,7 @@ export function createStore(path) {
2486
2502
  needsEmbedding,
2487
2503
  searchVector,
2488
2504
  saveDreamRun,
2505
+ lastDreamRunAt,
2489
2506
  getDreamRun,
2490
2507
  listDreamRuns,
2491
2508
  getLatestPolicyEpoch,
package/test/api.test.js CHANGED
@@ -663,8 +663,9 @@ test("GET /api/dsh-mneme/features returns empty overrides and effective config d
663
663
  // issue #257 新增 sleepMaxTokens、issue #258 新增总览路由两键与
664
664
  // dreamSummaryMaxInputs、issue #230 新增 documentMemoryEnabled/
665
665
  // documentInjectBudget、issue #249 第一批新增 injectGuidanceEnabled/
666
- // pinnedInjectBudget,v0.8.5 新增 disableMemorySearch/disableMemoryArchive)
667
- assert.equal(Object.keys(data.effective).length, 59 + 3 + 2 + 1 + 2 + 2 + 2);
666
+ // pinnedInjectBudget,v0.8.5 新增 disableMemorySearch/disableMemoryArchive,
667
+ // 本地嵌入池化新增 localEmbedPooling)
668
+ assert.equal(Object.keys(data.effective).length, 59 + 3 + 2 + 1 + 2 + 2 + 2 + 1);
668
669
  assert.equal(data.effective.dreamSkipInvalid, true);
669
670
  assert.equal(data.effective.allowCrossTypeMerge, false);
670
671
  assert.equal(data.effective.dreamMinIntervalMinutes, 0);
@@ -0,0 +1,101 @@
1
+ import test from "node:test";
2
+ import assert from "node:assert/strict";
3
+ import { createStore } from "../src/store.js";
4
+ import { createService } from "../src/service.js";
5
+ import { createDreamScheduler } from "../src/dream.js";
6
+ import { createSleepScheduler } from "../src/dream/sleep.js";
7
+
8
+ // 回归(issue #89):调度器的 lastRunAt 只活在内存里,进程重启即归零——
9
+ // 最小间隔/冷却闸对新实例放行,重启后 autoDream/sleep 立即连发(Sample A/C 的
10
+ // 根因)。修复 = 构造调度器时从 dream_runs 审计表恢复种子(store.lastDreamRunAt,
11
+ // failed/degraded 也算 run,审计表本来就是逐 run 落库)。本文件锁三点:
12
+ // 1. lastDreamRunAt 读回最近一次 run 的开跑时刻(含 run_type 过滤);
13
+ // 2. dream 调度器带种子构造后,重启场景下最小间隔闸门立即生效;
14
+ // 3. sleep 调度器带种子构造后,冷却期内不起跑、且按剩余窗口重排不丢触发权。
15
+
16
+ test("issue#89: store.lastDreamRunAt recovers the last run timestamp (with run_type filter)", () => {
17
+ const store = createStore(":memory:");
18
+ assert.equal(store.lastDreamRunAt(), 0, "empty trail → 0 (never ran)");
19
+ assert.equal(store.lastDreamRunAt("sleep"), 0);
20
+ store.saveDreamRun({ status: "ok", snapshot_hash: "h1", input_count: 1, receipt: "r", created_at: "2026-09-10T01:05:23.640Z", run_type: "auto" });
21
+ store.saveDreamRun({ status: "ok", snapshot_hash: "h2", input_count: 1, receipt: "r", created_at: "2026-09-09T20:00:00.000Z", run_type: "sleep" });
22
+ assert.equal(store.lastDreamRunAt(), Date.parse("2026-09-10T01:05:23.640Z"), "latest run regardless of type");
23
+ assert.equal(store.lastDreamRunAt("sleep"), Date.parse("2026-09-09T20:00:00.000Z"), "filtered by run_type");
24
+ assert.equal(store.lastDreamRunAt("auto"), Date.parse("2026-09-10T01:05:23.640Z"), "auto filter ignores sleep rows");
25
+ store.close();
26
+ });
27
+
28
+ test("issue#89: dream scheduler seeded from the audit trail — restart no longer bypasses the min interval", async () => {
29
+ const store = createStore(":memory:");
30
+ const service = createService({ store, mirror: null, config: {} });
31
+ // 模拟上一进程刚跑过一轮(10ms 前开跑)并留下审计行,然后「重启」:
32
+ // minIntervalMs 取 1min——断言窗口远宽于用例执行时间,真实时钟抖动(调度、
33
+ // 慢盘)不会把 10ms 偏移推出闸门外造成间歇性假失败。
34
+ store.saveDreamRun({
35
+ status: "ok", snapshot_hash: "h", input_count: 1, receipt: "r",
36
+ created_at: new Date(Date.now() - 10).toISOString(), run_type: "auto"
37
+ });
38
+ const dream = createDreamScheduler({
39
+ onRun: async () => ({ ok: false, error: "llm failed" }),
40
+ thresholdCount: 1, thresholdChars: 0, delayMs: 0, minIntervalMs: 60000,
41
+ lastRunAtSeed: store.lastDreamRunAt("auto"),
42
+ logger: { warn: () => {} }
43
+ });
44
+ service.saveWithDedupe({ type: "project", title: "a", content: "x" });
45
+ // 无种子时这里会返回 true(旧缺陷:重启后立刻连发);带种子必须被闸门拦下。
46
+ assert.equal(dream.maybeSchedule(service), false, "seeded gate blocks immediately after 'restart'");
47
+ store.close();
48
+ });
49
+
50
+ test("issue#89: sleep scheduler seeded — cooldown survives a restart", async () => {
51
+ let nowMs = 1_000_000;
52
+ const timers = [];
53
+ let seq = 1;
54
+ let runs = 0;
55
+ const lastRunAt = nowMs - 10 * 60_000; // 上一进程 10 分钟前跑过,CD 1h 还剩 50min
56
+ const sched = createSleepScheduler({
57
+ service: { enqueue: (fn) => fn() },
58
+ config: { sleepModeEnabled: true, sleepIdleMinutes: 5, sleepMinIntervalHours: 1 },
59
+ logger: { warn: () => {} },
60
+ onRun: async () => { runs++; return { ok: true }; },
61
+ now: () => nowMs,
62
+ lastRunAtSeed: lastRunAt,
63
+ setTimeoutFn: (fn, delay) => { const t = { id: seq++, at: nowMs + delay, fn }; timers.push(t); return t.id; },
64
+ clearTimeoutFn: (id) => { const i = timers.findIndex((t) => t.id === id); if (i >= 0) timers.splice(i, 1); }
65
+ });
66
+ nowMs += 1000;
67
+ sched.noteWrite(); // 挂 idle 闹钟
68
+ const idleTimer = timers[0];
69
+ assert.ok(idleTimer, "idle timer armed on write");
70
+ nowMs = idleTimer.at;
71
+ timers.length = 0;
72
+ await idleTimer.fn();
73
+ assert.equal(runs, 0, "restored cooldown blocks the run");
74
+ const rearm = timers[0];
75
+ assert.ok(rearm, "re-armed for the remaining cooldown window (not dropped, #187 semantics)");
76
+ assert.equal(rearm.at, lastRunAt + 3_600_000 + 1000, "re-arms at CD expiry (+1s guard)");
77
+ });
78
+
79
+ // --- CodeRabbit #291 findings 5/6:index.js 的 seed 注入是单行无分支接线,仓库
80
+ // 没有插件级挂载 harness(apply 需要完整宿主 ctx),起全量挂载只为准两条断言
81
+ // 不成比例——改为源码级锁:删掉 index.js 的对应注入行即红。import 用绝对路径,
82
+ // 不依赖测试进程 CWD。
83
+ import { readFileSync } from "node:fs";
84
+ import { fileURLToPath } from "node:url";
85
+ const indexSrc = readFileSync(fileURLToPath(new URL("../src/index.js", import.meta.url)), "utf8");
86
+
87
+ test("issue#89: index.js wires the dream scheduler seed from the audit trail", () => {
88
+ assert.match(
89
+ indexSrc,
90
+ /lastRunAtSeed:\s*store\.lastDreamRunAt\("auto"\)/,
91
+ "dream scheduler must seed its min-interval gate from dream_runs (run_type='auto') — remove this line and restart-bypass returns"
92
+ );
93
+ });
94
+
95
+ test("issue#89: index.js wires the sleep scheduler seed from the audit trail", () => {
96
+ assert.match(
97
+ indexSrc,
98
+ /lastRunAtSeed:\s*store\.lastDreamRunAt\("sleep"\)/,
99
+ "sleep scheduler must seed its cooldown gate from dream_runs (run_type='sleep') — remove this line and restart-bypass returns"
100
+ );
101
+ });
@@ -7,6 +7,7 @@ import { createDreamScheduler } from "../src/dream.js";
7
7
  import { createSummarizer } from "../src/summarize.js";
8
8
  import { createApi } from "../src/api.js";
9
9
  import { createSettings } from "../src/settings.js";
10
+ import { createEntityStreamAdapter } from "../src/index.js";
10
11
  import { mockCtx } from "./helpers/dream-mock.js";
11
12
 
12
13
  // Bug8: LLM audit trail. Every background LLM call (autoDream consolidation +
@@ -460,3 +461,96 @@ test("GET /api/dsh-mneme/semantic/llm-audit/stats aggregates by source and statu
460
461
  assert.equal(errStatus.c, 1);
461
462
  store.close();
462
463
  });
464
+
465
+ // ------------------------------------------- entity extraction (issue #250)
466
+ // 实体抽取是「每次写入记忆都会跑」的后台 LLM 链路,但 createEntityStreamAdapter
467
+ // 只拿到 llm 句柄、没有 service,所以从来没有写审计的能力——面板因此回答不了
468
+ // 「关掉实体抽取能省多少」。下面两条用例锁定 usage chunk 汇总落库,以及审计写失败
469
+ // 只 warn 不反噬(CONTRIBUTING 的 fail-safe 硬约定)。
470
+
471
+ test("entity extraction writes an llm_audit row with the streamed usage tokens", async () => {
472
+ const store = createStore(":memory:");
473
+ const service = createService({ store, mirror: null, config: {} });
474
+ const streamEntityText = createEntityStreamAdapter({
475
+ llm: {
476
+ async *stream() {
477
+ yield { type: "text-delta", index: 0, text: '{"entities":[],"relations":[]}' };
478
+ yield { type: "usage", usage: { inputTokens: 700, outputTokens: 180, totalTokens: 880 } };
479
+ yield { type: "finish", reason: { kind: "stop" } };
480
+ }
481
+ },
482
+ agentDefaultModel: { currentSelection: () => ({ provider: "mock", model: "mock-model" }) },
483
+ logger: { warn: () => {} },
484
+ service
485
+ });
486
+ const text = await streamEntityText(
487
+ [{ role: "user", content: [{ type: "text", text: "记忆内容" }] }],
488
+ {}
489
+ );
490
+ assert.ok(text.includes("entities"), "the extracted text still reaches the extractor");
491
+
492
+ const row = service.listLlmAudits({ source: "entityExtract" })[0];
493
+ assert.ok(row, "entity_extract audit row present");
494
+ assert.equal(row.trigger_source, "entityExtract");
495
+ assert.equal(row.operation_type, "entity_extract");
496
+ assert.equal(row.status, "success");
497
+ assert.equal(row.model_id, "mock:mock-model", "the resolved route is recorded");
498
+ assert.equal(row.input_tokens, 700, "input tokens come from chunk.usage");
499
+ assert.equal(row.output_tokens, 180, "output tokens come from chunk.usage");
500
+ assert.equal(row.total_tokens, 880);
501
+ store.close();
502
+ });
503
+
504
+ test("entity extraction audit write failure only warns and never sinks the extraction", async () => {
505
+ const store = createStore(":memory:");
506
+ const service = createService({ store, mirror: null, config: {} });
507
+ // 审计表写不进去(只读库 / 磁盘满)绝不能把抽取本身拖失败—— fail-safe 是硬约定。
508
+ service.saveLlmAudit = () => { throw new Error("attempt to write a readonly database"); };
509
+ const warnings = [];
510
+ const streamEntityText = createEntityStreamAdapter({
511
+ llm: {
512
+ async *stream() {
513
+ yield { type: "text-delta", index: 0, text: '{"entities":[],"relations":[]}' };
514
+ yield { type: "usage", usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 } };
515
+ yield { type: "finish", reason: { kind: "stop" } };
516
+ }
517
+ },
518
+ agentDefaultModel: { currentSelection: () => ({ provider: "mock", model: "mock-model" }) },
519
+ logger: { warn: (m) => warnings.push(String(m)) },
520
+ service
521
+ });
522
+ const text = await streamEntityText(
523
+ [{ role: "user", content: [{ type: "text", text: "记忆内容" }] }],
524
+ {}
525
+ );
526
+ assert.ok(text.includes("entities"), "extraction text survives an audit write failure");
527
+ assert.ok(warnings.some((w) => w.includes("llm audit write failed")),
528
+ "the audit failure is logged rather than swallowed silently");
529
+ store.close();
530
+ });
531
+
532
+ test("entity extraction writes no audit row when llmAudit.enabled === false", async () => {
533
+ const store = createStore(":memory:");
534
+ const service = createService({ store, mirror: null, config: {} });
535
+ const streamEntityText = createEntityStreamAdapter({
536
+ llm: {
537
+ async *stream() {
538
+ yield { type: "text-delta", index: 0, text: '{"entities":[],"relations":[]}' };
539
+ yield { type: "usage", usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 } };
540
+ yield { type: "finish", reason: { kind: "stop" } };
541
+ }
542
+ },
543
+ agentDefaultModel: { currentSelection: () => ({ provider: "mock", model: "mock-model" }) },
544
+ logger: { warn: () => {} },
545
+ service,
546
+ // 单闸门:与 dream / sleep / summarize 同一个 llmAudit.enabled,不新增第二道。
547
+ config: { llmAudit: { enabled: false } }
548
+ });
549
+ const text = await streamEntityText(
550
+ [{ role: "user", content: [{ type: "text", text: "记忆内容" }] }],
551
+ {}
552
+ );
553
+ assert.ok(text.includes("entities"), "turning the audit off must not break extraction");
554
+ assert.equal(service.listLlmAudits().length, 0, "no audit rows when disabled");
555
+ store.close();
556
+ });
@@ -79,7 +79,7 @@ test("LocalEmbedder init threads resilientModelDownload and logger to engineFact
79
79
  assert.equal(seen.resilientModelDownload, false, "直连构造不显式给 true = 保持现状");
80
80
  });
81
81
 
82
- test("LocalEmbedder embed returns [n, dim] with mean pooling and chunking", async () => {
82
+ test("LocalEmbedder embed returns [n, dim] with the model's pooling and chunking", async () => {
83
83
  const calls = [];
84
84
  const e = new LocalEmbedder({
85
85
  model: "Xenova/bge-small-zh-v1.5",
@@ -95,6 +95,18 @@ test("LocalEmbedder embed returns [n, dim] with mean pooling and chunking", asyn
95
95
  }
96
96
  // 3 texts at batchSize 2 -> two extractor calls.
97
97
  assert.deepEqual(calls.map((c) => c.count), [2, 1]);
98
+ // BGE 系按 CLS 训练(模型自带 1_Pooling/config.json: pooling_mode_cls_token=true)
99
+ assert.deepEqual(calls[0].opts, { pooling: "cls", normalize: true });
100
+ });
101
+
102
+ test("LocalEmbedder keeps mean pooling for models outside the CLS families", async () => {
103
+ const calls = [];
104
+ const e = new LocalEmbedder({
105
+ model: "sentence-transformers/all-MiniLM-L6-v2",
106
+ engineFactory: async () => makeFakeExtractor(384, calls)
107
+ });
108
+ await e.init();
109
+ await e.embed(["x"]);
98
110
  assert.deepEqual(calls[0].opts, { pooling: "mean", normalize: true });
99
111
  });
100
112
 
@@ -105,14 +117,45 @@ test("LocalEmbedder embedSingle returns a single vector", async () => {
105
117
  assert.equal(v.length, 384);
106
118
  });
107
119
 
120
+ test("LocalEmbedder resolves pooling by model family, explicit value wins", async () => {
121
+ const cases = [
122
+ [{ model: "Xenova/bge-small-zh-v1.5" }, "cls", "BGE 默认 cls"],
123
+ [{ model: "BAAI/bge-m3" }, "cls", "BGE 大小写无关"],
124
+ [{ model: "sentence-transformers/all-MiniLM-L6-v2" }, "mean", "非 BGE 族保持 mean"],
125
+ [{ model: "Xenova/bge-small-zh-v1.5", pooling: "mean" }, "mean", "显式覆盖 auto"],
126
+ [{ model: "sentence-transformers/all-MiniLM-L6-v2", pooling: "cls" }, "cls", "显式覆盖非 BGE"],
127
+ [{ model: "Xenova/bge-small-zh-v1.5", pooling: "AUTO" }, "cls", "大小写/空白容错"],
128
+ [{ model: "Xenova/bge-small-zh-v1.5", pooling: " bogus " }, "cls", "未知值退回 auto"]
129
+ ];
130
+ for (const [opts, expected, why] of cases) {
131
+ const calls = [];
132
+ const e = new LocalEmbedder({ ...opts, engineFactory: async () => makeFakeExtractor(512, calls) });
133
+ assert.equal(e.pooling, expected, why);
134
+ await e.init();
135
+ await e.embed(["x"]);
136
+ assert.equal(calls[0].opts.pooling, expected, `${why}: 必须真的传进 extractor`);
137
+ }
138
+ });
139
+
108
140
  test("LocalEmbedder dimension and modelHash are stable", () => {
109
141
  const a = new LocalEmbedder({ model: "Xenova/bge-small-zh-v1.5" });
110
142
  const b = new LocalEmbedder({ model: "Xenova/bge-small-zh-v1.5" });
111
143
  assert.equal(a.dimension, 512);
112
144
  assert.equal(a.modelHash, b.modelHash);
113
- assert.match(a.modelHash, /^Xenova\/bge-small-zh-v1\.5#[0-9a-f]+$/);
145
+ // BGE 解析为 cls,且池化进指纹(否则旧索引不会被判失配、不会重建)
146
+ assert.match(a.modelHash, /^Xenova\/bge-small-zh-v1\.5@cls#[0-9a-f]+$/);
114
147
  // Different model -> different hash.
115
148
  assert.notEqual(a.modelHash, new LocalEmbedder({ model: "other/model" }).modelHash);
149
+ // 显式 mean 保持历史指纹形状,不给未受影响的索引制造无谓重建
150
+ assert.match(
151
+ new LocalEmbedder({ model: "Xenova/bge-small-zh-v1.5", pooling: "mean" }).modelHash,
152
+ /^Xenova\/bge-small-zh-v1\.5#[0-9a-f]+$/
153
+ );
154
+ // 两种池化是不同向量空间,绝不能共用一个指纹
155
+ assert.notEqual(
156
+ a.modelHash,
157
+ new LocalEmbedder({ model: "Xenova/bge-small-zh-v1.5", pooling: "mean" }).modelHash
158
+ );
116
159
  });
117
160
 
118
161
  test("LocalEmbedder throws before init and after dispose", async () => {