@modusensus/dsh-mneme 0.8.4 → 0.8.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/README.md +45 -8
  2. package/bin/dsh-mneme-mcp.mjs +11 -3
  3. package/lib/api-standalone.js +11 -0
  4. package/lib/api.js +21 -0
  5. package/lib/client.js +213 -31
  6. package/lib/config.js +106 -5
  7. package/lib/document.js +221 -0
  8. package/lib/dream/sleep.js +128 -12
  9. package/lib/dream.js +46 -11
  10. package/lib/guide.js +44 -0
  11. package/lib/heat.js +2 -0
  12. package/lib/index.js +65 -3
  13. package/lib/inject.js +91 -15
  14. package/lib/lang.js +6 -0
  15. package/lib/local-embedder.js +44 -6
  16. package/lib/organize.js +450 -0
  17. package/lib/recall-stats.js +23 -1
  18. package/lib/runtime/layout.js +30 -0
  19. package/lib/runtime/loader.js +4 -1
  20. package/lib/runtime/verify.js +19 -6
  21. package/lib/search/adaptive.js +38 -0
  22. package/lib/service.js +165 -12
  23. package/lib/settings.js +35 -4
  24. package/lib/store.js +126 -8
  25. package/lib/summarize.js +253 -8
  26. package/lib/tools.js +136 -8
  27. package/package.json +6 -6
  28. package/runtime-manifest.json +104 -174
  29. package/scripts/build-runtime-manifest.mjs +8 -1
  30. package/scripts/sync-test-badge.mjs +18 -12
  31. package/src/api-standalone.js +11 -0
  32. package/src/api.js +21 -0
  33. package/src/config.js +106 -5
  34. package/src/document.js +221 -0
  35. package/src/dream/sleep.js +128 -12
  36. package/src/dream.js +46 -11
  37. package/src/guide.js +44 -0
  38. package/src/heat.js +2 -0
  39. package/src/index.js +65 -3
  40. package/src/inject.js +91 -15
  41. package/src/lang.js +6 -0
  42. package/src/local-embedder.js +44 -6
  43. package/src/organize.js +450 -0
  44. package/src/recall-stats.js +23 -1
  45. package/src/runtime/layout.js +30 -0
  46. package/src/runtime/loader.js +4 -1
  47. package/src/runtime/verify.js +19 -6
  48. package/src/search/adaptive.js +38 -0
  49. package/src/service.js +165 -12
  50. package/src/settings.js +35 -4
  51. package/src/store.js +126 -8
  52. package/src/summarize.js +253 -8
  53. package/src/tools.js +136 -8
  54. package/test/adaptive-inject-budget.test.js +85 -0
  55. package/test/api.test.js +66 -3
  56. package/test/client.test.js +63 -2
  57. package/test/document.test.js +523 -0
  58. package/test/dream.test.js +51 -0
  59. package/test/inject-content-cap.test.js +5 -3
  60. package/test/inject-pools.test.js +170 -0
  61. package/test/inject-recall.test.js +105 -0
  62. package/test/inject.test.js +35 -1
  63. package/test/lastrunat-persist.test.js +101 -0
  64. package/test/llm-audit.test.js +247 -0
  65. package/test/local-embedder.test.js +45 -2
  66. package/test/organize.test.js +418 -0
  67. package/test/recall-stats.test.js +7 -1
  68. package/test/runtime-loader.test.js +18 -0
  69. package/test/runtime-manifest.test.js +2 -1
  70. package/test/runtime-verify.test.js +65 -0
  71. package/test/sleep.test.js +154 -0
  72. package/test/store.test.js +13 -0
  73. package/test/summarize.test.js +407 -3
  74. package/test/tools-dto-schema-matrix.test.js +325 -0
  75. package/test/tools.test.js +36 -3
package/lib/dream.js CHANGED
@@ -6,7 +6,6 @@ import { createHash, randomUUID } from "node:crypto";
6
6
  import { STR, langOf } from "./lang.js";
7
7
  export { validateDecisions, applyDecisions, withEffortFallback, describeStreamFailure, resolveDreamEffort, resolveRoute };
8
8
 
9
-
10
9
  // Extract the first JSON array from LLM output, tolerating markdown fences,
11
10
  // leading/trailing prose, and common wrapper noise. Returns an array or null.
12
11
  function extractJsonArray(text) {
@@ -232,7 +231,12 @@ async function streamText(ctx, options, onUsage, onStreamError) {
232
231
  let text = "";
233
232
  for await (const chunk of ctx.llm.stream(options)) {
234
233
  if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
235
- if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk);
234
+ // DSH 的 StreamChunk 契约把用量嵌在 chunk.usage(TokenUsage:inputTokens /
235
+ // outputTokens / …),chunk 顶层没有 token 字段——把整个 chunk 当用量对象交给
236
+ // reporter,input/output 恒为 undefined,审计行落 0(v0.8.4 实测:7 天 49 次
237
+ // success 调用 token 全 0,面板「LLM 消耗」长期显示 0)。`?? chunk` 兜底把用量
238
+ // 平铺在顶层的替身。
239
+ if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk.usage ?? chunk);
236
240
  if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
237
241
  // dsh-llm rc.1 turns adapter-stage failures (unknown provider route,
238
242
  // UNSUPPORTED_REASONING_EFFORT from resolveCallWithInfo, …) into a
@@ -382,7 +386,7 @@ async function reEmbedMemory(semantic, memory, logger) {
382
386
  * 任何失败降级为 0 条,绝不反噬 dream 主流程。
383
387
  */
384
388
  async function generateNarratives({ ctx, service, config, route, language, effort, semantic, logger }) {
385
- const inputs = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary");
389
+ const inputs = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document");
386
390
  const clusters = clusterByTag(inputs, { minCluster: config.dreamNarrativeMinCluster ?? 3 });
387
391
  if (!clusters.length) return 0;
388
392
 
@@ -740,7 +744,7 @@ export async function maintainIndexAfterDream(decisions, service, semantic) {
740
744
  if (embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
741
745
  }
742
746
 
743
- export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChars = 5000, delayMs = 2000, minIntervalMs = 0, logger, semantic = null }) {
747
+ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChars = 5000, delayMs = 2000, minIntervalMs = 0, logger, semantic = null, lastRunAtSeed = 0 }) {
744
748
  let pendingTimer = null;
745
749
  let running = false;
746
750
  let disposed = false;
@@ -748,10 +752,12 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
748
752
  let inFlight = null;
749
753
  // Issue #89(请求 2):上一次实际开跑时刻。失败/degraded 的 run 也占用
750
754
  // 最小间隔——节流的目的正是防止失败调用连发;间隔内的触发静默跳过。
751
- let lastRunAt = 0;
755
+ // lastRunAtSeed:调用方从 dream_runs 审计表读出的上次开跑时刻——
756
+ // 内存变量进程重启即归零,闸门对新实例放行 → 重启后立刻连发(#89 根因)。
757
+ let lastRunAt = lastRunAtSeed;
752
758
 
753
759
  function shouldTrigger(service) {
754
- const memories = service.all().filter((m) => !m.archived && m.type !== "summary");
760
+ const memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
755
761
  const count = memories.length;
756
762
  const chars = totalChars(memories);
757
763
  const overBase = count >= baseline.count + thresholdCount || chars >= baseline.chars + thresholdChars;
@@ -814,8 +820,10 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
814
820
  async function runDream(ctx, service, config) {
815
821
  const language = langOf(config);
816
822
  const logger = ctx.logger;
817
- let memories = service.all().filter((m) => !m.archived && m.type !== "summary");
823
+ let memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
818
824
  if (memories.length === 0) return { ok: true, applied: 0, skipped: true, summary: false };
825
+ // Issue #89:开跑时刻既喂给审计行(created_at),也是调度器闸门的时间基准。
826
+ const dreamStartedAt = Date.now();
819
827
  // v0.4.4 滑动窗口:只 consolidation 最近 dreamMaxSnapshotSize 条记忆,
820
828
  // 窗口外的旧记忆不进 snapshot(大记忆量下全量快照会撑爆 LLM 输入,配合
821
829
  // 隐式 keep 让 run 始终可收敛)。按 updated_at 倒序取前 maxSize 条。
@@ -882,6 +890,9 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
882
890
  try {
883
891
  service.saveDreamRun({
884
892
  id: runId,
893
+ // Issue #89:审计行记开跑时刻而非完成时刻——lastDreamRunAt 以它做
894
+ // 重启后的间隔种子,落完成时刻会让 run 耗时白白计入下一轮最小间隔。
895
+ created_at: new Date(dreamStartedAt).toISOString(),
885
896
  status,
886
897
  error: result.error,
887
898
  provider: route?.provider,
@@ -1206,15 +1217,21 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1206
1217
  let summaryStreamFailure = "";
1207
1218
  const runSummary = (withEffort) => {
1208
1219
  summaryStreamFailure = "";
1220
+ // Issue #258:总览阶段可用 dreamSummaryProvider/dreamSummaryModel 走独立路由
1221
+ //(默认空 = 沿用 dream 路由);输入侧有 dreamSummaryMaxInputs 硬上限,
1222
+ // 按窗口同款 updated_at 倒序保留最新的(见下方 summaryInputs)。
1223
+ const summaryRoute = (config.dreamSummaryProvider && config.dreamSummaryModel)
1224
+ ? { provider: config.dreamSummaryProvider, model: config.dreamSummaryModel }
1225
+ : route;
1209
1226
  return runAuditedLlm(ctx, service, config, {
1210
1227
  triggerSource: "autoDream",
1211
1228
  operationType: "dream_summarize",
1212
- modelId: `${route.provider}:${route.model}`,
1229
+ modelId: `${summaryRoute.provider}:${summaryRoute.model}`,
1213
1230
  relatedMemoryIds: [],
1214
1231
  streamError: () => summaryStreamFailure
1215
1232
  }, (reportUsage) => streamText(ctx, {
1216
- provider: route.provider,
1217
- model: route.model,
1233
+ provider: summaryRoute.provider,
1234
+ model: summaryRoute.model,
1218
1235
  purpose: "compaction",
1219
1236
  maxTokens: config.dreamMaxTokens ?? 2048,
1220
1237
  ...(withEffort && effort ? { reasoningEffort: effort } : {}),
@@ -1229,7 +1246,25 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1229
1246
  // v1, content_history traceable). Its 口径 (what snapshot produced it) is
1230
1247
  // stamped into the content so the standing answer is always auditable; the
1231
1248
  // formal evidence column lands with the narrative-bars batch.
1232
- const summaryInputs = service.all().filter((m) => !m.archived && m.type !== "summary");
1249
+ // Issue #258:输入硬上限(条数)——原全库无界,把 dream 路由指向小 ctx
1250
+ // 模型时 120k token 当场 CONTEXT_WINDOW_EXCEEDED(#258 实测),且随库增长
1251
+ // 渐进恶化。倒序排序与 consolidate 窗口同款(updated_at desc, id tiebreak),
1252
+ // 保留最新的;0 = 关闭上限(回归全库行为,调用方自担 ctx)。
1253
+ // #230:document 指针行不进总览(与 dream/sleep 五个候选池的同款排除)。
1254
+ const summaryAll = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
1255
+ const summaryMaxInputs = Number.isInteger(config.dreamSummaryMaxInputs) ? config.dreamSummaryMaxInputs : 0;
1256
+ const summaryInputs = summaryMaxInputs > 0 && summaryAll.length > summaryMaxInputs
1257
+ ? [...summaryAll]
1258
+ .sort((a, b) => {
1259
+ const ta = String(a.updated_at ?? "");
1260
+ const tb = String(b.updated_at ?? "");
1261
+ if (ta < tb) return 1;
1262
+ if (ta > tb) return -1;
1263
+ return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
1264
+ })
1265
+ .slice(0, summaryMaxInputs)
1266
+ : summaryAll;
1267
+
1233
1268
  const summaryScope = STR.summaryScope[language](
1234
1269
  summaryInputs.length,
1235
1270
  runId.slice(0, 8),
package/lib/guide.js ADDED
@@ -0,0 +1,44 @@
1
+ // #249(第一批):能力说明——写给模型的「怎么用这套记忆」,不是给用户的文案。
2
+ //
3
+ // 为什么英文、且只留一份正本:注入指引的参照实现全部用英文(ACP 的
4
+ // ACP_SYSTEM_PROMPT + HOW_TO_COMPRESS_RULES、mnemon 的 ROUTING_GUIDANCE、
5
+ // 宿主压缩摘要规则),仓库既有先例也是 src/tools.js 的工具描述硬编码英文。
6
+ // src/lang.js 的 memory.language 管的是「生成出来的记忆内容与块内标题」,与本
7
+ // 模块是两件事,故不并入 STR、不做 zh/en 双写(双写只会让两份文本日后漂移)。
8
+ //
9
+ // 两个承载位(#249 §5):①工具描述——常驻、零注入成本,放「何时用这个工具」
10
+ // 这类单工具指引(TOOL_GUIDE);②系统提示段 order 150——放工具描述装不下的
11
+ // 总则(优先序、何时查 / 何时写 / 何时 no-op)。该段必须同会话内稳定,否则
12
+ // 每轮变化会作废其后的前缀缓存,因此这里全是常量,不含任何运行时插值。
13
+ //
14
+ // 前缀 `[dsh-mneme memory]` 是机械校验用的:测试按它断言「只在开时出现、且不
15
+ // 逐轮复读」,也让维护者能一眼认出这段文本的归属。
16
+
17
+ const SECTION_LINES = [
18
+ "[dsh-mneme memory] Read once; it applies to every turn of this session.",
19
+ "1. Precedence: the current instruction and the repository's actual state outrank any stored memory. " +
20
+ "When a memory contradicts either, check the instruction or the repository itself before relying on it — memory_search " +
21
+ "searches stored memories only, so it finds earlier context, never the present state. Never treat an old memory as current fact.",
22
+ "2. Recall on demand: call memory_search when the task depends on earlier decisions, user preferences, or project history " +
23
+ "that is not already in context. Do not search for facts you can read directly from the repository.",
24
+ "3. Write back sparingly: use memory_save for durable, cross-session value — a preference, a decision with its rationale, " +
25
+ "an engineering constraint, a pitfall with its root cause. Trivial single-turn work is not worth a memory.",
26
+ "4. When unsure, do nothing. Not acting is a valid outcome: a useless memory is paid for by every future session.",
27
+ "5. Prefer the reversible tools. memory_archive hides an entry from lists, search, injection and consolidation, and " +
28
+ "memory_forget suppresses it from injection, search results and lists — both are recoverable. memory_delete is permanent, " +
29
+ "so reach for it only when an entry is wrong or unwanted, not merely stale."
30
+ ];
31
+
32
+ /** 系统提示段(order 150)的总则文本。常量:同会话内稳定是硬约束。 */
33
+ export const MEMORY_GUIDE_SECTION = SECTION_LINES.join("\n");
34
+
35
+ /** 单工具判断指引,追加到对应工具描述尾部(`injectGuidanceEnabled` 开启时)。 */
36
+ export const TOOL_GUIDE = {
37
+ memory_search:
38
+ " Use this when the task depends on earlier decisions, preferences, or project history that is not already in context, " +
39
+ "or to look for a newer memory behind one that seems stale. " +
40
+ "Skip it for facts you can read directly from the repository.",
41
+ memory_save:
42
+ " Save only durable, cross-session value (a preference, a decision with its rationale, an engineering constraint, " +
43
+ "a pitfall with its root cause). Trivial single-turn work does not belong here, and when unsure, do not save."
44
+ };
package/lib/heat.js CHANGED
@@ -16,6 +16,8 @@ export const TYPE_DECAY_DEFAULTS = Object.freeze({
16
16
  preference: 0, // 免疫:用户画像需长期保持
17
17
  pattern: 0, // 免疫:发现型稳定规律
18
18
  summary: 0, // 免疫:已是压缩产物
19
+ document: 0, // 免疫:指针行(#230)——全文在文件系统,行失效靠 supersede
20
+ // 记账而非遗忘;遗忘曲线灭掉指针 = 按需读全文的入口消失
19
21
  project: 0.0008, // 慢衰减
20
22
  decision: 0.002, // 中速衰减
21
23
  history: 0.006, // 较快(会话摘要不断被合并)
package/lib/index.js CHANGED
@@ -43,7 +43,15 @@ export { Config };
43
43
  // unit-testable; the extractor only ever sees a callLLM(messages, options)
44
44
  // => Promise<string>. The route always carries a real provider/model (dsh-llm
45
45
  // GenerateOptions requires both) — never a bare stream.
46
- export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
46
+ //
47
+ // Issue #250: it also accounts for itself. Entity extraction is the background
48
+ // LLM path that runs on every memory write, yet it never reached
49
+ // llm_audit_logs — the adapter only ever held an llm handle, so it had no way
50
+ // to call service.saveLlmAudit. It now takes service (+ config for the shared
51
+ // llmAudit.enabled gate) and writes one row per stream attempt, matching the
52
+ // contract of runAuditedLlm in dream.js: a rejected effort attempt records its
53
+ // own error row and the retry records its own success row.
54
+ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger, service, config }) {
47
55
  return async function streamEntityText(messages, options = {}) {
48
56
  let route = {};
49
57
  if (options.provider) route.provider = options.provider;
@@ -58,8 +66,38 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
58
66
  } catch { /* fall through to whatever route we already have */ }
59
67
  }
60
68
  const effort = options.reasoningEffort;
69
+ // 没有解析出 provider/model 就没有可归属的模型——与 dream 的 resolveRoute
70
+ // 无路由早退同口径,那种情况不写审计行。
71
+ const modelId = route.provider && route.model ? `${route.provider}:${route.model}` : "";
61
72
  const tryStream = (withEffort) => {
62
73
  let text = "";
74
+ let inputTokens = 0;
75
+ let outputTokens = 0;
76
+ const startedAt = Date.now();
77
+ const timestamp = new Date(startedAt).toISOString();
78
+ // 记账是 best-effort:写审计行失败只 warn,绝不反噬抽取本身(CONTRIBUTING
79
+ // 的 fail-safe 硬约定)。
80
+ const writeAudit = (status, errorMessage) => {
81
+ if (config?.llmAudit?.enabled === false || !modelId || typeof service?.saveLlmAudit !== "function") return;
82
+ try {
83
+ service.saveLlmAudit({
84
+ timestamp,
85
+ trigger_source: "entityExtract",
86
+ operation_type: "entity_extract",
87
+ model_id: modelId,
88
+ input_tokens: inputTokens,
89
+ output_tokens: outputTokens,
90
+ total_tokens: inputTokens + outputTokens,
91
+ cost_usd: 0,
92
+ duration_ms: Date.now() - startedAt,
93
+ status,
94
+ error_message: errorMessage,
95
+ related_memory_ids: []
96
+ });
97
+ } catch (auditError) {
98
+ logger?.warn?.(`dsh-mneme: entity extraction llm audit write failed: ${String(auditError)}`);
99
+ }
100
+ };
63
101
  return (async () => {
64
102
  for await (const chunk of llm.stream({
65
103
  ...route,
@@ -68,11 +106,26 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger }) {
68
106
  messages
69
107
  })) {
70
108
  if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
71
- if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) return undefined;
109
+ // DSH 的 StreamChunk 契约把用量嵌在 chunk.usage(TokenUsage);兼容直接
110
+ // 平铺在 chunk 上的旧形态。归一放在这里,下面只需认平铺形状——与
111
+ // dream.js 的 streamText 同一处理。
112
+ if (chunk.type === "usage") {
113
+ const u = chunk.usage ?? chunk;
114
+ const i = u.input_tokens ?? u.inputTokens ?? u.prompt_tokens ?? u.promptTokens;
115
+ const o = u.output_tokens ?? u.outputTokens ?? u.completion_tokens ?? u.completionTokens;
116
+ if (Number.isFinite(i)) inputTokens = i;
117
+ if (Number.isFinite(o)) outputTokens = o;
118
+ }
119
+ if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
120
+ writeAudit("error", "llm stream aborted or errored");
121
+ return undefined;
122
+ }
72
123
  }
124
+ writeAudit("success", null);
73
125
  return text;
74
126
  })().catch((err) => {
75
127
  logger?.warn?.(`[dsh-mneme] entity extraction llm stream failed: ${String(err)}`);
128
+ writeAudit("error", String(err?.message ?? err));
76
129
  return undefined;
77
130
  });
78
131
  };
@@ -273,6 +326,9 @@ export const apply = (ctx, config) => {
273
326
  dimension: cfg.localEmbedDimension,
274
327
  device: cfg.localEmbedDevice,
275
328
  batchSize: cfg.localEmbedBatchSize,
329
+ // 池化方式必须与模型的训练口径一致(BGE 系 = CLS)。它既进 embed() 的调用,
330
+ // 也进 modelHash —— 池化改了就是换向量空间,既有索引会被判失配并重建。
331
+ pooling: cfg.localEmbedPooling,
276
332
  cacheDir: cfg.embedModelCacheDir,
277
333
  runtimeDir: cfg.runtimeDir,
278
334
  // #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
@@ -403,6 +459,7 @@ export const apply = (ctx, config) => {
403
459
  minIntervalMs: (cfg.dreamMinIntervalMinutes ?? 0) * 60000,
404
460
  logger: ctx.logger,
405
461
  semantic: { embedder, vectorIndex },
462
+ lastRunAtSeed: store.lastDreamRunAt("auto"),
406
463
  onRun: () => (dream ? dream.runDream(ctx, service, cfg) : Promise.resolve({ ok: true, skipped: true }))
407
464
  });
408
465
  service.setDreamHook(() => dream.maybeSchedule(service));
@@ -419,6 +476,7 @@ export const apply = (ctx, config) => {
419
476
  service,
420
477
  config: cfg,
421
478
  logger: ctx.logger,
479
+ lastRunAtSeed: store.lastDreamRunAt("sleep"),
422
480
  onRun: (signal) => (sleep ? runSleep(ctx, service, cfg, ctx.logger, { embedder, vectorIndex }, signal) : Promise.resolve({ ok: true, skipped: true }))
423
481
  });
424
482
  service.setSleepHook(() => sleep.noteWrite());
@@ -442,7 +500,11 @@ export const apply = (ctx, config) => {
442
500
  const streamEntityText = createEntityStreamAdapter({
443
501
  llm: ctx.llm,
444
502
  agentDefaultModel: ctx.agentDefaultModel,
445
- logger: ctx.logger
503
+ logger: ctx.logger,
504
+ // Issue #250: the adapter needs service to write its llm_audit_logs row
505
+ // (it never had it) and config for the shared llmAudit.enabled gate.
506
+ service,
507
+ config: cfg
446
508
  });
447
509
  service.setEntityExtractor((memory) =>
448
510
  extractEntities(memory, { store, config: cfg, callLLM: streamEntityText, logger: ctx.logger })
package/lib/inject.js CHANGED
@@ -1,6 +1,19 @@
1
1
  import { createScopeResolver } from "./scope.js";
2
2
  import { createHotMemory } from "./hot-memory.js";
3
3
  import { STR, langOf } from "./lang.js";
4
+ import { MEMORY_GUIDE_SECTION } from "./guide.js";
5
+ import { adaptiveInjectBudget } from "./search/adaptive.js";
6
+
7
+ // Issue #179:注入预览的数据底座。systemPrompt 渲染是同步回调,面板只能事后
8
+ // 拉取,所以在这里旁路缓存「最近一帧组装」——快照就是本次渲染用过的同一份
9
+ // candidates/hotText/body,不重新检索、不二次组装,零额外开销。模块级单例:
10
+ // 注入器全局唯一(index.js 仅在 autoInject 开启时注册一个),面板经 api.js 的
11
+ // 只读 getter 读取;autoInject 关闭时注入器不存在,快照保持 null——「预览
12
+ // 不可用」本身就是「注入关闭」的诚实呈现。
13
+ let injectionSnapshot = null;
14
+ export function getInjectionSnapshot() {
15
+ return injectionSnapshot;
16
+ }
4
17
 
5
18
  // Best-effort extraction of the current user's latest message text from the
6
19
  // live session, for semantic-first injection (Bug4). The system-prompt
@@ -92,7 +105,7 @@ function extractRounds(ctx, maxRounds) {
92
105
 
93
106
  export function createInjector(ctx, service, settings, config) {
94
107
  const language = langOf(config);
95
- const maxItems = config.maxInjectedItems ?? 5;
108
+ const baseMaxItems = config.maxInjectedItems ?? 5;
96
109
  const threshold = config.importanceThreshold ?? 3;
97
110
  // Issue #205:注入位跨轮轮换。rotationTurns = 最近 N 个「不同用户查询」轮次
98
111
  // 注入过的记忆本轮不再优先(0 = 关闭,保持既有行为)。历史按会话维护——
@@ -143,13 +156,18 @@ export function createInjector(ctx, service, settings, config) {
143
156
  // stale 1500) and an entry that would exceed it collapses to its title only.
144
157
  const maxContent = config.injectContentMaxChars ?? 300;
145
158
  const MAX_BLOCK = Math.max(1500, maxContent + 600);
159
+ // #249 第一批:pin 条目的正文硬顶。逐字保真不等于无界——单条上限取
160
+ // injectContentMaxChars 与 2000 的较大者,默认档下约束/偏好不会因常规截断
161
+ // 失真,同时给「一条超长约束每轮吃满常驻段」留一道闸(超顶仍带截断提示)。
162
+ const PINNED_CONTENT_MAX = Math.max(maxContent, 2000);
146
163
 
147
- // Compressed injection (v0.5.0 2.1): a sleep-demoted row already carries its
148
- // summary in `content` with the original parked in `_full_content` — inject
149
- // the summary verbatim instead of re-truncating the (already short) text.
150
- // Regular long rows keep the hard truncate.
164
+ // Compressed injection (v0.5.0 2.1): a sleep-demoted row carries its summary in
165
+ // `content` with the original parked in `_full_content` — inject the summary
166
+ // verbatim instead of re-truncating the (already short) text. 但「降级过」不等于
167
+ // 「天然够短」:原先那条 `_full_content` 早返回的唯一效果就是**跳过 maxLength**
168
+ // (两条分支取的都是 m.content),于是 pin 的硬顶在这一路上形同不存在(#266 评审)。
169
+ // 现在只有一条截断路径:超限照旧带提示,原文永远可通过 memory_get 取回。
151
170
  function injectMemory(m, maxLength = maxContent) {
152
- if (m?._full_content) return String(m.content ?? "");
153
171
  const text = String(m?.content ?? "");
154
172
  if (text.length <= maxLength) return text;
155
173
  return `${text.slice(0, maxLength)}…${STR.truncatedHint[language](maxLength, text.length, m.id)}`;
@@ -192,26 +210,41 @@ export function createInjector(ctx, service, settings, config) {
192
210
  return escapePromptVars(STR.hotHeader[language](rounds.length, body));
193
211
  }
194
212
 
195
- function render(candidates) {
213
+ function render(candidates, pinnedStats = null) {
196
214
  if (!candidates.length) return "";
197
215
  const header = STR.memoryHeader[language];
198
216
  const lines = [header];
199
217
  let budget = MAX_BLOCK - header.length;
200
- for (const m of candidates) {
218
+ // #249 第一批:pin 池(约束/偏好)排在块头且不参与块的塌缩预算——「与情景
219
+ // 日志抢同一份预算后被降级」正是本议题要修的结构缺陷。单条仍有硬顶
220
+ // (PINNED_CONTENT_MAX),超出照旧带截断提示,不静默。
221
+ const pinnedCount = Math.max(0, Math.min(candidates.length, pinnedStats?.shown ?? 0));
222
+ for (let i = 0; i < candidates.length; i++) {
223
+ const m = candidates[i];
224
+ const pinned = i < pinnedCount;
201
225
  // Epistemic trust (v0.4.5): when enabled, measured observations are
202
226
  // flagged so the agent can weigh them above guesses/opinions.
203
227
  const verified = config.trustEpistemicWeighting === true && m.epistemic_status === "observation"
204
228
  ? STR.verified[language]
205
229
  : "";
206
230
  const title = STR.entryTitle[language](m.title, m.importance);
207
- const content = injectMemory(m);
231
+ const content = injectMemory(m, pinned ? PINNED_CONTENT_MAX : maxContent);
208
232
  const full = STR.entryLine[language](m.type, verified, title, content);
209
- if (budget - full.length >= 0) {
233
+ if (pinned) {
234
+ // pin 不扣块预算(上面那条设计注释的落地):pin 一条就够击穿 MAX_BLOCK
235
+ // (PINNED_CONTENT_MAX 2000 > MAX_BLOCK 1500),照扣会把 budget 压成负数,
236
+ // 同一轮随后的普通候选全部退化成标题行——正是本议题要修的结构缺陷。
237
+ lines.push(full);
238
+ } else if (budget - full.length >= 0) {
210
239
  lines.push(full);
211
240
  budget -= full.length;
212
241
  } else {
213
242
  lines.push(`- [${m.type}] ${verified}${title}`);
214
243
  }
244
+ if (pinned && i === pinnedCount - 1) {
245
+ const hidden = Math.max(0, pinnedStats?.suppressed ?? 0);
246
+ if (hidden > 0) lines.push(STR.pinnedOverflow[language](hidden));
247
+ }
215
248
  }
216
249
  return escapePromptVars(lines.join("\n"));
217
250
  }
@@ -250,7 +283,17 @@ export function createInjector(ctx, service, settings, config) {
250
283
  return escapePromptVars(lines.join("\n"));
251
284
  }
252
285
 
286
+ // #249 第一批:能力说明的系统提示段。order 150 = 插件指引段的既有惯例
287
+ // (ACP 的 ACP_SYSTEM_PROMPT 与 mnemon 的 routing 段都在 150)。文本是常量、
288
+ // 同会话内不随轮次变化,故不影响其后的前缀缓存,也不逐轮复读。宿主若不提供
289
+ // section seam 就静默跳过——#249 §3 的口径是「只用现成 seam,拿不到位的语义
290
+ // 降级处理」:能力说明仍落在工具描述上,不算失败。
291
+ const guideSection = config.injectGuidanceEnabled === true && typeof ctx.systemPrompt?.section === "function"
292
+ ? ctx.systemPrompt.section({ name: "memory-guide", order: 150, text: MEMORY_GUIDE_SECTION })
293
+ : null;
294
+
253
295
  const disposers = [
296
+ ...(guideSection ? [guideSection] : []),
254
297
  ctx.systemPrompt.context({
255
298
  name: "memory",
256
299
  order: 90,
@@ -270,16 +313,48 @@ export function createInjector(ctx, service, settings, config) {
270
313
  const rotate = recentInjectedIds(sessionId, query);
271
314
  // rotateWindow 随集合一起传入:候选池按 maxItems×(N+1) 扩容——窗口要的
272
315
  // 牌比既有 maxItems×2 池多时,轮换才有新鲜牌可换(#205 补测)。
273
- const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold, scope, rotate, rotateWindow: rotationTurns });
274
- recordInjection(sessionId, query, candidates);
316
+ // Issue #239(第 5 项):注入条数的查询自适应(默认关)。确定性强的话题收缩
317
+ // 条数、模糊话题维持上限——单向收缩,绝不越过 maxInjectedItems;判据只看查询
318
+ // 本身,不做额外检索(先探针检索等于白付一次 fuseRecall)。
319
+ const maxItems = config.injectUncertaintyAdaptive === true
320
+ ? adaptiveInjectBudget(query, baseMaxItems)
321
+ : baseMaxItems;
322
+ // #249 第一批:pin 池统计走可选出参,不动 injectCandidates 的数组契约。
323
+ const pinnedStats = {};
324
+ const candidates = service.injectCandidates({ query, queryVector, maxItems, threshold, scope, rotate, rotateWindow: rotationTurns, pinnedStats });
325
+ // #249:pin 条目不进轮换历史——它们每轮固定前置,记进去只会占满轮换
326
+ // 窗口、挤掉情景候选的新鲜度(验收:pin 不参与跨轮轮换)。
327
+ recordInjection(sessionId, query, pinnedStats.shown > 0 ? candidates.slice(pinnedStats.shown) : candidates);
275
328
  // Hot memory (v0.5.0 1.3) leads the single memory block: the agent
276
329
  // sees the short-term rounds first, then the cross-session recall —
277
330
  // the documented injection order 1→2. Folding it here (instead of a
278
331
  // separate context) keeps the prompt assembly stable at two blocks.
279
332
  const hotText = renderHotContext(ctx);
280
- const body = render(candidates);
281
- if (!hotText) return body;
282
- return body ? `${hotText}\n\n${body}` : hotText;
333
+ const body = render(candidates, pinnedStats);
334
+ const finalBody = !hotText ? body : body ? `${hotText}\n\n${body}` : hotText;
335
+ // Issue #179:旁路缓存一帧——面板「注入预览」卡据此展示构成与体积。
336
+ // chars 为条目内容的截断后近似值(与 render 同一函数计长),totalChars
337
+ // 是本次实际返回块的精确长度。maxItems 反映自适应收缩后的生效值。
338
+ injectionSnapshot = {
339
+ at: Date.now(),
340
+ sessionId,
341
+ query: query || "",
342
+ maxItems,
343
+ threshold,
344
+ adaptive: config.injectUncertaintyAdaptive === true,
345
+ scoped: scope && (scope.agent_scope || scope.workspace_scope) ? scope : null,
346
+ rotated: rotate ? rotate.size : 0,
347
+ hotChars: hotText.length,
348
+ entries: candidates.map((m) => ({
349
+ id: m.id,
350
+ type: m.type,
351
+ title: m.title,
352
+ importance: m.importance,
353
+ chars: injectMemory(m).length
354
+ })),
355
+ totalChars: finalBody.length
356
+ };
357
+ return finalBody;
283
358
  }
284
359
  }),
285
360
  ctx.systemPrompt.context({
@@ -292,6 +367,7 @@ export function createInjector(ctx, service, settings, config) {
292
367
  return () => {
293
368
  queryVectorCache.clear();
294
369
  rotationHistory.clear();
370
+ injectionSnapshot = null; // 注入器卸载即失效:快照不得跨生命周期存留
295
371
  for (const dispose of disposers) {
296
372
  if (typeof dispose === "function") dispose();
297
373
  }
package/lib/lang.js CHANGED
@@ -75,6 +75,12 @@ export const STR = {
75
75
  zh: (max, len, id) => `〔已截断:上限 ${max},原 ${len} 字符;全文用 memory_get "${id}" 查看〕`,
76
76
  en: (max, len, id) => `[truncated: limit ${max}, original ${len} chars; memory_get "${id}" for full text]`
77
77
  },
78
+ // Issue #249(第一批):pin 池超预算时的如实标注——分层不等于丢弃,也不允许
79
+ // 静默省略未展示条数(agent 才知道库里的约束不止这几条,需按需 memory_search)。
80
+ pinnedOverflow: {
81
+ zh: (n) => `〔约束/偏好类另有 ${n} 条未展示〕`,
82
+ en: (n) => `[${n} more constraint/preference entries not shown]`
83
+ },
78
84
  userSettingsHeader: {
79
85
  zh: "[用户设置] 来自 dsh-mneme 的用户画像与规则:",
80
86
  en: "[User settings] Profile and rules from dsh-mneme:"
@@ -21,6 +21,32 @@ function modelHash(model) {
21
21
  return `${model}#${hashString(model)}`;
22
22
  }
23
23
 
24
+ /**
25
+ * Model families whose embedding is the CLS token rather than a mean over tokens.
26
+ *
27
+ * BGE (BAAI/bge-*) is trained that way: the model's own `1_Pooling/config.json`
28
+ * declares `pooling_mode_cls_token: true` / `pooling_mode_mean_tokens: false`, and
29
+ * the official README says "select the last hidden state of the first token" plus
30
+ * L2 normalization. transformers.js's `feature-extraction` defaults to `mean`, so
31
+ * feeding it a BGE model silently produces slightly-off vectors — no error, just a
32
+ * worse ranking. This table is the extension point: a family goes in only once its
33
+ * pooling is confirmed against the model's own `1_Pooling` (or equivalent) config.
34
+ */
35
+ const CLS_POOLED_MODEL_PATTERNS = [/bge/i];
36
+
37
+ /**
38
+ * Resolve the pooling to embed with.
39
+ *
40
+ * `auto` (default) picks by model family — BGE → `cls`, everything else → `mean`,
41
+ * so models that were never affected keep their exact previous behavior. An
42
+ * explicit `cls` / `mean` wins over the family table.
43
+ */
44
+ export function resolvePooling(pooling, model) {
45
+ const mode = String(pooling ?? "auto").trim().toLowerCase();
46
+ if (mode === "cls" || mode === "mean") return mode;
47
+ return CLS_POOLED_MODEL_PATTERNS.some((re) => re.test(String(model ?? ""))) ? "cls" : "mean";
48
+ }
49
+
24
50
  /**
25
51
  * Lazy default loader: dynamic import keeps module load cheap.
26
52
  *
@@ -71,8 +97,9 @@ function tensorToRows(tensor) {
71
97
 
72
98
  /**
73
99
  * ONNX text embedder backed by transformers.js (onnxruntime-node underneath).
74
- * Runs fully offline with mean pooling + L2 normalization for BERT-style
75
- * models like bge-small-zh. `engineFactory` is injectable for tests.
100
+ * Runs fully offline; pooling is resolved per model family (BGE → CLS, else mean —
101
+ * see resolvePooling) and vectors are L2-normalized. `engineFactory` is injectable
102
+ * for tests.
76
103
  */
77
104
  export class LocalEmbedder {
78
105
  constructor(opts = {}) {
@@ -87,6 +114,10 @@ export class LocalEmbedder {
87
114
  // (含测试)保持裸 fetch 的现状,配置默认值由 index.js 传进来。
88
115
  this.resilientModelDownload = opts.resilientModelDownload === true;
89
116
  this.useDtype = opts.useDtype || "q8";
117
+ // 池化:配置值(auto/cls/mean)→ 生效值(cls/mean)。解析结果既进 embed(),
118
+ // 也进 modelHash —— 换池化就是换向量空间,旧索引必须被判失配后重建。
119
+ this.poolingMode = String(opts.pooling ?? "auto").trim().toLowerCase();
120
+ this.pooling = resolvePooling(this.poolingMode, this.model);
90
121
  // 模型镜像下载源(embedModelMirror,#188 补充):空 = transformers 默认。
91
122
  this.remoteHost = String(opts.remoteHost ?? "").trim();
92
123
  this.logger = opts.logger ?? null;
@@ -114,19 +145,19 @@ export class LocalEmbedder {
114
145
  this.extractor = await this.engineFactory("feature-extraction", this.model, options);
115
146
  this.ready = true; // service reads this to flush queued re-embeds
116
147
  this.logger?.info?.(
117
- `[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device})`
148
+ `[dsh-mneme] local embedder ready: ${this.model} (dim=${this._dimension}, device=${this.device}, pooling=${this.pooling})`
118
149
  );
119
150
  return this;
120
151
  }
121
152
 
122
- /** Embed many texts with mean pooling; chunks at batchSize. */
153
+ /** Embed many texts with the model's pooling; chunks at batchSize. */
123
154
  async embed(texts) {
124
155
  if (!Array.isArray(texts)) throw new TypeError("embed expects an array of strings");
125
156
  if (!this.extractor) throw new Error("LocalEmbedder not initialized");
126
157
  const out = [];
127
158
  for (let i = 0; i < texts.length; i += this.batchSize) {
128
159
  const chunk = texts.slice(i, i + this.batchSize);
129
- const tensor = await this.extractor(chunk, { pooling: "mean", normalize: true });
160
+ const tensor = await this.extractor(chunk, { pooling: this.pooling, normalize: true });
130
161
  out.push(...tensorToRows(tensor));
131
162
  }
132
163
  return out;
@@ -142,7 +173,14 @@ export class LocalEmbedder {
142
173
  }
143
174
 
144
175
  get modelHash() {
145
- return modelHash(this.model);
176
+ // Pooling is part of the vector space: a CLS index cannot be queried with mean
177
+ // vectors (or vice versa). Keep the historical `model#hash` shape for `mean` so
178
+ // indexes that were never affected are not invalidated needlessly; every other
179
+ // pooling gets a fingerprint of its own, which makes the index-consistency gate
180
+ // flag the stale mean-space index and rebuild it.
181
+ return this.pooling === "mean"
182
+ ? modelHash(this.model)
183
+ : modelHash(`${this.model}@${this.pooling}`);
146
184
  }
147
185
 
148
186
  dispose() {