@modusensus/dsh-mneme 0.9.4 → 0.9.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +53 -10
  2. package/bin/cli.mjs +215 -2
  3. package/docs/CONFIGURATION.md +4 -2
  4. package/docs/DAEMON.md +9 -2
  5. package/docs/DISTILL.md +66 -4
  6. package/docs/SHARE.md +552 -0
  7. package/docs/SLEEP.md +1 -1
  8. package/docs/STORAGE.md +1 -1
  9. package/lib/api-standalone.js +286 -8
  10. package/lib/api.js +195 -1
  11. package/lib/client.js +783 -30
  12. package/lib/document.js +15 -2
  13. package/lib/dream/decisions.js +88 -3
  14. package/lib/dream/sleep.js +63 -13
  15. package/lib/dream.js +65 -9
  16. package/lib/lang.js +23 -6
  17. package/lib/memory-source.js +9 -0
  18. package/lib/organize.js +5 -1
  19. package/lib/pipeline.js +62 -1
  20. package/lib/provenance-http.js +126 -0
  21. package/lib/serve.js +13 -0
  22. package/lib/service.js +67 -4
  23. package/lib/share-http.js +281 -0
  24. package/lib/share-import.js +234 -0
  25. package/lib/share.js +335 -0
  26. package/lib/store.js +272 -11
  27. package/lib/summarize.js +60 -11
  28. package/package.json +1 -1
  29. package/src/api-standalone.js +286 -8
  30. package/src/api.js +195 -1
  31. package/src/document.js +15 -2
  32. package/src/dream/decisions.js +88 -3
  33. package/src/dream/sleep.js +63 -13
  34. package/src/dream.js +65 -9
  35. package/src/lang.js +23 -6
  36. package/src/memory-source.js +9 -0
  37. package/src/organize.js +5 -1
  38. package/src/pipeline.js +62 -1
  39. package/src/provenance-http.js +126 -0
  40. package/src/serve.js +13 -0
  41. package/src/service.js +67 -4
  42. package/src/share-http.js +281 -0
  43. package/src/share-import.js +234 -0
  44. package/src/share.js +335 -0
  45. package/src/store.js +272 -11
  46. package/src/summarize.js +60 -11
  47. package/test/api.test.js +370 -2
  48. package/test/audit.test.js +2 -2
  49. package/test/cli-receive.test.js +390 -0
  50. package/test/cli-share.test.js +437 -0
  51. package/test/client-share.test.js +225 -0
  52. package/test/client.test.js +139 -0
  53. package/test/dedupe-candidate-window.test.js +180 -0
  54. package/test/dream-candidate-hybrid.test.js +2 -2
  55. package/test/dream-fence.test.js +164 -0
  56. package/test/dream.test.js +87 -5
  57. package/test/helpers/dream-mock.js +6 -2
  58. package/test/llm-message-source.test.js +1 -1
  59. package/test/mint-source.test.js +1 -1
  60. package/test/pipeline-r1r2.test.js +129 -0
  61. package/test/policy-epoch.test.js +2 -2
  62. package/test/provenance-http.test.js +184 -0
  63. package/test/readme-claims.test.js +45 -0
  64. package/test/reasoning-effort.test.js +10 -10
  65. package/test/scope-candidates.test.js +7 -4
  66. package/test/scope-state.test.js +130 -1
  67. package/test/serve.test.js +62 -0
  68. package/test/share-foreign-rows.test.js +144 -0
  69. package/test/share-http.test.js +144 -0
  70. package/test/share-import.test.js +577 -0
  71. package/test/share-imported-guard.test.js +311 -0
  72. package/test/share-imported-identity.test.js +111 -0
  73. package/test/share-pack.test.js +511 -0
  74. package/test/share-receive-http.test.js +756 -0
  75. package/test/share-routes.test.js +337 -0
  76. package/test/share-store.test.js +133 -0
  77. package/test/share-validate.test.js +101 -0
  78. package/test/standalone-api.test.js +524 -5
  79. package/test/summarize-parse-diagnostics.test.js +75 -0
  80. package/test/summarize.test.js +15 -2
package/src/document.js CHANGED
@@ -16,6 +16,7 @@ import { homedir } from "node:os";
16
16
  import { isAbsolute, resolve, sep } from "node:path";
17
17
  import { scopeKeyOf } from "./scope.js";
18
18
  import { cosineSimilarity } from "./dream/clustering.js";
19
+ import { isImportedRow } from "./memory-source.js";
19
20
 
20
21
  // C2 vector 档阈值(#164 拍板 0.92,与 findSessionDuplicate 的 vector 档同源)。
21
22
  // 刻意不做 24h 时间窗:那个窗是会话去重的「近邻界」(findSessionDuplicate 注释),
@@ -82,7 +83,7 @@ const ABSORB_EXEMPT_TYPES = new Set(["document", "summary"]);
82
83
  *
83
84
  * @returns {(payload: object, opts?: object) => Promise<object>}
84
85
  * `{ action: "created"|"superseded", memory, superseded?, evidence_kept,
85
- * evidence_dropped, evidence_archived, degraded }`
86
+ * evidence_dropped, evidence_archived, evidence_skipped, degraded }`
86
87
  * @throws flag 关 / 路径非法或文件缺失 / title 或 summary 为空 / evidence 全部
87
88
  * 捏造 / 仅 vector 近重复(疑似重复外档,交 agent 裁决而不是静默二选一)。
88
89
  */
@@ -262,6 +263,10 @@ export function createDocumentRegistrar({ store, config, embedQuery, pushContent
262
263
  let superseded;
263
264
  const loser = explicitTarget ? store.getById(explicitTarget.id) : null;
264
265
  if (loser && !loser.archived) {
266
+ // 这里不挂 isImportedRow:loser 只会是 document 行(scopeMatched 全来自
267
+ // store.list({ type: "document" })),而 document 行永远不可能是外来行——
268
+ // 分享包格式排除 document,存储层的 document 只由 saveDocument 从这里铸造。
269
+ // 加谓词是空转,还会让下一个读者误以为 document 行有外来形态。
265
270
  store.update(loser.id, {
266
271
  content: `${loser.content}${supersededByPointer(created.id)}`,
267
272
  content_history: pushContentHistory(loser, "superseded")
@@ -276,18 +281,25 @@ export function createDocumentRegistrar({ store, config, embedQuery, pushContent
276
281
  // ABSORB_EXEMPT_TYPES)不吸收,吸收的是原子条;④只翻标志位、内容与审计全留
277
282
  // (可恢复),opt-out 走 archiveEvidence。
278
283
  // pinnedTypes 拿不到就不做这一步:宁可少做,也不能把保真池当普通行收走。
284
+ // 外来行不吸收(spec §15 一,车道 C8):这条旁路不经任何 service.all() 行池,
285
+ // 车道 B 的九池静态锁看不见它;证据 id 由调用方给,循环里的 setArchived 是
286
+ // 机器发起的批量归档——正是「本库模型不改写别人的记忆」要禁的动作类。
287
+ // 引用本身照旧落库(文档合法地为它背书),只是不替别人撤下活跃面,
288
+ // 跳过条数走 evidence_skipped 报出来,不做静默跳过。
279
289
  let evidenceArchived = 0;
290
+ let evidenceSkipped = 0;
280
291
  if (archiveEvidence && pinnedTypes && typeof pinnedTypes.has === "function") {
281
292
  for (const id of kept) {
282
293
  // 事务内重读:期间被别的进程归档/遗忘的行不重复计数,也不误伤 pinned。
283
294
  const row = store.getById(id);
284
295
  if (!row || row.archived || row.forgotten) continue;
296
+ if (isImportedRow(row)) { evidenceSkipped += 1; continue; }
285
297
  if (pinnedTypes.has(row.type) || ABSORB_EXEMPT_TYPES.has(row.type)) continue;
286
298
  store.setArchived(id, true);
287
299
  evidenceArchived += 1;
288
300
  }
289
301
  }
290
- return { created, superseded, evidenceArchived };
302
+ return { created, superseded, evidenceArchived, evidenceSkipped };
291
303
  });
292
304
  finalize(result.superseded ? [result.created, result.superseded] : [result.created]);
293
305
  return {
@@ -297,6 +309,7 @@ export function createDocumentRegistrar({ store, config, embedQuery, pushContent
297
309
  evidence_kept: kept.length,
298
310
  evidence_dropped: dropped.length,
299
311
  evidence_archived: result.evidenceArchived,
312
+ evidence_skipped: result.evidenceSkipped,
300
313
  degraded: dropped.length > 0
301
314
  };
302
315
  };
@@ -19,6 +19,78 @@ const ARCHIVE_GUARDED_TYPES = new Set(["preference", "pattern", "rejected_soluti
19
19
  // 「重复 / 过时」类理由的关键词(中英双语)。命中才允许归档长保留类型;
20
20
  // 词表宁可放宽也不误拦——护栏只跳过单条并落明细,不整单拒绝。
21
21
  const ARCHIVE_RATIONALE_RE = /重复|过时|陈旧|过期|已被取代|被取代|冗余|不再使用|旧版|废弃|stale|outdated|obsolete|dup(?:licate|licat)?|superseded|redundant|deprecated|expired|replaced/i;
22
+ // Issue #522 评审:上面那张词表只认关键词,不认否定——「并不重复,只是主题相近」
23
+ // 与 "not a duplicate" 都会命中「重复」/"duplicate" 子串。archive 护栏从 #104 起
24
+ // 就带着这个洞(既有行为,改它会扩大范围,先记在这里),但 #521 给 merge 加的
25
+ // 逃生门是本 PR 新开的口子,必须自己堵上:否定表述一票否决。第二轮评审又指出
26
+ // 词表里「过时 / stale / outdated」同样是钥匙,只挡「重复」等于留了半边门——
27
+ // 「并不过时,只是主题相近」照样能换到合并许可,所以否定词按整张词表铺开。
28
+ const NEGATED_RATIONALE_RE = /(?:不|并不|不是|非)\s*(?:重复|过时|陈旧|过期|已被取代|被取代|冗余|旧版|废弃)|\b(?:not|no|isn't|is not|never)\s+(?:a\s+|an\s+)?(?:duplicat\w*|stale|outdated|obsolete|superseded|redundant|deprecated|expired|replaced)/i;
29
+
30
+ // Issue #521:把 skipped[].error 归一化成可分组的原因。原样字符串不能直接
31
+ // GROUP BY——前缀带 decision[N]: 序号,同一种原因会被拆成 N 组(#521 实测 280
32
+ // 条被拆成 20 组),面板上看不出「一整类动作 100% 失败」。这里只剥三样东西:
33
+ // 序号前缀、模型可控的引号文本、具体类型集合,其余措辞保持原样,好让人一眼
34
+ // 认出是哪个判据。
35
+ export function normalizeSkipReason(error) {
36
+ // 一条决策可能攒下多条错误,skipped[].error 用 "; " 把它们拼成一句(见下方
37
+ // skipped.push)。所以先按 decision[N]: 的边界切开、逐段归一化再拼回去——
38
+ // 原先的正则 ^ 锚定,只剥得掉第一个前缀,后面的 decision[1]: 会留在字符串
39
+ // 中间,聚合时又被拆成多组(#522 评审指出)。
40
+ // 抹引号必须排在切分之前:模型写的理由原文里可能正好含 "; decision[0]: "
41
+ // (引文、邮箱都算),先切分就会把这段自由文本切成独立的一段,而那一段不带
42
+ // 任何可识别的边界,后面对它无从下手。先抹掉引号里的内容,切分就只看得到
43
+ // 我们自己拼进去的分隔符。
44
+ return maskModelText(String(error ?? ""))
45
+ .split(/;\s*(?=decision\[\d+\]:\s*)/)
46
+ .map(normalizeSkipSegment)
47
+ .join("; ")
48
+ .trim();
49
+ }
50
+
51
+ // 引号内的内容一律是模型可控的:id、action 名、理由原文都由 JSON.stringify
52
+ // 加引号,所以「引号内 = 不可信」是这里唯一的判据。成对的那段整段抹成占位符;
53
+ // 万一剩下一个孤立引号(旧版把理由截到 60 字再拼进 error,可能正好断在字符串
54
+ // 中间),从那里砍到结尾——宁可少一句尾巴,也不让记忆正文随 open 路由出网关。
55
+ // 刻意不按「, got: 」这类分隔符猜边界(#522 三轮评审):分隔符本身可能出现在
56
+ // id 里(unknown id "a@b.c, got: x"),猜错就会把引号截断,脱敏反而失效。
57
+ function maskModelText(text) {
58
+ return String(text ?? "")
59
+ .replace(/"(?:[^"\\]|\\.)*"/g, "\u0000")
60
+ .replace(/"(?:[\s\S])*$/, "\u0000")
61
+ .replace(/\u0000/g, '"…"');
62
+ }
63
+
64
+ // 单段归一化:剥序号前缀、丢掉已经抹空的自由文本尾巴、把具体类型集合抹成占位符。
65
+ function normalizeSkipSegment(segment) {
66
+ return String(segment ?? "")
67
+ .replace(/^decision\[\d+\]:\s*/, "")
68
+ // 抹完之后「, got: "…"」是个固定字面量(maskModelText 的产物),可以整段
69
+ // 砍掉:它只说明「校验器当时回了一句理由」,内容已经不在字符串里了。
70
+ .replace(/,\s*got:\s*"…"$/, "")
71
+ // 类型集合同理,但单个类型名保留——「被拦的是哪一类」本身是有效信息
72
+ .replace(/\(([a-z_]+(?:,\s*[a-z_]+)+)\)/g, "(…)")
73
+ .trim();
74
+ }
75
+
76
+ // Issue #521:按归一化原因聚合 skipped,供面板直接显示。按 (action, reason)
77
+ // 二元组计数——同一句原因在 merge 和 archive 上是两回事,混在一起会误导。
78
+ export function summarizeSkipped(skipped) {
79
+ const buckets = new Map();
80
+ for (const item of Array.isArray(skipped) ? skipped : []) {
81
+ const reason = normalizeSkipReason(item?.error);
82
+ if (!reason) continue;
83
+ // #522 评审:action 也走白名单。这条聚合结果会随 open 路由出网关,放任意字符串
84
+ // 过去等于让上游的脏数据(或将来某个新字段)自己决定响应形状;不在枚举里的一律
85
+ // 归成空串,面板少显示一个动作标签,比显示一段来路不明的文本安全。
86
+ const action = ACTIONS.has(item?.action) ? item.action : "";
87
+ const key = action + "\u0000" + reason;
88
+ const bucket = buckets.get(key);
89
+ if (bucket) bucket.count += 1;
90
+ else buckets.set(key, { action, reason, count: 1 });
91
+ }
92
+ return [...buckets.values()].sort((a, b) => b.count - a.count || a.reason.localeCompare(b.reason));
93
+ }
22
94
 
23
95
  /**
24
96
  * Validate a dream decision list against a snapshot of eligible memories.
@@ -149,7 +221,7 @@ export function validateDecisions(decisions, snapshot, options = {}) {
149
221
  if (mem && ARCHIVE_GUARDED_TYPES.has(mem.type)) guardedTypes.add(mem.type);
150
222
  }
151
223
  if (guardedTypes.size > 0 && !ARCHIVE_RATIONALE_RE.test(String(d.reason ?? ""))) {
152
- local.push(`${at}: archive of long-retention type(s) ${[...guardedTypes].join(", ")} needs a duplicate/outdated rationale (重复/过时), got: ${JSON.stringify(String(d.reason ?? "")).slice(0, 60)}`);
224
+ local.push(`${at}: archive of long-retention type(s) ${[...guardedTypes].join(", ")} needs a duplicate/outdated rationale (重复/过时), got: ${JSON.stringify(String(d.reason ?? "").slice(0, 60))}`);
153
225
  }
154
226
  }
155
227
  if (local.length === 0 && d?.action === "update") {
@@ -222,8 +294,21 @@ export function validateDecisions(decisions, snapshot, options = {}) {
222
294
  const mem = snapshot.get(id);
223
295
  if (mem && ARCHIVE_GUARDED_TYPES.has(mem.type)) guardedTypes.add(mem.type);
224
296
  }
225
- if (guardedTypes.size > 0) {
226
- local.push(`${at}: merge of long-retention type(s) ${[...guardedTypes].join(", ")} is blocked by dreamMergeGuard`);
297
+ // Issue #521:与上方 archive 护栏对齐——archive 早就允许「带重复/过时
298
+ // 理由」的长保留类型通过,merge 却一律拦死,于是一份真重复的
299
+ // preference 永远合不掉,而提示词还在催模型合并它们。实测(#521)46 轮
300
+ // 巩固里 280 条 merge 决策全被这条拦下、merge 恒为 0,模型每轮把同一
301
+ // 簇条目再提一次,满载 token 空转。逃生门条件与 archive 护栏同一条词表
302
+ // 且要求同类型(跨类型合并本就另受 allowCrossTypeMerge 约束,逃生门
303
+ // 不跟着放宽)。reason 只写「主题相近」不会命中词表——这是故意的:
304
+ // 相近不等于重复,只有确属重复/过时才值得牺牲两条独立表述。
305
+ const sameType = new Set(d.ids.map((id) => snapshot.get(id)?.type)).size === 1;
306
+ const reasonText = String(d.reason ?? "");
307
+ // #522 评审:词表命中还不够——「并不重复」「not a duplicate」也含关键词,
308
+ // 放它们过闸等于让模型用一句否认换来合并许可。否定表述直接判不重复。
309
+ const duplicated = !NEGATED_RATIONALE_RE.test(reasonText) && ARCHIVE_RATIONALE_RE.test(reasonText);
310
+ if (guardedTypes.size > 0 && !(sameType && duplicated)) {
311
+ local.push(`${at}: merge of long-retention type(s) ${[...guardedTypes].join(", ")} is blocked by dreamMergeGuard (needs a duplicate/outdated rationale, 重复/过时)`);
227
312
  }
228
313
  }
229
314
  }
@@ -21,8 +21,12 @@ import { randomUUID, createHash } from "node:crypto";
21
21
  import { validateDecisions, applyDecisions, ACTIONS } from "./decisions.js";
22
22
  import { findPotentialConflicts, cosineSimilarity } from "./clustering.js";
23
23
  import { scopeKeyOf } from "../scope.js";
24
- import { buildReceipt, describeStreamFailure, resolveDreamEffort, withEffortFallback, withMaxTokensFallback, maintainIndexAfterDream } from "../dream.js";
24
+ import { buildReceipt, describeStreamFailure, resolveDreamEffort, withEffortFallback, withMaxTokensFallback, maintainIndexAfterDream, fenceListBody, parseFailureMeta } from "../dream.js";
25
25
  import { computeHeat } from "../heat.js";
26
+ // 四个相位读的都是行池,而动行的写手不经汇流口:phaseDemotion 直接 setArchived /
27
+ // demoteToSummary,phasePatterns 还会 mint 出新行,所以这条契约要在每处各挡一遍
28
+ // (清单与静态锁见 test/share-imported-guard.test.js)。
29
+ import { isImportedRow } from "../memory-source.js";
26
30
 
27
31
  // Issue #126 review(Copilot):默认档(sleepActionSet="conflict")允许的动作 = 全局
28
32
  // ACTIONS 去掉 #126 新增的两个。从 ACTIONS 派生而非硬编码,将来再扩动作集时自动跟随。
@@ -102,8 +106,11 @@ async function auditedSleepLlm(ctx, service, config, logger, spec, body) {
102
106
  let outputTokens = 0;
103
107
  let status = "success";
104
108
  let errorMessage = null;
109
+ // 提到 try 之外:finally 里的 metadataFrom 要读这一轮的原始输出(dream.js 的
110
+ // runAuditedLlm 就是这么声明的)。留在 try 里,finally 看不见它,读数会静默变空。
111
+ let result;
105
112
  try {
106
- const result = await body((usage) => {
113
+ result = await body((usage) => {
107
114
  if (!usage) return;
108
115
  const i = usage.input_tokens ?? usage.inputTokens ?? usage.prompt_tokens ?? usage.promptTokens;
109
116
  const o = usage.output_tokens ?? usage.outputTokens ?? usage.completion_tokens ?? usage.completionTokens;
@@ -129,6 +136,17 @@ async function auditedSleepLlm(ctx, service, config, logger, spec, body) {
129
136
  errorMessage = String(error?.message ?? error);
130
137
  throw error;
131
138
  } finally {
139
+ // spec.metadataFrom(result):与 dream.js 的 runAuditedLlm 同一形状(这里是一份并行
140
+ // 实现,不是同一个函数——两份审计写法的字段必须一致,否则面板按 operation_type
141
+ // 筛出来的读数会时有时没有)。判据函数只读 result,不读闭包,理由同 dream.js。
142
+ let metadata;
143
+ if (result !== undefined && typeof spec.metadataFrom === "function") {
144
+ try {
145
+ metadata = spec.metadataFrom(result) ?? undefined;
146
+ } catch {
147
+ metadata = undefined;
148
+ }
149
+ }
132
150
  try {
133
151
  service.saveLlmAudit({
134
152
  timestamp,
@@ -142,7 +160,8 @@ async function auditedSleepLlm(ctx, service, config, logger, spec, body) {
142
160
  duration_ms: Date.now() - startedAt,
143
161
  status,
144
162
  error_message: errorMessage,
145
- related_memory_ids: spec.relatedMemoryIds ?? []
163
+ related_memory_ids: spec.relatedMemoryIds ?? [],
164
+ ...(metadata ? { metadata } : {})
146
165
  });
147
166
  } catch (auditError) {
148
167
  logger?.warn?.(`dsh-mneme sleep: llm audit write failed: ${String(auditError)}`);
@@ -205,7 +224,7 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
205
224
  }
206
225
  const strictness = config.sleepConflictStrictness ?? "normal";
207
226
  const threshold = CONFLICT_THRESHOLDS[strictness] ?? CONFLICT_THRESHOLDS.normal;
208
- const memories = withoutPaused(service, service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document"));
227
+ const memories = withoutPaused(service, service.all().filter((m) => !isImportedRow(m) && !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document"));
209
228
  if (memories.length < 2) return { status: "skipped", reason: "too few memories" };
210
229
  if (signal?.aborted) return { status: "aborted", reason: "user activity" };
211
230
 
@@ -342,7 +361,10 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
342
361
  // 解析结果存进闭包给下面的控制流用——审计关闭时 auditedSleepLlm 整段早退、
343
362
  // 这个钩子不会被调用,控制流会跟着失效(dream 的既有坑)。代价是审计开启时
344
363
  // 多解析一次,换来两条路径彻底解耦。
345
- auditError: (text) => (parseJsonArray(text) ? null : "no json array in llm output")
364
+ auditError: (text) => (parseJsonArray(text) ? null : "no json array in llm output"),
365
+ // 纯判据,与 auditError 同一条约定:重解析一次、不读闭包(审计关掉时那些闭包
366
+ // 根本不会被赋值,读它会让「审计说了什么」依赖调用方走到了哪一行)。
367
+ metadataFrom: (text) => (parseJsonArray(text) ? null : parseFailureMeta(text))
346
368
  }, (reportUsage) => streamText(ctx, {
347
369
  provider: route.provider,
348
370
  model: route.model,
@@ -351,7 +373,7 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
351
373
  ...(withEffort && sleepEffort ? { reasoningEffort: sleepEffort } : {}),
352
374
  messages: [
353
375
  { role: "system", content: [{ type: "text", text: conflictPrompt }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
354
- { role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
376
+ { role: "user", content: [{ type: "text", text: fenceListBody(language, listText) }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
355
377
  ]
356
378
  }, reportUsage, (reason) => { conflictStreamFailure = describeStreamFailure(reason); }));
357
379
  };
@@ -444,7 +466,10 @@ function phaseDemotion(service, config, logger, runId, signal = null) {
444
466
  // 政策轴在这里尤其要紧:下面两行会真改归档面(setArchived / demoteToSummary)。
445
467
  // 把过滤放在循环之外、而不是在循环体里 continue:一旦将来有人在循环体里加
446
468
  // 副作用(现已有),漏掉那条 continue 就是静默改到被按下的行。
447
- for (const m of withoutPaused(service, service.all())) {
469
+ // 外来行也在循环之外挡掉,同一个理由:它不经本方蒸馏,时间分层无权动它的原文或
470
+ // 归档面;而下面两行动作(setArchived / demoteToSummary)不经 saveWithDedupe,
471
+ // 汇流口那道闸门到这里已经管不到了。
472
+ for (const m of withoutPaused(service, service.all().filter((m) => !isImportedRow(m)))) {
448
473
  if (signal?.aborted) break;
449
474
  if (m.archived || m.forgotten) continue;
450
475
  const ref = m.last_accessed_at ?? m.created_at;
@@ -495,9 +520,31 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
495
520
  // 先生效的话,指针行一多就会把普通记忆挤出扫描窗,池子饿到门槛以下。
496
521
  // 政策轴同理必须排在 `.slice(0, limit)` **之前**,否则被按下的行照样占掉
497
522
  // sleepPatternMinMemories 的名额,而它们既不会被扫、也不该挤掉别人(#471)。
498
- const memories = withoutPaused(service, service
499
- .list({ limit: 200, includeForgotten: false, excludeTypes: ["document"] })
500
- .filter((m) => !m.archived && m.type !== "summary" && m.type !== "pattern"))
523
+ //
524
+ // archived 这一轴不在下面出现,不是漏了:list() 默认就带 `archived = 0`
525
+ // (store.js list 的 includeArchived 默认 false、onlyArchived 默认 false),
526
+ // 旧写法那句 `!m.archived` 一直是重复过滤——查询层已经把它挡在外面。
527
+ // 窗口至少等于本次要扫的名额:`sleepPatternMinMemories` 的合法上限是 1000
528
+ // (src/config.js:612)。窗口若还是死的 200,那么把这个值配到 200 以上的人
529
+ // **每一轮**都会触发下面那次无界重取,跟闸门按没按下无关——那是把这条修复
530
+ // 从「按需」变成「常付」。
531
+ const poolQuery = {
532
+ limit: Math.max(200, limit), includeForgotten: false,
533
+ // 三条政策轴全部下推到查询层:留在 LIMIT 之后过滤,等于让被丢掉的行为窗口占名额
534
+ // (模式扫描池会因此饿到 sleepPatternMinMemories 以下而静默跳过)。同一取舍见
535
+ // #230 的 document 轴与本文件 #471 的暂停轴,src/service.js:2247 那条注释记着前例。
536
+ excludeTypes: ["document", "summary", "pattern"],
537
+ excludeImported: true
538
+ };
539
+ const gather = (query) => withoutPaused(service, service.list(query));
540
+ let pool = gather(poolQuery);
541
+ // #515:暂停闸门要读 scope_state 并按「这一维有标注」匹配,下推进 SQL 就等于让判定
542
+ // 长出第二份实现(issue 里写过为什么否掉,`*` 那一行是关键的不可下推处)。所以这里
543
+ // 不动判定、只放宽取数:**只有窗口过滤后不够本次要扫的名额**才不带窗口重取一次。
544
+ // 常态仍只付一次有界查询;多付的那一次恰好就是原本会静默跳过的那一次——
545
+ // 症状从「库里没东西可扫」变成「扫了」,这才是这条闸门该有的行为。
546
+ if (pool.length < limit) pool = gather({ ...poolQuery, limit: null });
547
+ const memories = pool
501
548
  .sort((a, b) => (a.updated_at < b.updated_at ? 1 : -1))
502
549
  .slice(0, limit);
503
550
  if (memories.length === 0) return { status: "skipped", reason: "no memories to scan" };
@@ -518,7 +565,10 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
518
565
  relatedMemoryIds: memories.map((m) => m.id),
519
566
  streamError: () => patternStreamFailure,
520
567
  // 纯判据,同 conflict 阶段:不把解析结果存进闭包供控制流使用。
521
- auditError: (text) => (parseJsonArray(text) ? null : "no json array in llm output")
568
+ auditError: (text) => (parseJsonArray(text) ? null : "no json array in llm output"),
569
+ // 纯判据,与 auditError 同一条约定:重解析一次、不读闭包(审计关掉时那些闭包
570
+ // 根本不会被赋值,读它会让「审计说了什么」依赖调用方走到了哪一行)。
571
+ metadataFrom: (text) => (parseJsonArray(text) ? null : parseFailureMeta(text))
522
572
  }, (reportUsage) => streamText(ctx, {
523
573
  provider: route.provider,
524
574
  model: route.model,
@@ -527,7 +577,7 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
527
577
  ...(withEffort && sleepEffort ? { reasoningEffort: sleepEffort } : {}),
528
578
  messages: [
529
579
  { role: "system", content: [{ type: "text", text: STR.prompts.pattern[language].replace("N", String(maxPatterns)) }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
530
- { role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
580
+ { role: "user", content: [{ type: "text", text: fenceListBody(language, listText) }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
531
581
  ]
532
582
  }, reportUsage, (reason) => { patternStreamFailure = describeStreamFailure(reason); }));
533
583
  };
@@ -586,7 +636,7 @@ function phaseRelations(service, config, logger, runId, signal = null) {
586
636
  if (entities.length < 2) return { status: "skipped", reason: "too few entities" };
587
637
  const orphans = entities.filter((e) => (service.getRelations(e.id) ?? []).length === 0);
588
638
  if (orphans.length === 0) return { status: "skipped", reason: "no orphan entities" };
589
- const memories = withoutPaused(service, service.all().filter((m) => !m.archived && !m.forgotten));
639
+ const memories = withoutPaused(service, service.all().filter((m) => !isImportedRow(m) && !m.archived && !m.forgotten));
590
640
  const seen = new Set();
591
641
  const related = [];
592
642
  const MAX_RELATIONS_PER_ORPHAN = 3;
package/src/dream.js CHANGED
@@ -10,6 +10,10 @@ import { scopeKeyOf } from "./scope.js";
10
10
  import { isInPeakWindow, nextOffPeakAt } from "./peak-hours.js";
11
11
  import { createHash, randomUUID } from "node:crypto";
12
12
  import { STR, langOf } from "./lang.js";
13
+ // 巩固读的是行池,而动行的写手不经汇流口:applyDecisions 归档败者直接走
14
+ // service.setArchived(dream/decisions.js),所以「不许改写外来行」这条契约要在
15
+ // 每个池子各挡一遍。清单与静态锁见 share-imported-guard.test.js。
16
+ import { isImportedRow } from "./memory-source.js";
13
17
  export { validateDecisions, applyDecisions, withEffortFallback, withMaxTokensFallback, parseMaxTokensCap, describeStreamFailure, resolveDreamEffort, resolveRoute, streamText, runAuditedLlm };
14
18
 
15
19
  // Extract the first JSON array from LLM output, tolerating markdown fences,
@@ -57,7 +61,7 @@ function totalChars(memories) {
57
61
  // 两处若各写一份过滤条件,重启播种读到的基线就会和运行期判定口径漂移(一个把
58
62
  // document 算进去、一个不算),阈值随之失真,且这种漂移只会在重启后才显形。
59
63
  function activeStoreSize(service) {
60
- const memories = withoutPaused(service, service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document"));
64
+ const memories = withoutPaused(service, service.all().filter((m) => !isImportedRow(m) && !m.archived && m.type !== "summary" && m.type !== "document"));
61
65
  return { count: memories.length, chars: totalChars(memories) };
62
66
  }
63
67
 
@@ -309,6 +313,45 @@ function describeStreamFailure(reason) {
309
313
  * operation_type, model_id, related_memory_ids); `body(reportUsage)` performs
310
314
  * the actual stream consumption and is handed a usage reporter for the chunks.
311
315
  */
316
+ /**
317
+ * 把喂给模型的清单包进围栏(#493 的另一半:dream / sleep 那 5 处 user 消息以前是裸放)。
318
+ * 三个细节各自有原因,别当装饰删:
319
+ * - 角色声明落在**开头**:判断「这段文本是什么」的是它所在的那条 user 消息,只写在 system
320
+ * 里实测救不回来(#493 的 A/C 臂:只加 system 约束仍 0/2 出 JSON);
321
+ * - 正文里的 `<<<` / `>>>` 用零宽空格断链——**只中和标记,不删内容**。不这么做时,一条写着
322
+ * 「<<<记忆清单结束>>>」的记忆就能提前闭合围栏,把后面的文字升格成指令;而记忆正文本来就
323
+ * 是模型的可影响输入,这是最省事的注入入口;
324
+ * - 纯函数、不带 nonce:同一批记忆在下一次巩固阈值到来时会被原样再喂一次,重试收到的 prompt
325
+ * 必须与首次逐字节相同,否则「同输入同错」这条诊断依据就没了(#493 的重试 20 次现场就靠它)。
326
+ * @param {"json"|"text"} [shape] 该次调用的**输出契约**:巩固/叙述/冲突/模式要 JSON 数组,
327
+ * 总览摘要要散文。角色声明里那句「只输出 JSON 数组」必须跟着契约走——对散文契约说它,
328
+ * 模型会回 `[]`,而 `summaryText.trim()` 判非空通过,`[]` 就被当成摘要存进库(评审实测)。
329
+ */
330
+ export function fenceListBody(language, text, shape = "json") {
331
+ const open = STR.memoryFenceOpen[language] ?? STR.memoryFenceOpen.en;
332
+ const close = STR.memoryFenceClose[language] ?? STR.memoryFenceClose.en;
333
+ const table = shape === "text" ? STR.dreamTextRoleReminder : STR.dreamRoleReminder;
334
+ const reminder = table[language] ?? table.en;
335
+ const body = String(text ?? "").replace(/<<<|>>>/g, (run) => `${run[0]}\u200B${run.slice(1)}`);
336
+ return `${reminder}\n\n${open}\n${body}\n${close}`;
337
+ }
338
+
339
+ /**
340
+ * 解析失败行的审计读数(#493 同款,报告者提的第 2 条)。dream 的 warn 日志本来就带
341
+ * raw length + 前 300 字,缺的是**可查询**的那一面:日志要翻、按不了、也进不了面板的
342
+ * 「LLM 消耗」。`raw_has_bracket` 是区分「模型压根没按格式写」与「格式对但内容救不回来」
343
+ * 的唯一低成本判据。
344
+ */
345
+ export function parseFailureMeta(text) {
346
+ const raw = typeof text === "string" ? text : "";
347
+ return {
348
+ parse_failure: true,
349
+ raw_len: raw.length,
350
+ raw_head: raw.slice(0, 300).replace(/\s+/g, " ").trim(),
351
+ raw_has_bracket: raw.includes("[")
352
+ };
353
+ }
354
+
312
355
  async function runAuditedLlm(ctx, service, config, spec, body) {
313
356
  const audit = config?.llmAudit;
314
357
  if (audit?.enabled === false || typeof service?.saveLlmAudit !== "function") return body(() => {});
@@ -351,6 +394,17 @@ async function runAuditedLlm(ctx, service, config, spec, body) {
351
394
  errorMessage = String(error?.message ?? error);
352
395
  throw error;
353
396
  } finally {
397
+ // spec.metadataFrom(result):调用方按**这一轮的原始输出**决定要不要带读数(解析成功就
398
+ // 返回 null → 整键不写)。判据函数必须是纯的:它读的是 result,不读控制流里的缓存,
399
+ // 否则「审计说了什么」会随调用方走到哪一行而变(sleep.js 同一条硬约定)。
400
+ let metadata;
401
+ if (result !== undefined && typeof spec.metadataFrom === "function") {
402
+ try {
403
+ metadata = spec.metadataFrom(result) ?? undefined;
404
+ } catch {
405
+ metadata = undefined; // 读数失败不许把审计行整个弄没
406
+ }
407
+ }
354
408
  try {
355
409
  service.saveLlmAudit({
356
410
  timestamp,
@@ -364,7 +418,8 @@ async function runAuditedLlm(ctx, service, config, spec, body) {
364
418
  duration_ms: Date.now() - startedAt,
365
419
  status,
366
420
  error_message: errorMessage,
367
- related_memory_ids: spec.relatedMemoryIds ?? []
421
+ related_memory_ids: spec.relatedMemoryIds ?? [],
422
+ ...(metadata ? { metadata } : {})
368
423
  });
369
424
  } catch (auditError) {
370
425
  ctx.logger?.warn?.(`dsh-mneme: llm audit write failed: ${String(auditError)}`);
@@ -486,7 +541,7 @@ async function reEmbedMemory(semantic, memory, logger) {
486
541
  * 任何失败降级为 0 条,绝不反噬 dream 主流程。
487
542
  */
488
543
  async function generateNarratives({ ctx, service, config, route, language, effort, semantic, logger }) {
489
- const inputs = withoutPaused(service, service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document"));
544
+ const inputs = withoutPaused(service, service.all().filter((m) => !isImportedRow(m) && !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document"));
490
545
  const clusters = clusterByTag(inputs, { minCluster: config.dreamNarrativeMinCluster ?? 3 });
491
546
  if (!clusters.length) return 0;
492
547
 
@@ -511,7 +566,7 @@ async function generateNarratives({ ctx, service, config, route, language, effor
511
566
  ...(withEffort && effort ? { reasoningEffort: effort } : {}),
512
567
  messages: [
513
568
  { role: "system", content: [{ type: "text", text: STR.prompts.narrative[language] }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
514
- { role: "user", content: [{ type: "text", text: listing }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
569
+ { role: "user", content: [{ type: "text", text: fenceListBody(language, listing) }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
515
570
  ]
516
571
  }, reportUsage, (reason) => { streamFailure = describeStreamFailure(reason); }));
517
572
  };
@@ -1014,7 +1069,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1014
1069
  async function runDream(ctx, service, config) {
1015
1070
  const language = langOf(config);
1016
1071
  const logger = ctx.logger;
1017
- let memories = withoutPaused(service, service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document"));
1072
+ let memories = withoutPaused(service, service.all().filter((m) => !isImportedRow(m) && !m.archived && m.type !== "summary" && m.type !== "document"));
1018
1073
  if (memories.length === 0) return { ok: true, applied: 0, skipped: true, summary: false };
1019
1074
  // Issue #89:开跑时刻既喂给审计行(created_at),也是调度器闸门的时间基准。
1020
1075
  const dreamStartedAt = Date.now();
@@ -1206,7 +1261,8 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1206
1261
  auditError: (text) => {
1207
1262
  decisions = extractJsonArray(text);
1208
1263
  return Array.isArray(decisions) ? null : "no json array in llm output";
1209
- }
1264
+ },
1265
+ metadataFrom: (text) => (Array.isArray(decisions) ? null : parseFailureMeta(text))
1210
1266
  }, (reportUsage) => streamText(ctx, {
1211
1267
  provider: route.provider,
1212
1268
  model: route.model,
@@ -1215,7 +1271,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1215
1271
  ...(withEffort && effort ? { reasoningEffort: effort } : {}),
1216
1272
  messages: [
1217
1273
  { role: "system", content: [{ type: "text", text: consolidationPrompt }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
1218
- { role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
1274
+ { role: "user", content: [{ type: "text", text: fenceListBody(language, listText) }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
1219
1275
  ]
1220
1276
  }, reportUsage, (reason) => { streamFailure = describeStreamFailure(reason); }));
1221
1277
  };
@@ -1458,7 +1514,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1458
1514
  ...(withEffort && effort ? { reasoningEffort: effort } : {}),
1459
1515
  messages: [
1460
1516
  { role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } },
1461
- { role: "user", content: [{ type: "text", text: summaryInputs.map((m) => `- ${m.title}: ${m.content}`).join("\n") }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
1517
+ { role: "user", content: [{ type: "text", text: fenceListBody(language, summaryInputs.map((m) => `- ${m.title}: ${m.content}`).join("\n"), "text") }], source: { kind: "plugin:dsh-mneme", plugin: "dsh-mneme" } }
1462
1518
  ]
1463
1519
  }, reportUsage, (reason) => { summaryStreamFailure = describeStreamFailure(reason); }));
1464
1520
  };
@@ -1476,7 +1532,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
1476
1532
  // (saveWithDedupe 不带 agent_scope/workspace_scope),所以被按下作用域的内容
1477
1533
  // 一旦进了总览,注入侧按作用域过滤就再也拦不住它:内容已经洗成了全局行。
1478
1534
  // 别的入口漏了只是「那一条漏过去」,这里漏了是「洗一遍之后一直漏」。
1479
- const summaryAll = withoutPaused(service, service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document"));
1535
+ const summaryAll = withoutPaused(service, service.all().filter((m) => !isImportedRow(m) && !m.archived && m.type !== "summary" && m.type !== "document"));
1480
1536
  const summaryMaxInputs = Number.isInteger(config.dreamSummaryMaxInputs) ? config.dreamSummaryMaxInputs : 0;
1481
1537
  const summaryInputs = summaryMaxInputs > 0 && summaryAll.length > summaryMaxInputs
1482
1538
  ? [...summaryAll]