dsh-memu-memory 1.2.1 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/AGENTS.md +1 -1
  2. package/README.md +12 -5
  3. package/README.zh.md +4 -2
  4. package/docs/contributing.md +16 -3
  5. package/docs/host-api.md +3 -1
  6. package/docs/memory-semantics.md +21 -2
  7. package/docs/troubleshooting.md +42 -4
  8. package/lib/backup.js +24 -2
  9. package/lib/binding-diagnostics.js +2 -2
  10. package/lib/client.js +1361 -146
  11. package/lib/consolidation-worker.js +30 -8
  12. package/lib/curator-evidence.js +131 -0
  13. package/lib/curator-policy.js +81 -0
  14. package/lib/curator.js +1252 -91
  15. package/lib/cycle.js +3 -0
  16. package/lib/embedding-download.js +39 -4
  17. package/lib/engine.js +6 -3
  18. package/lib/global-graph.js +282 -0
  19. package/lib/graph.js +51 -9
  20. package/lib/inbound.js +19 -0
  21. package/lib/inject.js +277 -17
  22. package/lib/lexicon.js +1115 -0
  23. package/lib/local-vector-service.js +8 -6
  24. package/lib/managed-runtime.js +33 -2
  25. package/lib/memory.js +282 -17
  26. package/lib/memu.js +2 -2
  27. package/lib/migrate.js +83 -2
  28. package/lib/private-links.js +587 -0
  29. package/lib/private.js +32 -3
  30. package/lib/prompts/global-curator.md +76 -19
  31. package/lib/prompts/private-consolidation.coding.md +4 -1
  32. package/lib/prompts/private-consolidation.md +3 -1
  33. package/lib/prompts/private-consolidation.mixed.md +3 -1
  34. package/lib/prompts/private-consolidation.rp.md +3 -1
  35. package/lib/prompts.js +1 -1
  36. package/lib/rpc.js +577 -9
  37. package/lib/state.js +4 -1
  38. package/lib/text-retrieval.js +661 -0
  39. package/lib/text-similarity.js +36 -18
  40. package/lib/tools.js +168 -10
  41. package/lib/webui/standalone.css +17 -0
  42. package/lib/webui/standalone.js +2 -0
  43. package/package.json +1 -1
  44. package/scripts/acceptance-local.mjs +8 -5
  45. package/scripts/curator-pair-calibrate.mjs +259 -0
  46. package/scripts/lexicon-calibrate.mjs +230 -0
  47. package/scripts/lexicon-dedupe.mjs +318 -0
  48. package/tests/absorb-review.test.mjs +353 -0
  49. package/tests/appendix-b2.test.mjs +2 -2
  50. package/tests/client-export.test.mjs +520 -0
  51. package/tests/client-render.test.mjs +19 -3
  52. package/tests/consolidation-fork.test.mjs +14 -12
  53. package/tests/consolidation-snapshot.test.mjs +39 -0
  54. package/tests/curator-absorb.test.mjs +443 -0
  55. package/tests/curator-calibrate.test.mjs +173 -0
  56. package/tests/curator-clusters.test.mjs +281 -0
  57. package/tests/curator-commit.test.mjs +52 -5
  58. package/tests/curator-evidence.test.mjs +102 -0
  59. package/tests/curator-link.test.mjs +316 -0
  60. package/tests/curator-pairing.test.mjs +252 -0
  61. package/tests/curator-probe.test.mjs +15 -6
  62. package/tests/curator-review-mode.test.mjs +314 -0
  63. package/tests/disclosure.test.mjs +2 -1
  64. package/tests/fixtures/curator-pairs/labeled.v1.jsonl +90 -0
  65. package/tests/fixtures/lexicon-stub-hooks.mjs +15 -0
  66. package/tests/fixtures/lexicon-stub.mjs +284 -0
  67. package/tests/graph-cascade.test.mjs +35 -2
  68. package/tests/graph.test.mjs +96 -0
  69. package/tests/help-page.test.mjs +415 -0
  70. package/tests/inbound.test.mjs +6 -3
  71. package/tests/inject-guide.test.mjs +28 -0
  72. package/tests/inject.test.mjs +157 -5
  73. package/tests/lexicon-calibrate.test.mjs +48 -0
  74. package/tests/lexicon-dedupe.test.mjs +171 -0
  75. package/tests/lexicon-frequency.test.mjs +179 -0
  76. package/tests/lexicon-inject.test.mjs +410 -0
  77. package/tests/lexicon-perf.test.mjs +436 -0
  78. package/tests/lexicon-ui.test.mjs +547 -0
  79. package/tests/lexicon.test.mjs +922 -0
  80. package/tests/managed-runtime.test.mjs +32 -0
  81. package/tests/private-links.test.mjs +445 -0
  82. package/tests/prompts.test.mjs +35 -0
  83. package/tests/recall-dedup.test.mjs +33 -4
  84. package/tests/tool-schema.test.mjs +1 -1
  85. package/tests/tools.test.mjs +112 -3
  86. package/tests/webui-auth.test.mjs +348 -2
  87. package/tests/webui.test.mjs +188 -0
package/lib/curator.js CHANGED
@@ -1,11 +1,48 @@
1
+ import { createHash } from 'node:crypto'
1
2
  import {
2
3
  globalDbPath, loadGlobalIndex, updateGlobalIndex, newMemoryId, getSettings, effectiveVectorRoute,
3
4
  } from './state.js'
4
5
  import { commit, forget } from './memu.js'
5
6
  import { loadPrompt } from './prompts.js'
6
- import { retryStoreWrite, redactSecrets, warn } from './paths.js'
7
+ import { retryStoreWrite, redactSecrets, warn, isSafeId } from './paths.js'
7
8
  import { record } from './activity-log.js'
8
9
  import { activeEntryStamp, assertVectorWriteAllowed, hasActiveVectorGeneration, readVectorState, withVectorWrite } from './vector-profile.js'
10
+ import { upsertLexiconTerm } from './lexicon.js'
11
+ import { clean, bigrams, dice, similarTexts, DUP_REJECT_DICE, TOPIC_DICE } from './text-similarity.js'
12
+ import { estTokens } from './budget.js'
13
+ // —— 包 F1(spec §4.1-§4.9/§6.6):候选检索、证据落地、全局 link ——
14
+ import {
15
+ buildCandidateClusters,
16
+ buildCorpus,
17
+ bm25,
18
+ globalConflictCheck,
19
+ pairFeatures,
20
+ pairScore,
21
+ rankPairCandidates,
22
+ tokenize,
23
+ } from './text-retrieval.js'
24
+ import { buildEvidenceCorpus, groundEvidence } from './curator-evidence.js'
25
+ import { applyGlobalLinks, validateGlobalLink, appendAudit } from './global-graph.js'
26
+ import {
27
+ ABSORB_ABSTRACTIVE_COVERAGE,
28
+ ABSORB_AUTO_SOURCE_COVERAGE,
29
+ ABSORB_AUTO_UNION_COVERAGE,
30
+ ABSORB_MAX_CHARS,
31
+ ABSORB_MAX_SOURCES,
32
+ ABSORB_MIN_LENGTH_RATIO,
33
+ ABSORB_REVIEW_SOURCE_COVERAGE,
34
+ ABSORB_REVIEW_UNION_COVERAGE,
35
+ CLUSTER_MIN_PAIR,
36
+ CURATOR_CLUSTER_TOKENS,
37
+ CURATOR_INDEX_MAX_ENTRIES,
38
+ CURATOR_INDEX_TOKENS,
39
+ CURATOR_MAX_LINK_ABSORB,
40
+ EVIDENCE_MAX,
41
+ PAIR_ABSORB_MIN,
42
+ PAIR_DUPLICATE_MIN,
43
+ PAIR_LINK_MIN,
44
+ PAIR_SCAN_LIMIT,
45
+ } from './curator-policy.js'
9
46
 
10
47
  const MAX_SESSIONS = 10
11
48
  const MAX_TOTAL_CHARS = 90_000
@@ -97,20 +134,98 @@ export function condenseEvents(events, maxChars = MAX_SESSION_CHARS) {
97
134
  return joined
98
135
  }
99
136
 
100
- function compactGlobalIndex(index) {
101
- return Object.entries(index.entries || {})
102
- .filter(([, e]) => e.status !== 'archived')
103
- .slice(0, 200)
104
- .map(([id, e]) => ({
105
- id,
106
- track: e.track || 'memory',
107
- domain: e.domain || 'unknown',
108
- applies_to: Array.isArray(e.appliesTo) ? e.appliesTo : [],
109
- title: e.title || '',
110
- content: truncate(e.content || '', 600),
111
- confidence: e.confidence ?? null,
112
- status: e.status || 'active',
113
- }))
137
+ // —— §4.5 curator 上下文打包(包 F1)——
138
+ // 旧 compactGlobalIndex 是"插入顺序前 200 条 ×600 字"的平铺,第 201 条后永远不可见、
139
+ // 无预算。这里改为:BM25(会话文本) top80 → 追加最近 30 / universal 10 / pending 10,
140
+ // 去重后按 BM25 desc → updatedAt desc → id 排序,estTokens 累计超预算即停(丢低分,
141
+ // 不截断 JSON)。同输入同输出,不依赖对象插入顺序。
142
+ function renderIndexEntry(id, entry, contentCap) {
143
+ const links = Array.isArray(entry?.links) ? entry.links : []
144
+ const item = {
145
+ id,
146
+ track: entry?.track || 'memory',
147
+ domain: entry?.domain || 'unknown',
148
+ applies_to: Array.isArray(entry?.appliesTo) ? entry.appliesTo.slice(0, 20) : [],
149
+ title: entry?.title || '',
150
+ content: truncate(entry?.content || '', contentCap),
151
+ confidence: entry?.confidence ?? null,
152
+ status: entry?.status || 'active',
153
+ updatedAt: entry?.updatedAt || null,
154
+ linkCount: links.length,
155
+ linkedTo: links.slice(0, 5).map((l) => ({ to: l?.to || null, kind: l?.kind || null })),
156
+ }
157
+ if (Array.isArray(entry?.absorbedFrom) && entry.absorbedFrom.length) item.absorbedFrom = entry.absorbedFrom
158
+ return item
159
+ }
160
+
161
+ export function packCuratorIndexContext({ index, sessionText, tokenBudget = CURATOR_INDEX_TOKENS, maxEntries = CURATOR_INDEX_MAX_ENTRIES } = {}) {
162
+ const pool = []
163
+ for (const [id, entry] of Object.entries(index?.entries || {})) {
164
+ if (!entry || entry.status === 'archived') continue
165
+ if (entry.reviewKind) continue // 本版没有 link 伪条目;防御性过滤
166
+ pool.push({ id, entry, updatedAt: String(entry.updatedAt || ''), score: 0 })
167
+ }
168
+ if (pool.length) {
169
+ const corpus = buildCorpus(pool.map((p) => ({ ...p.entry, id: p.id })))
170
+ const query = tokenize(sessionText || '')
171
+ for (const p of pool) {
172
+ const doc = corpus.docs.get(p.id)
173
+ p.score = doc ? bm25(query, doc, corpus) : 0
174
+ }
175
+ }
176
+ const byScore = [...pool].sort((a, b) => b.score - a.score || b.updatedAt.localeCompare(a.updatedAt) || (a.id < b.id ? -1 : 1))
177
+ const byRecent = [...pool].sort((a, b) => b.updatedAt.localeCompare(a.updatedAt) || (a.id < b.id ? -1 : 1))
178
+ const selected = new Map()
179
+ const take = (list) => { for (const p of list) if (!selected.has(p.id)) selected.set(p.id, p) }
180
+ take(byScore.slice(0, 80)) // BM25 top 80
181
+ take(byRecent.slice(0, 30)) // 最近 30
182
+ take(byRecent.filter((p) => (p.entry?.domain || '') === 'universal').slice(0, 10)) // universal 10
183
+ take(pool.filter((p) => (p.entry?.status || '') === 'pending_review' && (p.entry?.track || 'memory') === 'memory')
184
+ .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt) || (a.id < b.id ? -1 : 1)).slice(0, 10)) // pending_review 记忆 10
185
+
186
+ const ordered = [...selected.values()].sort((a, b) => b.score - a.score || b.updatedAt.localeCompare(a.updatedAt) || (a.id < b.id ? -1 : 1))
187
+ const entries = []
188
+ let tokens = 0
189
+ for (let i = 0; i < ordered.length && entries.length < maxEntries; i++) {
190
+ const item = renderIndexEntry(ordered[i].id, ordered[i].entry, i < 20 ? 900 : 600)
191
+ const cost = estTokens(JSON.stringify(item))
192
+ if (entries.length > 0 && tokens + cost > tokenBudget) break
193
+ entries.push(item)
194
+ tokens += cost
195
+ }
196
+ return { entries, tokens, omitted: Math.max(0, ordered.length - entries.length), includedIds: entries.map((e) => e.id) }
197
+ }
198
+
199
+ // candidate_clusters 的 prompt 打包(§4.5):预算 2000 tokens,簇内正文截 300 字,
200
+ // pair_scores 保留 2 位;簇已按 avg_score 降序,超预算丢低分簇。
201
+ export function packCuratorClusterContext(clusterResult, { tokenBudget = CURATOR_CLUSTER_TOKENS } = {}) {
202
+ const clusters = []
203
+ let tokens = 0
204
+ for (const cluster of clusterResult?.clusters || []) {
205
+ const rendered = {
206
+ cluster_id: cluster.cluster_id,
207
+ suggested_op: cluster.suggested_op,
208
+ medoid_id: cluster.medoid_id,
209
+ avg_score: cluster.avg_score,
210
+ pair_scores: cluster.pair_scores,
211
+ members: (cluster.members || []).map((m) => ({
212
+ id: m.id, title: m.title || '', track: m.track || 'memory',
213
+ domain: m.domain || 'unknown', updatedAt: m.updatedAt || null,
214
+ content: truncate(m.content || '', 300),
215
+ })),
216
+ }
217
+ const cost = estTokens(JSON.stringify(rendered))
218
+ if (clusters.length > 0 && tokens + cost > tokenBudget) break
219
+ clusters.push(rendered)
220
+ tokens += cost
221
+ }
222
+ const total = clusterResult?.clusters?.length || 0
223
+ return {
224
+ clusters,
225
+ tokens,
226
+ omitted: Math.max(0, total - clusters.length),
227
+ truncated: (clusterResult?.truncated === true) || (total - clusters.length) > 0,
228
+ }
114
229
  }
115
230
 
116
231
  async function readSession(sessionQuery, id) {
@@ -251,25 +366,238 @@ async function compensateCuratorCommit(originalError, { files, route }) {
251
366
  }
252
367
  }
253
368
 
254
- // Exported for the audit regression suite: merging the commit decision logic
255
- // into startGlobalCuration would require standing up subagents to test it.
256
- export async function commitCuratorOutput(output, { job }) {
257
- const memories = Array.isArray(output?.memories) ? output.memories : []
258
- if (memories.length === 0) return { files: [], created: 0, merged: 0, superseded: 0, pending: 0, skipped: [] }
259
- const index = loadGlobalIndex()
260
- const files = []
261
- const updates = []
369
+ // 词典 op 识别(§2.1):op 显式标 'lexicon',或任何带 libraryId / term+kind 的提案。
370
+ // 全局 curator 的输出里出现这些字段只能说明模型越权,拒绝而不是猜它的意图。
371
+ function isLexiconOp(memory) {
372
+ if (!memory || typeof memory !== 'object') return false
373
+ if (memory.op === 'lexicon') return true
374
+ if (typeof memory.libraryId === 'string' && memory.libraryId.trim()) return true
375
+ return typeof memory.term === 'string' && memory.term.trim() && ['slang', 'meme', 'trigger', 'term'].includes(memory.kind)
376
+ }
377
+
378
+ /**
379
+ * 私有整理链路的词典写入(§2.1-§2.2):validated ops 白名单校验 + 落库。
380
+ *
381
+ * 锁顺序(刻意注释,谁改谁读):本函数先拿 curator 侧的 managed 写闸
382
+ * (withManagedCuratorWrite —— 受管向量代际一致性),**在其内部**再让
383
+ * upsertLexiconTerm 拿词典自己的 withFileLock。两把锁的嵌套只有一个方向:
384
+ * curator(vector) → lexicon(file);lexicon.js 从不回头拿向量锁,所以无死锁环。
385
+ * 词典写从不跨 await 持锁——withFileLock 的 action 是同步的。
386
+ *
387
+ * 校验规则(§2.1 curator 行 + 契约):
388
+ * * 只接受 { op:'lexicon', action:'upsert', ... };action 其它值(尤其 delete)
389
+ * 一律拒绝——删除是用户/管理操作,模型没有删除路径。
390
+ * * 必须 evidence(quote 或 id 至少其一非空),无证据直接拒绝;
391
+ * "同一转录同词 ≥2 次才建议"是提示词层的事,代码层只验证据存在。
392
+ * * libraryId 必须显式给出且过 isSafeId;op 里带了别的 libraryId 视为越权拒绝。
393
+ * * needs_review === true → status:'pending',否则 'active'。
394
+ * * sensitive trigger 由 lexicon.js 的 LEXICON_SENSITIVE_FORBIDDEN 兜底(curator
395
+ * 来源理论上不该提,但来源字段在模型手里,存储层必须自己设防)。
396
+ */
397
+ export async function commitPrivateLexiconOps(ops, { libraryId, job } = {}) {
398
+ const list = Array.isArray(ops) ? ops : [ops]
399
+ const committed = []
400
+ const rejected = []
262
401
  const skipped = []
263
- const pending = []
264
- const now = new Date().toISOString()
402
+ if (!isSafeId(libraryId)) {
403
+ for (const raw of list) rejected.push({ term: raw?.term || null, code: 'LEXICON_INVALID_LIBRARY_ID', reason: 'lexicon ops need a valid private library id' })
404
+ return { committed, rejected, skipped }
405
+ }
406
+ const run = async () => {
407
+ for (const raw of list) {
408
+ const op = raw && typeof raw === 'object' ? raw : null
409
+ if (!op || op.op !== 'lexicon') {
410
+ skipped.push({ term: op?.term || null, reason: 'not a lexicon op (op must be "lexicon")' })
411
+ continue
412
+ }
413
+ if (op.action !== 'upsert') {
414
+ rejected.push({ term: op.term || null, code: 'LEXICON_ACTION_FORBIDDEN', reason: `lexicon action "${op.action}" is not allowed for consolidation; only upsert, never delete` })
415
+ continue
416
+ }
417
+ const evidence = Array.isArray(op.evidence)
418
+ ? op.evidence.filter(item => item && typeof item === 'object' && ((typeof item.quote === 'string' && item.quote.trim()) || (typeof item.id === 'string' && item.id.trim())))
419
+ : []
420
+ if (!evidence.length) {
421
+ rejected.push({ term: op.term || null, code: 'LEXICON_EVIDENCE_REQUIRED', reason: 'lexicon upserts need evidence from the transcript' })
422
+ continue
423
+ }
424
+ if (typeof op.libraryId === 'string' && op.libraryId && op.libraryId !== libraryId) {
425
+ rejected.push({ term: op.term || null, code: 'LEXICON_LIBRARY_MISMATCH', reason: `lexicon op targets library ${op.libraryId}, but this consolidation is bound to ${libraryId}` })
426
+ continue
427
+ }
428
+ // 敏感触发词不允许模型猜(§0.2/§2.1):curator 也是模型,整理链路一律不给
429
+ // sensitive trigger(trigger 缺省就是 sensitive,除非显式降为 normal)。
430
+ if (op.kind === 'trigger' && op.sensitivity !== 'normal') {
431
+ rejected.push({ term: op.term || null, code: 'LEXICON_SENSITIVE_FORBIDDEN', reason: 'consolidation cannot create sensitive triggers; sensitive vocabulary needs an explicit user write' })
432
+ continue
433
+ }
434
+ try {
435
+ const result = await upsertLexiconTerm({
436
+ libraryId,
437
+ term: op.term,
438
+ kind: op.kind,
439
+ meaning: op.meaning,
440
+ usage: op.usage,
441
+ aliases: op.aliases,
442
+ examples: op.examples,
443
+ context: op.context,
444
+ knownBy: op.knownBy,
445
+ visibility: op.visibility,
446
+ sensitivity: op.sensitivity,
447
+ confidence: op.confidence,
448
+ evidence,
449
+ source: 'curator',
450
+ status: op.needs_review === true ? 'pending' : 'active',
451
+ })
452
+ committed.push({
453
+ termId: result.term.termId,
454
+ term: result.term.term,
455
+ status: result.term.status,
456
+ created: result.created === true,
457
+ notOwned: result.notOwned === true,
458
+ pendingRevision: result.pendingRevision === true,
459
+ })
460
+ } catch (error) {
461
+ rejected.push({ term: op.term || null, code: error?.code || 'LEXICON_OP_FAILED', reason: String(error?.message || error) })
462
+ }
463
+ }
464
+ return { committed, rejected, skipped }
465
+ }
466
+ if (job) job.step = 'committing lexicon ops'
467
+ // 先过 curator 侧 managed 写闸,再进词典自己的文件锁(顺序见函数头注释)。
468
+ return withManagedCuratorWrite(run)
469
+ }
470
+
471
+ /**
472
+ * 审核决策(spec §4.10):硬门 + 风险分。
473
+ *
474
+ * 硬门恒 pending,不因审核模式放宽:conflict(模型自己承认两段会话打架)、
475
+ * 证据为空(可回放性是底线)、merge/supersede/absorb 的 target 不存在。
476
+ * 非法 target 在这里返回 skip——调用方必须跳过该提案并且**不得**用同一份内容
477
+ * 继续 create(旧代码缺 continue,会把失败的合并悄悄变成一条重复记忆,§0.2)。
478
+ *
479
+ * 风险分只拦"会改写既有内容"的操作:merge/supersede/absorb 在 auto 下风险
480
+ * ≥0.70 才 pending;create 默认通过,needs_review/低置信只记进 risk 供审计与
481
+ * 复核排序。review-all 把全部内容操作都送进复核。风险分上限 2。
482
+ *
483
+ * @returns {{ review: boolean, risk: number, reasons: string[], hard?: boolean } |
484
+ * { skip: true, reason: string }}
485
+ */
486
+ export function reviewDecision({ m, op, evidence, targetId, index, mode }) {
487
+ // 硬门(恒 pending;不因 mode 放宽)
488
+ if (op === 'conflict') return { review: true, hard: true, risk: 2, reasons: ['conflict'] }
489
+ if (!evidence.length) return { review: true, hard: true, risk: 2, reasons: ['missing evidence'] }
490
+ if (op === 'merge' || op === 'supersede' || op === 'absorb') {
491
+ if (!targetId || !index.entries[targetId]) {
492
+ return { skip: true, reason: `target not found: ${targetId || '(empty)'}` }
493
+ }
494
+ }
495
+ // 风险分
496
+ let risk = 0
497
+ const reasons = []
498
+ if (m?.needs_review === true) { risk += 0.70; reasons.push('needs_review') }
499
+ if (Number(m?.confidence) < 0.50) { risk += 0.70; reasons.push('confidence<0.50') }
500
+ else if (Number(m?.confidence) < 0.60) { risk += 0.20; reasons.push('confidence<0.60') }
501
+ if (op === 'merge' || op === 'supersede') { risk += 0.20; reasons.push(op) }
502
+ if (op === 'absorb' && (m?.source_ids?.length || 0) > 3) { risk += 0.20; reasons.push('absorb>3 sources') }
503
+ if (m?.review_reason) { risk += 0.20; reasons.push('review_reason') }
504
+ if (evidence.length === 1) { risk += 0.10; reasons.push('single evidence') }
505
+
506
+ const rewrite = op === 'merge' || op === 'supersede' || op === 'absorb'
507
+ const review = mode === 'review-all' || (mode === 'auto' && rewrite && risk >= 0.70)
508
+ return { review, risk: Math.min(2, Number(risk.toFixed(2))), reasons }
509
+ }
510
+
511
+ // —— §4.8 去重门/冲突门(包 F1)——
512
+ // 门语料只含 active + pending_review:archived/superseded 不是"已存在的事实",
513
+ // 不该拦 create,也不该成为 best。LLM 只裁决;提交层用同一把 pair 尺子复核。
514
+ // buildCorpus 的约定是条目自带 id/title/content(track/domain/appliesTo 等
515
+ // 随条目一起落在 doc.entry 上),所以这里直接展开索引条目再补 id。
516
+ function buildGateCorpus(index) {
517
+ const rows = []
518
+ for (const [id, entry] of Object.entries(index?.entries || {})) {
519
+ if (!entry || (entry.status !== 'active' && entry.status !== 'pending_review')) continue
520
+ rows.push({ ...entry, id })
521
+ }
522
+ return buildCorpus(rows)
523
+ }
265
524
 
525
+ const OPS_ALLOWLIST = ['create', 'merge', 'supersede', 'noop', 'conflict', 'link', 'absorb']
526
+ const SCORE_EPSILON = 1e-9
527
+
528
+ /**
529
+ * 规划阶段(spec §7.2 planCuratorOps):把模型输出拆成 writes / pendings / links /
530
+ * skipped 四类计划;全部语义复核在此完成,提交阶段只做事务。逐条流程:
531
+ * lexicon op → skipped;unknown op → skipped(绝不降级 create);noop → 静默跳过;
532
+ * link → validateGlobalLink 预检(快照索引 + 证据落地),事务内再复核并发面;
533
+ * absorb → skipped 'absorb lands in a later pass'(F2 的 commitCuratorPlans 负责,
534
+ * 它直接从原始输出里取 absorb ops,不受这里的 skip 影响);
535
+ * create/merge/supersede/conflict → 证据落地(§4.6)→ reviewDecision(§4.10)→
536
+ * §4.8 候选门(rankPairCandidates top1):
537
+ * create ≥0.90 同 track/domain → skip 'duplicate of <id>'(pending 同域 →
538
+ * 'duplicate of pending <id>'),warn 日志;
539
+ * create conflict(pairDecision 或 globalConflictCheck 预筛,absorb 带不重复查)→
540
+ * pending 'conflict detected';
541
+ * create 0.82-0.90 → duplicateOf+relatedTo;0.55-0.82 → relatedTo;
542
+ * merge/supersede best 是另一条且高 ≥0.05 → pending 'closest target differs';
543
+ * conflict 恒 pending,target 缺失时用 best。
544
+ */
545
+ export function planCuratorOps(memories, { index, mode, corpus, sessionCorpus = null, now }) {
546
+ const writes = []
547
+ const pendings = []
548
+ const links = []
549
+ const skipped = []
550
+ const entries = index?.entries || {}
266
551
  for (const m of memories) {
267
- // 'conflict' is the curator admitting two sessions disagree. Auto-picking a
268
- // winner would silently rewrite shared memory, so it waits for a human.
269
- const evidence = Array.isArray(m?.evidence) ? m.evidence.filter(item => typeof item?.session_id === 'string' && item.session_id && typeof item?.quote === 'string' && item.quote.trim()).slice(0, 20) : []
270
- const wantsReview = m?.op === 'conflict' || m?.needs_review === true || !evidence.length
271
- const op = ['create', 'merge', 'supersede', 'noop', 'conflict'].includes(m?.op) ? m.op : 'create'
552
+ // 词典 op 是私有整理的专属面(§2.1):全局 curator 只写全局记忆。带 libraryId /
553
+ // lexicon 标记的条目在这里一律拒绝——不进向量提交、不进全局索引,只在 skipped 里
554
+ // 留下可读的原因。私有写入走 commitPrivateLexiconOps。
555
+ if (isLexiconOp(m)) {
556
+ skipped.push({ title: m?.term || m?.title, reason: 'lexicon op rejected: the global curator only writes global memory; lexicon ops belong to the private consolidation chain' })
557
+ continue
558
+ }
559
+ // 未知 op 一律拒绝(§4.8):allowlist 之外的 op 不得静默降级成 create——
560
+ // 模型写错 op 名时,降级会凭空写条目。'noop' 是模型的"无可奉告",直接跳过。
561
+ const op = OPS_ALLOWLIST.includes(m?.op) ? m.op : null
272
562
  if (op === 'noop') continue
563
+ if (!op) {
564
+ skipped.push({ title: m?.title, reason: `unknown op: ${m?.op ?? '(missing)'}` })
565
+ continue
566
+ }
567
+ // —— link 分支(§4.9,包 F1):预检通过才进 links 计划;落库在提交事务里,
568
+ // 与普通 updates 同一次 updateGlobalIndex;不进 files、不触发向量提交。
569
+ if (op === 'link') {
570
+ const verdict = validateGlobalLink({
571
+ index,
572
+ fromId: m?.from_id,
573
+ toId: m?.to_id,
574
+ kind: m?.kind,
575
+ note: m?.note,
576
+ confidence: m?.confidence,
577
+ evidence: Array.isArray(m?.evidence) ? m.evidence : [],
578
+ sessionCorpus,
579
+ })
580
+ if (!verdict.ok) {
581
+ skipped.push({ title: m?.note || m?.to_id || m?.title, reason: `link rejected: ${verdict.reason}` })
582
+ continue
583
+ }
584
+ links.push({ fromId: verdict.fromId, toId: verdict.toId, kind: verdict.kind, symmetric: verdict.symmetric, link: verdict.link })
585
+ continue
586
+ }
587
+ // absorb 提交属于 F2 的 commitCuratorPlans(从原始 output 自取 absorb ops);
588
+ // 这里显式跳过——planCuratorOps 不把 absorb 当成 create 走内容提交。
589
+ // 注意:absorb 已接线(commitCuratorOutput 里调用 commitCuratorPlans),
590
+ // 这个 skip 只是内容半边的路由,不是"未实现"。
591
+ if (op === 'absorb') {
592
+ skipped.push({ title: m?.title, reason: 'absorb handled by commitCuratorPlans' })
593
+ continue
594
+ }
595
+
596
+ // 证据落地(§4.6):quote 必须能回放到本次策划的会话转录;无 corpus(测试/直呼)
597
+ // 时只做形状校验,status='unverified'。有效证据为空是硬门(reviewDecision)。
598
+ const grounded = groundEvidence(Array.isArray(m?.evidence) ? m.evidence : [], sessionCorpus)
599
+ const evidence = grounded.valid.slice(0, EVIDENCE_MAX)
600
+ .map((item) => ({ sessionId: item.sessionId, quote: truncate(item.quote, 500) }))
273
601
  const domain = String(m?.domain || '').trim()
274
602
  if (!validDomain(domain)) {
275
603
  skipped.push({ title: m?.title, reason: `invalid domain: ${domain}` })
@@ -283,21 +611,73 @@ export async function commitCuratorOutput(output, { job }) {
283
611
  continue
284
612
  }
285
613
  const targetId = String(m?.target_id || '').trim()
286
- let id
287
- let previous = null
288
- if ((op === 'merge' || op === 'supersede') && targetId && index.entries[targetId]) {
289
- previous = index.entries[targetId]
290
- // A reviewed (held) entry must never occupy the target's id: rejecting the
291
- // review would archive the only copy and the original active memory would
292
- // be gone. Held entries always get their own id; only a clean merge writes
293
- // in place.
294
- id = newMemoryId('gmem')
295
- } else {
296
- if (op === 'merge' || op === 'supersede') {
297
- skipped.push({ title, reason: `target not found: ${targetId || '(empty)'}` })
298
- }
299
- id = newMemoryId('gmem')
614
+ const decision = reviewDecision({ m, op, evidence, targetId, index, mode })
615
+ if (decision.skip) {
616
+ // 非法 merge/supersede target 到此为止(§0.2 修复):skipped + continue,
617
+ // 绝不带着同一份内容继续 create。
618
+ skipped.push({ title, reason: decision.reason })
619
+ continue
300
620
  }
621
+
622
+ // —— §4.8 候选复核门:每次规划用同一语料、同一把尺子。——
623
+ const proposal = {
624
+ id: '__curator_proposal__',
625
+ title, content, track, domain,
626
+ appliesTo: Array.isArray(m?.applies_to) ? m.applies_to : [],
627
+ }
628
+ const candidates = corpus && corpus.size ? rankPairCandidates({ corpus, proposal, limit: PAIR_SCAN_LIMIT }) : []
629
+ const best = candidates[0] || null
630
+ let review = decision.review
631
+ // reviewReason 优先级:硬门 > 候选门(conflict detected / closest target differs)
632
+ // > 模型 review_reason > 'risk threshold'。
633
+ let reviewReason = decision.hard
634
+ ? decision.reasons[0]
635
+ : (String(m?.review_reason || '').trim() || 'risk threshold')
636
+ let conflictingWith = targetId || null
637
+ const tags = {}
638
+ if (op === 'create' && best) {
639
+ if (best.score >= PAIR_DUPLICATE_MIN
640
+ && (best.entry?.track || 'memory') === track
641
+ && (best.entry?.domain || 'unknown') === domain) {
642
+ const isPending = best.entry?.status === 'pending_review'
643
+ warn(`curator: create skipped, ${isPending ? 'duplicate of pending' : 'duplicate of'} ${best.id} (pair score ${best.score.toFixed(2)})`)
644
+ skipped.push({ title, reason: isPending ? `duplicate of pending ${best.id}` : `duplicate of ${best.id}` })
645
+ continue
646
+ }
647
+ // 冲突检测:pairDecision 的 conflict(同标题正文迥异),或 §4.8 预筛
648
+ // (negation/value/same_title)。duplicate/absorb 带(≥0.82)不重复查——
649
+ // 同标题高相似是同一事实的改写,不是冲突(§4.8 豁免条款的推广)。
650
+ const conflictHit = best.decision === 'conflict' || (best.score < PAIR_ABSORB_MIN
651
+ && globalConflictCheck(proposal, best.entry, best.features))
652
+ if (conflictHit) {
653
+ review = true
654
+ reviewReason = 'conflict detected'
655
+ conflictingWith = best.id
656
+ } else if (best.score >= PAIR_ABSORB_MIN) {
657
+ tags.duplicateOf = best.id
658
+ tags.relatedTo = best.id
659
+ } else if (best.score >= PAIR_LINK_MIN) {
660
+ tags.relatedTo = best.id
661
+ }
662
+ }
663
+ if ((op === 'merge' || op === 'supersede') && best) {
664
+ // best 是另一条且分数比模型 target 高 ≥0.05 → 最近目标存疑,转人审。
665
+ const targetEntry = entries[targetId]
666
+ const targetScore = targetEntry ? pairScore(pairFeatures(proposal, targetEntry)) : 0
667
+ if (best.id !== targetId && best.score >= targetScore + 0.05 - SCORE_EPSILON) {
668
+ review = true
669
+ reviewReason = 'closest target differs'
670
+ conflictingWith = targetId
671
+ }
672
+ }
673
+ if (op === 'conflict' && !targetId && best) conflictingWith = best.id
674
+
675
+ const previous = (op === 'merge' || op === 'supersede') && targetId && entries[targetId] ? entries[targetId] : null
676
+ // A reviewed (held) entry must never occupy the target's id: rejecting the
677
+ // review would archive the only copy and the original active memory would
678
+ // be gone. Held entries always get their own id; only a clean merge writes
679
+ // in place.
680
+ const id = newMemoryId('gmem')
301
681
  const entry = {
302
682
  track,
303
683
  domain,
@@ -306,35 +686,141 @@ export async function commitCuratorOutput(output, { job }) {
306
686
  content: redactSecrets(content),
307
687
  confidence: Number.isFinite(Number(m?.confidence)) ? Math.max(0, Math.min(1, Number(m.confidence))) : 0.6,
308
688
  evidence: evidence.map((e) => ({
309
- sessionId: String(e?.session_id || ''),
689
+ sessionId: String(e?.sessionId || ''),
310
690
  quote: truncate(e?.quote || '', 500),
311
691
  })),
312
692
  status: 'active',
313
693
  supersedes: op === 'supersede' ? targetId || null : previous?.supersedes || null,
694
+ // 风险分写到所有条目(active 也写,§4.10):它是审计/复核排序的统一口径,
695
+ // 不是 pending 的专属属性。旧条目没有这两个字段时按缺省读取,不需要迁移。
696
+ reviewRisk: decision.risk,
697
+ reviewRiskReasons: decision.reasons,
698
+ // 证据是否通过 grounding(§3.1):unverified(直呼/测试)时为 false。
699
+ evidenceVerified: grounded.status === 'grounded',
314
700
  createdAt: previous?.createdAt || now,
315
701
  updatedAt: now,
316
702
  }
317
- if (wantsReview) {
703
+ if (tags.duplicateOf) entry.duplicateOf = tags.duplicateOf
704
+ if (tags.relatedTo) entry.relatedTo = tags.relatedTo
705
+ if (review) {
318
706
  // Held in the index only: never committed to MemU, so a contested claim
319
707
  // cannot be recalled as shared memory until someone decides.
320
708
  entry.status = 'pending_review'
321
- entry.reviewReason = !evidence.length ? 'missing evidence' : op === 'conflict' ? 'conflict' : String(m?.review_reason || 'low confidence')
322
- entry.conflictingWith = targetId || previous?.id || null
323
- pending.push({ id, entry })
709
+ entry.reviewReason = reviewReason
710
+ entry.conflictingWith = conflictingWith
711
+ pendings.push({ id, entry })
324
712
  continue
325
713
  }
326
- files.push({ name: id, track, description: truncate(title, 180), content })
327
- updates.push({ id, entry, op, targetId, previous })
714
+ writes.push({
715
+ id, entry, op, targetId, previous,
716
+ file: { name: id, track, description: truncate(title, 180), content },
717
+ })
718
+ }
719
+ return { writes, pendings, links, skipped }
720
+ }
721
+
722
+ // Exported for the audit regression suite: merging the commit decision logic
723
+ // into startGlobalCuration would require standing up subagents to test it.
724
+ export async function commitCuratorOutput(output, { job, sessionCorpus = null } = {}) {
725
+ const memories = Array.isArray(output?.memories) ? output.memories : []
726
+ if (memories.length === 0) {
727
+ return { files: [], created: 0, merged: 0, superseded: 0, pending: 0, linked: 0, skipped: [], reviewRisk: { max: 0, flagged: 0, reasons: {} } }
728
+ }
729
+ const index = loadGlobalIndex()
730
+ // 模式只在循环外读一次(§4.10):一次策划运行用同一个口径,中途改设置不影响本批。
731
+ const mode = getSettings().curatorReviewMode || 'auto'
732
+ const gateCorpus = buildGateCorpus(index)
733
+ const plan = planCuratorOps(memories, { index, mode, corpus: gateCorpus, sessionCorpus, now: new Date().toISOString() })
734
+ const contentResult = await commitPlannedOps(plan, { job })
735
+ // absorb 半边(F2):planCuratorOps 对 absorb 显式 skip(避免在内容提交里被误当成
736
+ // create),absorb ops 由 commitCuratorPlans 从原始 output 自取并直接提交。
737
+ const absorbResult = await commitCuratorPlans(output, { job })
738
+ return {
739
+ ...contentResult,
740
+ absorbed: absorbResult.absorbed,
741
+ pending: contentResult.pending + absorbResult.pending,
742
+ held: absorbResult.held,
743
+ demoted: absorbResult.demoted,
744
+ skipped: [...contentResult.skipped, ...absorbResult.skipped],
745
+ }
746
+ }
747
+
748
+ /**
749
+ * 提交阶段(spec §7.2 commitCuratorPlans 的内容操作半边;absorb 半边是 F2 的
750
+ * 同名函数,直接吃原始 output)。单次 updateGlobalIndex 事务内按序应用:
751
+ * pendings(索引 only)→ writes(held clone 检查 / merge 兜底 / supersede)→
752
+ * links(applyGlobalLinks,只有 link 时也要走一次索引事务,§4.9)。
753
+ * link 不进 files、不触发向量提交;两档审核模式都自动应用(C2)。
754
+ */
755
+ async function commitPlannedOps({ writes, pendings, links, skipped }, { job }) {
756
+ const files = writes.map((w) => w.file)
757
+ const now = new Date().toISOString()
758
+
759
+ // 风险统计(§4.10):本批所有写入条目(active + pending)的审计口径——最高分、
760
+ // 达到复核阈值(≥0.70)的条目数、每类原因出现的次数。
761
+ const written = [...writes.map((w) => w.entry), ...pendings.map((p) => p.entry)]
762
+ const riskReasons = {}
763
+ for (const entry of written) {
764
+ for (const reason of entry.reviewRiskReasons || []) riskReasons[reason] = (riskReasons[reason] || 0) + 1
765
+ }
766
+ const reviewRiskSummary = {
767
+ max: written.length ? Math.max(...written.map((entry) => Number(entry.reviewRisk) || 0)) : 0,
768
+ flagged: written.filter((entry) => (Number(entry.reviewRisk) || 0) >= 0.70).length,
769
+ reasons: riskReasons,
770
+ }
771
+
772
+ // 单事务:pendings + writes + links 一次落索引。返回本事务实际落地的 link 列表,
773
+ // 供活动日志逐条记录(retry 重跑时以成功那次的返回为准)。
774
+ const applyPlan = (fresh) => {
775
+ for (const { id, entry } of pendings) fresh.entries[id] = entry
776
+ for (const { id, entry, op, targetId, previous } of writes) {
777
+ if (previous && JSON.stringify(fresh.entries[targetId]) !== JSON.stringify(previous)) {
778
+ const heldId = newMemoryId('gmem')
779
+ // 提交成功的只有 `id` 这一行,held 克隆从未进过向量库——它绝不能继承
780
+ // "提交成功即落戳"的 entry.vector:向量库里的行名是 `id` 而不是 heldId,
781
+ // 带戳会让 membership 把一条不存在的行计进来,而真实存在的 `id` 行反而
782
+ // 无处归属(它根本不会按这个克隆写进索引)。
783
+ const held = { ...entry, status: 'pending_review', conflictingWith: targetId, reviewReason: 'target changed while curator was running' }
784
+ delete held.vector
785
+ fresh.entries[heldId] = held
786
+ continue
787
+ }
788
+ if (op === 'merge' && previous) {
789
+ entry.content = entry.content || previous.content
790
+ entry.title = entry.title || previous.title
791
+ entry.evidence = [...(previous.evidence || []), ...(entry.evidence || [])].slice(-40)
792
+ }
793
+ fresh.entries[id] = entry
794
+ if ((op === 'supersede' || op === 'merge') && targetId && fresh.entries[targetId]) {
795
+ fresh.entries[targetId].status = 'superseded'
796
+ fresh.entries[targetId].supersededBy = id
797
+ fresh.entries[targetId].validTo = now
798
+ fresh.entries[targetId].updatedAt = now
799
+ }
800
+ }
801
+ return applyGlobalLinks(fresh.entries, links, { now })
328
802
  }
329
803
 
330
804
  if (files.length === 0) {
331
- if (pending.length) await withManagedCuratorWrite(() => retryStoreWrite(() => updateGlobalIndex(fresh => {
332
- for (const { id, entry } of pending) fresh.entries[id] = entry
333
- })))
334
- return { files: [], created: 0, merged: 0, superseded: 0, pending: pending.length, skipped }
805
+ let linkResult = { applied: [], skipped: [] }
806
+ if (pendings.length || links.length) {
807
+ // 只有 pending/link 时同样要走一次索引事务(§4.9:只有 link 也要调一次
808
+ // updateGlobalIndex);withManagedCuratorWrite 闸门照过,保持锁纪律一致。
809
+ await withManagedCuratorWrite(() => retryStoreWrite(() => updateGlobalIndex((fresh) => {
810
+ linkResult = applyPlan(fresh)
811
+ })))
812
+ }
813
+ for (const applied of linkResult.applied) {
814
+ record({ kind: 'curate', scope: 'global', ok: true, message: `全局关联已落地:${applied.fromId} -(${applied.kind})-> ${applied.toId}`, detail: { op: 'link', from: applied.fromId, to: applied.toId, kind: applied.kind, linkId: applied.linkId } })
815
+ }
816
+ return {
817
+ files: [], created: 0, merged: 0, superseded: 0, pending: pendings.length,
818
+ linked: linkResult.applied.length, skipped, reviewRisk: reviewRiskSummary,
819
+ }
335
820
  }
336
821
 
337
822
  job.step = `committing ${files.length} global memories`
823
+ let linkResult = { applied: [], skipped: [] }
338
824
  await withManagedCuratorWrite(async () => {
339
825
  // Managed agent mode still commits the exact same files to the active local
340
826
  // vector database; only pending_review entries remain index-only.
@@ -350,37 +836,12 @@ export async function commitCuratorOutput(output, { job }) {
350
836
  // 整理写入与单条写入同口径:提交成功就在条目上落"已入向量库"戳,否则这批
351
837
  // 全局记忆在向量计数(membership 的戳腿)里永远缺失。pending 不提交,不戳。
352
838
  const vector = activeEntryStamp(route)
353
- if (vector) for (const update of updates) update.entry.vector = vector
839
+ if (vector) for (const write of writes) write.entry.vector = vector
354
840
  }
355
841
 
356
842
  if (job?.abort?.signal?.aborted) throw new Error('cancelled by user')
357
- await retryStoreWrite(() => updateGlobalIndex(fresh => {
358
- for (const { id, entry } of pending) fresh.entries[id] = entry
359
- for (const { id, entry, op, targetId, previous } of updates) {
360
- if (previous && JSON.stringify(fresh.entries[targetId]) !== JSON.stringify(previous)) {
361
- const heldId = newMemoryId('gmem')
362
- // 提交成功的只有 `id` 这一行,held 克隆从未进过向量库——它绝不能继承
363
- // "提交成功即落戳"的 entry.vector:向量库里的行名是 `id` 而不是 heldId,
364
- // 带戳会让 membership 把一条不存在的行计进来,而真实存在的 `id` 行反而
365
- // 无处归属(它根本不会按这个克隆写进索引)。
366
- const held = { ...entry, status: 'pending_review', conflictingWith: targetId, reviewReason: 'target changed while curator was running' }
367
- delete held.vector
368
- fresh.entries[heldId] = held
369
- continue
370
- }
371
- if (op === 'merge' && previous) {
372
- entry.content = entry.content || previous.content
373
- entry.title = entry.title || previous.title
374
- entry.evidence = [...(previous.evidence || []), ...(entry.evidence || [])].slice(-40)
375
- }
376
- fresh.entries[id] = entry
377
- if ((op === 'supersede' || op === 'merge') && targetId && fresh.entries[targetId]) {
378
- fresh.entries[targetId].status = 'superseded'
379
- fresh.entries[targetId].supersededBy = id
380
- fresh.entries[targetId].validTo = now
381
- fresh.entries[targetId].updatedAt = now
382
- }
383
- }
843
+ await retryStoreWrite(() => updateGlobalIndex((fresh) => {
844
+ linkResult = applyPlan(fresh)
384
845
  }))
385
846
  } catch (error) {
386
847
  // 提交成功之后(abort 检查 / 索引写)的任何失败都会留下孤儿行:尽力撤销。
@@ -388,25 +849,707 @@ export async function commitCuratorOutput(output, { job }) {
388
849
  throw error
389
850
  }
390
851
  })
852
+ for (const applied of linkResult.applied) {
853
+ record({ kind: 'curate', scope: 'global', ok: true, message: `全局关联已落地:${applied.fromId} -(${applied.kind})-> ${applied.toId}`, detail: { op: 'link', from: applied.fromId, to: applied.toId, kind: applied.kind, linkId: applied.linkId } })
854
+ }
391
855
  const summary = {
392
856
  files,
393
- created: updates.filter((u) => u.op === 'create').length,
394
- merged: updates.filter((u) => u.op === 'merge').length,
395
- superseded: updates.filter((u) => u.op === 'supersede').length,
396
- pending: pending.length,
857
+ created: writes.filter((w) => w.op === 'create').length,
858
+ merged: writes.filter((w) => w.op === 'merge').length,
859
+ superseded: writes.filter((w) => w.op === 'supersede').length,
860
+ pending: pendings.length,
861
+ linked: linkResult.applied.length,
397
862
  skipped,
863
+ reviewRisk: reviewRiskSummary,
398
864
  }
399
865
  record({
400
866
  kind: 'curate',
401
867
  scope: 'global',
402
868
  ok: true,
403
869
  message: `全局提炼:新增 ${summary.created} / 合并 ${summary.merged} / 待复核 ${summary.pending}`
870
+ + (summary.linked ? ` / 关联 ${summary.linked}` : '')
404
871
  + (summary.skipped.length ? ` / 跳过 ${summary.skipped.length}` : ''),
405
872
  detail: { ...summary, skipped: summary.skipped.slice(0, 20) },
406
873
  refs: files.map((file) => file.name).slice(0, 20),
407
874
  })
408
875
  return summary
409
876
  }
877
+
878
+ // ============================================================================
879
+ // 包 F2(spec §4.7/§4.11/§6.7):absorb 合并稿、提交事务、复核与撤销。
880
+ // ============================================================================
881
+
882
+ // —— absorb 常量(spec §4.12)——单点定义在 lib/curator-policy.js(文件头部 import)。
883
+ // 句级去重的 charDice 阈值只属于合并稿这一处,留在本地。
884
+ const ABSORB_SENTENCE_DICE = 0.9
885
+
886
+ // —— F1 接口(spec §4.1):tokenize/buildCorpus 已由 lib/text-retrieval.js 落地
887
+ //(文件头部 import)。buildCorpus 返回的是 BM25 语料(df/size),这里按 F1 的
888
+ // idfOf 同一公式折算成合并稿 MMR 用的 idf 表。
889
+ function tokensOf(text) {
890
+ return new Set(tokenize(text))
891
+ }
892
+
893
+ function idfMapFromCorpus(corpus) {
894
+ const idf = new Map()
895
+ if (!corpus?.df) return idf
896
+ for (const [term, df] of corpus.df) {
897
+ idf.set(term, Math.log(1 + (corpus.size - df + 0.5) / (df + 0.5)))
898
+ }
899
+ return idf
900
+ }
901
+
902
+ // idf 语料取自当前全局索引的 active 条目(与候选簇同一口径)。
903
+ function absorbIdf(index) {
904
+ const docs = Object.entries(index?.entries || {})
905
+ .filter(([, entry]) => entry && entry.status === 'active')
906
+ .map(([id, entry]) => ({ id, title: entry.title || '', content: entry.content || '' }))
907
+ return idfMapFromCorpus(buildCorpus(docs))
908
+ }
909
+
910
+ // —— absorb 合并稿(spec §4.7)——
911
+ const SENTENCE_RE = /[^。!?!?;;\n]+[。!?!?;;]?/gu
912
+
913
+ function splitSentences(text) {
914
+ const out = []
915
+ for (const raw of String(text || '').match(SENTENCE_RE) || []) {
916
+ const sentence = raw.trim()
917
+ if (sentence) out.push(sentence)
918
+ }
919
+ return out
920
+ }
921
+
922
+ // token 覆盖率:source 的词元有多大比例出现在 content 里。
923
+ function tokenCoverage(sourceTokens, contentTokens) {
924
+ if (!sourceTokens || sourceTokens.size === 0) return 1
925
+ let hit = 0
926
+ for (const token of sourceTokens) if (contentTokens.has(token)) hit += 1
927
+ return hit / sourceTokens.size
928
+ }
929
+
930
+ // char 覆盖率:union 的字符 bigram 有多大比例出现在 proposed 里。
931
+ function charCoverage(unionText, proposedText) {
932
+ const unionGrams = bigrams(unionText)
933
+ if (unionGrams.size === 0) return 1
934
+ const proposedGrams = bigrams(proposedText)
935
+ let hit = 0
936
+ for (const gram of unionGrams) if (proposedGrams.has(gram)) hit += 1
937
+ return hit / unionGrams.size
938
+ }
939
+
940
+ function sentenceInfo(sentence, idf, coveredTokens = null) {
941
+ let sum = 0
942
+ for (const token of tokensOf(sentence)) {
943
+ if (coveredTokens && coveredTokens.has(token)) continue
944
+ sum += Number(idf?.get?.(token) ?? 1)
945
+ }
946
+ return sum
947
+ }
948
+
949
+ /**
950
+ * 吸收合并稿(spec §4.7)。输入 target、sources、模型的 proposed 稿;
951
+ * 输出 { content, mode, coverage, sourceCoverage, unionCoverage, truncated, omitted }。
952
+ * 句级去重(charDice ≥0.90 或互含丢弃)→ union(target 在前)→ proposed 覆盖
953
+ * ≥0.75 且不超长时用 proposed(abstractive),否则 extractive(放得下直接拼接,
954
+ * 放不下做句子级 MMR:info − 0.7·redundancy·maxInfo 贪心,被丢关键句记 truncated)。
955
+ * idf 由调用方通过 buildCorpus 提供(F1);缺省按 1 计(spec 已兼容 idf=null)。
956
+ */
957
+ export function composeAbsorbContent({ target, sources = [], proposed = '', maxChars = ABSORB_MAX_CHARS, idf = null } = {}) {
958
+ const result = { content: '', mode: 'extractive', coverage: 0, sourceCoverage: 1, unionCoverage: 1, truncated: false, omitted: [] }
959
+ // 1-3. 句子切分 + 句级去重;target 在前,构成 union。origin 记录每个句子的出处
960
+ //(-1 = target,其余 = sources 下标),供关键句判定使用。
961
+ const union = []
962
+ const origins = []
963
+ const keptKeys = []
964
+ const pushSentence = (raw, origin) => {
965
+ const sentence = String(raw || '').trim()
966
+ const key = clean(sentence)
967
+ if (!key) return
968
+ for (const kept of keptKeys) {
969
+ if (kept === key || kept.includes(key) || key.includes(kept) || dice(bigrams(key), bigrams(kept)) >= ABSORB_SENTENCE_DICE) return
970
+ }
971
+ keptKeys.push(key)
972
+ union.push(sentence)
973
+ origins.push(origin)
974
+ }
975
+ const sourceList = Array.isArray(sources) ? sources : []
976
+ for (const sentence of splitSentences(target?.content)) pushSentence(sentence, -1)
977
+ sourceList.forEach((source, index) => {
978
+ for (const sentence of splitSentences(source?.content)) pushSentence(sentence, index)
979
+ })
980
+ if (!union.length) return result
981
+ const unionText = union.join('')
982
+ const proposedText = String(proposed || '').trim()
983
+
984
+ // 4. proposed 覆盖足够且不超长 → abstractive。
985
+ if (proposedText) {
986
+ result.coverage = Math.max(charCoverage(unionText, proposedText), tokenCoverage(tokensOf(unionText), tokensOf(proposedText)))
987
+ if (result.coverage >= ABSORB_ABSTRACTIVE_COVERAGE && proposedText.length <= maxChars) {
988
+ result.content = proposedText
989
+ result.mode = 'abstractive'
990
+ }
991
+ }
992
+
993
+ // 5. extractive:总长放得下直接拼接;否则句子级 MMR 贪心到 maxChars。
994
+ if (result.mode === 'extractive') {
995
+ const infos = union.map((sentence) => sentenceInfo(sentence, idf))
996
+ const maxInfo = infos.reduce((acc, value) => Math.max(acc, value), 0)
997
+ const picked = []
998
+ const covered = new Set()
999
+ let length = 0
1000
+ const remaining = new Set(union.map((_, index) => index))
1001
+ while (remaining.size > 0) {
1002
+ let bestIndex = -1
1003
+ let bestScore = -Infinity
1004
+ for (const index of remaining) {
1005
+ let redundancy = 0
1006
+ for (const chosen of picked) redundancy = Math.max(redundancy, dice(bigrams(union[index]), bigrams(union[chosen])))
1007
+ const score = sentenceInfo(union[index], idf, covered) - 0.7 * redundancy * maxInfo
1008
+ if (score > bestScore) { bestScore = score; bestIndex = index }
1009
+ }
1010
+ remaining.delete(bestIndex)
1011
+ if (length + union[bestIndex].length > maxChars) continue
1012
+ picked.push(bestIndex)
1013
+ length += union[bestIndex].length
1014
+ for (const token of tokensOf(union[bestIndex])) covered.add(token)
1015
+ }
1016
+ if (!picked.length && union.length) picked.push(0)
1017
+ // 关键句:target 全部句子 + 每条 source 的 info 前 3 句;被 MMR 丢掉即 truncated。
1018
+ const byOrigin = new Map()
1019
+ union.forEach((_, index) => {
1020
+ if (!byOrigin.has(origins[index])) byOrigin.set(origins[index], [])
1021
+ byOrigin.get(origins[index]).push(index)
1022
+ })
1023
+ const keyIndices = new Set(byOrigin.get(-1) || [])
1024
+ for (const [origin, indices] of byOrigin) {
1025
+ if (origin === -1) continue
1026
+ for (const index of [...indices].sort((a, b) => infos[b] - infos[a]).slice(0, 3)) keyIndices.add(index)
1027
+ }
1028
+ const missingKeys = [...keyIndices].filter((index) => !picked.includes(index)).sort((a, b) => a - b)
1029
+ if (missingKeys.length) {
1030
+ result.truncated = true
1031
+ result.omitted = missingKeys.slice(0, 10).map((index) => union[index])
1032
+ }
1033
+ result.content = picked.sort((a, b) => a - b).map((index) => union[index]).join('')
1034
+ }
1035
+
1036
+ // 6. 覆盖率:sourceCoverage = 每条 source 的 token 覆盖率取最小;unionCoverage = 全部
1037
+ // source 词元并集的覆盖率。coverage 是 proposed 对 union 的覆盖(§4.7.4)。
1038
+ const contentTokens = tokensOf(result.content)
1039
+ const presentSources = sourceList.filter((source) => String(source?.content || '').trim())
1040
+ if (presentSources.length) {
1041
+ result.sourceCoverage = Math.min(...presentSources.map((source) => tokenCoverage(tokensOf(String(source.content || '')), contentTokens)))
1042
+ const unionSourceTokens = new Set()
1043
+ for (const source of presentSources) {
1044
+ for (const token of tokensOf(String(source.content || ''))) unionSourceTokens.add(token)
1045
+ }
1046
+ result.unionCoverage = tokenCoverage(unionSourceTokens, contentTokens)
1047
+ }
1048
+ return result
1049
+ }
1050
+
1051
+ // absorbOpId(spec §3.1):sha1('absorb|' + sortedIds + '|' + clean(content)).slice(0,16)。
1052
+ // 内容用脱敏后的合并稿,保证从已存条目重算能得到同一个 opId(幂等门靠它对账)。
1053
+ function absorbOpIdOf(sortedIds, content) {
1054
+ return createHash('sha1').update('absorb|' + [...sortedIds].sort().join(',') + '|' + clean(content)).digest('hex').slice(0, 16)
1055
+ }
1056
+
1057
+ function appliesToSubset(sourceApplies, targetApplies) {
1058
+ const target = Array.isArray(targetApplies) ? targetApplies.filter(Boolean) : []
1059
+ if (target.length === 0) return true // target 不限 ⇒ 更宽,任何 source 都算子集
1060
+ const source = Array.isArray(sourceApplies) ? sourceApplies.filter(Boolean) : []
1061
+ return source.every((item) => target.includes(item) || target.includes('all'))
1062
+ }
1063
+
1064
+ const round4 = (value) => Math.round(Number(value) * 10000) / 10000
1065
+
1066
+ /**
1067
+ * absorb 的索引单事务(spec §4.11 auto absorb 步骤 3b-3f)。在 updateGlobalIndex 的
1068
+ * 同步 mutator 里运行;expectedUpdatedAt 任一不匹配时不做任何修改并返回
1069
+ * { landed:false, changed }(调用方决定 held clone 还是抛 'review changed')。
1070
+ * 匹配时:merged 入索引(含出边并入 + absorbSnapshot)、target superseded +
1071
+ * absorbedInto、sources archived + absorbedBy、簇外入边改指 mergedId。
1072
+ */
1073
+ export function applyAbsorbIndexTransaction(fresh, { mergedId, merged, targetId, sourceIds = [], expectedUpdatedAt = {}, now }) {
1074
+ const entries = fresh.entries
1075
+ const memberIds = [targetId, ...sourceIds].filter(Boolean)
1076
+ const changed = []
1077
+ for (const memberId of memberIds) {
1078
+ const current = entries[memberId]
1079
+ const expected = expectedUpdatedAt[memberId]
1080
+ if (!current || current.status !== 'active' || (expected !== undefined && current.updatedAt !== expected)) {
1081
+ changed.push(memberId || '(missing id)')
1082
+ }
1083
+ }
1084
+ if (changed.length) return { landed: false, changed }
1085
+
1086
+ const target = entries[targetId]
1087
+ const sourceRefs = sourceIds.map((id) => ({ id, entry: entries[id] }))
1088
+ const clusterIds = new Set(memberIds)
1089
+ const linksOf = (entry) => (Array.isArray(entry?.links) ? entry.links : [])
1090
+
1091
+ // 吸收前状态快照(§4.11 f):rewired 存簇外条目的完整 links,撤销整体恢复。
1092
+ const rewired = []
1093
+ for (const [otherId, other] of Object.entries(entries)) {
1094
+ if (clusterIds.has(otherId) || otherId === mergedId) continue
1095
+ if (linksOf(other).some((link) => clusterIds.has(link?.to))) {
1096
+ rewired.push({ id: otherId, links: structuredClone(linksOf(other)) })
1097
+ }
1098
+ }
1099
+ // 出边并入 merged:target/sources 自己的 links 按 (to,kind) 去重、cap 20;
1100
+ // 指向簇内的边在吸收后是悬挂边,改指 mergedId,自环直接丢。
1101
+ const outbound = []
1102
+ const seenPairs = new Set()
1103
+ for (const member of [target, ...sourceRefs.map((ref) => ref.entry)]) {
1104
+ for (const link of linksOf(member)) {
1105
+ if (!link || !link.to) continue
1106
+ const key = `${link.to}|${link.kind || 'related'}`
1107
+ if (seenPairs.has(key)) continue
1108
+ seenPairs.add(key)
1109
+ outbound.push(structuredClone(link))
1110
+ }
1111
+ }
1112
+ const mergedLinks = outbound
1113
+ .map((link) => (clusterIds.has(link.to) ? { ...link, to: mergedId } : link))
1114
+ .filter((link) => link.to !== mergedId)
1115
+ .slice(0, CURATOR_MAX_LINK_ABSORB)
1116
+
1117
+ const mergedEntry = {
1118
+ ...merged,
1119
+ links: mergedLinks,
1120
+ updatedAt: now,
1121
+ absorbSnapshot: {
1122
+ target: { id: targetId, status: target.status, links: structuredClone(linksOf(target)), updatedAt: target.updatedAt },
1123
+ sources: sourceRefs.map(({ id, entry }) => ({ id, status: entry.status, links: structuredClone(linksOf(entry)), updatedAt: entry.updatedAt })),
1124
+ rewired,
1125
+ mergedLinks: structuredClone(mergedLinks),
1126
+ at: now,
1127
+ },
1128
+ }
1129
+
1130
+ entries[mergedId] = mergedEntry
1131
+ target.status = 'superseded'
1132
+ target.supersededBy = mergedId
1133
+ target.absorbedInto = mergedId
1134
+ target.validTo = now
1135
+ target.updatedAt = now
1136
+ const audit = { op: 'absorb', at: now, mergedId, targetId, sourceIds: [...sourceIds] }
1137
+ appendAudit(mergedEntry, audit)
1138
+ appendAudit(target, audit)
1139
+ for (const { entry } of sourceRefs) {
1140
+ entry.status = 'archived'
1141
+ entry.absorbedBy = mergedId
1142
+ entry.absorbedAt = now
1143
+ entry.validTo = now
1144
+ entry.updatedAt = now
1145
+ appendAudit(entry, audit)
1146
+ }
1147
+ for (const { id } of rewired) {
1148
+ entries[id].links = entries[id].links.map((link) => (clusterIds.has(link?.to) ? { ...link, to: mergedId } : link))
1149
+ }
1150
+ return { landed: true, at: now }
1151
+ }
1152
+
1153
+ /**
1154
+ * absorb 计划提交(spec §6.7 + §4.7 + §4.10 + §4.11)。输入 curator 输出,只处理
1155
+ * op === 'absorb' 的条目;每条走完 协议校验 → 降级(domain/appliesTo)→ medoid →
1156
+ * 合并稿 → 覆盖率分段 → 重复门 → 风险门,auto 直接落 §4.11 事务,其余生成
1157
+ * §3.3 pendingAbsorb 提案(只进索引,不进向量库)。
1158
+ * 并发变化:不落 active,held pending clone,已提交向量行锁外 best-effort forget。
1159
+ */
1160
+ export async function commitCuratorPlans(output, { job } = {}) {
1161
+ const memories = Array.isArray(output?.memories) ? output.memories : []
1162
+ const plans = memories.filter((m) => m && m.op === 'absorb')
1163
+ const result = { absorbed: 0, pending: 0, held: 0, skipped: [], demoted: [] }
1164
+ if (!plans.length) return result
1165
+ const index = loadGlobalIndex()
1166
+ // 模式只在循环外读一次(§4.10)。
1167
+ const mode = getSettings().curatorReviewMode || 'auto'
1168
+ const now = new Date().toISOString()
1169
+ const confOf = (value) => (Number.isFinite(Number(value)) ? Number(value) : 0.6)
1170
+ const pendingEntries = []
1171
+ const autoCommits = []
1172
+ const heldNames = []
1173
+
1174
+ for (const plan of plans) {
1175
+ const label = String(plan?.title || plan?.target_id || '(absorb)')
1176
+ // —— 协议校验(§6.7):target active;source 1–5;target 不在 source_ids。——
1177
+ const targetId = String(plan?.target_id || '').trim()
1178
+ const target = targetId ? index.entries[targetId] : null
1179
+ if (!target || target.status !== 'active') {
1180
+ result.skipped.push({ title: label, reason: `absorb target not found or not active: ${targetId || '(empty)'}` })
1181
+ continue
1182
+ }
1183
+ const requestedSources = [...new Set((Array.isArray(plan?.source_ids) ? plan.source_ids : [])
1184
+ .map((id) => String(id || '').trim()).filter(Boolean))]
1185
+ if (requestedSources.length < 1 || requestedSources.length > ABSORB_MAX_SOURCES) {
1186
+ result.skipped.push({ title: label, reason: `absorb needs 1-${ABSORB_MAX_SOURCES} distinct source ids, got ${requestedSources.length}` })
1187
+ continue
1188
+ }
1189
+ if (requestedSources.includes(targetId)) {
1190
+ result.skipped.push({ title: label, reason: 'absorb target must not appear in source_ids' })
1191
+ continue
1192
+ }
1193
+ // —— 降级(§4.7.11):source 的 appliesTo 必须是 target 子集;最终只允许同 domain。
1194
+ // 不满足的 source 不 absorb,降级 link(link 落地由 F1 的 commitCuratorOutput link 分支做)。
1195
+ const sourceRefs = []
1196
+ for (const sourceId of requestedSources) {
1197
+ const source = index.entries[sourceId]
1198
+ if (!source || source.status !== 'active') {
1199
+ result.demoted.push({ targetId, sourceId, reason: 'source not found or not active' })
1200
+ continue
1201
+ }
1202
+ if ((source.track || 'memory') !== (target.track || 'memory')) {
1203
+ result.demoted.push({ targetId, sourceId, reason: 'track differs' })
1204
+ continue
1205
+ }
1206
+ if ((source.domain || 'unknown') !== (target.domain || 'unknown')) {
1207
+ result.demoted.push({ targetId, sourceId, reason: 'domain differs; absorb is same-domain only (link candidate)' })
1208
+ continue
1209
+ }
1210
+ if (!appliesToSubset(source.appliesTo, target.appliesTo)) {
1211
+ result.demoted.push({ targetId, sourceId, reason: 'source appliesTo is not a subset of target; demoted to link candidate' })
1212
+ continue
1213
+ }
1214
+ sourceRefs.push({ id: sourceId, entry: source })
1215
+ }
1216
+ if (!sourceRefs.length) {
1217
+ result.skipped.push({ title: label, reason: 'no eligible source left after domain/appliesTo demotion' })
1218
+ continue
1219
+ }
1220
+ const sources = sourceRefs.map((ref) => ({ ...ref.entry, id: ref.id }))
1221
+
1222
+ // —— medoid/representative(C4):算 medoid,模型 target 只作 provenance,
1223
+ // medoid 平均相似度 <0.55 整簇跳过。——
1224
+ const members = [{ id: targetId, entry: target }, ...sourceRefs.map((ref) => ({ id: ref.id, entry: ref.entry }))]
1225
+ const memberText = new Map(members.map((member) => [member.id, `${member.entry.title || ''} ${member.entry.content || ''}`]))
1226
+ let medoid = null
1227
+ for (const member of members) {
1228
+ let sum = 0
1229
+ for (const other of members) {
1230
+ if (other.id === member.id) continue
1231
+ sum += similarTexts(memberText.get(member.id), memberText.get(other.id))
1232
+ }
1233
+ const avg = sum / Math.max(1, members.length - 1)
1234
+ if (!medoid || avg > medoid.avg) medoid = { id: member.id, avg }
1235
+ }
1236
+ if (!medoid || medoid.avg < CLUSTER_MIN_PAIR) {
1237
+ result.skipped.push({ title: label, reason: `medoid average similarity ${medoid ? medoid.avg.toFixed(2) : 'n/a'} is below the pair floor` })
1238
+ continue
1239
+ }
1240
+ const representativeId = medoid.id
1241
+
1242
+ // —— evidence(§4.6/§4.7.8):plan 证据(带 sourceId)优先,簇内既有证据并集,
1243
+ // 去重 ≤20;每条 source 至少一条 plan 证据覆盖,否则只进复核。——
1244
+ const planEvidence = (Array.isArray(plan?.evidence) ? plan.evidence : [])
1245
+ .filter((item) => typeof item?.session_id === 'string' && item.session_id && typeof item?.quote === 'string' && item.quote.trim())
1246
+ .map((item) => ({
1247
+ sessionId: item.session_id,
1248
+ quote: truncate(item.quote, 500),
1249
+ sourceId: typeof item?.source_id === 'string' && item.source_id.trim() ? item.source_id.trim() : null,
1250
+ }))
1251
+ .slice(0, 20)
1252
+ const clusterEvidence = []
1253
+ const seenEvidence = new Set()
1254
+ for (const item of [...planEvidence, ...members.flatMap((member) => (Array.isArray(member.entry.evidence) ? member.entry.evidence : [])
1255
+ .map((e) => ({ sessionId: e?.sessionId, quote: e?.quote, sourceId: member.id })))] ) {
1256
+ if (clusterEvidence.length >= 20) break
1257
+ const sessionId = typeof item?.sessionId === 'string' ? item.sessionId : null
1258
+ const quote = typeof item?.quote === 'string' ? item.quote : ''
1259
+ if (!sessionId || !quote.trim()) continue
1260
+ const key = `${sessionId}|${clean(quote)}|${item.sourceId || ''}`
1261
+ if (seenEvidence.has(key)) continue
1262
+ seenEvidence.add(key)
1263
+ clusterEvidence.push({ sessionId, quote: truncate(quote, 500), sourceId: item.sourceId || null })
1264
+ }
1265
+ const uncoveredSources = sourceRefs.filter((ref) => !planEvidence.some((item) => item.sourceId === ref.id))
1266
+
1267
+ // —— 合并稿 + 覆盖率分段(§4.7.6)。——
1268
+ const idf = absorbIdf(index)
1269
+ const composed = composeAbsorbContent({ target, sources, proposed: plan?.content, maxChars: ABSORB_MAX_CHARS, idf })
1270
+ const mergedContent = redactSecrets(composed.content)
1271
+ if (!mergedContent.trim()) {
1272
+ result.skipped.push({ title: label, reason: 'absorb produced an empty merged draft' })
1273
+ continue
1274
+ }
1275
+ if (composed.sourceCoverage < ABSORB_REVIEW_SOURCE_COVERAGE || composed.unionCoverage < ABSORB_REVIEW_UNION_COVERAGE) {
1276
+ result.skipped.push({ title: label, reason: `absorb coverage below floor (source ${composed.sourceCoverage.toFixed(2)} / union ${composed.unionCoverage.toFixed(2)})` })
1277
+ continue
1278
+ }
1279
+ const longestSource = Math.max(0, ...sourceRefs.map((ref) => String(ref.entry.content || '').length))
1280
+ const belowLengthFloor = mergedContent.length < ABSORB_MIN_LENGTH_RATIO * longestSource
1281
+
1282
+ // —— 幂等门(§6.7):同 opId 的提案/合并稿已存在 → skip。——
1283
+ const memberIdsSorted = [targetId, ...sourceRefs.map((ref) => ref.id)].sort()
1284
+ const opId = absorbOpIdOf(memberIdsSorted, mergedContent)
1285
+ if (Object.values(index.entries).some((e) => e?.absorbOpId === opId || e?.pendingAbsorb?.opId === opId)) {
1286
+ result.skipped.push({ title: label, reason: `duplicate absorb opId ${opId}; already proposed or absorbed` })
1287
+ continue
1288
+ }
1289
+
1290
+ // —— title(§4.7.7)与重复门(§6.7):与簇外 active 条目 similarTexts ≥0.90 →
1291
+ // pending 'absorb duplicate of <id>';0.55–0.90 → 新条目打 duplicateOf。——
1292
+ const titleCandidate = String(target.title || '').trim()
1293
+ || String(plan?.title || '').trim()
1294
+ || String(index.entries[representativeId]?.title || '').trim()
1295
+ const title = truncate(titleCandidate || label, 180)
1296
+ let absorbDuplicate = null
1297
+ for (const [otherId, other] of Object.entries(index.entries || {})) {
1298
+ if (memberIdsSorted.includes(otherId)) continue
1299
+ if (other?.status !== 'active') continue
1300
+ const similarity = similarTexts(`${title} ${mergedContent}`, `${other.title || ''} ${other.content || ''}`)
1301
+ if (similarity > (absorbDuplicate?.similarity ?? 0)) absorbDuplicate = { id: otherId, similarity }
1302
+ }
1303
+
1304
+ // —— 风险分(§4.10):写到所有条目(active 也写)。——
1305
+ const decision = reviewDecision({ m: plan, op: 'absorb', evidence: planEvidence, targetId, index, mode })
1306
+
1307
+ let reviewReason = null
1308
+ if (composed.sourceCoverage < ABSORB_AUTO_SOURCE_COVERAGE || composed.unionCoverage < ABSORB_AUTO_UNION_COVERAGE
1309
+ || belowLengthFloor || (composed.truncated && composed.omitted.length)) {
1310
+ reviewReason = 'absorb coverage'
1311
+ } else if (absorbDuplicate && absorbDuplicate.similarity >= DUP_REJECT_DICE) {
1312
+ reviewReason = `absorb duplicate of ${absorbDuplicate.id}`
1313
+ } else if (uncoveredSources.length || !planEvidence.length) {
1314
+ reviewReason = decision.hard ? decision.reasons[0] : 'evidence not grounded'
1315
+ } else if (decision.review) {
1316
+ reviewReason = decision.hard ? decision.reasons[0] : (String(plan?.review_reason || '').trim() || 'risk threshold')
1317
+ }
1318
+
1319
+ const confidence = Math.min(0.95, Math.max(confOf(target.confidence), ...sourceRefs.map((ref) => confOf(ref.entry.confidence)), confOf(plan?.confidence)))
1320
+ const clusterCreatedAt = members.map((member) => member.entry.createdAt).filter(Boolean).sort()[0] || now
1321
+ const expectedUpdatedAt = {
1322
+ [targetId]: target.updatedAt,
1323
+ ...Object.fromEntries(sourceRefs.map((ref) => [ref.id, ref.entry.updatedAt])),
1324
+ }
1325
+ const reviewRiskReasons = reviewReason === 'absorb coverage' ? [...decision.reasons, 'absorb coverage'] : decision.reasons
1326
+ const entryBase = {
1327
+ track: target.track || 'memory',
1328
+ domain: target.domain || 'general',
1329
+ appliesTo: Array.isArray(target.appliesTo) ? structuredClone(target.appliesTo).slice(0, 20) : [],
1330
+ title,
1331
+ content: mergedContent,
1332
+ confidence,
1333
+ evidence: clusterEvidence,
1334
+ status: 'active',
1335
+ absorbedFrom: memberIdsSorted,
1336
+ absorbedAt: now,
1337
+ absorbOpId: opId,
1338
+ representativeId,
1339
+ mergeMode: composed.mode,
1340
+ mergeCoverage: round4(composed.coverage),
1341
+ sourceCoverage: round4(composed.sourceCoverage),
1342
+ unionCoverage: round4(composed.unionCoverage),
1343
+ reviewRisk: decision.risk,
1344
+ reviewRiskReasons,
1345
+ createdAt: clusterCreatedAt,
1346
+ updatedAt: now,
1347
+ }
1348
+ if (absorbDuplicate && absorbDuplicate.similarity >= TOPIC_DICE && absorbDuplicate.similarity < DUP_REJECT_DICE) {
1349
+ entryBase.duplicateOf = absorbDuplicate.id
1350
+ }
1351
+
1352
+ if (reviewReason) {
1353
+ // pending absorb(§3.3):只进索引,绝不进向量库;accept 时才提交提案 id 本身。
1354
+ pendingEntries.push({
1355
+ id: newMemoryId('gmem'),
1356
+ entry: {
1357
+ ...entryBase,
1358
+ status: 'pending_review',
1359
+ reviewReason,
1360
+ conflictingWith: targetId,
1361
+ pendingAbsorb: { targetId, sourceIds: sourceRefs.map((ref) => ref.id), opId, representativeId, expectedUpdatedAt },
1362
+ source: 'curator',
1363
+ },
1364
+ })
1365
+ result.pending += 1
1366
+ continue
1367
+ }
1368
+ autoCommits.push({
1369
+ mergedId: newMemoryId('gmem'),
1370
+ entry: entryBase,
1371
+ targetId,
1372
+ sourceIds: sourceRefs.map((ref) => ref.id),
1373
+ expectedUpdatedAt,
1374
+ })
1375
+ }
1376
+
1377
+ if (pendingEntries.length) {
1378
+ if (job) job.step = `writing ${pendingEntries.length} pending absorb proposals`
1379
+ await withManagedCuratorWrite(() => retryStoreWrite(() => updateGlobalIndex((fresh) => {
1380
+ for (const { id, entry } of pendingEntries) fresh.entries[id] = entry
1381
+ })))
1382
+ }
1383
+
1384
+ if (autoCommits.length) {
1385
+ if (job) job.step = `committing ${autoCommits.length} absorb merges`
1386
+ await withManagedCuratorWrite(async () => {
1387
+ const route = engineRoute()
1388
+ const committedNames = []
1389
+ try {
1390
+ // 先向量后索引(§4.11 auto absorb 步骤 2→3):索引失败时能补偿已提交行。
1391
+ if (route !== 'lexical') {
1392
+ await commit({
1393
+ payload: {
1394
+ recall_files: autoCommits.map(({ mergedId, entry }) => ({ name: mergedId, track: entry.track, description: entry.title, content: entry.content })),
1395
+ resource: [],
1396
+ user: {},
1397
+ },
1398
+ route,
1399
+ db: globalDbPath(),
1400
+ })
1401
+ for (const c of autoCommits) committedNames.push(c.mergedId)
1402
+ const vector = activeEntryStamp(route)
1403
+ if (vector) for (const c of autoCommits) c.entry.vector = vector
1404
+ }
1405
+ await retryStoreWrite(() => updateGlobalIndex((fresh) => {
1406
+ for (const c of autoCommits) {
1407
+ const outcome = applyAbsorbIndexTransaction(fresh, {
1408
+ mergedId: c.mergedId, merged: c.entry, targetId: c.targetId,
1409
+ sourceIds: c.sourceIds, expectedUpdatedAt: c.expectedUpdatedAt, now,
1410
+ })
1411
+ if (!outcome.landed) {
1412
+ // 并发变化(§4.11 步骤 5):不落 active,生成 held pending clone;
1413
+ // expectedUpdatedAt 刷新为当前值,人审时再做快照校验。
1414
+ const heldId = newMemoryId('gmem')
1415
+ const refreshed = {}
1416
+ for (const memberId of [c.targetId, ...c.sourceIds]) {
1417
+ refreshed[memberId] = fresh.entries[memberId]?.updatedAt || c.expectedUpdatedAt[memberId]
1418
+ }
1419
+ const held = {
1420
+ ...c.entry,
1421
+ status: 'pending_review',
1422
+ reviewReason: 'cluster changed while curator was running',
1423
+ conflictingWith: c.targetId,
1424
+ pendingAbsorb: { targetId: c.targetId, sourceIds: c.sourceIds, opId: c.entry.absorbOpId, representativeId: c.entry.representativeId, expectedUpdatedAt: refreshed },
1425
+ updatedAt: now,
1426
+ }
1427
+ // 已提交向量库的是 mergedId 这一行;held 克隆从未进过向量库,
1428
+ // 绝不能继承"提交成功即落戳"的 entry.vector(同 curator-commit 的坑)。
1429
+ delete held.vector
1430
+ fresh.entries[heldId] = held
1431
+ heldNames.push(c.mergedId)
1432
+ result.held += 1
1433
+ continue
1434
+ }
1435
+ result.absorbed += 1
1436
+ }
1437
+ }))
1438
+ } catch (error) {
1439
+ if (committedNames.length) await compensateCuratorCommit(error, { files: committedNames.map((name) => ({ name })), route })
1440
+ throw error
1441
+ }
1442
+ // held clone 的已提交行 best-effort forget(锁外 await,修 D11);只在向量路由
1443
+ // 真的提交过行时才需要。删不掉必须可见。
1444
+ if (route !== 'lexical') {
1445
+ for (const name of heldNames) {
1446
+ try {
1447
+ await forget({ db: globalDbPath(), name })
1448
+ } catch (forgetError) {
1449
+ record({
1450
+ kind: 'curate', scope: 'global', ok: false, level: 'error',
1451
+ message: `集合吸收并发保护:held clone 已生成,但向量行 ${name} 撤销失败(引擎库残留孤儿行,需手动清理)`,
1452
+ detail: { op: 'absorb', heldForgetFailed: true, name, error: String(forgetError?.message || forgetError) },
1453
+ refs: [name],
1454
+ })
1455
+ }
1456
+ }
1457
+ }
1458
+ })
1459
+ }
1460
+
1461
+ if (result.absorbed || result.pending || result.held || result.skipped.length || result.demoted.length) {
1462
+ record({
1463
+ kind: 'curate',
1464
+ scope: 'global',
1465
+ ok: true,
1466
+ message: `集合吸收:合并 ${result.absorbed} / 待复核 ${result.pending}`
1467
+ + (result.held ? ` / 并发暂扣 ${result.held}` : '')
1468
+ + (result.skipped.length ? ` / 跳过 ${result.skipped.length}` : '')
1469
+ + (result.demoted.length ? ` / 降级 link ${result.demoted.length}` : ''),
1470
+ detail: { op: 'absorb', ...result, skipped: result.skipped.slice(0, 20), demoted: result.demoted.slice(0, 20) },
1471
+ refs: autoCommits.map((c) => c.mergedId).slice(0, 20),
1472
+ })
1473
+ }
1474
+ return result
1475
+ }
1476
+
1477
+ /**
1478
+ * 撤销集合吸收(spec §4.11 undo-absorb)。merged 必须 active 且带 absorbSnapshot;
1479
+ * 同一事务恢复 target/sources/rewired,merged 归档(保留正文)。吸收后又被人工改过
1480
+ * (updatedAt > absorbedAt)的成员拒绝整体撤销并返回可读错误——部分恢复比不恢复更
1481
+ * 难对账。不删除、不写向量。
1482
+ */
1483
+ export async function undoGlobalAbsorb({ id } = {}) {
1484
+ const mergedId = String(id || '').trim()
1485
+ if (!mergedId) throw new Error('undo-absorb needs the merged entry id')
1486
+ const index = loadGlobalIndex()
1487
+ const merged = index.entries[mergedId]
1488
+ if (!merged || merged.status !== 'active' || !merged.absorbSnapshot) {
1489
+ throw new Error('undo-absorb needs an active merged entry with an absorb snapshot')
1490
+ }
1491
+ const snapshot = merged.absorbSnapshot
1492
+ const memberRefs = [snapshot.target, ...(Array.isArray(snapshot.sources) ? snapshot.sources : [])].filter(Boolean)
1493
+ const changedMembers = []
1494
+ for (const member of memberRefs) {
1495
+ const current = index.entries[member.id]
1496
+ if (!current) continue
1497
+ const absorbedAt = Date.parse(current.absorbedAt || '') || Date.parse(snapshot.at || '') || 0
1498
+ if ((Date.parse(current.updatedAt || '') || 0) > absorbedAt) changedMembers.push(member.id)
1499
+ }
1500
+ if (changedMembers.length) {
1501
+ throw new Error(`absorb undo refused: ${changedMembers.join(', ')} changed after absorption; restore them manually or resolve each edit first`)
1502
+ }
1503
+ await retryStoreWrite(() => updateGlobalIndex((fresh) => {
1504
+ if (JSON.stringify(fresh.entries[mergedId]) !== JSON.stringify(merged)) throw new Error('absorb entry changed; reload and retry')
1505
+ const at = new Date().toISOString()
1506
+ const targetSnap = snapshot.target
1507
+ const currentTarget = targetSnap?.id ? fresh.entries[targetSnap.id] : null
1508
+ if (currentTarget && currentTarget.status === 'superseded' && currentTarget.supersededBy === mergedId) {
1509
+ currentTarget.status = targetSnap.status || 'active'
1510
+ if (Array.isArray(targetSnap.links)) currentTarget.links = structuredClone(targetSnap.links)
1511
+ delete currentTarget.supersededBy
1512
+ delete currentTarget.absorbedInto
1513
+ delete currentTarget.validTo
1514
+ currentTarget.updatedAt = at
1515
+ }
1516
+ for (const sourceSnap of Array.isArray(snapshot.sources) ? snapshot.sources : []) {
1517
+ const current = fresh.entries[sourceSnap.id]
1518
+ if (!current || current.status !== 'archived' || current.absorbedBy !== mergedId) continue
1519
+ current.status = sourceSnap.status || 'active'
1520
+ if (Array.isArray(sourceSnap.links)) current.links = structuredClone(sourceSnap.links)
1521
+ delete current.absorbedBy
1522
+ delete current.absorbedAt
1523
+ delete current.validTo
1524
+ current.updatedAt = at
1525
+ }
1526
+ for (const rewiredSnap of Array.isArray(snapshot.rewired) ? snapshot.rewired : []) {
1527
+ const current = fresh.entries[rewiredSnap.id]
1528
+ if (current && Array.isArray(rewiredSnap.links)) current.links = structuredClone(rewiredSnap.links)
1529
+ }
1530
+ const stored = fresh.entries[mergedId]
1531
+ stored.status = 'archived'
1532
+ stored.absorbUndoneAt = at
1533
+ stored.updatedAt = at
1534
+ const audit = { op: 'undo-absorb', at, mergedId }
1535
+ appendAudit(currentTarget, audit)
1536
+ for (const sourceSnap of Array.isArray(snapshot.sources) ? snapshot.sources : []) {
1537
+ appendAudit(fresh.entries[sourceSnap.id], audit)
1538
+ }
1539
+ appendAudit(stored, audit)
1540
+ }))
1541
+ // 活动日志 CURATOR_ABSORB_UNDO(§4.11 撤销 4):以 detail.op 标记,kind 仍是 curate。
1542
+ record({
1543
+ kind: 'curate',
1544
+ scope: 'global',
1545
+ ok: true,
1546
+ message: `撤销集合吸收 ${mergedId}:target 与来源记忆已恢复,merged 已归档`,
1547
+ detail: { op: 'undo-absorb', id: mergedId, snapshotAt: snapshot.at || null },
1548
+ refs: [mergedId],
1549
+ })
1550
+ return { id: mergedId, status: 'archived', undone: true }
1551
+ }
1552
+
410
1553
  export function startGlobalCuration(ctx, { sessionIds, parentSessionId, model } = {}) {
411
1554
  const ids = [...new Set((Array.isArray(sessionIds) ? sessionIds : []).filter((s) => typeof s === 'string' && s.trim()))]
412
1555
  const jobId = `curate-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`
@@ -465,8 +1608,20 @@ export function startGlobalCuration(ctx, { sessionIds, parentSessionId, model }
465
1608
  if (total >= MAX_TOTAL_CHARS) break
466
1609
  }
467
1610
 
468
- const existing = compactGlobalIndex(loadGlobalIndex())
469
- const input = JSON.stringify({ sessions, existing_global_index: existing }, null, 2)
1611
+ // —— §4.5/§6.6(包 F1):相关性打包索引(4000 tokens)+ 候选簇(2000 tokens),
1612
+ // 取代旧 compactGlobalIndex 的"插入顺序前 200 条"平铺。sessionCorpus 同步构建,
1613
+ // 提交层证据落地(§4.6)用同一份转录做回放校验。
1614
+ const globalIndex = loadGlobalIndex()
1615
+ const sessionText = sessions.map((s) => s.transcript).join('\n')
1616
+ const packedIndex = packCuratorIndexContext({ index: globalIndex, sessionText })
1617
+ const packedClusters = packCuratorClusterContext(buildCandidateClusters(globalIndex))
1618
+ const input = JSON.stringify({
1619
+ sessions,
1620
+ existing_global_index: packedIndex.entries,
1621
+ index_truncated: packedIndex.omitted > 0,
1622
+ candidate_clusters: packedClusters.clusters,
1623
+ clusters_truncated: packedClusters.truncated,
1624
+ }, null, 2)
470
1625
  const prompt = loadPrompt('global-curator', { input }).text
471
1626
  job.step = 'curator running'
472
1627
  log('starting non-main curator subagent')
@@ -499,7 +1654,12 @@ export function startGlobalCuration(ctx, { sessionIds, parentSessionId, model }
499
1654
  throw new Error(`curator did not return valid JSON: ${truncate(text, 800)}`)
500
1655
  }
501
1656
  job.step = 'committing'
502
- const committed = await commitCuratorOutput(output, { job })
1657
+ // 生产调用必须传会话语料(§4.6.6):证据 quote 要能在本次策划的转录里回放,
1658
+ // 否则一律 pending。transcripts 已在 condenseEvents 里脱敏。
1659
+ const committed = await commitCuratorOutput(output, {
1660
+ job,
1661
+ sessionCorpus: buildEvidenceCorpus(sessions.map((s) => ({ id: s.id, transcript: s.transcript }))),
1662
+ })
503
1663
  job.status = 'completed'
504
1664
  job.endedAt = new Date().toISOString()
505
1665
  job.result = {
@@ -508,9 +1668,10 @@ export function startGlobalCuration(ctx, { sessionIds, parentSessionId, model }
508
1668
  created: committed.created,
509
1669
  merged: committed.merged,
510
1670
  superseded: committed.superseded,
1671
+ linked: committed.linked || 0,
511
1672
  skipped: committed.skipped,
512
1673
  }
513
- log(`done: ${committed.created} created, ${committed.merged} merged, ${committed.superseded} superseded`)
1674
+ log(`done: ${committed.created} created, ${committed.merged} merged, ${committed.superseded} superseded, ${committed.linked || 0} linked`)
514
1675
  } catch (error) {
515
1676
  const aborted = abort.signal.aborted || /cancelled by user/.test(String(error?.message || ''))
516
1677
  job.status = aborted ? 'cancelled' : 'error'