dsh-memu-memory 1.2.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/README.md +12 -5
- package/README.zh.md +4 -2
- package/docs/contributing.md +16 -3
- package/docs/host-api.md +3 -1
- package/docs/memory-semantics.md +21 -2
- package/docs/troubleshooting.md +42 -4
- package/lib/backup.js +24 -2
- package/lib/binding-diagnostics.js +2 -2
- package/lib/client.js +1361 -146
- package/lib/consolidation-worker.js +30 -8
- package/lib/curator-evidence.js +131 -0
- package/lib/curator-policy.js +81 -0
- package/lib/curator.js +1252 -91
- package/lib/cycle.js +3 -0
- package/lib/embedding-download.js +39 -4
- package/lib/engine.js +6 -3
- package/lib/global-graph.js +282 -0
- package/lib/graph.js +51 -9
- package/lib/inbound.js +19 -0
- package/lib/inject.js +277 -17
- package/lib/lexicon.js +1115 -0
- package/lib/local-vector-service.js +8 -6
- package/lib/managed-runtime.js +33 -2
- package/lib/memory.js +282 -17
- package/lib/memu.js +2 -2
- package/lib/migrate.js +83 -2
- package/lib/private-links.js +587 -0
- package/lib/private.js +32 -3
- package/lib/prompts/global-curator.md +76 -19
- package/lib/prompts/private-consolidation.coding.md +4 -1
- package/lib/prompts/private-consolidation.md +3 -1
- package/lib/prompts/private-consolidation.mixed.md +3 -1
- package/lib/prompts/private-consolidation.rp.md +3 -1
- package/lib/prompts.js +1 -1
- package/lib/rpc.js +577 -9
- package/lib/state.js +4 -1
- package/lib/text-retrieval.js +661 -0
- package/lib/text-similarity.js +36 -18
- package/lib/tools.js +168 -10
- package/lib/webui/standalone.css +17 -0
- package/lib/webui/standalone.js +2 -0
- package/package.json +1 -1
- package/scripts/acceptance-local.mjs +8 -5
- package/scripts/curator-pair-calibrate.mjs +259 -0
- package/scripts/lexicon-calibrate.mjs +230 -0
- package/scripts/lexicon-dedupe.mjs +318 -0
- package/tests/absorb-review.test.mjs +353 -0
- package/tests/appendix-b2.test.mjs +2 -2
- package/tests/client-export.test.mjs +520 -0
- package/tests/client-render.test.mjs +19 -3
- package/tests/consolidation-fork.test.mjs +14 -12
- package/tests/consolidation-snapshot.test.mjs +39 -0
- package/tests/curator-absorb.test.mjs +443 -0
- package/tests/curator-calibrate.test.mjs +173 -0
- package/tests/curator-clusters.test.mjs +281 -0
- package/tests/curator-commit.test.mjs +52 -5
- package/tests/curator-evidence.test.mjs +102 -0
- package/tests/curator-link.test.mjs +316 -0
- package/tests/curator-pairing.test.mjs +252 -0
- package/tests/curator-probe.test.mjs +15 -6
- package/tests/curator-review-mode.test.mjs +314 -0
- package/tests/disclosure.test.mjs +2 -1
- package/tests/fixtures/curator-pairs/labeled.v1.jsonl +90 -0
- package/tests/fixtures/lexicon-stub-hooks.mjs +15 -0
- package/tests/fixtures/lexicon-stub.mjs +284 -0
- package/tests/graph-cascade.test.mjs +35 -2
- package/tests/graph.test.mjs +96 -0
- package/tests/help-page.test.mjs +415 -0
- package/tests/inbound.test.mjs +6 -3
- package/tests/inject-guide.test.mjs +28 -0
- package/tests/inject.test.mjs +157 -5
- package/tests/lexicon-calibrate.test.mjs +48 -0
- package/tests/lexicon-dedupe.test.mjs +171 -0
- package/tests/lexicon-frequency.test.mjs +179 -0
- package/tests/lexicon-inject.test.mjs +410 -0
- package/tests/lexicon-perf.test.mjs +436 -0
- package/tests/lexicon-ui.test.mjs +547 -0
- package/tests/lexicon.test.mjs +922 -0
- package/tests/managed-runtime.test.mjs +32 -0
- package/tests/private-links.test.mjs +445 -0
- package/tests/prompts.test.mjs +35 -0
- package/tests/recall-dedup.test.mjs +33 -4
- package/tests/tool-schema.test.mjs +1 -1
- package/tests/tools.test.mjs +112 -3
- package/tests/webui-auth.test.mjs +348 -2
- package/tests/webui.test.mjs +188 -0
package/lib/curator.js
CHANGED
|
@@ -1,11 +1,48 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
1
2
|
import {
|
|
2
3
|
globalDbPath, loadGlobalIndex, updateGlobalIndex, newMemoryId, getSettings, effectiveVectorRoute,
|
|
3
4
|
} from './state.js'
|
|
4
5
|
import { commit, forget } from './memu.js'
|
|
5
6
|
import { loadPrompt } from './prompts.js'
|
|
6
|
-
import { retryStoreWrite, redactSecrets, warn } from './paths.js'
|
|
7
|
+
import { retryStoreWrite, redactSecrets, warn, isSafeId } from './paths.js'
|
|
7
8
|
import { record } from './activity-log.js'
|
|
8
9
|
import { activeEntryStamp, assertVectorWriteAllowed, hasActiveVectorGeneration, readVectorState, withVectorWrite } from './vector-profile.js'
|
|
10
|
+
import { upsertLexiconTerm } from './lexicon.js'
|
|
11
|
+
import { clean, bigrams, dice, similarTexts, DUP_REJECT_DICE, TOPIC_DICE } from './text-similarity.js'
|
|
12
|
+
import { estTokens } from './budget.js'
|
|
13
|
+
// —— 包 F1(spec §4.1-§4.9/§6.6):候选检索、证据落地、全局 link ——
|
|
14
|
+
import {
|
|
15
|
+
buildCandidateClusters,
|
|
16
|
+
buildCorpus,
|
|
17
|
+
bm25,
|
|
18
|
+
globalConflictCheck,
|
|
19
|
+
pairFeatures,
|
|
20
|
+
pairScore,
|
|
21
|
+
rankPairCandidates,
|
|
22
|
+
tokenize,
|
|
23
|
+
} from './text-retrieval.js'
|
|
24
|
+
import { buildEvidenceCorpus, groundEvidence } from './curator-evidence.js'
|
|
25
|
+
import { applyGlobalLinks, validateGlobalLink, appendAudit } from './global-graph.js'
|
|
26
|
+
import {
|
|
27
|
+
ABSORB_ABSTRACTIVE_COVERAGE,
|
|
28
|
+
ABSORB_AUTO_SOURCE_COVERAGE,
|
|
29
|
+
ABSORB_AUTO_UNION_COVERAGE,
|
|
30
|
+
ABSORB_MAX_CHARS,
|
|
31
|
+
ABSORB_MAX_SOURCES,
|
|
32
|
+
ABSORB_MIN_LENGTH_RATIO,
|
|
33
|
+
ABSORB_REVIEW_SOURCE_COVERAGE,
|
|
34
|
+
ABSORB_REVIEW_UNION_COVERAGE,
|
|
35
|
+
CLUSTER_MIN_PAIR,
|
|
36
|
+
CURATOR_CLUSTER_TOKENS,
|
|
37
|
+
CURATOR_INDEX_MAX_ENTRIES,
|
|
38
|
+
CURATOR_INDEX_TOKENS,
|
|
39
|
+
CURATOR_MAX_LINK_ABSORB,
|
|
40
|
+
EVIDENCE_MAX,
|
|
41
|
+
PAIR_ABSORB_MIN,
|
|
42
|
+
PAIR_DUPLICATE_MIN,
|
|
43
|
+
PAIR_LINK_MIN,
|
|
44
|
+
PAIR_SCAN_LIMIT,
|
|
45
|
+
} from './curator-policy.js'
|
|
9
46
|
|
|
10
47
|
const MAX_SESSIONS = 10
|
|
11
48
|
const MAX_TOTAL_CHARS = 90_000
|
|
@@ -97,20 +134,98 @@ export function condenseEvents(events, maxChars = MAX_SESSION_CHARS) {
|
|
|
97
134
|
return joined
|
|
98
135
|
}
|
|
99
136
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
137
|
+
// —— §4.5 curator 上下文打包(包 F1)——
|
|
138
|
+
// 旧 compactGlobalIndex 是"插入顺序前 200 条 ×600 字"的平铺,第 201 条后永远不可见、
|
|
139
|
+
// 无预算。这里改为:BM25(会话文本) top80 → 追加最近 30 / universal 10 / pending 10,
|
|
140
|
+
// 去重后按 BM25 desc → updatedAt desc → id 排序,estTokens 累计超预算即停(丢低分,
|
|
141
|
+
// 不截断 JSON)。同输入同输出,不依赖对象插入顺序。
|
|
142
|
+
function renderIndexEntry(id, entry, contentCap) {
|
|
143
|
+
const links = Array.isArray(entry?.links) ? entry.links : []
|
|
144
|
+
const item = {
|
|
145
|
+
id,
|
|
146
|
+
track: entry?.track || 'memory',
|
|
147
|
+
domain: entry?.domain || 'unknown',
|
|
148
|
+
applies_to: Array.isArray(entry?.appliesTo) ? entry.appliesTo.slice(0, 20) : [],
|
|
149
|
+
title: entry?.title || '',
|
|
150
|
+
content: truncate(entry?.content || '', contentCap),
|
|
151
|
+
confidence: entry?.confidence ?? null,
|
|
152
|
+
status: entry?.status || 'active',
|
|
153
|
+
updatedAt: entry?.updatedAt || null,
|
|
154
|
+
linkCount: links.length,
|
|
155
|
+
linkedTo: links.slice(0, 5).map((l) => ({ to: l?.to || null, kind: l?.kind || null })),
|
|
156
|
+
}
|
|
157
|
+
if (Array.isArray(entry?.absorbedFrom) && entry.absorbedFrom.length) item.absorbedFrom = entry.absorbedFrom
|
|
158
|
+
return item
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export function packCuratorIndexContext({ index, sessionText, tokenBudget = CURATOR_INDEX_TOKENS, maxEntries = CURATOR_INDEX_MAX_ENTRIES } = {}) {
|
|
162
|
+
const pool = []
|
|
163
|
+
for (const [id, entry] of Object.entries(index?.entries || {})) {
|
|
164
|
+
if (!entry || entry.status === 'archived') continue
|
|
165
|
+
if (entry.reviewKind) continue // 本版没有 link 伪条目;防御性过滤
|
|
166
|
+
pool.push({ id, entry, updatedAt: String(entry.updatedAt || ''), score: 0 })
|
|
167
|
+
}
|
|
168
|
+
if (pool.length) {
|
|
169
|
+
const corpus = buildCorpus(pool.map((p) => ({ ...p.entry, id: p.id })))
|
|
170
|
+
const query = tokenize(sessionText || '')
|
|
171
|
+
for (const p of pool) {
|
|
172
|
+
const doc = corpus.docs.get(p.id)
|
|
173
|
+
p.score = doc ? bm25(query, doc, corpus) : 0
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
const byScore = [...pool].sort((a, b) => b.score - a.score || b.updatedAt.localeCompare(a.updatedAt) || (a.id < b.id ? -1 : 1))
|
|
177
|
+
const byRecent = [...pool].sort((a, b) => b.updatedAt.localeCompare(a.updatedAt) || (a.id < b.id ? -1 : 1))
|
|
178
|
+
const selected = new Map()
|
|
179
|
+
const take = (list) => { for (const p of list) if (!selected.has(p.id)) selected.set(p.id, p) }
|
|
180
|
+
take(byScore.slice(0, 80)) // BM25 top 80
|
|
181
|
+
take(byRecent.slice(0, 30)) // 最近 30
|
|
182
|
+
take(byRecent.filter((p) => (p.entry?.domain || '') === 'universal').slice(0, 10)) // universal 10
|
|
183
|
+
take(pool.filter((p) => (p.entry?.status || '') === 'pending_review' && (p.entry?.track || 'memory') === 'memory')
|
|
184
|
+
.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt) || (a.id < b.id ? -1 : 1)).slice(0, 10)) // pending_review 记忆 10
|
|
185
|
+
|
|
186
|
+
const ordered = [...selected.values()].sort((a, b) => b.score - a.score || b.updatedAt.localeCompare(a.updatedAt) || (a.id < b.id ? -1 : 1))
|
|
187
|
+
const entries = []
|
|
188
|
+
let tokens = 0
|
|
189
|
+
for (let i = 0; i < ordered.length && entries.length < maxEntries; i++) {
|
|
190
|
+
const item = renderIndexEntry(ordered[i].id, ordered[i].entry, i < 20 ? 900 : 600)
|
|
191
|
+
const cost = estTokens(JSON.stringify(item))
|
|
192
|
+
if (entries.length > 0 && tokens + cost > tokenBudget) break
|
|
193
|
+
entries.push(item)
|
|
194
|
+
tokens += cost
|
|
195
|
+
}
|
|
196
|
+
return { entries, tokens, omitted: Math.max(0, ordered.length - entries.length), includedIds: entries.map((e) => e.id) }
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// candidate_clusters 的 prompt 打包(§4.5):预算 2000 tokens,簇内正文截 300 字,
|
|
200
|
+
// pair_scores 保留 2 位;簇已按 avg_score 降序,超预算丢低分簇。
|
|
201
|
+
export function packCuratorClusterContext(clusterResult, { tokenBudget = CURATOR_CLUSTER_TOKENS } = {}) {
|
|
202
|
+
const clusters = []
|
|
203
|
+
let tokens = 0
|
|
204
|
+
for (const cluster of clusterResult?.clusters || []) {
|
|
205
|
+
const rendered = {
|
|
206
|
+
cluster_id: cluster.cluster_id,
|
|
207
|
+
suggested_op: cluster.suggested_op,
|
|
208
|
+
medoid_id: cluster.medoid_id,
|
|
209
|
+
avg_score: cluster.avg_score,
|
|
210
|
+
pair_scores: cluster.pair_scores,
|
|
211
|
+
members: (cluster.members || []).map((m) => ({
|
|
212
|
+
id: m.id, title: m.title || '', track: m.track || 'memory',
|
|
213
|
+
domain: m.domain || 'unknown', updatedAt: m.updatedAt || null,
|
|
214
|
+
content: truncate(m.content || '', 300),
|
|
215
|
+
})),
|
|
216
|
+
}
|
|
217
|
+
const cost = estTokens(JSON.stringify(rendered))
|
|
218
|
+
if (clusters.length > 0 && tokens + cost > tokenBudget) break
|
|
219
|
+
clusters.push(rendered)
|
|
220
|
+
tokens += cost
|
|
221
|
+
}
|
|
222
|
+
const total = clusterResult?.clusters?.length || 0
|
|
223
|
+
return {
|
|
224
|
+
clusters,
|
|
225
|
+
tokens,
|
|
226
|
+
omitted: Math.max(0, total - clusters.length),
|
|
227
|
+
truncated: (clusterResult?.truncated === true) || (total - clusters.length) > 0,
|
|
228
|
+
}
|
|
114
229
|
}
|
|
115
230
|
|
|
116
231
|
async function readSession(sessionQuery, id) {
|
|
@@ -251,25 +366,238 @@ async function compensateCuratorCommit(originalError, { files, route }) {
|
|
|
251
366
|
}
|
|
252
367
|
}
|
|
253
368
|
|
|
254
|
-
//
|
|
255
|
-
//
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
if (
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
369
|
+
// 词典 op 识别(§2.1):op 显式标 'lexicon',或任何带 libraryId / term+kind 的提案。
|
|
370
|
+
// 全局 curator 的输出里出现这些字段只能说明模型越权,拒绝而不是猜它的意图。
|
|
371
|
+
function isLexiconOp(memory) {
|
|
372
|
+
if (!memory || typeof memory !== 'object') return false
|
|
373
|
+
if (memory.op === 'lexicon') return true
|
|
374
|
+
if (typeof memory.libraryId === 'string' && memory.libraryId.trim()) return true
|
|
375
|
+
return typeof memory.term === 'string' && memory.term.trim() && ['slang', 'meme', 'trigger', 'term'].includes(memory.kind)
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* 私有整理链路的词典写入(§2.1-§2.2):validated ops 白名单校验 + 落库。
|
|
380
|
+
*
|
|
381
|
+
* 锁顺序(刻意注释,谁改谁读):本函数先拿 curator 侧的 managed 写闸
|
|
382
|
+
* (withManagedCuratorWrite —— 受管向量代际一致性),**在其内部**再让
|
|
383
|
+
* upsertLexiconTerm 拿词典自己的 withFileLock。两把锁的嵌套只有一个方向:
|
|
384
|
+
* curator(vector) → lexicon(file);lexicon.js 从不回头拿向量锁,所以无死锁环。
|
|
385
|
+
* 词典写从不跨 await 持锁——withFileLock 的 action 是同步的。
|
|
386
|
+
*
|
|
387
|
+
* 校验规则(§2.1 curator 行 + 契约):
|
|
388
|
+
* * 只接受 { op:'lexicon', action:'upsert', ... };action 其它值(尤其 delete)
|
|
389
|
+
* 一律拒绝——删除是用户/管理操作,模型没有删除路径。
|
|
390
|
+
* * 必须 evidence(quote 或 id 至少其一非空),无证据直接拒绝;
|
|
391
|
+
* "同一转录同词 ≥2 次才建议"是提示词层的事,代码层只验证据存在。
|
|
392
|
+
* * libraryId 必须显式给出且过 isSafeId;op 里带了别的 libraryId 视为越权拒绝。
|
|
393
|
+
* * needs_review === true → status:'pending',否则 'active'。
|
|
394
|
+
* * sensitive trigger 由 lexicon.js 的 LEXICON_SENSITIVE_FORBIDDEN 兜底(curator
|
|
395
|
+
* 来源理论上不该提,但来源字段在模型手里,存储层必须自己设防)。
|
|
396
|
+
*/
|
|
397
|
+
export async function commitPrivateLexiconOps(ops, { libraryId, job } = {}) {
|
|
398
|
+
const list = Array.isArray(ops) ? ops : [ops]
|
|
399
|
+
const committed = []
|
|
400
|
+
const rejected = []
|
|
262
401
|
const skipped = []
|
|
263
|
-
|
|
264
|
-
|
|
402
|
+
if (!isSafeId(libraryId)) {
|
|
403
|
+
for (const raw of list) rejected.push({ term: raw?.term || null, code: 'LEXICON_INVALID_LIBRARY_ID', reason: 'lexicon ops need a valid private library id' })
|
|
404
|
+
return { committed, rejected, skipped }
|
|
405
|
+
}
|
|
406
|
+
const run = async () => {
|
|
407
|
+
for (const raw of list) {
|
|
408
|
+
const op = raw && typeof raw === 'object' ? raw : null
|
|
409
|
+
if (!op || op.op !== 'lexicon') {
|
|
410
|
+
skipped.push({ term: op?.term || null, reason: 'not a lexicon op (op must be "lexicon")' })
|
|
411
|
+
continue
|
|
412
|
+
}
|
|
413
|
+
if (op.action !== 'upsert') {
|
|
414
|
+
rejected.push({ term: op.term || null, code: 'LEXICON_ACTION_FORBIDDEN', reason: `lexicon action "${op.action}" is not allowed for consolidation; only upsert, never delete` })
|
|
415
|
+
continue
|
|
416
|
+
}
|
|
417
|
+
const evidence = Array.isArray(op.evidence)
|
|
418
|
+
? op.evidence.filter(item => item && typeof item === 'object' && ((typeof item.quote === 'string' && item.quote.trim()) || (typeof item.id === 'string' && item.id.trim())))
|
|
419
|
+
: []
|
|
420
|
+
if (!evidence.length) {
|
|
421
|
+
rejected.push({ term: op.term || null, code: 'LEXICON_EVIDENCE_REQUIRED', reason: 'lexicon upserts need evidence from the transcript' })
|
|
422
|
+
continue
|
|
423
|
+
}
|
|
424
|
+
if (typeof op.libraryId === 'string' && op.libraryId && op.libraryId !== libraryId) {
|
|
425
|
+
rejected.push({ term: op.term || null, code: 'LEXICON_LIBRARY_MISMATCH', reason: `lexicon op targets library ${op.libraryId}, but this consolidation is bound to ${libraryId}` })
|
|
426
|
+
continue
|
|
427
|
+
}
|
|
428
|
+
// 敏感触发词不允许模型猜(§0.2/§2.1):curator 也是模型,整理链路一律不给
|
|
429
|
+
// sensitive trigger(trigger 缺省就是 sensitive,除非显式降为 normal)。
|
|
430
|
+
if (op.kind === 'trigger' && op.sensitivity !== 'normal') {
|
|
431
|
+
rejected.push({ term: op.term || null, code: 'LEXICON_SENSITIVE_FORBIDDEN', reason: 'consolidation cannot create sensitive triggers; sensitive vocabulary needs an explicit user write' })
|
|
432
|
+
continue
|
|
433
|
+
}
|
|
434
|
+
try {
|
|
435
|
+
const result = await upsertLexiconTerm({
|
|
436
|
+
libraryId,
|
|
437
|
+
term: op.term,
|
|
438
|
+
kind: op.kind,
|
|
439
|
+
meaning: op.meaning,
|
|
440
|
+
usage: op.usage,
|
|
441
|
+
aliases: op.aliases,
|
|
442
|
+
examples: op.examples,
|
|
443
|
+
context: op.context,
|
|
444
|
+
knownBy: op.knownBy,
|
|
445
|
+
visibility: op.visibility,
|
|
446
|
+
sensitivity: op.sensitivity,
|
|
447
|
+
confidence: op.confidence,
|
|
448
|
+
evidence,
|
|
449
|
+
source: 'curator',
|
|
450
|
+
status: op.needs_review === true ? 'pending' : 'active',
|
|
451
|
+
})
|
|
452
|
+
committed.push({
|
|
453
|
+
termId: result.term.termId,
|
|
454
|
+
term: result.term.term,
|
|
455
|
+
status: result.term.status,
|
|
456
|
+
created: result.created === true,
|
|
457
|
+
notOwned: result.notOwned === true,
|
|
458
|
+
pendingRevision: result.pendingRevision === true,
|
|
459
|
+
})
|
|
460
|
+
} catch (error) {
|
|
461
|
+
rejected.push({ term: op.term || null, code: error?.code || 'LEXICON_OP_FAILED', reason: String(error?.message || error) })
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
return { committed, rejected, skipped }
|
|
465
|
+
}
|
|
466
|
+
if (job) job.step = 'committing lexicon ops'
|
|
467
|
+
// 先过 curator 侧 managed 写闸,再进词典自己的文件锁(顺序见函数头注释)。
|
|
468
|
+
return withManagedCuratorWrite(run)
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* 审核决策(spec §4.10):硬门 + 风险分。
|
|
473
|
+
*
|
|
474
|
+
* 硬门恒 pending,不因审核模式放宽:conflict(模型自己承认两段会话打架)、
|
|
475
|
+
* 证据为空(可回放性是底线)、merge/supersede/absorb 的 target 不存在。
|
|
476
|
+
* 非法 target 在这里返回 skip——调用方必须跳过该提案并且**不得**用同一份内容
|
|
477
|
+
* 继续 create(旧代码缺 continue,会把失败的合并悄悄变成一条重复记忆,§0.2)。
|
|
478
|
+
*
|
|
479
|
+
* 风险分只拦"会改写既有内容"的操作:merge/supersede/absorb 在 auto 下风险
|
|
480
|
+
* ≥0.70 才 pending;create 默认通过,needs_review/低置信只记进 risk 供审计与
|
|
481
|
+
* 复核排序。review-all 把全部内容操作都送进复核。风险分上限 2。
|
|
482
|
+
*
|
|
483
|
+
* @returns {{ review: boolean, risk: number, reasons: string[], hard?: boolean } |
|
|
484
|
+
* { skip: true, reason: string }}
|
|
485
|
+
*/
|
|
486
|
+
export function reviewDecision({ m, op, evidence, targetId, index, mode }) {
|
|
487
|
+
// 硬门(恒 pending;不因 mode 放宽)
|
|
488
|
+
if (op === 'conflict') return { review: true, hard: true, risk: 2, reasons: ['conflict'] }
|
|
489
|
+
if (!evidence.length) return { review: true, hard: true, risk: 2, reasons: ['missing evidence'] }
|
|
490
|
+
if (op === 'merge' || op === 'supersede' || op === 'absorb') {
|
|
491
|
+
if (!targetId || !index.entries[targetId]) {
|
|
492
|
+
return { skip: true, reason: `target not found: ${targetId || '(empty)'}` }
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
// 风险分
|
|
496
|
+
let risk = 0
|
|
497
|
+
const reasons = []
|
|
498
|
+
if (m?.needs_review === true) { risk += 0.70; reasons.push('needs_review') }
|
|
499
|
+
if (Number(m?.confidence) < 0.50) { risk += 0.70; reasons.push('confidence<0.50') }
|
|
500
|
+
else if (Number(m?.confidence) < 0.60) { risk += 0.20; reasons.push('confidence<0.60') }
|
|
501
|
+
if (op === 'merge' || op === 'supersede') { risk += 0.20; reasons.push(op) }
|
|
502
|
+
if (op === 'absorb' && (m?.source_ids?.length || 0) > 3) { risk += 0.20; reasons.push('absorb>3 sources') }
|
|
503
|
+
if (m?.review_reason) { risk += 0.20; reasons.push('review_reason') }
|
|
504
|
+
if (evidence.length === 1) { risk += 0.10; reasons.push('single evidence') }
|
|
505
|
+
|
|
506
|
+
const rewrite = op === 'merge' || op === 'supersede' || op === 'absorb'
|
|
507
|
+
const review = mode === 'review-all' || (mode === 'auto' && rewrite && risk >= 0.70)
|
|
508
|
+
return { review, risk: Math.min(2, Number(risk.toFixed(2))), reasons }
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
// —— §4.8 去重门/冲突门(包 F1)——
|
|
512
|
+
// 门语料只含 active + pending_review:archived/superseded 不是"已存在的事实",
|
|
513
|
+
// 不该拦 create,也不该成为 best。LLM 只裁决;提交层用同一把 pair 尺子复核。
|
|
514
|
+
// buildCorpus 的约定是条目自带 id/title/content(track/domain/appliesTo 等
|
|
515
|
+
// 随条目一起落在 doc.entry 上),所以这里直接展开索引条目再补 id。
|
|
516
|
+
function buildGateCorpus(index) {
|
|
517
|
+
const rows = []
|
|
518
|
+
for (const [id, entry] of Object.entries(index?.entries || {})) {
|
|
519
|
+
if (!entry || (entry.status !== 'active' && entry.status !== 'pending_review')) continue
|
|
520
|
+
rows.push({ ...entry, id })
|
|
521
|
+
}
|
|
522
|
+
return buildCorpus(rows)
|
|
523
|
+
}
|
|
265
524
|
|
|
525
|
+
const OPS_ALLOWLIST = ['create', 'merge', 'supersede', 'noop', 'conflict', 'link', 'absorb']
|
|
526
|
+
const SCORE_EPSILON = 1e-9
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* 规划阶段(spec §7.2 planCuratorOps):把模型输出拆成 writes / pendings / links /
|
|
530
|
+
* skipped 四类计划;全部语义复核在此完成,提交阶段只做事务。逐条流程:
|
|
531
|
+
* lexicon op → skipped;unknown op → skipped(绝不降级 create);noop → 静默跳过;
|
|
532
|
+
* link → validateGlobalLink 预检(快照索引 + 证据落地),事务内再复核并发面;
|
|
533
|
+
* absorb → skipped 'absorb lands in a later pass'(F2 的 commitCuratorPlans 负责,
|
|
534
|
+
* 它直接从原始输出里取 absorb ops,不受这里的 skip 影响);
|
|
535
|
+
* create/merge/supersede/conflict → 证据落地(§4.6)→ reviewDecision(§4.10)→
|
|
536
|
+
* §4.8 候选门(rankPairCandidates top1):
|
|
537
|
+
* create ≥0.90 同 track/domain → skip 'duplicate of <id>'(pending 同域 →
|
|
538
|
+
* 'duplicate of pending <id>'),warn 日志;
|
|
539
|
+
* create conflict(pairDecision 或 globalConflictCheck 预筛,absorb 带不重复查)→
|
|
540
|
+
* pending 'conflict detected';
|
|
541
|
+
* create 0.82-0.90 → duplicateOf+relatedTo;0.55-0.82 → relatedTo;
|
|
542
|
+
* merge/supersede best 是另一条且高 ≥0.05 → pending 'closest target differs';
|
|
543
|
+
* conflict 恒 pending,target 缺失时用 best。
|
|
544
|
+
*/
|
|
545
|
+
export function planCuratorOps(memories, { index, mode, corpus, sessionCorpus = null, now }) {
|
|
546
|
+
const writes = []
|
|
547
|
+
const pendings = []
|
|
548
|
+
const links = []
|
|
549
|
+
const skipped = []
|
|
550
|
+
const entries = index?.entries || {}
|
|
266
551
|
for (const m of memories) {
|
|
267
|
-
//
|
|
268
|
-
//
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
552
|
+
// 词典 op 是私有整理的专属面(§2.1):全局 curator 只写全局记忆。带 libraryId /
|
|
553
|
+
// lexicon 标记的条目在这里一律拒绝——不进向量提交、不进全局索引,只在 skipped 里
|
|
554
|
+
// 留下可读的原因。私有写入走 commitPrivateLexiconOps。
|
|
555
|
+
if (isLexiconOp(m)) {
|
|
556
|
+
skipped.push({ title: m?.term || m?.title, reason: 'lexicon op rejected: the global curator only writes global memory; lexicon ops belong to the private consolidation chain' })
|
|
557
|
+
continue
|
|
558
|
+
}
|
|
559
|
+
// 未知 op 一律拒绝(§4.8):allowlist 之外的 op 不得静默降级成 create——
|
|
560
|
+
// 模型写错 op 名时,降级会凭空写条目。'noop' 是模型的"无可奉告",直接跳过。
|
|
561
|
+
const op = OPS_ALLOWLIST.includes(m?.op) ? m.op : null
|
|
272
562
|
if (op === 'noop') continue
|
|
563
|
+
if (!op) {
|
|
564
|
+
skipped.push({ title: m?.title, reason: `unknown op: ${m?.op ?? '(missing)'}` })
|
|
565
|
+
continue
|
|
566
|
+
}
|
|
567
|
+
// —— link 分支(§4.9,包 F1):预检通过才进 links 计划;落库在提交事务里,
|
|
568
|
+
// 与普通 updates 同一次 updateGlobalIndex;不进 files、不触发向量提交。
|
|
569
|
+
if (op === 'link') {
|
|
570
|
+
const verdict = validateGlobalLink({
|
|
571
|
+
index,
|
|
572
|
+
fromId: m?.from_id,
|
|
573
|
+
toId: m?.to_id,
|
|
574
|
+
kind: m?.kind,
|
|
575
|
+
note: m?.note,
|
|
576
|
+
confidence: m?.confidence,
|
|
577
|
+
evidence: Array.isArray(m?.evidence) ? m.evidence : [],
|
|
578
|
+
sessionCorpus,
|
|
579
|
+
})
|
|
580
|
+
if (!verdict.ok) {
|
|
581
|
+
skipped.push({ title: m?.note || m?.to_id || m?.title, reason: `link rejected: ${verdict.reason}` })
|
|
582
|
+
continue
|
|
583
|
+
}
|
|
584
|
+
links.push({ fromId: verdict.fromId, toId: verdict.toId, kind: verdict.kind, symmetric: verdict.symmetric, link: verdict.link })
|
|
585
|
+
continue
|
|
586
|
+
}
|
|
587
|
+
// absorb 提交属于 F2 的 commitCuratorPlans(从原始 output 自取 absorb ops);
|
|
588
|
+
// 这里显式跳过——planCuratorOps 不把 absorb 当成 create 走内容提交。
|
|
589
|
+
// 注意:absorb 已接线(commitCuratorOutput 里调用 commitCuratorPlans),
|
|
590
|
+
// 这个 skip 只是内容半边的路由,不是"未实现"。
|
|
591
|
+
if (op === 'absorb') {
|
|
592
|
+
skipped.push({ title: m?.title, reason: 'absorb handled by commitCuratorPlans' })
|
|
593
|
+
continue
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
// 证据落地(§4.6):quote 必须能回放到本次策划的会话转录;无 corpus(测试/直呼)
|
|
597
|
+
// 时只做形状校验,status='unverified'。有效证据为空是硬门(reviewDecision)。
|
|
598
|
+
const grounded = groundEvidence(Array.isArray(m?.evidence) ? m.evidence : [], sessionCorpus)
|
|
599
|
+
const evidence = grounded.valid.slice(0, EVIDENCE_MAX)
|
|
600
|
+
.map((item) => ({ sessionId: item.sessionId, quote: truncate(item.quote, 500) }))
|
|
273
601
|
const domain = String(m?.domain || '').trim()
|
|
274
602
|
if (!validDomain(domain)) {
|
|
275
603
|
skipped.push({ title: m?.title, reason: `invalid domain: ${domain}` })
|
|
@@ -283,21 +611,73 @@ export async function commitCuratorOutput(output, { job }) {
|
|
|
283
611
|
continue
|
|
284
612
|
}
|
|
285
613
|
const targetId = String(m?.target_id || '').trim()
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
// be gone. Held entries always get their own id; only a clean merge writes
|
|
293
|
-
// in place.
|
|
294
|
-
id = newMemoryId('gmem')
|
|
295
|
-
} else {
|
|
296
|
-
if (op === 'merge' || op === 'supersede') {
|
|
297
|
-
skipped.push({ title, reason: `target not found: ${targetId || '(empty)'}` })
|
|
298
|
-
}
|
|
299
|
-
id = newMemoryId('gmem')
|
|
614
|
+
const decision = reviewDecision({ m, op, evidence, targetId, index, mode })
|
|
615
|
+
if (decision.skip) {
|
|
616
|
+
// 非法 merge/supersede target 到此为止(§0.2 修复):skipped + continue,
|
|
617
|
+
// 绝不带着同一份内容继续 create。
|
|
618
|
+
skipped.push({ title, reason: decision.reason })
|
|
619
|
+
continue
|
|
300
620
|
}
|
|
621
|
+
|
|
622
|
+
// —— §4.8 候选复核门:每次规划用同一语料、同一把尺子。——
|
|
623
|
+
const proposal = {
|
|
624
|
+
id: '__curator_proposal__',
|
|
625
|
+
title, content, track, domain,
|
|
626
|
+
appliesTo: Array.isArray(m?.applies_to) ? m.applies_to : [],
|
|
627
|
+
}
|
|
628
|
+
const candidates = corpus && corpus.size ? rankPairCandidates({ corpus, proposal, limit: PAIR_SCAN_LIMIT }) : []
|
|
629
|
+
const best = candidates[0] || null
|
|
630
|
+
let review = decision.review
|
|
631
|
+
// reviewReason 优先级:硬门 > 候选门(conflict detected / closest target differs)
|
|
632
|
+
// > 模型 review_reason > 'risk threshold'。
|
|
633
|
+
let reviewReason = decision.hard
|
|
634
|
+
? decision.reasons[0]
|
|
635
|
+
: (String(m?.review_reason || '').trim() || 'risk threshold')
|
|
636
|
+
let conflictingWith = targetId || null
|
|
637
|
+
const tags = {}
|
|
638
|
+
if (op === 'create' && best) {
|
|
639
|
+
if (best.score >= PAIR_DUPLICATE_MIN
|
|
640
|
+
&& (best.entry?.track || 'memory') === track
|
|
641
|
+
&& (best.entry?.domain || 'unknown') === domain) {
|
|
642
|
+
const isPending = best.entry?.status === 'pending_review'
|
|
643
|
+
warn(`curator: create skipped, ${isPending ? 'duplicate of pending' : 'duplicate of'} ${best.id} (pair score ${best.score.toFixed(2)})`)
|
|
644
|
+
skipped.push({ title, reason: isPending ? `duplicate of pending ${best.id}` : `duplicate of ${best.id}` })
|
|
645
|
+
continue
|
|
646
|
+
}
|
|
647
|
+
// 冲突检测:pairDecision 的 conflict(同标题正文迥异),或 §4.8 预筛
|
|
648
|
+
// (negation/value/same_title)。duplicate/absorb 带(≥0.82)不重复查——
|
|
649
|
+
// 同标题高相似是同一事实的改写,不是冲突(§4.8 豁免条款的推广)。
|
|
650
|
+
const conflictHit = best.decision === 'conflict' || (best.score < PAIR_ABSORB_MIN
|
|
651
|
+
&& globalConflictCheck(proposal, best.entry, best.features))
|
|
652
|
+
if (conflictHit) {
|
|
653
|
+
review = true
|
|
654
|
+
reviewReason = 'conflict detected'
|
|
655
|
+
conflictingWith = best.id
|
|
656
|
+
} else if (best.score >= PAIR_ABSORB_MIN) {
|
|
657
|
+
tags.duplicateOf = best.id
|
|
658
|
+
tags.relatedTo = best.id
|
|
659
|
+
} else if (best.score >= PAIR_LINK_MIN) {
|
|
660
|
+
tags.relatedTo = best.id
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
if ((op === 'merge' || op === 'supersede') && best) {
|
|
664
|
+
// best 是另一条且分数比模型 target 高 ≥0.05 → 最近目标存疑,转人审。
|
|
665
|
+
const targetEntry = entries[targetId]
|
|
666
|
+
const targetScore = targetEntry ? pairScore(pairFeatures(proposal, targetEntry)) : 0
|
|
667
|
+
if (best.id !== targetId && best.score >= targetScore + 0.05 - SCORE_EPSILON) {
|
|
668
|
+
review = true
|
|
669
|
+
reviewReason = 'closest target differs'
|
|
670
|
+
conflictingWith = targetId
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
if (op === 'conflict' && !targetId && best) conflictingWith = best.id
|
|
674
|
+
|
|
675
|
+
const previous = (op === 'merge' || op === 'supersede') && targetId && entries[targetId] ? entries[targetId] : null
|
|
676
|
+
// A reviewed (held) entry must never occupy the target's id: rejecting the
|
|
677
|
+
// review would archive the only copy and the original active memory would
|
|
678
|
+
// be gone. Held entries always get their own id; only a clean merge writes
|
|
679
|
+
// in place.
|
|
680
|
+
const id = newMemoryId('gmem')
|
|
301
681
|
const entry = {
|
|
302
682
|
track,
|
|
303
683
|
domain,
|
|
@@ -306,35 +686,141 @@ export async function commitCuratorOutput(output, { job }) {
|
|
|
306
686
|
content: redactSecrets(content),
|
|
307
687
|
confidence: Number.isFinite(Number(m?.confidence)) ? Math.max(0, Math.min(1, Number(m.confidence))) : 0.6,
|
|
308
688
|
evidence: evidence.map((e) => ({
|
|
309
|
-
sessionId: String(e?.
|
|
689
|
+
sessionId: String(e?.sessionId || ''),
|
|
310
690
|
quote: truncate(e?.quote || '', 500),
|
|
311
691
|
})),
|
|
312
692
|
status: 'active',
|
|
313
693
|
supersedes: op === 'supersede' ? targetId || null : previous?.supersedes || null,
|
|
694
|
+
// 风险分写到所有条目(active 也写,§4.10):它是审计/复核排序的统一口径,
|
|
695
|
+
// 不是 pending 的专属属性。旧条目没有这两个字段时按缺省读取,不需要迁移。
|
|
696
|
+
reviewRisk: decision.risk,
|
|
697
|
+
reviewRiskReasons: decision.reasons,
|
|
698
|
+
// 证据是否通过 grounding(§3.1):unverified(直呼/测试)时为 false。
|
|
699
|
+
evidenceVerified: grounded.status === 'grounded',
|
|
314
700
|
createdAt: previous?.createdAt || now,
|
|
315
701
|
updatedAt: now,
|
|
316
702
|
}
|
|
317
|
-
if (
|
|
703
|
+
if (tags.duplicateOf) entry.duplicateOf = tags.duplicateOf
|
|
704
|
+
if (tags.relatedTo) entry.relatedTo = tags.relatedTo
|
|
705
|
+
if (review) {
|
|
318
706
|
// Held in the index only: never committed to MemU, so a contested claim
|
|
319
707
|
// cannot be recalled as shared memory until someone decides.
|
|
320
708
|
entry.status = 'pending_review'
|
|
321
|
-
entry.reviewReason =
|
|
322
|
-
entry.conflictingWith =
|
|
323
|
-
|
|
709
|
+
entry.reviewReason = reviewReason
|
|
710
|
+
entry.conflictingWith = conflictingWith
|
|
711
|
+
pendings.push({ id, entry })
|
|
324
712
|
continue
|
|
325
713
|
}
|
|
326
|
-
|
|
327
|
-
|
|
714
|
+
writes.push({
|
|
715
|
+
id, entry, op, targetId, previous,
|
|
716
|
+
file: { name: id, track, description: truncate(title, 180), content },
|
|
717
|
+
})
|
|
718
|
+
}
|
|
719
|
+
return { writes, pendings, links, skipped }
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
// Exported for the audit regression suite: merging the commit decision logic
|
|
723
|
+
// into startGlobalCuration would require standing up subagents to test it.
|
|
724
|
+
export async function commitCuratorOutput(output, { job, sessionCorpus = null } = {}) {
|
|
725
|
+
const memories = Array.isArray(output?.memories) ? output.memories : []
|
|
726
|
+
if (memories.length === 0) {
|
|
727
|
+
return { files: [], created: 0, merged: 0, superseded: 0, pending: 0, linked: 0, skipped: [], reviewRisk: { max: 0, flagged: 0, reasons: {} } }
|
|
728
|
+
}
|
|
729
|
+
const index = loadGlobalIndex()
|
|
730
|
+
// 模式只在循环外读一次(§4.10):一次策划运行用同一个口径,中途改设置不影响本批。
|
|
731
|
+
const mode = getSettings().curatorReviewMode || 'auto'
|
|
732
|
+
const gateCorpus = buildGateCorpus(index)
|
|
733
|
+
const plan = planCuratorOps(memories, { index, mode, corpus: gateCorpus, sessionCorpus, now: new Date().toISOString() })
|
|
734
|
+
const contentResult = await commitPlannedOps(plan, { job })
|
|
735
|
+
// absorb 半边(F2):planCuratorOps 对 absorb 显式 skip(避免在内容提交里被误当成
|
|
736
|
+
// create),absorb ops 由 commitCuratorPlans 从原始 output 自取并直接提交。
|
|
737
|
+
const absorbResult = await commitCuratorPlans(output, { job })
|
|
738
|
+
return {
|
|
739
|
+
...contentResult,
|
|
740
|
+
absorbed: absorbResult.absorbed,
|
|
741
|
+
pending: contentResult.pending + absorbResult.pending,
|
|
742
|
+
held: absorbResult.held,
|
|
743
|
+
demoted: absorbResult.demoted,
|
|
744
|
+
skipped: [...contentResult.skipped, ...absorbResult.skipped],
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
/**
|
|
749
|
+
* 提交阶段(spec §7.2 commitCuratorPlans 的内容操作半边;absorb 半边是 F2 的
|
|
750
|
+
* 同名函数,直接吃原始 output)。单次 updateGlobalIndex 事务内按序应用:
|
|
751
|
+
* pendings(索引 only)→ writes(held clone 检查 / merge 兜底 / supersede)→
|
|
752
|
+
* links(applyGlobalLinks,只有 link 时也要走一次索引事务,§4.9)。
|
|
753
|
+
* link 不进 files、不触发向量提交;两档审核模式都自动应用(C2)。
|
|
754
|
+
*/
|
|
755
|
+
async function commitPlannedOps({ writes, pendings, links, skipped }, { job }) {
|
|
756
|
+
const files = writes.map((w) => w.file)
|
|
757
|
+
const now = new Date().toISOString()
|
|
758
|
+
|
|
759
|
+
// 风险统计(§4.10):本批所有写入条目(active + pending)的审计口径——最高分、
|
|
760
|
+
// 达到复核阈值(≥0.70)的条目数、每类原因出现的次数。
|
|
761
|
+
const written = [...writes.map((w) => w.entry), ...pendings.map((p) => p.entry)]
|
|
762
|
+
const riskReasons = {}
|
|
763
|
+
for (const entry of written) {
|
|
764
|
+
for (const reason of entry.reviewRiskReasons || []) riskReasons[reason] = (riskReasons[reason] || 0) + 1
|
|
765
|
+
}
|
|
766
|
+
const reviewRiskSummary = {
|
|
767
|
+
max: written.length ? Math.max(...written.map((entry) => Number(entry.reviewRisk) || 0)) : 0,
|
|
768
|
+
flagged: written.filter((entry) => (Number(entry.reviewRisk) || 0) >= 0.70).length,
|
|
769
|
+
reasons: riskReasons,
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
// 单事务:pendings + writes + links 一次落索引。返回本事务实际落地的 link 列表,
|
|
773
|
+
// 供活动日志逐条记录(retry 重跑时以成功那次的返回为准)。
|
|
774
|
+
const applyPlan = (fresh) => {
|
|
775
|
+
for (const { id, entry } of pendings) fresh.entries[id] = entry
|
|
776
|
+
for (const { id, entry, op, targetId, previous } of writes) {
|
|
777
|
+
if (previous && JSON.stringify(fresh.entries[targetId]) !== JSON.stringify(previous)) {
|
|
778
|
+
const heldId = newMemoryId('gmem')
|
|
779
|
+
// 提交成功的只有 `id` 这一行,held 克隆从未进过向量库——它绝不能继承
|
|
780
|
+
// "提交成功即落戳"的 entry.vector:向量库里的行名是 `id` 而不是 heldId,
|
|
781
|
+
// 带戳会让 membership 把一条不存在的行计进来,而真实存在的 `id` 行反而
|
|
782
|
+
// 无处归属(它根本不会按这个克隆写进索引)。
|
|
783
|
+
const held = { ...entry, status: 'pending_review', conflictingWith: targetId, reviewReason: 'target changed while curator was running' }
|
|
784
|
+
delete held.vector
|
|
785
|
+
fresh.entries[heldId] = held
|
|
786
|
+
continue
|
|
787
|
+
}
|
|
788
|
+
if (op === 'merge' && previous) {
|
|
789
|
+
entry.content = entry.content || previous.content
|
|
790
|
+
entry.title = entry.title || previous.title
|
|
791
|
+
entry.evidence = [...(previous.evidence || []), ...(entry.evidence || [])].slice(-40)
|
|
792
|
+
}
|
|
793
|
+
fresh.entries[id] = entry
|
|
794
|
+
if ((op === 'supersede' || op === 'merge') && targetId && fresh.entries[targetId]) {
|
|
795
|
+
fresh.entries[targetId].status = 'superseded'
|
|
796
|
+
fresh.entries[targetId].supersededBy = id
|
|
797
|
+
fresh.entries[targetId].validTo = now
|
|
798
|
+
fresh.entries[targetId].updatedAt = now
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
return applyGlobalLinks(fresh.entries, links, { now })
|
|
328
802
|
}
|
|
329
803
|
|
|
330
804
|
if (files.length === 0) {
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
805
|
+
let linkResult = { applied: [], skipped: [] }
|
|
806
|
+
if (pendings.length || links.length) {
|
|
807
|
+
// 只有 pending/link 时同样要走一次索引事务(§4.9:只有 link 也要调一次
|
|
808
|
+
// updateGlobalIndex);withManagedCuratorWrite 闸门照过,保持锁纪律一致。
|
|
809
|
+
await withManagedCuratorWrite(() => retryStoreWrite(() => updateGlobalIndex((fresh) => {
|
|
810
|
+
linkResult = applyPlan(fresh)
|
|
811
|
+
})))
|
|
812
|
+
}
|
|
813
|
+
for (const applied of linkResult.applied) {
|
|
814
|
+
record({ kind: 'curate', scope: 'global', ok: true, message: `全局关联已落地:${applied.fromId} -(${applied.kind})-> ${applied.toId}`, detail: { op: 'link', from: applied.fromId, to: applied.toId, kind: applied.kind, linkId: applied.linkId } })
|
|
815
|
+
}
|
|
816
|
+
return {
|
|
817
|
+
files: [], created: 0, merged: 0, superseded: 0, pending: pendings.length,
|
|
818
|
+
linked: linkResult.applied.length, skipped, reviewRisk: reviewRiskSummary,
|
|
819
|
+
}
|
|
335
820
|
}
|
|
336
821
|
|
|
337
822
|
job.step = `committing ${files.length} global memories`
|
|
823
|
+
let linkResult = { applied: [], skipped: [] }
|
|
338
824
|
await withManagedCuratorWrite(async () => {
|
|
339
825
|
// Managed agent mode still commits the exact same files to the active local
|
|
340
826
|
// vector database; only pending_review entries remain index-only.
|
|
@@ -350,37 +836,12 @@ export async function commitCuratorOutput(output, { job }) {
|
|
|
350
836
|
// 整理写入与单条写入同口径:提交成功就在条目上落"已入向量库"戳,否则这批
|
|
351
837
|
// 全局记忆在向量计数(membership 的戳腿)里永远缺失。pending 不提交,不戳。
|
|
352
838
|
const vector = activeEntryStamp(route)
|
|
353
|
-
if (vector) for (const
|
|
839
|
+
if (vector) for (const write of writes) write.entry.vector = vector
|
|
354
840
|
}
|
|
355
841
|
|
|
356
842
|
if (job?.abort?.signal?.aborted) throw new Error('cancelled by user')
|
|
357
|
-
await retryStoreWrite(() => updateGlobalIndex(fresh => {
|
|
358
|
-
|
|
359
|
-
for (const { id, entry, op, targetId, previous } of updates) {
|
|
360
|
-
if (previous && JSON.stringify(fresh.entries[targetId]) !== JSON.stringify(previous)) {
|
|
361
|
-
const heldId = newMemoryId('gmem')
|
|
362
|
-
// 提交成功的只有 `id` 这一行,held 克隆从未进过向量库——它绝不能继承
|
|
363
|
-
// "提交成功即落戳"的 entry.vector:向量库里的行名是 `id` 而不是 heldId,
|
|
364
|
-
// 带戳会让 membership 把一条不存在的行计进来,而真实存在的 `id` 行反而
|
|
365
|
-
// 无处归属(它根本不会按这个克隆写进索引)。
|
|
366
|
-
const held = { ...entry, status: 'pending_review', conflictingWith: targetId, reviewReason: 'target changed while curator was running' }
|
|
367
|
-
delete held.vector
|
|
368
|
-
fresh.entries[heldId] = held
|
|
369
|
-
continue
|
|
370
|
-
}
|
|
371
|
-
if (op === 'merge' && previous) {
|
|
372
|
-
entry.content = entry.content || previous.content
|
|
373
|
-
entry.title = entry.title || previous.title
|
|
374
|
-
entry.evidence = [...(previous.evidence || []), ...(entry.evidence || [])].slice(-40)
|
|
375
|
-
}
|
|
376
|
-
fresh.entries[id] = entry
|
|
377
|
-
if ((op === 'supersede' || op === 'merge') && targetId && fresh.entries[targetId]) {
|
|
378
|
-
fresh.entries[targetId].status = 'superseded'
|
|
379
|
-
fresh.entries[targetId].supersededBy = id
|
|
380
|
-
fresh.entries[targetId].validTo = now
|
|
381
|
-
fresh.entries[targetId].updatedAt = now
|
|
382
|
-
}
|
|
383
|
-
}
|
|
843
|
+
await retryStoreWrite(() => updateGlobalIndex((fresh) => {
|
|
844
|
+
linkResult = applyPlan(fresh)
|
|
384
845
|
}))
|
|
385
846
|
} catch (error) {
|
|
386
847
|
// 提交成功之后(abort 检查 / 索引写)的任何失败都会留下孤儿行:尽力撤销。
|
|
@@ -388,25 +849,707 @@ export async function commitCuratorOutput(output, { job }) {
|
|
|
388
849
|
throw error
|
|
389
850
|
}
|
|
390
851
|
})
|
|
852
|
+
for (const applied of linkResult.applied) {
|
|
853
|
+
record({ kind: 'curate', scope: 'global', ok: true, message: `全局关联已落地:${applied.fromId} -(${applied.kind})-> ${applied.toId}`, detail: { op: 'link', from: applied.fromId, to: applied.toId, kind: applied.kind, linkId: applied.linkId } })
|
|
854
|
+
}
|
|
391
855
|
const summary = {
|
|
392
856
|
files,
|
|
393
|
-
created:
|
|
394
|
-
merged:
|
|
395
|
-
superseded:
|
|
396
|
-
pending:
|
|
857
|
+
created: writes.filter((w) => w.op === 'create').length,
|
|
858
|
+
merged: writes.filter((w) => w.op === 'merge').length,
|
|
859
|
+
superseded: writes.filter((w) => w.op === 'supersede').length,
|
|
860
|
+
pending: pendings.length,
|
|
861
|
+
linked: linkResult.applied.length,
|
|
397
862
|
skipped,
|
|
863
|
+
reviewRisk: reviewRiskSummary,
|
|
398
864
|
}
|
|
399
865
|
record({
|
|
400
866
|
kind: 'curate',
|
|
401
867
|
scope: 'global',
|
|
402
868
|
ok: true,
|
|
403
869
|
message: `全局提炼:新增 ${summary.created} / 合并 ${summary.merged} / 待复核 ${summary.pending}`
|
|
870
|
+
+ (summary.linked ? ` / 关联 ${summary.linked}` : '')
|
|
404
871
|
+ (summary.skipped.length ? ` / 跳过 ${summary.skipped.length}` : ''),
|
|
405
872
|
detail: { ...summary, skipped: summary.skipped.slice(0, 20) },
|
|
406
873
|
refs: files.map((file) => file.name).slice(0, 20),
|
|
407
874
|
})
|
|
408
875
|
return summary
|
|
409
876
|
}
|
|
877
|
+
|
|
878
|
+
// ============================================================================
|
|
879
|
+
// 包 F2(spec §4.7/§4.11/§6.7):absorb 合并稿、提交事务、复核与撤销。
|
|
880
|
+
// ============================================================================
|
|
881
|
+
|
|
882
|
+
// —— absorb 常量(spec §4.12)——单点定义在 lib/curator-policy.js(文件头部 import)。
|
|
883
|
+
// 句级去重的 charDice 阈值只属于合并稿这一处,留在本地。
|
|
884
|
+
const ABSORB_SENTENCE_DICE = 0.9
|
|
885
|
+
|
|
886
|
+
// —— F1 接口(spec §4.1):tokenize/buildCorpus 已由 lib/text-retrieval.js 落地
|
|
887
|
+
//(文件头部 import)。buildCorpus 返回的是 BM25 语料(df/size),这里按 F1 的
|
|
888
|
+
// idfOf 同一公式折算成合并稿 MMR 用的 idf 表。
|
|
889
|
+
function tokensOf(text) {
|
|
890
|
+
return new Set(tokenize(text))
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
function idfMapFromCorpus(corpus) {
|
|
894
|
+
const idf = new Map()
|
|
895
|
+
if (!corpus?.df) return idf
|
|
896
|
+
for (const [term, df] of corpus.df) {
|
|
897
|
+
idf.set(term, Math.log(1 + (corpus.size - df + 0.5) / (df + 0.5)))
|
|
898
|
+
}
|
|
899
|
+
return idf
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
// idf 语料取自当前全局索引的 active 条目(与候选簇同一口径)。
|
|
903
|
+
function absorbIdf(index) {
|
|
904
|
+
const docs = Object.entries(index?.entries || {})
|
|
905
|
+
.filter(([, entry]) => entry && entry.status === 'active')
|
|
906
|
+
.map(([id, entry]) => ({ id, title: entry.title || '', content: entry.content || '' }))
|
|
907
|
+
return idfMapFromCorpus(buildCorpus(docs))
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
// —— absorb 合并稿(spec §4.7)——
|
|
911
|
+
const SENTENCE_RE = /[^。!?!?;;\n]+[。!?!?;;]?/gu
|
|
912
|
+
|
|
913
|
+
function splitSentences(text) {
|
|
914
|
+
const out = []
|
|
915
|
+
for (const raw of String(text || '').match(SENTENCE_RE) || []) {
|
|
916
|
+
const sentence = raw.trim()
|
|
917
|
+
if (sentence) out.push(sentence)
|
|
918
|
+
}
|
|
919
|
+
return out
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
// token 覆盖率:source 的词元有多大比例出现在 content 里。
|
|
923
|
+
function tokenCoverage(sourceTokens, contentTokens) {
|
|
924
|
+
if (!sourceTokens || sourceTokens.size === 0) return 1
|
|
925
|
+
let hit = 0
|
|
926
|
+
for (const token of sourceTokens) if (contentTokens.has(token)) hit += 1
|
|
927
|
+
return hit / sourceTokens.size
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
// char 覆盖率:union 的字符 bigram 有多大比例出现在 proposed 里。
|
|
931
|
+
function charCoverage(unionText, proposedText) {
|
|
932
|
+
const unionGrams = bigrams(unionText)
|
|
933
|
+
if (unionGrams.size === 0) return 1
|
|
934
|
+
const proposedGrams = bigrams(proposedText)
|
|
935
|
+
let hit = 0
|
|
936
|
+
for (const gram of unionGrams) if (proposedGrams.has(gram)) hit += 1
|
|
937
|
+
return hit / unionGrams.size
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
function sentenceInfo(sentence, idf, coveredTokens = null) {
|
|
941
|
+
let sum = 0
|
|
942
|
+
for (const token of tokensOf(sentence)) {
|
|
943
|
+
if (coveredTokens && coveredTokens.has(token)) continue
|
|
944
|
+
sum += Number(idf?.get?.(token) ?? 1)
|
|
945
|
+
}
|
|
946
|
+
return sum
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
/**
|
|
950
|
+
* 吸收合并稿(spec §4.7)。输入 target、sources、模型的 proposed 稿;
|
|
951
|
+
* 输出 { content, mode, coverage, sourceCoverage, unionCoverage, truncated, omitted }。
|
|
952
|
+
* 句级去重(charDice ≥0.90 或互含丢弃)→ union(target 在前)→ proposed 覆盖
|
|
953
|
+
* ≥0.75 且不超长时用 proposed(abstractive),否则 extractive(放得下直接拼接,
|
|
954
|
+
* 放不下做句子级 MMR:info − 0.7·redundancy·maxInfo 贪心,被丢关键句记 truncated)。
|
|
955
|
+
* idf 由调用方通过 buildCorpus 提供(F1);缺省按 1 计(spec 已兼容 idf=null)。
|
|
956
|
+
*/
|
|
957
|
+
export function composeAbsorbContent({ target, sources = [], proposed = '', maxChars = ABSORB_MAX_CHARS, idf = null } = {}) {
|
|
958
|
+
const result = { content: '', mode: 'extractive', coverage: 0, sourceCoverage: 1, unionCoverage: 1, truncated: false, omitted: [] }
|
|
959
|
+
// 1-3. 句子切分 + 句级去重;target 在前,构成 union。origin 记录每个句子的出处
|
|
960
|
+
//(-1 = target,其余 = sources 下标),供关键句判定使用。
|
|
961
|
+
const union = []
|
|
962
|
+
const origins = []
|
|
963
|
+
const keptKeys = []
|
|
964
|
+
const pushSentence = (raw, origin) => {
|
|
965
|
+
const sentence = String(raw || '').trim()
|
|
966
|
+
const key = clean(sentence)
|
|
967
|
+
if (!key) return
|
|
968
|
+
for (const kept of keptKeys) {
|
|
969
|
+
if (kept === key || kept.includes(key) || key.includes(kept) || dice(bigrams(key), bigrams(kept)) >= ABSORB_SENTENCE_DICE) return
|
|
970
|
+
}
|
|
971
|
+
keptKeys.push(key)
|
|
972
|
+
union.push(sentence)
|
|
973
|
+
origins.push(origin)
|
|
974
|
+
}
|
|
975
|
+
const sourceList = Array.isArray(sources) ? sources : []
|
|
976
|
+
for (const sentence of splitSentences(target?.content)) pushSentence(sentence, -1)
|
|
977
|
+
sourceList.forEach((source, index) => {
|
|
978
|
+
for (const sentence of splitSentences(source?.content)) pushSentence(sentence, index)
|
|
979
|
+
})
|
|
980
|
+
if (!union.length) return result
|
|
981
|
+
const unionText = union.join('')
|
|
982
|
+
const proposedText = String(proposed || '').trim()
|
|
983
|
+
|
|
984
|
+
// 4. proposed 覆盖足够且不超长 → abstractive。
|
|
985
|
+
if (proposedText) {
|
|
986
|
+
result.coverage = Math.max(charCoverage(unionText, proposedText), tokenCoverage(tokensOf(unionText), tokensOf(proposedText)))
|
|
987
|
+
if (result.coverage >= ABSORB_ABSTRACTIVE_COVERAGE && proposedText.length <= maxChars) {
|
|
988
|
+
result.content = proposedText
|
|
989
|
+
result.mode = 'abstractive'
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
// 5. extractive:总长放得下直接拼接;否则句子级 MMR 贪心到 maxChars。
|
|
994
|
+
if (result.mode === 'extractive') {
|
|
995
|
+
const infos = union.map((sentence) => sentenceInfo(sentence, idf))
|
|
996
|
+
const maxInfo = infos.reduce((acc, value) => Math.max(acc, value), 0)
|
|
997
|
+
const picked = []
|
|
998
|
+
const covered = new Set()
|
|
999
|
+
let length = 0
|
|
1000
|
+
const remaining = new Set(union.map((_, index) => index))
|
|
1001
|
+
while (remaining.size > 0) {
|
|
1002
|
+
let bestIndex = -1
|
|
1003
|
+
let bestScore = -Infinity
|
|
1004
|
+
for (const index of remaining) {
|
|
1005
|
+
let redundancy = 0
|
|
1006
|
+
for (const chosen of picked) redundancy = Math.max(redundancy, dice(bigrams(union[index]), bigrams(union[chosen])))
|
|
1007
|
+
const score = sentenceInfo(union[index], idf, covered) - 0.7 * redundancy * maxInfo
|
|
1008
|
+
if (score > bestScore) { bestScore = score; bestIndex = index }
|
|
1009
|
+
}
|
|
1010
|
+
remaining.delete(bestIndex)
|
|
1011
|
+
if (length + union[bestIndex].length > maxChars) continue
|
|
1012
|
+
picked.push(bestIndex)
|
|
1013
|
+
length += union[bestIndex].length
|
|
1014
|
+
for (const token of tokensOf(union[bestIndex])) covered.add(token)
|
|
1015
|
+
}
|
|
1016
|
+
if (!picked.length && union.length) picked.push(0)
|
|
1017
|
+
// 关键句:target 全部句子 + 每条 source 的 info 前 3 句;被 MMR 丢掉即 truncated。
|
|
1018
|
+
const byOrigin = new Map()
|
|
1019
|
+
union.forEach((_, index) => {
|
|
1020
|
+
if (!byOrigin.has(origins[index])) byOrigin.set(origins[index], [])
|
|
1021
|
+
byOrigin.get(origins[index]).push(index)
|
|
1022
|
+
})
|
|
1023
|
+
const keyIndices = new Set(byOrigin.get(-1) || [])
|
|
1024
|
+
for (const [origin, indices] of byOrigin) {
|
|
1025
|
+
if (origin === -1) continue
|
|
1026
|
+
for (const index of [...indices].sort((a, b) => infos[b] - infos[a]).slice(0, 3)) keyIndices.add(index)
|
|
1027
|
+
}
|
|
1028
|
+
const missingKeys = [...keyIndices].filter((index) => !picked.includes(index)).sort((a, b) => a - b)
|
|
1029
|
+
if (missingKeys.length) {
|
|
1030
|
+
result.truncated = true
|
|
1031
|
+
result.omitted = missingKeys.slice(0, 10).map((index) => union[index])
|
|
1032
|
+
}
|
|
1033
|
+
result.content = picked.sort((a, b) => a - b).map((index) => union[index]).join('')
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
// 6. 覆盖率:sourceCoverage = 每条 source 的 token 覆盖率取最小;unionCoverage = 全部
|
|
1037
|
+
// source 词元并集的覆盖率。coverage 是 proposed 对 union 的覆盖(§4.7.4)。
|
|
1038
|
+
const contentTokens = tokensOf(result.content)
|
|
1039
|
+
const presentSources = sourceList.filter((source) => String(source?.content || '').trim())
|
|
1040
|
+
if (presentSources.length) {
|
|
1041
|
+
result.sourceCoverage = Math.min(...presentSources.map((source) => tokenCoverage(tokensOf(String(source.content || '')), contentTokens)))
|
|
1042
|
+
const unionSourceTokens = new Set()
|
|
1043
|
+
for (const source of presentSources) {
|
|
1044
|
+
for (const token of tokensOf(String(source.content || ''))) unionSourceTokens.add(token)
|
|
1045
|
+
}
|
|
1046
|
+
result.unionCoverage = tokenCoverage(unionSourceTokens, contentTokens)
|
|
1047
|
+
}
|
|
1048
|
+
return result
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
// absorbOpId(spec §3.1):sha1('absorb|' + sortedIds + '|' + clean(content)).slice(0,16)。
|
|
1052
|
+
// 内容用脱敏后的合并稿,保证从已存条目重算能得到同一个 opId(幂等门靠它对账)。
|
|
1053
|
+
function absorbOpIdOf(sortedIds, content) {
|
|
1054
|
+
return createHash('sha1').update('absorb|' + [...sortedIds].sort().join(',') + '|' + clean(content)).digest('hex').slice(0, 16)
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
function appliesToSubset(sourceApplies, targetApplies) {
|
|
1058
|
+
const target = Array.isArray(targetApplies) ? targetApplies.filter(Boolean) : []
|
|
1059
|
+
if (target.length === 0) return true // target 不限 ⇒ 更宽,任何 source 都算子集
|
|
1060
|
+
const source = Array.isArray(sourceApplies) ? sourceApplies.filter(Boolean) : []
|
|
1061
|
+
return source.every((item) => target.includes(item) || target.includes('all'))
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
const round4 = (value) => Math.round(Number(value) * 10000) / 10000
|
|
1065
|
+
|
|
1066
|
+
/**
|
|
1067
|
+
* absorb 的索引单事务(spec §4.11 auto absorb 步骤 3b-3f)。在 updateGlobalIndex 的
|
|
1068
|
+
* 同步 mutator 里运行;expectedUpdatedAt 任一不匹配时不做任何修改并返回
|
|
1069
|
+
* { landed:false, changed }(调用方决定 held clone 还是抛 'review changed')。
|
|
1070
|
+
* 匹配时:merged 入索引(含出边并入 + absorbSnapshot)、target superseded +
|
|
1071
|
+
* absorbedInto、sources archived + absorbedBy、簇外入边改指 mergedId。
|
|
1072
|
+
*/
|
|
1073
|
+
export function applyAbsorbIndexTransaction(fresh, { mergedId, merged, targetId, sourceIds = [], expectedUpdatedAt = {}, now }) {
|
|
1074
|
+
const entries = fresh.entries
|
|
1075
|
+
const memberIds = [targetId, ...sourceIds].filter(Boolean)
|
|
1076
|
+
const changed = []
|
|
1077
|
+
for (const memberId of memberIds) {
|
|
1078
|
+
const current = entries[memberId]
|
|
1079
|
+
const expected = expectedUpdatedAt[memberId]
|
|
1080
|
+
if (!current || current.status !== 'active' || (expected !== undefined && current.updatedAt !== expected)) {
|
|
1081
|
+
changed.push(memberId || '(missing id)')
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
if (changed.length) return { landed: false, changed }
|
|
1085
|
+
|
|
1086
|
+
const target = entries[targetId]
|
|
1087
|
+
const sourceRefs = sourceIds.map((id) => ({ id, entry: entries[id] }))
|
|
1088
|
+
const clusterIds = new Set(memberIds)
|
|
1089
|
+
const linksOf = (entry) => (Array.isArray(entry?.links) ? entry.links : [])
|
|
1090
|
+
|
|
1091
|
+
// 吸收前状态快照(§4.11 f):rewired 存簇外条目的完整 links,撤销整体恢复。
|
|
1092
|
+
const rewired = []
|
|
1093
|
+
for (const [otherId, other] of Object.entries(entries)) {
|
|
1094
|
+
if (clusterIds.has(otherId) || otherId === mergedId) continue
|
|
1095
|
+
if (linksOf(other).some((link) => clusterIds.has(link?.to))) {
|
|
1096
|
+
rewired.push({ id: otherId, links: structuredClone(linksOf(other)) })
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
// 出边并入 merged:target/sources 自己的 links 按 (to,kind) 去重、cap 20;
|
|
1100
|
+
// 指向簇内的边在吸收后是悬挂边,改指 mergedId,自环直接丢。
|
|
1101
|
+
const outbound = []
|
|
1102
|
+
const seenPairs = new Set()
|
|
1103
|
+
for (const member of [target, ...sourceRefs.map((ref) => ref.entry)]) {
|
|
1104
|
+
for (const link of linksOf(member)) {
|
|
1105
|
+
if (!link || !link.to) continue
|
|
1106
|
+
const key = `${link.to}|${link.kind || 'related'}`
|
|
1107
|
+
if (seenPairs.has(key)) continue
|
|
1108
|
+
seenPairs.add(key)
|
|
1109
|
+
outbound.push(structuredClone(link))
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
const mergedLinks = outbound
|
|
1113
|
+
.map((link) => (clusterIds.has(link.to) ? { ...link, to: mergedId } : link))
|
|
1114
|
+
.filter((link) => link.to !== mergedId)
|
|
1115
|
+
.slice(0, CURATOR_MAX_LINK_ABSORB)
|
|
1116
|
+
|
|
1117
|
+
const mergedEntry = {
|
|
1118
|
+
...merged,
|
|
1119
|
+
links: mergedLinks,
|
|
1120
|
+
updatedAt: now,
|
|
1121
|
+
absorbSnapshot: {
|
|
1122
|
+
target: { id: targetId, status: target.status, links: structuredClone(linksOf(target)), updatedAt: target.updatedAt },
|
|
1123
|
+
sources: sourceRefs.map(({ id, entry }) => ({ id, status: entry.status, links: structuredClone(linksOf(entry)), updatedAt: entry.updatedAt })),
|
|
1124
|
+
rewired,
|
|
1125
|
+
mergedLinks: structuredClone(mergedLinks),
|
|
1126
|
+
at: now,
|
|
1127
|
+
},
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
entries[mergedId] = mergedEntry
|
|
1131
|
+
target.status = 'superseded'
|
|
1132
|
+
target.supersededBy = mergedId
|
|
1133
|
+
target.absorbedInto = mergedId
|
|
1134
|
+
target.validTo = now
|
|
1135
|
+
target.updatedAt = now
|
|
1136
|
+
const audit = { op: 'absorb', at: now, mergedId, targetId, sourceIds: [...sourceIds] }
|
|
1137
|
+
appendAudit(mergedEntry, audit)
|
|
1138
|
+
appendAudit(target, audit)
|
|
1139
|
+
for (const { entry } of sourceRefs) {
|
|
1140
|
+
entry.status = 'archived'
|
|
1141
|
+
entry.absorbedBy = mergedId
|
|
1142
|
+
entry.absorbedAt = now
|
|
1143
|
+
entry.validTo = now
|
|
1144
|
+
entry.updatedAt = now
|
|
1145
|
+
appendAudit(entry, audit)
|
|
1146
|
+
}
|
|
1147
|
+
for (const { id } of rewired) {
|
|
1148
|
+
entries[id].links = entries[id].links.map((link) => (clusterIds.has(link?.to) ? { ...link, to: mergedId } : link))
|
|
1149
|
+
}
|
|
1150
|
+
return { landed: true, at: now }
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
/**
|
|
1154
|
+
* absorb 计划提交(spec §6.7 + §4.7 + §4.10 + §4.11)。输入 curator 输出,只处理
|
|
1155
|
+
* op === 'absorb' 的条目;每条走完 协议校验 → 降级(domain/appliesTo)→ medoid →
|
|
1156
|
+
* 合并稿 → 覆盖率分段 → 重复门 → 风险门,auto 直接落 §4.11 事务,其余生成
|
|
1157
|
+
* §3.3 pendingAbsorb 提案(只进索引,不进向量库)。
|
|
1158
|
+
* 并发变化:不落 active,held pending clone,已提交向量行锁外 best-effort forget。
|
|
1159
|
+
*/
|
|
1160
|
+
export async function commitCuratorPlans(output, { job } = {}) {
|
|
1161
|
+
const memories = Array.isArray(output?.memories) ? output.memories : []
|
|
1162
|
+
const plans = memories.filter((m) => m && m.op === 'absorb')
|
|
1163
|
+
const result = { absorbed: 0, pending: 0, held: 0, skipped: [], demoted: [] }
|
|
1164
|
+
if (!plans.length) return result
|
|
1165
|
+
const index = loadGlobalIndex()
|
|
1166
|
+
// 模式只在循环外读一次(§4.10)。
|
|
1167
|
+
const mode = getSettings().curatorReviewMode || 'auto'
|
|
1168
|
+
const now = new Date().toISOString()
|
|
1169
|
+
const confOf = (value) => (Number.isFinite(Number(value)) ? Number(value) : 0.6)
|
|
1170
|
+
const pendingEntries = []
|
|
1171
|
+
const autoCommits = []
|
|
1172
|
+
const heldNames = []
|
|
1173
|
+
|
|
1174
|
+
for (const plan of plans) {
|
|
1175
|
+
const label = String(plan?.title || plan?.target_id || '(absorb)')
|
|
1176
|
+
// —— 协议校验(§6.7):target active;source 1–5;target 不在 source_ids。——
|
|
1177
|
+
const targetId = String(plan?.target_id || '').trim()
|
|
1178
|
+
const target = targetId ? index.entries[targetId] : null
|
|
1179
|
+
if (!target || target.status !== 'active') {
|
|
1180
|
+
result.skipped.push({ title: label, reason: `absorb target not found or not active: ${targetId || '(empty)'}` })
|
|
1181
|
+
continue
|
|
1182
|
+
}
|
|
1183
|
+
const requestedSources = [...new Set((Array.isArray(plan?.source_ids) ? plan.source_ids : [])
|
|
1184
|
+
.map((id) => String(id || '').trim()).filter(Boolean))]
|
|
1185
|
+
if (requestedSources.length < 1 || requestedSources.length > ABSORB_MAX_SOURCES) {
|
|
1186
|
+
result.skipped.push({ title: label, reason: `absorb needs 1-${ABSORB_MAX_SOURCES} distinct source ids, got ${requestedSources.length}` })
|
|
1187
|
+
continue
|
|
1188
|
+
}
|
|
1189
|
+
if (requestedSources.includes(targetId)) {
|
|
1190
|
+
result.skipped.push({ title: label, reason: 'absorb target must not appear in source_ids' })
|
|
1191
|
+
continue
|
|
1192
|
+
}
|
|
1193
|
+
// —— 降级(§4.7.11):source 的 appliesTo 必须是 target 子集;最终只允许同 domain。
|
|
1194
|
+
// 不满足的 source 不 absorb,降级 link(link 落地由 F1 的 commitCuratorOutput link 分支做)。
|
|
1195
|
+
const sourceRefs = []
|
|
1196
|
+
for (const sourceId of requestedSources) {
|
|
1197
|
+
const source = index.entries[sourceId]
|
|
1198
|
+
if (!source || source.status !== 'active') {
|
|
1199
|
+
result.demoted.push({ targetId, sourceId, reason: 'source not found or not active' })
|
|
1200
|
+
continue
|
|
1201
|
+
}
|
|
1202
|
+
if ((source.track || 'memory') !== (target.track || 'memory')) {
|
|
1203
|
+
result.demoted.push({ targetId, sourceId, reason: 'track differs' })
|
|
1204
|
+
continue
|
|
1205
|
+
}
|
|
1206
|
+
if ((source.domain || 'unknown') !== (target.domain || 'unknown')) {
|
|
1207
|
+
result.demoted.push({ targetId, sourceId, reason: 'domain differs; absorb is same-domain only (link candidate)' })
|
|
1208
|
+
continue
|
|
1209
|
+
}
|
|
1210
|
+
if (!appliesToSubset(source.appliesTo, target.appliesTo)) {
|
|
1211
|
+
result.demoted.push({ targetId, sourceId, reason: 'source appliesTo is not a subset of target; demoted to link candidate' })
|
|
1212
|
+
continue
|
|
1213
|
+
}
|
|
1214
|
+
sourceRefs.push({ id: sourceId, entry: source })
|
|
1215
|
+
}
|
|
1216
|
+
if (!sourceRefs.length) {
|
|
1217
|
+
result.skipped.push({ title: label, reason: 'no eligible source left after domain/appliesTo demotion' })
|
|
1218
|
+
continue
|
|
1219
|
+
}
|
|
1220
|
+
const sources = sourceRefs.map((ref) => ({ ...ref.entry, id: ref.id }))
|
|
1221
|
+
|
|
1222
|
+
// —— medoid/representative(C4):算 medoid,模型 target 只作 provenance,
|
|
1223
|
+
// medoid 平均相似度 <0.55 整簇跳过。——
|
|
1224
|
+
const members = [{ id: targetId, entry: target }, ...sourceRefs.map((ref) => ({ id: ref.id, entry: ref.entry }))]
|
|
1225
|
+
const memberText = new Map(members.map((member) => [member.id, `${member.entry.title || ''} ${member.entry.content || ''}`]))
|
|
1226
|
+
let medoid = null
|
|
1227
|
+
for (const member of members) {
|
|
1228
|
+
let sum = 0
|
|
1229
|
+
for (const other of members) {
|
|
1230
|
+
if (other.id === member.id) continue
|
|
1231
|
+
sum += similarTexts(memberText.get(member.id), memberText.get(other.id))
|
|
1232
|
+
}
|
|
1233
|
+
const avg = sum / Math.max(1, members.length - 1)
|
|
1234
|
+
if (!medoid || avg > medoid.avg) medoid = { id: member.id, avg }
|
|
1235
|
+
}
|
|
1236
|
+
if (!medoid || medoid.avg < CLUSTER_MIN_PAIR) {
|
|
1237
|
+
result.skipped.push({ title: label, reason: `medoid average similarity ${medoid ? medoid.avg.toFixed(2) : 'n/a'} is below the pair floor` })
|
|
1238
|
+
continue
|
|
1239
|
+
}
|
|
1240
|
+
const representativeId = medoid.id
|
|
1241
|
+
|
|
1242
|
+
// —— evidence(§4.6/§4.7.8):plan 证据(带 sourceId)优先,簇内既有证据并集,
|
|
1243
|
+
// 去重 ≤20;每条 source 至少一条 plan 证据覆盖,否则只进复核。——
|
|
1244
|
+
const planEvidence = (Array.isArray(plan?.evidence) ? plan.evidence : [])
|
|
1245
|
+
.filter((item) => typeof item?.session_id === 'string' && item.session_id && typeof item?.quote === 'string' && item.quote.trim())
|
|
1246
|
+
.map((item) => ({
|
|
1247
|
+
sessionId: item.session_id,
|
|
1248
|
+
quote: truncate(item.quote, 500),
|
|
1249
|
+
sourceId: typeof item?.source_id === 'string' && item.source_id.trim() ? item.source_id.trim() : null,
|
|
1250
|
+
}))
|
|
1251
|
+
.slice(0, 20)
|
|
1252
|
+
const clusterEvidence = []
|
|
1253
|
+
const seenEvidence = new Set()
|
|
1254
|
+
for (const item of [...planEvidence, ...members.flatMap((member) => (Array.isArray(member.entry.evidence) ? member.entry.evidence : [])
|
|
1255
|
+
.map((e) => ({ sessionId: e?.sessionId, quote: e?.quote, sourceId: member.id })))] ) {
|
|
1256
|
+
if (clusterEvidence.length >= 20) break
|
|
1257
|
+
const sessionId = typeof item?.sessionId === 'string' ? item.sessionId : null
|
|
1258
|
+
const quote = typeof item?.quote === 'string' ? item.quote : ''
|
|
1259
|
+
if (!sessionId || !quote.trim()) continue
|
|
1260
|
+
const key = `${sessionId}|${clean(quote)}|${item.sourceId || ''}`
|
|
1261
|
+
if (seenEvidence.has(key)) continue
|
|
1262
|
+
seenEvidence.add(key)
|
|
1263
|
+
clusterEvidence.push({ sessionId, quote: truncate(quote, 500), sourceId: item.sourceId || null })
|
|
1264
|
+
}
|
|
1265
|
+
const uncoveredSources = sourceRefs.filter((ref) => !planEvidence.some((item) => item.sourceId === ref.id))
|
|
1266
|
+
|
|
1267
|
+
// —— 合并稿 + 覆盖率分段(§4.7.6)。——
|
|
1268
|
+
const idf = absorbIdf(index)
|
|
1269
|
+
const composed = composeAbsorbContent({ target, sources, proposed: plan?.content, maxChars: ABSORB_MAX_CHARS, idf })
|
|
1270
|
+
const mergedContent = redactSecrets(composed.content)
|
|
1271
|
+
if (!mergedContent.trim()) {
|
|
1272
|
+
result.skipped.push({ title: label, reason: 'absorb produced an empty merged draft' })
|
|
1273
|
+
continue
|
|
1274
|
+
}
|
|
1275
|
+
if (composed.sourceCoverage < ABSORB_REVIEW_SOURCE_COVERAGE || composed.unionCoverage < ABSORB_REVIEW_UNION_COVERAGE) {
|
|
1276
|
+
result.skipped.push({ title: label, reason: `absorb coverage below floor (source ${composed.sourceCoverage.toFixed(2)} / union ${composed.unionCoverage.toFixed(2)})` })
|
|
1277
|
+
continue
|
|
1278
|
+
}
|
|
1279
|
+
const longestSource = Math.max(0, ...sourceRefs.map((ref) => String(ref.entry.content || '').length))
|
|
1280
|
+
const belowLengthFloor = mergedContent.length < ABSORB_MIN_LENGTH_RATIO * longestSource
|
|
1281
|
+
|
|
1282
|
+
// —— 幂等门(§6.7):同 opId 的提案/合并稿已存在 → skip。——
|
|
1283
|
+
const memberIdsSorted = [targetId, ...sourceRefs.map((ref) => ref.id)].sort()
|
|
1284
|
+
const opId = absorbOpIdOf(memberIdsSorted, mergedContent)
|
|
1285
|
+
if (Object.values(index.entries).some((e) => e?.absorbOpId === opId || e?.pendingAbsorb?.opId === opId)) {
|
|
1286
|
+
result.skipped.push({ title: label, reason: `duplicate absorb opId ${opId}; already proposed or absorbed` })
|
|
1287
|
+
continue
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
// —— title(§4.7.7)与重复门(§6.7):与簇外 active 条目 similarTexts ≥0.90 →
|
|
1291
|
+
// pending 'absorb duplicate of <id>';0.55–0.90 → 新条目打 duplicateOf。——
|
|
1292
|
+
const titleCandidate = String(target.title || '').trim()
|
|
1293
|
+
|| String(plan?.title || '').trim()
|
|
1294
|
+
|| String(index.entries[representativeId]?.title || '').trim()
|
|
1295
|
+
const title = truncate(titleCandidate || label, 180)
|
|
1296
|
+
let absorbDuplicate = null
|
|
1297
|
+
for (const [otherId, other] of Object.entries(index.entries || {})) {
|
|
1298
|
+
if (memberIdsSorted.includes(otherId)) continue
|
|
1299
|
+
if (other?.status !== 'active') continue
|
|
1300
|
+
const similarity = similarTexts(`${title} ${mergedContent}`, `${other.title || ''} ${other.content || ''}`)
|
|
1301
|
+
if (similarity > (absorbDuplicate?.similarity ?? 0)) absorbDuplicate = { id: otherId, similarity }
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
// —— 风险分(§4.10):写到所有条目(active 也写)。——
|
|
1305
|
+
const decision = reviewDecision({ m: plan, op: 'absorb', evidence: planEvidence, targetId, index, mode })
|
|
1306
|
+
|
|
1307
|
+
let reviewReason = null
|
|
1308
|
+
if (composed.sourceCoverage < ABSORB_AUTO_SOURCE_COVERAGE || composed.unionCoverage < ABSORB_AUTO_UNION_COVERAGE
|
|
1309
|
+
|| belowLengthFloor || (composed.truncated && composed.omitted.length)) {
|
|
1310
|
+
reviewReason = 'absorb coverage'
|
|
1311
|
+
} else if (absorbDuplicate && absorbDuplicate.similarity >= DUP_REJECT_DICE) {
|
|
1312
|
+
reviewReason = `absorb duplicate of ${absorbDuplicate.id}`
|
|
1313
|
+
} else if (uncoveredSources.length || !planEvidence.length) {
|
|
1314
|
+
reviewReason = decision.hard ? decision.reasons[0] : 'evidence not grounded'
|
|
1315
|
+
} else if (decision.review) {
|
|
1316
|
+
reviewReason = decision.hard ? decision.reasons[0] : (String(plan?.review_reason || '').trim() || 'risk threshold')
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
const confidence = Math.min(0.95, Math.max(confOf(target.confidence), ...sourceRefs.map((ref) => confOf(ref.entry.confidence)), confOf(plan?.confidence)))
|
|
1320
|
+
const clusterCreatedAt = members.map((member) => member.entry.createdAt).filter(Boolean).sort()[0] || now
|
|
1321
|
+
const expectedUpdatedAt = {
|
|
1322
|
+
[targetId]: target.updatedAt,
|
|
1323
|
+
...Object.fromEntries(sourceRefs.map((ref) => [ref.id, ref.entry.updatedAt])),
|
|
1324
|
+
}
|
|
1325
|
+
const reviewRiskReasons = reviewReason === 'absorb coverage' ? [...decision.reasons, 'absorb coverage'] : decision.reasons
|
|
1326
|
+
const entryBase = {
|
|
1327
|
+
track: target.track || 'memory',
|
|
1328
|
+
domain: target.domain || 'general',
|
|
1329
|
+
appliesTo: Array.isArray(target.appliesTo) ? structuredClone(target.appliesTo).slice(0, 20) : [],
|
|
1330
|
+
title,
|
|
1331
|
+
content: mergedContent,
|
|
1332
|
+
confidence,
|
|
1333
|
+
evidence: clusterEvidence,
|
|
1334
|
+
status: 'active',
|
|
1335
|
+
absorbedFrom: memberIdsSorted,
|
|
1336
|
+
absorbedAt: now,
|
|
1337
|
+
absorbOpId: opId,
|
|
1338
|
+
representativeId,
|
|
1339
|
+
mergeMode: composed.mode,
|
|
1340
|
+
mergeCoverage: round4(composed.coverage),
|
|
1341
|
+
sourceCoverage: round4(composed.sourceCoverage),
|
|
1342
|
+
unionCoverage: round4(composed.unionCoverage),
|
|
1343
|
+
reviewRisk: decision.risk,
|
|
1344
|
+
reviewRiskReasons,
|
|
1345
|
+
createdAt: clusterCreatedAt,
|
|
1346
|
+
updatedAt: now,
|
|
1347
|
+
}
|
|
1348
|
+
if (absorbDuplicate && absorbDuplicate.similarity >= TOPIC_DICE && absorbDuplicate.similarity < DUP_REJECT_DICE) {
|
|
1349
|
+
entryBase.duplicateOf = absorbDuplicate.id
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
if (reviewReason) {
|
|
1353
|
+
// pending absorb(§3.3):只进索引,绝不进向量库;accept 时才提交提案 id 本身。
|
|
1354
|
+
pendingEntries.push({
|
|
1355
|
+
id: newMemoryId('gmem'),
|
|
1356
|
+
entry: {
|
|
1357
|
+
...entryBase,
|
|
1358
|
+
status: 'pending_review',
|
|
1359
|
+
reviewReason,
|
|
1360
|
+
conflictingWith: targetId,
|
|
1361
|
+
pendingAbsorb: { targetId, sourceIds: sourceRefs.map((ref) => ref.id), opId, representativeId, expectedUpdatedAt },
|
|
1362
|
+
source: 'curator',
|
|
1363
|
+
},
|
|
1364
|
+
})
|
|
1365
|
+
result.pending += 1
|
|
1366
|
+
continue
|
|
1367
|
+
}
|
|
1368
|
+
autoCommits.push({
|
|
1369
|
+
mergedId: newMemoryId('gmem'),
|
|
1370
|
+
entry: entryBase,
|
|
1371
|
+
targetId,
|
|
1372
|
+
sourceIds: sourceRefs.map((ref) => ref.id),
|
|
1373
|
+
expectedUpdatedAt,
|
|
1374
|
+
})
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
if (pendingEntries.length) {
|
|
1378
|
+
if (job) job.step = `writing ${pendingEntries.length} pending absorb proposals`
|
|
1379
|
+
await withManagedCuratorWrite(() => retryStoreWrite(() => updateGlobalIndex((fresh) => {
|
|
1380
|
+
for (const { id, entry } of pendingEntries) fresh.entries[id] = entry
|
|
1381
|
+
})))
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
if (autoCommits.length) {
|
|
1385
|
+
if (job) job.step = `committing ${autoCommits.length} absorb merges`
|
|
1386
|
+
await withManagedCuratorWrite(async () => {
|
|
1387
|
+
const route = engineRoute()
|
|
1388
|
+
const committedNames = []
|
|
1389
|
+
try {
|
|
1390
|
+
// 先向量后索引(§4.11 auto absorb 步骤 2→3):索引失败时能补偿已提交行。
|
|
1391
|
+
if (route !== 'lexical') {
|
|
1392
|
+
await commit({
|
|
1393
|
+
payload: {
|
|
1394
|
+
recall_files: autoCommits.map(({ mergedId, entry }) => ({ name: mergedId, track: entry.track, description: entry.title, content: entry.content })),
|
|
1395
|
+
resource: [],
|
|
1396
|
+
user: {},
|
|
1397
|
+
},
|
|
1398
|
+
route,
|
|
1399
|
+
db: globalDbPath(),
|
|
1400
|
+
})
|
|
1401
|
+
for (const c of autoCommits) committedNames.push(c.mergedId)
|
|
1402
|
+
const vector = activeEntryStamp(route)
|
|
1403
|
+
if (vector) for (const c of autoCommits) c.entry.vector = vector
|
|
1404
|
+
}
|
|
1405
|
+
await retryStoreWrite(() => updateGlobalIndex((fresh) => {
|
|
1406
|
+
for (const c of autoCommits) {
|
|
1407
|
+
const outcome = applyAbsorbIndexTransaction(fresh, {
|
|
1408
|
+
mergedId: c.mergedId, merged: c.entry, targetId: c.targetId,
|
|
1409
|
+
sourceIds: c.sourceIds, expectedUpdatedAt: c.expectedUpdatedAt, now,
|
|
1410
|
+
})
|
|
1411
|
+
if (!outcome.landed) {
|
|
1412
|
+
// 并发变化(§4.11 步骤 5):不落 active,生成 held pending clone;
|
|
1413
|
+
// expectedUpdatedAt 刷新为当前值,人审时再做快照校验。
|
|
1414
|
+
const heldId = newMemoryId('gmem')
|
|
1415
|
+
const refreshed = {}
|
|
1416
|
+
for (const memberId of [c.targetId, ...c.sourceIds]) {
|
|
1417
|
+
refreshed[memberId] = fresh.entries[memberId]?.updatedAt || c.expectedUpdatedAt[memberId]
|
|
1418
|
+
}
|
|
1419
|
+
const held = {
|
|
1420
|
+
...c.entry,
|
|
1421
|
+
status: 'pending_review',
|
|
1422
|
+
reviewReason: 'cluster changed while curator was running',
|
|
1423
|
+
conflictingWith: c.targetId,
|
|
1424
|
+
pendingAbsorb: { targetId: c.targetId, sourceIds: c.sourceIds, opId: c.entry.absorbOpId, representativeId: c.entry.representativeId, expectedUpdatedAt: refreshed },
|
|
1425
|
+
updatedAt: now,
|
|
1426
|
+
}
|
|
1427
|
+
// 已提交向量库的是 mergedId 这一行;held 克隆从未进过向量库,
|
|
1428
|
+
// 绝不能继承"提交成功即落戳"的 entry.vector(同 curator-commit 的坑)。
|
|
1429
|
+
delete held.vector
|
|
1430
|
+
fresh.entries[heldId] = held
|
|
1431
|
+
heldNames.push(c.mergedId)
|
|
1432
|
+
result.held += 1
|
|
1433
|
+
continue
|
|
1434
|
+
}
|
|
1435
|
+
result.absorbed += 1
|
|
1436
|
+
}
|
|
1437
|
+
}))
|
|
1438
|
+
} catch (error) {
|
|
1439
|
+
if (committedNames.length) await compensateCuratorCommit(error, { files: committedNames.map((name) => ({ name })), route })
|
|
1440
|
+
throw error
|
|
1441
|
+
}
|
|
1442
|
+
// held clone 的已提交行 best-effort forget(锁外 await,修 D11);只在向量路由
|
|
1443
|
+
// 真的提交过行时才需要。删不掉必须可见。
|
|
1444
|
+
if (route !== 'lexical') {
|
|
1445
|
+
for (const name of heldNames) {
|
|
1446
|
+
try {
|
|
1447
|
+
await forget({ db: globalDbPath(), name })
|
|
1448
|
+
} catch (forgetError) {
|
|
1449
|
+
record({
|
|
1450
|
+
kind: 'curate', scope: 'global', ok: false, level: 'error',
|
|
1451
|
+
message: `集合吸收并发保护:held clone 已生成,但向量行 ${name} 撤销失败(引擎库残留孤儿行,需手动清理)`,
|
|
1452
|
+
detail: { op: 'absorb', heldForgetFailed: true, name, error: String(forgetError?.message || forgetError) },
|
|
1453
|
+
refs: [name],
|
|
1454
|
+
})
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
})
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
if (result.absorbed || result.pending || result.held || result.skipped.length || result.demoted.length) {
|
|
1462
|
+
record({
|
|
1463
|
+
kind: 'curate',
|
|
1464
|
+
scope: 'global',
|
|
1465
|
+
ok: true,
|
|
1466
|
+
message: `集合吸收:合并 ${result.absorbed} / 待复核 ${result.pending}`
|
|
1467
|
+
+ (result.held ? ` / 并发暂扣 ${result.held}` : '')
|
|
1468
|
+
+ (result.skipped.length ? ` / 跳过 ${result.skipped.length}` : '')
|
|
1469
|
+
+ (result.demoted.length ? ` / 降级 link ${result.demoted.length}` : ''),
|
|
1470
|
+
detail: { op: 'absorb', ...result, skipped: result.skipped.slice(0, 20), demoted: result.demoted.slice(0, 20) },
|
|
1471
|
+
refs: autoCommits.map((c) => c.mergedId).slice(0, 20),
|
|
1472
|
+
})
|
|
1473
|
+
}
|
|
1474
|
+
return result
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
/**
|
|
1478
|
+
* 撤销集合吸收(spec §4.11 undo-absorb)。merged 必须 active 且带 absorbSnapshot;
|
|
1479
|
+
* 同一事务恢复 target/sources/rewired,merged 归档(保留正文)。吸收后又被人工改过
|
|
1480
|
+
* (updatedAt > absorbedAt)的成员拒绝整体撤销并返回可读错误——部分恢复比不恢复更
|
|
1481
|
+
* 难对账。不删除、不写向量。
|
|
1482
|
+
*/
|
|
1483
|
+
export async function undoGlobalAbsorb({ id } = {}) {
|
|
1484
|
+
const mergedId = String(id || '').trim()
|
|
1485
|
+
if (!mergedId) throw new Error('undo-absorb needs the merged entry id')
|
|
1486
|
+
const index = loadGlobalIndex()
|
|
1487
|
+
const merged = index.entries[mergedId]
|
|
1488
|
+
if (!merged || merged.status !== 'active' || !merged.absorbSnapshot) {
|
|
1489
|
+
throw new Error('undo-absorb needs an active merged entry with an absorb snapshot')
|
|
1490
|
+
}
|
|
1491
|
+
const snapshot = merged.absorbSnapshot
|
|
1492
|
+
const memberRefs = [snapshot.target, ...(Array.isArray(snapshot.sources) ? snapshot.sources : [])].filter(Boolean)
|
|
1493
|
+
const changedMembers = []
|
|
1494
|
+
for (const member of memberRefs) {
|
|
1495
|
+
const current = index.entries[member.id]
|
|
1496
|
+
if (!current) continue
|
|
1497
|
+
const absorbedAt = Date.parse(current.absorbedAt || '') || Date.parse(snapshot.at || '') || 0
|
|
1498
|
+
if ((Date.parse(current.updatedAt || '') || 0) > absorbedAt) changedMembers.push(member.id)
|
|
1499
|
+
}
|
|
1500
|
+
if (changedMembers.length) {
|
|
1501
|
+
throw new Error(`absorb undo refused: ${changedMembers.join(', ')} changed after absorption; restore them manually or resolve each edit first`)
|
|
1502
|
+
}
|
|
1503
|
+
await retryStoreWrite(() => updateGlobalIndex((fresh) => {
|
|
1504
|
+
if (JSON.stringify(fresh.entries[mergedId]) !== JSON.stringify(merged)) throw new Error('absorb entry changed; reload and retry')
|
|
1505
|
+
const at = new Date().toISOString()
|
|
1506
|
+
const targetSnap = snapshot.target
|
|
1507
|
+
const currentTarget = targetSnap?.id ? fresh.entries[targetSnap.id] : null
|
|
1508
|
+
if (currentTarget && currentTarget.status === 'superseded' && currentTarget.supersededBy === mergedId) {
|
|
1509
|
+
currentTarget.status = targetSnap.status || 'active'
|
|
1510
|
+
if (Array.isArray(targetSnap.links)) currentTarget.links = structuredClone(targetSnap.links)
|
|
1511
|
+
delete currentTarget.supersededBy
|
|
1512
|
+
delete currentTarget.absorbedInto
|
|
1513
|
+
delete currentTarget.validTo
|
|
1514
|
+
currentTarget.updatedAt = at
|
|
1515
|
+
}
|
|
1516
|
+
for (const sourceSnap of Array.isArray(snapshot.sources) ? snapshot.sources : []) {
|
|
1517
|
+
const current = fresh.entries[sourceSnap.id]
|
|
1518
|
+
if (!current || current.status !== 'archived' || current.absorbedBy !== mergedId) continue
|
|
1519
|
+
current.status = sourceSnap.status || 'active'
|
|
1520
|
+
if (Array.isArray(sourceSnap.links)) current.links = structuredClone(sourceSnap.links)
|
|
1521
|
+
delete current.absorbedBy
|
|
1522
|
+
delete current.absorbedAt
|
|
1523
|
+
delete current.validTo
|
|
1524
|
+
current.updatedAt = at
|
|
1525
|
+
}
|
|
1526
|
+
for (const rewiredSnap of Array.isArray(snapshot.rewired) ? snapshot.rewired : []) {
|
|
1527
|
+
const current = fresh.entries[rewiredSnap.id]
|
|
1528
|
+
if (current && Array.isArray(rewiredSnap.links)) current.links = structuredClone(rewiredSnap.links)
|
|
1529
|
+
}
|
|
1530
|
+
const stored = fresh.entries[mergedId]
|
|
1531
|
+
stored.status = 'archived'
|
|
1532
|
+
stored.absorbUndoneAt = at
|
|
1533
|
+
stored.updatedAt = at
|
|
1534
|
+
const audit = { op: 'undo-absorb', at, mergedId }
|
|
1535
|
+
appendAudit(currentTarget, audit)
|
|
1536
|
+
for (const sourceSnap of Array.isArray(snapshot.sources) ? snapshot.sources : []) {
|
|
1537
|
+
appendAudit(fresh.entries[sourceSnap.id], audit)
|
|
1538
|
+
}
|
|
1539
|
+
appendAudit(stored, audit)
|
|
1540
|
+
}))
|
|
1541
|
+
// 活动日志 CURATOR_ABSORB_UNDO(§4.11 撤销 4):以 detail.op 标记,kind 仍是 curate。
|
|
1542
|
+
record({
|
|
1543
|
+
kind: 'curate',
|
|
1544
|
+
scope: 'global',
|
|
1545
|
+
ok: true,
|
|
1546
|
+
message: `撤销集合吸收 ${mergedId}:target 与来源记忆已恢复,merged 已归档`,
|
|
1547
|
+
detail: { op: 'undo-absorb', id: mergedId, snapshotAt: snapshot.at || null },
|
|
1548
|
+
refs: [mergedId],
|
|
1549
|
+
})
|
|
1550
|
+
return { id: mergedId, status: 'archived', undone: true }
|
|
1551
|
+
}
|
|
1552
|
+
|
|
410
1553
|
export function startGlobalCuration(ctx, { sessionIds, parentSessionId, model } = {}) {
|
|
411
1554
|
const ids = [...new Set((Array.isArray(sessionIds) ? sessionIds : []).filter((s) => typeof s === 'string' && s.trim()))]
|
|
412
1555
|
const jobId = `curate-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`
|
|
@@ -465,8 +1608,20 @@ export function startGlobalCuration(ctx, { sessionIds, parentSessionId, model }
|
|
|
465
1608
|
if (total >= MAX_TOTAL_CHARS) break
|
|
466
1609
|
}
|
|
467
1610
|
|
|
468
|
-
|
|
469
|
-
|
|
1611
|
+
// —— §4.5/§6.6(包 F1):相关性打包索引(4000 tokens)+ 候选簇(2000 tokens),
|
|
1612
|
+
// 取代旧 compactGlobalIndex 的"插入顺序前 200 条"平铺。sessionCorpus 同步构建,
|
|
1613
|
+
// 提交层证据落地(§4.6)用同一份转录做回放校验。
|
|
1614
|
+
const globalIndex = loadGlobalIndex()
|
|
1615
|
+
const sessionText = sessions.map((s) => s.transcript).join('\n')
|
|
1616
|
+
const packedIndex = packCuratorIndexContext({ index: globalIndex, sessionText })
|
|
1617
|
+
const packedClusters = packCuratorClusterContext(buildCandidateClusters(globalIndex))
|
|
1618
|
+
const input = JSON.stringify({
|
|
1619
|
+
sessions,
|
|
1620
|
+
existing_global_index: packedIndex.entries,
|
|
1621
|
+
index_truncated: packedIndex.omitted > 0,
|
|
1622
|
+
candidate_clusters: packedClusters.clusters,
|
|
1623
|
+
clusters_truncated: packedClusters.truncated,
|
|
1624
|
+
}, null, 2)
|
|
470
1625
|
const prompt = loadPrompt('global-curator', { input }).text
|
|
471
1626
|
job.step = 'curator running'
|
|
472
1627
|
log('starting non-main curator subagent')
|
|
@@ -499,7 +1654,12 @@ export function startGlobalCuration(ctx, { sessionIds, parentSessionId, model }
|
|
|
499
1654
|
throw new Error(`curator did not return valid JSON: ${truncate(text, 800)}`)
|
|
500
1655
|
}
|
|
501
1656
|
job.step = 'committing'
|
|
502
|
-
|
|
1657
|
+
// 生产调用必须传会话语料(§4.6.6):证据 quote 要能在本次策划的转录里回放,
|
|
1658
|
+
// 否则一律 pending。transcripts 已在 condenseEvents 里脱敏。
|
|
1659
|
+
const committed = await commitCuratorOutput(output, {
|
|
1660
|
+
job,
|
|
1661
|
+
sessionCorpus: buildEvidenceCorpus(sessions.map((s) => ({ id: s.id, transcript: s.transcript }))),
|
|
1662
|
+
})
|
|
503
1663
|
job.status = 'completed'
|
|
504
1664
|
job.endedAt = new Date().toISOString()
|
|
505
1665
|
job.result = {
|
|
@@ -508,9 +1668,10 @@ export function startGlobalCuration(ctx, { sessionIds, parentSessionId, model }
|
|
|
508
1668
|
created: committed.created,
|
|
509
1669
|
merged: committed.merged,
|
|
510
1670
|
superseded: committed.superseded,
|
|
1671
|
+
linked: committed.linked || 0,
|
|
511
1672
|
skipped: committed.skipped,
|
|
512
1673
|
}
|
|
513
|
-
log(`done: ${committed.created} created, ${committed.merged} merged, ${committed.superseded} superseded`)
|
|
1674
|
+
log(`done: ${committed.created} created, ${committed.merged} merged, ${committed.superseded} superseded, ${committed.linked || 0} linked`)
|
|
514
1675
|
} catch (error) {
|
|
515
1676
|
const aborted = abort.signal.aborted || /cancelled by user/.test(String(error?.message || ''))
|
|
516
1677
|
job.status = aborted ? 'cancelled' : 'error'
|