dsh-memu-memory 1.2.1 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/AGENTS.md +1 -1
  2. package/README.md +12 -5
  3. package/README.zh.md +4 -2
  4. package/docs/contributing.md +16 -3
  5. package/docs/host-api.md +3 -1
  6. package/docs/memory-semantics.md +21 -2
  7. package/docs/troubleshooting.md +42 -4
  8. package/lib/backup.js +24 -2
  9. package/lib/binding-diagnostics.js +2 -2
  10. package/lib/client.js +1361 -146
  11. package/lib/consolidation-worker.js +30 -8
  12. package/lib/curator-evidence.js +131 -0
  13. package/lib/curator-policy.js +81 -0
  14. package/lib/curator.js +1252 -91
  15. package/lib/cycle.js +3 -0
  16. package/lib/embedding-download.js +39 -4
  17. package/lib/engine.js +6 -3
  18. package/lib/global-graph.js +282 -0
  19. package/lib/graph.js +51 -9
  20. package/lib/inbound.js +19 -0
  21. package/lib/inject.js +277 -17
  22. package/lib/lexicon.js +1115 -0
  23. package/lib/local-vector-service.js +8 -6
  24. package/lib/managed-runtime.js +33 -2
  25. package/lib/memory.js +282 -17
  26. package/lib/memu.js +2 -2
  27. package/lib/migrate.js +83 -2
  28. package/lib/private-links.js +587 -0
  29. package/lib/private.js +32 -3
  30. package/lib/prompts/global-curator.md +76 -19
  31. package/lib/prompts/private-consolidation.coding.md +4 -1
  32. package/lib/prompts/private-consolidation.md +3 -1
  33. package/lib/prompts/private-consolidation.mixed.md +3 -1
  34. package/lib/prompts/private-consolidation.rp.md +3 -1
  35. package/lib/prompts.js +1 -1
  36. package/lib/rpc.js +577 -9
  37. package/lib/state.js +4 -1
  38. package/lib/text-retrieval.js +661 -0
  39. package/lib/text-similarity.js +36 -18
  40. package/lib/tools.js +168 -10
  41. package/lib/webui/standalone.css +17 -0
  42. package/lib/webui/standalone.js +2 -0
  43. package/package.json +1 -1
  44. package/scripts/acceptance-local.mjs +8 -5
  45. package/scripts/curator-pair-calibrate.mjs +259 -0
  46. package/scripts/lexicon-calibrate.mjs +230 -0
  47. package/scripts/lexicon-dedupe.mjs +318 -0
  48. package/tests/absorb-review.test.mjs +353 -0
  49. package/tests/appendix-b2.test.mjs +2 -2
  50. package/tests/client-export.test.mjs +520 -0
  51. package/tests/client-render.test.mjs +19 -3
  52. package/tests/consolidation-fork.test.mjs +14 -12
  53. package/tests/consolidation-snapshot.test.mjs +39 -0
  54. package/tests/curator-absorb.test.mjs +443 -0
  55. package/tests/curator-calibrate.test.mjs +173 -0
  56. package/tests/curator-clusters.test.mjs +281 -0
  57. package/tests/curator-commit.test.mjs +52 -5
  58. package/tests/curator-evidence.test.mjs +102 -0
  59. package/tests/curator-link.test.mjs +316 -0
  60. package/tests/curator-pairing.test.mjs +252 -0
  61. package/tests/curator-probe.test.mjs +15 -6
  62. package/tests/curator-review-mode.test.mjs +314 -0
  63. package/tests/disclosure.test.mjs +2 -1
  64. package/tests/fixtures/curator-pairs/labeled.v1.jsonl +90 -0
  65. package/tests/fixtures/lexicon-stub-hooks.mjs +15 -0
  66. package/tests/fixtures/lexicon-stub.mjs +284 -0
  67. package/tests/graph-cascade.test.mjs +35 -2
  68. package/tests/graph.test.mjs +96 -0
  69. package/tests/help-page.test.mjs +415 -0
  70. package/tests/inbound.test.mjs +6 -3
  71. package/tests/inject-guide.test.mjs +28 -0
  72. package/tests/inject.test.mjs +157 -5
  73. package/tests/lexicon-calibrate.test.mjs +48 -0
  74. package/tests/lexicon-dedupe.test.mjs +171 -0
  75. package/tests/lexicon-frequency.test.mjs +179 -0
  76. package/tests/lexicon-inject.test.mjs +410 -0
  77. package/tests/lexicon-perf.test.mjs +436 -0
  78. package/tests/lexicon-ui.test.mjs +547 -0
  79. package/tests/lexicon.test.mjs +922 -0
  80. package/tests/managed-runtime.test.mjs +32 -0
  81. package/tests/private-links.test.mjs +445 -0
  82. package/tests/prompts.test.mjs +35 -0
  83. package/tests/recall-dedup.test.mjs +33 -4
  84. package/tests/tool-schema.test.mjs +1 -1
  85. package/tests/tools.test.mjs +112 -3
  86. package/tests/webui-auth.test.mjs +348 -2
  87. package/tests/webui.test.mjs +188 -0
@@ -89,17 +89,19 @@ function profileFor(catalogId) {
89
89
  }
90
90
 
91
91
  function installationFor(id, sharedRuntime = null) {
92
- const status = getDownloader().getStatus(id, { deep: false })
92
+ const status = getDownloader().getStatus(id, { deep: true })
93
93
  // 所有可用模型 pin 的是同一个受管 runtime;每个模型各走一遍运行时目录树(几百个文件)
94
94
  // 会让每次 getState 多花几百毫秒。上层已经算过一次就复用。
95
95
  const runtime = sharedRuntime || localVectorRuntimeStatus({ modelId: id })
96
+ const degraded = status.state === 'installed' && status.verified === false
96
97
  return {
97
- status: status.state,
98
- installed: status.installed === true,
99
- modelDir: status.installed ? path.join(root, id) : null,
98
+ status: degraded ? 'degraded' : status.state,
99
+ installed: status.installed === true && !degraded,
100
+ verified: status.verified ?? null,
101
+ modelDir: status.installed && !degraded ? path.join(root, id) : null,
100
102
  runtimeDir: runtime.ready ? runtime.path : null,
101
103
  fingerprintMatches: status.fingerprintMatches ?? null,
102
- error: status.error || null,
104
+ error: degraded ? { code: 'EMBEDDING_INSTALL_TAMPERED', message: 'installed embedding files failed deep verification' } : (status.error || null),
103
105
  progress: status.progress || null,
104
106
  }
105
107
  }
@@ -164,7 +166,7 @@ export function getLocalVectorState() {
164
166
 
165
167
  export function selectLocalVectorModel(id) {
166
168
  const status = getDownloader().getStatus(String(id), { deep: true })
167
- if (status.state !== 'installed' || status.fingerprintMatches === false) {
169
+ if (status.state !== 'installed' || status.verified !== true || status.fingerprintMatches === false) {
168
170
  const error = new Error('install and verify the local embedding model before selecting it')
169
171
  error.code = 'LOCAL_VECTOR_MODEL_NOT_INSTALLED'
170
172
  throw error
@@ -20,6 +20,28 @@ export function verifyRuntimeArchive(data, digest) {
20
20
  if (crypto.createHash('sha256').update(data).digest('hex') !== digest) throw new Error('managed runtime checksum mismatch; refusing to execute')
21
21
  }
22
22
 
23
+ function executableFingerprint(file) {
24
+ try {
25
+ const stat = fs.lstatSync(file)
26
+ if (!stat.isFile() || stat.isSymbolicLink()) return null
27
+ const sha256 = crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex')
28
+ return { bytes: stat.size, sha256 }
29
+ } catch {
30
+ return null
31
+ }
32
+ }
33
+
34
+ function verifiedRuntimeMetadata(file, asset) {
35
+ try {
36
+ const metadata = JSON.parse(fs.readFileSync(file, 'utf8'))
37
+ return metadata && metadata.version === UV_VERSION && metadata.archive === asset[0] && metadata.archiveSha256 === asset[1] && Number.isInteger(metadata.executableBytes) && typeof metadata.executableSha256 === 'string'
38
+ ? metadata
39
+ : null
40
+ } catch {
41
+ return null
42
+ }
43
+ }
44
+
23
45
  export async function ensureManagedPython({ run, log = () => {}, fetcher = fetch } = {}) {
24
46
  const platform = `${process.platform}-${process.arch}`
25
47
  const asset = ASSETS[platform]
@@ -33,7 +55,14 @@ export async function ensureManagedPython({ run, log = () => {}, fetcher = fetch
33
55
  UV_PYTHON_BIN_DIR: ensureDir(path.join(root, 'bin')),
34
56
  UV_NO_PROGRESS: '1', UV_PYTHON_PREFERENCE: 'only-managed', UV_NO_MODIFY_PATH: '1', UV_NO_CONFIG: '1',
35
57
  }
36
- if (!fs.existsSync(executable)) {
58
+ const metadataFile = path.join(bin, 'verified.json')
59
+ const metadata = verifiedRuntimeMetadata(metadataFile, asset)
60
+ const fingerprint = executableFingerprint(executable)
61
+ const reusable = Boolean(metadata && fingerprint && metadata.executableBytes === fingerprint.bytes && metadata.executableSha256 === fingerprint.sha256)
62
+ if (!reusable) {
63
+ if (fingerprint || fs.existsSync(executable)) log(`managed uv ${UV_VERSION} is missing trusted metadata or was modified; replacing it`)
64
+ try { fs.rmSync(executable, { force: true }) } catch {}
65
+ try { fs.rmSync(metadataFile, { force: true }) } catch {}
37
66
  log(`no usable Python with pip; downloading pinned uv ${UV_VERSION}`)
38
67
  const response = await fetcher(`https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/${asset[0]}`, { signal: AbortSignal.timeout(120_000) })
39
68
  if (!response.ok || Number(response.headers.get('content-length')) > 64 * 1024 * 1024) throw new Error('managed runtime download failed or is too large')
@@ -63,7 +92,9 @@ export async function ensureManagedPython({ run, log = () => {}, fetcher = fetch
63
92
  }
64
93
  if (fs.existsSync(temporary)) fs.renameSync(temporary, executable)
65
94
  if (process.platform !== 'win32') fs.chmodSync(executable, 0o700)
66
- writeJson(path.join(bin, 'verified.json'), { version: UV_VERSION, archive: asset[0], sha256: asset[1] })
95
+ const installedFingerprint = executableFingerprint(executable)
96
+ if (!installedFingerprint) throw new Error('managed runtime executable is not a regular file')
97
+ writeJson(metadataFile, { version: UV_VERSION, archive: asset[0], archiveSha256: asset[1], executableBytes: installedFingerprint.bytes, executableSha256: installedFingerprint.sha256 })
67
98
  }
68
99
  const version = await run(executable, ['--version'], { env, timeoutMs: 20_000 })
69
100
  if (version.code !== 0 || !version.stdout.startsWith(`uv ${UV_VERSION} `) && version.stdout.trim() !== `uv ${UV_VERSION}`) throw new Error('managed uv version mismatch')
package/lib/memory.js CHANGED
@@ -20,6 +20,7 @@ import { attachmentRefs } from './media.js'
20
20
  import { normalizeMemoryContext, contextVisible, sameFactContext } from './context.js'
21
21
  import { activeEntryStamp, assertGeneration, assertVectorWriteAllowed, captureGeneration, enrollActiveLibrary, hasActiveVectorGeneration, isVectorStateError, readVectorState, withVectorWrite } from './vector-profile.js'
22
22
  import { bigrams, dice, similarTexts, DUP_DICE, DUP_REJECT_DICE, TOPIC_DICE } from './text-similarity.js'
23
+ import { applyAbsorbIndexTransaction } from './curator.js'
23
24
 
24
25
  const HALF_LIFE_DAYS = 14
25
26
  // A short-term entry is meant to live for the length of a conversation, not a
@@ -595,6 +596,48 @@ export function findConflict(entries, candidate) {
595
596
  return null
596
597
  }
597
598
 
599
+ // Duplicate/conflict gates may only consult entries this conversation could
600
+ // actually surface. `secret` never auto-surfaces; knownBy/context-hidden rows are
601
+ // equally invisible. A hidden hit must not leak its id (5.1).
602
+ function visibleForDedupe(entry, sessionId) {
603
+ if (!entry || entry.visibility === 'secret') return false
604
+ return sessionId ? visibleToSession(entry, sessionId) : true
605
+ }
606
+
607
+ // Critical-section re-check for the duplicate gate (5.4): the pre-flight scan
608
+ // runs before the backend await, so a concurrent writer can land a twin while the
609
+ // commit is in flight. Same ruler, same visible scope, one pass inside the lock.
610
+ function findNearDuplicate(entries, { id, title, content, context, sessionId, excludeId = null }) {
611
+ const text = `${title || ''} ${content || ''}`
612
+ let duplicate = null
613
+ let hidden = false
614
+ for (const [otherId, other] of Object.entries(entries || {})) {
615
+ if (otherId === id || otherId === excludeId) continue
616
+ if (!isMemoryEligible(other)) continue
617
+ if (!sameFactContext(other.context, context)) continue
618
+ const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
619
+ if (!visibleForDedupe(other, sessionId)) {
620
+ if (similarity >= DUP_REJECT_DICE) hidden = true
621
+ continue
622
+ }
623
+ if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
624
+ }
625
+ return { duplicate, hidden }
626
+ }
627
+
628
+ // The visible tail that the rewrite adds/changes, for a rejection response. Only
629
+ // ever computed against a VISIBLE duplicate — a hidden twin's text must not leak.
630
+ function nearDuplicateDiff(existingText, candidateText, max = 180) {
631
+ const a = String(existingText || '')
632
+ const b = String(candidateText || '')
633
+ let start = 0
634
+ while (start < a.length && start < b.length && a[start] === b[start]) start += 1
635
+ let end = 0
636
+ while (end < a.length - start && end < b.length - start && a[a.length - 1 - end] === b[b.length - 1 - end]) end += 1
637
+ const added = b.slice(start, b.length - end).trim()
638
+ return clampText(added || b, max)
639
+ }
640
+
598
641
  function mirrorContent(file) {
599
642
  const raw = file?.content
600
643
  if (typeof raw === 'string' && raw.trim()) return raw
@@ -1350,7 +1393,10 @@ async function privateRememberImpl({
1350
1393
  // the library is not silently appended: it lands as pending_review with the id it
1351
1394
  // collides with, so the conflict surfaces instead of both copies being read as
1352
1395
  // true. Same mechanism the global curator uses; until now the private side had none.
1353
- const conflict = findConflict(Object.fromEntries(Object.entries(index.entries).filter(([key]) => key !== id)), entry)
1396
+ // Hidden rows are excluded (5.1): otherwise a visible write could learn an id
1397
+ // through conflictingWith, and a hidden claim could hold an unrelated new one.
1398
+ const memorySession = memorySessionId(exec)
1399
+ const conflict = findConflict(Object.fromEntries(Object.entries(index.entries).filter(([key, other]) => key !== id && visibleForDedupe(other, memorySession))), entry)
1354
1400
  if (conflict) {
1355
1401
  entry.status = 'pending_review'
1356
1402
  entry.conflictingWith = conflict.id
@@ -1366,24 +1412,58 @@ async function privateRememberImpl({
1366
1412
  // Never deletes or rewrites anything: a hit either refuses the new entry, or tags
1367
1413
  // it duplicateOf so recall can keep one seat per fact.
1368
1414
  let duplicate = null
1415
+ let hiddenDuplicate = null
1369
1416
  if (!conflict && allowDuplicate !== true) {
1370
1417
  const text = `${entry.title || ''} ${content || ''}`
1371
- const grams = bigrams(text)
1372
1418
  for (const [otherId, other] of Object.entries(index.entries)) {
1373
- // Same filter set the recall side uses: only an entry a recall could actually
1374
- // surface can be "the same fact" — and a duplicateOf tag must point at one.
1375
1419
  if (otherId === id || !isMemoryEligible(other)) continue
1376
1420
  if (!sameFactContext(other.context, entry.context)) continue
1377
1421
  const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
1422
+ if (!visibleForDedupe(other, memorySession)) {
1423
+ // Hidden near-duplicate: refuse generically, never return its id/similarity.
1424
+ if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
1425
+ continue
1426
+ }
1378
1427
  if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
1379
1428
  }
1380
1429
  }
1381
- if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
1430
+ if (hiddenDuplicate || (duplicate && duplicate.similarity >= DUP_REJECT_DICE)) {
1431
+ // 5.3:拒写不丢文本——新正文作为 pending_review 暂存,进入现有复核队列;
1432
+ // 可见重复给出 id 与差异片段,隐藏重复只说“有重复”,不泄露 id/正文。
1433
+ const visibleTarget = duplicate ? index.entries[duplicate.id] || null : null
1434
+ const pendingEntry = {
1435
+ ...entry,
1436
+ status: 'pending_review',
1437
+ reviewReason: 'near_duplicate',
1438
+ conflictingWith: duplicate ? duplicate.id : null,
1439
+ ...(duplicate ? { duplicateOf: duplicate.id } : {}),
1440
+ }
1441
+ await retryStoreWrite(() => updatePrivateIndex(scope.libraryId, (fresh) => {
1442
+ if (fresh.entries[id]) throw new Error('memory changed concurrently; retry')
1443
+ fresh.entries[id] = pendingEntry
1444
+ }))
1445
+ if (duplicate) {
1446
+ const diff = nearDuplicateDiff(
1447
+ `${visibleTarget?.title || ''} ${visibleTarget?.content || ''}`,
1448
+ `${entry.title || ''} ${entry.content || ''}`,
1449
+ )
1450
+ return {
1451
+ id,
1452
+ entry: pendingEntry,
1453
+ pending: true,
1454
+ deduped: true,
1455
+ existingId: duplicate.id,
1456
+ similarity: duplicate.similarity,
1457
+ hint: `库内已有一条几乎相同的记忆;新正文已暂存待复核(near_duplicate)。差异片段:${diff || '(无可见差异)'}。若确属两次独立观测,带 allowDuplicate=true 重写;要合并请改用 memu_private_update。`,
1458
+ }
1459
+ }
1382
1460
  return {
1461
+ id,
1462
+ entry: pendingEntry,
1463
+ pending: true,
1383
1464
  deduped: true,
1384
- existingId: duplicate.id,
1385
- similarity: duplicate.similarity,
1386
- hint: '库内已有一条几乎相同的记忆;改用 memu_private_update 合并内容。若确属两次独立观测,带 allowDuplicate=true 重写',
1465
+ similarityBand: '0.90+',
1466
+ hint: '库内已有一条几乎相同的记忆,但它不在当前会话的可见范围内;新正文已暂存待复核(near_duplicate)。',
1387
1467
  }
1388
1468
  }
1389
1469
  if (duplicate && duplicate.similarity >= TOPIC_DICE) entry.duplicateOf = duplicate.id
@@ -1418,13 +1498,27 @@ async function privateRememberImpl({
1418
1498
  if (sessionIdOf(exec) && (!currentBinding || currentBinding.libraryId !== scope.libraryId)) throw new Error('memory scope changed during write; retry')
1419
1499
  if (currentBinding) assertScopeEnabled(currentBinding)
1420
1500
  if (previousHabit && JSON.stringify(fresh.entries[id]) !== JSON.stringify(previousHabit[1])) throw new Error('habit changed concurrently; recall and retry')
1421
- const latestConflict = findConflict(Object.fromEntries(Object.entries(fresh.entries).filter(([key]) => key !== id)), entry)
1501
+ const latestConflict = findConflict(Object.fromEntries(Object.entries(fresh.entries).filter(([key, other]) => key !== id && visibleForDedupe(other, memorySession))), entry)
1422
1502
  if (latestConflict && !conflict) {
1423
1503
  if (db) throw new Error('memory conflict changed during write; retry for review')
1424
1504
  entry.status = 'pending_review'
1425
1505
  entry.conflictingWith = latestConflict.id
1426
1506
  entry.reviewReason = 'concurrent conflicting memory; review required'
1427
1507
  }
1508
+ // 5.4:预检在 await 之前,提交期间另一个 turn 可能刚写入同事实。临界区用同一
1509
+ // 尺子重扫;命中就 retry(有向量则 throw 让 compensation 撤掉刚提交的行),
1510
+ // lexical 路由直接暂扣为 near_duplicate,不双写也不丢文本。
1511
+ const latestDuplicate = allowDuplicate
1512
+ ? { duplicate: null, hidden: false }
1513
+ : findNearDuplicate(fresh.entries, {
1514
+ id, title: entry.title, content: entry.content, context: entry.context, sessionId: memorySession,
1515
+ })
1516
+ if (latestDuplicate.hidden || (latestDuplicate.duplicate && latestDuplicate.duplicate.similarity >= DUP_REJECT_DICE)) {
1517
+ if (db) throw new Error('memory duplicate changed during write; retry')
1518
+ entry.status = 'pending_review'
1519
+ entry.reviewReason = 'near_duplicate'
1520
+ entry.conflictingWith = latestDuplicate.duplicate?.id || null
1521
+ }
1428
1522
  if (layer === 'must_remember' && pinSource === 'self') {
1429
1523
  const pinned = Object.values(fresh.entries).filter(e => e.layer === 'must_remember' && isMemoryEligible(e)).length
1430
1524
  if (pinned >= Number(getSettings().mustRememberCap || 12)) throw new Error('must_remember soft cap reached; retry as long_term')
@@ -1745,6 +1839,7 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
1745
1839
  const scope = resolveLibrary({ exec, libraryId })
1746
1840
  if (scope.binding) assertScopeEnabled(scope.binding)
1747
1841
  const index = loadPrivateIndex(scope.libraryId)
1842
+ const memorySession = memorySessionId(exec)
1748
1843
  const id_ = resolveEntryId(index, id)
1749
1844
  const entry = index.entries[id_]
1750
1845
  assertEntryAccess(entry, exec)
@@ -1776,29 +1871,46 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
1776
1871
  const nextContext = context === undefined ? entry.context : normalizeMemoryContext(context)
1777
1872
  const scanned = textChanged && entry.status !== 'pending_review'
1778
1873
  let duplicate = null
1874
+ let hiddenDuplicate = null
1779
1875
  let text = null
1780
1876
  if (scanned) {
1781
- const others = Object.fromEntries(Object.entries(index.entries).filter(([key]) => key !== id_))
1782
- const conflict = findConflict(others, { ...entry, title: nextTitle, content: nextContent ?? entry.content })
1877
+ const allOthers = Object.fromEntries(Object.entries(index.entries).filter(([key]) => key !== id_))
1878
+ const visibleOthers = Object.fromEntries(Object.entries(allOthers).filter(([, other]) => visibleForDedupe(other, memorySession)))
1879
+ const conflict = findConflict(visibleOthers, { ...entry, title: nextTitle, content: nextContent ?? entry.content })
1783
1880
  if (conflict) throw new Error(`content conflicts with ${conflict.id}; write a new memory for review`)
1784
1881
  text = `${nextTitle} ${nextContent ?? entry.content}`
1785
- for (const [otherId, other] of Object.entries(others)) {
1882
+ for (const [otherId, other] of Object.entries(allOthers)) {
1786
1883
  // 与 remember 的门同一过滤集(isMemoryEligible + sameFactContext),再排除它原本的
1787
1884
  // duplicateOf 关系——那对条目是写入时就获准共存的,向它继续靠拢不该被拦。
1788
1885
  if (!isMemoryEligible(other)) continue
1789
1886
  if (otherId === entry.duplicateOf) continue
1790
1887
  if (!sameFactContext(other.context, nextContext)) continue
1791
1888
  const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
1889
+ if (!visibleForDedupe(other, memorySession)) {
1890
+ if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
1891
+ continue
1892
+ }
1792
1893
  if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
1793
1894
  }
1895
+ if (hiddenDuplicate) {
1896
+ record({
1897
+ kind: 'write', scope: 'private', sessionId: sessionIdOf(exec) || null, libraryId: scope.libraryId,
1898
+ message: '拒绝重复更新:改写后与当前会话不可见的记忆几乎相同',
1899
+ detail: { id: id_, deduped: true, hidden: true, similarityBand: '0.90+' },
1900
+ refs: [id_],
1901
+ })
1902
+ throw new Error('update refused: a nearly identical memory exists outside this conversation\'s visible scope; rewrite the text or archive one of the two instead')
1903
+ }
1794
1904
  if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
1905
+ const target = index.entries[duplicate.id]
1906
+ const diff = nearDuplicateDiff(`${target?.title || ''} ${target?.content || ''}`, text)
1795
1907
  record({
1796
1908
  kind: 'write', scope: 'private', sessionId: sessionIdOf(exec) || null, libraryId: scope.libraryId,
1797
1909
  message: '拒绝重复更新:' + id_ + ' 改写后与 ' + duplicate.id + ' 几乎相同',
1798
1910
  detail: { id: id_, deduped: true, existingId: duplicate.id, similarity: duplicate.similarity },
1799
1911
  refs: [id_, duplicate.id],
1800
1912
  })
1801
- throw new Error(`update refused: the rewritten text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); merge into ${duplicate.id} or archive one of the two instead`)
1913
+ throw new Error(`update refused: the rewritten text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); differences: ${diff || '(none visible)'}; merge into ${duplicate.id} or archive one of the two instead`)
1802
1914
  }
1803
1915
  }
1804
1916
  const route = engineRoute()
@@ -1831,6 +1943,16 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
1831
1943
  const currentBinding = peekBinding(exec)
1832
1944
  if (sessionIdOf(exec) && (!currentBinding || currentBinding.libraryId !== scope.libraryId)) throw new Error('memory scope changed during update; retry')
1833
1945
  if (currentBinding) assertScopeEnabled(currentBinding)
1946
+ if (scanned) {
1947
+ const latestDuplicate = findNearDuplicate(fresh.entries, {
1948
+ id: id_, title: nextTitle, content: nextContent ?? entry.content, context: nextContext,
1949
+ sessionId: memorySession, excludeId: entry.duplicateOf,
1950
+ })
1951
+ if (latestDuplicate.hidden || (latestDuplicate.duplicate && latestDuplicate.duplicate.similarity >= DUP_REJECT_DICE)) {
1952
+ throw new Error('duplicate changed during update; retry')
1953
+ }
1954
+ if (latestDuplicate.duplicate) duplicate = latestDuplicate.duplicate
1955
+ }
1834
1956
  if (title !== undefined) target.title = clampText(title, 120)
1835
1957
  if (nextContent !== undefined) target.content = nextContent
1836
1958
  // 打标带与 remember 同口径:[0.55, 0.90) 改写后仍与另一条同主题 → duplicateOf。
@@ -1947,14 +2069,19 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
1947
2069
  }
1948
2070
 
1949
2071
  /** Review activation and invalidation of the previous claim are one index commit. */
1950
- async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolution = 'replace', reason = '', content, title }) {
2072
+ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolution = 'replace', reason = '', content, title, sessionId = null }) {
1951
2073
  if (!['accept', 'reject'].includes(action)) throw new Error('action must be accept or reject')
1952
- if (!['replace', 'coexist'].includes(resolution)) throw new Error('resolution must be replace or coexist')
1953
- if (resolution === 'coexist' && String(reason).trim().length < 8) throw new Error('coexist needs a reason describing the distinct conditions (at least 8 characters)')
1954
2074
  if (libraryId && !getLibrary(libraryId)) throw new Error('library not found')
1955
2075
  const index = libraryId ? loadPrivateIndex(libraryId) : loadGlobalIndex()
1956
2076
  const entry = index.entries[id]
1957
2077
  if (!entry || entry.status !== 'pending_review') throw new Error('no such pending review')
2078
+ // pendingAbsorb(spec §4.11 review-all / 安全门触发的 pending absorb):走专用复核
2079
+ // 分支——提案是"整簇合并稿",accept/reject 的落库动作与普通复核完全不同。
2080
+ if (!libraryId && entry.pendingAbsorb) {
2081
+ return resolveAbsorbReviewImpl({ id, entry, index, action, resolution, reason, content, title, sessionId })
2082
+ }
2083
+ if (!['replace', 'coexist'].includes(resolution)) throw new Error('resolution must be replace or coexist')
2084
+ if (resolution === 'coexist' && String(reason).trim().length < 8) throw new Error('coexist needs a reason describing the distinct conditions (at least 8 characters)')
1958
2085
  const next = { ...entry }
1959
2086
  if (action === 'accept') {
1960
2087
  if (typeof content === 'string' && content.trim()) next.content = content
@@ -1966,21 +2093,37 @@ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolutio
1966
2093
  // 复核的本来含义,不是重复。
1967
2094
  const text = `${next.title || ''} ${next.content || ''}`
1968
2095
  let duplicate = null
2096
+ let hiddenDuplicate = null
1969
2097
  for (const [otherId, other] of Object.entries(index.entries || {})) {
1970
2098
  if (otherId === id || otherId === entry.conflictingWith) continue
1971
2099
  if (!isMemoryEligible(other)) continue
1972
2100
  if (!sameFactContext(other.context, next.context)) continue
1973
2101
  const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
2102
+ if (!visibleForDedupe(other, sessionId)) {
2103
+ if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
2104
+ continue
2105
+ }
1974
2106
  if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
1975
2107
  }
2108
+ if (hiddenDuplicate) {
2109
+ record({
2110
+ kind: 'write', scope: libraryId ? 'private' : 'global', libraryId: libraryId || null,
2111
+ message: '拒绝复核接受:接受稿与当前会话不可见的记忆几乎相同',
2112
+ detail: { id, deduped: true, hidden: true, similarityBand: '0.90+' },
2113
+ refs: [id],
2114
+ })
2115
+ throw new Error('review accept refused: a nearly identical memory exists outside the visible scope; pass different content or change the review scope first')
2116
+ }
1976
2117
  if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
2118
+ const target = index.entries[duplicate.id]
2119
+ const diff = nearDuplicateDiff(`${target?.title || ''} ${target?.content || ''}`, text)
1977
2120
  record({
1978
2121
  kind: 'write', scope: libraryId ? 'private' : 'global', libraryId: libraryId || null,
1979
2122
  message: '拒绝复核接受:' + id + ' 与 ' + duplicate.id + ' 几乎相同',
1980
2123
  detail: { id, deduped: true, existingId: duplicate.id, similarity: duplicate.similarity },
1981
2124
  refs: [id, duplicate.id],
1982
2125
  })
1983
- throw new Error(`review accept refused: the accepted text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); pass different content or archive ${duplicate.id} first`)
2126
+ throw new Error(`review accept refused: the accepted text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); differences: ${diff || '(none visible)'}; pass different content or archive ${duplicate.id} first`)
1984
2127
  }
1985
2128
  if (duplicate && duplicate.similarity >= TOPIC_DICE) next.duplicateOf = duplicate.id
1986
2129
  const reviewRoute = engineRoute()
@@ -2000,6 +2143,15 @@ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolutio
2000
2143
  || JSON.stringify(fresh.entries[entry.conflictingWith]) !== JSON.stringify(index.entries[entry.conflictingWith])) {
2001
2144
  throw new Error('review changed; reload and retry')
2002
2145
  }
2146
+ if (action === 'accept') {
2147
+ const latestDuplicate = findNearDuplicate(fresh.entries, {
2148
+ id, title: next.title, content: next.content, context: next.context,
2149
+ sessionId, excludeId: entry.conflictingWith,
2150
+ })
2151
+ if (latestDuplicate.hidden || (latestDuplicate.duplicate && latestDuplicate.duplicate.similarity >= DUP_REJECT_DICE)) {
2152
+ throw new Error('review changed; reload and retry')
2153
+ }
2154
+ }
2003
2155
  const at = stamp()
2004
2156
  next.status = action === 'accept' ? 'active' : 'archived'
2005
2157
  next.reviewedAt = at
@@ -2024,6 +2176,119 @@ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolutio
2024
2176
  return { id, status: next.status, resolution }
2025
2177
  }
2026
2178
 
2179
+ /**
2180
+ * pendingAbsorb 复核分支(spec §3.3/§4.11/§6.7)。只作用于全局队列(私有索引里
2181
+ * 不存在 pendingAbsorb)。
2182
+ *
2183
+ * accept:resolution 必须 replace——合并稿不能与自己的来源"并存",coexist 直接抛错;
2184
+ * 重复门排除 [id, conflictingWith, ...sourceIds],否则合并稿会被自己的来源判重;
2185
+ * 快照校验 target/sources 仍 active 且 updatedAt === expectedUpdatedAt,否则抛
2186
+ * 'review changed; reload and retry';向量库只提交提案 id 本身(绝不提交 sources,
2187
+ * §6.7 复核);然后单事务落 §4.11 步骤 3c-6 并清 pendingAbsorb/reviewReason/
2188
+ * conflictingWith。索引写失败时 pending 保持、已提交向量行未激活(现有行为,
2189
+ * 本版不额外补偿)。
2190
+ *
2191
+ * reject:提案 archived;target/sources 原样。
2192
+ */
2193
+ async function resolveAbsorbReviewImpl({ id, entry, index, action, resolution, reason, content, title, sessionId = null }) {
2194
+ const pa = entry.pendingAbsorb || {}
2195
+ const targetId = String(pa.targetId || '')
2196
+ const sourceIds = (Array.isArray(pa.sourceIds) ? pa.sourceIds : []).map(String)
2197
+ if (action === 'reject') {
2198
+ await retryStoreWrite(() => updateGlobalIndex((fresh) => {
2199
+ if (JSON.stringify(fresh.entries[id]) !== JSON.stringify(entry)) throw new Error('review changed; reload and retry')
2200
+ const at = stamp()
2201
+ const next = { ...entry, status: 'archived', reviewedAt: at, updatedAt: at, reviewResolution: resolution }
2202
+ if (reason) next.reviewNote = String(reason).trim().slice(0, 1000)
2203
+ fresh.entries[id] = next
2204
+ }))
2205
+ return { id, status: 'archived', resolution }
2206
+ }
2207
+ // —— accept ——
2208
+ if (resolution !== 'replace') {
2209
+ throw new Error('absorb review accept requires the replace resolution; a merged entry cannot coexist with its own sources')
2210
+ }
2211
+ const next = { ...entry }
2212
+ if (typeof content === 'string' && content.trim()) next.content = content
2213
+ if (typeof title === 'string' && title.trim()) next.title = title
2214
+ const exclude = new Set([id, entry.conflictingWith, targetId, ...sourceIds].filter(Boolean))
2215
+ const text = `${next.title || ''} ${next.content || ''}`
2216
+ let duplicate = null
2217
+ let hiddenDuplicate = null
2218
+ for (const [otherId, other] of Object.entries(index.entries || {})) {
2219
+ if (exclude.has(otherId)) continue
2220
+ if (!isMemoryEligible(other)) continue
2221
+ if (!sameFactContext(other.context, next.context)) continue
2222
+ const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
2223
+ if (!visibleForDedupe(other, sessionId)) {
2224
+ if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
2225
+ continue
2226
+ }
2227
+ if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
2228
+ }
2229
+ if (hiddenDuplicate) {
2230
+ record({
2231
+ kind: 'write', scope: 'global', libraryId: null,
2232
+ message: '拒绝集合吸收接受:接受稿与当前会话不可见的记忆几乎相同',
2233
+ detail: { id, deduped: true, hidden: true, similarityBand: '0.90+' },
2234
+ refs: [id],
2235
+ })
2236
+ throw new Error('review accept refused: a nearly identical memory exists outside the visible scope; pass different content or change the review scope first')
2237
+ }
2238
+ if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
2239
+ const target = index.entries[duplicate.id]
2240
+ const diff = nearDuplicateDiff(`${target?.title || ''} ${target?.content || ''}`, text)
2241
+ record({
2242
+ kind: 'write', scope: 'global', libraryId: null,
2243
+ message: '拒绝集合吸收接受:' + id + ' 与 ' + duplicate.id + ' 几乎相同',
2244
+ detail: { id, deduped: true, existingId: duplicate.id, similarity: duplicate.similarity },
2245
+ refs: [id, duplicate.id],
2246
+ })
2247
+ throw new Error(`review accept refused: the accepted text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); differences: ${diff || '(none visible)'}; pass different content or archive ${duplicate.id} first`)
2248
+ }
2249
+ if (duplicate && duplicate.similarity >= TOPIC_DICE) next.duplicateOf = duplicate.id
2250
+ // 快照校验(§4.11 accept 3):target/sources 仍 active 且 updatedAt 未被并发改写。
2251
+ const expected = pa.expectedUpdatedAt || {}
2252
+ for (const memberId of [targetId, ...sourceIds].filter(Boolean)) {
2253
+ const current = index.entries[memberId]
2254
+ if (!current || current.status !== 'active'
2255
+ || (expected[memberId] !== undefined && current.updatedAt !== expected[memberId])) {
2256
+ throw new Error('review changed; reload and retry')
2257
+ }
2258
+ }
2259
+ // 向量提交:只提交提案 id,绝不提交 sources(§6.7 复核)。
2260
+ const reviewRoute = engineRoute()
2261
+ if (reviewRoute !== 'lexical') {
2262
+ await commit({
2263
+ payload: { recall_files: [{ name: id, track: next.track || 'memory', description: clampText(next.title, 180), content: next.content }], resource: [], user: {} },
2264
+ route: reviewRoute,
2265
+ db: globalDbPath(),
2266
+ })
2267
+ next.vector = activeEntryStamp(reviewRoute) || next.vector
2268
+ }
2269
+ // 单事务:§4.11 步骤 3c-6。merged 入索引、target superseded、sources archived、
2270
+ // rewire links、写 absorbSnapshot,然后清 pendingAbsorb/reviewReason/conflictingWith。
2271
+ await retryStoreWrite(() => updateGlobalIndex((fresh) => {
2272
+ if (JSON.stringify(fresh.entries[id]) !== JSON.stringify(entry)) throw new Error('review changed; reload and retry')
2273
+ const at = stamp()
2274
+ next.status = 'active'
2275
+ next.updatedAt = at
2276
+ next.reviewedAt = at
2277
+ next.reviewResolution = resolution
2278
+ next.validFrom = at
2279
+ if (reason) next.reviewNote = String(reason).trim().slice(0, 1000)
2280
+ const outcome = applyAbsorbIndexTransaction(fresh, {
2281
+ mergedId: id, merged: next, targetId, sourceIds, expectedUpdatedAt: expected, now: at,
2282
+ })
2283
+ if (!outcome.landed) throw new Error('review changed; reload and retry')
2284
+ const stored = fresh.entries[id]
2285
+ delete stored.pendingAbsorb
2286
+ delete stored.reviewReason
2287
+ delete stored.conflictingWith
2288
+ }))
2289
+ return { id, status: 'active', resolution }
2290
+ }
2291
+
2027
2292
  /**
2028
2293
  * Short-term entries expire instead of being deleted. Deleting would throw away the
2029
2294
  * only copy before the consolidation pass could decide what it was worth; flipping
package/lib/memu.js CHANGED
@@ -2,7 +2,7 @@ import fs from 'node:fs'
2
2
  import path from 'node:path'
3
3
  import { spawn } from 'node:child_process'
4
4
  import { dataDir, ensureDir, redactSecrets, windowsSystemEnv } from './paths.js'
5
- import { readCurrent, verifyEngine } from './engine.js'
5
+ import { ENGINE_VERSION, readCurrent, verifyEngine } from './engine.js'
6
6
  import { getSettings } from './state.js'
7
7
  import { activeProfile, captureVectorGeneration, assertVectorGeneration, assertProfileVerified, assertMigrationProfile } from './vector-profile.js'
8
8
  import { embeddingWorkerManager } from './local-embedding-manager.js'
@@ -219,6 +219,6 @@ export async function forget({ db, name, timeoutMs = 60_000 } = {}) {
219
219
 
220
220
  export async function probe() {
221
221
  const current = currentEngine()
222
- const verified = await verifyEngine(current.python)
222
+ const verified = await verifyEngine(current.python, { expectedVersion: current.version || ENGINE_VERSION })
223
223
  return { ...verified, current }
224
224
  }
package/lib/migrate.js CHANGED
@@ -7,11 +7,18 @@
7
7
  // the pass exists to rewrite the files: a library nobody writes to again would otherwise
8
8
  // keep the dead `sessionId` fields on disk forever, and an auditor reading the JSON would
9
9
  // have to know about a level the code no longer has.
10
+ //
11
+ // The lexicon half (§2.1 of the construction spec) is the same shape: older builds
12
+ // wrote `pending` proposals and revision drafts; the current lifecycle has only active/archived. The rewrite rule
13
+ // lives in `lib/lexicon.js` (`migrateLegacyLexiconStatuses`); this file only finds the
14
+ // files and runs it, one library at a time, so a single corrupt lexicon cannot take the
15
+ // sweep — or startup — down with it.
10
16
  import fs from 'node:fs'
11
17
  import path from 'node:path'
12
- import { indexDir, readJson, stamp } from './paths.js'
18
+ import { dataDir, indexDir, isSafeId, readJson, readJsonStrict, stamp } from './paths.js'
13
19
  import { listLibrariesAll, updatePrivateIndex } from './state.js'
14
20
  import { countLegacyInRaw, migrateEntryVisibility } from './visibility.js'
21
+ import { migrateLegacyLexiconStatuses } from './lexicon.js'
15
22
 
16
23
  const DEFAULT_MIN_INTERVAL_MS = 60_000
17
24
  const lastRun = { at: 0 }
@@ -109,11 +116,85 @@ export function maybeMigrateLibraries({ minIntervalMs = DEFAULT_MIN_INTERVAL_MS,
109
116
  return migrateLibraries({ now })
110
117
  }
111
118
 
119
+ // ── lexicon lifecycle (§2.1): pending → active/archived ─────────────────────────
120
+ // The directory holds exactly `<libraryId>.json`; a name that does not pass `isSafeId`
121
+ // is skipped rather than guessed at, the same path boundary `lib/lexicon.js` enforces
122
+ // before it builds a filename.
123
+
124
+ function lexiconDir() {
125
+ return path.join(dataDir(), 'lexicon')
126
+ }
127
+
128
+ /** Library ids that own a lexicon file. Sorted, safe names only. */
129
+ export function scanLexiconLibraries() {
130
+ let names = []
131
+ try { names = fs.readdirSync(lexiconDir()) } catch { return [] }
132
+ return names
133
+ .filter((name) => name.endsWith('.json'))
134
+ .map((name) => name.slice(0, -'.json'.length))
135
+ .filter((id) => isSafeId(id))
136
+ .sort()
137
+ }
138
+
139
+ // Counted on the raw file, before any load-time interpretation, so a sweep can tell a
140
+ // library that still needs the rewrite from one that is already migrated. `readJsonStrict`
141
+ // (not `readJson`) on purpose: a scan must not move files, and `readJson` quarantines a
142
+ // corrupt one as a side effect. A file that does not parse counts 0 — it cannot hold
143
+ // statuses to migrate, and `lib/lexicon.js` already blocks writes to it until restored.
144
+ function legacyLexiconCount(libraryId) {
145
+ const result = readJsonStrict(path.join(lexiconDir(), `${libraryId}.json`))
146
+ if (!result.ok) return 0
147
+ const terms = result.value?.terms
148
+ if (!terms || typeof terms !== 'object') return 0
149
+ return Object.values(terms).filter((entry) => entry && entry.status === 'pending').length
150
+ }
151
+
152
+ /** Lexicon files that still hold legacy `pending` entries. */
153
+ export function scanLegacyLexiconLibraries() {
154
+ return scanLexiconLibraries()
155
+ .map((libraryId) => ({ libraryId, legacy: legacyLexiconCount(libraryId) }))
156
+ .filter((report) => report.legacy > 0)
157
+ }
158
+
159
+ /**
160
+ * Rewrite every lexicon that still carries `pending` entries.
161
+ *
162
+ * Per-library try/catch is the point: a single corrupt or write-protected lexicon must
163
+ * not stop the others, and must never abort startup (the pass runs from the cycle tick).
164
+ * A library with nothing to do is not written at all — `migrateLegacyLexiconStatuses`
165
+ * skips the write (and the revision bump) when no entry changed.
166
+ */
167
+ export function migrateLexiconLibraries() {
168
+ const reports = []
169
+ for (const { libraryId, legacy } of scanLegacyLexiconLibraries()) {
170
+ try {
171
+ const result = migrateLegacyLexiconStatuses(libraryId)
172
+ reports.push({
173
+ libraryId,
174
+ before: legacy,
175
+ migrated: result.migrated,
176
+ ...(result.details ? { details: result.details } : {}),
177
+ })
178
+ } catch (error) {
179
+ reports.push({ libraryId, before: legacy, migrated: 0, error: String(error?.message || error) })
180
+ }
181
+ }
182
+ return reports
183
+ }
184
+
112
185
  export function runVisibilityMigrationPass({ now = stamp(), log = () => {} } = {}) {
113
186
  const reports = migrateLibraries({ now })
114
187
  for (const report of reports) {
115
188
  if (report.error) log(`visibility migration failed for ${report.libraryId}: ${report.error}`)
116
189
  else if (report.migrated > 0) log(`migrated ${report.migrated} legacy memory/memories to shared in ${report.libraryId}`)
117
190
  }
118
- return { reports }
191
+ // §2.1: legacy lexicons may still carry pending proposals/revision drafts. The rewrite is
192
+ // idempotent and only writes files that change, so running it on every cycle tick
193
+ // costs a directory read plus one JSON parse per file that still has pending entries.
194
+ const lexicon = migrateLexiconLibraries()
195
+ for (const report of lexicon) {
196
+ if (report.error) log(`lexicon lifecycle migration failed for ${report.libraryId}: ${report.error}`)
197
+ else if (report.migrated > 0) log(`migrated ${report.migrated} legacy lexicon term(s) to the active/archived lifecycle in ${report.libraryId}`)
198
+ }
199
+ return { reports, lexicon }
119
200
  }