dsh-memu-memory 1.2.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/README.md +12 -5
- package/README.zh.md +4 -2
- package/docs/contributing.md +16 -3
- package/docs/host-api.md +3 -1
- package/docs/memory-semantics.md +21 -2
- package/docs/troubleshooting.md +42 -4
- package/lib/backup.js +24 -2
- package/lib/binding-diagnostics.js +2 -2
- package/lib/client.js +1361 -146
- package/lib/consolidation-worker.js +30 -8
- package/lib/curator-evidence.js +131 -0
- package/lib/curator-policy.js +81 -0
- package/lib/curator.js +1252 -91
- package/lib/cycle.js +3 -0
- package/lib/embedding-download.js +39 -4
- package/lib/engine.js +6 -3
- package/lib/global-graph.js +282 -0
- package/lib/graph.js +51 -9
- package/lib/inbound.js +19 -0
- package/lib/inject.js +277 -17
- package/lib/lexicon.js +1115 -0
- package/lib/local-vector-service.js +8 -6
- package/lib/managed-runtime.js +33 -2
- package/lib/memory.js +282 -17
- package/lib/memu.js +2 -2
- package/lib/migrate.js +83 -2
- package/lib/private-links.js +587 -0
- package/lib/private.js +32 -3
- package/lib/prompts/global-curator.md +76 -19
- package/lib/prompts/private-consolidation.coding.md +4 -1
- package/lib/prompts/private-consolidation.md +3 -1
- package/lib/prompts/private-consolidation.mixed.md +3 -1
- package/lib/prompts/private-consolidation.rp.md +3 -1
- package/lib/prompts.js +1 -1
- package/lib/rpc.js +577 -9
- package/lib/state.js +4 -1
- package/lib/text-retrieval.js +661 -0
- package/lib/text-similarity.js +36 -18
- package/lib/tools.js +168 -10
- package/lib/webui/standalone.css +17 -0
- package/lib/webui/standalone.js +2 -0
- package/package.json +1 -1
- package/scripts/acceptance-local.mjs +8 -5
- package/scripts/curator-pair-calibrate.mjs +259 -0
- package/scripts/lexicon-calibrate.mjs +230 -0
- package/scripts/lexicon-dedupe.mjs +318 -0
- package/tests/absorb-review.test.mjs +353 -0
- package/tests/appendix-b2.test.mjs +2 -2
- package/tests/client-export.test.mjs +520 -0
- package/tests/client-render.test.mjs +19 -3
- package/tests/consolidation-fork.test.mjs +14 -12
- package/tests/consolidation-snapshot.test.mjs +39 -0
- package/tests/curator-absorb.test.mjs +443 -0
- package/tests/curator-calibrate.test.mjs +173 -0
- package/tests/curator-clusters.test.mjs +281 -0
- package/tests/curator-commit.test.mjs +52 -5
- package/tests/curator-evidence.test.mjs +102 -0
- package/tests/curator-link.test.mjs +316 -0
- package/tests/curator-pairing.test.mjs +252 -0
- package/tests/curator-probe.test.mjs +15 -6
- package/tests/curator-review-mode.test.mjs +314 -0
- package/tests/disclosure.test.mjs +2 -1
- package/tests/fixtures/curator-pairs/labeled.v1.jsonl +90 -0
- package/tests/fixtures/lexicon-stub-hooks.mjs +15 -0
- package/tests/fixtures/lexicon-stub.mjs +284 -0
- package/tests/graph-cascade.test.mjs +35 -2
- package/tests/graph.test.mjs +96 -0
- package/tests/help-page.test.mjs +415 -0
- package/tests/inbound.test.mjs +6 -3
- package/tests/inject-guide.test.mjs +28 -0
- package/tests/inject.test.mjs +157 -5
- package/tests/lexicon-calibrate.test.mjs +48 -0
- package/tests/lexicon-dedupe.test.mjs +171 -0
- package/tests/lexicon-frequency.test.mjs +179 -0
- package/tests/lexicon-inject.test.mjs +410 -0
- package/tests/lexicon-perf.test.mjs +436 -0
- package/tests/lexicon-ui.test.mjs +547 -0
- package/tests/lexicon.test.mjs +922 -0
- package/tests/managed-runtime.test.mjs +32 -0
- package/tests/private-links.test.mjs +445 -0
- package/tests/prompts.test.mjs +35 -0
- package/tests/recall-dedup.test.mjs +33 -4
- package/tests/tool-schema.test.mjs +1 -1
- package/tests/tools.test.mjs +112 -3
- package/tests/webui-auth.test.mjs +348 -2
- package/tests/webui.test.mjs +188 -0
|
@@ -89,17 +89,19 @@ function profileFor(catalogId) {
|
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
function installationFor(id, sharedRuntime = null) {
|
|
92
|
-
const status = getDownloader().getStatus(id, { deep:
|
|
92
|
+
const status = getDownloader().getStatus(id, { deep: true })
|
|
93
93
|
// 所有可用模型 pin 的是同一个受管 runtime;每个模型各走一遍运行时目录树(几百个文件)
|
|
94
94
|
// 会让每次 getState 多花几百毫秒。上层已经算过一次就复用。
|
|
95
95
|
const runtime = sharedRuntime || localVectorRuntimeStatus({ modelId: id })
|
|
96
|
+
const degraded = status.state === 'installed' && status.verified === false
|
|
96
97
|
return {
|
|
97
|
-
status: status.state,
|
|
98
|
-
installed: status.installed === true,
|
|
99
|
-
|
|
98
|
+
status: degraded ? 'degraded' : status.state,
|
|
99
|
+
installed: status.installed === true && !degraded,
|
|
100
|
+
verified: status.verified ?? null,
|
|
101
|
+
modelDir: status.installed && !degraded ? path.join(root, id) : null,
|
|
100
102
|
runtimeDir: runtime.ready ? runtime.path : null,
|
|
101
103
|
fingerprintMatches: status.fingerprintMatches ?? null,
|
|
102
|
-
error: status.error || null,
|
|
104
|
+
error: degraded ? { code: 'EMBEDDING_INSTALL_TAMPERED', message: 'installed embedding files failed deep verification' } : (status.error || null),
|
|
103
105
|
progress: status.progress || null,
|
|
104
106
|
}
|
|
105
107
|
}
|
|
@@ -164,7 +166,7 @@ export function getLocalVectorState() {
|
|
|
164
166
|
|
|
165
167
|
export function selectLocalVectorModel(id) {
|
|
166
168
|
const status = getDownloader().getStatus(String(id), { deep: true })
|
|
167
|
-
if (status.state !== 'installed' || status.fingerprintMatches === false) {
|
|
169
|
+
if (status.state !== 'installed' || status.verified !== true || status.fingerprintMatches === false) {
|
|
168
170
|
const error = new Error('install and verify the local embedding model before selecting it')
|
|
169
171
|
error.code = 'LOCAL_VECTOR_MODEL_NOT_INSTALLED'
|
|
170
172
|
throw error
|
package/lib/managed-runtime.js
CHANGED
|
@@ -20,6 +20,28 @@ export function verifyRuntimeArchive(data, digest) {
|
|
|
20
20
|
if (crypto.createHash('sha256').update(data).digest('hex') !== digest) throw new Error('managed runtime checksum mismatch; refusing to execute')
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
function executableFingerprint(file) {
|
|
24
|
+
try {
|
|
25
|
+
const stat = fs.lstatSync(file)
|
|
26
|
+
if (!stat.isFile() || stat.isSymbolicLink()) return null
|
|
27
|
+
const sha256 = crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex')
|
|
28
|
+
return { bytes: stat.size, sha256 }
|
|
29
|
+
} catch {
|
|
30
|
+
return null
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function verifiedRuntimeMetadata(file, asset) {
|
|
35
|
+
try {
|
|
36
|
+
const metadata = JSON.parse(fs.readFileSync(file, 'utf8'))
|
|
37
|
+
return metadata && metadata.version === UV_VERSION && metadata.archive === asset[0] && metadata.archiveSha256 === asset[1] && Number.isInteger(metadata.executableBytes) && typeof metadata.executableSha256 === 'string'
|
|
38
|
+
? metadata
|
|
39
|
+
: null
|
|
40
|
+
} catch {
|
|
41
|
+
return null
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
23
45
|
export async function ensureManagedPython({ run, log = () => {}, fetcher = fetch } = {}) {
|
|
24
46
|
const platform = `${process.platform}-${process.arch}`
|
|
25
47
|
const asset = ASSETS[platform]
|
|
@@ -33,7 +55,14 @@ export async function ensureManagedPython({ run, log = () => {}, fetcher = fetch
|
|
|
33
55
|
UV_PYTHON_BIN_DIR: ensureDir(path.join(root, 'bin')),
|
|
34
56
|
UV_NO_PROGRESS: '1', UV_PYTHON_PREFERENCE: 'only-managed', UV_NO_MODIFY_PATH: '1', UV_NO_CONFIG: '1',
|
|
35
57
|
}
|
|
36
|
-
|
|
58
|
+
const metadataFile = path.join(bin, 'verified.json')
|
|
59
|
+
const metadata = verifiedRuntimeMetadata(metadataFile, asset)
|
|
60
|
+
const fingerprint = executableFingerprint(executable)
|
|
61
|
+
const reusable = Boolean(metadata && fingerprint && metadata.executableBytes === fingerprint.bytes && metadata.executableSha256 === fingerprint.sha256)
|
|
62
|
+
if (!reusable) {
|
|
63
|
+
if (fingerprint || fs.existsSync(executable)) log(`managed uv ${UV_VERSION} is missing trusted metadata or was modified; replacing it`)
|
|
64
|
+
try { fs.rmSync(executable, { force: true }) } catch {}
|
|
65
|
+
try { fs.rmSync(metadataFile, { force: true }) } catch {}
|
|
37
66
|
log(`no usable Python with pip; downloading pinned uv ${UV_VERSION}`)
|
|
38
67
|
const response = await fetcher(`https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/${asset[0]}`, { signal: AbortSignal.timeout(120_000) })
|
|
39
68
|
if (!response.ok || Number(response.headers.get('content-length')) > 64 * 1024 * 1024) throw new Error('managed runtime download failed or is too large')
|
|
@@ -63,7 +92,9 @@ export async function ensureManagedPython({ run, log = () => {}, fetcher = fetch
|
|
|
63
92
|
}
|
|
64
93
|
if (fs.existsSync(temporary)) fs.renameSync(temporary, executable)
|
|
65
94
|
if (process.platform !== 'win32') fs.chmodSync(executable, 0o700)
|
|
66
|
-
|
|
95
|
+
const installedFingerprint = executableFingerprint(executable)
|
|
96
|
+
if (!installedFingerprint) throw new Error('managed runtime executable is not a regular file')
|
|
97
|
+
writeJson(metadataFile, { version: UV_VERSION, archive: asset[0], archiveSha256: asset[1], executableBytes: installedFingerprint.bytes, executableSha256: installedFingerprint.sha256 })
|
|
67
98
|
}
|
|
68
99
|
const version = await run(executable, ['--version'], { env, timeoutMs: 20_000 })
|
|
69
100
|
if (version.code !== 0 || !version.stdout.startsWith(`uv ${UV_VERSION} `) && version.stdout.trim() !== `uv ${UV_VERSION}`) throw new Error('managed uv version mismatch')
|
package/lib/memory.js
CHANGED
|
@@ -20,6 +20,7 @@ import { attachmentRefs } from './media.js'
|
|
|
20
20
|
import { normalizeMemoryContext, contextVisible, sameFactContext } from './context.js'
|
|
21
21
|
import { activeEntryStamp, assertGeneration, assertVectorWriteAllowed, captureGeneration, enrollActiveLibrary, hasActiveVectorGeneration, isVectorStateError, readVectorState, withVectorWrite } from './vector-profile.js'
|
|
22
22
|
import { bigrams, dice, similarTexts, DUP_DICE, DUP_REJECT_DICE, TOPIC_DICE } from './text-similarity.js'
|
|
23
|
+
import { applyAbsorbIndexTransaction } from './curator.js'
|
|
23
24
|
|
|
24
25
|
const HALF_LIFE_DAYS = 14
|
|
25
26
|
// A short-term entry is meant to live for the length of a conversation, not a
|
|
@@ -595,6 +596,48 @@ export function findConflict(entries, candidate) {
|
|
|
595
596
|
return null
|
|
596
597
|
}
|
|
597
598
|
|
|
599
|
+
// Duplicate/conflict gates may only consult entries this conversation could
|
|
600
|
+
// actually surface. `secret` never auto-surfaces; knownBy/context-hidden rows are
|
|
601
|
+
// equally invisible. A hidden hit must not leak its id (5.1).
|
|
602
|
+
function visibleForDedupe(entry, sessionId) {
|
|
603
|
+
if (!entry || entry.visibility === 'secret') return false
|
|
604
|
+
return sessionId ? visibleToSession(entry, sessionId) : true
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
// Critical-section re-check for the duplicate gate (5.4): the pre-flight scan
|
|
608
|
+
// runs before the backend await, so a concurrent writer can land a twin while the
|
|
609
|
+
// commit is in flight. Same ruler, same visible scope, one pass inside the lock.
|
|
610
|
+
function findNearDuplicate(entries, { id, title, content, context, sessionId, excludeId = null }) {
|
|
611
|
+
const text = `${title || ''} ${content || ''}`
|
|
612
|
+
let duplicate = null
|
|
613
|
+
let hidden = false
|
|
614
|
+
for (const [otherId, other] of Object.entries(entries || {})) {
|
|
615
|
+
if (otherId === id || otherId === excludeId) continue
|
|
616
|
+
if (!isMemoryEligible(other)) continue
|
|
617
|
+
if (!sameFactContext(other.context, context)) continue
|
|
618
|
+
const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
|
|
619
|
+
if (!visibleForDedupe(other, sessionId)) {
|
|
620
|
+
if (similarity >= DUP_REJECT_DICE) hidden = true
|
|
621
|
+
continue
|
|
622
|
+
}
|
|
623
|
+
if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
|
|
624
|
+
}
|
|
625
|
+
return { duplicate, hidden }
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
// The visible tail that the rewrite adds/changes, for a rejection response. Only
|
|
629
|
+
// ever computed against a VISIBLE duplicate — a hidden twin's text must not leak.
|
|
630
|
+
function nearDuplicateDiff(existingText, candidateText, max = 180) {
|
|
631
|
+
const a = String(existingText || '')
|
|
632
|
+
const b = String(candidateText || '')
|
|
633
|
+
let start = 0
|
|
634
|
+
while (start < a.length && start < b.length && a[start] === b[start]) start += 1
|
|
635
|
+
let end = 0
|
|
636
|
+
while (end < a.length - start && end < b.length - start && a[a.length - 1 - end] === b[b.length - 1 - end]) end += 1
|
|
637
|
+
const added = b.slice(start, b.length - end).trim()
|
|
638
|
+
return clampText(added || b, max)
|
|
639
|
+
}
|
|
640
|
+
|
|
598
641
|
function mirrorContent(file) {
|
|
599
642
|
const raw = file?.content
|
|
600
643
|
if (typeof raw === 'string' && raw.trim()) return raw
|
|
@@ -1350,7 +1393,10 @@ async function privateRememberImpl({
|
|
|
1350
1393
|
// the library is not silently appended: it lands as pending_review with the id it
|
|
1351
1394
|
// collides with, so the conflict surfaces instead of both copies being read as
|
|
1352
1395
|
// true. Same mechanism the global curator uses; until now the private side had none.
|
|
1353
|
-
|
|
1396
|
+
// Hidden rows are excluded (5.1): otherwise a visible write could learn an id
|
|
1397
|
+
// through conflictingWith, and a hidden claim could hold an unrelated new one.
|
|
1398
|
+
const memorySession = memorySessionId(exec)
|
|
1399
|
+
const conflict = findConflict(Object.fromEntries(Object.entries(index.entries).filter(([key, other]) => key !== id && visibleForDedupe(other, memorySession))), entry)
|
|
1354
1400
|
if (conflict) {
|
|
1355
1401
|
entry.status = 'pending_review'
|
|
1356
1402
|
entry.conflictingWith = conflict.id
|
|
@@ -1366,24 +1412,58 @@ async function privateRememberImpl({
|
|
|
1366
1412
|
// Never deletes or rewrites anything: a hit either refuses the new entry, or tags
|
|
1367
1413
|
// it duplicateOf so recall can keep one seat per fact.
|
|
1368
1414
|
let duplicate = null
|
|
1415
|
+
let hiddenDuplicate = null
|
|
1369
1416
|
if (!conflict && allowDuplicate !== true) {
|
|
1370
1417
|
const text = `${entry.title || ''} ${content || ''}`
|
|
1371
|
-
const grams = bigrams(text)
|
|
1372
1418
|
for (const [otherId, other] of Object.entries(index.entries)) {
|
|
1373
|
-
// Same filter set the recall side uses: only an entry a recall could actually
|
|
1374
|
-
// surface can be "the same fact" — and a duplicateOf tag must point at one.
|
|
1375
1419
|
if (otherId === id || !isMemoryEligible(other)) continue
|
|
1376
1420
|
if (!sameFactContext(other.context, entry.context)) continue
|
|
1377
1421
|
const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
|
|
1422
|
+
if (!visibleForDedupe(other, memorySession)) {
|
|
1423
|
+
// Hidden near-duplicate: refuse generically, never return its id/similarity.
|
|
1424
|
+
if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
|
|
1425
|
+
continue
|
|
1426
|
+
}
|
|
1378
1427
|
if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
|
|
1379
1428
|
}
|
|
1380
1429
|
}
|
|
1381
|
-
if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
|
|
1430
|
+
if (hiddenDuplicate || (duplicate && duplicate.similarity >= DUP_REJECT_DICE)) {
|
|
1431
|
+
// 5.3:拒写不丢文本——新正文作为 pending_review 暂存,进入现有复核队列;
|
|
1432
|
+
// 可见重复给出 id 与差异片段,隐藏重复只说“有重复”,不泄露 id/正文。
|
|
1433
|
+
const visibleTarget = duplicate ? index.entries[duplicate.id] || null : null
|
|
1434
|
+
const pendingEntry = {
|
|
1435
|
+
...entry,
|
|
1436
|
+
status: 'pending_review',
|
|
1437
|
+
reviewReason: 'near_duplicate',
|
|
1438
|
+
conflictingWith: duplicate ? duplicate.id : null,
|
|
1439
|
+
...(duplicate ? { duplicateOf: duplicate.id } : {}),
|
|
1440
|
+
}
|
|
1441
|
+
await retryStoreWrite(() => updatePrivateIndex(scope.libraryId, (fresh) => {
|
|
1442
|
+
if (fresh.entries[id]) throw new Error('memory changed concurrently; retry')
|
|
1443
|
+
fresh.entries[id] = pendingEntry
|
|
1444
|
+
}))
|
|
1445
|
+
if (duplicate) {
|
|
1446
|
+
const diff = nearDuplicateDiff(
|
|
1447
|
+
`${visibleTarget?.title || ''} ${visibleTarget?.content || ''}`,
|
|
1448
|
+
`${entry.title || ''} ${entry.content || ''}`,
|
|
1449
|
+
)
|
|
1450
|
+
return {
|
|
1451
|
+
id,
|
|
1452
|
+
entry: pendingEntry,
|
|
1453
|
+
pending: true,
|
|
1454
|
+
deduped: true,
|
|
1455
|
+
existingId: duplicate.id,
|
|
1456
|
+
similarity: duplicate.similarity,
|
|
1457
|
+
hint: `库内已有一条几乎相同的记忆;新正文已暂存待复核(near_duplicate)。差异片段:${diff || '(无可见差异)'}。若确属两次独立观测,带 allowDuplicate=true 重写;要合并请改用 memu_private_update。`,
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1382
1460
|
return {
|
|
1461
|
+
id,
|
|
1462
|
+
entry: pendingEntry,
|
|
1463
|
+
pending: true,
|
|
1383
1464
|
deduped: true,
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
hint: '库内已有一条几乎相同的记忆;改用 memu_private_update 合并内容。若确属两次独立观测,带 allowDuplicate=true 重写',
|
|
1465
|
+
similarityBand: '0.90+',
|
|
1466
|
+
hint: '库内已有一条几乎相同的记忆,但它不在当前会话的可见范围内;新正文已暂存待复核(near_duplicate)。',
|
|
1387
1467
|
}
|
|
1388
1468
|
}
|
|
1389
1469
|
if (duplicate && duplicate.similarity >= TOPIC_DICE) entry.duplicateOf = duplicate.id
|
|
@@ -1418,13 +1498,27 @@ async function privateRememberImpl({
|
|
|
1418
1498
|
if (sessionIdOf(exec) && (!currentBinding || currentBinding.libraryId !== scope.libraryId)) throw new Error('memory scope changed during write; retry')
|
|
1419
1499
|
if (currentBinding) assertScopeEnabled(currentBinding)
|
|
1420
1500
|
if (previousHabit && JSON.stringify(fresh.entries[id]) !== JSON.stringify(previousHabit[1])) throw new Error('habit changed concurrently; recall and retry')
|
|
1421
|
-
const latestConflict = findConflict(Object.fromEntries(Object.entries(fresh.entries).filter(([key]) => key !== id)), entry)
|
|
1501
|
+
const latestConflict = findConflict(Object.fromEntries(Object.entries(fresh.entries).filter(([key, other]) => key !== id && visibleForDedupe(other, memorySession))), entry)
|
|
1422
1502
|
if (latestConflict && !conflict) {
|
|
1423
1503
|
if (db) throw new Error('memory conflict changed during write; retry for review')
|
|
1424
1504
|
entry.status = 'pending_review'
|
|
1425
1505
|
entry.conflictingWith = latestConflict.id
|
|
1426
1506
|
entry.reviewReason = 'concurrent conflicting memory; review required'
|
|
1427
1507
|
}
|
|
1508
|
+
// 5.4:预检在 await 之前,提交期间另一个 turn 可能刚写入同事实。临界区用同一
|
|
1509
|
+
// 尺子重扫;命中就 retry(有向量则 throw 让 compensation 撤掉刚提交的行),
|
|
1510
|
+
// lexical 路由直接暂扣为 near_duplicate,不双写也不丢文本。
|
|
1511
|
+
const latestDuplicate = allowDuplicate
|
|
1512
|
+
? { duplicate: null, hidden: false }
|
|
1513
|
+
: findNearDuplicate(fresh.entries, {
|
|
1514
|
+
id, title: entry.title, content: entry.content, context: entry.context, sessionId: memorySession,
|
|
1515
|
+
})
|
|
1516
|
+
if (latestDuplicate.hidden || (latestDuplicate.duplicate && latestDuplicate.duplicate.similarity >= DUP_REJECT_DICE)) {
|
|
1517
|
+
if (db) throw new Error('memory duplicate changed during write; retry')
|
|
1518
|
+
entry.status = 'pending_review'
|
|
1519
|
+
entry.reviewReason = 'near_duplicate'
|
|
1520
|
+
entry.conflictingWith = latestDuplicate.duplicate?.id || null
|
|
1521
|
+
}
|
|
1428
1522
|
if (layer === 'must_remember' && pinSource === 'self') {
|
|
1429
1523
|
const pinned = Object.values(fresh.entries).filter(e => e.layer === 'must_remember' && isMemoryEligible(e)).length
|
|
1430
1524
|
if (pinned >= Number(getSettings().mustRememberCap || 12)) throw new Error('must_remember soft cap reached; retry as long_term')
|
|
@@ -1745,6 +1839,7 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
|
|
|
1745
1839
|
const scope = resolveLibrary({ exec, libraryId })
|
|
1746
1840
|
if (scope.binding) assertScopeEnabled(scope.binding)
|
|
1747
1841
|
const index = loadPrivateIndex(scope.libraryId)
|
|
1842
|
+
const memorySession = memorySessionId(exec)
|
|
1748
1843
|
const id_ = resolveEntryId(index, id)
|
|
1749
1844
|
const entry = index.entries[id_]
|
|
1750
1845
|
assertEntryAccess(entry, exec)
|
|
@@ -1776,29 +1871,46 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
|
|
|
1776
1871
|
const nextContext = context === undefined ? entry.context : normalizeMemoryContext(context)
|
|
1777
1872
|
const scanned = textChanged && entry.status !== 'pending_review'
|
|
1778
1873
|
let duplicate = null
|
|
1874
|
+
let hiddenDuplicate = null
|
|
1779
1875
|
let text = null
|
|
1780
1876
|
if (scanned) {
|
|
1781
|
-
const
|
|
1782
|
-
const
|
|
1877
|
+
const allOthers = Object.fromEntries(Object.entries(index.entries).filter(([key]) => key !== id_))
|
|
1878
|
+
const visibleOthers = Object.fromEntries(Object.entries(allOthers).filter(([, other]) => visibleForDedupe(other, memorySession)))
|
|
1879
|
+
const conflict = findConflict(visibleOthers, { ...entry, title: nextTitle, content: nextContent ?? entry.content })
|
|
1783
1880
|
if (conflict) throw new Error(`content conflicts with ${conflict.id}; write a new memory for review`)
|
|
1784
1881
|
text = `${nextTitle} ${nextContent ?? entry.content}`
|
|
1785
|
-
for (const [otherId, other] of Object.entries(
|
|
1882
|
+
for (const [otherId, other] of Object.entries(allOthers)) {
|
|
1786
1883
|
// 与 remember 的门同一过滤集(isMemoryEligible + sameFactContext),再排除它原本的
|
|
1787
1884
|
// duplicateOf 关系——那对条目是写入时就获准共存的,向它继续靠拢不该被拦。
|
|
1788
1885
|
if (!isMemoryEligible(other)) continue
|
|
1789
1886
|
if (otherId === entry.duplicateOf) continue
|
|
1790
1887
|
if (!sameFactContext(other.context, nextContext)) continue
|
|
1791
1888
|
const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
|
|
1889
|
+
if (!visibleForDedupe(other, memorySession)) {
|
|
1890
|
+
if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
|
|
1891
|
+
continue
|
|
1892
|
+
}
|
|
1792
1893
|
if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
|
|
1793
1894
|
}
|
|
1895
|
+
if (hiddenDuplicate) {
|
|
1896
|
+
record({
|
|
1897
|
+
kind: 'write', scope: 'private', sessionId: sessionIdOf(exec) || null, libraryId: scope.libraryId,
|
|
1898
|
+
message: '拒绝重复更新:改写后与当前会话不可见的记忆几乎相同',
|
|
1899
|
+
detail: { id: id_, deduped: true, hidden: true, similarityBand: '0.90+' },
|
|
1900
|
+
refs: [id_],
|
|
1901
|
+
})
|
|
1902
|
+
throw new Error('update refused: a nearly identical memory exists outside this conversation\'s visible scope; rewrite the text or archive one of the two instead')
|
|
1903
|
+
}
|
|
1794
1904
|
if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
|
|
1905
|
+
const target = index.entries[duplicate.id]
|
|
1906
|
+
const diff = nearDuplicateDiff(`${target?.title || ''} ${target?.content || ''}`, text)
|
|
1795
1907
|
record({
|
|
1796
1908
|
kind: 'write', scope: 'private', sessionId: sessionIdOf(exec) || null, libraryId: scope.libraryId,
|
|
1797
1909
|
message: '拒绝重复更新:' + id_ + ' 改写后与 ' + duplicate.id + ' 几乎相同',
|
|
1798
1910
|
detail: { id: id_, deduped: true, existingId: duplicate.id, similarity: duplicate.similarity },
|
|
1799
1911
|
refs: [id_, duplicate.id],
|
|
1800
1912
|
})
|
|
1801
|
-
throw new Error(`update refused: the rewritten text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); merge into ${duplicate.id} or archive one of the two instead`)
|
|
1913
|
+
throw new Error(`update refused: the rewritten text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); differences: ${diff || '(none visible)'}; merge into ${duplicate.id} or archive one of the two instead`)
|
|
1802
1914
|
}
|
|
1803
1915
|
}
|
|
1804
1916
|
const route = engineRoute()
|
|
@@ -1831,6 +1943,16 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
|
|
|
1831
1943
|
const currentBinding = peekBinding(exec)
|
|
1832
1944
|
if (sessionIdOf(exec) && (!currentBinding || currentBinding.libraryId !== scope.libraryId)) throw new Error('memory scope changed during update; retry')
|
|
1833
1945
|
if (currentBinding) assertScopeEnabled(currentBinding)
|
|
1946
|
+
if (scanned) {
|
|
1947
|
+
const latestDuplicate = findNearDuplicate(fresh.entries, {
|
|
1948
|
+
id: id_, title: nextTitle, content: nextContent ?? entry.content, context: nextContext,
|
|
1949
|
+
sessionId: memorySession, excludeId: entry.duplicateOf,
|
|
1950
|
+
})
|
|
1951
|
+
if (latestDuplicate.hidden || (latestDuplicate.duplicate && latestDuplicate.duplicate.similarity >= DUP_REJECT_DICE)) {
|
|
1952
|
+
throw new Error('duplicate changed during update; retry')
|
|
1953
|
+
}
|
|
1954
|
+
if (latestDuplicate.duplicate) duplicate = latestDuplicate.duplicate
|
|
1955
|
+
}
|
|
1834
1956
|
if (title !== undefined) target.title = clampText(title, 120)
|
|
1835
1957
|
if (nextContent !== undefined) target.content = nextContent
|
|
1836
1958
|
// 打标带与 remember 同口径:[0.55, 0.90) 改写后仍与另一条同主题 → duplicateOf。
|
|
@@ -1947,14 +2069,19 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
|
|
|
1947
2069
|
}
|
|
1948
2070
|
|
|
1949
2071
|
/** Review activation and invalidation of the previous claim are one index commit. */
|
|
1950
|
-
async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolution = 'replace', reason = '', content, title }) {
|
|
2072
|
+
async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolution = 'replace', reason = '', content, title, sessionId = null }) {
|
|
1951
2073
|
if (!['accept', 'reject'].includes(action)) throw new Error('action must be accept or reject')
|
|
1952
|
-
if (!['replace', 'coexist'].includes(resolution)) throw new Error('resolution must be replace or coexist')
|
|
1953
|
-
if (resolution === 'coexist' && String(reason).trim().length < 8) throw new Error('coexist needs a reason describing the distinct conditions (at least 8 characters)')
|
|
1954
2074
|
if (libraryId && !getLibrary(libraryId)) throw new Error('library not found')
|
|
1955
2075
|
const index = libraryId ? loadPrivateIndex(libraryId) : loadGlobalIndex()
|
|
1956
2076
|
const entry = index.entries[id]
|
|
1957
2077
|
if (!entry || entry.status !== 'pending_review') throw new Error('no such pending review')
|
|
2078
|
+
// pendingAbsorb(spec §4.11 review-all / 安全门触发的 pending absorb):走专用复核
|
|
2079
|
+
// 分支——提案是"整簇合并稿",accept/reject 的落库动作与普通复核完全不同。
|
|
2080
|
+
if (!libraryId && entry.pendingAbsorb) {
|
|
2081
|
+
return resolveAbsorbReviewImpl({ id, entry, index, action, resolution, reason, content, title, sessionId })
|
|
2082
|
+
}
|
|
2083
|
+
if (!['replace', 'coexist'].includes(resolution)) throw new Error('resolution must be replace or coexist')
|
|
2084
|
+
if (resolution === 'coexist' && String(reason).trim().length < 8) throw new Error('coexist needs a reason describing the distinct conditions (at least 8 characters)')
|
|
1958
2085
|
const next = { ...entry }
|
|
1959
2086
|
if (action === 'accept') {
|
|
1960
2087
|
if (typeof content === 'string' && content.trim()) next.content = content
|
|
@@ -1966,21 +2093,37 @@ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolutio
|
|
|
1966
2093
|
// 复核的本来含义,不是重复。
|
|
1967
2094
|
const text = `${next.title || ''} ${next.content || ''}`
|
|
1968
2095
|
let duplicate = null
|
|
2096
|
+
let hiddenDuplicate = null
|
|
1969
2097
|
for (const [otherId, other] of Object.entries(index.entries || {})) {
|
|
1970
2098
|
if (otherId === id || otherId === entry.conflictingWith) continue
|
|
1971
2099
|
if (!isMemoryEligible(other)) continue
|
|
1972
2100
|
if (!sameFactContext(other.context, next.context)) continue
|
|
1973
2101
|
const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
|
|
2102
|
+
if (!visibleForDedupe(other, sessionId)) {
|
|
2103
|
+
if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
|
|
2104
|
+
continue
|
|
2105
|
+
}
|
|
1974
2106
|
if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
|
|
1975
2107
|
}
|
|
2108
|
+
if (hiddenDuplicate) {
|
|
2109
|
+
record({
|
|
2110
|
+
kind: 'write', scope: libraryId ? 'private' : 'global', libraryId: libraryId || null,
|
|
2111
|
+
message: '拒绝复核接受:接受稿与当前会话不可见的记忆几乎相同',
|
|
2112
|
+
detail: { id, deduped: true, hidden: true, similarityBand: '0.90+' },
|
|
2113
|
+
refs: [id],
|
|
2114
|
+
})
|
|
2115
|
+
throw new Error('review accept refused: a nearly identical memory exists outside the visible scope; pass different content or change the review scope first')
|
|
2116
|
+
}
|
|
1976
2117
|
if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
|
|
2118
|
+
const target = index.entries[duplicate.id]
|
|
2119
|
+
const diff = nearDuplicateDiff(`${target?.title || ''} ${target?.content || ''}`, text)
|
|
1977
2120
|
record({
|
|
1978
2121
|
kind: 'write', scope: libraryId ? 'private' : 'global', libraryId: libraryId || null,
|
|
1979
2122
|
message: '拒绝复核接受:' + id + ' 与 ' + duplicate.id + ' 几乎相同',
|
|
1980
2123
|
detail: { id, deduped: true, existingId: duplicate.id, similarity: duplicate.similarity },
|
|
1981
2124
|
refs: [id, duplicate.id],
|
|
1982
2125
|
})
|
|
1983
|
-
throw new Error(`review accept refused: the accepted text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); pass different content or archive ${duplicate.id} first`)
|
|
2126
|
+
throw new Error(`review accept refused: the accepted text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); differences: ${diff || '(none visible)'}; pass different content or archive ${duplicate.id} first`)
|
|
1984
2127
|
}
|
|
1985
2128
|
if (duplicate && duplicate.similarity >= TOPIC_DICE) next.duplicateOf = duplicate.id
|
|
1986
2129
|
const reviewRoute = engineRoute()
|
|
@@ -2000,6 +2143,15 @@ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolutio
|
|
|
2000
2143
|
|| JSON.stringify(fresh.entries[entry.conflictingWith]) !== JSON.stringify(index.entries[entry.conflictingWith])) {
|
|
2001
2144
|
throw new Error('review changed; reload and retry')
|
|
2002
2145
|
}
|
|
2146
|
+
if (action === 'accept') {
|
|
2147
|
+
const latestDuplicate = findNearDuplicate(fresh.entries, {
|
|
2148
|
+
id, title: next.title, content: next.content, context: next.context,
|
|
2149
|
+
sessionId, excludeId: entry.conflictingWith,
|
|
2150
|
+
})
|
|
2151
|
+
if (latestDuplicate.hidden || (latestDuplicate.duplicate && latestDuplicate.duplicate.similarity >= DUP_REJECT_DICE)) {
|
|
2152
|
+
throw new Error('review changed; reload and retry')
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
2003
2155
|
const at = stamp()
|
|
2004
2156
|
next.status = action === 'accept' ? 'active' : 'archived'
|
|
2005
2157
|
next.reviewedAt = at
|
|
@@ -2024,6 +2176,119 @@ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolutio
|
|
|
2024
2176
|
return { id, status: next.status, resolution }
|
|
2025
2177
|
}
|
|
2026
2178
|
|
|
2179
|
+
/**
|
|
2180
|
+
* pendingAbsorb 复核分支(spec §3.3/§4.11/§6.7)。只作用于全局队列(私有索引里
|
|
2181
|
+
* 不存在 pendingAbsorb)。
|
|
2182
|
+
*
|
|
2183
|
+
* accept:resolution 必须 replace——合并稿不能与自己的来源"并存",coexist 直接抛错;
|
|
2184
|
+
* 重复门排除 [id, conflictingWith, ...sourceIds],否则合并稿会被自己的来源判重;
|
|
2185
|
+
* 快照校验 target/sources 仍 active 且 updatedAt === expectedUpdatedAt,否则抛
|
|
2186
|
+
* 'review changed; reload and retry';向量库只提交提案 id 本身(绝不提交 sources,
|
|
2187
|
+
* §6.7 复核);然后单事务落 §4.11 步骤 3c-6 并清 pendingAbsorb/reviewReason/
|
|
2188
|
+
* conflictingWith。索引写失败时 pending 保持、已提交向量行未激活(现有行为,
|
|
2189
|
+
* 本版不额外补偿)。
|
|
2190
|
+
*
|
|
2191
|
+
* reject:提案 archived;target/sources 原样。
|
|
2192
|
+
*/
|
|
2193
|
+
async function resolveAbsorbReviewImpl({ id, entry, index, action, resolution, reason, content, title, sessionId = null }) {
|
|
2194
|
+
const pa = entry.pendingAbsorb || {}
|
|
2195
|
+
const targetId = String(pa.targetId || '')
|
|
2196
|
+
const sourceIds = (Array.isArray(pa.sourceIds) ? pa.sourceIds : []).map(String)
|
|
2197
|
+
if (action === 'reject') {
|
|
2198
|
+
await retryStoreWrite(() => updateGlobalIndex((fresh) => {
|
|
2199
|
+
if (JSON.stringify(fresh.entries[id]) !== JSON.stringify(entry)) throw new Error('review changed; reload and retry')
|
|
2200
|
+
const at = stamp()
|
|
2201
|
+
const next = { ...entry, status: 'archived', reviewedAt: at, updatedAt: at, reviewResolution: resolution }
|
|
2202
|
+
if (reason) next.reviewNote = String(reason).trim().slice(0, 1000)
|
|
2203
|
+
fresh.entries[id] = next
|
|
2204
|
+
}))
|
|
2205
|
+
return { id, status: 'archived', resolution }
|
|
2206
|
+
}
|
|
2207
|
+
// —— accept ——
|
|
2208
|
+
if (resolution !== 'replace') {
|
|
2209
|
+
throw new Error('absorb review accept requires the replace resolution; a merged entry cannot coexist with its own sources')
|
|
2210
|
+
}
|
|
2211
|
+
const next = { ...entry }
|
|
2212
|
+
if (typeof content === 'string' && content.trim()) next.content = content
|
|
2213
|
+
if (typeof title === 'string' && title.trim()) next.title = title
|
|
2214
|
+
const exclude = new Set([id, entry.conflictingWith, targetId, ...sourceIds].filter(Boolean))
|
|
2215
|
+
const text = `${next.title || ''} ${next.content || ''}`
|
|
2216
|
+
let duplicate = null
|
|
2217
|
+
let hiddenDuplicate = null
|
|
2218
|
+
for (const [otherId, other] of Object.entries(index.entries || {})) {
|
|
2219
|
+
if (exclude.has(otherId)) continue
|
|
2220
|
+
if (!isMemoryEligible(other)) continue
|
|
2221
|
+
if (!sameFactContext(other.context, next.context)) continue
|
|
2222
|
+
const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
|
|
2223
|
+
if (!visibleForDedupe(other, sessionId)) {
|
|
2224
|
+
if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
|
|
2225
|
+
continue
|
|
2226
|
+
}
|
|
2227
|
+
if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
|
|
2228
|
+
}
|
|
2229
|
+
if (hiddenDuplicate) {
|
|
2230
|
+
record({
|
|
2231
|
+
kind: 'write', scope: 'global', libraryId: null,
|
|
2232
|
+
message: '拒绝集合吸收接受:接受稿与当前会话不可见的记忆几乎相同',
|
|
2233
|
+
detail: { id, deduped: true, hidden: true, similarityBand: '0.90+' },
|
|
2234
|
+
refs: [id],
|
|
2235
|
+
})
|
|
2236
|
+
throw new Error('review accept refused: a nearly identical memory exists outside the visible scope; pass different content or change the review scope first')
|
|
2237
|
+
}
|
|
2238
|
+
if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
|
|
2239
|
+
const target = index.entries[duplicate.id]
|
|
2240
|
+
const diff = nearDuplicateDiff(`${target?.title || ''} ${target?.content || ''}`, text)
|
|
2241
|
+
record({
|
|
2242
|
+
kind: 'write', scope: 'global', libraryId: null,
|
|
2243
|
+
message: '拒绝集合吸收接受:' + id + ' 与 ' + duplicate.id + ' 几乎相同',
|
|
2244
|
+
detail: { id, deduped: true, existingId: duplicate.id, similarity: duplicate.similarity },
|
|
2245
|
+
refs: [id, duplicate.id],
|
|
2246
|
+
})
|
|
2247
|
+
throw new Error(`review accept refused: the accepted text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); differences: ${diff || '(none visible)'}; pass different content or archive ${duplicate.id} first`)
|
|
2248
|
+
}
|
|
2249
|
+
if (duplicate && duplicate.similarity >= TOPIC_DICE) next.duplicateOf = duplicate.id
|
|
2250
|
+
// 快照校验(§4.11 accept 3):target/sources 仍 active 且 updatedAt 未被并发改写。
|
|
2251
|
+
const expected = pa.expectedUpdatedAt || {}
|
|
2252
|
+
for (const memberId of [targetId, ...sourceIds].filter(Boolean)) {
|
|
2253
|
+
const current = index.entries[memberId]
|
|
2254
|
+
if (!current || current.status !== 'active'
|
|
2255
|
+
|| (expected[memberId] !== undefined && current.updatedAt !== expected[memberId])) {
|
|
2256
|
+
throw new Error('review changed; reload and retry')
|
|
2257
|
+
}
|
|
2258
|
+
}
|
|
2259
|
+
// 向量提交:只提交提案 id,绝不提交 sources(§6.7 复核)。
|
|
2260
|
+
const reviewRoute = engineRoute()
|
|
2261
|
+
if (reviewRoute !== 'lexical') {
|
|
2262
|
+
await commit({
|
|
2263
|
+
payload: { recall_files: [{ name: id, track: next.track || 'memory', description: clampText(next.title, 180), content: next.content }], resource: [], user: {} },
|
|
2264
|
+
route: reviewRoute,
|
|
2265
|
+
db: globalDbPath(),
|
|
2266
|
+
})
|
|
2267
|
+
next.vector = activeEntryStamp(reviewRoute) || next.vector
|
|
2268
|
+
}
|
|
2269
|
+
// 单事务:§4.11 步骤 3c-6。merged 入索引、target superseded、sources archived、
|
|
2270
|
+
// rewire links、写 absorbSnapshot,然后清 pendingAbsorb/reviewReason/conflictingWith。
|
|
2271
|
+
await retryStoreWrite(() => updateGlobalIndex((fresh) => {
|
|
2272
|
+
if (JSON.stringify(fresh.entries[id]) !== JSON.stringify(entry)) throw new Error('review changed; reload and retry')
|
|
2273
|
+
const at = stamp()
|
|
2274
|
+
next.status = 'active'
|
|
2275
|
+
next.updatedAt = at
|
|
2276
|
+
next.reviewedAt = at
|
|
2277
|
+
next.reviewResolution = resolution
|
|
2278
|
+
next.validFrom = at
|
|
2279
|
+
if (reason) next.reviewNote = String(reason).trim().slice(0, 1000)
|
|
2280
|
+
const outcome = applyAbsorbIndexTransaction(fresh, {
|
|
2281
|
+
mergedId: id, merged: next, targetId, sourceIds, expectedUpdatedAt: expected, now: at,
|
|
2282
|
+
})
|
|
2283
|
+
if (!outcome.landed) throw new Error('review changed; reload and retry')
|
|
2284
|
+
const stored = fresh.entries[id]
|
|
2285
|
+
delete stored.pendingAbsorb
|
|
2286
|
+
delete stored.reviewReason
|
|
2287
|
+
delete stored.conflictingWith
|
|
2288
|
+
}))
|
|
2289
|
+
return { id, status: 'active', resolution }
|
|
2290
|
+
}
|
|
2291
|
+
|
|
2027
2292
|
/**
|
|
2028
2293
|
* Short-term entries expire instead of being deleted. Deleting would throw away the
|
|
2029
2294
|
* only copy before the consolidation pass could decide what it was worth; flipping
|
package/lib/memu.js
CHANGED
|
@@ -2,7 +2,7 @@ import fs from 'node:fs'
|
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { spawn } from 'node:child_process'
|
|
4
4
|
import { dataDir, ensureDir, redactSecrets, windowsSystemEnv } from './paths.js'
|
|
5
|
-
import { readCurrent, verifyEngine } from './engine.js'
|
|
5
|
+
import { ENGINE_VERSION, readCurrent, verifyEngine } from './engine.js'
|
|
6
6
|
import { getSettings } from './state.js'
|
|
7
7
|
import { activeProfile, captureVectorGeneration, assertVectorGeneration, assertProfileVerified, assertMigrationProfile } from './vector-profile.js'
|
|
8
8
|
import { embeddingWorkerManager } from './local-embedding-manager.js'
|
|
@@ -219,6 +219,6 @@ export async function forget({ db, name, timeoutMs = 60_000 } = {}) {
|
|
|
219
219
|
|
|
220
220
|
export async function probe() {
|
|
221
221
|
const current = currentEngine()
|
|
222
|
-
const verified = await verifyEngine(current.python)
|
|
222
|
+
const verified = await verifyEngine(current.python, { expectedVersion: current.version || ENGINE_VERSION })
|
|
223
223
|
return { ...verified, current }
|
|
224
224
|
}
|
package/lib/migrate.js
CHANGED
|
@@ -7,11 +7,18 @@
|
|
|
7
7
|
// the pass exists to rewrite the files: a library nobody writes to again would otherwise
|
|
8
8
|
// keep the dead `sessionId` fields on disk forever, and an auditor reading the JSON would
|
|
9
9
|
// have to know about a level the code no longer has.
|
|
10
|
+
//
|
|
11
|
+
// The lexicon half (§2.1 of the construction spec) is the same shape: older builds
|
|
12
|
+
// wrote `pending` proposals and revision drafts; the current lifecycle has only active/archived. The rewrite rule
|
|
13
|
+
// lives in `lib/lexicon.js` (`migrateLegacyLexiconStatuses`); this file only finds the
|
|
14
|
+
// files and runs it, one library at a time, so a single corrupt lexicon cannot take the
|
|
15
|
+
// sweep — or startup — down with it.
|
|
10
16
|
import fs from 'node:fs'
|
|
11
17
|
import path from 'node:path'
|
|
12
|
-
import { indexDir, readJson, stamp } from './paths.js'
|
|
18
|
+
import { dataDir, indexDir, isSafeId, readJson, readJsonStrict, stamp } from './paths.js'
|
|
13
19
|
import { listLibrariesAll, updatePrivateIndex } from './state.js'
|
|
14
20
|
import { countLegacyInRaw, migrateEntryVisibility } from './visibility.js'
|
|
21
|
+
import { migrateLegacyLexiconStatuses } from './lexicon.js'
|
|
15
22
|
|
|
16
23
|
const DEFAULT_MIN_INTERVAL_MS = 60_000
|
|
17
24
|
const lastRun = { at: 0 }
|
|
@@ -109,11 +116,85 @@ export function maybeMigrateLibraries({ minIntervalMs = DEFAULT_MIN_INTERVAL_MS,
|
|
|
109
116
|
return migrateLibraries({ now })
|
|
110
117
|
}
|
|
111
118
|
|
|
119
|
+
// ── lexicon lifecycle (§2.1): pending → active/archived ─────────────────────────
|
|
120
|
+
// The directory holds exactly `<libraryId>.json`; a name that does not pass `isSafeId`
|
|
121
|
+
// is skipped rather than guessed at, the same path boundary `lib/lexicon.js` enforces
|
|
122
|
+
// before it builds a filename.
|
|
123
|
+
|
|
124
|
+
function lexiconDir() {
|
|
125
|
+
return path.join(dataDir(), 'lexicon')
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/** Library ids that own a lexicon file. Sorted, safe names only. */
|
|
129
|
+
export function scanLexiconLibraries() {
|
|
130
|
+
let names = []
|
|
131
|
+
try { names = fs.readdirSync(lexiconDir()) } catch { return [] }
|
|
132
|
+
return names
|
|
133
|
+
.filter((name) => name.endsWith('.json'))
|
|
134
|
+
.map((name) => name.slice(0, -'.json'.length))
|
|
135
|
+
.filter((id) => isSafeId(id))
|
|
136
|
+
.sort()
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Counted on the raw file, before any load-time interpretation, so a sweep can tell a
|
|
140
|
+
// library that still needs the rewrite from one that is already migrated. `readJsonStrict`
|
|
141
|
+
// (not `readJson`) on purpose: a scan must not move files, and `readJson` quarantines a
|
|
142
|
+
// corrupt one as a side effect. A file that does not parse counts 0 — it cannot hold
|
|
143
|
+
// statuses to migrate, and `lib/lexicon.js` already blocks writes to it until restored.
|
|
144
|
+
function legacyLexiconCount(libraryId) {
|
|
145
|
+
const result = readJsonStrict(path.join(lexiconDir(), `${libraryId}.json`))
|
|
146
|
+
if (!result.ok) return 0
|
|
147
|
+
const terms = result.value?.terms
|
|
148
|
+
if (!terms || typeof terms !== 'object') return 0
|
|
149
|
+
return Object.values(terms).filter((entry) => entry && entry.status === 'pending').length
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Lexicon files that still hold legacy `pending` entries. */
|
|
153
|
+
export function scanLegacyLexiconLibraries() {
|
|
154
|
+
return scanLexiconLibraries()
|
|
155
|
+
.map((libraryId) => ({ libraryId, legacy: legacyLexiconCount(libraryId) }))
|
|
156
|
+
.filter((report) => report.legacy > 0)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Rewrite every lexicon that still carries `pending` entries.
|
|
161
|
+
*
|
|
162
|
+
* Per-library try/catch is the point: a single corrupt or write-protected lexicon must
|
|
163
|
+
* not stop the others, and must never abort startup (the pass runs from the cycle tick).
|
|
164
|
+
* A library with nothing to do is not written at all — `migrateLegacyLexiconStatuses`
|
|
165
|
+
* skips the write (and the revision bump) when no entry changed.
|
|
166
|
+
*/
|
|
167
|
+
export function migrateLexiconLibraries() {
|
|
168
|
+
const reports = []
|
|
169
|
+
for (const { libraryId, legacy } of scanLegacyLexiconLibraries()) {
|
|
170
|
+
try {
|
|
171
|
+
const result = migrateLegacyLexiconStatuses(libraryId)
|
|
172
|
+
reports.push({
|
|
173
|
+
libraryId,
|
|
174
|
+
before: legacy,
|
|
175
|
+
migrated: result.migrated,
|
|
176
|
+
...(result.details ? { details: result.details } : {}),
|
|
177
|
+
})
|
|
178
|
+
} catch (error) {
|
|
179
|
+
reports.push({ libraryId, before: legacy, migrated: 0, error: String(error?.message || error) })
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return reports
|
|
183
|
+
}
|
|
184
|
+
|
|
112
185
|
export function runVisibilityMigrationPass({ now = stamp(), log = () => {} } = {}) {
|
|
113
186
|
const reports = migrateLibraries({ now })
|
|
114
187
|
for (const report of reports) {
|
|
115
188
|
if (report.error) log(`visibility migration failed for ${report.libraryId}: ${report.error}`)
|
|
116
189
|
else if (report.migrated > 0) log(`migrated ${report.migrated} legacy memory/memories to shared in ${report.libraryId}`)
|
|
117
190
|
}
|
|
118
|
-
|
|
191
|
+
// §2.1: legacy lexicons may still carry pending proposals/revision drafts. The rewrite is
|
|
192
|
+
// idempotent and only writes files that change, so running it on every cycle tick
|
|
193
|
+
// costs a directory read plus one JSON parse per file that still has pending entries.
|
|
194
|
+
const lexicon = migrateLexiconLibraries()
|
|
195
|
+
for (const report of lexicon) {
|
|
196
|
+
if (report.error) log(`lexicon lifecycle migration failed for ${report.libraryId}: ${report.error}`)
|
|
197
|
+
else if (report.migrated > 0) log(`migrated ${report.migrated} legacy lexicon term(s) to the active/archived lifecycle in ${report.libraryId}`)
|
|
198
|
+
}
|
|
199
|
+
return { reports, lexicon }
|
|
119
200
|
}
|