dsh-memu-memory 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -6
- package/README.md +50 -5
- package/README.zh.md +49 -5
- package/docs/active-memory-ab.md +385 -0
- package/docs/binding-lifecycle.md +99 -0
- package/docs/contributing.md +6 -4
- package/docs/emotion-model-spec.md +281 -0
- package/docs/env-and-commands.md +13 -0
- package/docs/host-api.md +35 -3
- package/docs/memory-semantics.md +40 -11
- package/docs/recall-benchmark.md +238 -0
- package/docs/troubleshooting.md +395 -366
- package/lib/active-memory.js +165 -0
- package/lib/activity-log.js +190 -0
- package/lib/backup.js +1103 -0
- package/lib/binding-diagnostics.js +635 -0
- package/lib/client.js +728 -58
- package/lib/consolidation-ledger.js +347 -0
- package/lib/consolidation-receipt.js +284 -0
- package/lib/consolidation-snapshot.js +274 -0
- package/lib/consolidation-worker.js +115 -0
- package/lib/curator.js +81 -37
- package/lib/cycle.js +43 -6
- package/lib/embedding-catalog.js +763 -0
- package/lib/embedding-download.js +407 -0
- package/lib/embedding-runtime.js +710 -0
- package/lib/engine.js +2 -2
- package/lib/index.js +36 -0
- package/lib/inject.js +232 -103
- package/lib/local-embedding-entry.js +145 -0
- package/lib/local-embedding-manager.js +267 -0
- package/lib/local-embedding-protocol.js +173 -0
- package/lib/local-embedding-worker.js +652 -0
- package/lib/local-vector-service.js +283 -0
- package/lib/memory.js +599 -57
- package/lib/memu-config.js +10 -22
- package/lib/memu.js +53 -19
- package/lib/mood.js +268 -39
- package/lib/paths.js +24 -0
- package/lib/preset-scope.js +1 -1
- package/lib/private.js +450 -29
- package/lib/prompts/private-consolidation.coding.md +1 -0
- package/lib/prompts/private-consolidation.md +1 -0
- package/lib/prompts/private-consolidation.mixed.md +1 -0
- package/lib/prompts/private-consolidation.rp.md +1 -0
- package/lib/prompts.js +4 -4
- package/lib/remote-summary.js +51 -25
- package/lib/rpc.js +54 -5
- package/lib/state.js +137 -172
- package/lib/tools.js +58 -7
- package/lib/vector-migration.js +878 -0
- package/lib/vector-profile.js +968 -0
- package/package.json +4 -2
- package/scripts/acceptance-local.mjs +7 -5
- package/scripts/active-memory-ab-live.mjs +655 -0
- package/scripts/active-memory-ab.mjs +1302 -0
- package/scripts/chain-matrix.mjs +537 -0
- package/scripts/check-all.mjs +29 -1
- package/scripts/check-doc-drift.mjs +353 -0
- package/scripts/diagnose-bindings.mjs +176 -0
- package/scripts/local-runtime-install.mjs +142 -0
- package/scripts/local-runtime-verify.mjs +56 -0
- package/scripts/local-vector-verify.mjs +93 -0
- package/scripts/recall-bench.mjs +813 -0
- package/scripts/repetition-ab-scenarios.mjs +93 -0
- package/scripts/repetition-detector-snapshot.mjs +314 -0
- package/scripts/repetition-eval-live.mjs +360 -0
- package/scripts/repetition-eval.mjs +702 -0
- package/tests/activation.test.mjs +89 -0
- package/tests/active-memory-ab.test.mjs +745 -0
- package/tests/active-memory.test.mjs +129 -0
- package/tests/activity-log.test.mjs +126 -0
- package/tests/activity-wiring.test.mjs +103 -0
- package/tests/appendix-b.test.mjs +212 -0
- package/tests/appendix-b2.test.mjs +166 -0
- package/tests/audit-regression.test.mjs +272 -0
- package/tests/audit-safety.test.mjs +227 -0
- package/tests/backend-compensation.test.mjs +272 -0
- package/tests/backup.test.mjs +793 -0
- package/tests/binding-diagnostics.test.mjs +720 -0
- package/tests/browser.test.mjs +366 -0
- package/tests/client-render.test.mjs +892 -0
- package/tests/coalesce.test.mjs +113 -0
- package/tests/config.test.mjs +159 -0
- package/tests/consolidation-credential.test.mjs +320 -0
- package/tests/consolidation-fork.test.mjs +357 -0
- package/tests/consolidation-ledger.test.mjs +298 -0
- package/tests/consolidation-snapshot.test.mjs +321 -0
- package/tests/consolidation-value.test.mjs +89 -0
- package/tests/context-scope.test.mjs +40 -0
- package/tests/cue.test.mjs +44 -0
- package/tests/curator-probe.test.mjs +433 -0
- package/tests/curator.test.mjs +150 -0
- package/tests/cycle.test.mjs +411 -0
- package/tests/disclosure.test.mjs +231 -0
- package/tests/doc-drift.test.mjs +222 -0
- package/tests/embedding-catalog.test.mjs +266 -0
- package/tests/embedding-download.test.mjs +391 -0
- package/tests/embedding-runtime.test.mjs +389 -0
- package/tests/emotion-chain.test.mjs +84 -0
- package/tests/emotion-contract.test.mjs +445 -0
- package/tests/engine.test.mjs +35 -0
- package/tests/fixtures/active-memory-ab/README.md +56 -0
- package/tests/fixtures/active-memory-ab/cases/01-clean-dense-turn.json +351 -0
- package/tests/fixtures/active-memory-ab/cases/02-clean-thin-turn.json +122 -0
- package/tests/fixtures/active-memory-ab/cases/03-clean-starved-small.json +125 -0
- package/tests/fixtures/active-memory-ab/cases/04-clean-large-thin.json +137 -0
- package/tests/fixtures/active-memory-ab/cases/05-clean-half-step.json +166 -0
- package/tests/fixtures/active-memory-ab/cases/06-clean-duplicate-reread.json +197 -0
- package/tests/fixtures/active-memory-ab/cases/07-clean-recent-recall.json +125 -0
- package/tests/fixtures/active-memory-ab/cases/08-global-relevant.json +111 -0
- package/tests/fixtures/active-memory-ab/cases/09-global-irrelevant-conflict.json +158 -0
- package/tests/fixtures/active-memory-ab/cases/10-no-human-turn.json +97 -0
- package/tests/fixtures/active-memory-ab/cases/11-clean-scale-100.json +137 -0
- package/tests/fixtures/active-memory-ab/cases/12-clean-scale-1000.json +137 -0
- package/tests/fixtures/active-memory-ab/cases/13-clean-scale-5000.json +137 -0
- package/tests/fixtures/active-memory-ab/cases/14-clean-scale-20000.json +137 -0
- package/tests/fixtures/active-memory-ab/lead-ai-reviewed-reference.json +191 -0
- package/tests/fixtures/active-memory-ab/manifest.json +199 -0
- package/tests/fixtures/recall-bench/01-paraphrase.json +184 -0
- package/tests/fixtures/recall-bench/02-exact.json +153 -0
- package/tests/fixtures/recall-bench/03-negation.json +173 -0
- package/tests/fixtures/recall-bench/04-no-answer.json +162 -0
- package/tests/fixtures/recall-bench/05-conflict.json +194 -0
- package/tests/fixtures/recall-bench/06-boundary.json +273 -0
- package/tests/fixtures/recall-bench/07-secret.json +191 -0
- package/tests/fixtures/recall-bench/08-coding.json +164 -0
- package/tests/fixtures/recall-bench-gold-candidates/01-paraphrase.json +193 -0
- package/tests/fixtures/recall-bench-gold-candidates/02-exact.json +197 -0
- package/tests/fixtures/recall-bench-gold-candidates/03-negation.json +244 -0
- package/tests/fixtures/recall-bench-gold-candidates/04-no-answer.json +189 -0
- package/tests/fixtures/recall-bench-gold-candidates/05-conflict.json +245 -0
- package/tests/fixtures/recall-bench-gold-candidates/06-boundary.json +279 -0
- package/tests/fixtures/recall-bench-gold-candidates/07-secret.json +224 -0
- package/tests/fixtures/recall-bench-gold-candidates/08-coding.json +218 -0
- package/tests/fixtures/repetition-eval/01-zh-rp.json +93 -0
- package/tests/fixtures/repetition-eval/02-en.json +97 -0
- package/tests/fixtures/repetition-eval/03-code.json +91 -0
- package/tests/fixtures/repetition-eval/04-names.json +116 -0
- package/tests/fixtures/repetition-eval/05-user-quote.json +96 -0
- package/tests/fixtures/repetition-eval/06-agreement.json +86 -0
- package/tests/graph-view.test.mjs +187 -0
- package/tests/graph.test.mjs +77 -0
- package/tests/hook-order.test.mjs +72 -0
- package/tests/humanlike.test.mjs +232 -0
- package/tests/hybrid-recall.test.mjs +92 -0
- package/tests/inbound.test.mjs +393 -0
- package/tests/inject-guide.test.mjs +364 -0
- package/tests/inject.test.mjs +222 -0
- package/tests/injection-budget-settings.test.mjs +35 -0
- package/tests/injection-budget.test.mjs +64 -0
- package/tests/injection-chain.test.mjs +191 -0
- package/tests/loader-register.mjs +3 -0
- package/tests/loader.mjs +12 -0
- package/tests/local-embedding-entry.test.mjs +95 -0
- package/tests/local-embedding-manager.test.mjs +476 -0
- package/tests/local-embedding-protocol.test.mjs +95 -0
- package/tests/local-embedding-worker.test.mjs +346 -0
- package/tests/managed-memory.test.mjs +536 -0
- package/tests/managed-runtime.test.mjs +40 -0
- package/tests/media.test.mjs +132 -0
- package/tests/migration-payload.test.mjs +42 -0
- package/tests/mood.test.mjs +109 -0
- package/tests/multiprocess.test.mjs +29 -0
- package/tests/new-session-gate.test.mjs +361 -0
- package/tests/package-shipping.test.mjs +84 -0
- package/tests/preset-scope.test.mjs +212 -0
- package/tests/prompts.test.mjs +250 -0
- package/tests/react-snapshot.test.mjs +23 -0
- package/tests/recall-bench.test.mjs +306 -0
- package/tests/remote.test.mjs +140 -0
- package/tests/rename.test.mjs +30 -0
- package/tests/repetition-detector-snapshot.test.mjs +231 -0
- package/tests/repetition-eval-live.test.mjs +177 -0
- package/tests/repetition-eval.test.mjs +373 -0
- package/tests/salient-date.test.mjs +142 -0
- package/tests/settings-beat.test.mjs +221 -0
- package/tests/short-term.test.mjs +166 -0
- package/tests/sidebar-beat.test.mjs +208 -0
- package/tests/sidebar-refresh.test.mjs +123 -0
- package/tests/sidebar.test.mjs +70 -0
- package/tests/snippet.test.mjs +47 -0
- package/tests/source-kind.test.mjs +52 -0
- package/tests/state.test.mjs +150 -0
- package/tests/stats.test.mjs +112 -0
- package/tests/storage.test.mjs +125 -0
- package/tests/stubs/dsh-llm.js +5 -0
- package/tests/stubs/dsh-session.js +7 -0
- package/tests/stubs/dsh-tools.js +101 -0
- package/tests/tool-schema.test.mjs +73 -0
- package/tests/tools.test.mjs +184 -0
- package/tests/ui-defaults.test.mjs +57 -0
- package/tests/update-fields.test.mjs +138 -0
- package/tests/vector-migration.test.mjs +733 -0
- package/tests/vector-profile.test.mjs +425 -0
- package/tests/visibility-migration.test.mjs +152 -0
- package/tests/webui-auth.test.mjs +454 -0
- package/tests/webui.test.mjs +164 -0
- package/tests/windows-env.test.mjs +89 -0
|
@@ -0,0 +1,793 @@
|
|
|
1
|
+
import test from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import fs from 'node:fs'
|
|
4
|
+
import os from 'node:os'
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { spawnSync } from 'node:child_process'
|
|
7
|
+
|
|
8
|
+
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-memu-memory-backup-'))
|
|
9
|
+
process.env.DSH_MEMU_HOME = home
|
|
10
|
+
|
|
11
|
+
const state = await import('../lib/state.js')
|
|
12
|
+
|
|
13
|
+
test.after(() => {
|
|
14
|
+
try { state.flushState() } catch {}
|
|
15
|
+
try { fs.rmSync(home, { recursive: true, force: true }) } catch {}
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
test('a backup carries every store and a checksum manifest', () => {
|
|
19
|
+
const binding = state.ensureSession('backup-session', { cwd: path.join(home, 'proj') })
|
|
20
|
+
state.updatePrivateIndex(binding.libraryId, (index) => {
|
|
21
|
+
index.entries['mem-keep'] = { id: 'mem-keep', content: '别丢', layer: 'long_term', status: 'active' }
|
|
22
|
+
})
|
|
23
|
+
state.setWorkspacePolicy(path.join(home, 'proj'), { useLibrary: true })
|
|
24
|
+
|
|
25
|
+
const dest = state.backupDatabases()
|
|
26
|
+
assert.ok(fs.existsSync(path.join(dest, 'state.json')))
|
|
27
|
+
assert.ok(fs.existsSync(path.join(dest, 'index', `${binding.libraryId}.json`)))
|
|
28
|
+
|
|
29
|
+
const manifest = JSON.parse(fs.readFileSync(path.join(dest, 'manifest.json'), 'utf8'))
|
|
30
|
+
assert.ok(manifest.createdAt)
|
|
31
|
+
assert.ok(manifest.files['state.json'].sha256)
|
|
32
|
+
assert.ok(manifest.files[`index/${binding.libraryId}.json`].sha256)
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
test('restore puts the backed-up content back and keeps a safety copy', () => {
|
|
36
|
+
const binding = state.getSessionBinding('backup-session')
|
|
37
|
+
const backups = state.listBackups()
|
|
38
|
+
assert.ok(backups.length >= 1)
|
|
39
|
+
const target = backups[0]
|
|
40
|
+
assert.equal(target.hasManifest, true)
|
|
41
|
+
|
|
42
|
+
// Change the live data after the backup was taken.
|
|
43
|
+
state.updatePrivateIndex(binding.libraryId, (index) => {
|
|
44
|
+
index.entries['mem-after'] = { id: 'mem-after', content: '后来才写的', layer: 'long_term', status: 'active' }
|
|
45
|
+
})
|
|
46
|
+
assert.ok(state.loadPrivateIndex(binding.libraryId).entries['mem-after'])
|
|
47
|
+
|
|
48
|
+
const result = state.restoreBackup(target.name)
|
|
49
|
+
assert.ok(result.safety, 'a restore backs the live data up first')
|
|
50
|
+
const restored = state.loadPrivateIndex(binding.libraryId)
|
|
51
|
+
assert.ok(restored.entries['mem-keep'], 'the backed-up entry is back')
|
|
52
|
+
assert.equal(restored.entries['mem-after'], undefined, 'the later write is gone')
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
test('a backup with a broken checksum is refused', () => {
|
|
56
|
+
const backups = state.listBackups()
|
|
57
|
+
const target = backups[0]
|
|
58
|
+
const manifestPath = path.join(target.dir, 'manifest.json')
|
|
59
|
+
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'))
|
|
60
|
+
manifest.files['state.json'].sha256 = '0'.repeat(64)
|
|
61
|
+
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2))
|
|
62
|
+
|
|
63
|
+
assert.throws(() => state.restoreBackup(target.name), /corrupt|checksum/i)
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
test('an incomplete backup is refused', () => {
|
|
67
|
+
const dest = state.backupDatabases()
|
|
68
|
+
const name = path.basename(dest)
|
|
69
|
+
fs.rmSync(path.join(dest, 'state.json'))
|
|
70
|
+
assert.throws(() => state.restoreBackup(name), /incomplete/i)
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
test('backup names that try to escape the backups directory are rejected', () => {
|
|
74
|
+
assert.throws(() => state.restoreBackup('../../etc'), /invalid backup name/i)
|
|
75
|
+
assert.throws(() => state.deleteBackup('../../etc'), /invalid backup name/i)
|
|
76
|
+
})
|
|
77
|
+
|
|
78
|
+
test('a backup can be deleted, and only that one', () => {
|
|
79
|
+
const keep = state.backupDatabases()
|
|
80
|
+
const drop = state.backupDatabases()
|
|
81
|
+
const before = state.listBackups().length
|
|
82
|
+
assert.equal(state.deleteBackup(path.basename(drop)).deleted, path.basename(drop))
|
|
83
|
+
assert.equal(fs.existsSync(drop), false)
|
|
84
|
+
assert.equal(fs.existsSync(keep), true, 'other backups are untouched')
|
|
85
|
+
assert.equal(state.listBackups().length, before - 1)
|
|
86
|
+
assert.throws(() => state.deleteBackup(path.basename(drop)), /not found/i)
|
|
87
|
+
})
|
|
88
|
+
|
|
89
|
+
// ---------------------------------------------------------------------------
|
|
90
|
+
// Retention, staging and recovery (本地向量施工文档 §14.9).
|
|
91
|
+
// Every test below runs in its own temporary data root: no real user backup is
|
|
92
|
+
// created, pruned or restored by this file.
|
|
93
|
+
// ---------------------------------------------------------------------------
|
|
94
|
+
const backup = await import('../lib/backup.js')
|
|
95
|
+
|
|
96
|
+
function tempRoot(tag) {
|
|
97
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dsh-memu-memory-' + tag + '-'))
|
|
98
|
+
process.env.DSH_MEMU_HOME = dir
|
|
99
|
+
delete process.env.DSH_MEMU_BACKUP_NO_PYTHON
|
|
100
|
+
delete process.env.DSH_MEMU_PYTHON
|
|
101
|
+
backup.resetBackupPythonProbe()
|
|
102
|
+
state.clearCaches()
|
|
103
|
+
return dir
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function releaseRoot(dir) {
|
|
107
|
+
state.clearCaches()
|
|
108
|
+
process.env.DSH_MEMU_HOME = home
|
|
109
|
+
delete process.env.DSH_MEMU_BACKUP_NO_PYTHON
|
|
110
|
+
delete process.env.DSH_MEMU_PYTHON
|
|
111
|
+
backup.resetBackupPythonProbe()
|
|
112
|
+
try { fs.rmSync(dir, { recursive: true, force: true }) } catch {}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function seed(root, text) {
|
|
116
|
+
fs.mkdirSync(path.join(root, 'index'), { recursive: true })
|
|
117
|
+
fs.writeFileSync(path.join(root, 'state.json'), JSON.stringify({ version: 1, marker: text }))
|
|
118
|
+
fs.writeFileSync(path.join(root, 'index', 'lib-a.json'), JSON.stringify({ marker: text, entries: {} }))
|
|
119
|
+
return root
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function completeNames() {
|
|
123
|
+
return backup.scanBackups({ verify: true }).filter((entry) => entry.status === 'complete').map((entry) => entry.name)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function statusOf(name) {
|
|
127
|
+
const entry = backup.scanBackups().find((item) => item.name === name)
|
|
128
|
+
return entry ? entry.status : null
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
test('retention keeps only the newest three complete backups', () => {
|
|
132
|
+
const root = tempRoot('keep3')
|
|
133
|
+
try {
|
|
134
|
+
seed(root, 'v0')
|
|
135
|
+
assert.deepEqual(backup.pruneBackups().deleted, [], 'an empty root prunes nothing')
|
|
136
|
+
const made = []
|
|
137
|
+
for (let round = 1; round <= 6; round += 1) {
|
|
138
|
+
seed(root, 'v' + round)
|
|
139
|
+
made.push(backup.createBackup().name)
|
|
140
|
+
const complete = completeNames()
|
|
141
|
+
assert.ok(complete.length <= 3, 'at most three complete backups after ' + round + ' creations')
|
|
142
|
+
assert.deepEqual(complete.slice().sort(), made.slice(-3).slice().sort(), 'the newest three are the ones kept')
|
|
143
|
+
}
|
|
144
|
+
assert.equal(made.length, 6)
|
|
145
|
+
for (const old of made.slice(0, 3)) assert.equal(fs.existsSync(path.join(root, 'backups', old)), false, 'older backups are cleaned up')
|
|
146
|
+
for (const kept of made.slice(3)) assert.equal(fs.existsSync(path.join(root, 'backups', kept)), true)
|
|
147
|
+
} finally { releaseRoot(root) }
|
|
148
|
+
})
|
|
149
|
+
|
|
150
|
+
test('backups created in the same millisecond stay distinct and ordered', () => {
|
|
151
|
+
const root = tempRoot('samems')
|
|
152
|
+
try {
|
|
153
|
+
seed(root, 'same')
|
|
154
|
+
const first = backup.createBackup().name
|
|
155
|
+
const second = backup.createBackup().name
|
|
156
|
+
assert.notEqual(first, second, 'the name is unique even within one millisecond')
|
|
157
|
+
const entries = backup.scanBackups()
|
|
158
|
+
assert.equal(entries.length, 2)
|
|
159
|
+
assert.notEqual(entries[0].uniqueId, entries[1].uniqueId)
|
|
160
|
+
assert.deepEqual(entries.map((entry) => entry.name), [second, first].sort().reverse(), 'newest first and deterministic')
|
|
161
|
+
assert.match(entries[0].name, /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{3}Z/)
|
|
162
|
+
} finally { releaseRoot(root) }
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
test('a failed creation publishes nothing and deletes no usable backup', () => {
|
|
166
|
+
const root = tempRoot('failed-create')
|
|
167
|
+
try {
|
|
168
|
+
seed(root, 'good')
|
|
169
|
+
const kept = backup.createBackup().name
|
|
170
|
+
// A directory where a file belongs: the copy must fail, not paper over it.
|
|
171
|
+
fs.mkdirSync(path.join(root, 'prompts', 'manifest.json'), { recursive: true })
|
|
172
|
+
assert.throws(() => backup.createBackup(), /non-file|symbolic/i)
|
|
173
|
+
assert.deepEqual(completeNames(), [kept], 'the previous backup is intact')
|
|
174
|
+
assert.equal(backup.scanBackups().length, 1, 'nothing was published')
|
|
175
|
+
assert.deepEqual(fs.readdirSync(path.join(root, 'backups', '.staging')), [], 'the staging directory was discarded')
|
|
176
|
+
} finally { releaseRoot(root) }
|
|
177
|
+
})
|
|
178
|
+
|
|
179
|
+
test('leftovers of a crashed run are reclaimed, never published', () => {
|
|
180
|
+
const root = tempRoot('crash')
|
|
181
|
+
try {
|
|
182
|
+
seed(root, 'crash')
|
|
183
|
+
const backups = path.join(root, 'backups')
|
|
184
|
+
const abandoned = path.join(backups, '.staging', '2020-01-01T00-00-00-000Z.999999.crashed')
|
|
185
|
+
fs.mkdirSync(abandoned, { recursive: true })
|
|
186
|
+
fs.writeFileSync(path.join(abandoned, 'state.json'), '{}')
|
|
187
|
+
// Power loss between copying the stores and writing the completion marker.
|
|
188
|
+
const unpublished = path.join(backups, '2020-01-01T00-00-00-000Z-powerloss')
|
|
189
|
+
fs.mkdirSync(unpublished, { recursive: true })
|
|
190
|
+
fs.writeFileSync(path.join(unpublished, 'state.json'), '{}')
|
|
191
|
+
const created = backup.createBackup()
|
|
192
|
+
assert.equal(fs.existsSync(abandoned), false, 'staging of a dead process is reclaimed')
|
|
193
|
+
assert.equal(fs.existsSync(unpublished), true, 'an unpublished directory is reported, never deleted')
|
|
194
|
+
assert.equal(statusOf(path.basename(unpublished)), 'unknown')
|
|
195
|
+
assert.equal(statusOf(created.name), 'complete')
|
|
196
|
+
const pruned = backup.pruneBackups({ keep: 1 })
|
|
197
|
+
assert.deepEqual(pruned.deleted, [], 'an unpublished directory never counts as a backup')
|
|
198
|
+
assert.equal(fs.existsSync(unpublished), true)
|
|
199
|
+
assert.ok(pruned.skipped.some((item) => item.name === path.basename(unpublished)), 'it is reported as skipped')
|
|
200
|
+
} finally { releaseRoot(root) }
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
test('a backup whose bytes drifted is not counted as valid, and is not silently deleted', () => {
|
|
204
|
+
const root = tempRoot('corrupt')
|
|
205
|
+
try {
|
|
206
|
+
seed(root, 'one')
|
|
207
|
+
const first = backup.createBackup({ keep: 10 }).name
|
|
208
|
+
seed(root, 'two')
|
|
209
|
+
backup.createBackup({ keep: 10 })
|
|
210
|
+
seed(root, 'three')
|
|
211
|
+
backup.createBackup({ keep: 10 })
|
|
212
|
+
seed(root, 'four')
|
|
213
|
+
const newest = backup.createBackup({ keep: 10 }).name
|
|
214
|
+
const victim = path.join(root, 'backups', first, 'state.json')
|
|
215
|
+
const original = fs.readFileSync(victim, 'utf8')
|
|
216
|
+
const flipped = original.slice(0, -2) + (original.slice(-2) === '1}' ? '2}' : '1}')
|
|
217
|
+
assert.equal(Buffer.byteLength(flipped), Buffer.byteLength(original), 'same size, different bytes')
|
|
218
|
+
fs.writeFileSync(victim, flipped)
|
|
219
|
+
assert.throws(() => backup.restoreFromBackup(first, { keep: 10 }), /corrupt|checksum/i)
|
|
220
|
+
const pruned = backup.pruneBackups()
|
|
221
|
+
assert.equal(fs.existsSync(path.join(root, 'backups', first)), true, 'a corrupt backup is reported, not quietly removed')
|
|
222
|
+
assert.ok(pruned.skipped.some((item) => item.name === first && item.status === 'corrupt'), 'it is reported as corrupt')
|
|
223
|
+
assert.equal(completeNames().length, 3, 'only intact backups occupy the three slots')
|
|
224
|
+
assert.ok(completeNames().includes(newest))
|
|
225
|
+
} finally { releaseRoot(root) }
|
|
226
|
+
})
|
|
227
|
+
|
|
228
|
+
test('unknown and legacy directories survive retention untouched', () => {
|
|
229
|
+
const root = tempRoot('foreign')
|
|
230
|
+
try {
|
|
231
|
+
seed(root, 'x')
|
|
232
|
+
const good = backup.createBackup({ keep: 10 }).name
|
|
233
|
+
const backups = path.join(root, 'backups')
|
|
234
|
+
const handMade01 = path.join(backups, 'my-own-backup')
|
|
235
|
+
fs.mkdirSync(handMade01, { recursive: true })
|
|
236
|
+
fs.writeFileSync(path.join(handMade01, 'state.json'), '{}')
|
|
237
|
+
// A backup written by an older engine: manifest without the completion marker.
|
|
238
|
+
const legacyName = '2020-01-01T00-00-00-000Z-legacy'
|
|
239
|
+
const legacy = path.join(backups, legacyName)
|
|
240
|
+
fs.mkdirSync(legacy, { recursive: true })
|
|
241
|
+
fs.writeFileSync(path.join(legacy, 'state.json'), '{}')
|
|
242
|
+
fs.writeFileSync(path.join(legacy, 'manifest.json'), JSON.stringify({ version: 1, createdAt: '2020-01-01T00:00:00.000Z', files: { 'state.json': { bytes: 2, sha256: 'x' } } }))
|
|
243
|
+
const pruned = backup.pruneBackups({ keep: 0 })
|
|
244
|
+
assert.deepEqual(pruned.deleted, [good])
|
|
245
|
+
assert.equal(fs.existsSync(handMade01), true, 'an unmanaged directory is never deleted')
|
|
246
|
+
assert.equal(fs.existsSync(legacy), true, 'an incomplete backup is never deleted')
|
|
247
|
+
assert.equal(statusOf('my-own-backup'), 'unknown')
|
|
248
|
+
assert.equal(statusOf(legacyName), 'incomplete')
|
|
249
|
+
} finally { releaseRoot(root) }
|
|
250
|
+
})
|
|
251
|
+
|
|
252
|
+
test('a backup that cannot be deleted safely is reported and left in place', () => {
|
|
253
|
+
const root = tempRoot('delete-fail')
|
|
254
|
+
try {
|
|
255
|
+
seed(root, 'a')
|
|
256
|
+
const older = backup.createBackup({ keep: 10 }).name
|
|
257
|
+
seed(root, 'b')
|
|
258
|
+
const newer = backup.createBackup({ keep: 10 }).name
|
|
259
|
+
const nested = path.join(root, 'backups', older, 'attachments')
|
|
260
|
+
fs.mkdirSync(nested, { recursive: true })
|
|
261
|
+
const outside = path.join(root, 'outside')
|
|
262
|
+
fs.mkdirSync(outside, { recursive: true })
|
|
263
|
+
fs.writeFileSync(path.join(outside, 'keep.txt'), 'important')
|
|
264
|
+
// A junction inside the backup makes a recursive delete unsafe.
|
|
265
|
+
fs.symlinkSync(outside, path.join(nested, 'link'), 'junction')
|
|
266
|
+
const pruned = backup.pruneBackups({ keep: 1 })
|
|
267
|
+
assert.deepEqual(pruned.deleted, [])
|
|
268
|
+
assert.equal(pruned.failed.length, 1, 'the refusal is reported')
|
|
269
|
+
assert.match(pruned.failed[0].error, /link/i)
|
|
270
|
+
assert.equal(fs.existsSync(path.join(root, 'backups', older)), true, 'nothing was deleted')
|
|
271
|
+
assert.equal(fs.existsSync(path.join(root, 'backups', newer)), true)
|
|
272
|
+
assert.equal(fs.readFileSync(path.join(outside, 'keep.txt'), 'utf8'), 'important', 'the link target is untouched')
|
|
273
|
+
} finally { releaseRoot(root) }
|
|
274
|
+
})
|
|
275
|
+
|
|
276
|
+
test('one lock serializes create, restore, delete and prune', () => {
|
|
277
|
+
const root = tempRoot('lock')
|
|
278
|
+
try {
|
|
279
|
+
seed(root, 'lock')
|
|
280
|
+
// Reentrant: a restore makes its own safety backup while holding the lock.
|
|
281
|
+
const inner = backup.withBackupLock(() => backup.createBackup({ keep: 10 }).name)
|
|
282
|
+
assert.ok(inner)
|
|
283
|
+
const lock = path.join(root, 'backups', '.backup.lock')
|
|
284
|
+
fs.writeFileSync(lock, String(process.pid))
|
|
285
|
+
try {
|
|
286
|
+
assert.throws(() => backup.createBackup(), /busy/i)
|
|
287
|
+
} finally { fs.rmSync(lock, { force: true }) }
|
|
288
|
+
let deadPid = null
|
|
289
|
+
for (let candidate = 999999; candidate < 1000100 && deadPid === null; candidate += 1) {
|
|
290
|
+
try { process.kill(candidate, 0) } catch (error) { if (error && error.code === 'ESRCH') deadPid = candidate }
|
|
291
|
+
}
|
|
292
|
+
if (deadPid !== null) {
|
|
293
|
+
fs.writeFileSync(lock, String(deadPid))
|
|
294
|
+
const recovered = backup.createBackup({ keep: 10 })
|
|
295
|
+
assert.ok(fs.existsSync(recovered.dir), 'a lock left by a dead process is reclaimed')
|
|
296
|
+
assert.equal(fs.existsSync(lock), false, 'the lock is released')
|
|
297
|
+
}
|
|
298
|
+
} finally { releaseRoot(root) }
|
|
299
|
+
})
|
|
300
|
+
|
|
301
|
+
test('restoring the oldest of four keeps the target and its safety backup', () => {
|
|
302
|
+
const root = tempRoot('restore-oldest')
|
|
303
|
+
try {
|
|
304
|
+
seed(root, 'oldest')
|
|
305
|
+
const target = backup.createBackup({ keep: 10, label: 'manual' }).name
|
|
306
|
+
for (const text of ['second', 'third', 'fourth']) {
|
|
307
|
+
seed(root, text)
|
|
308
|
+
backup.createBackup({ keep: 10 })
|
|
309
|
+
}
|
|
310
|
+
assert.equal(completeNames().length, 4)
|
|
311
|
+
const result = backup.restoreFromBackup(target, { keep: 3 })
|
|
312
|
+
assert.equal(result.restored, target)
|
|
313
|
+
assert.ok(result.safety, 'a safety backup was made')
|
|
314
|
+
assert.equal(fs.existsSync(path.join(root, 'backups', path.basename(result.safety))), true)
|
|
315
|
+
assert.equal(fs.existsSync(path.join(root, 'backups', target)), true, 'the restore target is never pruned')
|
|
316
|
+
assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'state.json'), 'utf8')).marker, 'oldest', 'the backed-up text is back')
|
|
317
|
+
assert.equal(completeNames().length, 4, 'target plus safety may exceed three during the restore')
|
|
318
|
+
const pruned = backup.pruneBackups({ keep: 3 })
|
|
319
|
+
assert.equal(pruned.kept.length, 3, 'retention is reclaimed right after the operation')
|
|
320
|
+
assert.equal(pruned.pendingCleanup, 0)
|
|
321
|
+
} finally { releaseRoot(root) }
|
|
322
|
+
})
|
|
323
|
+
|
|
324
|
+
test('a restore validates the whole inventory before it writes anything', () => {
|
|
325
|
+
const root = tempRoot('restore-guard')
|
|
326
|
+
try {
|
|
327
|
+
seed(root, 'backup')
|
|
328
|
+
const name = backup.createBackup({ keep: 10 }).name
|
|
329
|
+
seed(root, 'live')
|
|
330
|
+
const liveBefore = fs.readFileSync(path.join(root, 'state.json'), 'utf8')
|
|
331
|
+
const liveIndex = fs.readFileSync(path.join(root, 'index', 'lib-a.json'), 'utf8')
|
|
332
|
+
fs.rmSync(path.join(root, 'backups', name, 'index', 'lib-a.json'))
|
|
333
|
+
assert.throws(() => backup.restoreFromBackup(name, { keep: 10 }), /incomplete/i)
|
|
334
|
+
assert.equal(fs.readFileSync(path.join(root, 'state.json'), 'utf8'), liveBefore, 'the live root is untouched')
|
|
335
|
+
assert.equal(fs.readFileSync(path.join(root, 'index', 'lib-a.json'), 'utf8'), liveIndex)
|
|
336
|
+
assert.equal(fs.existsSync(path.join(root, 'backups', name, 'state.json')), true)
|
|
337
|
+
} finally { releaseRoot(root) }
|
|
338
|
+
})
|
|
339
|
+
|
|
340
|
+
test('a restore that fails while writing stays recoverable through its safety backup', () => {
|
|
341
|
+
const root = tempRoot('restore-fail')
|
|
342
|
+
try {
|
|
343
|
+
seed(root, 'backup')
|
|
344
|
+
fs.mkdirSync(path.join(root, 'attachments'), { recursive: true })
|
|
345
|
+
fs.writeFileSync(path.join(root, 'attachments', 'a.txt'), 'attachment')
|
|
346
|
+
const name = backup.createBackup({ keep: 10 }).name
|
|
347
|
+
seed(root, 'live')
|
|
348
|
+
fs.writeFileSync(path.join(root, 'attachments', 'a.txt'), 'attachment')
|
|
349
|
+
// Make one restore target unwritable after the safety backup exists.
|
|
350
|
+
fs.rmSync(path.join(root, 'attachments'), { recursive: true, force: true })
|
|
351
|
+
fs.writeFileSync(path.join(root, 'attachments'), 'not a directory')
|
|
352
|
+
let caught = null
|
|
353
|
+
try { backup.restoreFromBackup(name, { keep: 10 }) } catch (error) { caught = error }
|
|
354
|
+
assert.ok(caught, 'the restore failed')
|
|
355
|
+
assert.equal(caught.code, 'DSH_MEMU_BACKUP_RESTORE_FAILED')
|
|
356
|
+
assert.ok(caught.safety, 'the failure names the safety backup it kept')
|
|
357
|
+
assert.match(caught.message, /saved as/i)
|
|
358
|
+
const safety = path.join(root, 'backups', caught.safety)
|
|
359
|
+
assert.equal(fs.existsSync(path.join(safety, 'state.json')), true)
|
|
360
|
+
// Recovery: repair the obstacle, then roll back with the safety backup.
|
|
361
|
+
fs.rmSync(path.join(root, 'attachments'), { force: true })
|
|
362
|
+
fs.mkdirSync(path.join(root, 'attachments'), { recursive: true })
|
|
363
|
+
const rollback = backup.restoreFromBackup(caught.safety, { keep: 10 })
|
|
364
|
+
assert.equal(rollback.restored, caught.safety)
|
|
365
|
+
assert.equal(JSON.parse(fs.readFileSync(path.join(root, 'state.json'), 'utf8')).marker, 'live')
|
|
366
|
+
} finally { releaseRoot(root) }
|
|
367
|
+
})
|
|
368
|
+
|
|
369
|
+
test('a backup from before the vector journal retires the stale live pointer', () => {
|
|
370
|
+
const root = tempRoot('vector')
|
|
371
|
+
try {
|
|
372
|
+
seed(root, 'text-only')
|
|
373
|
+
const name = backup.createBackup({ keep: 10 }).name
|
|
374
|
+
const live = path.join(root, 'index', 'vector-active.json')
|
|
375
|
+
fs.writeFileSync(live, JSON.stringify({
|
|
376
|
+
version: 1,
|
|
377
|
+
generation: 2,
|
|
378
|
+
active: {
|
|
379
|
+
status: 'active',
|
|
380
|
+
generation: 2,
|
|
381
|
+
fingerprint: 'abcdef01',
|
|
382
|
+
profile: { id: 'small' },
|
|
383
|
+
databases: { global: { path: 'libraries/global.abcdef01.g2.sqlite3' }, libraries: {} },
|
|
384
|
+
},
|
|
385
|
+
}))
|
|
386
|
+
const result = backup.restoreFromBackup(name, { keep: 10 })
|
|
387
|
+
assert.equal(result.vectorActive, 'cleared')
|
|
388
|
+
assert.equal(fs.existsSync(live), false, 'the stale pointer is gone')
|
|
389
|
+
assert.ok(fs.readdirSync(path.join(root, 'index')).some((file) => file.indexOf('vector-active.json.stale-') === 0), 'it was moved aside, not lost')
|
|
390
|
+
} finally { releaseRoot(root) }
|
|
391
|
+
})
|
|
392
|
+
|
|
393
|
+
test('retention and restore refuse paths that leave the data root', () => {
|
|
394
|
+
const root = tempRoot('traversal')
|
|
395
|
+
try {
|
|
396
|
+
seed(root, 'tv')
|
|
397
|
+
const name = backup.createBackup({ keep: 10 }).name
|
|
398
|
+
const manifestPath = path.join(root, 'backups', name, 'manifest.json')
|
|
399
|
+
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'))
|
|
400
|
+
manifest.files['../escaped.json'] = { bytes: 2, sha256: 'x' }
|
|
401
|
+
fs.writeFileSync(manifestPath, JSON.stringify(manifest))
|
|
402
|
+
assert.throws(() => backup.restoreFromBackup(name, { keep: 10 }), /invalid backup manifest path/)
|
|
403
|
+
assert.throws(() => backup.deleteBackupEntry('..'), /invalid backup name/)
|
|
404
|
+
assert.throws(() => backup.deleteBackupEntry('.staging'), /invalid backup name/)
|
|
405
|
+
const outside = path.join(root, 'outside')
|
|
406
|
+
fs.mkdirSync(outside, { recursive: true })
|
|
407
|
+
const linkedName = '2020-01-01T00-00-00-000Z-linked'
|
|
408
|
+
fs.symlinkSync(outside, path.join(root, 'backups', linkedName), 'junction')
|
|
409
|
+
assert.equal(statusOf(linkedName), 'unknown', 'a link is never treated as a backup')
|
|
410
|
+
const pruned = backup.pruneBackups({ keep: 0 })
|
|
411
|
+
assert.deepEqual(pruned.deleted, [], 'a manifest with an escaping path is never deleted')
|
|
412
|
+
assert.ok(pruned.skipped.some((item) => item.name === name), 'and it is reported instead')
|
|
413
|
+
assert.equal(fs.existsSync(path.join(root, 'backups', linkedName)), true, 'the link is not followed or deleted')
|
|
414
|
+
assert.equal(fs.existsSync(outside), true)
|
|
415
|
+
} finally { releaseRoot(root) }
|
|
416
|
+
})
|
|
417
|
+
|
|
418
|
+
test('the state API keeps its shape and reports the retention policy', () => {
|
|
419
|
+
const root = tempRoot('api')
|
|
420
|
+
try {
|
|
421
|
+
const binding = state.ensureSession('api-session', { cwd: path.join(root, 'proj') })
|
|
422
|
+
state.updatePrivateIndex(binding.libraryId, (index) => {
|
|
423
|
+
index.entries['mem-api'] = { id: 'mem-api', content: '接口', layer: 'long_term', status: 'active' }
|
|
424
|
+
})
|
|
425
|
+
const dir = state.backupDatabases()
|
|
426
|
+
assert.equal(typeof dir, 'string')
|
|
427
|
+
assert.ok(fs.existsSync(path.join(dir, 'manifest.json')))
|
|
428
|
+
for (let round = 0; round < 4; round += 1) state.backupDatabases()
|
|
429
|
+
const listed = state.listBackups()
|
|
430
|
+
assert.ok(Array.isArray(listed))
|
|
431
|
+
assert.ok(listed.length >= 1)
|
|
432
|
+
assert.ok(listed.filter((item) => item.status === 'complete').length <= 3, 'at most three complete backups')
|
|
433
|
+
for (const item of listed) {
|
|
434
|
+
assert.equal(typeof item.name, 'string')
|
|
435
|
+
assert.equal(typeof item.dir, 'string')
|
|
436
|
+
assert.ok(item.createdAt)
|
|
437
|
+
assert.equal(typeof item.files, 'number')
|
|
438
|
+
assert.equal(typeof item.bytes, 'number')
|
|
439
|
+
assert.equal(typeof item.hasManifest, 'boolean')
|
|
440
|
+
}
|
|
441
|
+
const report = state.lastBackupReportView()
|
|
442
|
+
assert.equal(report.action, 'create')
|
|
443
|
+
assert.match(report.notice, /at most 3/)
|
|
444
|
+
const oldest = listed[listed.length - 1]
|
|
445
|
+
const restored = state.restoreBackup(oldest.name)
|
|
446
|
+
assert.equal(restored.restored, oldest.name)
|
|
447
|
+
assert.ok(restored.safety)
|
|
448
|
+
assert.ok(state.loadPrivateIndex(binding.libraryId).entries['mem-api'], 'the restored library is readable')
|
|
449
|
+
} finally { releaseRoot(root) }
|
|
450
|
+
})
|
|
451
|
+
|
|
452
|
+
test('the backup manifest carries identity, never a credential', () => {
|
|
453
|
+
const root = tempRoot('secrets')
|
|
454
|
+
try {
|
|
455
|
+
seed(root, 'secret')
|
|
456
|
+
fs.writeFileSync(path.join(root, 'state.json'), JSON.stringify({
|
|
457
|
+
version: 1,
|
|
458
|
+
settings: { apiKey: 'sk-abcdef0123456789abcdef', endpoint: 'https://user:pw@example.invalid/v1' },
|
|
459
|
+
}))
|
|
460
|
+
const name = backup.createBackup({ keep: 10 }).name
|
|
461
|
+
const manifestText = fs.readFileSync(path.join(root, 'backups', name, 'manifest.json'), 'utf8')
|
|
462
|
+
assert.equal(manifestText.indexOf('sk-abcdef0123456789abcdef'), -1, 'no key in the manifest')
|
|
463
|
+
assert.equal(manifestText.indexOf('user:pw'), -1, 'no credential in a URL')
|
|
464
|
+
const manifest = JSON.parse(manifestText)
|
|
465
|
+
assert.equal(manifest.complete, true)
|
|
466
|
+
assert.equal(typeof manifest.uniqueId, 'string')
|
|
467
|
+
assert.ok(manifest.files['state.json'].sha256, 'the state file itself is still backed up')
|
|
468
|
+
} finally { releaseRoot(root) }
|
|
469
|
+
})
|
|
470
|
+
// ---------------------------------------------------------------------------
|
|
471
|
+
// Live SQLite snapshotting (本地向量施工文档 §14.9, rule 2).
|
|
472
|
+
//
|
|
473
|
+
// A non-empty database may only be snapshotted by SQLite's own backup API, run
|
|
474
|
+
// by the plugin's isolated engine/<version>/venv interpreter. These tests prove
|
|
475
|
+
// the two halves of that rule: the engine snapshot is what lands in the backup,
|
|
476
|
+
// and every path where the engine cannot prove a snapshot FAILS CLOSED - no
|
|
477
|
+
// byte copy, no db+wal copy, no global Python, nothing published.
|
|
478
|
+
// ---------------------------------------------------------------------------
|
|
479
|
+
|
|
480
|
+
// 16 bytes of SQLite magic: enough for the header check, never a real database.
|
|
481
|
+
function sqliteFile(extra = '') {
|
|
482
|
+
return Buffer.concat([Buffer.from('SQLite format 3', 'latin1'), Buffer.from([0]), Buffer.from(extra || 'x', 'latin1')])
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
function seedDatabase(root, rel, bytes) {
|
|
486
|
+
const full = path.join(root, rel.split('/').join(path.sep))
|
|
487
|
+
fs.mkdirSync(path.dirname(full), { recursive: true })
|
|
488
|
+
fs.writeFileSync(full, bytes)
|
|
489
|
+
return full
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/** An engine install as engine.js leaves it: venv on disk plus current.json. */
|
|
493
|
+
function fakeEngine(root, version = '9.9.9') {
|
|
494
|
+
const venv = path.join(root, 'engine', version, 'venv')
|
|
495
|
+
const bin = process.platform === 'win32' ? path.join(venv, 'Scripts') : path.join(venv, 'bin')
|
|
496
|
+
fs.mkdirSync(bin, { recursive: true })
|
|
497
|
+
const python = path.join(bin, process.platform === 'win32' ? 'python.exe' : 'python3')
|
|
498
|
+
fs.writeFileSync(python, '')
|
|
499
|
+
fs.writeFileSync(path.join(root, 'engine', 'current.json'), JSON.stringify({ version, venv, python }))
|
|
500
|
+
return { venv, python }
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Stands in for the venv interpreter. A two-argument call is the probe, a
|
|
505
|
+
* four-argument call is the backup API. \`snapshot\` selects the behaviour.
|
|
506
|
+
*/
|
|
507
|
+
function fakeInterpreter(venv, { snapshot = 'ok', calls = [] } = {}) {
|
|
508
|
+
return (cmd, args) => {
|
|
509
|
+
calls.push({ cmd, args })
|
|
510
|
+
if (args.length === 2) {
|
|
511
|
+
return { status: 0, stdout: JSON.stringify({ ok: true, prefix: venv, sqlite: '3.45.0' }), stderr: '' }
|
|
512
|
+
}
|
|
513
|
+
const target = args[3]
|
|
514
|
+
if (snapshot === 'ok') {
|
|
515
|
+
fs.writeFileSync(target, Buffer.concat([sqliteFile(), Buffer.from('ENGINE-SNAPSHOT', 'latin1')]))
|
|
516
|
+
return { status: 0, stdout: JSON.stringify({ ok: true, quickCheck: 'ok' }), stderr: '' }
|
|
517
|
+
}
|
|
518
|
+
if (snapshot === 'io-error') return { status: 0, stdout: JSON.stringify({ ok: false, error: 'disk I/O error' }), stderr: '' }
|
|
519
|
+
if (snapshot === 'bad-check') {
|
|
520
|
+
fs.writeFileSync(target, sqliteFile())
|
|
521
|
+
return { status: 0, stdout: JSON.stringify({ ok: true, quickCheck: 'database disk image is malformed' }), stderr: '' }
|
|
522
|
+
}
|
|
523
|
+
if (snapshot === 'no-header') {
|
|
524
|
+
fs.writeFileSync(target, 'not a database')
|
|
525
|
+
return { status: 0, stdout: JSON.stringify({ ok: true, quickCheck: 'ok' }), stderr: '' }
|
|
526
|
+
}
|
|
527
|
+
return { status: 2, stdout: '', stderr: 'Traceback: interpreter crashed' }
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
function stagedFiles(root) {
|
|
532
|
+
return fs.readdirSync(path.join(root, 'backups', '.staging'))
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
test('a non-empty database is snapshotted by the engine, never copied', () => {
|
|
536
|
+
const root = tempRoot('engine-snapshot')
|
|
537
|
+
try {
|
|
538
|
+
seed(root, 'live')
|
|
539
|
+
const live = seedDatabase(root, 'libraries/lib-a.sqlite3', sqliteFile('LIVE-BYTES-ONLY'))
|
|
540
|
+
seedDatabase(root, 'libraries/lib-a.sqlite3-wal', Buffer.alloc(64, 7))
|
|
541
|
+
const engine = fakeEngine(root)
|
|
542
|
+
const calls = []
|
|
543
|
+
const created = backup.createBackup({ keep: 10, pythonExec: fakeInterpreter(engine.venv, { calls }) })
|
|
544
|
+
|
|
545
|
+
const copy = fs.readFileSync(path.join(created.dir, 'libraries', 'lib-a.sqlite3'))
|
|
546
|
+
assert.equal(copy.toString('latin1').includes('ENGINE-SNAPSHOT'), true, 'the published file is the engine snapshot')
|
|
547
|
+
assert.equal(copy.toString('latin1').includes('LIVE-BYTES-ONLY'), false, 'the live bytes were not copied')
|
|
548
|
+
assert.equal(fs.existsSync(path.join(created.dir, 'libraries', 'lib-a.sqlite3-wal')), false, 'no WAL sidecar is published')
|
|
549
|
+
assert.equal(fs.readFileSync(live, 'latin1').includes('LIVE-BYTES-ONLY'), true, 'the live database is untouched')
|
|
550
|
+
|
|
551
|
+
const manifest = JSON.parse(fs.readFileSync(path.join(created.dir, 'manifest.json'), 'utf8'))
|
|
552
|
+
const snapshot = manifest.snapshot.databases.find((item) => item.file === 'libraries/lib-a.sqlite3')
|
|
553
|
+
assert.equal(snapshot.mode, 'sqlite-backup-api')
|
|
554
|
+
assert.equal(snapshot.quickCheck, 'ok')
|
|
555
|
+
assert.equal(snapshot.engine, '9.9.9', 'the manifest names the engine that took the snapshot')
|
|
556
|
+
assert.equal(created.snapshots[0].engine, '9.9.9')
|
|
557
|
+
assert.ok(calls.some((call) => call.args.length === 4), 'the engine interpreter ran the backup API')
|
|
558
|
+
assert.ok(calls.every((call) => call.cmd === engine.python), 'and it is the venv interpreter, not something else')
|
|
559
|
+
} finally { releaseRoot(root) }
|
|
560
|
+
})
|
|
561
|
+
|
|
562
|
+
test('without a verified engine the backup fails and publishes nothing', () => {
|
|
563
|
+
const root = tempRoot('no-engine')
|
|
564
|
+
try {
|
|
565
|
+
seed(root, 'live')
|
|
566
|
+
const live = seedDatabase(root, 'libraries/lib-a.sqlite3', sqliteFile('LIVE-BYTES'))
|
|
567
|
+
const before = fs.readFileSync(live)
|
|
568
|
+
let caught = null
|
|
569
|
+
try { backup.createBackup({ keep: 10 }) } catch (error) { caught = error }
|
|
570
|
+
assert.ok(caught, 'the backup refused')
|
|
571
|
+
assert.equal(caught.code, 'DSH_MEMU_BACKUP_ENGINE_REQUIRED')
|
|
572
|
+
assert.match(caught.message, /engine/)
|
|
573
|
+
assert.match(caught.message, /Nothing was copied/)
|
|
574
|
+
assert.equal(backup.scanBackups().length, 0, 'no backup directory appeared')
|
|
575
|
+
assert.deepEqual(stagedFiles(root), [], 'the staging directory was discarded')
|
|
576
|
+
assert.deepEqual(fs.readFileSync(live), before, 'the live database was not touched')
|
|
577
|
+
} finally { releaseRoot(root) }
|
|
578
|
+
})
|
|
579
|
+
|
|
580
|
+
test('a global interpreter is never borrowed, and a shim is caught by sys.prefix', () => {
|
|
581
|
+
const root = tempRoot('global-python')
|
|
582
|
+
try {
|
|
583
|
+
seed(root, 'live')
|
|
584
|
+
seedDatabase(root, 'libraries/lib-a.sqlite3', sqliteFile('LIVE-BYTES'))
|
|
585
|
+
const engine = fakeEngine(root)
|
|
586
|
+
// DSH_MEMU_PYTHON is what the old resolution order trusted first. It must
|
|
587
|
+
// not even be consulted now.
|
|
588
|
+
const decoy = path.join(root, 'global-python.exe')
|
|
589
|
+
fs.writeFileSync(decoy, '')
|
|
590
|
+
process.env.DSH_MEMU_PYTHON = decoy
|
|
591
|
+
const calls = []
|
|
592
|
+
// A venv python that hands the work to the global interpreter reports the
|
|
593
|
+
// global sys.prefix, which is the proof the isolation check needs.
|
|
594
|
+
const outside = path.join(os.tmpdir(), 'not-an-engine')
|
|
595
|
+
fs.mkdirSync(outside, { recursive: true })
|
|
596
|
+
const shim = (cmd, args) => {
|
|
597
|
+
calls.push({ cmd, args })
|
|
598
|
+
if (args.length === 2) return { status: 0, stdout: JSON.stringify({ ok: true, prefix: outside, sqlite: '3.45.0' }), stderr: '' }
|
|
599
|
+
fs.writeFileSync(args[3], sqliteFile('WRONG'))
|
|
600
|
+
return { status: 0, stdout: JSON.stringify({ ok: true, quickCheck: 'ok' }), stderr: '' }
|
|
601
|
+
}
|
|
602
|
+
let caught = null
|
|
603
|
+
try { backup.createBackup({ keep: 10, pythonExec: shim }) } catch (error) { caught = error }
|
|
604
|
+
assert.ok(caught, 'the backup refused')
|
|
605
|
+
assert.equal(caught.code, 'DSH_MEMU_BACKUP_ENGINE_REQUIRED')
|
|
606
|
+
assert.match(caught.message, /sys\.prefix/)
|
|
607
|
+
assert.ok(calls.every((call) => call.cmd !== decoy), 'the decoy interpreter was never executed')
|
|
608
|
+
assert.ok(calls.every((call) => call.args.length === 2), 'no snapshot was attempted through the shim')
|
|
609
|
+
assert.equal(backup.scanBackups().length, 0)
|
|
610
|
+
assert.equal(fs.existsSync(path.join(root, 'backups', 'libraries')), false)
|
|
611
|
+
} finally { releaseRoot(root) }
|
|
612
|
+
})
|
|
613
|
+
|
|
614
|
+
test('an engine that cannot prove its snapshot fails the whole backup', () => {
|
|
615
|
+
for (const [snapshot, expected] of [
|
|
616
|
+
['io-error', /disk I\/O error/],
|
|
617
|
+
['bad-check', /quick_check/],
|
|
618
|
+
['no-header', /no SQLite database header/],
|
|
619
|
+
['crashes', /interpreter crashed/],
|
|
620
|
+
]) {
|
|
621
|
+
const root = tempRoot('snapshot-' + snapshot)
|
|
622
|
+
try {
|
|
623
|
+
seed(root, snapshot)
|
|
624
|
+
seedDatabase(root, 'libraries/lib-a.sqlite3', sqliteFile('LIVE-BYTES'))
|
|
625
|
+
const engine = fakeEngine(root)
|
|
626
|
+
let caught = null
|
|
627
|
+
try { backup.createBackup({ keep: 10, pythonExec: fakeInterpreter(engine.venv, { snapshot }) }) } catch (error) { caught = error }
|
|
628
|
+
assert.ok(caught, snapshot + ': the backup refused')
|
|
629
|
+
assert.equal(caught.code, 'DSH_MEMU_BACKUP_SNAPSHOT_FAILED', snapshot + ': the refusal names the snapshot step')
|
|
630
|
+
assert.match(caught.message, expected)
|
|
631
|
+
assert.equal(backup.scanBackups().length, 0, snapshot + ': nothing was published')
|
|
632
|
+
assert.deepEqual(stagedFiles(root), [], snapshot + ': no partial snapshot is left behind')
|
|
633
|
+
assert.equal(fs.existsSync(path.join(root, 'backups', 'libraries')), false, snapshot + ': no torn copy exists anywhere')
|
|
634
|
+
} finally { releaseRoot(root) }
|
|
635
|
+
}
|
|
636
|
+
})
|
|
637
|
+
|
|
638
|
+
test('a zero-byte placeholder is copied and needs no engine', () => {
|
|
639
|
+
const root = tempRoot('placeholder')
|
|
640
|
+
try {
|
|
641
|
+
seed(root, 'empty')
|
|
642
|
+
seedDatabase(root, 'global.sqlite3', Buffer.alloc(0))
|
|
643
|
+
const created = backup.createBackup({ keep: 10 })
|
|
644
|
+
const manifest = JSON.parse(fs.readFileSync(path.join(created.dir, 'manifest.json'), 'utf8'))
|
|
645
|
+
const snapshot = manifest.snapshot.databases.find((item) => item.file === 'global.sqlite3')
|
|
646
|
+
assert.equal(snapshot.mode, 'empty-placeholder')
|
|
647
|
+
assert.equal(snapshot.quickCheck, 'empty')
|
|
648
|
+
assert.equal(fs.statSync(path.join(created.dir, 'global.sqlite3')).size, 0)
|
|
649
|
+
} finally { releaseRoot(root) }
|
|
650
|
+
})
|
|
651
|
+
|
|
652
|
+
test('a non-empty file that is not SQLite is refused instead of copied', () => {
|
|
653
|
+
const root = tempRoot('not-sqlite')
|
|
654
|
+
try {
|
|
655
|
+
seed(root, 'junk')
|
|
656
|
+
seedDatabase(root, 'global.sqlite3', Buffer.alloc(4096, 65))
|
|
657
|
+
let caught = null
|
|
658
|
+
try { backup.createBackup({ keep: 10 }) } catch (error) { caught = error }
|
|
659
|
+
assert.ok(caught)
|
|
660
|
+
assert.equal(caught.code, 'DSH_MEMU_BACKUP_NOT_SQLITE')
|
|
661
|
+
assert.match(caught.message, /neither an empty placeholder nor a SQLite database/)
|
|
662
|
+
assert.equal(backup.scanBackups().length, 0)
|
|
663
|
+
} finally { releaseRoot(root) }
|
|
664
|
+
})
|
|
665
|
+
|
|
666
|
+
test('even with an installed engine, force-unavailable sqlite refuses rather than copies', () => {
|
|
667
|
+
const root = tempRoot('no-python-switch')
|
|
668
|
+
try {
|
|
669
|
+
seed(root, 'live')
|
|
670
|
+
seedDatabase(root, 'libraries/lib-a.sqlite3', sqliteFile('LIVE-BYTES'))
|
|
671
|
+
const engine = fakeEngine(root)
|
|
672
|
+
process.env.DSH_MEMU_BACKUP_NO_PYTHON = '1'
|
|
673
|
+
backup.resetBackupPythonProbe()
|
|
674
|
+
let caught = null
|
|
675
|
+
try { backup.createBackup({ keep: 10, pythonExec: fakeInterpreter(engine.venv) }) } catch (error) { caught = error }
|
|
676
|
+
assert.ok(caught, 'the forced-unavailable engine refuses')
|
|
677
|
+
assert.equal(caught.code, 'DSH_MEMU_BACKUP_ENGINE_REQUIRED')
|
|
678
|
+
assert.equal(backup.scanBackups().length, 0)
|
|
679
|
+
} finally { releaseRoot(root) }
|
|
680
|
+
})
|
|
681
|
+
|
|
682
|
+
test('a refused backup is visible through the state API, not swallowed', () => {
|
|
683
|
+
const root = tempRoot('refusal-report')
|
|
684
|
+
try {
|
|
685
|
+
seed(root, 'live')
|
|
686
|
+
seedDatabase(root, 'libraries/lib-a.sqlite3', sqliteFile('LIVE-BYTES'))
|
|
687
|
+
assert.throws(() => state.backupDatabases(), /engine/i)
|
|
688
|
+
const report = state.lastBackupReportView()
|
|
689
|
+
assert.equal(report.action, 'create')
|
|
690
|
+
assert.equal(report.status, 'failed')
|
|
691
|
+
assert.equal(report.code, 'DSH_MEMU_BACKUP_ENGINE_REQUIRED')
|
|
692
|
+
assert.match(report.error, /engine/)
|
|
693
|
+
assert.equal(report.dir, null)
|
|
694
|
+
} finally { releaseRoot(root) }
|
|
695
|
+
})
|
|
696
|
+
// Opt-in end-to-end proof against a real venv and a real WAL database:
|
|
697
|
+
// DSH_MEMU_BACKUP_REAL_ENGINE=1 npm test
|
|
698
|
+
// It is skipped by default for the same reason every other real-engine case in
|
|
699
|
+
// this repo is: it needs a base Python to create the venv with (the venv is
|
|
700
|
+
// built exactly the way lib/engine.js builds it) and it takes a few seconds.
|
|
701
|
+
test('a real venv interpreter folds a real live WAL into the backup', (t) => {
|
|
702
|
+
if (process.env.DSH_MEMU_BACKUP_REAL_ENGINE !== '1') {
|
|
703
|
+
t.skip('set DSH_MEMU_BACKUP_REAL_ENGINE=1 to run the real-engine case')
|
|
704
|
+
return
|
|
705
|
+
}
|
|
706
|
+
const base = [
|
|
707
|
+
process.platform === 'win32' ? ['py', ['-3']] : ['python3', []],
|
|
708
|
+
['python3', []],
|
|
709
|
+
['python', []],
|
|
710
|
+
].map(([cmd, prefix]) => ({ cmd, prefix })).find((candidate) => {
|
|
711
|
+
try {
|
|
712
|
+
const probe = spawnSync(candidate.cmd, candidate.prefix.concat(['-c', 'import sqlite3, sys']), { encoding: 'utf8', windowsHide: true })
|
|
713
|
+
return probe.status === 0
|
|
714
|
+
} catch { return false }
|
|
715
|
+
})
|
|
716
|
+
if (!base) { t.skip('no base Python available to build a venv'); return }
|
|
717
|
+
|
|
718
|
+
const root = tempRoot('real-engine')
|
|
719
|
+
try {
|
|
720
|
+
const version = '9.9.9'
|
|
721
|
+
const venv = path.join(root, 'engine', version, 'venv')
|
|
722
|
+
const built = spawnSync(base.cmd, base.prefix.concat(['-m', 'venv', '--without-pip', venv]), { encoding: 'utf8', windowsHide: true })
|
|
723
|
+
assert.equal(built.status, 0, 'the venv was created: ' + String(built.stderr || ''))
|
|
724
|
+
const python = process.platform === 'win32' ? path.join(venv, 'Scripts', 'python.exe') : path.join(venv, 'bin', 'python3')
|
|
725
|
+
assert.equal(fs.existsSync(python), true)
|
|
726
|
+
fs.writeFileSync(path.join(root, 'engine', 'current.json'), JSON.stringify({ version, venv, python }))
|
|
727
|
+
|
|
728
|
+
seed(root, 'real')
|
|
729
|
+
const db = seedDatabase(root, 'libraries/lib-real.sqlite3', '')
|
|
730
|
+
const made = spawnSync(python, ['-c', [
|
|
731
|
+
'import os, sqlite3',
|
|
732
|
+
'conn = sqlite3.connect(os.environ["DB"])',
|
|
733
|
+
'conn.execute("pragma journal_mode=wal")',
|
|
734
|
+
'conn.execute("create table t (id integer primary key, v text)")',
|
|
735
|
+
'conn.executemany("insert into t (v) values (?)", [("a",), ("b",), ("c",)])',
|
|
736
|
+
'conn.commit()',
|
|
737
|
+
'os._exit(0)',
|
|
738
|
+
].join(String.fromCharCode(10))], { encoding: 'utf8', windowsHide: true, env: { ...process.env, DB: db } })
|
|
739
|
+
assert.equal(made.status, 0, 'the live database was written: ' + String(made.stderr || ''))
|
|
740
|
+
assert.ok(fs.statSync(db + '-wal').size > 0, 'the database really has a live WAL to fold in')
|
|
741
|
+
|
|
742
|
+
const created = backup.createBackup({ keep: 10 })
|
|
743
|
+
const manifest = JSON.parse(fs.readFileSync(path.join(created.dir, 'manifest.json'), 'utf8'))
|
|
744
|
+
const snapshot = manifest.snapshot.databases.find((item) => item.file === 'libraries/lib-real.sqlite3')
|
|
745
|
+
assert.equal(snapshot.mode, 'sqlite-backup-api')
|
|
746
|
+
assert.equal(snapshot.quickCheck, 'ok')
|
|
747
|
+
assert.equal(snapshot.engine, version)
|
|
748
|
+
assert.equal(fs.existsSync(path.join(created.dir, 'libraries', 'lib-real.sqlite3-wal')), false, 'the WAL was folded in, not copied beside it')
|
|
749
|
+
|
|
750
|
+
// The published file must be a self-contained database holding the rows
|
|
751
|
+
// that were still in the WAL when the backup started.
|
|
752
|
+
const read = spawnSync(python, ['-c', [
|
|
753
|
+
'import os, sqlite3',
|
|
754
|
+
'conn = sqlite3.connect("file:" + os.environ["DB"] + "?mode=ro", uri=True)',
|
|
755
|
+
'print(conn.execute("pragma quick_check").fetchone()[0])',
|
|
756
|
+
'print(conn.execute("select count(*) from t").fetchone()[0])',
|
|
757
|
+
'conn.close()',
|
|
758
|
+
].join(String.fromCharCode(10))], { encoding: 'utf8', windowsHide: true, env: { ...process.env, DB: path.join(created.dir, 'libraries', 'lib-real.sqlite3') }, cwd: path.join(created.dir, 'libraries') })
|
|
759
|
+
const lines = String(read.stdout || '').trim().split(/\r?\n/)
|
|
760
|
+
assert.equal(lines[0], 'ok', 'the snapshot passes quick_check')
|
|
761
|
+
assert.equal(lines[1], '3', 'every committed row made it in, including the WAL frames')
|
|
762
|
+
} finally { releaseRoot(root) }
|
|
763
|
+
})
|
|
764
|
+
test('an engine still installing (venv.tmp-*) can take the snapshot too', () => {
|
|
765
|
+
// The repair path: lib/engine.js backs up with beforeSwitch while the new
|
|
766
|
+
// venv is still `venv.tmp-*` and current.json still points at the broken one.
|
|
767
|
+
const root = tempRoot('repair')
|
|
768
|
+
try {
|
|
769
|
+
seed(root, 'live')
|
|
770
|
+
seedDatabase(root, 'libraries/lib-a.sqlite3', sqliteFile('LIVE-BYTES'))
|
|
771
|
+
const venv = path.join(root, 'engine', '0.11.0b3', 'venv.tmp-' + process.pid + '-1')
|
|
772
|
+
const bin = process.platform === 'win32' ? path.join(venv, 'Scripts') : path.join(venv, 'bin')
|
|
773
|
+
fs.mkdirSync(bin, { recursive: true })
|
|
774
|
+
const python = path.join(bin, process.platform === 'win32' ? 'python.exe' : 'python3')
|
|
775
|
+
fs.writeFileSync(python, '')
|
|
776
|
+
const created = backup.createBackup({ keep: 10, pythonExec: fakeInterpreter(venv) })
|
|
777
|
+
const manifest = JSON.parse(fs.readFileSync(path.join(created.dir, 'manifest.json'), 'utf8'))
|
|
778
|
+
const snapshot = manifest.snapshot.databases.find((item) => item.file === 'libraries/lib-a.sqlite3')
|
|
779
|
+
assert.equal(snapshot.mode, 'sqlite-backup-api')
|
|
780
|
+
assert.equal(snapshot.engine, '0.11.0b3')
|
|
781
|
+
} finally { releaseRoot(root) }
|
|
782
|
+
})
|
|
783
|
+
|
|
784
|
+
test('lib/backup.js cannot reach a global interpreter or a copy fallback', () => {
|
|
785
|
+
const src = fs.readFileSync(path.join(process.cwd(), 'lib', 'backup.js'), 'utf8')
|
|
786
|
+
assert.equal(src.includes('DSH_MEMU_PYTHON'), false, 'the env override is not consulted at all')
|
|
787
|
+
assert.equal(src.includes('pythonCandidates'), false, 'the PATH/py-launcher candidate list is gone')
|
|
788
|
+
assert.equal(src.includes("'-3'"), false, 'no py-launcher flag survives')
|
|
789
|
+
assert.ok(src.includes("path.join(venv, 'bin', 'python3')"), true, 'the only interpreter is the one inside the venv')
|
|
790
|
+
assert.equal(src.includes('COPY_ATTEMPTS'), false, 'the retry-copy loop is gone')
|
|
791
|
+
assert.equal(src.includes('guarded-copy'), false, 'and so is the guarded copy mode')
|
|
792
|
+
assert.equal(src.includes('signatureOf'), false, 'nor the size+mtime stability check that used to justify a copy')
|
|
793
|
+
});
|