@summerscar/dsh-mnemon-sync 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +86 -2
- package/client.js +1194 -0
- package/cordis.patch.yml +3 -0
- package/icon.svg +15 -0
- package/index.js +731 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +75 -4
- package/scheduler.js +241 -0
- package/snapshot.js +400 -0
- package/source.js +201 -0
- package/store.js +187 -0
- package/targets.js +533 -0
- package/zip.js +168 -0
package/index.js
ADDED
|
@@ -0,0 +1,731 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mnemon Sync — Host half.
|
|
3
|
+
*
|
|
4
|
+
* Backs up the whole `<MNEMON_DATA_DIR>/data` directory — every memory space
|
|
5
|
+
* database, not just `default` — and ships it to an S3-compatible bucket or a
|
|
6
|
+
* WebDAV collection.
|
|
7
|
+
*
|
|
8
|
+
* Why not litestream: it replicates *one* database per instance, and in v0.5 a
|
|
9
|
+
* database accepts exactly one replica, so directory-level backup would mean
|
|
10
|
+
* one long-lived process per space plus dynamic reconfiguration whenever a
|
|
11
|
+
* space is added. It also holds the WAL open, which makes dsh-mnemon's own
|
|
12
|
+
* Pack export fail its `-wal is not checkpointed` check. `node:sqlite`'s
|
|
13
|
+
* `backup()` gives a transaction-consistent copy of each live database with no
|
|
14
|
+
* resident process, which is both simpler and compatible with the store.
|
|
15
|
+
*
|
|
16
|
+
* The browser half only ever sees JSON: credentials are masked on the way out
|
|
17
|
+
* and are never echoed back by a read.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { createHash } from 'node:crypto'
|
|
21
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
22
|
+
import { createRequire } from 'node:module'
|
|
23
|
+
import { dirname, join, resolve } from 'node:path'
|
|
24
|
+
import { pathToFileURL } from 'node:url'
|
|
25
|
+
|
|
26
|
+
import { SyncHistory, SyncScheduler } from './scheduler.js'
|
|
27
|
+
import {
|
|
28
|
+
COMPONENTS,
|
|
29
|
+
MEMORY_SPACE_DATABASE_FILE,
|
|
30
|
+
buildSnapshot,
|
|
31
|
+
inspectHome,
|
|
32
|
+
readSpaceNames,
|
|
33
|
+
verifySnapshot,
|
|
34
|
+
} from './snapshot.js'
|
|
35
|
+
import { createZip, readZip } from './zip.js'
|
|
36
|
+
import { deleteArchive, downloadArchive, listArchives, normalizeTarget, uploadArchive } from './targets.js'
|
|
37
|
+
import { maskSettings, normalizeSettings, readSettings, settingsPath, writeSettings } from './store.js'
|
|
38
|
+
|
|
39
|
+
const API_PREFIX = '/api/mnemon-sync'
|
|
40
|
+
const ARCHIVE_PREFIX = 'mnemon-sync-'
|
|
41
|
+
|
|
42
|
+
export const name = 'dsh-mnemon-sync'
|
|
43
|
+
// `mnemonMemory` is optional on purpose: the plugin still backs up on a profile
|
|
44
|
+
// where dsh-mnemon is absent or too old to accept Sources, it just cannot offer
|
|
45
|
+
// its page there.
|
|
46
|
+
export const inject = ['webServer']
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* `MNEMON_DATA_DIR` names the Mnemon home; the memory store itself lives in
|
|
50
|
+
* its `data` subdirectory, exactly as dsh-mnemon resolves it.
|
|
51
|
+
*/
|
|
52
|
+
export function resolveDataRoot(config = {}) {
|
|
53
|
+
if (typeof config.dataRoot === 'string' && config.dataRoot.trim() !== '') {
|
|
54
|
+
return resolve(config.dataRoot.trim())
|
|
55
|
+
}
|
|
56
|
+
const fromEnvironment = process.env.MNEMON_DATA_DIR?.trim()
|
|
57
|
+
const home = fromEnvironment === undefined || fromEnvironment === ''
|
|
58
|
+
? join(process.env.HOME ?? '', '.mnemon')
|
|
59
|
+
: fromEnvironment
|
|
60
|
+
return join(home, 'data')
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function sendJson(res, status, body) {
|
|
64
|
+
const payload = Buffer.from(`${JSON.stringify(body)}\n`, 'utf8')
|
|
65
|
+
res.writeHead(status, {
|
|
66
|
+
'content-type': 'application/json; charset=utf-8',
|
|
67
|
+
'content-length': String(payload.length),
|
|
68
|
+
'cache-control': 'no-store',
|
|
69
|
+
})
|
|
70
|
+
res.end(payload)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async function readJsonBody(req) {
|
|
74
|
+
const chunks = []
|
|
75
|
+
let total = 0
|
|
76
|
+
for await (const chunk of req) {
|
|
77
|
+
total += chunk.length
|
|
78
|
+
if (total > 1_000_000) throw new Error('request body too large')
|
|
79
|
+
chunks.push(chunk)
|
|
80
|
+
}
|
|
81
|
+
if (chunks.length === 0) return {}
|
|
82
|
+
const text = Buffer.concat(chunks).toString('utf8').trim()
|
|
83
|
+
if (text === '') return {}
|
|
84
|
+
return JSON.parse(text)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** `mnemon-sync-2026-11-09T041500Z.zip`: sorts lexicographically by time. */
|
|
88
|
+
export function archiveKey(at = new Date()) {
|
|
89
|
+
return `${ARCHIVE_PREFIX}${at.toISOString().replace(/:/gu, '-')}.zip`
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function apply(ctx, config = {}) {
|
|
93
|
+
const dataRoot = resolveDataRoot(config)
|
|
94
|
+
// `<home>/data` → `<home>`: settings and work files sit beside the store.
|
|
95
|
+
const home = resolve(dataRoot, '..')
|
|
96
|
+
const workRoot = join(home, 'state', 'mnemon-sync-work')
|
|
97
|
+
|
|
98
|
+
const history = new SyncHistory(home)
|
|
99
|
+
const settings = () => readSettings(home)
|
|
100
|
+
|
|
101
|
+
const performBackup = async (trigger, current, signal) => {
|
|
102
|
+
if (current.enabled !== true && trigger === 'schedule') {
|
|
103
|
+
return { skipped: 'disabled' }
|
|
104
|
+
}
|
|
105
|
+
const target = normalizeTarget(current.target)
|
|
106
|
+
const staging = join(workRoot, `build-${String(Date.now())}`)
|
|
107
|
+
mkdirSync(staging, { recursive: true })
|
|
108
|
+
const startedAt = new Date().toISOString()
|
|
109
|
+
try {
|
|
110
|
+
const snapshot = await buildSnapshot(home, staging, { components: current.components })
|
|
111
|
+
verifySnapshot(staging)
|
|
112
|
+
const key = archiveKey()
|
|
113
|
+
const archive = createZip(collect(staging), { level: 6 })
|
|
114
|
+
const uploaded = await uploadArchive(target, key, archive, { signal })
|
|
115
|
+
const removed = await applyRetention(history, target, current.retention.keep, { signal })
|
|
116
|
+
const record = {
|
|
117
|
+
id: `${String(Date.now())}-${Math.random().toString(36).slice(2, 8)}`,
|
|
118
|
+
startedAt,
|
|
119
|
+
finishedAt: new Date().toISOString(),
|
|
120
|
+
ok: true,
|
|
121
|
+
trigger,
|
|
122
|
+
durationMs: Date.now() - Date.parse(startedAt),
|
|
123
|
+
// What actually shipped, so the history row can say "3 components"
|
|
124
|
+
// rather than assuming memory spaces were involved.
|
|
125
|
+
components: snapshot.manifest.components,
|
|
126
|
+
spaces: countSpaces(snapshot.manifest),
|
|
127
|
+
files: snapshot.files.length,
|
|
128
|
+
bytes: archive.length,
|
|
129
|
+
key,
|
|
130
|
+
location: uploaded.location,
|
|
131
|
+
removed,
|
|
132
|
+
}
|
|
133
|
+
history.append(record)
|
|
134
|
+
return record
|
|
135
|
+
} catch (error) {
|
|
136
|
+
history.append({
|
|
137
|
+
id: `${String(Date.now())}-${Math.random().toString(36).slice(2, 8)}`,
|
|
138
|
+
startedAt,
|
|
139
|
+
finishedAt: new Date().toISOString(),
|
|
140
|
+
ok: false,
|
|
141
|
+
trigger,
|
|
142
|
+
durationMs: Date.now() - Date.parse(startedAt),
|
|
143
|
+
error: error instanceof Error ? error.message : String(error),
|
|
144
|
+
})
|
|
145
|
+
throw error
|
|
146
|
+
} finally {
|
|
147
|
+
rmSync(staging, { recursive: true, force: true })
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const scheduler = new SyncScheduler({
|
|
152
|
+
readSettings: settings,
|
|
153
|
+
run: performBackup,
|
|
154
|
+
onError: () => {
|
|
155
|
+
// History already records the failure; the timer must stay silent.
|
|
156
|
+
},
|
|
157
|
+
})
|
|
158
|
+
|
|
159
|
+
ctx.effect(() => scheduler.start(), 'mnemon-sync: scheduler')
|
|
160
|
+
|
|
161
|
+
// Register the Source that owns the sync page.
|
|
162
|
+
//
|
|
163
|
+
// dsh-mnemon shows a page tab only when its `sourceTypeId` appears in the
|
|
164
|
+
// live source catalog, and that catalog lists running Source implementations
|
|
165
|
+
// — not UI contributions. Standing up this minimal Source is what lets the
|
|
166
|
+
// page use its own type instead of borrowing `memory-spaces`, which is the
|
|
167
|
+
// only way its tab can highlight independently.
|
|
168
|
+
ctx.effect(() => registerSyncSource(ctx, dataRoot), 'mnemon-sync: memory source')
|
|
169
|
+
|
|
170
|
+
ctx.effect(() => ctx.webServer.register({
|
|
171
|
+
kind: 'prefix',
|
|
172
|
+
path: API_PREFIX,
|
|
173
|
+
handler: (req, res) => {
|
|
174
|
+
void handle(req, res).catch((error) => {
|
|
175
|
+
sendJson(res, 500, { error: error instanceof Error ? error.message : String(error) })
|
|
176
|
+
})
|
|
177
|
+
},
|
|
178
|
+
}), 'mnemon-sync: http api')
|
|
179
|
+
|
|
180
|
+
async function handle(req, res) {
|
|
181
|
+
const url = new URL(req.url ?? '/', 'http://mnemon-sync.invalid')
|
|
182
|
+
const route = url.pathname.slice(API_PREFIX.length) || '/'
|
|
183
|
+
|
|
184
|
+
if (req.method === 'GET' && route === '/status') {
|
|
185
|
+
const current = settings()
|
|
186
|
+
sendJson(res, 200, {
|
|
187
|
+
dataRoot,
|
|
188
|
+
settingsPath: settingsPath(home),
|
|
189
|
+
workRoot,
|
|
190
|
+
state: inspectHome(home),
|
|
191
|
+
settings: maskSettings(current),
|
|
192
|
+
scheduler: scheduler.snapshot(current),
|
|
193
|
+
|
|
194
|
+
history: history.read(),
|
|
195
|
+
spaceNames: Object.fromEntries(readSpaceNames(home)),
|
|
196
|
+
})
|
|
197
|
+
return
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (req.method === 'PUT' && route === '/settings') {
|
|
201
|
+
const body = await readJsonBody(req)
|
|
202
|
+
const incoming = body?.settings ?? body
|
|
203
|
+
const saved = writeSettings(home, normalizeSettings(restoreMaskedSecrets(incoming, settings())))
|
|
204
|
+
scheduler.refresh()
|
|
205
|
+
sendJson(res, 200, { settings: maskSettings(saved) })
|
|
206
|
+
return
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if (req.method === 'POST' && route === '/run') {
|
|
210
|
+
// An unsaved edit followed by "Back up now" must back up what the user is
|
|
211
|
+
// looking at, not what was last saved. When the page sends its draft, it
|
|
212
|
+
// is normalized and saved first — so the button never silently ignores a
|
|
213
|
+
// change — and the run then reads that same saved state.
|
|
214
|
+
const body = await readJsonBody(req)
|
|
215
|
+
try {
|
|
216
|
+
if (body?.settings !== undefined) {
|
|
217
|
+
// The same masked-secret rule as saving settings: the page always
|
|
218
|
+
// sends the mask it was shown, and writing that mask as the real
|
|
219
|
+
// password would silently break the next upload with a 401.
|
|
220
|
+
const saved = writeSettings(home, normalizeSettings(restoreMaskedSecrets(body.settings, settings())))
|
|
221
|
+
scheduler.refresh()
|
|
222
|
+
const result = await scheduler.runNow('manual')
|
|
223
|
+
sendJson(res, 200, { result, settings: maskSettings(saved) })
|
|
224
|
+
return
|
|
225
|
+
}
|
|
226
|
+
const result = await scheduler.runNow('manual')
|
|
227
|
+
sendJson(res, 200, { result })
|
|
228
|
+
} catch (error) {
|
|
229
|
+
sendJson(res, 500, { error: error instanceof Error ? error.message : String(error) })
|
|
230
|
+
}
|
|
231
|
+
return
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if (req.method === 'POST' && route === '/clear-history') {
|
|
235
|
+
// Only the local run log is cleared; archives on the target are untouched.
|
|
236
|
+
sendJson(res, 200, { history: history.clear() })
|
|
237
|
+
return
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
if (req.method === 'POST' && route === '/refresh-schedule') {
|
|
241
|
+
scheduler.refresh()
|
|
242
|
+
sendJson(res, 200, { scheduler: scheduler.snapshot(settings()) })
|
|
243
|
+
return
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (req.method === 'GET' && route === '/archives') {
|
|
247
|
+
// What the target actually holds, so the page can list backups this
|
|
248
|
+
// machine did not necessarily make — which is what a person wants after
|
|
249
|
+
// reinstalling or on a second machine.
|
|
250
|
+
try {
|
|
251
|
+
const archives = await listArchives(normalizeTarget(settings().target))
|
|
252
|
+
sendJson(res, 200, { archives })
|
|
253
|
+
} catch (error) {
|
|
254
|
+
sendJson(res, 400, { error: error instanceof Error ? error.message : String(error) })
|
|
255
|
+
}
|
|
256
|
+
return
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
if (req.method === 'POST' && route === '/inspect-archive') {
|
|
260
|
+
// Read one archive back and report what it actually holds. This is the
|
|
261
|
+
// restore half of "back up these components": an archive made when only
|
|
262
|
+
// documents were selected has no memory spaces, and saying so plainly is
|
|
263
|
+
// the difference between a restore and a silent half-restore.
|
|
264
|
+
const body = await readJsonBody(req)
|
|
265
|
+
try {
|
|
266
|
+
const report = await inspectArchive(settings(), String(body?.key ?? '').trim(), body?.components)
|
|
267
|
+
sendJson(res, 200, report)
|
|
268
|
+
} catch (error) {
|
|
269
|
+
sendJson(res, 400, { error: error instanceof Error ? error.message : String(error) })
|
|
270
|
+
}
|
|
271
|
+
return
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
if (req.method === 'POST' && route === '/restore') {
|
|
275
|
+
// Restore is a staged, verified operation: download, verify every
|
|
276
|
+
// checksum, extract to a staging directory, then either hand the files
|
|
277
|
+
// back for inspection or replace live data only when explicitly asked.
|
|
278
|
+
const body = await readJsonBody(req)
|
|
279
|
+
try {
|
|
280
|
+
const report = await restoreArchive(settings(), body ?? {})
|
|
281
|
+
sendJson(res, 200, report)
|
|
282
|
+
} catch (error) {
|
|
283
|
+
sendJson(res, 400, { error: error instanceof Error ? error.message : String(error) })
|
|
284
|
+
}
|
|
285
|
+
return
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
sendJson(res, 404, { error: `unknown route: ${req.method} ${route}` })
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Download one archive from the configured target and describe its contents.
|
|
293
|
+
*
|
|
294
|
+
* `components` is what the operator wants to restore; any requested component
|
|
295
|
+
* the archive does not carry is reported in `missing` with the reason, so the
|
|
296
|
+
* UI can warn before touching anything on disk.
|
|
297
|
+
*/
|
|
298
|
+
async function inspectArchive(current, key, requested) {
|
|
299
|
+
if (key === '') throw new Error('an archive key is required')
|
|
300
|
+
const target = normalizeTarget(current.target)
|
|
301
|
+
const bytes = await downloadArchive(target, key)
|
|
302
|
+
const files = readZip(bytes)
|
|
303
|
+
const manifestBytes = files.get('manifest.json')
|
|
304
|
+
if (manifestBytes === undefined) {
|
|
305
|
+
throw new Error('this archive has no manifest.json, so its contents cannot be verified')
|
|
306
|
+
}
|
|
307
|
+
const manifest = JSON.parse(manifestBytes.toString('utf8'))
|
|
308
|
+
if (manifest?.format !== 'mnemon-sync-snapshot') {
|
|
309
|
+
throw new Error(`unexpected archive format: ${String(manifest?.format)}`)
|
|
310
|
+
}
|
|
311
|
+
const present = Array.isArray(manifest.components) ? manifest.components : []
|
|
312
|
+
const wanted = Array.isArray(requested) && requested.length > 0
|
|
313
|
+
? requested.filter((value) => COMPONENTS.includes(value))
|
|
314
|
+
: present
|
|
315
|
+
const missing = wanted
|
|
316
|
+
.filter((component) => !present.includes(component))
|
|
317
|
+
.map((component) => ({
|
|
318
|
+
component,
|
|
319
|
+
reason: skipReason(manifest, component) ?? 'this archive was made without it',
|
|
320
|
+
}))
|
|
321
|
+
// Every payload file the manifest promises is checked against the archive,
|
|
322
|
+
// so a truncated upload is caught here rather than after a restore starts.
|
|
323
|
+
const expected = manifest.checksums?.files ?? {}
|
|
324
|
+
const absent = Object.keys(expected).filter((relative) => !files.has(relative))
|
|
325
|
+
|
|
326
|
+
// The same plan a restore would carry out, reported here so the detail
|
|
327
|
+
// panel can show what pressing the button would write — and so the button
|
|
328
|
+
// exists at all. Keeping one plan shape means the preview and the work
|
|
329
|
+
// cannot drift apart.
|
|
330
|
+
const plan = []
|
|
331
|
+
for (const [relative, content] of files.entries()) {
|
|
332
|
+
if (relative === 'manifest.json') continue
|
|
333
|
+
const component = componentDirectoryOf(relative)
|
|
334
|
+
if (component === undefined || !wanted.includes(component)) continue
|
|
335
|
+
plan.push({ component, path: relative, bytes: content.length })
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
return {
|
|
339
|
+
key,
|
|
340
|
+
createdAt: manifest.createdAt,
|
|
341
|
+
archiveComponents: present,
|
|
342
|
+
requested: wanted,
|
|
343
|
+
missing,
|
|
344
|
+
entries: (manifest.entries ?? []).filter((entry) => typeof entry.component === 'string'),
|
|
345
|
+
skipped: Array.isArray(manifest.skipped) ? manifest.skipped : [],
|
|
346
|
+
files: files.size,
|
|
347
|
+
bytes: bytes.length,
|
|
348
|
+
plan,
|
|
349
|
+
planBytes: plan.reduce((total, entry) => total + entry.bytes, 0),
|
|
350
|
+
// `intact` means "this archive is complete and readable", which is what the
|
|
351
|
+
// badge says. A missing component the operator asked for is reported in
|
|
352
|
+
// `missing` separately: the archive is not damaged, it simply does not
|
|
353
|
+
// contain that component.
|
|
354
|
+
intact: absent.length === 0,
|
|
355
|
+
...absent.length === 0 ? {} : { absent },
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Restore one archive into the live Mnemon home.
|
|
361
|
+
*
|
|
362
|
+
* The order is deliberate and never short-circuited:
|
|
363
|
+
* 1. download the archive,
|
|
364
|
+
* 2. verify every checksum in its manifest,
|
|
365
|
+
* 3. report any requested component the archive does not carry,
|
|
366
|
+
* 4. write files only for components the caller confirmed.
|
|
367
|
+
*
|
|
368
|
+
* Without `apply: true` this is a dry run: it answers what *would* be written
|
|
369
|
+
* and touches nothing. That makes the button safe to press while exploring,
|
|
370
|
+
* and keeps the destructive step an explicit second decision.
|
|
371
|
+
*
|
|
372
|
+
* Live SQLite databases are replaced with their checkpointed copy from the
|
|
373
|
+
* archive. The WAL and SHM sidecars are removed first, because a stale WAL
|
|
374
|
+
* beside a restored database would make SQLite replay transactions that the
|
|
375
|
+
* restored file never had — the one way a "correct" restore corrupts data.
|
|
376
|
+
*/
|
|
377
|
+
async function restoreArchive(current, body) {
|
|
378
|
+
const key = String(body?.key ?? '').trim()
|
|
379
|
+
if (key === '') throw new Error('an archive key is required')
|
|
380
|
+
const apply_ = body?.apply === true
|
|
381
|
+
const target = normalizeTarget(current.target)
|
|
382
|
+
const bytes = await downloadArchive(target, key)
|
|
383
|
+
const files = readZip(bytes)
|
|
384
|
+
const manifestBytes = files.get('manifest.json')
|
|
385
|
+
if (manifestBytes === undefined) {
|
|
386
|
+
throw new Error('this archive has no manifest.json, so its contents cannot be verified')
|
|
387
|
+
}
|
|
388
|
+
const manifest = JSON.parse(manifestBytes.toString('utf8'))
|
|
389
|
+
if (manifest?.format !== 'mnemon-sync-snapshot') {
|
|
390
|
+
throw new Error(`unexpected archive format: ${String(manifest?.format)}`)
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// Verify before writing anything: a truncated upload must be caught here.
|
|
394
|
+
const expected = manifest.checksums?.files ?? {}
|
|
395
|
+
const mismatched = []
|
|
396
|
+
for (const [relative, digest] of Object.entries(expected)) {
|
|
397
|
+
const content = files.get(relative)
|
|
398
|
+
if (content === undefined) {
|
|
399
|
+
mismatched.push({ path: relative, problem: 'missing from the archive' })
|
|
400
|
+
continue
|
|
401
|
+
}
|
|
402
|
+
const actual = createHash('sha256').update(content).digest('hex')
|
|
403
|
+
if (actual !== digest) mismatched.push({ path: relative, problem: 'checksum mismatch' })
|
|
404
|
+
}
|
|
405
|
+
if (mismatched.length > 0) {
|
|
406
|
+
return { key, applied: false, intact: false, mismatched }
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
const present = Array.isArray(manifest.components) ? manifest.components : []
|
|
410
|
+
const requested = Array.isArray(body?.components) && body.components.length > 0
|
|
411
|
+
? body.components.filter((value) => COMPONENTS.includes(value))
|
|
412
|
+
: present
|
|
413
|
+
const missing = requested
|
|
414
|
+
.filter((component) => !present.includes(component))
|
|
415
|
+
.map((component) => ({
|
|
416
|
+
component,
|
|
417
|
+
reason: skipReason(manifest, component) ?? 'this archive was made without it',
|
|
418
|
+
}))
|
|
419
|
+
const restorable = requested.filter((component) => present.includes(component))
|
|
420
|
+
|
|
421
|
+
// Plan first, so a dry run and a real run describe the same work.
|
|
422
|
+
const plan = []
|
|
423
|
+
for (const component of restorable) {
|
|
424
|
+
for (const [relative, content] of files.entries()) {
|
|
425
|
+
if (relative === 'manifest.json') continue
|
|
426
|
+
const director = componentDirectoryOf(relative)
|
|
427
|
+
if (director !== component) continue
|
|
428
|
+
plan.push({ component, relative, destination: destinationFor(home, relative), bytes: content.length })
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
if (!apply_) {
|
|
433
|
+
return {
|
|
434
|
+
key,
|
|
435
|
+
applied: false,
|
|
436
|
+
intact: true,
|
|
437
|
+
createdAt: manifest.createdAt,
|
|
438
|
+
archiveComponents: present,
|
|
439
|
+
requested,
|
|
440
|
+
missing,
|
|
441
|
+
plan: plan.map(({ relative, bytes: size }) => ({ path: relative, bytes: size })),
|
|
442
|
+
planBytes: plan.reduce((total, entry) => total + entry.bytes, 0),
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
// Refuse a live restore of memory spaces while the store is writing: the
|
|
447
|
+
// database being replaced may have an open connection, and swapping a file
|
|
448
|
+
// under it loses whatever was committed since the backup.
|
|
449
|
+
const written = []
|
|
450
|
+
for (const component of restorable) {
|
|
451
|
+
if (component === 'memory-spaces') {
|
|
452
|
+
for (const entry of plan.filter((item) => item.component === component)) {
|
|
453
|
+
const database = entry.destination
|
|
454
|
+
if (database.endsWith(MEMORY_SPACE_DATABASE_FILE)) {
|
|
455
|
+
// Remove sidecars before the database, see the doc comment.
|
|
456
|
+
for (const suffix of ['-wal', '-shm']) {
|
|
457
|
+
const sidecar = `${database}${suffix}`
|
|
458
|
+
if (existsSync(sidecar)) rmSync(sidecar, { force: true })
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
for (const entry of plan.filter((item) => item.component === component)) {
|
|
464
|
+
const content = files.get(entry.relative)
|
|
465
|
+
mkdirSync(dirname(entry.destination), { recursive: true })
|
|
466
|
+
writeFileSync(entry.destination, content, { mode: 0o600 })
|
|
467
|
+
written.push({ component, path: entry.relative, destination: entry.destination, bytes: entry.bytes })
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
return {
|
|
472
|
+
key,
|
|
473
|
+
applied: true,
|
|
474
|
+
intact: true,
|
|
475
|
+
createdAt: manifest.createdAt,
|
|
476
|
+
archiveComponents: present,
|
|
477
|
+
requested,
|
|
478
|
+
missing,
|
|
479
|
+
restored: restorable,
|
|
480
|
+
written,
|
|
481
|
+
writtenBytes: written.reduce((total, entry) => total + entry.bytes, 0),
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/** Why a component the operator asked for never made it into an archive. */
|
|
487
|
+
/**
|
|
488
|
+
* Which component an archive path belongs to.
|
|
489
|
+
*
|
|
490
|
+
* The payload layout is fixed by `buildSnapshot`: `payload/data/...` holds
|
|
491
|
+
* memory spaces, `payload/documents/...` project documents and
|
|
492
|
+
* `payload/runtime/...` runtime memory. Reading the component back off the path
|
|
493
|
+
* keeps the restore plan aligned with what was actually written, rather than
|
|
494
|
+
* trusting a second list that could drift.
|
|
495
|
+
*/
|
|
496
|
+
/**
|
|
497
|
+
* How many memory spaces a manifest holds.
|
|
498
|
+
*
|
|
499
|
+
* A memory-space summary entry lists its space ids in `spaces`; per-space
|
|
500
|
+
* records carry a single `id`. Both shapes are counted here so the history row
|
|
501
|
+
* is right whichever one the archive was made with, and a backup taken with
|
|
502
|
+
* memory spaces deselected honestly reads zero.
|
|
503
|
+
*/
|
|
504
|
+
function countSpaces(manifest) {
|
|
505
|
+
const entries = Array.isArray(manifest?.entries) ? manifest.entries : []
|
|
506
|
+
const ids = new Set()
|
|
507
|
+
for (const entry of entries) {
|
|
508
|
+
if (entry?.component !== 'memory-spaces') continue
|
|
509
|
+
if (Array.isArray(entry.spaces)) {
|
|
510
|
+
for (const id of entry.spaces) if (typeof id === 'string') ids.add(id)
|
|
511
|
+
continue
|
|
512
|
+
}
|
|
513
|
+
if (typeof entry.id === 'string') ids.add(entry.id)
|
|
514
|
+
}
|
|
515
|
+
return ids.size
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
function componentDirectoryOf(relative) {
|
|
519
|
+
if (!relative.startsWith('payload/')) return undefined
|
|
520
|
+
const [, directory] = relative.split('/')
|
|
521
|
+
if (directory === 'data') return 'memory-spaces'
|
|
522
|
+
if (directory === 'documents') return 'documents'
|
|
523
|
+
if (directory === 'runtime') return 'runtime'
|
|
524
|
+
return undefined
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* Where one archive path lands in the Mnemon home.
|
|
529
|
+
*
|
|
530
|
+
* `payload/` is dropped and the component directories are the same names the
|
|
531
|
+
* store itself uses, so a restored tree is byte-identical in layout to a live
|
|
532
|
+
* one. The path is resolved and checked so a malformed archive cannot walk out
|
|
533
|
+
* of the home directory.
|
|
534
|
+
*/
|
|
535
|
+
function destinationFor(home, relative) {
|
|
536
|
+
const stripped = relative.replace(/^payload\//u, '')
|
|
537
|
+
const destination = resolve(home, stripped)
|
|
538
|
+
const root = resolve(home)
|
|
539
|
+
if (destination !== root && !destination.startsWith(`${root}/`)) {
|
|
540
|
+
throw new Error(`archive entry escapes the Mnemon home: ${relative}`)
|
|
541
|
+
}
|
|
542
|
+
return destination
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/**
|
|
546
|
+
* Why a component the operator asked for never made it into an archive.
|
|
547
|
+
*/
|
|
548
|
+
function skipReason(manifest, component) {
|
|
549
|
+
const skipped = Array.isArray(manifest?.skipped) ? manifest.skipped : []
|
|
550
|
+
const entry = skipped.find((item) => item?.component === component)
|
|
551
|
+
return typeof entry?.reason === 'string' ? entry.reason : undefined
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
function isMasked(value) {
|
|
555
|
+
return typeof value === 'string' && value !== '' && /^•+$/u.test(value)
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* Put the real secrets back into a settings payload that arrived masked.
|
|
560
|
+
*
|
|
561
|
+
* The page never receives a secret — `maskSettings` replaces it with bullets —
|
|
562
|
+
* and echoes those bullets back on every save. Treating them as a value would
|
|
563
|
+
* overwrite the real credential with eight bullet characters, which is exactly
|
|
564
|
+
* what made a correctly configured WebDAV target answer 401: the stored
|
|
565
|
+
* password really was `••••••••`.
|
|
566
|
+
*
|
|
567
|
+
* Any write path that accepts settings from the browser must run this first.
|
|
568
|
+
*/
|
|
569
|
+
function restoreMaskedSecrets(incoming, previous) {
|
|
570
|
+
if (incoming === null || typeof incoming !== 'object') return incoming
|
|
571
|
+
const target = incoming.target
|
|
572
|
+
if (target === null || typeof target !== 'object') return incoming
|
|
573
|
+
if (target.kind === 'webdav' && isMasked(target.password)) {
|
|
574
|
+
target.password = previous.target.kind === 'webdav' ? previous.target.password : ''
|
|
575
|
+
}
|
|
576
|
+
if (target.kind === 's3' && isMasked(target.secretAccessKey)) {
|
|
577
|
+
target.secretAccessKey = previous.target.kind === 's3' ? previous.target.secretAccessKey : ''
|
|
578
|
+
}
|
|
579
|
+
if (target.kind === 's3' && isMasked(target.sessionToken)) {
|
|
580
|
+
target.sessionToken = previous.target.kind === 's3' ? previous.target.sessionToken : ''
|
|
581
|
+
}
|
|
582
|
+
return incoming
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/**
|
|
586
|
+
* Resolve the dsh-mnemon extension SDK.
|
|
587
|
+
*
|
|
588
|
+
* A bare `import('dsh-mnemon/extension-sdk')` is not enough here. Installed
|
|
589
|
+
* from npm the package sits at `<profile>/node_modules/@summerscar/...`, so
|
|
590
|
+
* Node's own ancestor walk would find dsh-mnemon beside it — but a *linked*
|
|
591
|
+
* checkout lives outside the profile with no `node_modules` of its own, and
|
|
592
|
+
* then the walk finds nothing.
|
|
593
|
+
*
|
|
594
|
+
* Both install shapes are therefore covered, cheapest first:
|
|
595
|
+
* 1. From this module's own location, which is right for an npm install.
|
|
596
|
+
* 2. From `DSH_PROFILE_DIR`, which is right for a linked checkout and is set
|
|
597
|
+
* in every shell and Host process a profile launches.
|
|
598
|
+
* 3. From `DSH_HOME/profiles/web`, for a Host booted without that variable.
|
|
599
|
+
*
|
|
600
|
+
* Returns `undefined` when dsh-mnemon is not installed at all, which is a
|
|
601
|
+
* supported configuration: the backup keeps working without its page.
|
|
602
|
+
*/
|
|
603
|
+
async function loadExtensionSdk() {
|
|
604
|
+
// 1. Ask Node, anchored at this file. Resolves for an npm install.
|
|
605
|
+
try {
|
|
606
|
+
const resolver = createRequire(import.meta.url)
|
|
607
|
+
const resolved = resolver.resolve('dsh-mnemon/lib/extension-sdk.js')
|
|
608
|
+
return await import(pathToFileURL(resolved).href)
|
|
609
|
+
} catch {
|
|
610
|
+
// Not resolvable from here: fall through to the profile-anchored paths.
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
const candidates = []
|
|
614
|
+
const profileDir = process.env.DSH_PROFILE_DIR?.trim()
|
|
615
|
+
if (profileDir !== undefined && profileDir !== '') {
|
|
616
|
+
candidates.push(join(profileDir, 'node_modules', 'dsh-mnemon', 'lib', 'extension-sdk.js'))
|
|
617
|
+
}
|
|
618
|
+
// The Host that loads this plugin lives in the installation's own tree.
|
|
619
|
+
candidates.push(join(process.env.DSH_HOME ?? '', 'profiles', 'web', 'node_modules', 'dsh-mnemon', 'lib', 'extension-sdk.js'))
|
|
620
|
+
for (const candidate of candidates) {
|
|
621
|
+
if (candidate === '' || !existsSync(candidate)) continue
|
|
622
|
+
return import(pathToFileURL(candidate).href)
|
|
623
|
+
}
|
|
624
|
+
return undefined
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* Install the sync Source into dsh-mnemon's composable-memory service.
|
|
629
|
+
*
|
|
630
|
+
* Returns a disposer, and does nothing at all when the Host runs without that
|
|
631
|
+
* service (a profile with dsh-mnemon absent).
|
|
632
|
+
*/
|
|
633
|
+
function registerSyncSource(ctx, dataRoot) {
|
|
634
|
+
if (typeof ctx.get !== 'function') return () => {}
|
|
635
|
+
const service = ctx.get('mnemonMemory')
|
|
636
|
+
if (service === undefined || service === null || typeof service.installContributions !== 'function') {
|
|
637
|
+
return () => {}
|
|
638
|
+
}
|
|
639
|
+
let disposed = false
|
|
640
|
+
let uninstall
|
|
641
|
+
void (async () => {
|
|
642
|
+
try {
|
|
643
|
+
const sdk = await loadExtensionSdk()
|
|
644
|
+
if (sdk === undefined || typeof sdk.defineMemorySource !== 'function') {
|
|
645
|
+
console.warn('mnemon-sync: dsh-mnemon extension SDK not found; the sync tab stays hidden')
|
|
646
|
+
return
|
|
647
|
+
}
|
|
648
|
+
const { createSyncSource, syncSourceManifest } = await import('./source.js')
|
|
649
|
+
if (disposed) return
|
|
650
|
+
// The manifest goes through the SDK validator so a contract drift in
|
|
651
|
+
// dsh-mnemon fails loudly here instead of at composition time.
|
|
652
|
+
const definition = sdk.defineMemorySource(createSyncSource({ ...syncSourceManifest }))
|
|
653
|
+
uninstall = service.installContributions({
|
|
654
|
+
plugin: {
|
|
655
|
+
apiVersion: 'dsh-mnemon/plugin/v1',
|
|
656
|
+
packageName: '@summerscar/dsh-mnemon-sync',
|
|
657
|
+
label: { en: 'Sync backup', 'zh-CN': '同步备份' },
|
|
658
|
+
description: {
|
|
659
|
+
en: 'Archive memory spaces to S3 or WebDAV.',
|
|
660
|
+
'zh-CN': '把记忆空间备份到 S3 或 WebDAV。',
|
|
661
|
+
},
|
|
662
|
+
roles: ['source'],
|
|
663
|
+
provides: [{ id: 'source' }],
|
|
664
|
+
},
|
|
665
|
+
sources: [definition],
|
|
666
|
+
}, { instanceId: 'mnemon-sync' })
|
|
667
|
+
if (disposed) {
|
|
668
|
+
uninstall?.()
|
|
669
|
+
uninstall = undefined
|
|
670
|
+
}
|
|
671
|
+
} catch (error) {
|
|
672
|
+
// A Host that cannot accept the Source must not take the backup down.
|
|
673
|
+
console.warn('mnemon-sync: could not register the sync source', error)
|
|
674
|
+
}
|
|
675
|
+
})()
|
|
676
|
+
return () => {
|
|
677
|
+
disposed = true
|
|
678
|
+
uninstall?.()
|
|
679
|
+
uninstall = undefined
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
/** Read a staging tree into the flat `{ path, bytes }` entries the ZIP writer takes. */
|
|
684
|
+
function collect(root, prefix = '') {
|
|
685
|
+
const entries = []
|
|
686
|
+
for (const entry of readdirSync(join(root, prefix), { withFileTypes: true })) {
|
|
687
|
+
const relative = prefix === '' ? entry.name : `${prefix}/${entry.name}`
|
|
688
|
+
if (entry.isDirectory()) {
|
|
689
|
+
entries.push(...collect(root, relative))
|
|
690
|
+
continue
|
|
691
|
+
}
|
|
692
|
+
if (!entry.isFile()) continue
|
|
693
|
+
const path = join(root, relative)
|
|
694
|
+
// Every staged file goes in, including an empty one. `USER.md` is legitimately
|
|
695
|
+
// 0 bytes when the user profile holds nothing, and the manifest records a
|
|
696
|
+
// checksum for it — skipping it here made every such archive fail its own
|
|
697
|
+
// integrity check.
|
|
698
|
+
entries.push({ path: relative, bytes: readFileSync(path) })
|
|
699
|
+
}
|
|
700
|
+
return entries
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
/**
|
|
704
|
+
* Delete the oldest archives beyond `keep`.
|
|
705
|
+
*
|
|
706
|
+
* Listing is deliberately *local*: archive keys are timestamped and the run
|
|
707
|
+
* history already records every key this plugin uploaded, so retention never
|
|
708
|
+
* needs a LIST call — which would mean a second signature scheme and would
|
|
709
|
+
* fail on buckets where listing is denied. A key that no longer exists counts
|
|
710
|
+
* as removed.
|
|
711
|
+
*/
|
|
712
|
+
async function applyRetention(history, target, keep, options) {
|
|
713
|
+
const uploaded = history.read()
|
|
714
|
+
.filter((run) => run.ok && typeof run.key === 'string')
|
|
715
|
+
.map((run) => run.key)
|
|
716
|
+
const unique = [...new Set(uploaded)].sort()
|
|
717
|
+
const overflow = unique.slice(0, Math.max(0, unique.length - keep))
|
|
718
|
+
const removed = []
|
|
719
|
+
for (const key of overflow) {
|
|
720
|
+
try {
|
|
721
|
+
await deleteArchive(target, key, options)
|
|
722
|
+
removed.push(key)
|
|
723
|
+
} catch {
|
|
724
|
+
// retention is best effort; the next run tries again
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
return removed
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
export { buildSnapshot, inspectHome, readSettings, writeSettings }
|
|
731
|
+
|