advisers 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh.md +6 -4
- package/README.md +6 -6
- package/THIRD-PARTY-NOTICES.md +1 -1
- package/bin/advisers.js +3148 -369
- package/index.js +1 -1
- package/lib/agent.js +529 -80
- package/lib/evidence.js +150 -17
- package/lib/fs-safe.js +284 -0
- package/lib/mcp-server.js +46 -21
- package/lib/model.js +73 -1
- package/lib/ocr.js +29 -9
- package/lib/pet-brief.js +24 -25
- package/lib/safety/classifier.js +55 -2
- package/lib/safety/rules.js +28 -0
- package/lib/sqlite.js +1614 -129
- package/lib/sync.js +398 -71
- package/lib/timestamp.js +29 -9
- package/lib/untrusted.js +186 -0
- package/package.json +4 -4
package/lib/sync.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* P2: file-based end-to-end encrypted snapshot sync.
|
|
2
|
+
* P2: file-based end-to-end encrypted snapshot sync (0.2.2), hardened in 0.2.3
|
|
3
|
+
* (robustness audit 2026-11-17 P1-1/P1-2/P1-3).
|
|
3
4
|
*
|
|
4
5
|
* The user chooses a directory that their existing sync tool (OneDrive, Dropbox,
|
|
5
6
|
* Syncthing, rsync, a NAS) already replicates. Advisers only writes encrypted
|
|
@@ -7,39 +8,48 @@
|
|
|
7
8
|
*
|
|
8
9
|
* Scope: whole-database snapshots, manual push/pull, last-snapshot-wins.
|
|
9
10
|
* Not provided: continuous merge, multi-writer conflict resolution, cloud accounts.
|
|
10
|
-
* Pull always
|
|
11
|
+
* Pull always keeps a verified backup before replacing the local database.
|
|
12
|
+
*
|
|
13
|
+
* 0.2.3 changes (all additive to the public exports):
|
|
14
|
+
* - Snapshot selection uses the snapshot's created time, not the file mtime,
|
|
15
|
+
* so a sync tool/clock that rewrites timestamps cannot make `pull --force`
|
|
16
|
+
* silently install an older snapshot. New snapshots carry an envelope-level
|
|
17
|
+
* `sync.createdAt` copy for cheap ordering; it is checked against the
|
|
18
|
+
* authenticated payload of the chosen snapshot. Legacy snapshots (no
|
|
19
|
+
* envelope metadata) are ranked by their authenticated payload createdAt.
|
|
20
|
+
* When the mtime order and createdAt order disagree by more than 5 minutes
|
|
21
|
+
* the chosen snapshot is still the content-newest one, with a warning; an
|
|
22
|
+
* unreadable/undecidable snapshot set requires an explicit `--file`.
|
|
23
|
+
* - `pullSnapshot` reports `localEmpty` / `snapshotOlder` / `status`, lets a
|
|
24
|
+
* brand-new device bootstrap (empty ledger) without `--force`, and warns
|
|
25
|
+
* (never silently) when `--force` replaces a local ledger with an older
|
|
26
|
+
* snapshot or when `--file` selects a non-newest snapshot.
|
|
27
|
+
* - Pull staging files are written durably through lib/fs-safe.js (short-write
|
|
28
|
+
* loop, fsync, rename retry) and the staging file plus its -wal/-shm siblings
|
|
29
|
+
* are always removed; `cleanupStalePullTemps()` (bin/doctor hook) sweeps
|
|
30
|
+
* leftovers from a crashed pull.
|
|
11
31
|
*/
|
|
12
|
-
import { chmodSync,
|
|
32
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, statSync } from 'node:fs'
|
|
13
33
|
import { createHash } from 'node:crypto'
|
|
14
34
|
import { basename, dirname, join } from 'node:path'
|
|
15
35
|
import { DatabaseSync } from 'node:sqlite'
|
|
16
36
|
import { decryptJson, encryptJson, isVaultEnvelope } from './vault.js'
|
|
37
|
+
import { atomicWriteFileSync, copyFileWithRetrySync, fsyncDirSync, renameWithRetrySync, rmWithRetrySync, writeAllToTempSync } from './fs-safe.js'
|
|
17
38
|
|
|
18
|
-
/**
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
39
|
+
/** Prefix of the staged (plaintext) database copy used while a pull verifies. */
|
|
40
|
+
export const PULL_TMP_PREFIX = '.advisers-pull-verify-'
|
|
41
|
+
/** Staged pull files older than this (and from a dead pid) are safe to remove. */
|
|
42
|
+
export const STALE_PULL_TMP_MS = 60 * 60 * 1000
|
|
43
|
+
/** mtime-vs-createdAt disagreement above this is reported/blocked as skew. */
|
|
44
|
+
export const SNAPSHOT_CLOCK_SKEW_MS = 5 * 60 * 1000
|
|
45
|
+
/** Envelope-level (unencrypted) copy of the snapshot metadata for cheap ordering. */
|
|
46
|
+
const ENVELOPE_META_VERSION = 1
|
|
47
|
+
const SYNC_ENVELOPE_META_KEY = 'sync'
|
|
25
48
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
mkdirSync(join(file, '..'), { recursive: true, mode: 0o700 })
|
|
29
|
-
const tmp = `${file}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 8)}`
|
|
30
|
-
try {
|
|
31
|
-
writeFileSync(tmp, data, { mode, flush: true })
|
|
32
|
-
renameSync(tmp, file)
|
|
33
|
-
} catch (error) {
|
|
34
|
-
try { if (existsSync(tmp)) rmSync(tmp) } catch {}
|
|
35
|
-
throw error
|
|
36
|
-
}
|
|
37
|
-
try { chmodSync(file, mode) } catch {}
|
|
38
|
-
fsyncDir(join(file, '..'))
|
|
49
|
+
function randomSuffix() {
|
|
50
|
+
return Math.random().toString(36).slice(2, 10)
|
|
39
51
|
}
|
|
40
52
|
|
|
41
|
-
const PULL_TMP_PREFIX = '.advisers-pull-verify-'
|
|
42
|
-
|
|
43
53
|
/**
|
|
44
54
|
* Verify that a database file is a structurally valid SQLite database before it
|
|
45
55
|
* is allowed to replace the main database (robustness audit P1-4, 2026-11-17).
|
|
@@ -67,12 +77,42 @@ function assertDatabaseIntegrity(file) {
|
|
|
67
77
|
}
|
|
68
78
|
}
|
|
69
79
|
|
|
80
|
+
/**
|
|
81
|
+
* Open a database read-only without leaving SQLite sidecars behind.
|
|
82
|
+
* A read-only open of a WAL-mode database may still make SQLite create an empty
|
|
83
|
+
* <db>-wal and a <db>-shm in a writable directory; those are inspection
|
|
84
|
+
* artifacts (not part of the ledger) and would otherwise be copied into pull
|
|
85
|
+
* backups or counted as plaintext copies. Only files that did not exist before
|
|
86
|
+
* the open are removed; a pre-existing pending WAL is never touched.
|
|
87
|
+
*/
|
|
88
|
+
function withReadOnlyDatabase(file, fn) {
|
|
89
|
+
const preexisting = new Set(['-wal', '-shm'].filter((suffix) => existsSync(file + suffix)))
|
|
90
|
+
let db = null
|
|
91
|
+
try {
|
|
92
|
+
db = new DatabaseSync(file, { readOnly: true })
|
|
93
|
+
return fn(db)
|
|
94
|
+
} finally {
|
|
95
|
+
try { db?.close() } catch {}
|
|
96
|
+
for (const suffix of ['-wal', '-shm']) {
|
|
97
|
+
if (preexisting.has(suffix)) continue
|
|
98
|
+
const artifact = file + suffix
|
|
99
|
+
try {
|
|
100
|
+
if (!existsSync(artifact)) continue
|
|
101
|
+
// A read-only connection cannot append committed frames; only remove an
|
|
102
|
+
// empty WAL (and the rebuildable shared-memory index).
|
|
103
|
+
if (suffix === '-wal' && statSync(artifact).size > 0) continue
|
|
104
|
+
rmWithRetrySync(artifact)
|
|
105
|
+
} catch {}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
70
110
|
/** Put the pre-pull database (and its WAL/SHM siblings) back after a failed replace. */
|
|
71
111
|
function restoreBackup(backup, dbPath) {
|
|
72
|
-
|
|
112
|
+
copyFileWithRetrySync(backup, dbPath)
|
|
73
113
|
for (const suffix of ['-wal', '-shm']) {
|
|
74
|
-
if (existsSync(backup + suffix))
|
|
75
|
-
else { try {
|
|
114
|
+
if (existsSync(backup + suffix)) copyFileWithRetrySync(backup + suffix, dbPath + suffix)
|
|
115
|
+
else { try { rmWithRetrySync(dbPath + suffix) } catch {} }
|
|
76
116
|
}
|
|
77
117
|
try { chmodSync(dbPath, 0o600) } catch {}
|
|
78
118
|
}
|
|
@@ -89,29 +129,276 @@ export function snapshotName(device, date = new Date()) {
|
|
|
89
129
|
return `${SYNC_PREFIX}${sanitizeDevice(device)}-${stamp}${SYNC_SUFFIX}`
|
|
90
130
|
}
|
|
91
131
|
|
|
132
|
+
/**
|
|
133
|
+
* A snapshot must carry every committed row. bin/ checkpoints the WAL before
|
|
134
|
+
* calling this, but the library API is also used directly: refuse a push while
|
|
135
|
+
* a non-empty -wal still holds possibly-committed data (it would silently
|
|
136
|
+
* produce a snapshot missing the newest writes).
|
|
137
|
+
*/
|
|
138
|
+
function assertSnapshotSourceComplete(dbPath) {
|
|
139
|
+
const walPath = `${dbPath}-wal`
|
|
140
|
+
try {
|
|
141
|
+
const st = statSync(walPath)
|
|
142
|
+
if (st.size > 0) throw new Error(`refusing to push a snapshot of ${dbPath}: the WAL sidecar still holds ${st.size} bytes of possibly-committed data; close other advisers processes and retry (a checkpoint is required first)`)
|
|
143
|
+
} catch (error) {
|
|
144
|
+
if (error?.code !== 'ENOENT') throw error
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
92
148
|
export function createSnapshot({ dbPath, passphrase, device = 'device', createdAt = new Date() }) {
|
|
93
149
|
if (!existsSync(dbPath)) throw new Error(`no database at ${dbPath}`)
|
|
150
|
+
assertSnapshotSourceComplete(dbPath)
|
|
94
151
|
const bytes = readFileSync(dbPath)
|
|
95
152
|
const sha256 = createHash('sha256').update(bytes).digest('hex')
|
|
96
153
|
const payload = { kind: 'advisers-sync-db', version: 1, device: sanitizeDevice(device), createdAt: createdAt.toISOString(), sha256, db: bytes.toString('base64') }
|
|
97
|
-
|
|
154
|
+
const envelope = encryptJson(payload, passphrase)
|
|
155
|
+
// 0.2.3 P1-1: an envelope-level copy of createdAt so `sync status` and the
|
|
156
|
+
// default snapshot choice do not depend on file mtimes. It is NOT trusted on
|
|
157
|
+
// its own: pullSnapshot re-decrypts the payload and refuses a mismatch.
|
|
158
|
+
envelope[SYNC_ENVELOPE_META_KEY] = { v: ENVELOPE_META_VERSION, createdAt: payload.createdAt, device: payload.device }
|
|
159
|
+
return { name: snapshotName(device, createdAt), payload, envelope }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function pushSnapshot({ dir, dbPath, passphrase, device = 'device', createdAt = new Date() }) {
|
|
163
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 })
|
|
164
|
+
const snap = createSnapshot({ dbPath, passphrase, device, createdAt })
|
|
165
|
+
const out = join(dir, snap.name)
|
|
166
|
+
const body = JSON.stringify(snap.envelope) + '\n'
|
|
167
|
+
atomicWriteFileSync(out, body, { mode: 0o600 })
|
|
168
|
+
return { file: out, sha256: snap.payload.sha256, bytes: Buffer.byteLength(body) }
|
|
98
169
|
}
|
|
99
170
|
|
|
171
|
+
/** Cheap listing for `sync status`: file metadata plus the envelope createdAt (if any). */
|
|
100
172
|
export function listSnapshots(dir) {
|
|
101
173
|
if (!existsSync(dir)) return []
|
|
102
174
|
return readdirSync(dir)
|
|
103
175
|
.filter((n) => n.startsWith(SYNC_PREFIX) && n.endsWith(SYNC_SUFFIX))
|
|
104
|
-
.map((name) => {
|
|
176
|
+
.map((name) => {
|
|
177
|
+
const file = join(dir, name)
|
|
178
|
+
const st = statSync(file)
|
|
179
|
+
const entry = { name, mtimeMs: st.mtimeMs, bytes: st.size }
|
|
180
|
+
try {
|
|
181
|
+
const envelope = JSON.parse(readFileSync(file, 'utf8'))
|
|
182
|
+
const meta = envelope?.[SYNC_ENVELOPE_META_KEY]
|
|
183
|
+
if (meta && typeof meta.createdAt === 'string' && Number.isFinite(Date.parse(meta.createdAt))) {
|
|
184
|
+
entry.createdAt = meta.createdAt
|
|
185
|
+
entry.createdAtSource = 'envelope'
|
|
186
|
+
}
|
|
187
|
+
} catch {}
|
|
188
|
+
return entry
|
|
189
|
+
})
|
|
105
190
|
.sort((a, b) => b.mtimeMs - a.mtimeMs)
|
|
106
191
|
}
|
|
107
192
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
193
|
+
function entryTimeMs(entry) {
|
|
194
|
+
if (Number.isFinite(entry.authenticatedCreatedAtMs)) return entry.authenticatedCreatedAtMs
|
|
195
|
+
if (Number.isFinite(entry.createdAtMs)) return entry.createdAtMs
|
|
196
|
+
return NaN
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Inspect every snapshot file: file metadata, envelope-level createdAt, and
|
|
201
|
+
* (with a passphrase) the authenticated payload createdAt/device. Options are
|
|
202
|
+
* extracted and the decrypted payload is dropped, so memory stays bounded.
|
|
203
|
+
*/
|
|
204
|
+
export function inspectSnapshots(dir, { passphrase } = {}) {
|
|
205
|
+
const result = { snapshots: [], warnings: [] }
|
|
206
|
+
if (!existsSync(dir)) return result
|
|
207
|
+
for (const name of readdirSync(dir).filter((n) => n.startsWith(SYNC_PREFIX) && n.endsWith(SYNC_SUFFIX)).sort()) {
|
|
208
|
+
const file = join(dir, name)
|
|
209
|
+
let st
|
|
210
|
+
try { st = statSync(file) } catch (error) { result.warnings.push(`cannot stat snapshot ${name}: ${String(error?.message ?? error).slice(0, 120)}`); continue }
|
|
211
|
+
const entry = {
|
|
212
|
+
name,
|
|
213
|
+
file,
|
|
214
|
+
mtimeMs: st.mtimeMs,
|
|
215
|
+
bytes: st.size,
|
|
216
|
+
createdAt: null,
|
|
217
|
+
createdAtMs: null,
|
|
218
|
+
createdAtSource: null,
|
|
219
|
+
authenticatedCreatedAt: null,
|
|
220
|
+
authenticatedCreatedAtMs: null,
|
|
221
|
+
device: null,
|
|
222
|
+
readable: false,
|
|
223
|
+
error: null,
|
|
224
|
+
}
|
|
225
|
+
let envelope = null
|
|
226
|
+
try { envelope = JSON.parse(readFileSync(file, 'utf8')) } catch (error) { entry.error = `not valid JSON (${String(error?.message ?? error).slice(0, 120)})` }
|
|
227
|
+
if (envelope) {
|
|
228
|
+
if (!isVaultEnvelope(envelope)) {
|
|
229
|
+
entry.error = 'not an Advisers encrypted envelope'
|
|
230
|
+
} else {
|
|
231
|
+
const meta = envelope?.[SYNC_ENVELOPE_META_KEY]
|
|
232
|
+
if (meta && typeof meta.createdAt === 'string' && Number.isFinite(Date.parse(meta.createdAt))) {
|
|
233
|
+
entry.createdAt = meta.createdAt
|
|
234
|
+
entry.createdAtMs = Date.parse(meta.createdAt)
|
|
235
|
+
entry.createdAtSource = 'envelope'
|
|
236
|
+
}
|
|
237
|
+
if (meta?.device) entry.device = sanitizeDevice(meta.device)
|
|
238
|
+
if (passphrase) {
|
|
239
|
+
try {
|
|
240
|
+
const payload = decryptJson(envelope, passphrase)
|
|
241
|
+
if (payload?.kind !== 'advisers-sync-db' || typeof payload.db !== 'string') throw new Error('not an Advisers database snapshot')
|
|
242
|
+
const payloadAt = Date.parse(payload.createdAt ?? '')
|
|
243
|
+
entry.readable = true
|
|
244
|
+
if (Number.isFinite(payloadAt)) {
|
|
245
|
+
entry.authenticatedCreatedAt = payload.createdAt
|
|
246
|
+
entry.authenticatedCreatedAtMs = payloadAt
|
|
247
|
+
}
|
|
248
|
+
entry.device = sanitizeDevice(payload.device ?? entry.device)
|
|
249
|
+
if (entry.createdAtMs != null && Number.isFinite(payloadAt) && Math.abs(entry.createdAtMs - payloadAt) > 1000) {
|
|
250
|
+
entry.error = `envelope metadata createdAt ${entry.createdAt} does not match the authenticated payload ${payload.createdAt} (the sync directory may have been tampered with)`
|
|
251
|
+
}
|
|
252
|
+
} catch (error) {
|
|
253
|
+
entry.readable = false
|
|
254
|
+
entry.error = String(error?.message ?? error).slice(0, 200)
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
result.snapshots.push(entry)
|
|
260
|
+
}
|
|
261
|
+
return result
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Choose which snapshot a pull should install.
|
|
266
|
+
*
|
|
267
|
+
* Rules (P1-1):
|
|
268
|
+
* - `file` is an explicit user choice; it must be readable.
|
|
269
|
+
* - Otherwise the snapshot with the newest authenticated createdAt wins. Ties
|
|
270
|
+
* fall back to file mtime.
|
|
271
|
+
* - If any candidate is unreadable, the newest cannot be proven: require
|
|
272
|
+
* `--file` instead of guessing.
|
|
273
|
+
* - If the newest file mtime belongs to a different snapshot than the newest
|
|
274
|
+
* createdAt and their times disagree by more than SNAPSHOT_CLOCK_SKEW_MS, the
|
|
275
|
+
* timestamps are not trustworthy: require `--file` (an ordinary skew with an
|
|
276
|
+
* agreed order only warns).
|
|
277
|
+
*/
|
|
278
|
+
export function selectSnapshot({ dir, passphrase, file, now = Date.now() } = {}) {
|
|
279
|
+
const { snapshots, warnings } = inspectSnapshots(dir, { passphrase })
|
|
280
|
+
if (!snapshots.length) throw new Error(`no Advisers snapshots in ${dir}`)
|
|
281
|
+
const explicit = Boolean(file)
|
|
282
|
+
let chosen
|
|
283
|
+
if (explicit) {
|
|
284
|
+
chosen = snapshots.find((entry) => entry.name === basename(file))
|
|
285
|
+
if (!chosen) throw new Error(`snapshot not found: ${file}`)
|
|
286
|
+
if (chosen.error) throw new Error(`snapshot ${chosen.name} cannot be used: ${chosen.error}`)
|
|
287
|
+
} else {
|
|
288
|
+
const unreadable = snapshots.filter((entry) => entry.error)
|
|
289
|
+
if (unreadable.length) {
|
|
290
|
+
const detail = unreadable.map((entry) => `${entry.name} (${entry.error})`).join('; ')
|
|
291
|
+
throw new Error(`cannot determine the newest snapshot: ${detail}; re-run with --file <snapshot> to choose one explicitly`)
|
|
292
|
+
}
|
|
293
|
+
const undetermined = snapshots.filter((entry) => !Number.isFinite(entryTimeMs(entry)))
|
|
294
|
+
if (undetermined.length) {
|
|
295
|
+
const names = undetermined.map((entry) => entry.name).join(', ')
|
|
296
|
+
throw new Error(`cannot determine the newest snapshot: ${names} has no readable createdAt; re-run with --file <snapshot>`)
|
|
297
|
+
}
|
|
298
|
+
const ordered = [...snapshots].sort((a, b) => entryTimeMs(b) - entryTimeMs(a) || b.mtimeMs - a.mtimeMs)
|
|
299
|
+
chosen = ordered[0]
|
|
300
|
+
const mtimeTop = [...snapshots].sort((a, b) => b.mtimeMs - a.mtimeMs)[0]
|
|
301
|
+
if (mtimeTop && mtimeTop.name !== chosen.name) {
|
|
302
|
+
const mtimeTopTime = entryTimeMs(mtimeTop)
|
|
303
|
+
if (mtimeTopTime > entryTimeMs(chosen) + SNAPSHOT_CLOCK_SKEW_MS) {
|
|
304
|
+
const minutes = SNAPSHOT_CLOCK_SKEW_MS / 60000
|
|
305
|
+
throw new Error(`cannot determine the newest snapshot: ${mtimeTop.name} has the newest file mtime but ${chosen.name} has the newest embedded createdAt (disagreement > ${minutes} min); file timestamps may be skewed by a sync tool or a clock change — re-run with --file <snapshot>`)
|
|
306
|
+
}
|
|
307
|
+
warnings.push(`snapshot file mtimes disagree with their embedded createdAt (${mtimeTop.name} is newest by mtime, ${chosen.name} by createdAt); using ${chosen.name} (a sync tool may have rewritten timestamps)`)
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
if (explicit && !chosen.error) {
|
|
311
|
+
const newest = [...snapshots].filter((entry) => !entry.error && Number.isFinite(entryTimeMs(entry))).sort((a, b) => entryTimeMs(b) - entryTimeMs(a))[0]
|
|
312
|
+
if (newest && newest.name !== chosen.name) {
|
|
313
|
+
warnings.push(`--file selected ${chosen.name}, but ${newest.name} has a newer createdAt (${newest.authenticatedCreatedAt ?? newest.createdAt}); --force does not change snapshot selection`)
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
return { chosen, snapshots, warnings, explicit }
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** Tables that mean "this local ledger has user data" (used for bootstrap). */
|
|
320
|
+
const LOCAL_CONTENT_TABLES = [
|
|
321
|
+
'matters', 'facts', 'commitments', 'preferences', 'links', 'updates', 'routines', 'routine_patches',
|
|
322
|
+
'turns', 'session_summaries', 'sessions', 'deliveries', 'task_runs', 'nudges', 'calendar_events',
|
|
323
|
+
'memory_candidates', 'research_runs', 'model_calls',
|
|
324
|
+
]
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* Is the local database empty (a brand-new device) or does it hold user data?
|
|
328
|
+
* A TOS-gate/init-created database has schema tables but no rows; the gate's
|
|
329
|
+
* audit rows (tos.*) and sync.* audit rows do not count as user data. Empty
|
|
330
|
+
* local ledgers are allowed to be replaced by a pull WITHOUT --force (P1-1):
|
|
331
|
+
* on a new device, the empty DB created by `init`/TOS is always newer by mtime
|
|
332
|
+
* than a snapshot taken elsewhere, which used to make documented bootstrap
|
|
333
|
+
* impossible. Errors (locked/corrupt DB) are treated as NOT empty (fail-safe).
|
|
334
|
+
*/
|
|
335
|
+
export function inspectLocalLedger(dbPath) {
|
|
336
|
+
const result = { exists: false, empty: true, counts: {}, userRows: 0, auditRows: 0, meaningfulAuditRows: 0, error: null }
|
|
337
|
+
if (!dbPath) { result.empty = false; result.error = 'no database path'; return result }
|
|
338
|
+
let st
|
|
339
|
+
try { st = statSync(dbPath) } catch (error) {
|
|
340
|
+
if (error?.code === 'ENOENT') return result
|
|
341
|
+
result.empty = false
|
|
342
|
+
result.error = String(error?.message ?? error).slice(0, 200)
|
|
343
|
+
return result
|
|
344
|
+
}
|
|
345
|
+
result.exists = true
|
|
346
|
+
if (!st.isFile() || st.size === 0) {
|
|
347
|
+
result.empty = false
|
|
348
|
+
result.error = 'the local database is not a regular non-empty file'
|
|
349
|
+
return result
|
|
350
|
+
}
|
|
351
|
+
try {
|
|
352
|
+
return withReadOnlyDatabase(dbPath, (db) => {
|
|
353
|
+
const tableNames = new Set(db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map((row) => row.name))
|
|
354
|
+
let userRows = 0
|
|
355
|
+
for (const table of LOCAL_CONTENT_TABLES) {
|
|
356
|
+
if (!tableNames.has(table)) continue
|
|
357
|
+
const n = Number(db.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get()?.n ?? 0)
|
|
358
|
+
result.counts[table] = n
|
|
359
|
+
userRows += n
|
|
360
|
+
}
|
|
361
|
+
result.userRows = userRows
|
|
362
|
+
if (tableNames.has('audit')) {
|
|
363
|
+
result.auditRows = Number(db.prepare('SELECT COUNT(*) AS n FROM audit').get()?.n ?? 0)
|
|
364
|
+
result.meaningfulAuditRows = Number(db.prepare("SELECT COUNT(*) AS n FROM audit WHERE action NOT LIKE 'tos.%' AND action NOT LIKE 'sync.%'").get()?.n ?? 0)
|
|
365
|
+
result.counts.audit = result.auditRows
|
|
366
|
+
}
|
|
367
|
+
result.empty = userRows === 0 && result.meaningfulAuditRows === 0
|
|
368
|
+
return result
|
|
369
|
+
})
|
|
370
|
+
} catch (error) {
|
|
371
|
+
// Fail safe: a locked/corrupt database is never treated as disposable.
|
|
372
|
+
result.empty = false
|
|
373
|
+
result.error = String(error?.message ?? error).slice(0, 200)
|
|
374
|
+
return result
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Remove staged pull files left behind by a crashed/killed pull (P1-3). Only
|
|
380
|
+
* files with the pull staging prefix are touched, and only when they are older
|
|
381
|
+
* than maxAgeMs AND their pid is not alive, so a concurrent pull is never
|
|
382
|
+
* disturbed. bin/doctor should call this on the HOME directory.
|
|
383
|
+
*/
|
|
384
|
+
export function cleanupStalePullTemps(dir, { maxAgeMs = STALE_PULL_TMP_MS, now = Date.now() } = {}) {
|
|
385
|
+
const result = { removed: [], kept: [], errors: [] }
|
|
386
|
+
let names
|
|
387
|
+
try { names = readdirSync(dir) } catch (error) { result.errors.push(String(error?.message ?? error).slice(0, 160)); return result }
|
|
388
|
+
for (const name of names) {
|
|
389
|
+
if (!name.startsWith(PULL_TMP_PREFIX)) continue
|
|
390
|
+
const file = join(dir, name)
|
|
391
|
+
const pid = Number.parseInt(name.slice(PULL_TMP_PREFIX.length).split('-')[0], 10)
|
|
392
|
+
let st
|
|
393
|
+
try { st = statSync(file) } catch (error) { if (error?.code !== 'ENOENT') result.errors.push(`${file}: ${String(error?.message ?? error).slice(0, 120)}`); continue }
|
|
394
|
+
if (now - st.mtimeMs < maxAgeMs) { result.kept.push(file); continue }
|
|
395
|
+
if (Number.isFinite(pid) && pid > 0) {
|
|
396
|
+
if (pid === process.pid) { result.kept.push(file); continue }
|
|
397
|
+
try { process.kill(pid, 0); result.kept.push(file); continue } catch (error) { if (error?.code !== 'ESRCH') { result.kept.push(file); continue } }
|
|
398
|
+
}
|
|
399
|
+
try { rmWithRetrySync(file); result.removed.push(file) } catch (error) { result.errors.push(`${file}: ${String(error?.message ?? error).slice(0, 120)}`) }
|
|
400
|
+
}
|
|
401
|
+
return result
|
|
115
402
|
}
|
|
116
403
|
|
|
117
404
|
function latestDbMtime(dbPath) {
|
|
@@ -122,53 +409,91 @@ function latestDbMtime(dbPath) {
|
|
|
122
409
|
return latest
|
|
123
410
|
}
|
|
124
411
|
|
|
412
|
+
function emitWarnings(warnings) {
|
|
413
|
+
for (const warning of warnings ?? []) {
|
|
414
|
+
try { console.warn(`advisers: warning: ${warning}`) } catch {}
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
125
418
|
/**
|
|
126
|
-
* Replace the local database with the
|
|
419
|
+
* Replace the local database with the selected snapshot.
|
|
127
420
|
*
|
|
128
|
-
* Safety
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
421
|
+
* Safety rules:
|
|
422
|
+
* - a local ledger with user data is only replaced when (a) the snapshot was
|
|
423
|
+
* pushed from this same HOME (intentional restore), (b) it is not older than
|
|
424
|
+
* the local write, or (c) the caller passed force:true (a verified backup is
|
|
425
|
+
* kept on every path);
|
|
426
|
+
* - a new device (no local DB, or an empty ledger) bootstraps without --force,
|
|
427
|
+
* still keeping a backup when a local file existed;
|
|
428
|
+
* - the snapshot is staged next to the database and passes quick_check before
|
|
429
|
+
* anything replaces the live library;
|
|
430
|
+
* - the result reports { changed, status, localEmpty, snapshotOlder, sameDevice,
|
|
431
|
+
* backup, warnings } so bin/ can explain what happened. `status` is one of
|
|
432
|
+
* unchanged | pulled | bootstrapped | forced.
|
|
433
|
+
* - --force never changes WHICH snapshot is selected (selection is always the
|
|
434
|
+
* newest readable one unless --file names one), and replacing the local
|
|
435
|
+
* ledger with an older snapshot is warned about loudly.
|
|
133
436
|
*/
|
|
134
|
-
export function pullSnapshot({ dir, dbPath, passphrase, file, force = false, localMtimeMs, localDevice }) {
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
const chosen = file ? snapshots.find((s) => s.name === basename(file)) : snapshots[0]
|
|
138
|
-
if (!chosen) throw new Error(`snapshot not found: ${file}`)
|
|
139
|
-
const envelope = JSON.parse(readFileSync(join(dir, chosen.name), 'utf8'))
|
|
437
|
+
export function pullSnapshot({ dir, dbPath, passphrase, file, force = false, localMtimeMs, localDevice, now = Date.now() }) {
|
|
438
|
+
const { chosen, warnings: selectionWarnings } = selectSnapshot({ dir, passphrase, file, now })
|
|
439
|
+
const envelope = JSON.parse(readFileSync(chosen.file, 'utf8'))
|
|
140
440
|
if (!isVaultEnvelope(envelope)) throw new Error('snapshot is not an Advisers encrypted envelope')
|
|
141
441
|
const payload = decryptJson(envelope, passphrase)
|
|
142
442
|
if (payload.kind !== 'advisers-sync-db' || typeof payload.db !== 'string') throw new Error('snapshot is not an Advisers database snapshot')
|
|
143
443
|
const bytes = Buffer.from(payload.db, 'base64')
|
|
144
444
|
const sha256 = createHash('sha256').update(bytes).digest('hex')
|
|
145
445
|
if (sha256 !== payload.sha256) throw new Error('snapshot checksum mismatch')
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
446
|
+
const snapshotAt = Date.parse(payload.createdAt ?? '')
|
|
447
|
+
const sameDevice = Boolean(localDevice) && Boolean(payload.device) && sanitizeDevice(payload.device) === sanitizeDevice(localDevice)
|
|
448
|
+
const local = inspectLocalLedger(dbPath)
|
|
449
|
+
const localEmpty = local.empty
|
|
450
|
+
const localAt = Number.isFinite(localMtimeMs) && localMtimeMs > 0 ? localMtimeMs : latestDbMtime(dbPath)
|
|
451
|
+
// Only meaningful for a non-empty local ledger: a new device's empty DB is
|
|
452
|
+
// always 'newer' by mtime but has no writes worth protecting.
|
|
453
|
+
const snapshotOlder = !localEmpty && Number.isFinite(snapshotAt) && localAt > 0 && localAt > snapshotAt
|
|
454
|
+
const warnings = selectionWarnings.slice()
|
|
455
|
+
const base = { file: chosen.name, sha256, device: payload.device, createdAt: payload.createdAt, localEmpty, snapshotOlder, sameDevice, explicit: Boolean(file), warnings, backup: null }
|
|
456
|
+
|
|
457
|
+
// Same bytes: no work to do. Kept force-sensitive for CLI compatibility
|
|
458
|
+
// (`sync pull --force` has always replaced the file even when identical).
|
|
459
|
+
if (local.exists && !force) {
|
|
460
|
+
let localBytes = null
|
|
461
|
+
try { localBytes = readFileSync(dbPath) } catch {}
|
|
462
|
+
if (localBytes && localBytes.equals(bytes)) {
|
|
463
|
+
return { ...base, changed: false, status: 'unchanged' }
|
|
154
464
|
}
|
|
155
465
|
}
|
|
466
|
+
|
|
467
|
+
if (local.exists && !localEmpty && !sameDevice && !force) {
|
|
468
|
+
if (!Number.isFinite(snapshotAt)) throw new Error(`cannot tell whether the local database is newer than ${chosen.name} (snapshot has no valid createdAt); re-run with --force to overwrite and keep a backup`)
|
|
469
|
+
if (snapshotOlder) throw new Error(`refusing to overwrite: the local database was written after snapshot ${chosen.name} (local ${new Date(localAt).toISOString()} > snapshot ${new Date(snapshotAt).toISOString()}). If those local writes matter, back them up/export them first; re-run with --force to overwrite anyway (a backup is kept).`)
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
let status = 'pulled'
|
|
473
|
+
if (localEmpty) status = 'bootstrapped'
|
|
474
|
+
else if (force) status = 'forced'
|
|
475
|
+
if (local.exists && !localEmpty && snapshotOlder) {
|
|
476
|
+
const reason = force ? '--force was given' : 'the snapshot was pushed from this device'
|
|
477
|
+
warnings.push(`replacing a newer local database (${new Date(localAt).toISOString()}) with older snapshot ${chosen.name} (${new Date(snapshotAt).toISOString()}) because ${reason}; a backup is kept`)
|
|
478
|
+
}
|
|
479
|
+
emitWarnings(warnings)
|
|
480
|
+
|
|
156
481
|
const parent = dirname(dbPath)
|
|
157
482
|
mkdirSync(parent, { recursive: true, mode: 0o700 })
|
|
158
|
-
//
|
|
159
|
-
//
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
const tmp = join(parent, `${PULL_TMP_PREFIX}${process.pid}-${Math.random().toString(36).slice(2, 8)}`)
|
|
483
|
+
// Sweep leftovers from crashed pulls before staging a fresh one (only stale
|
|
484
|
+
// files whose pid is gone are removed; a concurrent pull is never disturbed).
|
|
485
|
+
try { cleanupStalePullTemps(parent, { now }) } catch {}
|
|
486
|
+
const tmp = join(parent, `${PULL_TMP_PREFIX}${process.pid}-${randomSuffix()}`)
|
|
163
487
|
let backup = null
|
|
164
488
|
try {
|
|
165
|
-
|
|
489
|
+
// Durable staging write (fs-safe short-write loop + fsync + size check).
|
|
490
|
+
writeAllToTempSync(dbPath, bytes, { mode: 0o600, tempPath: tmp })
|
|
166
491
|
const hadLocal = existsSync(dbPath)
|
|
167
492
|
if (hadLocal) {
|
|
168
493
|
backup = `${dbPath}.before-sync-${Date.now()}`
|
|
169
|
-
|
|
494
|
+
copyFileWithRetrySync(dbPath, backup)
|
|
170
495
|
for (const suffix of ['-wal', '-shm']) {
|
|
171
|
-
try { if (existsSync(dbPath + suffix))
|
|
496
|
+
try { if (existsSync(dbPath + suffix)) copyFileWithRetrySync(dbPath + suffix, backup + suffix) } catch {}
|
|
172
497
|
}
|
|
173
498
|
}
|
|
174
499
|
try {
|
|
@@ -176,12 +501,12 @@ export function pullSnapshot({ dir, dbPath, passphrase, file, force = false, loc
|
|
|
176
501
|
} catch (error) {
|
|
177
502
|
throw new Error(`refusing to replace the local database: ${error.message}`)
|
|
178
503
|
}
|
|
179
|
-
for (const suffix of ['-wal', '-shm']) { try {
|
|
504
|
+
for (const suffix of ['-wal', '-shm']) { try { rmWithRetrySync(dbPath + suffix) } catch {} }
|
|
180
505
|
try {
|
|
181
506
|
// Rename the verified bytes into place (same directory: atomic).
|
|
182
|
-
|
|
507
|
+
renameWithRetrySync(tmp, dbPath)
|
|
183
508
|
try { chmodSync(dbPath, 0o600) } catch {}
|
|
184
|
-
|
|
509
|
+
fsyncDirSync(parent)
|
|
185
510
|
} catch (error) {
|
|
186
511
|
// A failure after the rename must never leave a half-installed library:
|
|
187
512
|
// restore the pre-pull database and keep the backup for inspection.
|
|
@@ -191,15 +516,17 @@ export function pullSnapshot({ dir, dbPath, passphrase, file, force = false, loc
|
|
|
191
516
|
restoreBackup(backup, dbPath)
|
|
192
517
|
restored = true
|
|
193
518
|
} else {
|
|
194
|
-
|
|
519
|
+
rmWithRetrySync(dbPath)
|
|
195
520
|
restored = true
|
|
196
521
|
}
|
|
197
522
|
} catch {}
|
|
198
523
|
const hint = restored ? 'the previous local database was restored' : 'restore the backup manually from ' + String(backup)
|
|
199
524
|
throw new Error(`failed to install the verified snapshot; ${hint}: ${error.message}`)
|
|
200
525
|
}
|
|
201
|
-
return {
|
|
526
|
+
return { ...base, changed: true, backup, status, replacedLocalEmpty: local.exists && localEmpty }
|
|
202
527
|
} finally {
|
|
203
|
-
|
|
528
|
+
for (const suffix of ['', '-wal', '-shm']) {
|
|
529
|
+
try { if (existsSync(tmp + suffix)) rmWithRetrySync(tmp + suffix) } catch {}
|
|
530
|
+
}
|
|
204
531
|
}
|
|
205
532
|
}
|
package/lib/timestamp.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* Without it the feature fails closed; no network is used by Advisers.
|
|
8
8
|
*/
|
|
9
9
|
import { spawn } from 'node:child_process'
|
|
10
|
-
import { statSync } from 'node:fs'
|
|
10
|
+
import { realpathSync, statSync } from 'node:fs'
|
|
11
11
|
import { isAbsolute, relative, resolve } from 'node:path'
|
|
12
12
|
|
|
13
13
|
/**
|
|
@@ -41,16 +41,36 @@ export function timestampConfigured() {
|
|
|
41
41
|
return Boolean(process.env.ADVISERS_TIMESTAMP_CMD)
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
const
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
44
|
+
function resolveContainedFile(file, cwd, label) {
|
|
45
|
+
if (typeof file !== 'string' || !file) throw new Error(label + ' path is required')
|
|
46
|
+
const rootLexical = resolve(cwd)
|
|
47
|
+
let root
|
|
48
|
+
try {
|
|
49
|
+
root = realpathSync(rootLexical)
|
|
50
|
+
} catch {
|
|
51
|
+
throw new Error(label + ' workspace does not exist: ' + cwd)
|
|
52
|
+
}
|
|
53
|
+
const abs = resolve(rootLexical, file)
|
|
54
|
+
const lexicalRel = relative(rootLexical, abs)
|
|
55
|
+
if (lexicalRel.startsWith('..') || isAbsolute(lexicalRel)) throw new Error(label + ' file escapes workspace: ' + file)
|
|
56
|
+
let real
|
|
57
|
+
try {
|
|
58
|
+
real = realpathSync(abs)
|
|
59
|
+
} catch (error) {
|
|
60
|
+
if (error && error.code === 'ENOENT') throw new Error(label + ' file not found: ' + file)
|
|
61
|
+
throw error
|
|
62
|
+
}
|
|
63
|
+
const realRel = relative(root, real)
|
|
64
|
+
if (realRel.startsWith('..') || isAbsolute(realRel)) throw new Error(label + ' file escapes workspace through a symlink: ' + file)
|
|
65
|
+
const info = statSync(real)
|
|
66
|
+
if (!info.isFile()) throw new Error('not a file: ' + file)
|
|
67
|
+
if (info.size > 10 * 1024 * 1024) throw new Error(label + ' file is larger than 10 MB')
|
|
68
|
+
return real
|
|
52
69
|
}
|
|
53
70
|
|
|
71
|
+
export function assertProofFile(file, cwd = process.cwd()) {
|
|
72
|
+
return resolveContainedFile(file, cwd, 'timestamp')
|
|
73
|
+
}
|
|
54
74
|
export async function runLocalTimestamp(file, { cmd = process.env.ADVISERS_TIMESTAMP_CMD, timeoutMs = 30_000, cwd = process.cwd() } = {}) {
|
|
55
75
|
if (!cmd) throw new Error('timestamp is not configured; set ADVISERS_TIMESTAMP_CMD to a local timestamp/signing executable')
|
|
56
76
|
const abs = assertProofFile(file, cwd)
|