advisers 0.2.0-preview.71 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/sqlite.js CHANGED
@@ -1,6 +1,71 @@
1
1
  import { DatabaseSync } from 'node:sqlite'
2
+ import { chmodSync, copyFileSync } from 'node:fs'
2
3
  import { createHash, createHmac, randomUUID } from 'node:crypto'
3
4
 
5
+ /**
6
+ * Open + write hardening (2026-11-17 robustness audit P0-1/P0-3/P1-2):
7
+ * - busy_timeout is set before any lock-taking pragma; journal_mode=WAL is
8
+ * idempotent and retried with bounded backoff while another process holds
9
+ * the database lock.
10
+ * - Opening no longer runs a full 'PRAGMA quick_check' (that is O(db) and made
11
+ * every command slow). Integrity checks live in assertIntegrity() and are
12
+ * run by 'advisers doctor' / 'advisers audit --check-db'.
13
+ * - appendAudit() and saveWithAudit() run in BEGIN IMMEDIATE transactions that
14
+ * re-read the previous hash before inserting, so the audit chain cannot fork
15
+ * under concurrent writers and a ledger write and its audit row commit
16
+ * together or not at all.
17
+ * - Migrations are refused for a schema newer than this CLI and are preceded
18
+ * by a same-directory backup of the old database.
19
+ */
20
+ export const SCHEMA_VERSION = 18
21
+ const DEFAULT_BUSY_TIMEOUT_MS = 5000
22
+ const OPEN_RETRIES = 6
23
+ const BUSY_ERRCODES = new Set([5, 6]) // SQLITE_BUSY, SQLITE_LOCKED
24
+
25
+ function isBusyError(error) {
26
+ if (!error) return false
27
+ if (typeof error.errcode === 'number' && BUSY_ERRCODES.has(error.errcode)) return true
28
+ return /SQLITE_BUSY|SQLITE_LOCKED|database (?:table )?is locked/i.test(String(error.message ?? error))
29
+ }
30
+
31
+ function sleepSync(ms) {
32
+ if (!ms || ms <= 0) return
33
+ try {
34
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
35
+ } catch {
36
+ const end = Date.now() + ms
37
+ while (Date.now() < end) {}
38
+ }
39
+ }
40
+
41
+ function backoffMs(attempt) {
42
+ return Math.min(2000, 50 * 2 ** attempt)
43
+ }
44
+
45
+ // Ledger tables are written as a whole snapshot by save(). The id column lets
46
+ // save() merge with rows other processes committed after our load().
47
+ const LEDGER_TABLES = [
48
+ ['matters', 'id'],
49
+ ['facts', 'id'],
50
+ ['commitments', 'id'],
51
+ ['preferences', 'id'],
52
+ ['links', 'id'],
53
+ ['updates', 'id'],
54
+ ['routines', 'id'],
55
+ ['routine_patches', 'id'],
56
+ ['routes', 'key'],
57
+ ]
58
+
59
+ function idsFor(snapshot) {
60
+ const ids = {}
61
+ for (const [table, key] of LEDGER_TABLES) {
62
+ const value = snapshot?.[table]
63
+ const rows = table === 'routes' ? Object.keys(value ?? {}) : (value ?? []).map((row) => row?.id)
64
+ ids[table] = new Set(rows.filter((id) => id != null))
65
+ }
66
+ return ids
67
+ }
68
+
4
69
  const SCHEMA = `
5
70
  CREATE TABLE IF NOT EXISTS matters (
6
71
  id TEXT PRIMARY KEY, title TEXT NOT NULL, status TEXT NOT NULL, domain TEXT NOT NULL,
@@ -17,7 +82,7 @@ CREATE TABLE IF NOT EXISTS facts (
17
82
  CREATE TABLE IF NOT EXISTS commitments (
18
83
  id TEXT PRIMARY KEY, matter_id TEXT NOT NULL, debtor TEXT NOT NULL, creditor TEXT NOT NULL,
19
84
  action TEXT NOT NULL, due_at TEXT, timezone TEXT NOT NULL, status TEXT NOT NULL,
20
- evidence TEXT NOT NULL, capability TEXT NOT NULL, done_at TEXT
85
+ evidence TEXT NOT NULL, capability TEXT NOT NULL, done_at TEXT, reminder_offsets TEXT
21
86
  );
22
87
  CREATE TABLE IF NOT EXISTS preferences (
23
88
  id TEXT PRIMARY KEY, kind TEXT NOT NULL, statement TEXT NOT NULL, scope TEXT NOT NULL,
@@ -121,6 +186,30 @@ CREATE TABLE IF NOT EXISTS calendar_events (
121
186
  CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
122
187
  `
123
188
 
189
+ /**
190
+ * P2-1: secondary indexes for the hot read/write paths (tick, today, history,
191
+ * deliveries). Indexes are layout only, not a schema change, so they are
192
+ * created with IF NOT EXISTS after migrate() on every open: new databases get
193
+ * them at creation time and existing databases get them the first time they
194
+ * are opened by this version. Each one was a full table SCAN + TEMP B-TREE in
195
+ * the 2026-11-17 performance audit (nudges/facts/deliveries/calendar_events/
196
+ * task_runs/sessions/model_calls/preferences/commitments).
197
+ */
198
+ const INDEXES = `
199
+ CREATE INDEX IF NOT EXISTS idx_nudges_commitment_at ON nudges(commitment_id, at);
200
+ CREATE INDEX IF NOT EXISTS idx_nudges_local_day_level ON nudges(local_day, level);
201
+ CREATE INDEX IF NOT EXISTS idx_nudges_at ON nudges(at);
202
+ CREATE INDEX IF NOT EXISTS idx_commitments_status_due ON commitments(status, due_at);
203
+ CREATE INDEX IF NOT EXISTS idx_facts_matter ON facts(matter_id);
204
+ CREATE INDEX IF NOT EXISTS idx_facts_status_deleted ON facts(status, deleted_at);
205
+ CREATE INDEX IF NOT EXISTS idx_preferences_status_expires ON preferences(status, expires_at);
206
+ CREATE INDEX IF NOT EXISTS idx_calendar_events_starts ON calendar_events(starts_at);
207
+ CREATE INDEX IF NOT EXISTS idx_deliveries_created ON deliveries(created_at);
208
+ CREATE INDEX IF NOT EXISTS idx_task_runs_created ON task_runs(created_at);
209
+ CREATE INDEX IF NOT EXISTS idx_sessions_updated ON sessions(updated_at);
210
+ CREATE INDEX IF NOT EXISTS idx_model_calls_created ON model_calls(created_at);
211
+ `
212
+
124
213
  const json = (v) => JSON.stringify(v ?? [])
125
214
  const parse = (v, fallback = []) => {
126
215
  if (v == null) return fallback
@@ -128,29 +217,140 @@ const parse = (v, fallback = []) => {
128
217
  }
129
218
 
130
219
  export class SqliteStore {
131
- constructor(path, { readOnly = false } = {}) {
220
+ constructor(path, { readOnly = false, checkIntegrity = false, busyTimeoutMs = Number(process.env.ADVISERS_SQLITE_BUSY_TIMEOUT_MS) || DEFAULT_BUSY_TIMEOUT_MS } = {}) {
221
+ this.path = path
222
+ this.busyTimeoutMs = busyTimeoutMs
223
+ this._transactionDepth = 0
224
+ this._baselineIds = null
132
225
  this.db = readOnly ? new DatabaseSync(path, { readOnly: true }) : new DatabaseSync(path)
226
+ // P0-1: sqlite only applies busy_timeout to statements that run after the
227
+ // pragma is set; concurrent first-opens used to fail on journal_mode=WAL
228
+ // because busy_timeout was set afterwards.
229
+ this.db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.floor(Number(busyTimeoutMs) || 0))}`)
133
230
  if (readOnly) {
134
- this.db.exec('PRAGMA busy_timeout = 5000')
135
- const integrity = this.db.prepare('PRAGMA quick_check').get()
136
- if (integrity && integrity.quick_check !== 'ok') throw new Error(`database integrity check failed: ${integrity.quick_check}`)
231
+ if (checkIntegrity) this.assertIntegrity()
137
232
  return
138
233
  }
139
- this.db.exec('PRAGMA journal_mode = WAL')
140
- this.db.exec('PRAGMA busy_timeout = 5000')
234
+ this.enableWal()
141
235
  this.db.exec('PRAGMA secure_delete = ON')
142
236
  this.db.exec('PRAGMA synchronous = NORMAL')
143
- const check = this.db.prepare('PRAGMA quick_check').get()
144
- if (check && check.quick_check !== 'ok') throw new Error(`database integrity check failed: ${check.quick_check}`)
145
- this.db.exec(SCHEMA)
146
- this.migrate()
237
+ this.initializeSchema()
238
+ }
239
+
240
+ /** Full-database integrity check; explicit because it is O(database size). */
241
+ assertIntegrity() {
242
+ const row = this.db.prepare('PRAGMA quick_check').get()
243
+ const verdict = row?.quick_check ?? 'unknown'
244
+ if (verdict !== 'ok') throw new Error(`database integrity check failed: ${verdict}`)
245
+ return verdict
246
+ }
247
+
248
+ /** WAL is persistent; skip the switch when already enabled and retry on busy. */
249
+ enableWal() {
250
+ if (!this.path || this.path === ':memory:') return 'memory'
251
+ const current = String(this.db.prepare('PRAGMA journal_mode').get()?.journal_mode ?? '').toLowerCase()
252
+ if (current === 'wal' || current === 'memory') return current
253
+ let lastError
254
+ for (let attempt = 0; attempt <= OPEN_RETRIES; attempt++) {
255
+ try {
256
+ const result = this.db.prepare('PRAGMA journal_mode = WAL').get()
257
+ const mode = String(result?.journal_mode ?? '').toLowerCase()
258
+ if (mode === 'wal' || mode === 'memory') return mode
259
+ lastError = new Error(`could not enable WAL journal mode (got '${mode || 'unknown'}')`)
260
+ } catch (error) {
261
+ lastError = error
262
+ if (!isBusyError(error)) throw error
263
+ }
264
+ sleepSync(backoffMs(attempt))
265
+ }
266
+ throw lastError
267
+ }
268
+
269
+ initializeSchema() {
270
+ this._transaction('IMMEDIATE', () => {
271
+ this.db.exec(SCHEMA)
272
+ this.migrate()
273
+ // Indexes come after migrate(): some index the columns migrate() adds
274
+ // (e.g. facts.deleted_at), so an old database must be upgraded first.
275
+ this.db.exec(INDEXES)
276
+ })
277
+ }
278
+
279
+ /**
280
+ * Run fn() in a write transaction with bounded SQLITE_BUSY retries.
281
+ * Nested calls join the outer transaction (no nested BEGIN).
282
+ */
283
+ _transaction(mode, fn, { retries = OPEN_RETRIES } = {}) {
284
+ if (this._transactionDepth > 0) return fn()
285
+ let lastError
286
+ for (let attempt = 0; attempt <= retries; attempt++) {
287
+ let began = false
288
+ try {
289
+ this.db.exec(`BEGIN ${mode}`)
290
+ began = true
291
+ this._transactionDepth++
292
+ let result
293
+ try {
294
+ result = fn()
295
+ } finally {
296
+ this._transactionDepth--
297
+ }
298
+ this.db.exec('COMMIT')
299
+ return result
300
+ } catch (error) {
301
+ if (began) { try { this.db.exec('ROLLBACK') } catch {} }
302
+ lastError = error
303
+ if (!isBusyError(error) || attempt === retries) throw error
304
+ }
305
+ sleepSync(backoffMs(attempt))
306
+ }
307
+ throw lastError
308
+ }
309
+
310
+ /** Public transaction helper for multi-statement invariants (returns fn result). */
311
+ transaction(fn, { mode = 'IMMEDIATE' } = {}) {
312
+ return this._transaction(mode, fn)
313
+ }
314
+
315
+ /**
316
+ * Checkpoint the WAL so a copied/sealed database file is self-contained.
317
+ * Returns busy=true when another connection prevented a full checkpoint.
318
+ */
319
+ checkpoint({ mode = 'TRUNCATE', retries = OPEN_RETRIES } = {}) {
320
+ let last = null
321
+ for (let attempt = 0; attempt <= retries; attempt++) {
322
+ last = this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get()
323
+ if (!last || Number(last.busy) === 0) {
324
+ return { busy: false, log: Number(last?.log ?? 0), checkpointed: Number(last?.checkpointed ?? 0) }
325
+ }
326
+ sleepSync(backoffMs(attempt))
327
+ }
328
+ return { busy: Number(last?.busy ?? 1) !== 0, log: Number(last?.log ?? 0), checkpointed: Number(last?.checkpointed ?? 0) }
147
329
  }
148
330
 
331
+ backupBeforeMigrate(current) {
332
+ if (!this.path || this.path === ':memory:') return
333
+ const target = `${this.path}.before-migrate-v${current}-${Date.now()}`
334
+ try {
335
+ copyFileSync(this.path, target)
336
+ try { chmodSync(target, 0o600) } catch {}
337
+ } catch (error) {
338
+ throw new Error(`refusing to migrate without a backup of ${this.path}: ${error.message}`)
339
+ }
340
+ }
341
+
342
+ /**
343
+ * Bring an older database up to SCHEMA_VERSION. Runs inside the caller's
344
+ * write transaction (initializeSchema). Refuses futures schemas and keeps
345
+ * a same-directory backup before altering an existing database.
346
+ */
149
347
  migrate() {
150
348
  const row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get('schema_version')
151
349
  const current = row ? Number(row.value) : 0
152
- if (current >= 17) return
153
- this.db.exec('BEGIN')
350
+ if (!Number.isFinite(current) || current < 0) throw new Error(`invalid schema_version in ${this.path}: ${row?.value}`)
351
+ if (current > SCHEMA_VERSION) throw new Error(`database schema v${current} is newer than this CLI (v${SCHEMA_VERSION}); upgrade advisers before opening ${this.path}`)
352
+ if (current >= SCHEMA_VERSION) return
353
+ if (current > 0) this.backupBeforeMigrate(current)
154
354
  try {
155
355
  this.db.exec(`CREATE TABLE IF NOT EXISTS seen_events (
156
356
  link_id TEXT NOT NULL, account_id TEXT NOT NULL, dedupe_key TEXT NOT NULL,
@@ -196,6 +396,8 @@ export class SqliteStore {
196
396
  try { this.db.exec("INSERT INTO turns_fts(turns_fts) VALUES('rebuild')") } catch {}
197
397
  const commitmentCols = this.db.prepare('PRAGMA table_info(commitments)').all().map((c) => c.name)
198
398
  if (!commitmentCols.includes('done_at')) this.db.exec('ALTER TABLE commitments ADD COLUMN done_at TEXT')
399
+ // 0.2.0: per-commitment custom reminder offsets (JSON array of strings).
400
+ if (!commitmentCols.includes('reminder_offsets')) this.db.exec('ALTER TABLE commitments ADD COLUMN reminder_offsets TEXT')
199
401
  const factCols = this.db.prepare('PRAGMA table_info(facts)').all().map((c) => c.name)
200
402
  for (const [name, type] of [['valid_from','TEXT'],['valid_to','TEXT'],['expired_at','TEXT'],['evidence','TEXT'],['supersedes','TEXT'],['superseded_by','TEXT'],['shadowed_by','TEXT'],['last_seen_at','TEXT'],['deleted_at','TEXT'],['deleted_reason','TEXT']]) {
201
403
  if (!factCols.includes(name)) this.db.exec(`ALTER TABLE facts ADD COLUMN ${name} ${type}`)
@@ -237,10 +439,8 @@ CREATE TABLE IF NOT EXISTS task_runs (
237
439
  all_day INTEGER NOT NULL DEFAULT 0, tzid TEXT, location TEXT, rrule TEXT,
238
440
  source TEXT, imported_at TEXT NOT NULL
239
441
  )`)
240
- this.db.prepare('INSERT OR REPLACE INTO meta(key,value) VALUES(?,?)').run('schema_version', '17')
241
- this.db.exec('COMMIT')
442
+ this.db.prepare('INSERT OR REPLACE INTO meta(key,value) VALUES(?,?)').run('schema_version', String(SCHEMA_VERSION))
242
443
  } catch (error) {
243
- this.db.exec('ROLLBACK')
244
444
  throw error
245
445
  }
246
446
  }
@@ -323,6 +523,76 @@ CREATE TABLE IF NOT EXISTS task_runs (
323
523
  return this.db.prepare('SELECT COUNT(*) AS n FROM nudges WHERE local_day = ? AND level = ?').get(localDay, level).n
324
524
  }
325
525
 
526
+ /**
527
+ * P0-2 (tick N+1 removal): insert every nudge for one tick in a single
528
+ * transaction with one prepared statement. INSERT OR IGNORE keeps the
529
+ * per-(local_day, level, commitment_id) dedupe key semantics of addNudge()
530
+ * without aborting the batch on an expected duplicate. Returns the number of
531
+ * rows actually inserted (existing rows are left untouched).
532
+ */
533
+ addNudges(rows) {
534
+ if (!rows?.length) return 0
535
+ return this._transaction('IMMEDIATE', () => {
536
+ const statement = this.db.prepare('INSERT OR IGNORE INTO nudges(id,at,local_day,level,commitment_id,dedupe_key,body) VALUES(?,?,?,?,?,?,?)')
537
+ const now = new Date().toISOString()
538
+ let inserted = 0
539
+ for (const row of rows) {
540
+ const key = `${row.localDay}:${row.level}:${row.commitmentId}`
541
+ const id = `nudge_${randomUUID().replace(/-/g, '').slice(0, 16)}`
542
+ inserted += Number(statement.run(id, row.at ?? now, row.localDay, row.level, row.commitmentId, key, row.body).changes)
543
+ }
544
+ return inserted
545
+ })
546
+ }
547
+
548
+ /**
549
+ * P0-2 (tick N+1 removal): the newest `limit` outcomes per commitment in one
550
+ * query (window function over idx_nudges_commitment_at). Replaces one
551
+ * listCommitmentNudges() round-trip per commitment.
552
+ */
553
+ recentNudgeOutcomes(limit = 3) {
554
+ const rows = this.db.prepare(`SELECT commitment_id, outcome, at FROM (
555
+ SELECT commitment_id, outcome, at, ROW_NUMBER() OVER (PARTITION BY commitment_id ORDER BY at DESC) AS rn
556
+ FROM nudges
557
+ ) WHERE rn <= ?`).all(limit)
558
+ const byCommitment = new Map()
559
+ for (const row of rows) {
560
+ const list = byCommitment.get(row.commitment_id)
561
+ if (list) list.push(row.outcome)
562
+ else byCommitment.set(row.commitment_id, [row.outcome])
563
+ }
564
+ return byCommitment
565
+ }
566
+
567
+ /** P0-2 (tick N+1 removal): today's nudge count per level in one query. */
568
+ countNudgesByDay(localDay) {
569
+ const rows = this.db.prepare('SELECT level, COUNT(*) AS n FROM nudges WHERE local_day = ? GROUP BY level').all(localDay)
570
+ return new Map(rows.map((row) => [row.level, Number(row.n)]))
571
+ }
572
+
573
+ /** P0-2 (tick N+1 removal): existing (commitment, level) nudge keys for a day. */
574
+ listNudgeLevelKeysByDay(localDay) {
575
+ const rows = this.db.prepare('SELECT commitment_id, level FROM nudges WHERE local_day = ?').all(localDay)
576
+ return new Set(rows.map((row) => `${row.commitment_id}\u0000${row.level}`))
577
+ }
578
+
579
+ /** P0-2 (tick N+1 removal): all meta rows with a prefix in one query (snooze:*). */
580
+ listMetaByPrefix(prefix) {
581
+ const rows = this.db.prepare('SELECT key, value FROM meta WHERE key LIKE ?').all(`${prefix}%`)
582
+ return new Map(rows.map((row) => [row.key, row.value]))
583
+ }
584
+
585
+ /** P0-2: delete a batch of meta keys in one transaction. */
586
+ deleteMetaKeys(keys) {
587
+ if (!keys?.length) return 0
588
+ return this._transaction('IMMEDIATE', () => {
589
+ const statement = this.db.prepare('DELETE FROM meta WHERE key = ?')
590
+ let deleted = 0
591
+ for (const key of keys) deleted += Number(statement.run(key).changes)
592
+ return deleted
593
+ })
594
+ }
595
+
326
596
  listNudges(limit = 20) {
327
597
  return this.db.prepare('SELECT * FROM nudges ORDER BY at DESC LIMIT ?').all(limit)
328
598
  }
@@ -528,7 +798,16 @@ CREATE TABLE IF NOT EXISTS task_runs (
528
798
  return this.db.prepare('SELECT COUNT(*) AS n FROM deliveries').get().n
529
799
  }
530
800
 
801
+ /**
802
+ * Append an audit row atomically. BEGIN IMMEDIATE serializes writers and the
803
+ * previous hash is re-read inside the transaction, so two concurrent writers
804
+ * can never both chain onto the same predecessor (audit fork, P0-3).
805
+ */
531
806
  appendAudit({ actor, action, allowed, reason }) {
807
+ return this._transaction('IMMEDIATE', () => this._appendAuditRow({ actor, action, allowed, reason }))
808
+ }
809
+
810
+ _appendAuditRow({ actor, action, allowed, reason }) {
532
811
  const last = this.db.prepare('SELECT hash FROM audit ORDER BY seq DESC LIMIT 1').get()
533
812
  const prev = last?.hash ?? ''
534
813
  const at = new Date().toISOString()
@@ -591,7 +870,7 @@ CREATE TABLE IF NOT EXISTS task_runs (
591
870
 
592
871
  load() {
593
872
  const all = (sql) => this.db.prepare(sql).all()
594
- return {
873
+ const snapshot = {
595
874
  version: 1,
596
875
  matters: all('SELECT * FROM matters').map((r) => ({
597
876
  id: r.id, title: r.title, status: r.status, domain: r.domain, secrecy: r.secrecy,
@@ -609,11 +888,19 @@ CREATE TABLE IF NOT EXISTS task_runs (
609
888
  shadowedBy: r.shadowed_by ?? undefined, lastSeenAt: r.last_seen_at ?? undefined,
610
889
  deletedAt: r.deleted_at ?? undefined, deletedReason: r.deleted_reason ?? undefined,
611
890
  })),
612
- commitments: all('SELECT * FROM commitments').map((r) => ({
613
- id: r.id, matterId: r.matter_id, debtor: r.debtor, creditor: r.creditor, action: r.action,
614
- dueAt: r.due_at ?? undefined, timezone: r.timezone, status: r.status,
615
- evidence: parse(r.evidence), capability: r.capability, doneAt: r.done_at ?? undefined,
616
- })),
891
+ commitments: all('SELECT * FROM commitments').map((r) => {
892
+ // NULL means "never customised": keep the field absent so old snapshots
893
+ // and the pre-0.2.0 commitment shape stay identical. Corrupt or
894
+ // non-array JSON degrades to absent rather than breaking the ledger.
895
+ const parsedOffsets = r.reminder_offsets == null ? null : parse(r.reminder_offsets, null)
896
+ const reminderOffsets = Array.isArray(parsedOffsets) ? parsedOffsets : undefined
897
+ return {
898
+ id: r.id, matterId: r.matter_id, debtor: r.debtor, creditor: r.creditor, action: r.action,
899
+ dueAt: r.due_at ?? undefined, timezone: r.timezone, status: r.status,
900
+ evidence: parse(r.evidence), capability: r.capability, doneAt: r.done_at ?? undefined,
901
+ ...(reminderOffsets === undefined ? {} : { reminderOffsets }),
902
+ }
903
+ }),
617
904
  preferences: all('SELECT * FROM preferences').map((r) => ({
618
905
  id: r.id, kind: r.kind, statement: r.statement, scope: parse(r.scope, {}),
619
906
  source: parse(r.source, {}), evidence: parse(r.evidence), confidence: r.confidence,
@@ -641,53 +928,90 @@ CREATE TABLE IF NOT EXISTS task_runs (
641
928
  })),
642
929
  routes: Object.fromEntries(all('SELECT * FROM routes').map((r) => [r.key, r.matter_id])),
643
930
  }
931
+ // Remember which rows this connection has seen (P1-1): save() only deletes
932
+ // rows this loader owned, so another process's concurrent additions are
933
+ // merged instead of being silently overwritten by a stale snapshot.
934
+ this._baselineIds = idsFor(snapshot)
935
+ return snapshot
644
936
  }
645
937
 
646
938
  save(snapshot) {
939
+ return this._transaction('IMMEDIATE', () => this._saveRows(snapshot))
940
+ }
941
+
942
+ /**
943
+ * Write the ledger snapshot and its audit row in one transaction, so the
944
+ * audit trail can never claim a write that rolled back (or miss one that
945
+ * committed). `inside` runs inside the same transaction before the audit row.
946
+ */
947
+ saveWithAudit(snapshot, audit, inside) {
948
+ return this._transaction('IMMEDIATE', () => {
949
+ this._saveRows(snapshot)
950
+ if (typeof inside === 'function') inside(this)
951
+ return this._appendAuditRow(audit)
952
+ })
953
+ }
954
+
955
+ _saveRows(snapshot) {
647
956
  const db = this.db
648
- db.exec('BEGIN')
957
+ const newIds = idsFor(snapshot)
958
+ const baseline = this._baselineIds
649
959
  try {
650
- for (const table of ['matters', 'facts', 'commitments', 'preferences', 'links', 'updates', 'routines', 'routine_patches', 'routes']) {
651
- db.exec(`DELETE FROM ${table}`)
960
+ if (baseline) {
961
+ // P1-1: delete only rows this connection loaded that the new snapshot
962
+ // dropped; rows another process added after our load() are kept.
963
+ for (const [table, key] of LEDGER_TABLES) {
964
+ const removed = [...(baseline[table] ?? [])].filter((id) => !newIds[table].has(id))
965
+ if (!removed.length) continue
966
+ const statement = db.prepare(`DELETE FROM ${table} WHERE ${key} = ?`)
967
+ for (const id of removed) statement.run(id)
968
+ }
969
+ } else {
970
+ // No prior load(): keep the historical full-replace semantics for
971
+ // direct library use (e.g. importing or seeding a ledger).
972
+ for (const [table] of LEDGER_TABLES) db.exec(`DELETE FROM ${table}`)
652
973
  }
653
- const m = db.prepare('INSERT INTO matters VALUES (?,?,?,?,?,?,?,?,?,?,?)')
974
+ const m = db.prepare('INSERT OR REPLACE INTO matters VALUES (?,?,?,?,?,?,?,?,?,?,?)')
654
975
  for (const x of snapshot.matters ?? []) {
655
976
  m.run(x.id, x.title, x.status, x.domain, x.secrecy, x.summary ?? null, json(x.participants), json(x.deadlines), json(x.openQuestions), x.createdAt, x.updatedAt)
656
977
  }
657
- const f = db.prepare(`INSERT INTO facts(id,matter_id,subject,predicate,object,recorded_at,expired_at,authority,confidence,source,evidence,status,valid_from,valid_to,supersedes,superseded_by,shadowed_by,last_seen_at,deleted_at,deleted_reason)
978
+ const f = db.prepare(`INSERT OR REPLACE INTO facts(id,matter_id,subject,predicate,object,recorded_at,expired_at,authority,confidence,source,evidence,status,valid_from,valid_to,supersedes,superseded_by,shadowed_by,last_seen_at,deleted_at,deleted_reason)
658
979
  VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
659
980
  for (const x of snapshot.facts ?? []) {
660
981
  f.run(x.id, x.matterId ?? null, x.subject, x.predicate, x.object, x.recordedAt, x.expiredAt ?? null, x.authority, x.confidence, x.source ? json(x.source) : null, x.evidence ? json(x.evidence) : null, x.status, x.validFrom ?? null, x.validTo ?? null, x.supersedes ?? null, x.supersededBy ?? null, x.shadowedBy ?? null, x.lastSeenAt ?? null, x.deletedAt ?? null, x.deletedReason ?? null)
661
982
  }
662
- const c = db.prepare('INSERT INTO commitments VALUES (?,?,?,?,?,?,?,?,?,?,?)')
983
+ const c = db.prepare(`INSERT OR REPLACE INTO commitments(id,matter_id,debtor,creditor,action,due_at,timezone,status,evidence,capability,done_at,reminder_offsets)
984
+ VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`)
663
985
  for (const x of snapshot.commitments ?? []) {
664
- c.run(x.id, x.matterId, x.debtor, x.creditor, x.action, x.dueAt ?? null, x.timezone, x.status, json(x.evidence), x.capability, x.doneAt ?? null)
986
+ // A missing/undefined list stays NULL so a round-trip keeps the pre-0.2.0
987
+ // "no field" shape; an explicit [] is preserved as its own value.
988
+ const reminderOffsets = Array.isArray(x.reminderOffsets) ? JSON.stringify(x.reminderOffsets) : null
989
+ c.run(x.id, x.matterId, x.debtor, x.creditor, x.action, x.dueAt ?? null, x.timezone, x.status, json(x.evidence), x.capability, x.doneAt ?? null, reminderOffsets)
665
990
  }
666
- const p = db.prepare('INSERT INTO preferences VALUES (?,?,?,?,?,?,?,?,?,?,?,?)')
991
+ const p = db.prepare('INSERT OR REPLACE INTO preferences VALUES (?,?,?,?,?,?,?,?,?,?,?,?)')
667
992
  for (const x of snapshot.preferences ?? []) {
668
993
  p.run(x.id, x.kind, x.statement, json(x.scope), json(x.source), json(x.evidence), x.confidence, x.status, x.validFrom, x.lastConfirmed ?? null, x.expiresAt ?? null, x.priority)
669
994
  }
670
- const l = db.prepare('INSERT INTO links VALUES (?,?,?,?,?,?,?,?,?)')
995
+ const l = db.prepare('INSERT OR REPLACE INTO links VALUES (?,?,?,?,?,?,?,?,?)')
671
996
  for (const x of snapshot.links ?? []) {
672
997
  l.run(x.id, x.fromMatter, x.toMatter, x.type, x.confidence, json(x.evidence), x.status, x.createdBy, x.createdAt)
673
998
  }
674
- const u = db.prepare('INSERT INTO updates VALUES (?,?,?,?,?,?,?)')
999
+ const u = db.prepare('INSERT OR REPLACE INTO updates VALUES (?,?,?,?,?,?,?)')
675
1000
  for (const x of snapshot.updates ?? []) {
676
1001
  u.run(x.id, x.sourceMatter, x.targetMatter, x.change, x.status, x.createdAt, x.rollback ?? null)
677
1002
  }
678
- const r = db.prepare('INSERT INTO routines VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)')
1003
+ const r = db.prepare('INSERT OR REPLACE INTO routines VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)')
679
1004
  for (const x of snapshot.routines ?? []) {
680
1005
  r.run(x.id, x.name, x.description, json(x.trigger), json(x.steps), json(x.preferences), json(x.pitfalls), json(x.verification), x.scope ?? null, x.version, x.status, x.createdBy, x.createdAt, x.updatedAt)
681
1006
  }
682
- const rp = db.prepare('INSERT INTO routine_patches VALUES (?,?,?,?,?,?,?,?)')
1007
+ const rp = db.prepare('INSERT OR REPLACE INTO routine_patches VALUES (?,?,?,?,?,?,?,?)')
683
1008
  for (const x of snapshot.routinePatches ?? []) {
684
1009
  rp.run(x.id, x.routineId, x.baseVersion, json(x.changes), json(x.before), x.diff, x.status, x.createdAt)
685
1010
  }
686
- const rt = db.prepare('INSERT INTO routes VALUES (?,?)')
1011
+ const rt = db.prepare('INSERT OR REPLACE INTO routes VALUES (?,?)')
687
1012
  for (const [key, matterId] of Object.entries(snapshot.routes ?? {})) rt.run(key, matterId)
688
- db.exec('COMMIT')
1013
+ this._baselineIds = newIds
689
1014
  } catch (error) {
690
- db.exec('ROLLBACK')
691
1015
  throw error
692
1016
  }
693
1017
  }
package/lib/sync.js CHANGED
@@ -9,11 +9,34 @@
9
9
  * Not provided: continuous merge, multi-writer conflict resolution, cloud accounts.
10
10
  * Pull always backs up the local DB before replacing it.
11
11
  */
12
- import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
12
+ import { chmodSync, closeSync, copyFileSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
13
13
  import { createHash } from 'node:crypto'
14
14
  import { basename, join } from 'node:path'
15
15
  import { decryptJson, encryptJson, isVaultEnvelope } from './vault.js'
16
16
 
17
+ /** fsync a directory entry so a rename survives a crash. */
18
+ function fsyncDir(dir) {
19
+ try {
20
+ const fd = openSync(dir, 'r')
21
+ try { fsyncSync(fd) } finally { closeSync(fd) }
22
+ } catch {}
23
+ }
24
+
25
+ /** Write a same-directory temp file, flush it, then rename it into place. */
26
+ function writeAtomic(file, data, { mode = 0o600 } = {}) {
27
+ mkdirSync(join(file, '..'), { recursive: true, mode: 0o700 })
28
+ const tmp = `${file}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 8)}`
29
+ try {
30
+ writeFileSync(tmp, data, { mode, flush: true })
31
+ renameSync(tmp, file)
32
+ } catch (error) {
33
+ try { if (existsSync(tmp)) rmSync(tmp) } catch {}
34
+ throw error
35
+ }
36
+ try { chmodSync(file, mode) } catch {}
37
+ fsyncDir(join(file, '..'))
38
+ }
39
+
17
40
  export const SYNC_PREFIX = 'advisers-'
18
41
  export const SYNC_SUFFIX = '.enc.json'
19
42
 
@@ -46,14 +69,29 @@ export function pushSnapshot({ dir, dbPath, passphrase, device = 'device', creat
46
69
  mkdirSync(dir, { recursive: true, mode: 0o700 })
47
70
  const snap = createSnapshot({ dbPath, passphrase, device, createdAt })
48
71
  const out = join(dir, snap.name)
49
- const tmp = `${out}.tmp-${process.pid}`
50
- writeFileSync(tmp, JSON.stringify(snap.envelope) + '\n', { mode: 0o600 })
51
- renameSync(tmp, out)
52
- try { chmodSync(out, 0o600) } catch {}
53
- return { file: out, sha256: snap.payload.sha256, bytes: Buffer.byteLength(JSON.stringify(snap.envelope)) }
72
+ const body = JSON.stringify(snap.envelope) + '\n'
73
+ writeAtomic(out, body, { mode: 0o600 })
74
+ return { file: out, sha256: snap.payload.sha256, bytes: Buffer.byteLength(body) }
54
75
  }
55
76
 
56
- export function pullSnapshot({ dir, dbPath, passphrase, file, force = false }) {
77
+ function latestDbMtime(dbPath) {
78
+ let latest = 0
79
+ for (const suffix of ['', '-wal']) {
80
+ try { latest = Math.max(latest, statSync(dbPath + suffix).mtimeMs) } catch {}
81
+ }
82
+ return latest
83
+ }
84
+
85
+ /**
86
+ * Replace the local database with the chosen snapshot.
87
+ *
88
+ * Safety (robustness audit P1-4): when the local database was written after
89
+ * the snapshot was taken on *another* device, overwriting it would silently
90
+ * drop those writes. Refuse unless force:true. A snapshot pushed from this
91
+ * HOME (device marker matches payload.device) is treated as an intentional
92
+ * restore, and no local database at all is always allowed.
93
+ */
94
+ export function pullSnapshot({ dir, dbPath, passphrase, file, force = false, localMtimeMs, localDevice }) {
57
95
  const snapshots = listSnapshots(dir)
58
96
  if (!snapshots.length) throw new Error(`no Advisers snapshots in ${dir}`)
59
97
  const chosen = file ? snapshots.find((s) => s.name === basename(file)) : snapshots[0]
@@ -65,20 +103,27 @@ export function pullSnapshot({ dir, dbPath, passphrase, file, force = false }) {
65
103
  const bytes = Buffer.from(payload.db, 'base64')
66
104
  const sha256 = createHash('sha256').update(bytes).digest('hex')
67
105
  if (sha256 !== payload.sha256) throw new Error('snapshot checksum mismatch')
68
- if (existsSync(dbPath) && !force && readFileSync(dbPath).equals(bytes)) {
69
- return { file: chosen.name, sha256, changed: false, backup: null }
106
+ if (existsSync(dbPath) && !force) {
107
+ if (readFileSync(dbPath).equals(bytes)) return { file: chosen.name, sha256, changed: false, backup: null }
108
+ const snapshotAt = Date.parse(payload.createdAt ?? '')
109
+ const localAt = Number.isFinite(localMtimeMs) && localMtimeMs > 0 ? localMtimeMs : latestDbMtime(dbPath)
110
+ const sameDevice = Boolean(localDevice) && Boolean(payload.device) && sanitizeDevice(payload.device) === sanitizeDevice(localDevice)
111
+ if (!sameDevice) {
112
+ if (!Number.isFinite(snapshotAt)) throw new Error(`cannot tell whether the local database is newer than ${chosen.name} (snapshot has no valid createdAt); re-run with --force to overwrite and keep a backup`)
113
+ if (localAt > snapshotAt) throw new Error(`local database was written after snapshot ${chosen.name} (local ${new Date(localAt).toISOString()} > snapshot ${new Date(snapshotAt).toISOString()}); refusing to overwrite. If those local writes matter, back them up/export them first; re-run with --force to overwrite anyway (a backup is kept).`)
114
+ }
70
115
  }
71
116
  let backup = null
72
117
  if (existsSync(dbPath)) {
73
118
  backup = `${dbPath}.before-sync-${Date.now()}`
74
119
  copyFileSync(dbPath, backup)
120
+ for (const suffix of ['-wal', '-shm']) {
121
+ try { if (existsSync(dbPath + suffix)) copyFileSync(dbPath + suffix, backup + suffix) } catch {}
122
+ }
75
123
  for (const suffix of ['-wal', '-shm']) { try { rmSync(dbPath + suffix) } catch {} }
76
124
  } else {
77
125
  mkdirSync(join(dbPath, '..'), { recursive: true, mode: 0o700 })
78
126
  }
79
- const tmp = `${dbPath}.sync-tmp-${process.pid}`
80
- writeFileSync(tmp, bytes, { mode: 0o600 })
81
- renameSync(tmp, dbPath)
82
- try { chmodSync(dbPath, 0o600) } catch {}
127
+ writeAtomic(dbPath, bytes, { mode: 0o600 })
83
128
  return { file: chosen.name, sha256, changed: true, backup, device: payload.device, createdAt: payload.createdAt }
84
129
  }