advisers 0.2.0-preview.71 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +130 -201
- package/NOTICE +29 -4
- package/README-zh.md +32 -6
- package/README.md +36 -7
- package/THIRD-PARTY-NOTICES.md +277 -0
- package/bin/advisers.js +1634 -290
- package/index.js +1 -1
- package/lib/attachments.js +153 -27
- package/lib/capture.js +82 -2
- package/lib/evidence.js +1180 -26
- package/lib/ical.js +542 -0
- package/lib/imap.js +82 -24
- package/lib/mcp-server.js +869 -0
- package/lib/model.js +151 -2
- package/lib/notify.js +129 -3
- package/lib/pdf.js +44 -6
- package/lib/pet-brief.js +78 -15
- package/lib/receipt-verify.js +624 -0
- package/lib/remind.js +261 -11
- package/lib/research.js +142 -18
- package/lib/safety/classifier.js +392 -0
- package/lib/safety/explain.js +157 -0
- package/lib/safety/index.js +333 -0
- package/lib/safety/redact.js +212 -0
- package/lib/safety/rules.js +824 -0
- package/lib/scheduler.js +399 -0
- package/lib/source.js +294 -0
- package/lib/sqlite.js +361 -37
- package/lib/sync.js +58 -13
- package/lib/tos/index.js +332 -0
- package/lib/tos/manifest.json +32 -0
- package/lib/tos/privacy.zh-CN.md +136 -0
- package/lib/tos/safety.zh-CN.md +101 -0
- package/lib/tos/summary.zh-CN.md +12 -0
- package/lib/tos/terms.zh-CN.md +439 -0
- package/package.json +7 -6
package/lib/sqlite.js
CHANGED
|
@@ -1,6 +1,71 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite'
|
|
2
|
+
import { chmodSync, copyFileSync } from 'node:fs'
|
|
2
3
|
import { createHash, createHmac, randomUUID } from 'node:crypto'
|
|
3
4
|
|
|
5
|
+
/**
|
|
6
|
+
* Open + write hardening (2026-11-17 robustness audit P0-1/P0-3/P1-2):
|
|
7
|
+
* - busy_timeout is set before any lock-taking pragma; journal_mode=WAL is
|
|
8
|
+
* idempotent and retried with bounded backoff while another process holds
|
|
9
|
+
* the database lock.
|
|
10
|
+
* - Opening no longer runs a full 'PRAGMA quick_check' (that is O(db) and made
|
|
11
|
+
* every command slow). Integrity checks live in assertIntegrity() and are
|
|
12
|
+
* run by 'advisers doctor' / 'advisers audit --check-db'.
|
|
13
|
+
* - appendAudit() and saveWithAudit() run in BEGIN IMMEDIATE transactions that
|
|
14
|
+
* re-read the previous hash before inserting, so the audit chain cannot fork
|
|
15
|
+
* under concurrent writers and a ledger write and its audit row commit
|
|
16
|
+
* together or not at all.
|
|
17
|
+
* - Migrations are refused for a schema newer than this CLI and are preceded
|
|
18
|
+
* by a same-directory backup of the old database.
|
|
19
|
+
*/
|
|
20
|
+
export const SCHEMA_VERSION = 18
|
|
21
|
+
const DEFAULT_BUSY_TIMEOUT_MS = 5000
|
|
22
|
+
const OPEN_RETRIES = 6
|
|
23
|
+
const BUSY_ERRCODES = new Set([5, 6]) // SQLITE_BUSY, SQLITE_LOCKED
|
|
24
|
+
|
|
25
|
+
function isBusyError(error) {
|
|
26
|
+
if (!error) return false
|
|
27
|
+
if (typeof error.errcode === 'number' && BUSY_ERRCODES.has(error.errcode)) return true
|
|
28
|
+
return /SQLITE_BUSY|SQLITE_LOCKED|database (?:table )?is locked/i.test(String(error.message ?? error))
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function sleepSync(ms) {
|
|
32
|
+
if (!ms || ms <= 0) return
|
|
33
|
+
try {
|
|
34
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
|
|
35
|
+
} catch {
|
|
36
|
+
const end = Date.now() + ms
|
|
37
|
+
while (Date.now() < end) {}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function backoffMs(attempt) {
|
|
42
|
+
return Math.min(2000, 50 * 2 ** attempt)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Ledger tables are written as a whole snapshot by save(). The id column lets
|
|
46
|
+
// save() merge with rows other processes committed after our load().
|
|
47
|
+
const LEDGER_TABLES = [
|
|
48
|
+
['matters', 'id'],
|
|
49
|
+
['facts', 'id'],
|
|
50
|
+
['commitments', 'id'],
|
|
51
|
+
['preferences', 'id'],
|
|
52
|
+
['links', 'id'],
|
|
53
|
+
['updates', 'id'],
|
|
54
|
+
['routines', 'id'],
|
|
55
|
+
['routine_patches', 'id'],
|
|
56
|
+
['routes', 'key'],
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
function idsFor(snapshot) {
|
|
60
|
+
const ids = {}
|
|
61
|
+
for (const [table, key] of LEDGER_TABLES) {
|
|
62
|
+
const value = snapshot?.[table]
|
|
63
|
+
const rows = table === 'routes' ? Object.keys(value ?? {}) : (value ?? []).map((row) => row?.id)
|
|
64
|
+
ids[table] = new Set(rows.filter((id) => id != null))
|
|
65
|
+
}
|
|
66
|
+
return ids
|
|
67
|
+
}
|
|
68
|
+
|
|
4
69
|
const SCHEMA = `
|
|
5
70
|
CREATE TABLE IF NOT EXISTS matters (
|
|
6
71
|
id TEXT PRIMARY KEY, title TEXT NOT NULL, status TEXT NOT NULL, domain TEXT NOT NULL,
|
|
@@ -17,7 +82,7 @@ CREATE TABLE IF NOT EXISTS facts (
|
|
|
17
82
|
CREATE TABLE IF NOT EXISTS commitments (
|
|
18
83
|
id TEXT PRIMARY KEY, matter_id TEXT NOT NULL, debtor TEXT NOT NULL, creditor TEXT NOT NULL,
|
|
19
84
|
action TEXT NOT NULL, due_at TEXT, timezone TEXT NOT NULL, status TEXT NOT NULL,
|
|
20
|
-
evidence TEXT NOT NULL, capability TEXT NOT NULL, done_at TEXT
|
|
85
|
+
evidence TEXT NOT NULL, capability TEXT NOT NULL, done_at TEXT, reminder_offsets TEXT
|
|
21
86
|
);
|
|
22
87
|
CREATE TABLE IF NOT EXISTS preferences (
|
|
23
88
|
id TEXT PRIMARY KEY, kind TEXT NOT NULL, statement TEXT NOT NULL, scope TEXT NOT NULL,
|
|
@@ -121,6 +186,30 @@ CREATE TABLE IF NOT EXISTS calendar_events (
|
|
|
121
186
|
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
122
187
|
`
|
|
123
188
|
|
|
189
|
+
/**
|
|
190
|
+
* P2-1: secondary indexes for the hot read/write paths (tick, today, history,
|
|
191
|
+
* deliveries). Indexes are layout only, not a schema change, so they are
|
|
192
|
+
* created with IF NOT EXISTS after migrate() on every open: new databases get
|
|
193
|
+
* them at creation time and existing databases get them the first time they
|
|
194
|
+
* are opened by this version. Each one was a full table SCAN + TEMP B-TREE in
|
|
195
|
+
* the 2026-11-17 performance audit (nudges/facts/deliveries/calendar_events/
|
|
196
|
+
* task_runs/sessions/model_calls/preferences/commitments).
|
|
197
|
+
*/
|
|
198
|
+
const INDEXES = `
|
|
199
|
+
CREATE INDEX IF NOT EXISTS idx_nudges_commitment_at ON nudges(commitment_id, at);
|
|
200
|
+
CREATE INDEX IF NOT EXISTS idx_nudges_local_day_level ON nudges(local_day, level);
|
|
201
|
+
CREATE INDEX IF NOT EXISTS idx_nudges_at ON nudges(at);
|
|
202
|
+
CREATE INDEX IF NOT EXISTS idx_commitments_status_due ON commitments(status, due_at);
|
|
203
|
+
CREATE INDEX IF NOT EXISTS idx_facts_matter ON facts(matter_id);
|
|
204
|
+
CREATE INDEX IF NOT EXISTS idx_facts_status_deleted ON facts(status, deleted_at);
|
|
205
|
+
CREATE INDEX IF NOT EXISTS idx_preferences_status_expires ON preferences(status, expires_at);
|
|
206
|
+
CREATE INDEX IF NOT EXISTS idx_calendar_events_starts ON calendar_events(starts_at);
|
|
207
|
+
CREATE INDEX IF NOT EXISTS idx_deliveries_created ON deliveries(created_at);
|
|
208
|
+
CREATE INDEX IF NOT EXISTS idx_task_runs_created ON task_runs(created_at);
|
|
209
|
+
CREATE INDEX IF NOT EXISTS idx_sessions_updated ON sessions(updated_at);
|
|
210
|
+
CREATE INDEX IF NOT EXISTS idx_model_calls_created ON model_calls(created_at);
|
|
211
|
+
`
|
|
212
|
+
|
|
124
213
|
const json = (v) => JSON.stringify(v ?? [])
|
|
125
214
|
const parse = (v, fallback = []) => {
|
|
126
215
|
if (v == null) return fallback
|
|
@@ -128,29 +217,140 @@ const parse = (v, fallback = []) => {
|
|
|
128
217
|
}
|
|
129
218
|
|
|
130
219
|
export class SqliteStore {
|
|
131
|
-
constructor(path, { readOnly = false } = {}) {
|
|
220
|
+
constructor(path, { readOnly = false, checkIntegrity = false, busyTimeoutMs = Number(process.env.ADVISERS_SQLITE_BUSY_TIMEOUT_MS) || DEFAULT_BUSY_TIMEOUT_MS } = {}) {
|
|
221
|
+
this.path = path
|
|
222
|
+
this.busyTimeoutMs = busyTimeoutMs
|
|
223
|
+
this._transactionDepth = 0
|
|
224
|
+
this._baselineIds = null
|
|
132
225
|
this.db = readOnly ? new DatabaseSync(path, { readOnly: true }) : new DatabaseSync(path)
|
|
226
|
+
// P0-1: sqlite only applies busy_timeout to statements that run after the
|
|
227
|
+
// pragma is set; concurrent first-opens used to fail on journal_mode=WAL
|
|
228
|
+
// because busy_timeout was set afterwards.
|
|
229
|
+
this.db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.floor(Number(busyTimeoutMs) || 0))}`)
|
|
133
230
|
if (readOnly) {
|
|
134
|
-
this.
|
|
135
|
-
const integrity = this.db.prepare('PRAGMA quick_check').get()
|
|
136
|
-
if (integrity && integrity.quick_check !== 'ok') throw new Error(`database integrity check failed: ${integrity.quick_check}`)
|
|
231
|
+
if (checkIntegrity) this.assertIntegrity()
|
|
137
232
|
return
|
|
138
233
|
}
|
|
139
|
-
this.
|
|
140
|
-
this.db.exec('PRAGMA busy_timeout = 5000')
|
|
234
|
+
this.enableWal()
|
|
141
235
|
this.db.exec('PRAGMA secure_delete = ON')
|
|
142
236
|
this.db.exec('PRAGMA synchronous = NORMAL')
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
237
|
+
this.initializeSchema()
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Full-database integrity check; explicit because it is O(database size). */
|
|
241
|
+
assertIntegrity() {
|
|
242
|
+
const row = this.db.prepare('PRAGMA quick_check').get()
|
|
243
|
+
const verdict = row?.quick_check ?? 'unknown'
|
|
244
|
+
if (verdict !== 'ok') throw new Error(`database integrity check failed: ${verdict}`)
|
|
245
|
+
return verdict
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** WAL is persistent; skip the switch when already enabled and retry on busy. */
|
|
249
|
+
enableWal() {
|
|
250
|
+
if (!this.path || this.path === ':memory:') return 'memory'
|
|
251
|
+
const current = String(this.db.prepare('PRAGMA journal_mode').get()?.journal_mode ?? '').toLowerCase()
|
|
252
|
+
if (current === 'wal' || current === 'memory') return current
|
|
253
|
+
let lastError
|
|
254
|
+
for (let attempt = 0; attempt <= OPEN_RETRIES; attempt++) {
|
|
255
|
+
try {
|
|
256
|
+
const result = this.db.prepare('PRAGMA journal_mode = WAL').get()
|
|
257
|
+
const mode = String(result?.journal_mode ?? '').toLowerCase()
|
|
258
|
+
if (mode === 'wal' || mode === 'memory') return mode
|
|
259
|
+
lastError = new Error(`could not enable WAL journal mode (got '${mode || 'unknown'}')`)
|
|
260
|
+
} catch (error) {
|
|
261
|
+
lastError = error
|
|
262
|
+
if (!isBusyError(error)) throw error
|
|
263
|
+
}
|
|
264
|
+
sleepSync(backoffMs(attempt))
|
|
265
|
+
}
|
|
266
|
+
throw lastError
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
initializeSchema() {
|
|
270
|
+
this._transaction('IMMEDIATE', () => {
|
|
271
|
+
this.db.exec(SCHEMA)
|
|
272
|
+
this.migrate()
|
|
273
|
+
// Indexes come after migrate(): some index the columns migrate() adds
|
|
274
|
+
// (e.g. facts.deleted_at), so an old database must be upgraded first.
|
|
275
|
+
this.db.exec(INDEXES)
|
|
276
|
+
})
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Run fn() in a write transaction with bounded SQLITE_BUSY retries.
|
|
281
|
+
* Nested calls join the outer transaction (no nested BEGIN).
|
|
282
|
+
*/
|
|
283
|
+
_transaction(mode, fn, { retries = OPEN_RETRIES } = {}) {
|
|
284
|
+
if (this._transactionDepth > 0) return fn()
|
|
285
|
+
let lastError
|
|
286
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
287
|
+
let began = false
|
|
288
|
+
try {
|
|
289
|
+
this.db.exec(`BEGIN ${mode}`)
|
|
290
|
+
began = true
|
|
291
|
+
this._transactionDepth++
|
|
292
|
+
let result
|
|
293
|
+
try {
|
|
294
|
+
result = fn()
|
|
295
|
+
} finally {
|
|
296
|
+
this._transactionDepth--
|
|
297
|
+
}
|
|
298
|
+
this.db.exec('COMMIT')
|
|
299
|
+
return result
|
|
300
|
+
} catch (error) {
|
|
301
|
+
if (began) { try { this.db.exec('ROLLBACK') } catch {} }
|
|
302
|
+
lastError = error
|
|
303
|
+
if (!isBusyError(error) || attempt === retries) throw error
|
|
304
|
+
}
|
|
305
|
+
sleepSync(backoffMs(attempt))
|
|
306
|
+
}
|
|
307
|
+
throw lastError
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** Public transaction helper for multi-statement invariants (returns fn result). */
|
|
311
|
+
transaction(fn, { mode = 'IMMEDIATE' } = {}) {
|
|
312
|
+
return this._transaction(mode, fn)
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Checkpoint the WAL so a copied/sealed database file is self-contained.
|
|
317
|
+
* Returns busy=true when another connection prevented a full checkpoint.
|
|
318
|
+
*/
|
|
319
|
+
checkpoint({ mode = 'TRUNCATE', retries = OPEN_RETRIES } = {}) {
|
|
320
|
+
let last = null
|
|
321
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
322
|
+
last = this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get()
|
|
323
|
+
if (!last || Number(last.busy) === 0) {
|
|
324
|
+
return { busy: false, log: Number(last?.log ?? 0), checkpointed: Number(last?.checkpointed ?? 0) }
|
|
325
|
+
}
|
|
326
|
+
sleepSync(backoffMs(attempt))
|
|
327
|
+
}
|
|
328
|
+
return { busy: Number(last?.busy ?? 1) !== 0, log: Number(last?.log ?? 0), checkpointed: Number(last?.checkpointed ?? 0) }
|
|
147
329
|
}
|
|
148
330
|
|
|
331
|
+
backupBeforeMigrate(current) {
|
|
332
|
+
if (!this.path || this.path === ':memory:') return
|
|
333
|
+
const target = `${this.path}.before-migrate-v${current}-${Date.now()}`
|
|
334
|
+
try {
|
|
335
|
+
copyFileSync(this.path, target)
|
|
336
|
+
try { chmodSync(target, 0o600) } catch {}
|
|
337
|
+
} catch (error) {
|
|
338
|
+
throw new Error(`refusing to migrate without a backup of ${this.path}: ${error.message}`)
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Bring an older database up to SCHEMA_VERSION. Runs inside the caller's
|
|
344
|
+
* write transaction (initializeSchema). Refuses futures schemas and keeps
|
|
345
|
+
* a same-directory backup before altering an existing database.
|
|
346
|
+
*/
|
|
149
347
|
migrate() {
|
|
150
348
|
const row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get('schema_version')
|
|
151
349
|
const current = row ? Number(row.value) : 0
|
|
152
|
-
if (current
|
|
153
|
-
this
|
|
350
|
+
if (!Number.isFinite(current) || current < 0) throw new Error(`invalid schema_version in ${this.path}: ${row?.value}`)
|
|
351
|
+
if (current > SCHEMA_VERSION) throw new Error(`database schema v${current} is newer than this CLI (v${SCHEMA_VERSION}); upgrade advisers before opening ${this.path}`)
|
|
352
|
+
if (current >= SCHEMA_VERSION) return
|
|
353
|
+
if (current > 0) this.backupBeforeMigrate(current)
|
|
154
354
|
try {
|
|
155
355
|
this.db.exec(`CREATE TABLE IF NOT EXISTS seen_events (
|
|
156
356
|
link_id TEXT NOT NULL, account_id TEXT NOT NULL, dedupe_key TEXT NOT NULL,
|
|
@@ -196,6 +396,8 @@ export class SqliteStore {
|
|
|
196
396
|
try { this.db.exec("INSERT INTO turns_fts(turns_fts) VALUES('rebuild')") } catch {}
|
|
197
397
|
const commitmentCols = this.db.prepare('PRAGMA table_info(commitments)').all().map((c) => c.name)
|
|
198
398
|
if (!commitmentCols.includes('done_at')) this.db.exec('ALTER TABLE commitments ADD COLUMN done_at TEXT')
|
|
399
|
+
// 0.2.0: per-commitment custom reminder offsets (JSON array of strings).
|
|
400
|
+
if (!commitmentCols.includes('reminder_offsets')) this.db.exec('ALTER TABLE commitments ADD COLUMN reminder_offsets TEXT')
|
|
199
401
|
const factCols = this.db.prepare('PRAGMA table_info(facts)').all().map((c) => c.name)
|
|
200
402
|
for (const [name, type] of [['valid_from','TEXT'],['valid_to','TEXT'],['expired_at','TEXT'],['evidence','TEXT'],['supersedes','TEXT'],['superseded_by','TEXT'],['shadowed_by','TEXT'],['last_seen_at','TEXT'],['deleted_at','TEXT'],['deleted_reason','TEXT']]) {
|
|
201
403
|
if (!factCols.includes(name)) this.db.exec(`ALTER TABLE facts ADD COLUMN ${name} ${type}`)
|
|
@@ -237,10 +439,8 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
237
439
|
all_day INTEGER NOT NULL DEFAULT 0, tzid TEXT, location TEXT, rrule TEXT,
|
|
238
440
|
source TEXT, imported_at TEXT NOT NULL
|
|
239
441
|
)`)
|
|
240
|
-
this.db.prepare('INSERT OR REPLACE INTO meta(key,value) VALUES(?,?)').run('schema_version',
|
|
241
|
-
this.db.exec('COMMIT')
|
|
442
|
+
this.db.prepare('INSERT OR REPLACE INTO meta(key,value) VALUES(?,?)').run('schema_version', String(SCHEMA_VERSION))
|
|
242
443
|
} catch (error) {
|
|
243
|
-
this.db.exec('ROLLBACK')
|
|
244
444
|
throw error
|
|
245
445
|
}
|
|
246
446
|
}
|
|
@@ -323,6 +523,76 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
323
523
|
return this.db.prepare('SELECT COUNT(*) AS n FROM nudges WHERE local_day = ? AND level = ?').get(localDay, level).n
|
|
324
524
|
}
|
|
325
525
|
|
|
526
|
+
/**
|
|
527
|
+
* P0-2 (tick N+1 removal): insert every nudge for one tick in a single
|
|
528
|
+
* transaction with one prepared statement. INSERT OR IGNORE keeps the
|
|
529
|
+
* per-(local_day, level, commitment_id) dedupe key semantics of addNudge()
|
|
530
|
+
* without aborting the batch on an expected duplicate. Returns the number of
|
|
531
|
+
* rows actually inserted (existing rows are left untouched).
|
|
532
|
+
*/
|
|
533
|
+
addNudges(rows) {
|
|
534
|
+
if (!rows?.length) return 0
|
|
535
|
+
return this._transaction('IMMEDIATE', () => {
|
|
536
|
+
const statement = this.db.prepare('INSERT OR IGNORE INTO nudges(id,at,local_day,level,commitment_id,dedupe_key,body) VALUES(?,?,?,?,?,?,?)')
|
|
537
|
+
const now = new Date().toISOString()
|
|
538
|
+
let inserted = 0
|
|
539
|
+
for (const row of rows) {
|
|
540
|
+
const key = `${row.localDay}:${row.level}:${row.commitmentId}`
|
|
541
|
+
const id = `nudge_${randomUUID().replace(/-/g, '').slice(0, 16)}`
|
|
542
|
+
inserted += Number(statement.run(id, row.at ?? now, row.localDay, row.level, row.commitmentId, key, row.body).changes)
|
|
543
|
+
}
|
|
544
|
+
return inserted
|
|
545
|
+
})
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* P0-2 (tick N+1 removal): the newest `limit` outcomes per commitment in one
|
|
550
|
+
* query (window function over idx_nudges_commitment_at). Replaces one
|
|
551
|
+
* listCommitmentNudges() round-trip per commitment.
|
|
552
|
+
*/
|
|
553
|
+
recentNudgeOutcomes(limit = 3) {
|
|
554
|
+
const rows = this.db.prepare(`SELECT commitment_id, outcome, at FROM (
|
|
555
|
+
SELECT commitment_id, outcome, at, ROW_NUMBER() OVER (PARTITION BY commitment_id ORDER BY at DESC) AS rn
|
|
556
|
+
FROM nudges
|
|
557
|
+
) WHERE rn <= ?`).all(limit)
|
|
558
|
+
const byCommitment = new Map()
|
|
559
|
+
for (const row of rows) {
|
|
560
|
+
const list = byCommitment.get(row.commitment_id)
|
|
561
|
+
if (list) list.push(row.outcome)
|
|
562
|
+
else byCommitment.set(row.commitment_id, [row.outcome])
|
|
563
|
+
}
|
|
564
|
+
return byCommitment
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/** P0-2 (tick N+1 removal): today's nudge count per level in one query. */
|
|
568
|
+
countNudgesByDay(localDay) {
|
|
569
|
+
const rows = this.db.prepare('SELECT level, COUNT(*) AS n FROM nudges WHERE local_day = ? GROUP BY level').all(localDay)
|
|
570
|
+
return new Map(rows.map((row) => [row.level, Number(row.n)]))
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/** P0-2 (tick N+1 removal): existing (commitment, level) nudge keys for a day. */
|
|
574
|
+
listNudgeLevelKeysByDay(localDay) {
|
|
575
|
+
const rows = this.db.prepare('SELECT commitment_id, level FROM nudges WHERE local_day = ?').all(localDay)
|
|
576
|
+
return new Set(rows.map((row) => `${row.commitment_id}\u0000${row.level}`))
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
/** P0-2 (tick N+1 removal): all meta rows with a prefix in one query (snooze:*). */
|
|
580
|
+
listMetaByPrefix(prefix) {
|
|
581
|
+
const rows = this.db.prepare('SELECT key, value FROM meta WHERE key LIKE ?').all(`${prefix}%`)
|
|
582
|
+
return new Map(rows.map((row) => [row.key, row.value]))
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
/** P0-2: delete a batch of meta keys in one transaction. */
|
|
586
|
+
deleteMetaKeys(keys) {
|
|
587
|
+
if (!keys?.length) return 0
|
|
588
|
+
return this._transaction('IMMEDIATE', () => {
|
|
589
|
+
const statement = this.db.prepare('DELETE FROM meta WHERE key = ?')
|
|
590
|
+
let deleted = 0
|
|
591
|
+
for (const key of keys) deleted += Number(statement.run(key).changes)
|
|
592
|
+
return deleted
|
|
593
|
+
})
|
|
594
|
+
}
|
|
595
|
+
|
|
326
596
|
listNudges(limit = 20) {
|
|
327
597
|
return this.db.prepare('SELECT * FROM nudges ORDER BY at DESC LIMIT ?').all(limit)
|
|
328
598
|
}
|
|
@@ -528,7 +798,16 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
528
798
|
return this.db.prepare('SELECT COUNT(*) AS n FROM deliveries').get().n
|
|
529
799
|
}
|
|
530
800
|
|
|
801
|
+
/**
|
|
802
|
+
* Append an audit row atomically. BEGIN IMMEDIATE serializes writers and the
|
|
803
|
+
* previous hash is re-read inside the transaction, so two concurrent writers
|
|
804
|
+
* can never both chain onto the same predecessor (audit fork, P0-3).
|
|
805
|
+
*/
|
|
531
806
|
appendAudit({ actor, action, allowed, reason }) {
|
|
807
|
+
return this._transaction('IMMEDIATE', () => this._appendAuditRow({ actor, action, allowed, reason }))
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
_appendAuditRow({ actor, action, allowed, reason }) {
|
|
532
811
|
const last = this.db.prepare('SELECT hash FROM audit ORDER BY seq DESC LIMIT 1').get()
|
|
533
812
|
const prev = last?.hash ?? ''
|
|
534
813
|
const at = new Date().toISOString()
|
|
@@ -591,7 +870,7 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
591
870
|
|
|
592
871
|
load() {
|
|
593
872
|
const all = (sql) => this.db.prepare(sql).all()
|
|
594
|
-
|
|
873
|
+
const snapshot = {
|
|
595
874
|
version: 1,
|
|
596
875
|
matters: all('SELECT * FROM matters').map((r) => ({
|
|
597
876
|
id: r.id, title: r.title, status: r.status, domain: r.domain, secrecy: r.secrecy,
|
|
@@ -609,11 +888,19 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
609
888
|
shadowedBy: r.shadowed_by ?? undefined, lastSeenAt: r.last_seen_at ?? undefined,
|
|
610
889
|
deletedAt: r.deleted_at ?? undefined, deletedReason: r.deleted_reason ?? undefined,
|
|
611
890
|
})),
|
|
612
|
-
commitments: all('SELECT * FROM commitments').map((r) =>
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
891
|
+
commitments: all('SELECT * FROM commitments').map((r) => {
|
|
892
|
+
// NULL means "never customised": keep the field absent so old snapshots
|
|
893
|
+
// and the pre-0.2.0 commitment shape stay identical. Corrupt or
|
|
894
|
+
// non-array JSON degrades to absent rather than breaking the ledger.
|
|
895
|
+
const parsedOffsets = r.reminder_offsets == null ? null : parse(r.reminder_offsets, null)
|
|
896
|
+
const reminderOffsets = Array.isArray(parsedOffsets) ? parsedOffsets : undefined
|
|
897
|
+
return {
|
|
898
|
+
id: r.id, matterId: r.matter_id, debtor: r.debtor, creditor: r.creditor, action: r.action,
|
|
899
|
+
dueAt: r.due_at ?? undefined, timezone: r.timezone, status: r.status,
|
|
900
|
+
evidence: parse(r.evidence), capability: r.capability, doneAt: r.done_at ?? undefined,
|
|
901
|
+
...(reminderOffsets === undefined ? {} : { reminderOffsets }),
|
|
902
|
+
}
|
|
903
|
+
}),
|
|
617
904
|
preferences: all('SELECT * FROM preferences').map((r) => ({
|
|
618
905
|
id: r.id, kind: r.kind, statement: r.statement, scope: parse(r.scope, {}),
|
|
619
906
|
source: parse(r.source, {}), evidence: parse(r.evidence), confidence: r.confidence,
|
|
@@ -641,53 +928,90 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
641
928
|
})),
|
|
642
929
|
routes: Object.fromEntries(all('SELECT * FROM routes').map((r) => [r.key, r.matter_id])),
|
|
643
930
|
}
|
|
931
|
+
// Remember which rows this connection has seen (P1-1): save() only deletes
|
|
932
|
+
// rows this loader owned, so another process's concurrent additions are
|
|
933
|
+
// merged instead of being silently overwritten by a stale snapshot.
|
|
934
|
+
this._baselineIds = idsFor(snapshot)
|
|
935
|
+
return snapshot
|
|
644
936
|
}
|
|
645
937
|
|
|
646
938
|
save(snapshot) {
|
|
939
|
+
return this._transaction('IMMEDIATE', () => this._saveRows(snapshot))
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* Write the ledger snapshot and its audit row in one transaction, so the
|
|
944
|
+
* audit trail can never claim a write that rolled back (or miss one that
|
|
945
|
+
* committed). `inside` runs inside the same transaction before the audit row.
|
|
946
|
+
*/
|
|
947
|
+
saveWithAudit(snapshot, audit, inside) {
|
|
948
|
+
return this._transaction('IMMEDIATE', () => {
|
|
949
|
+
this._saveRows(snapshot)
|
|
950
|
+
if (typeof inside === 'function') inside(this)
|
|
951
|
+
return this._appendAuditRow(audit)
|
|
952
|
+
})
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
_saveRows(snapshot) {
|
|
647
956
|
const db = this.db
|
|
648
|
-
|
|
957
|
+
const newIds = idsFor(snapshot)
|
|
958
|
+
const baseline = this._baselineIds
|
|
649
959
|
try {
|
|
650
|
-
|
|
651
|
-
|
|
960
|
+
if (baseline) {
|
|
961
|
+
// P1-1: delete only rows this connection loaded that the new snapshot
|
|
962
|
+
// dropped; rows another process added after our load() are kept.
|
|
963
|
+
for (const [table, key] of LEDGER_TABLES) {
|
|
964
|
+
const removed = [...(baseline[table] ?? [])].filter((id) => !newIds[table].has(id))
|
|
965
|
+
if (!removed.length) continue
|
|
966
|
+
const statement = db.prepare(`DELETE FROM ${table} WHERE ${key} = ?`)
|
|
967
|
+
for (const id of removed) statement.run(id)
|
|
968
|
+
}
|
|
969
|
+
} else {
|
|
970
|
+
// No prior load(): keep the historical full-replace semantics for
|
|
971
|
+
// direct library use (e.g. importing or seeding a ledger).
|
|
972
|
+
for (const [table] of LEDGER_TABLES) db.exec(`DELETE FROM ${table}`)
|
|
652
973
|
}
|
|
653
|
-
const m = db.prepare('INSERT INTO matters VALUES (?,?,?,?,?,?,?,?,?,?,?)')
|
|
974
|
+
const m = db.prepare('INSERT OR REPLACE INTO matters VALUES (?,?,?,?,?,?,?,?,?,?,?)')
|
|
654
975
|
for (const x of snapshot.matters ?? []) {
|
|
655
976
|
m.run(x.id, x.title, x.status, x.domain, x.secrecy, x.summary ?? null, json(x.participants), json(x.deadlines), json(x.openQuestions), x.createdAt, x.updatedAt)
|
|
656
977
|
}
|
|
657
|
-
const f = db.prepare(`INSERT INTO facts(id,matter_id,subject,predicate,object,recorded_at,expired_at,authority,confidence,source,evidence,status,valid_from,valid_to,supersedes,superseded_by,shadowed_by,last_seen_at,deleted_at,deleted_reason)
|
|
978
|
+
const f = db.prepare(`INSERT OR REPLACE INTO facts(id,matter_id,subject,predicate,object,recorded_at,expired_at,authority,confidence,source,evidence,status,valid_from,valid_to,supersedes,superseded_by,shadowed_by,last_seen_at,deleted_at,deleted_reason)
|
|
658
979
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
|
|
659
980
|
for (const x of snapshot.facts ?? []) {
|
|
660
981
|
f.run(x.id, x.matterId ?? null, x.subject, x.predicate, x.object, x.recordedAt, x.expiredAt ?? null, x.authority, x.confidence, x.source ? json(x.source) : null, x.evidence ? json(x.evidence) : null, x.status, x.validFrom ?? null, x.validTo ?? null, x.supersedes ?? null, x.supersededBy ?? null, x.shadowedBy ?? null, x.lastSeenAt ?? null, x.deletedAt ?? null, x.deletedReason ?? null)
|
|
661
982
|
}
|
|
662
|
-
const c = db.prepare(
|
|
983
|
+
const c = db.prepare(`INSERT OR REPLACE INTO commitments(id,matter_id,debtor,creditor,action,due_at,timezone,status,evidence,capability,done_at,reminder_offsets)
|
|
984
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`)
|
|
663
985
|
for (const x of snapshot.commitments ?? []) {
|
|
664
|
-
|
|
986
|
+
// A missing/undefined list stays NULL so a round-trip keeps the pre-0.2.0
|
|
987
|
+
// "no field" shape; an explicit [] is preserved as its own value.
|
|
988
|
+
const reminderOffsets = Array.isArray(x.reminderOffsets) ? JSON.stringify(x.reminderOffsets) : null
|
|
989
|
+
c.run(x.id, x.matterId, x.debtor, x.creditor, x.action, x.dueAt ?? null, x.timezone, x.status, json(x.evidence), x.capability, x.doneAt ?? null, reminderOffsets)
|
|
665
990
|
}
|
|
666
|
-
const p = db.prepare('INSERT INTO preferences VALUES (?,?,?,?,?,?,?,?,?,?,?,?)')
|
|
991
|
+
const p = db.prepare('INSERT OR REPLACE INTO preferences VALUES (?,?,?,?,?,?,?,?,?,?,?,?)')
|
|
667
992
|
for (const x of snapshot.preferences ?? []) {
|
|
668
993
|
p.run(x.id, x.kind, x.statement, json(x.scope), json(x.source), json(x.evidence), x.confidence, x.status, x.validFrom, x.lastConfirmed ?? null, x.expiresAt ?? null, x.priority)
|
|
669
994
|
}
|
|
670
|
-
const l = db.prepare('INSERT INTO links VALUES (?,?,?,?,?,?,?,?,?)')
|
|
995
|
+
const l = db.prepare('INSERT OR REPLACE INTO links VALUES (?,?,?,?,?,?,?,?,?)')
|
|
671
996
|
for (const x of snapshot.links ?? []) {
|
|
672
997
|
l.run(x.id, x.fromMatter, x.toMatter, x.type, x.confidence, json(x.evidence), x.status, x.createdBy, x.createdAt)
|
|
673
998
|
}
|
|
674
|
-
const u = db.prepare('INSERT INTO updates VALUES (?,?,?,?,?,?,?)')
|
|
999
|
+
const u = db.prepare('INSERT OR REPLACE INTO updates VALUES (?,?,?,?,?,?,?)')
|
|
675
1000
|
for (const x of snapshot.updates ?? []) {
|
|
676
1001
|
u.run(x.id, x.sourceMatter, x.targetMatter, x.change, x.status, x.createdAt, x.rollback ?? null)
|
|
677
1002
|
}
|
|
678
|
-
const r = db.prepare('INSERT INTO routines VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)')
|
|
1003
|
+
const r = db.prepare('INSERT OR REPLACE INTO routines VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)')
|
|
679
1004
|
for (const x of snapshot.routines ?? []) {
|
|
680
1005
|
r.run(x.id, x.name, x.description, json(x.trigger), json(x.steps), json(x.preferences), json(x.pitfalls), json(x.verification), x.scope ?? null, x.version, x.status, x.createdBy, x.createdAt, x.updatedAt)
|
|
681
1006
|
}
|
|
682
|
-
const rp = db.prepare('INSERT INTO routine_patches VALUES (?,?,?,?,?,?,?,?)')
|
|
1007
|
+
const rp = db.prepare('INSERT OR REPLACE INTO routine_patches VALUES (?,?,?,?,?,?,?,?)')
|
|
683
1008
|
for (const x of snapshot.routinePatches ?? []) {
|
|
684
1009
|
rp.run(x.id, x.routineId, x.baseVersion, json(x.changes), json(x.before), x.diff, x.status, x.createdAt)
|
|
685
1010
|
}
|
|
686
|
-
const rt = db.prepare('INSERT INTO routes VALUES (?,?)')
|
|
1011
|
+
const rt = db.prepare('INSERT OR REPLACE INTO routes VALUES (?,?)')
|
|
687
1012
|
for (const [key, matterId] of Object.entries(snapshot.routes ?? {})) rt.run(key, matterId)
|
|
688
|
-
|
|
1013
|
+
this._baselineIds = newIds
|
|
689
1014
|
} catch (error) {
|
|
690
|
-
db.exec('ROLLBACK')
|
|
691
1015
|
throw error
|
|
692
1016
|
}
|
|
693
1017
|
}
|
package/lib/sync.js
CHANGED
|
@@ -9,11 +9,34 @@
|
|
|
9
9
|
* Not provided: continuous merge, multi-writer conflict resolution, cloud accounts.
|
|
10
10
|
* Pull always backs up the local DB before replacing it.
|
|
11
11
|
*/
|
|
12
|
-
import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
12
|
+
import { chmodSync, closeSync, copyFileSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
13
13
|
import { createHash } from 'node:crypto'
|
|
14
14
|
import { basename, join } from 'node:path'
|
|
15
15
|
import { decryptJson, encryptJson, isVaultEnvelope } from './vault.js'
|
|
16
16
|
|
|
17
|
+
/** fsync a directory entry so a rename survives a crash. */
|
|
18
|
+
function fsyncDir(dir) {
|
|
19
|
+
try {
|
|
20
|
+
const fd = openSync(dir, 'r')
|
|
21
|
+
try { fsyncSync(fd) } finally { closeSync(fd) }
|
|
22
|
+
} catch {}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Write a same-directory temp file, flush it, then rename it into place. */
|
|
26
|
+
function writeAtomic(file, data, { mode = 0o600 } = {}) {
|
|
27
|
+
mkdirSync(join(file, '..'), { recursive: true, mode: 0o700 })
|
|
28
|
+
const tmp = `${file}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 8)}`
|
|
29
|
+
try {
|
|
30
|
+
writeFileSync(tmp, data, { mode, flush: true })
|
|
31
|
+
renameSync(tmp, file)
|
|
32
|
+
} catch (error) {
|
|
33
|
+
try { if (existsSync(tmp)) rmSync(tmp) } catch {}
|
|
34
|
+
throw error
|
|
35
|
+
}
|
|
36
|
+
try { chmodSync(file, mode) } catch {}
|
|
37
|
+
fsyncDir(join(file, '..'))
|
|
38
|
+
}
|
|
39
|
+
|
|
17
40
|
export const SYNC_PREFIX = 'advisers-'
|
|
18
41
|
export const SYNC_SUFFIX = '.enc.json'
|
|
19
42
|
|
|
@@ -46,14 +69,29 @@ export function pushSnapshot({ dir, dbPath, passphrase, device = 'device', creat
|
|
|
46
69
|
mkdirSync(dir, { recursive: true, mode: 0o700 })
|
|
47
70
|
const snap = createSnapshot({ dbPath, passphrase, device, createdAt })
|
|
48
71
|
const out = join(dir, snap.name)
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
try { chmodSync(out, 0o600) } catch {}
|
|
53
|
-
return { file: out, sha256: snap.payload.sha256, bytes: Buffer.byteLength(JSON.stringify(snap.envelope)) }
|
|
72
|
+
const body = JSON.stringify(snap.envelope) + '\n'
|
|
73
|
+
writeAtomic(out, body, { mode: 0o600 })
|
|
74
|
+
return { file: out, sha256: snap.payload.sha256, bytes: Buffer.byteLength(body) }
|
|
54
75
|
}
|
|
55
76
|
|
|
56
|
-
|
|
77
|
+
function latestDbMtime(dbPath) {
|
|
78
|
+
let latest = 0
|
|
79
|
+
for (const suffix of ['', '-wal']) {
|
|
80
|
+
try { latest = Math.max(latest, statSync(dbPath + suffix).mtimeMs) } catch {}
|
|
81
|
+
}
|
|
82
|
+
return latest
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Replace the local database with the chosen snapshot.
|
|
87
|
+
*
|
|
88
|
+
* Safety (robustness audit P1-4): when the local database was written after
|
|
89
|
+
* the snapshot was taken on *another* device, overwriting it would silently
|
|
90
|
+
* drop those writes. Refuse unless force:true. A snapshot pushed from this
|
|
91
|
+
* HOME (device marker matches payload.device) is treated as an intentional
|
|
92
|
+
* restore, and no local database at all is always allowed.
|
|
93
|
+
*/
|
|
94
|
+
export function pullSnapshot({ dir, dbPath, passphrase, file, force = false, localMtimeMs, localDevice }) {
|
|
57
95
|
const snapshots = listSnapshots(dir)
|
|
58
96
|
if (!snapshots.length) throw new Error(`no Advisers snapshots in ${dir}`)
|
|
59
97
|
const chosen = file ? snapshots.find((s) => s.name === basename(file)) : snapshots[0]
|
|
@@ -65,20 +103,27 @@ export function pullSnapshot({ dir, dbPath, passphrase, file, force = false }) {
|
|
|
65
103
|
const bytes = Buffer.from(payload.db, 'base64')
|
|
66
104
|
const sha256 = createHash('sha256').update(bytes).digest('hex')
|
|
67
105
|
if (sha256 !== payload.sha256) throw new Error('snapshot checksum mismatch')
|
|
68
|
-
if (existsSync(dbPath) && !force
|
|
69
|
-
return { file: chosen.name, sha256, changed: false, backup: null }
|
|
106
|
+
if (existsSync(dbPath) && !force) {
|
|
107
|
+
if (readFileSync(dbPath).equals(bytes)) return { file: chosen.name, sha256, changed: false, backup: null }
|
|
108
|
+
const snapshotAt = Date.parse(payload.createdAt ?? '')
|
|
109
|
+
const localAt = Number.isFinite(localMtimeMs) && localMtimeMs > 0 ? localMtimeMs : latestDbMtime(dbPath)
|
|
110
|
+
const sameDevice = Boolean(localDevice) && Boolean(payload.device) && sanitizeDevice(payload.device) === sanitizeDevice(localDevice)
|
|
111
|
+
if (!sameDevice) {
|
|
112
|
+
if (!Number.isFinite(snapshotAt)) throw new Error(`cannot tell whether the local database is newer than ${chosen.name} (snapshot has no valid createdAt); re-run with --force to overwrite and keep a backup`)
|
|
113
|
+
if (localAt > snapshotAt) throw new Error(`local database was written after snapshot ${chosen.name} (local ${new Date(localAt).toISOString()} > snapshot ${new Date(snapshotAt).toISOString()}); refusing to overwrite. If those local writes matter, back them up/export them first; re-run with --force to overwrite anyway (a backup is kept).`)
|
|
114
|
+
}
|
|
70
115
|
}
|
|
71
116
|
let backup = null
|
|
72
117
|
if (existsSync(dbPath)) {
|
|
73
118
|
backup = `${dbPath}.before-sync-${Date.now()}`
|
|
74
119
|
copyFileSync(dbPath, backup)
|
|
120
|
+
for (const suffix of ['-wal', '-shm']) {
|
|
121
|
+
try { if (existsSync(dbPath + suffix)) copyFileSync(dbPath + suffix, backup + suffix) } catch {}
|
|
122
|
+
}
|
|
75
123
|
for (const suffix of ['-wal', '-shm']) { try { rmSync(dbPath + suffix) } catch {} }
|
|
76
124
|
} else {
|
|
77
125
|
mkdirSync(join(dbPath, '..'), { recursive: true, mode: 0o700 })
|
|
78
126
|
}
|
|
79
|
-
|
|
80
|
-
writeFileSync(tmp, bytes, { mode: 0o600 })
|
|
81
|
-
renameSync(tmp, dbPath)
|
|
82
|
-
try { chmodSync(dbPath, 0o600) } catch {}
|
|
127
|
+
writeAtomic(dbPath, bytes, { mode: 0o600 })
|
|
83
128
|
return { file: chosen.name, sha256, changed: true, backup, device: payload.device, createdAt: payload.createdAt }
|
|
84
129
|
}
|