advisers 0.2.0-preview.71 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +130 -201
- package/NOTICE +29 -4
- package/README-zh.md +35 -8
- package/README.md +37 -8
- package/THIRD-PARTY-NOTICES.md +277 -0
- package/bin/advisers.js +2264 -350
- package/index.js +1 -1
- package/lib/attachments.js +158 -29
- package/lib/capture.js +82 -2
- package/lib/evidence.js +1180 -26
- package/lib/ical.js +542 -0
- package/lib/imap.js +82 -24
- package/lib/mcp-server.js +869 -0
- package/lib/model.js +151 -2
- package/lib/notify.js +156 -4
- package/lib/ocr.js +26 -1
- package/lib/pdf.js +52 -6
- package/lib/pet-brief.js +78 -15
- package/lib/receipt-verify.js +644 -0
- package/lib/remind.js +261 -11
- package/lib/research.js +148 -18
- package/lib/safety/classifier.js +444 -0
- package/lib/safety/explain.js +157 -0
- package/lib/safety/index.js +333 -0
- package/lib/safety/redact.js +212 -0
- package/lib/safety/rules.js +824 -0
- package/lib/scheduler.js +399 -0
- package/lib/source.js +294 -0
- package/lib/sqlite.js +702 -42
- package/lib/sync.js +141 -20
- package/lib/timestamp.js +26 -1
- package/lib/tos/index.js +332 -0
- package/lib/tos/manifest.json +32 -0
- package/lib/tos/privacy.zh-CN.md +136 -0
- package/lib/tos/safety.zh-CN.md +101 -0
- package/lib/tos/summary.zh-CN.md +12 -0
- package/lib/tos/terms.zh-CN.md +439 -0
- package/package.json +10 -8
package/lib/sqlite.js
CHANGED
|
@@ -1,5 +1,375 @@
|
|
|
1
1
|
import { DatabaseSync } from 'node:sqlite'
|
|
2
|
+
import { chmodSync, closeSync, copyFileSync, fsyncSync, mkdirSync, openSync, readFileSync, rmSync, statSync, unlinkSync, utimesSync, writeSync } from 'node:fs'
|
|
2
3
|
import { createHash, createHmac, randomUUID } from 'node:crypto'
|
|
4
|
+
import { dirname, join } from 'node:path'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Open + write hardening (2026-11-17 robustness audit P0-1/P0-3/P1-2):
|
|
8
|
+
* - busy_timeout is set before any lock-taking pragma; journal_mode=WAL is
|
|
9
|
+
* idempotent and retried with bounded backoff while another process holds
|
|
10
|
+
* the database lock.
|
|
11
|
+
* - Opening no longer runs a full 'PRAGMA quick_check' (that is O(db) and made
|
|
12
|
+
* every command slow). Integrity checks live in assertIntegrity() and are
|
|
13
|
+
* run by 'advisers doctor' / 'advisers audit --check-db'.
|
|
14
|
+
* - appendAudit() and saveWithAudit() run in BEGIN IMMEDIATE transactions that
|
|
15
|
+
* re-read the previous hash before inserting, so the audit chain cannot fork
|
|
16
|
+
* under concurrent writers and a ledger write and its audit row commit
|
|
17
|
+
* together or not at all.
|
|
18
|
+
* - Migrations are refused for a schema newer than this CLI and are preceded
|
|
19
|
+
* by a same-directory backup of the old database.
|
|
20
|
+
*/
|
|
21
|
+
export const SCHEMA_VERSION = 18
|
|
22
|
+
const DEFAULT_BUSY_TIMEOUT_MS = 5000
|
|
23
|
+
const OPEN_RETRIES = 6
|
|
24
|
+
const BUSY_ERRCODES = new Set([5, 6]) // SQLITE_BUSY, SQLITE_LOCKED
|
|
25
|
+
|
|
26
|
+
function isBusyError(error) {
|
|
27
|
+
if (!error) return false
|
|
28
|
+
if (typeof error.errcode === 'number' && BUSY_ERRCODES.has(error.errcode)) return true
|
|
29
|
+
return /SQLITE_BUSY|SQLITE_LOCKED|database (?:table )?is locked/i.test(String(error.message ?? error))
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function sleepSync(ms) {
|
|
33
|
+
if (!ms || ms <= 0) return
|
|
34
|
+
try {
|
|
35
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
|
|
36
|
+
} catch {
|
|
37
|
+
const end = Date.now() + ms
|
|
38
|
+
while (Date.now() < end) {}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function backoffMs(attempt) {
|
|
43
|
+
return Math.min(2000, 50 * 2 ** attempt)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// Cross-process single-writer instance lock (0.2.1 robustness audit P0-1).
|
|
48
|
+
//
|
|
49
|
+
// The ledger snapshot save() path (see _saveRows below) writes whole rows, so
|
|
50
|
+
// two ordinary CLI commands that both load() and then save() can overwrite each
|
|
51
|
+
// other's committed updates (and resurrect soft-deleted rows) even while the
|
|
52
|
+
// SQLite page-level writer serialization keeps the file structurally intact.
|
|
53
|
+
// The audit's recommended first fix is one writer per HOME: every process that
|
|
54
|
+
// opens the SQLite store takes ~/.advisers/instance.lock (O_EXCL) first, waits a
|
|
55
|
+
// bounded time for a live holder, reclaims a provably stale lock, and releases
|
|
56
|
+
// on exit/SIGINT/SIGTERM.
|
|
57
|
+
//
|
|
58
|
+
// Rules that keep the takeover safe:
|
|
59
|
+
// - a lock whose pid is alive is never stolen while its heartbeat is fresh;
|
|
60
|
+
// long-lived holders (ui/mcp/bridge) refresh the lock mtime periodically.
|
|
61
|
+
// - a stale lock is removed only under an O_EXCL reclaim guard directory, so
|
|
62
|
+
// two waiters can never both delete and recreate the lock under each other.
|
|
63
|
+
// - release() only unlinks the lock when its pid + startedAt still match ours.
|
|
64
|
+
// ---------------------------------------------------------------------------
|
|
65
|
+
export const INSTANCE_LOCK_FILE = 'instance.lock'
|
|
66
|
+
const DEFAULT_LOCK_TIMEOUT_MS = 10_000
|
|
67
|
+
const DEFAULT_LOCK_STALE_MS = 60 * 60 * 1000
|
|
68
|
+
const MAX_LOCK_TIMEOUT_MS = 24 * 60 * 60 * 1000
|
|
69
|
+
const MAX_LOCK_STALE_MS = 30 * 24 * 60 * 60 * 1000
|
|
70
|
+
const LOCK_POLL_INTERVAL_MS = 50
|
|
71
|
+
const LOCK_RECLAIM_GUARD_MS = 10_000
|
|
72
|
+
|
|
73
|
+
/** The database exists but cannot be trusted (0 bytes / not a SQLite file). */
|
|
74
|
+
export class DatabaseCorruptError extends Error {
|
|
75
|
+
constructor(path, detail) {
|
|
76
|
+
super(
|
|
77
|
+
`E_DB_CORRUPT: ${path}: ${detail}. Refusing to rebuild or overwrite it. ` +
|
|
78
|
+
'Back up the file first (do not delete it), then run `advisers doctor`; ' +
|
|
79
|
+
'restore a known-good copy (state.db.before-*) if you have one.',
|
|
80
|
+
)
|
|
81
|
+
this.name = 'DatabaseCorruptError'
|
|
82
|
+
this.code = 'E_DB_CORRUPT'
|
|
83
|
+
this.path = path
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Another live advisers process holds the HOME's single-writer instance lock. */
|
|
88
|
+
export class DatabaseLockedError extends Error {
|
|
89
|
+
constructor({ lockPath, holder, waitedMs, timeoutMs }) {
|
|
90
|
+
const who = holder
|
|
91
|
+
? ` (pid ${holder.pid}, started ${holder.startedAt || 'unknown'}${holder.command ? `, "${holder.command}"` : ''})`
|
|
92
|
+
: ''
|
|
93
|
+
super(
|
|
94
|
+
`E_DB_LOCKED: another advisers process is using this home via ${lockPath}${who}; ` +
|
|
95
|
+
`waited ${waitedMs}ms (ADVISERS_LOCK_TIMEOUT_MS=${timeoutMs}). No data was written. ` +
|
|
96
|
+
'Stop the other process (ui/mcp/bridge servers hold the lock while running) and retry, ' +
|
|
97
|
+
'or raise ADVISERS_LOCK_TIMEOUT_MS to wait longer. If the holder is gone, remove the stale lock file.',
|
|
98
|
+
)
|
|
99
|
+
this.name = 'DatabaseLockedError'
|
|
100
|
+
this.code = 'E_DB_LOCKED'
|
|
101
|
+
this.lockPath = lockPath
|
|
102
|
+
this.holder = holder
|
|
103
|
+
this.waitedMs = waitedMs
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function readEnvMs(name, fallback, max) {
|
|
108
|
+
const raw = process.env[name]
|
|
109
|
+
if (raw === undefined || raw === null || String(raw).trim() === '') return fallback
|
|
110
|
+
const value = Number(raw)
|
|
111
|
+
if (!Number.isFinite(value) || value < 0) return fallback
|
|
112
|
+
return Math.min(Math.floor(value), max)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Wait budget for a contended instance lock (`ADVISERS_LOCK_TIMEOUT_MS`, default 10s). */
|
|
116
|
+
export function instanceLockTimeoutMs() {
|
|
117
|
+
return readEnvMs('ADVISERS_LOCK_TIMEOUT_MS', DEFAULT_LOCK_TIMEOUT_MS, MAX_LOCK_TIMEOUT_MS)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Age after which an unrefreshed lock may be reclaimed (`ADVISERS_LOCK_STALE_MS`, default 1h). */
|
|
121
|
+
export function instanceLockStaleMs() {
|
|
122
|
+
return readEnvMs('ADVISERS_LOCK_STALE_MS', DEFAULT_LOCK_STALE_MS, MAX_LOCK_STALE_MS)
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Absolute path of the per-HOME instance lock. */
|
|
126
|
+
export function instanceLockPath(home) {
|
|
127
|
+
return join(home, INSTANCE_LOCK_FILE)
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function isPidAlive(pid) {
|
|
131
|
+
if (!Number.isInteger(pid) || pid <= 0) return false
|
|
132
|
+
try {
|
|
133
|
+
process.kill(pid, 0)
|
|
134
|
+
return true
|
|
135
|
+
} catch (error) {
|
|
136
|
+
// EPERM: the pid exists but belongs to another user; never steal from it.
|
|
137
|
+
return error?.code === 'EPERM'
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Read + stat the instance lock without creating anything; null when absent. */
|
|
142
|
+
export function readInstanceLock(home) {
|
|
143
|
+
const file = instanceLockPath(home)
|
|
144
|
+
let stat
|
|
145
|
+
try {
|
|
146
|
+
stat = statSync(file)
|
|
147
|
+
} catch (error) {
|
|
148
|
+
if (error?.code === 'ENOENT') return null
|
|
149
|
+
throw error
|
|
150
|
+
}
|
|
151
|
+
let record = null
|
|
152
|
+
try {
|
|
153
|
+
const parsed = JSON.parse(readFileSync(file, 'utf8'))
|
|
154
|
+
if (parsed && typeof parsed === 'object' && Number.isFinite(Number(parsed.pid)) && Number(parsed.pid) > 0) {
|
|
155
|
+
record = {
|
|
156
|
+
pid: Math.trunc(Number(parsed.pid)),
|
|
157
|
+
startedAt: typeof parsed.startedAt === 'string' ? parsed.startedAt : '',
|
|
158
|
+
command: typeof parsed.command === 'string' ? parsed.command : '',
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
} catch {}
|
|
162
|
+
return { file, record, mtimeMs: stat.mtimeMs, size: stat.size }
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* A lock is stale when its owner pid is gone, or when it has not been refreshed
|
|
167
|
+
* (heartbeat / normal short command) for longer than `staleMs`. A live pid with
|
|
168
|
+
* a fresh mtime is never stolen — ui/mcp/bridge holders legitimately keep the
|
|
169
|
+
* lock for hours.
|
|
170
|
+
*/
|
|
171
|
+
export function instanceLockIsStale(info, { now = Date.now(), staleMs = instanceLockStaleMs() } = {}) {
|
|
172
|
+
if (!info) return false
|
|
173
|
+
if (info.record && !isPidAlive(info.record.pid)) return true
|
|
174
|
+
return now - info.mtimeMs > staleMs
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function heartbeatIntervalMs(staleMs) {
|
|
178
|
+
return Math.max(1000, Math.min(5 * 60_000, Math.floor(staleMs / 4)))
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Remove a stale lock while serialising concurrent reclaimers with a tty-less guard dir. */
|
|
182
|
+
function reclaimStaleLock({ path, expected, staleMs, timeoutMs, pollMs }) {
|
|
183
|
+
const guard = path + '.reclaim'
|
|
184
|
+
const guardDeadline = Date.now() + timeoutMs
|
|
185
|
+
for (;;) {
|
|
186
|
+
try {
|
|
187
|
+
mkdirSync(guard, { mode: 0o700 })
|
|
188
|
+
break
|
|
189
|
+
} catch (error) {
|
|
190
|
+
if (error?.code !== 'EEXIST') throw error
|
|
191
|
+
let age = 0
|
|
192
|
+
try { age = Date.now() - statSync(guard).mtimeMs } catch {}
|
|
193
|
+
if (age > LOCK_RECLAIM_GUARD_MS) {
|
|
194
|
+
try { rmSync(guard, { recursive: true, force: true }) } catch {}
|
|
195
|
+
continue
|
|
196
|
+
}
|
|
197
|
+
if (Date.now() >= guardDeadline) return false
|
|
198
|
+
sleepSync(pollMs)
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
try {
|
|
202
|
+
const current = readInstanceLock(dirname(path))
|
|
203
|
+
if (!current) return true
|
|
204
|
+
if (!instanceLockIsStale(current, { staleMs })) return false
|
|
205
|
+
// Only delete the exact record we inspected; a new holder may have replaced it.
|
|
206
|
+
const same = expected
|
|
207
|
+
? current.record && expected.record &&
|
|
208
|
+
current.record.pid === expected.record.pid &&
|
|
209
|
+
current.record.startedAt === expected.record.startedAt
|
|
210
|
+
: !current.record
|
|
211
|
+
if (!same) return false
|
|
212
|
+
try { unlinkSync(path) } catch (error) { if (error?.code !== 'ENOENT') throw error }
|
|
213
|
+
return true
|
|
214
|
+
} finally {
|
|
215
|
+
try { rmSync(guard, { recursive: true, force: true }) } catch {}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export class InstanceLock {
|
|
220
|
+
constructor({ home, command = '', timeoutMs = instanceLockTimeoutMs(), staleMs = instanceLockStaleMs(), pollMs = LOCK_POLL_INTERVAL_MS } = {}) {
|
|
221
|
+
this.home = home
|
|
222
|
+
this.path = instanceLockPath(home)
|
|
223
|
+
this.timeoutMs = Math.max(0, Math.floor(Number(timeoutMs) || 0))
|
|
224
|
+
this.staleMs = Math.max(1, Math.floor(Number(staleMs) || DEFAULT_LOCK_STALE_MS))
|
|
225
|
+
this.pollMs = Math.max(5, Math.floor(Number(pollMs) || LOCK_POLL_INTERVAL_MS))
|
|
226
|
+
this.record = { pid: process.pid, startedAt: new Date().toISOString(), command: String(command ?? '').slice(0, 160) }
|
|
227
|
+
this._held = false
|
|
228
|
+
this._heartbeat = null
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
get held() {
|
|
232
|
+
return this._held
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Try once to create the lock file (O_EXCL); returns true on success. */
|
|
236
|
+
_create() {
|
|
237
|
+
mkdirSync(this.home, { recursive: true, mode: 0o700 })
|
|
238
|
+
let fd
|
|
239
|
+
try {
|
|
240
|
+
fd = openSync(this.path, 'wx', 0o600)
|
|
241
|
+
} catch (error) {
|
|
242
|
+
if (error?.code === 'EEXIST') return false
|
|
243
|
+
throw error
|
|
244
|
+
}
|
|
245
|
+
try {
|
|
246
|
+
writeSync(fd, JSON.stringify(this.record) + '\n')
|
|
247
|
+
fsyncSync(fd)
|
|
248
|
+
} catch (error) {
|
|
249
|
+
// Never leave a half-written lock file behind: a reader would see invalid
|
|
250
|
+
// JSON and could only recover by waiting out the (1h) stale threshold.
|
|
251
|
+
try { closeSync(fd) } catch {}
|
|
252
|
+
try { unlinkSync(this.path) } catch {}
|
|
253
|
+
throw error
|
|
254
|
+
}
|
|
255
|
+
closeSync(fd)
|
|
256
|
+
try { chmodSync(this.path, 0o600) } catch {}
|
|
257
|
+
return true
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
acquire() {
|
|
261
|
+
const deadline = Date.now() + this.timeoutMs
|
|
262
|
+
const started = Date.now()
|
|
263
|
+
for (;;) {
|
|
264
|
+
if (this._create()) {
|
|
265
|
+
this._held = true
|
|
266
|
+
this._startHeartbeat()
|
|
267
|
+
return this
|
|
268
|
+
}
|
|
269
|
+
const info = readInstanceLock(this.home)
|
|
270
|
+
if (info && instanceLockIsStale(info, { staleMs: this.staleMs })) {
|
|
271
|
+
if (reclaimStaleLock({ path: this.path, expected: info, staleMs: this.staleMs, timeoutMs: this.timeoutMs, pollMs: this.pollMs })) continue
|
|
272
|
+
}
|
|
273
|
+
const now = Date.now()
|
|
274
|
+
if (now >= deadline) {
|
|
275
|
+
throw new DatabaseLockedError({ lockPath: this.path, holder: info?.record ?? null, waitedMs: now - started, timeoutMs: this.timeoutMs })
|
|
276
|
+
}
|
|
277
|
+
sleepSync(Math.max(1, Math.min(this.pollMs, deadline - now)))
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Refresh the lock mtime so long-lived holders are not age-reclaimed. */
|
|
282
|
+
refresh() {
|
|
283
|
+
if (!this._held) return false
|
|
284
|
+
const info = readInstanceLock(this.home)
|
|
285
|
+
if (!info?.record || info.record.pid !== this.record.pid || info.record.startedAt !== this.record.startedAt) {
|
|
286
|
+
this._stopHeartbeat()
|
|
287
|
+
this._held = false
|
|
288
|
+
return false
|
|
289
|
+
}
|
|
290
|
+
const now = new Date()
|
|
291
|
+
try { utimesSync(this.path, now, now) } catch { this._stopHeartbeat() }
|
|
292
|
+
return true
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
_startHeartbeat() {
|
|
296
|
+
if (this._heartbeat || this.staleMs < 60_000) return
|
|
297
|
+
this._heartbeat = setInterval(() => this.refresh(), heartbeatIntervalMs(this.staleMs))
|
|
298
|
+
if (typeof this._heartbeat.unref === 'function') this._heartbeat.unref()
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
_stopHeartbeat() {
|
|
302
|
+
if (!this._heartbeat) return
|
|
303
|
+
clearInterval(this._heartbeat)
|
|
304
|
+
this._heartbeat = null
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/** Release the lock; only the exact record this process wrote is removed. */
|
|
308
|
+
release() {
|
|
309
|
+
this._stopHeartbeat()
|
|
310
|
+
if (!this._held) return false
|
|
311
|
+
this._held = false
|
|
312
|
+
let info
|
|
313
|
+
try { info = readInstanceLock(this.home) } catch { return false }
|
|
314
|
+
if (!info?.record) return false
|
|
315
|
+
if (info.record.pid !== this.record.pid || info.record.startedAt !== this.record.startedAt) return false
|
|
316
|
+
try {
|
|
317
|
+
unlinkSync(this.path)
|
|
318
|
+
return true
|
|
319
|
+
} catch (error) {
|
|
320
|
+
return error?.code === 'ENOENT'
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/** Take the single-writer lock for a HOME; throws E_DB_LOCKED after the wait budget. */
|
|
326
|
+
export function acquireInstanceLock({ home, command = '', timeoutMs, staleMs, pollMs } = {}) {
|
|
327
|
+
if (!home) throw new Error('acquireInstanceLock requires a home directory')
|
|
328
|
+
const lock = new InstanceLock({ home, command, timeoutMs, staleMs, pollMs })
|
|
329
|
+
return lock.acquire()
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** Throw E_DB_CORRUPT for an existing 0-byte / non-file database before SQLite touches it. */
|
|
333
|
+
export function assertDatabaseFileUsable(path) {
|
|
334
|
+
if (!path || path === ':memory:') return
|
|
335
|
+
let stat
|
|
336
|
+
try {
|
|
337
|
+
stat = statSync(path)
|
|
338
|
+
} catch (error) {
|
|
339
|
+
if (error?.code === 'ENOENT') return
|
|
340
|
+
throw error
|
|
341
|
+
}
|
|
342
|
+
if (stat.isDirectory()) throw new DatabaseCorruptError(path, 'the path is a directory, not a SQLite database file')
|
|
343
|
+
if (stat.size === 0) throw new DatabaseCorruptError(path, 'the database file is 0 bytes (truncated, or an interrupted first-time initialisation)')
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function isCorruptSqliteError(error) {
|
|
347
|
+
return /file is not a database|database disk image is malformed|malformed database|not a database/i.test(String(error?.message ?? error))
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// Ledger tables are written as a whole snapshot by save(). The id column lets
|
|
351
|
+
// save() merge with rows other processes committed after our load().
|
|
352
|
+
const LEDGER_TABLES = [
|
|
353
|
+
['matters', 'id'],
|
|
354
|
+
['facts', 'id'],
|
|
355
|
+
['commitments', 'id'],
|
|
356
|
+
['preferences', 'id'],
|
|
357
|
+
['links', 'id'],
|
|
358
|
+
['updates', 'id'],
|
|
359
|
+
['routines', 'id'],
|
|
360
|
+
['routine_patches', 'id'],
|
|
361
|
+
['routes', 'key'],
|
|
362
|
+
]
|
|
363
|
+
|
|
364
|
+
function idsFor(snapshot) {
|
|
365
|
+
const ids = {}
|
|
366
|
+
for (const [table, key] of LEDGER_TABLES) {
|
|
367
|
+
const value = snapshot?.[table]
|
|
368
|
+
const rows = table === 'routes' ? Object.keys(value ?? {}) : (value ?? []).map((row) => row?.id)
|
|
369
|
+
ids[table] = new Set(rows.filter((id) => id != null))
|
|
370
|
+
}
|
|
371
|
+
return ids
|
|
372
|
+
}
|
|
3
373
|
|
|
4
374
|
const SCHEMA = `
|
|
5
375
|
CREATE TABLE IF NOT EXISTS matters (
|
|
@@ -17,7 +387,7 @@ CREATE TABLE IF NOT EXISTS facts (
|
|
|
17
387
|
CREATE TABLE IF NOT EXISTS commitments (
|
|
18
388
|
id TEXT PRIMARY KEY, matter_id TEXT NOT NULL, debtor TEXT NOT NULL, creditor TEXT NOT NULL,
|
|
19
389
|
action TEXT NOT NULL, due_at TEXT, timezone TEXT NOT NULL, status TEXT NOT NULL,
|
|
20
|
-
evidence TEXT NOT NULL, capability TEXT NOT NULL, done_at TEXT
|
|
390
|
+
evidence TEXT NOT NULL, capability TEXT NOT NULL, done_at TEXT, reminder_offsets TEXT
|
|
21
391
|
);
|
|
22
392
|
CREATE TABLE IF NOT EXISTS preferences (
|
|
23
393
|
id TEXT PRIMARY KEY, kind TEXT NOT NULL, statement TEXT NOT NULL, scope TEXT NOT NULL,
|
|
@@ -121,6 +491,30 @@ CREATE TABLE IF NOT EXISTS calendar_events (
|
|
|
121
491
|
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
122
492
|
`
|
|
123
493
|
|
|
494
|
+
/**
|
|
495
|
+
* P2-1: secondary indexes for the hot read/write paths (tick, today, history,
|
|
496
|
+
* deliveries). Indexes are layout only, not a schema change, so they are
|
|
497
|
+
* created with IF NOT EXISTS after migrate() on every open: new databases get
|
|
498
|
+
* them at creation time and existing databases get them the first time they
|
|
499
|
+
* are opened by this version. Each one was a full table SCAN + TEMP B-TREE in
|
|
500
|
+
* the 2026-11-17 performance audit (nudges/facts/deliveries/calendar_events/
|
|
501
|
+
* task_runs/sessions/model_calls/preferences/commitments).
|
|
502
|
+
*/
|
|
503
|
+
const INDEXES = `
|
|
504
|
+
CREATE INDEX IF NOT EXISTS idx_nudges_commitment_at ON nudges(commitment_id, at);
|
|
505
|
+
CREATE INDEX IF NOT EXISTS idx_nudges_local_day_level ON nudges(local_day, level);
|
|
506
|
+
CREATE INDEX IF NOT EXISTS idx_nudges_at ON nudges(at);
|
|
507
|
+
CREATE INDEX IF NOT EXISTS idx_commitments_status_due ON commitments(status, due_at);
|
|
508
|
+
CREATE INDEX IF NOT EXISTS idx_facts_matter ON facts(matter_id);
|
|
509
|
+
CREATE INDEX IF NOT EXISTS idx_facts_status_deleted ON facts(status, deleted_at);
|
|
510
|
+
CREATE INDEX IF NOT EXISTS idx_preferences_status_expires ON preferences(status, expires_at);
|
|
511
|
+
CREATE INDEX IF NOT EXISTS idx_calendar_events_starts ON calendar_events(starts_at);
|
|
512
|
+
CREATE INDEX IF NOT EXISTS idx_deliveries_created ON deliveries(created_at);
|
|
513
|
+
CREATE INDEX IF NOT EXISTS idx_task_runs_created ON task_runs(created_at);
|
|
514
|
+
CREATE INDEX IF NOT EXISTS idx_sessions_updated ON sessions(updated_at);
|
|
515
|
+
CREATE INDEX IF NOT EXISTS idx_model_calls_created ON model_calls(created_at);
|
|
516
|
+
`
|
|
517
|
+
|
|
124
518
|
const json = (v) => JSON.stringify(v ?? [])
|
|
125
519
|
const parse = (v, fallback = []) => {
|
|
126
520
|
if (v == null) return fallback
|
|
@@ -128,29 +522,153 @@ const parse = (v, fallback = []) => {
|
|
|
128
522
|
}
|
|
129
523
|
|
|
130
524
|
export class SqliteStore {
|
|
131
|
-
constructor(path, { readOnly = false } = {}) {
|
|
132
|
-
this.
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
525
|
+
constructor(path, { readOnly = false, checkIntegrity = false, busyTimeoutMs = Number(process.env.ADVISERS_SQLITE_BUSY_TIMEOUT_MS) || DEFAULT_BUSY_TIMEOUT_MS } = {}) {
|
|
526
|
+
this.path = path
|
|
527
|
+
this.busyTimeoutMs = busyTimeoutMs
|
|
528
|
+
this._transactionDepth = 0
|
|
529
|
+
this._baselineIds = null
|
|
530
|
+
// P2-2 (0.2.1): an existing 0-byte file is treated as corrupt instead of a
|
|
531
|
+
// brand-new database. Before this, SQLite turned it into an empty schema and
|
|
532
|
+
// every command reported "no data" while silently rebuilding the ledger.
|
|
533
|
+
assertDatabaseFileUsable(path)
|
|
534
|
+
try {
|
|
535
|
+
this.db = readOnly ? new DatabaseSync(path, { readOnly: true }) : new DatabaseSync(path)
|
|
536
|
+
// P0-1: sqlite only applies busy_timeout to statements that run after the
|
|
537
|
+
// pragma is set; concurrent first-opens used to fail on journal_mode=WAL
|
|
538
|
+
// because busy_timeout was set afterwards.
|
|
539
|
+
this.db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.floor(Number(busyTimeoutMs) || 0))}`)
|
|
540
|
+
if (readOnly) {
|
|
541
|
+
if (checkIntegrity) this.assertIntegrity()
|
|
542
|
+
return
|
|
543
|
+
}
|
|
544
|
+
this.enableWal()
|
|
545
|
+
this.db.exec('PRAGMA secure_delete = ON')
|
|
546
|
+
this.db.exec('PRAGMA synchronous = NORMAL')
|
|
547
|
+
this.initializeSchema()
|
|
548
|
+
} catch (error) {
|
|
549
|
+
try { this.db?.close() } catch {}
|
|
550
|
+
if (error?.code === 'E_DB_CORRUPT') throw error
|
|
551
|
+
if (isCorruptSqliteError(error)) {
|
|
552
|
+
throw new DatabaseCorruptError(path, `SQLite reports that the file is not a valid database (${String(error?.message ?? error).slice(0, 200)})`)
|
|
553
|
+
}
|
|
554
|
+
throw error
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/** Full-database integrity check; explicit because it is O(database size). */
|
|
559
|
+
assertIntegrity() {
|
|
560
|
+
const row = this.db.prepare('PRAGMA quick_check').get()
|
|
561
|
+
const verdict = row?.quick_check ?? 'unknown'
|
|
562
|
+
if (verdict !== 'ok') throw new Error(`database integrity check failed: ${verdict}`)
|
|
563
|
+
return verdict
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/** WAL is persistent; skip the switch when already enabled and retry on busy. */
|
|
567
|
+
enableWal() {
|
|
568
|
+
if (!this.path || this.path === ':memory:') return 'memory'
|
|
569
|
+
const current = String(this.db.prepare('PRAGMA journal_mode').get()?.journal_mode ?? '').toLowerCase()
|
|
570
|
+
if (current === 'wal' || current === 'memory') return current
|
|
571
|
+
let lastError
|
|
572
|
+
for (let attempt = 0; attempt <= OPEN_RETRIES; attempt++) {
|
|
573
|
+
try {
|
|
574
|
+
const result = this.db.prepare('PRAGMA journal_mode = WAL').get()
|
|
575
|
+
const mode = String(result?.journal_mode ?? '').toLowerCase()
|
|
576
|
+
if (mode === 'wal' || mode === 'memory') return mode
|
|
577
|
+
lastError = new Error(`could not enable WAL journal mode (got '${mode || 'unknown'}')`)
|
|
578
|
+
} catch (error) {
|
|
579
|
+
lastError = error
|
|
580
|
+
if (!isBusyError(error)) throw error
|
|
581
|
+
}
|
|
582
|
+
sleepSync(backoffMs(attempt))
|
|
583
|
+
}
|
|
584
|
+
throw lastError
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
initializeSchema() {
|
|
588
|
+
this._transaction('IMMEDIATE', () => {
|
|
589
|
+
this.db.exec(SCHEMA)
|
|
590
|
+
this.migrate()
|
|
591
|
+
// Indexes come after migrate(): some index the columns migrate() adds
|
|
592
|
+
// (e.g. facts.deleted_at), so an old database must be upgraded first.
|
|
593
|
+
this.db.exec(INDEXES)
|
|
594
|
+
})
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
/**
|
|
598
|
+
* Run fn() in a write transaction with bounded SQLITE_BUSY retries.
|
|
599
|
+
* Nested calls join the outer transaction (no nested BEGIN).
|
|
600
|
+
*/
|
|
601
|
+
_transaction(mode, fn, { retries = OPEN_RETRIES } = {}) {
|
|
602
|
+
if (this._transactionDepth > 0) return fn()
|
|
603
|
+
let lastError
|
|
604
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
605
|
+
let began = false
|
|
606
|
+
try {
|
|
607
|
+
this.db.exec(`BEGIN ${mode}`)
|
|
608
|
+
began = true
|
|
609
|
+
this._transactionDepth++
|
|
610
|
+
let result
|
|
611
|
+
try {
|
|
612
|
+
result = fn()
|
|
613
|
+
} finally {
|
|
614
|
+
this._transactionDepth--
|
|
615
|
+
}
|
|
616
|
+
this.db.exec('COMMIT')
|
|
617
|
+
return result
|
|
618
|
+
} catch (error) {
|
|
619
|
+
if (began) { try { this.db.exec('ROLLBACK') } catch {} }
|
|
620
|
+
lastError = error
|
|
621
|
+
if (!isBusyError(error) || attempt === retries) throw error
|
|
622
|
+
}
|
|
623
|
+
sleepSync(backoffMs(attempt))
|
|
138
624
|
}
|
|
139
|
-
|
|
140
|
-
this.db.exec('PRAGMA busy_timeout = 5000')
|
|
141
|
-
this.db.exec('PRAGMA secure_delete = ON')
|
|
142
|
-
this.db.exec('PRAGMA synchronous = NORMAL')
|
|
143
|
-
const check = this.db.prepare('PRAGMA quick_check').get()
|
|
144
|
-
if (check && check.quick_check !== 'ok') throw new Error(`database integrity check failed: ${check.quick_check}`)
|
|
145
|
-
this.db.exec(SCHEMA)
|
|
146
|
-
this.migrate()
|
|
625
|
+
throw lastError
|
|
147
626
|
}
|
|
148
627
|
|
|
628
|
+
/** Public transaction helper for multi-statement invariants (returns fn result). */
|
|
629
|
+
transaction(fn, { mode = 'IMMEDIATE' } = {}) {
|
|
630
|
+
return this._transaction(mode, fn)
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/**
|
|
634
|
+
* Checkpoint the WAL so a copied/sealed database file is self-contained.
|
|
635
|
+
* Returns busy=true when another connection prevented a full checkpoint.
|
|
636
|
+
*/
|
|
637
|
+
checkpoint({ mode = 'TRUNCATE', retries = OPEN_RETRIES } = {}) {
|
|
638
|
+
let last = null
|
|
639
|
+
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
640
|
+
last = this.db.prepare(`PRAGMA wal_checkpoint(${mode})`).get()
|
|
641
|
+
if (!last || Number(last.busy) === 0) {
|
|
642
|
+
return { busy: false, log: Number(last?.log ?? 0), checkpointed: Number(last?.checkpointed ?? 0) }
|
|
643
|
+
}
|
|
644
|
+
sleepSync(backoffMs(attempt))
|
|
645
|
+
}
|
|
646
|
+
return { busy: Number(last?.busy ?? 1) !== 0, log: Number(last?.log ?? 0), checkpointed: Number(last?.checkpointed ?? 0) }
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
backupBeforeMigrate(current) {
|
|
650
|
+
if (!this.path || this.path === ':memory:') return
|
|
651
|
+
const target = `${this.path}.before-migrate-v${current}-${Date.now()}`
|
|
652
|
+
try {
|
|
653
|
+
copyFileSync(this.path, target)
|
|
654
|
+
try { chmodSync(target, 0o600) } catch {}
|
|
655
|
+
} catch (error) {
|
|
656
|
+
throw new Error(`refusing to migrate without a backup of ${this.path}: ${error.message}`)
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
/**
|
|
661
|
+
* Bring an older database up to SCHEMA_VERSION. Runs inside the caller's
|
|
662
|
+
* write transaction (initializeSchema). Refuses futures schemas and keeps
|
|
663
|
+
* a same-directory backup before altering an existing database.
|
|
664
|
+
*/
|
|
149
665
|
migrate() {
|
|
150
666
|
const row = this.db.prepare('SELECT value FROM meta WHERE key = ?').get('schema_version')
|
|
151
667
|
const current = row ? Number(row.value) : 0
|
|
152
|
-
if (current
|
|
153
|
-
this
|
|
668
|
+
if (!Number.isFinite(current) || current < 0) throw new Error(`invalid schema_version in ${this.path}: ${row?.value}`)
|
|
669
|
+
if (current > SCHEMA_VERSION) throw new Error(`database schema v${current} is newer than this CLI (v${SCHEMA_VERSION}); upgrade advisers before opening ${this.path}`)
|
|
670
|
+
if (current >= SCHEMA_VERSION) return
|
|
671
|
+
if (current > 0) this.backupBeforeMigrate(current)
|
|
154
672
|
try {
|
|
155
673
|
this.db.exec(`CREATE TABLE IF NOT EXISTS seen_events (
|
|
156
674
|
link_id TEXT NOT NULL, account_id TEXT NOT NULL, dedupe_key TEXT NOT NULL,
|
|
@@ -196,6 +714,8 @@ export class SqliteStore {
|
|
|
196
714
|
try { this.db.exec("INSERT INTO turns_fts(turns_fts) VALUES('rebuild')") } catch {}
|
|
197
715
|
const commitmentCols = this.db.prepare('PRAGMA table_info(commitments)').all().map((c) => c.name)
|
|
198
716
|
if (!commitmentCols.includes('done_at')) this.db.exec('ALTER TABLE commitments ADD COLUMN done_at TEXT')
|
|
717
|
+
// 0.2.0: per-commitment custom reminder offsets (JSON array of strings).
|
|
718
|
+
if (!commitmentCols.includes('reminder_offsets')) this.db.exec('ALTER TABLE commitments ADD COLUMN reminder_offsets TEXT')
|
|
199
719
|
const factCols = this.db.prepare('PRAGMA table_info(facts)').all().map((c) => c.name)
|
|
200
720
|
for (const [name, type] of [['valid_from','TEXT'],['valid_to','TEXT'],['expired_at','TEXT'],['evidence','TEXT'],['supersedes','TEXT'],['superseded_by','TEXT'],['shadowed_by','TEXT'],['last_seen_at','TEXT'],['deleted_at','TEXT'],['deleted_reason','TEXT']]) {
|
|
201
721
|
if (!factCols.includes(name)) this.db.exec(`ALTER TABLE facts ADD COLUMN ${name} ${type}`)
|
|
@@ -237,10 +757,8 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
237
757
|
all_day INTEGER NOT NULL DEFAULT 0, tzid TEXT, location TEXT, rrule TEXT,
|
|
238
758
|
source TEXT, imported_at TEXT NOT NULL
|
|
239
759
|
)`)
|
|
240
|
-
this.db.prepare('INSERT OR REPLACE INTO meta(key,value) VALUES(?,?)').run('schema_version',
|
|
241
|
-
this.db.exec('COMMIT')
|
|
760
|
+
this.db.prepare('INSERT OR REPLACE INTO meta(key,value) VALUES(?,?)').run('schema_version', String(SCHEMA_VERSION))
|
|
242
761
|
} catch (error) {
|
|
243
|
-
this.db.exec('ROLLBACK')
|
|
244
762
|
throw error
|
|
245
763
|
}
|
|
246
764
|
}
|
|
@@ -323,6 +841,94 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
323
841
|
return this.db.prepare('SELECT COUNT(*) AS n FROM nudges WHERE local_day = ? AND level = ?').get(localDay, level).n
|
|
324
842
|
}
|
|
325
843
|
|
|
844
|
+
/**
|
|
845
|
+
* P0-2 (tick N+1 removal): insert every nudge for one tick in a single
|
|
846
|
+
* transaction with one prepared statement. INSERT OR IGNORE keeps the
|
|
847
|
+
* per-(local_day, level, commitment_id) dedupe key semantics of addNudge()
|
|
848
|
+
* without aborting the batch on an expected duplicate. Returns the number of
|
|
849
|
+
* rows actually inserted (existing rows are left untouched).
|
|
850
|
+
*/
|
|
851
|
+
addNudges(rows) {
|
|
852
|
+
if (!rows?.length) return 0
|
|
853
|
+
return this._transaction('IMMEDIATE', () => {
|
|
854
|
+
const statement = this.db.prepare('INSERT OR IGNORE INTO nudges(id,at,local_day,level,commitment_id,dedupe_key,body) VALUES(?,?,?,?,?,?,?)')
|
|
855
|
+
const now = new Date().toISOString()
|
|
856
|
+
let inserted = 0
|
|
857
|
+
for (const row of rows) {
|
|
858
|
+
const key = `${row.localDay}:${row.level}:${row.commitmentId}`
|
|
859
|
+
const id = `nudge_${randomUUID().replace(/-/g, '').slice(0, 16)}`
|
|
860
|
+
inserted += Number(statement.run(id, row.at ?? now, row.localDay, row.level, row.commitmentId, key, row.body).changes)
|
|
861
|
+
}
|
|
862
|
+
return inserted
|
|
863
|
+
})
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
/**
|
|
867
|
+
* P0-2 (tick N+1 removal): the newest `limit` outcomes per commitment.
|
|
868
|
+
*
|
|
869
|
+
* P2-2 (0.2.1): the original form windowed over the WHOLE nudges table
|
|
870
|
+
* (scan + temp b-tree sort), so every tick paid for the entire nudge history:
|
|
871
|
+
* 271ms at 200k rows and growing linearly. Tick only needs outcomes for the
|
|
872
|
+
* commitments it is about to nudge, so callers that know that set pass
|
|
873
|
+
* `commitmentIds`; the read then becomes `limit` index seeks per commitment on
|
|
874
|
+
* idx_nudges_commitment_at, bounded by the candidate set instead of the
|
|
875
|
+
* history size. Passing no ids keeps the historical whole-table map for
|
|
876
|
+
* callers/tests that need it.
|
|
877
|
+
*/
|
|
878
|
+
recentNudgeOutcomes(limit = 3, commitmentIds = null) {
|
|
879
|
+
const newest = Math.max(1, Math.floor(Number(limit) || 3))
|
|
880
|
+
const byCommitment = new Map()
|
|
881
|
+
if (commitmentIds) {
|
|
882
|
+
const ids = [...new Set(commitmentIds)]
|
|
883
|
+
if (!ids.length) return byCommitment
|
|
884
|
+
const statement = this.db.prepare('SELECT outcome FROM nudges WHERE commitment_id = ? ORDER BY at DESC LIMIT ?')
|
|
885
|
+
for (const id of ids) {
|
|
886
|
+
const rows = statement.all(id, newest)
|
|
887
|
+
if (rows.length) byCommitment.set(id, rows.map((row) => row.outcome))
|
|
888
|
+
}
|
|
889
|
+
return byCommitment
|
|
890
|
+
}
|
|
891
|
+
const rows = this.db.prepare(`SELECT commitment_id, outcome, at FROM (
|
|
892
|
+
SELECT commitment_id, outcome, at, ROW_NUMBER() OVER (PARTITION BY commitment_id ORDER BY at DESC) AS rn
|
|
893
|
+
FROM nudges
|
|
894
|
+
) WHERE rn <= ?`).all(newest)
|
|
895
|
+
for (const row of rows) {
|
|
896
|
+
const list = byCommitment.get(row.commitment_id)
|
|
897
|
+
if (list) list.push(row.outcome)
|
|
898
|
+
else byCommitment.set(row.commitment_id, [row.outcome])
|
|
899
|
+
}
|
|
900
|
+
return byCommitment
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
/** P0-2 (tick N+1 removal): today's nudge count per level in one query. */
|
|
904
|
+
countNudgesByDay(localDay) {
|
|
905
|
+
const rows = this.db.prepare('SELECT level, COUNT(*) AS n FROM nudges WHERE local_day = ? GROUP BY level').all(localDay)
|
|
906
|
+
return new Map(rows.map((row) => [row.level, Number(row.n)]))
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
/** P0-2 (tick N+1 removal): existing (commitment, level) nudge keys for a day. */
|
|
910
|
+
listNudgeLevelKeysByDay(localDay) {
|
|
911
|
+
const rows = this.db.prepare('SELECT commitment_id, level FROM nudges WHERE local_day = ?').all(localDay)
|
|
912
|
+
return new Set(rows.map((row) => `${row.commitment_id}\u0000${row.level}`))
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
/** P0-2 (tick N+1 removal): all meta rows with a prefix in one query (snooze:*). */
|
|
916
|
+
listMetaByPrefix(prefix) {
|
|
917
|
+
const rows = this.db.prepare('SELECT key, value FROM meta WHERE key LIKE ?').all(`${prefix}%`)
|
|
918
|
+
return new Map(rows.map((row) => [row.key, row.value]))
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
/** P0-2: delete a batch of meta keys in one transaction. */
|
|
922
|
+
deleteMetaKeys(keys) {
|
|
923
|
+
if (!keys?.length) return 0
|
|
924
|
+
return this._transaction('IMMEDIATE', () => {
|
|
925
|
+
const statement = this.db.prepare('DELETE FROM meta WHERE key = ?')
|
|
926
|
+
let deleted = 0
|
|
927
|
+
for (const key of keys) deleted += Number(statement.run(key).changes)
|
|
928
|
+
return deleted
|
|
929
|
+
})
|
|
930
|
+
}
|
|
931
|
+
|
|
326
932
|
listNudges(limit = 20) {
|
|
327
933
|
return this.db.prepare('SELECT * FROM nudges ORDER BY at DESC LIMIT ?').all(limit)
|
|
328
934
|
}
|
|
@@ -528,7 +1134,16 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
528
1134
|
return this.db.prepare('SELECT COUNT(*) AS n FROM deliveries').get().n
|
|
529
1135
|
}
|
|
530
1136
|
|
|
1137
|
+
/**
|
|
1138
|
+
* Append an audit row atomically. BEGIN IMMEDIATE serializes writers and the
|
|
1139
|
+
* previous hash is re-read inside the transaction, so two concurrent writers
|
|
1140
|
+
* can never both chain onto the same predecessor (audit fork, P0-3).
|
|
1141
|
+
*/
|
|
531
1142
|
appendAudit({ actor, action, allowed, reason }) {
|
|
1143
|
+
return this._transaction('IMMEDIATE', () => this._appendAuditRow({ actor, action, allowed, reason }))
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
_appendAuditRow({ actor, action, allowed, reason }) {
|
|
532
1147
|
const last = this.db.prepare('SELECT hash FROM audit ORDER BY seq DESC LIMIT 1').get()
|
|
533
1148
|
const prev = last?.hash ?? ''
|
|
534
1149
|
const at = new Date().toISOString()
|
|
@@ -591,7 +1206,7 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
591
1206
|
|
|
592
1207
|
load() {
|
|
593
1208
|
const all = (sql) => this.db.prepare(sql).all()
|
|
594
|
-
|
|
1209
|
+
const snapshot = {
|
|
595
1210
|
version: 1,
|
|
596
1211
|
matters: all('SELECT * FROM matters').map((r) => ({
|
|
597
1212
|
id: r.id, title: r.title, status: r.status, domain: r.domain, secrecy: r.secrecy,
|
|
@@ -609,11 +1224,19 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
609
1224
|
shadowedBy: r.shadowed_by ?? undefined, lastSeenAt: r.last_seen_at ?? undefined,
|
|
610
1225
|
deletedAt: r.deleted_at ?? undefined, deletedReason: r.deleted_reason ?? undefined,
|
|
611
1226
|
})),
|
|
612
|
-
commitments: all('SELECT * FROM commitments').map((r) =>
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
1227
|
+
commitments: all('SELECT * FROM commitments').map((r) => {
|
|
1228
|
+
// NULL means "never customised": keep the field absent so old snapshots
|
|
1229
|
+
// and the pre-0.2.0 commitment shape stay identical. Corrupt or
|
|
1230
|
+
// non-array JSON degrades to absent rather than breaking the ledger.
|
|
1231
|
+
const parsedOffsets = r.reminder_offsets == null ? null : parse(r.reminder_offsets, null)
|
|
1232
|
+
const reminderOffsets = Array.isArray(parsedOffsets) ? parsedOffsets : undefined
|
|
1233
|
+
return {
|
|
1234
|
+
id: r.id, matterId: r.matter_id, debtor: r.debtor, creditor: r.creditor, action: r.action,
|
|
1235
|
+
dueAt: r.due_at ?? undefined, timezone: r.timezone, status: r.status,
|
|
1236
|
+
evidence: parse(r.evidence), capability: r.capability, doneAt: r.done_at ?? undefined,
|
|
1237
|
+
...(reminderOffsets === undefined ? {} : { reminderOffsets }),
|
|
1238
|
+
}
|
|
1239
|
+
}),
|
|
617
1240
|
preferences: all('SELECT * FROM preferences').map((r) => ({
|
|
618
1241
|
id: r.id, kind: r.kind, statement: r.statement, scope: parse(r.scope, {}),
|
|
619
1242
|
source: parse(r.source, {}), evidence: parse(r.evidence), confidence: r.confidence,
|
|
@@ -641,53 +1264,90 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
|
|
641
1264
|
})),
|
|
642
1265
|
routes: Object.fromEntries(all('SELECT * FROM routes').map((r) => [r.key, r.matter_id])),
|
|
643
1266
|
}
|
|
1267
|
+
// Remember which rows this connection has seen (P1-1): save() only deletes
|
|
1268
|
+
// rows this loader owned, so another process's concurrent additions are
|
|
1269
|
+
// merged instead of being silently overwritten by a stale snapshot.
|
|
1270
|
+
this._baselineIds = idsFor(snapshot)
|
|
1271
|
+
return snapshot
|
|
644
1272
|
}
|
|
645
1273
|
|
|
646
1274
|
save(snapshot) {
|
|
1275
|
+
return this._transaction('IMMEDIATE', () => this._saveRows(snapshot))
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
/**
|
|
1279
|
+
* Write the ledger snapshot and its audit row in one transaction, so the
|
|
1280
|
+
* audit trail can never claim a write that rolled back (or miss one that
|
|
1281
|
+
* committed). `inside` runs inside the same transaction before the audit row.
|
|
1282
|
+
*/
|
|
1283
|
+
saveWithAudit(snapshot, audit, inside) {
|
|
1284
|
+
return this._transaction('IMMEDIATE', () => {
|
|
1285
|
+
this._saveRows(snapshot)
|
|
1286
|
+
if (typeof inside === 'function') inside(this)
|
|
1287
|
+
return this._appendAuditRow(audit)
|
|
1288
|
+
})
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
_saveRows(snapshot) {
|
|
647
1292
|
const db = this.db
|
|
648
|
-
|
|
1293
|
+
const newIds = idsFor(snapshot)
|
|
1294
|
+
const baseline = this._baselineIds
|
|
649
1295
|
try {
|
|
650
|
-
|
|
651
|
-
|
|
1296
|
+
if (baseline) {
|
|
1297
|
+
// P1-1: delete only rows this connection loaded that the new snapshot
|
|
1298
|
+
// dropped; rows another process added after our load() are kept.
|
|
1299
|
+
for (const [table, key] of LEDGER_TABLES) {
|
|
1300
|
+
const removed = [...(baseline[table] ?? [])].filter((id) => !newIds[table].has(id))
|
|
1301
|
+
if (!removed.length) continue
|
|
1302
|
+
const statement = db.prepare(`DELETE FROM ${table} WHERE ${key} = ?`)
|
|
1303
|
+
for (const id of removed) statement.run(id)
|
|
1304
|
+
}
|
|
1305
|
+
} else {
|
|
1306
|
+
// No prior load(): keep the historical full-replace semantics for
|
|
1307
|
+
// direct library use (e.g. importing or seeding a ledger).
|
|
1308
|
+
for (const [table] of LEDGER_TABLES) db.exec(`DELETE FROM ${table}`)
|
|
652
1309
|
}
|
|
653
|
-
const m = db.prepare('INSERT INTO matters VALUES (?,?,?,?,?,?,?,?,?,?,?)')
|
|
1310
|
+
const m = db.prepare('INSERT OR REPLACE INTO matters VALUES (?,?,?,?,?,?,?,?,?,?,?)')
|
|
654
1311
|
for (const x of snapshot.matters ?? []) {
|
|
655
1312
|
m.run(x.id, x.title, x.status, x.domain, x.secrecy, x.summary ?? null, json(x.participants), json(x.deadlines), json(x.openQuestions), x.createdAt, x.updatedAt)
|
|
656
1313
|
}
|
|
657
|
-
const f = db.prepare(`INSERT INTO facts(id,matter_id,subject,predicate,object,recorded_at,expired_at,authority,confidence,source,evidence,status,valid_from,valid_to,supersedes,superseded_by,shadowed_by,last_seen_at,deleted_at,deleted_reason)
|
|
1314
|
+
const f = db.prepare(`INSERT OR REPLACE INTO facts(id,matter_id,subject,predicate,object,recorded_at,expired_at,authority,confidence,source,evidence,status,valid_from,valid_to,supersedes,superseded_by,shadowed_by,last_seen_at,deleted_at,deleted_reason)
|
|
658
1315
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
|
|
659
1316
|
for (const x of snapshot.facts ?? []) {
|
|
660
1317
|
f.run(x.id, x.matterId ?? null, x.subject, x.predicate, x.object, x.recordedAt, x.expiredAt ?? null, x.authority, x.confidence, x.source ? json(x.source) : null, x.evidence ? json(x.evidence) : null, x.status, x.validFrom ?? null, x.validTo ?? null, x.supersedes ?? null, x.supersededBy ?? null, x.shadowedBy ?? null, x.lastSeenAt ?? null, x.deletedAt ?? null, x.deletedReason ?? null)
|
|
661
1318
|
}
|
|
662
|
-
const c = db.prepare(
|
|
1319
|
+
const c = db.prepare(`INSERT OR REPLACE INTO commitments(id,matter_id,debtor,creditor,action,due_at,timezone,status,evidence,capability,done_at,reminder_offsets)
|
|
1320
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)`)
|
|
663
1321
|
for (const x of snapshot.commitments ?? []) {
|
|
664
|
-
|
|
1322
|
+
// A missing/undefined list stays NULL so a round-trip keeps the pre-0.2.0
|
|
1323
|
+
// "no field" shape; an explicit [] is preserved as its own value.
|
|
1324
|
+
const reminderOffsets = Array.isArray(x.reminderOffsets) ? JSON.stringify(x.reminderOffsets) : null
|
|
1325
|
+
c.run(x.id, x.matterId, x.debtor, x.creditor, x.action, x.dueAt ?? null, x.timezone, x.status, json(x.evidence), x.capability, x.doneAt ?? null, reminderOffsets)
|
|
665
1326
|
}
|
|
666
|
-
const p = db.prepare('INSERT INTO preferences VALUES (?,?,?,?,?,?,?,?,?,?,?,?)')
|
|
1327
|
+
const p = db.prepare('INSERT OR REPLACE INTO preferences VALUES (?,?,?,?,?,?,?,?,?,?,?,?)')
|
|
667
1328
|
for (const x of snapshot.preferences ?? []) {
|
|
668
1329
|
p.run(x.id, x.kind, x.statement, json(x.scope), json(x.source), json(x.evidence), x.confidence, x.status, x.validFrom, x.lastConfirmed ?? null, x.expiresAt ?? null, x.priority)
|
|
669
1330
|
}
|
|
670
|
-
const l = db.prepare('INSERT INTO links VALUES (?,?,?,?,?,?,?,?,?)')
|
|
1331
|
+
const l = db.prepare('INSERT OR REPLACE INTO links VALUES (?,?,?,?,?,?,?,?,?)')
|
|
671
1332
|
for (const x of snapshot.links ?? []) {
|
|
672
1333
|
l.run(x.id, x.fromMatter, x.toMatter, x.type, x.confidence, json(x.evidence), x.status, x.createdBy, x.createdAt)
|
|
673
1334
|
}
|
|
674
|
-
const u = db.prepare('INSERT INTO updates VALUES (?,?,?,?,?,?,?)')
|
|
1335
|
+
const u = db.prepare('INSERT OR REPLACE INTO updates VALUES (?,?,?,?,?,?,?)')
|
|
675
1336
|
for (const x of snapshot.updates ?? []) {
|
|
676
1337
|
u.run(x.id, x.sourceMatter, x.targetMatter, x.change, x.status, x.createdAt, x.rollback ?? null)
|
|
677
1338
|
}
|
|
678
|
-
const r = db.prepare('INSERT INTO routines VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)')
|
|
1339
|
+
const r = db.prepare('INSERT OR REPLACE INTO routines VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)')
|
|
679
1340
|
for (const x of snapshot.routines ?? []) {
|
|
680
1341
|
r.run(x.id, x.name, x.description, json(x.trigger), json(x.steps), json(x.preferences), json(x.pitfalls), json(x.verification), x.scope ?? null, x.version, x.status, x.createdBy, x.createdAt, x.updatedAt)
|
|
681
1342
|
}
|
|
682
|
-
const rp = db.prepare('INSERT INTO routine_patches VALUES (?,?,?,?,?,?,?,?)')
|
|
1343
|
+
const rp = db.prepare('INSERT OR REPLACE INTO routine_patches VALUES (?,?,?,?,?,?,?,?)')
|
|
683
1344
|
for (const x of snapshot.routinePatches ?? []) {
|
|
684
1345
|
rp.run(x.id, x.routineId, x.baseVersion, json(x.changes), json(x.before), x.diff, x.status, x.createdAt)
|
|
685
1346
|
}
|
|
686
|
-
const rt = db.prepare('INSERT INTO routes VALUES (?,?)')
|
|
1347
|
+
const rt = db.prepare('INSERT OR REPLACE INTO routes VALUES (?,?)')
|
|
687
1348
|
for (const [key, matterId] of Object.entries(snapshot.routes ?? {})) rt.run(key, matterId)
|
|
688
|
-
|
|
1349
|
+
this._baselineIds = newIds
|
|
689
1350
|
} catch (error) {
|
|
690
|
-
db.exec('ROLLBACK')
|
|
691
1351
|
throw error
|
|
692
1352
|
}
|
|
693
1353
|
}
|