advisers 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README-zh.md +13 -14
  2. package/README.md +32 -23
  3. package/THIRD-PARTY-NOTICES.md +1 -1
  4. package/bin/advisers.js +2180 -92
  5. package/index.js +1 -1
  6. package/lib/brief-archive.js +267 -0
  7. package/lib/client-pack.js +1016 -0
  8. package/lib/consent.js +390 -0
  9. package/lib/credentials.js +195 -0
  10. package/lib/error-codes.js +516 -7
  11. package/lib/evidence.js +697 -113
  12. package/lib/kb-extract.js +626 -0
  13. package/lib/kb-facts.js +429 -0
  14. package/lib/kb-pages.js +1084 -0
  15. package/lib/kb.js +1036 -0
  16. package/lib/mail-errors.js +46 -0
  17. package/lib/mail-extract.js +351 -0
  18. package/lib/mail-imap.js +446 -0
  19. package/lib/mail-import.js +176 -0
  20. package/lib/mail-parse.js +223 -0
  21. package/lib/mail-store.js +731 -0
  22. package/lib/mail.js +506 -0
  23. package/lib/mcp-server.js +196 -10
  24. package/lib/notify.js +265 -7
  25. package/lib/policy/decision.js +315 -0
  26. package/lib/policy/egress.js +268 -0
  27. package/lib/policy/errors.js +71 -0
  28. package/lib/policy/index.js +99 -0
  29. package/lib/policy/rules.js +468 -0
  30. package/lib/policy/surfaces.js +141 -0
  31. package/lib/receipt-verify.js +199 -19
  32. package/lib/receivables.js +1653 -0
  33. package/lib/scheduler.js +453 -9
  34. package/lib/shell.js +1406 -0
  35. package/lib/source.js +98 -3
  36. package/lib/sqlite.js +366 -49
  37. package/lib/tos/index.js +2 -2
  38. package/lib/tos/manifest.json +6 -6
  39. package/lib/tos/privacy.zh-CN.md +33 -20
  40. package/lib/tos/safety.zh-CN.md +7 -6
  41. package/lib/tos/summary.zh-CN.md +3 -2
  42. package/lib/tos/terms.zh-CN.md +31 -16
  43. package/lib/urgency.js +467 -0
  44. package/package.json +4 -4
@@ -0,0 +1,731 @@
1
+ /**
2
+ * SQLite persistence for the 0.4.0 mail entry (schema v22 tables owned by
3
+ * lib/sqlite.js: mail_accounts / mail_cursors / mail_messages /
4
+ * mail_sync_runs). This module never creates tables and never touches
5
+ * credentials; it only reads/writes the mail rows through an open
6
+ * SqliteStore (use store.mutate()/transaction() for the write helpers that
7
+ * are documented as transaction-internal).
8
+ *
9
+ * Invariants:
10
+ * - a message row keeps `body_text` only within MAIL_MAX_BODY_BYTES; clearing
11
+ * it (retention / purge) keeps the metadata, body_sha256 and dedupe key, so
12
+ * re-syncing the mailbox can never recreate the row;
13
+ * - dedupe key: Message-ID first, then (mailbox, UIDVALIDITY, UID), then a
14
+ * canonical content hash for .eml imports without a Message-ID. The body
15
+ * hash is NOT a dedupe key (signature/client differences would drift);
16
+ * - attachments are metadata only: [{ name, size, mime }]; no attachment
17
+ * bytes are stored anywhere under ADVISERS_HOME;
18
+ * - a sync batch writes messages + cursor + account last_sync_at + the run row
19
+ * in ONE BEGIN IMMEDIATE (see applyMailSyncBatch); the cursor only advances
20
+ * past successful rows, never past a failed UID.
21
+ */
22
+ import { createHash, randomBytes } from 'node:crypto'
23
+ import { mailError, MAIL_ERROR_CODES } from './mail-errors.js'
24
+
25
+ export const MAIL_DEFAULT_RETENTION_DAYS = 30
26
+ export const MAIL_MAX_RETENTION_DAYS = 3650
27
+ export const MAIL_DEFAULT_SYNC_LIMIT = 50
28
+ export const MAIL_MAX_SYNC_LIMIT = 200
29
+ export const MAIL_DEFAULT_LS_LIMIT = 50
30
+ export const MAIL_MAX_LS_LIMIT = 500
31
+ export const MAIL_MAX_BODY_BYTES = 256 * 1024
32
+ export const MAIL_MAX_MESSAGE_BYTES = 5 * 1024 * 1024
33
+ export const MAIL_MAX_ATTACHMENTS = 50
34
+ export const MAIL_TLS_MODES = Object.freeze(['tls', 'starttls'])
35
+ export const MAIL_ACCOUNT_STATUSES = Object.freeze(['active', 'removed'])
36
+ export const MAIL_MESSAGE_STATUSES = Object.freeze(['new', 'extracted', 'linked', 'dismissed', 'oversize', 'metadata'])
37
+ export const MAIL_LOCAL_ACCOUNT_ID = 'acct_local_eml'
38
+ export const MAIL_LOCAL_MAILBOX = 'local'
39
+ export const MAIL_MAX_ACCOUNT_NAME_CHARS = 120
40
+ export const MAIL_MAX_SUBJECT_CHARS = 500
41
+ export const MAIL_MAX_FROM_CHARS = 320
42
+
43
+ const REQUIRED_TABLES = ['mail_accounts', 'mail_cursors', 'mail_messages', 'mail_sync_runs']
44
+ const DAY_MS = 86_400_000
45
+ const ABSOLUTE_PATH_RE = /^(?:[A-Za-z]:[\\/]|\/|\\\\)/
46
+
47
+ function nowDate(now) {
48
+ const value = now instanceof Date ? now : new Date(now ?? Date.now())
49
+ if (!Number.isFinite(value.getTime())) throw mailError(MAIL_ERROR_CODES.CONFIG, 'invalid clock value')
50
+ return value
51
+ }
52
+
53
+ export function isoOf(now) {
54
+ return nowDate(now).toISOString()
55
+ }
56
+
57
+ function normalizeIso(value, field) {
58
+ if (value == null || value === '') return null
59
+ const parsed = value instanceof Date ? value : new Date(value)
60
+ if (!Number.isFinite(parsed.getTime())) throw mailError(MAIL_ERROR_CODES.CONFIG, field + ' is not a valid date')
61
+ return parsed.toISOString()
62
+ }
63
+
64
+ export function parseJson(value, fallback) {
65
+ if (value == null) return fallback
66
+ try {
67
+ const parsed = JSON.parse(value)
68
+ return parsed === null ? fallback : parsed
69
+ } catch { return fallback }
70
+ }
71
+
72
+ function sha256(text) {
73
+ return createHash('sha256').update(text, 'utf8').digest('hex')
74
+ }
75
+
76
+ function randomId(prefix) {
77
+ return prefix + randomBytes(8).toString('hex')
78
+ }
79
+
80
+ function singleLine(value, max) {
81
+ const text = String(value == null ? '' : value).replace(/[\u0000-\u001f\u007f]+/g, ' ').replace(/\s+/g, ' ').trim()
82
+ return text.length > max ? text.slice(0, max) : text
83
+ }
84
+
85
+ function normalizeMessageId(value) {
86
+ let text = String(value == null ? '' : value).trim()
87
+ if (!text) return ''
88
+ const wrapped = /^<([^>]+)>$/.exec(text)
89
+ if (wrapped) text = wrapped[1].trim()
90
+ if (!text || /\s/.test(text) || text.length > 512) return ''
91
+ return text
92
+ }
93
+
94
+ function canonicalText(value) {
95
+ return String(value == null ? '' : value)
96
+ .replace(/\r\n/g, '\n').replace(/\r/g, '\n')
97
+ .replace(/[ \t]+\n/g, '\n')
98
+ .replace(/\n{3,}/g, '\n\n')
99
+ .trim()
100
+ }
101
+
102
+ function truncateUtf8(text, maxBytes) {
103
+ const value = String(text == null ? '' : text)
104
+ if (Buffer.byteLength(value, 'utf8') <= maxBytes) return value
105
+ let low = 0
106
+ let high = value.length
107
+ while (low < high) {
108
+ const middle = Math.ceil((low + high) / 2)
109
+ if (Buffer.byteLength(value.slice(0, middle), 'utf8') <= maxBytes - 64) low = middle
110
+ else high = middle - 1
111
+ }
112
+ return value.slice(0, low) + '\n[body truncated by advisers]'
113
+ }
114
+
115
+ /** Fail with a stable code when the v22 mail tables are missing. */
116
+ export function assertMailTables(store) {
117
+ if (!store || typeof store.db?.prepare !== 'function') {
118
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail commands require the SQLite ledger (JSON store is not supported)')
119
+ }
120
+ let rows
121
+ try {
122
+ rows = store.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all()
123
+ } catch (error) {
124
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'the ledger database is not readable: ' + String(error?.message ?? error).slice(0, 200))
125
+ }
126
+ const present = new Set(rows.map((row) => row.name))
127
+ const missing = REQUIRED_TABLES.filter((name) => !present.has(name))
128
+ if (missing.length) {
129
+ throw mailError('E_SCHEMA_STALE', 'mail requires the v22 schema but this ledger is missing: ' + missing.join(', '), 'upgrade advisers and run `advisers init` to migrate the ledger')
130
+ }
131
+ }
132
+
133
+ function normalizeAttachments(attachments) {
134
+ if (!Array.isArray(attachments)) return []
135
+ const out = []
136
+ for (const item of attachments.slice(0, 500)) {
137
+ if (out.length >= MAIL_MAX_ATTACHMENTS) break
138
+ if (!item || typeof item !== 'object') continue
139
+ const name = singleLine(item.name, 120)
140
+ const size = Number(item.size)
141
+ const mime = singleLine(item.mime, 120)
142
+ if (!name) continue
143
+ out.push({ name, size: Number.isFinite(size) && size >= 0 ? Math.floor(size) : 0, mime: mime || 'application/octet-stream' })
144
+ }
145
+ return out
146
+ }
147
+
148
+ /* ------------------------------------------------------------------ */
149
+ /* Accounts */
150
+ /* ------------------------------------------------------------------ */
151
+
152
+ function validateAccountText(value, field, max) {
153
+ if (typeof value !== 'string') throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account ' + field + ' is required')
154
+ const text = singleLine(value, max + 1)
155
+ if (!text || text.length > max) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account ' + field + ' must be 1-' + max + ' characters')
156
+ return text
157
+ }
158
+
159
+ /**
160
+ * Validate and normalise a mail account config. Rejects cleartext IMAP
161
+ * (`secure: false` / `tls: 'none'`) with E_MAIL_CONFIG: RFC 8314 removed
162
+ * cleartext mail access and this module has no plaintext product path.
163
+ */
164
+ export function prepareMailAccount(config = {}, { now, id } = {}) {
165
+ if (!config || typeof config !== 'object') throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account config is required')
166
+ if (config.secure === false || config.secure === 0 || config.insecure === true) {
167
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'cleartext IMAP is not supported; mail sync requires implicit TLS (993) or STARTTLS', 'drop --insecure/secure=0 and use --port 993')
168
+ }
169
+ const name = validateAccountText(config.name, 'name', MAIL_MAX_ACCOUNT_NAME_CHARS)
170
+ if (typeof config.host !== 'string' || !config.host.trim()) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account host is required')
171
+ const host = singleLine(config.host, 256)
172
+ if (!host || /[\s/\\]/.test(host)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account host is not a valid hostname')
173
+ const requestedTls = config.tls == null ? undefined : String(config.tls).trim().toLowerCase()
174
+ let tls
175
+ if (requestedTls === undefined || requestedTls === '') tls = Number(config.port) === 143 ? 'starttls' : 'tls'
176
+ else if (requestedTls === 'ssl' || requestedTls === 'implicit' || requestedTls === 'tls') tls = 'tls'
177
+ else if (requestedTls === 'starttls') tls = 'starttls'
178
+ else if (['none', 'plain', 'insecure', 'cleartext', '0', 'false'].includes(requestedTls)) {
179
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'cleartext IMAP is not supported (tls=' + requestedTls + ')', 'use tls (implicit TLS, 993) or starttls (143)')
180
+ } else {
181
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'unknown mail account tls mode: ' + String(config.tls), 'use tls or starttls')
182
+ }
183
+ const defaultPort = tls === 'starttls' ? 143 : 993
184
+ const port = config.port == null ? defaultPort : Number(config.port)
185
+ if (!Number.isInteger(port) || port < 1 || port > 65535) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account port must be an integer 1-65535')
186
+ const username = validateAccountText(config.username, 'username', 320)
187
+ const mailbox = config.mailbox == null ? 'INBOX' : singleLine(config.mailbox, 200)
188
+ if (!mailbox) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account mailbox must not be empty')
189
+ let retentionDays = config.retentionDays == null ? MAIL_DEFAULT_RETENTION_DAYS : Number(config.retentionDays)
190
+ if (!Number.isInteger(retentionDays) || retentionDays < 0 || retentionDays > MAIL_MAX_RETENTION_DAYS) {
191
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'retention-days must be an integer 0-' + MAIL_MAX_RETENTION_DAYS, '0 keeps bodies until an explicit mail purge')
192
+ }
193
+ const entityId = (value, field) => {
194
+ if (value == null || value === '') return null
195
+ if (typeof value !== 'string' || value.length > 200 || ABSOLUTE_PATH_RE.test(value.trim())) {
196
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account ' + field + ' must be an entity id string')
197
+ }
198
+ return value.trim()
199
+ }
200
+ const at = isoOf(now)
201
+ return {
202
+ id: id ?? randomId('acct_'),
203
+ name,
204
+ host,
205
+ port,
206
+ tls,
207
+ username,
208
+ mailbox,
209
+ status: 'active',
210
+ defaultMatterId: entityId(config.defaultMatterId, 'default-matter'),
211
+ clientId: entityId(config.clientId, 'client'),
212
+ retentionDays,
213
+ lastSyncAt: null,
214
+ lastError: null,
215
+ createdAt: at,
216
+ updatedAt: at,
217
+ }
218
+ }
219
+
220
+ const mapAccountRow = (row) => ({
221
+ id: row.id,
222
+ name: row.name,
223
+ host: row.host,
224
+ port: Number(row.port),
225
+ tls: row.tls,
226
+ username: row.username,
227
+ mailbox: row.mailbox,
228
+ status: row.status,
229
+ defaultMatterId: row.default_matter_id ?? undefined,
230
+ clientId: row.client_id ?? undefined,
231
+ retentionDays: Number(row.retention_days),
232
+ lastSyncAt: row.last_sync_at ?? undefined,
233
+ lastError: row.last_error ?? undefined,
234
+ createdAt: row.created_at,
235
+ updatedAt: row.updated_at,
236
+ })
237
+
238
+ export function insertMailAccount(store, account) {
239
+ const duplicate = store.db.prepare("SELECT id FROM mail_accounts WHERE lower(name) = lower(?) AND status = 'active' LIMIT 1").get(account.name)
240
+ if (duplicate) throw mailError(MAIL_ERROR_CODES.CONFIG, 'an active mail account named ' + JSON.stringify(account.name) + ' already exists', 'use a different --name or remove the old account first')
241
+ store.db.prepare(
242
+ 'INSERT INTO mail_accounts(id,name,host,port,tls,username,mailbox,status,default_matter_id,client_id,retention_days,last_sync_at,last_error,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
243
+ ).run(account.id, account.name, account.host, account.port, account.tls, account.username, account.mailbox, account.status, account.defaultMatterId, account.clientId, account.retentionDays, account.lastSyncAt, account.lastError, account.createdAt, account.updatedAt)
244
+ return account
245
+ }
246
+
247
+ export function getMailAccount(store, ref, { includeRemoved = false } = {}) {
248
+ if (typeof ref !== 'string' || !ref.trim()) throw mailError(MAIL_ERROR_CODES.ACCOUNT_NOT_FOUND, 'mail account is required', 'run `advisers mail account ls` to list accounts')
249
+ const value = ref.trim()
250
+ const row = store.db.prepare('SELECT * FROM mail_accounts WHERE id = ? OR lower(name) = lower(?) ORDER BY CASE WHEN id = ? THEN 0 ELSE 1 END LIMIT 1').get(value, value, value)
251
+ if (!row || (!includeRemoved && row.status === 'removed')) {
252
+ throw mailError(MAIL_ERROR_CODES.ACCOUNT_NOT_FOUND, 'mail account not found: ' + value, 'run `advisers mail account ls` to list accounts')
253
+ }
254
+ return mapAccountRow(row)
255
+ }
256
+
257
+ export function listMailAccounts(store, { includeRemoved = false, limit = 100 } = {}) {
258
+ const cap = Math.max(1, Math.min(500, Number(limit) || 100))
259
+ const rows = includeRemoved
260
+ ? store.db.prepare('SELECT * FROM mail_accounts ORDER BY created_at ASC, id ASC LIMIT ?').all(cap)
261
+ : store.db.prepare("SELECT * FROM mail_accounts WHERE status = 'active' ORDER BY created_at ASC, id ASC LIMIT ?").all(cap)
262
+ return rows.map(mapAccountRow)
263
+ }
264
+
265
+ const ACCOUNT_PATCHES = {
266
+ name: 'name',
267
+ mailbox: 'mailbox',
268
+ status: 'status',
269
+ defaultMatterId: 'default_matter_id',
270
+ clientId: 'client_id',
271
+ retentionDays: 'retention_days',
272
+ lastSyncAt: 'last_sync_at',
273
+ lastError: 'last_error',
274
+ }
275
+
276
+ export function updateMailAccount(store, accountId, patch = {}, { now } = {}) {
277
+ const sets = []
278
+ const values = []
279
+ for (const [key, column] of Object.entries(ACCOUNT_PATCHES)) {
280
+ if (!(key in patch)) continue
281
+ let value = patch[key]
282
+ if (key === 'status') {
283
+ if (!MAIL_ACCOUNT_STATUSES.includes(value)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'unknown mail account status: ' + String(value))
284
+ } else if (key === 'retentionDays') {
285
+ value = Number(value)
286
+ if (!Number.isInteger(value) || value < 0 || value > MAIL_MAX_RETENTION_DAYS) throw mailError(MAIL_ERROR_CODES.CONFIG, 'retention-days must be an integer 0-' + MAIL_MAX_RETENTION_DAYS)
287
+ } else if (key === 'lastSyncAt') {
288
+ value = value == null ? null : normalizeIso(value, 'last-sync-at')
289
+ } else if (key === 'lastError') {
290
+ value = value == null ? null : singleLine(value, 300)
291
+ } else if (key === 'name') {
292
+ value = validateAccountText(value, 'name', MAIL_MAX_ACCOUNT_NAME_CHARS)
293
+ } else if (key === 'mailbox') {
294
+ value = singleLine(value, 200)
295
+ if (!value) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account mailbox must not be empty')
296
+ } else if (value != null && (typeof value !== 'string' || value.length > 200)) {
297
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account ' + key + ' must be an entity id string')
298
+ }
299
+ sets.push(column + ' = ?')
300
+ values.push(value)
301
+ }
302
+ if (!sets.length) return
303
+ sets.push('updated_at = ?')
304
+ values.push(isoOf(now))
305
+ values.push(accountId)
306
+ store.db.prepare('UPDATE mail_accounts SET ' + sets.join(', ') + ' WHERE id = ?').run(...values)
307
+ }
308
+
309
+ /** Transaction-internal: tombstone the account and delete its sync state. */
310
+ export function removeMailAccountData(store, accountId, { purgeMessages = false, now } = {}) {
311
+ const at = isoOf(now)
312
+ let messagesDeleted = 0
313
+ if (purgeMessages) {
314
+ const count = store.db.prepare('SELECT COUNT(*) AS n FROM mail_messages WHERE account_id = ?').get(accountId)
315
+ messagesDeleted = Number(count?.n ?? 0)
316
+ store.db.prepare('DELETE FROM mail_messages WHERE account_id = ?').run(accountId)
317
+ }
318
+ store.db.prepare('DELETE FROM mail_cursors WHERE account_id = ?').run(accountId)
319
+ store.db.prepare('DELETE FROM mail_sync_runs WHERE account_id = ?').run(accountId)
320
+ store.db.prepare("UPDATE mail_accounts SET status = 'removed', updated_at = ? WHERE id = ?").run(at, accountId)
321
+ return { messagesDeleted }
322
+ }
323
+
324
+ /** Built-in account for the no-network .eml/.mbox fallback import. */
325
+ export function ensureLocalMailAccount(store, { now } = {}) {
326
+ const existing = store.db.prepare('SELECT * FROM mail_accounts WHERE id = ?').get(MAIL_LOCAL_ACCOUNT_ID)
327
+ if (existing) return mapAccountRow(existing)
328
+ const at = isoOf(now)
329
+ store.db.prepare(
330
+ "INSERT OR IGNORE INTO mail_accounts(id,name,host,port,tls,username,mailbox,status,default_matter_id,client_id,retention_days,last_sync_at,last_error,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
331
+ ).run(MAIL_LOCAL_ACCOUNT_ID, 'Local .eml import', '(local)', 0, 'local', '(local)', MAIL_LOCAL_MAILBOX, 'active', null, null, MAIL_DEFAULT_RETENTION_DAYS, null, null, at, at)
332
+ return getMailAccount(store, MAIL_LOCAL_ACCOUNT_ID, { includeRemoved: true })
333
+ }
334
+
335
+ /* ------------------------------------------------------------------ */
336
+ /* Cursors */
337
+ /* ------------------------------------------------------------------ */
338
+
339
+ const mapCursorRow = (row) => ({
340
+ accountId: row.account_id,
341
+ mailbox: row.mailbox,
342
+ uidvalidity: row.uidvalidity ?? undefined,
343
+ lastUid: row.last_uid == null ? undefined : Number(row.last_uid),
344
+ uidnext: row.uidnext ?? undefined,
345
+ status: row.status,
346
+ updatedAt: row.updated_at,
347
+ })
348
+
349
+ export function getMailCursor(store, accountId, mailbox) {
350
+ const row = store.db.prepare('SELECT * FROM mail_cursors WHERE account_id = ? AND mailbox = ?').get(accountId, mailbox)
351
+ return row ? mapCursorRow(row) : undefined
352
+ }
353
+
354
+ /** Transaction-internal: upsert one mailbox cursor. */
355
+ export function putMailCursor(store, { accountId, mailbox, uidvalidity = null, lastUid = null, uidnext = null, status = 'ok', now } = {}) {
356
+ store.db.prepare(
357
+ 'INSERT INTO mail_cursors(account_id,mailbox,uidvalidity,last_uid,uidnext,status,updated_at) VALUES(?,?,?,?,?,?,?) ' +
358
+ 'ON CONFLICT(account_id,mailbox) DO UPDATE SET uidvalidity=excluded.uidvalidity,last_uid=excluded.last_uid,uidnext=excluded.uidnext,status=excluded.status,updated_at=excluded.updated_at',
359
+ ).run(accountId, mailbox, uidvalidity, lastUid, uidnext, status, isoOf(now))
360
+ }
361
+
362
+ /** Transaction-internal: mark the cursor as needing `mail sync --full`. */
363
+ export function markMailCursorNeedsResync(store, { accountId, mailbox, uidvalidity = null, uidnext = null, now } = {}) {
364
+ putMailCursor(store, { accountId, mailbox, uidvalidity, lastUid: null, uidnext, status: 'needs_resync', now })
365
+ }
366
+
367
+ /* ------------------------------------------------------------------ */
368
+ /* Messages */
369
+ /* ------------------------------------------------------------------ */
370
+
371
+ /**
372
+ * Stable dedupe key for one message (see file header).
373
+ * @returns {string} 64-char lowercase sha256 hex
374
+ */
375
+ export function mailDedupeKey({ accountId, mailbox = 'INBOX', uidvalidity = null, uid = null, messageId = '', from = '', subject = '', date = '', text = '' } = {}) {
376
+ const messageKey = normalizeMessageId(messageId)
377
+ if (messageKey) return sha256('message-id\u0000' + accountId + '\u0000' + messageKey)
378
+ if (Number.isInteger(uid) && uid > 0) return sha256('uid\u0000' + accountId + '\u0000' + mailbox + '\u0000' + String(uidvalidity ?? '') + '\u0000' + uid)
379
+ const bodyHash = sha256(canonicalText(text))
380
+ return sha256('content\u0000' + accountId + '\u0000' + canonicalText(from) + '\u0000' + canonicalText(subject) + '\u0000' + canonicalText(date) + '\u0000' + bodyHash)
381
+ }
382
+
383
+ export function mailMessageRowId(accountId, dedupeKey) {
384
+ return 'msg_' + sha256(accountId + '\u0000' + dedupeKey).slice(0, 32)
385
+ }
386
+
387
+ /**
388
+ * Build one mail_messages row from parsed mail. Does not touch the database.
389
+ * `status` defaults to 'new' when a body is retained, 'metadata' otherwise;
390
+ * callers pass 'oversize' for header-only rows.
391
+ */
392
+ export function buildMailMessageRow({
393
+ accountId, mailbox = 'INBOX', uid = null, uidvalidity = null, sizeBytes = null,
394
+ receivedAt, sentAt = null, from = '', subject = '', messageId = '', text = null,
395
+ attachments = [], status = null, clientId = null, projectId = null, now,
396
+ } = {}) {
397
+ if (typeof accountId !== 'string' || !accountId) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail message requires an account id')
398
+ const received = normalizeIso(receivedAt, 'received-at')
399
+ if (!received) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail message requires received-at')
400
+ const sent = normalizeIso(sentAt, 'sent-at')
401
+ const bodyText = text == null ? null : truncateUtf8(String(text), MAIL_MAX_BODY_BYTES)
402
+ const retained = bodyText != null && bodyText.trim() !== ''
403
+ const dedupeKey = mailDedupeKey({ accountId, mailbox, uidvalidity, uid, messageId, from, subject, date: sent ?? received, text: retained ? bodyText : '' })
404
+ const at = isoOf(now)
405
+ return {
406
+ id: mailMessageRowId(accountId, dedupeKey),
407
+ accountId,
408
+ mailbox,
409
+ uid: Number.isInteger(uid) && uid > 0 ? uid : null,
410
+ uidvalidity: uidvalidity == null ? null : String(uidvalidity),
411
+ messageId: normalizeMessageId(messageId) || null,
412
+ dedupeKey,
413
+ fromAddr: singleLine(from, MAIL_MAX_FROM_CHARS) || null,
414
+ subject: singleLine(subject, MAIL_MAX_SUBJECT_CHARS) || null,
415
+ sentAt: sent,
416
+ receivedAt: received,
417
+ sizeBytes: Number.isFinite(Number(sizeBytes)) && Number(sizeBytes) >= 0 ? Math.floor(Number(sizeBytes)) : null,
418
+ bodyText: retained ? bodyText : null,
419
+ bodySha256: retained ? sha256(bodyText) : null,
420
+ attachments: normalizeAttachments(attachments),
421
+ clientId: clientId ?? null,
422
+ projectId: projectId ?? null,
423
+ status: status ?? (retained ? 'new' : 'metadata'),
424
+ candidateIds: [],
425
+ purgedAt: null,
426
+ createdAt: at,
427
+ updatedAt: at,
428
+ }
429
+ }
430
+
431
+ export function findMailMessageDuplicate(store, accountId, row) {
432
+ const byKey = store.db.prepare('SELECT id, uid, dedupe_key FROM mail_messages WHERE account_id = ? AND dedupe_key = ? LIMIT 1').get(accountId, row.dedupeKey)
433
+ if (byKey) return byKey
434
+ if (Number.isInteger(row.uid) && row.uid > 0) {
435
+ const byUid = store.db.prepare('SELECT id, uid, dedupe_key FROM mail_messages WHERE account_id = ? AND mailbox = ? AND uidvalidity IS ? AND uid = ? LIMIT 1').get(accountId, row.mailbox, row.uidvalidity, row.uid)
436
+ if (byUid) return byUid
437
+ }
438
+ return undefined
439
+ }
440
+
441
+ /** Transaction-internal: insert one row; returns false when a unique index rejected it. */
442
+ export function insertMailMessage(store, row) {
443
+ const info = store.db.prepare(
444
+ 'INSERT OR IGNORE INTO mail_messages(id,account_id,mailbox,uid,uidvalidity,message_id,dedupe_key,from_addr,subject,sent_at,received_at,size_bytes,body_text,body_sha256,attachments,client_id,project_id,status,candidate_ids,purged_at,created_at,updated_at) ' +
445
+ 'VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
446
+ ).run(row.id, row.accountId, row.mailbox, row.uid, row.uidvalidity, row.messageId, row.dedupeKey, row.fromAddr, row.subject, row.sentAt, row.receivedAt, row.sizeBytes, row.bodyText, row.bodySha256, JSON.stringify(row.attachments ?? []), row.clientId, row.projectId, row.status, JSON.stringify(row.candidateIds ?? []), row.purgedAt, row.createdAt, row.updatedAt)
447
+ return Number(info.changes) > 0
448
+ }
449
+
450
+ const mapMessageRow = (row, { includeBody = false } = {}) => ({
451
+ id: row.id,
452
+ accountId: row.account_id,
453
+ mailbox: row.mailbox,
454
+ uid: Number(row.uid),
455
+ uidvalidity: row.uidvalidity ?? undefined,
456
+ messageId: row.message_id ?? undefined,
457
+ dedupeKey: row.dedupe_key,
458
+ from: row.from_addr ?? '',
459
+ subject: row.subject ?? '',
460
+ sentAt: row.sent_at ?? undefined,
461
+ receivedAt: row.received_at,
462
+ sizeBytes: row.size_bytes == null ? undefined : Number(row.size_bytes),
463
+ retained: row.body_text != null && row.purged_at == null,
464
+ purgedAt: row.purged_at ?? undefined,
465
+ bodySha256: row.body_sha256 ?? undefined,
466
+ attachments: parseJson(row.attachments, []),
467
+ clientId: row.client_id ?? undefined,
468
+ projectId: row.project_id ?? undefined,
469
+ status: row.status,
470
+ candidateIds: parseJson(row.candidate_ids, []),
471
+ createdAt: row.created_at,
472
+ updatedAt: row.updated_at,
473
+ ...(includeBody ? { bodyText: row.body_text ?? null } : {}),
474
+ })
475
+
476
+ export function getMailMessage(store, ref, { includeBody = false } = {}) {
477
+ if (typeof ref !== 'string' || !ref.trim()) throw mailError(MAIL_ERROR_CODES.MESSAGE_NOT_FOUND, 'mail message id is required')
478
+ const value = ref.trim()
479
+ const stripped = normalizeMessageId(value) || value
480
+ const row = value.startsWith('msg_')
481
+ ? store.db.prepare('SELECT * FROM mail_messages WHERE id = ?').get(value)
482
+ : store.db.prepare('SELECT * FROM mail_messages WHERE message_id = ? ORDER BY received_at DESC LIMIT 1').get(stripped)
483
+ if (!row) throw mailError(MAIL_ERROR_CODES.MESSAGE_NOT_FOUND, 'mail message not found: ' + value, 'run `advisers mail ls` to list messages')
484
+ return mapMessageRow(row, { includeBody })
485
+ }
486
+
487
+ const MESSAGE_PATCHES = {
488
+ status: 'status',
489
+ clientId: 'client_id',
490
+ projectId: 'project_id',
491
+ bodyText: 'body_text',
492
+ bodySha256: 'body_sha256',
493
+ purgedAt: 'purged_at',
494
+ }
495
+
496
+ /** Transaction-internal: update one message row. */
497
+ export function updateMailMessage(store, id, patch = {}, { now } = {}) {
498
+ const sets = []
499
+ const values = []
500
+ for (const [key, column] of Object.entries(MESSAGE_PATCHES)) {
501
+ if (!(key in patch)) continue
502
+ let value = patch[key]
503
+ if (key === 'status') {
504
+ if (!MAIL_MESSAGE_STATUSES.includes(value)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'unknown mail message status: ' + String(value))
505
+ } else if (key === 'bodyText') {
506
+ value = value == null ? null : truncateUtf8(String(value), MAIL_MAX_BODY_BYTES)
507
+ }
508
+ sets.push(column + ' = ?')
509
+ values.push(value)
510
+ }
511
+ if ('candidateIds' in patch && Array.isArray(patch.candidateIds)) {
512
+ sets.push('candidate_ids = ?')
513
+ values.push(JSON.stringify([...new Set(patch.candidateIds.filter((candidateId) => typeof candidateId === 'string' && candidateId))]))
514
+ }
515
+ if (!sets.length) return getMailMessage(store, id)
516
+ sets.push('updated_at = ?')
517
+ values.push(isoOf(now))
518
+ values.push(id)
519
+ store.db.prepare('UPDATE mail_messages SET ' + sets.join(', ') + ' WHERE id = ?').run(...values)
520
+ return getMailMessage(store, id)
521
+ }
522
+
523
+ /** Transaction-internal: append candidate ids to a message's reverse pointer. */
524
+ export function addMailCandidateIds(store, id, candidateIds, { now } = {}) {
525
+ const current = store.db.prepare('SELECT candidate_ids FROM mail_messages WHERE id = ?').get(id)
526
+ if (!current) return
527
+ const merged = [...new Set([...parseJson(current.candidate_ids, []), ...candidateIds])]
528
+ store.db.prepare('UPDATE mail_messages SET candidate_ids = ?, updated_at = ? WHERE id = ?').run(JSON.stringify(merged), isoOf(now), id)
529
+ }
530
+
531
+ function messageWhere(filters = {}) {
532
+ const where = []
533
+ const params = []
534
+ if (filters.accountId) { where.push('account_id = ?'); params.push(filters.accountId) }
535
+ if (filters.clientId) { where.push('client_id = ?'); params.push(filters.clientId) }
536
+ if (filters.projectId) { where.push('project_id = ?'); params.push(filters.projectId) }
537
+ if (filters.status) {
538
+ const statuses = Array.isArray(filters.status) ? filters.status : [filters.status]
539
+ where.push('status IN (' + statuses.map(() => '?').join(',') + ')')
540
+ params.push(...statuses)
541
+ }
542
+ if (filters.retained === true) where.push('body_text IS NOT NULL')
543
+ if (filters.retained === false) where.push('body_text IS NULL')
544
+ if (filters.sinceDays != null && Number(filters.sinceDays) >= 0) {
545
+ const cutoff = new Date(nowDate(filters.now).getTime() - Number(filters.sinceDays) * DAY_MS).toISOString()
546
+ where.push('received_at >= ?')
547
+ params.push(cutoff)
548
+ }
549
+ return { sql: where.length ? ' WHERE ' + where.join(' AND ') : '', params }
550
+ }
551
+
552
+ /**
553
+ * List message summaries (never the body). `newest first`, capped page.
554
+ */
555
+ export function listMailMessages(store, filters = {}) {
556
+ const limit = Math.max(1, Math.min(MAIL_MAX_LS_LIMIT, Number(filters.limit) || MAIL_DEFAULT_LS_LIMIT))
557
+ const offset = Math.max(0, Math.floor(Number(filters.offset) || 0))
558
+ const where = messageWhere(filters)
559
+ const total = Number(store.db.prepare('SELECT COUNT(*) AS n FROM mail_messages' + where.sql).get(...where.params)?.n ?? 0)
560
+ const rows = store.db.prepare('SELECT * FROM mail_messages' + where.sql + ' ORDER BY received_at DESC, uid DESC LIMIT ? OFFSET ?').all(...where.params, limit, offset)
561
+ return { messages: rows.map((row) => mapMessageRow(row)), total, limit, offset, truncated: offset + rows.length < total }
562
+ }
563
+
564
+ /** Transaction-read: rows a `mail extract` run should scan (body retained). */
565
+ export function listMessagesForExtraction(store, { accountId, ids, all = false, limit = 20, now } = {}) {
566
+ const cap = Math.max(1, Math.min(MAIL_MAX_SYNC_LIMIT, Number(limit) || 20))
567
+ if (Array.isArray(ids) && ids.length) {
568
+ const rows = []
569
+ for (const ref of ids.slice(0, cap)) {
570
+ const value = String(ref).trim()
571
+ const stripped = normalizeMessageId(value) || value
572
+ const row = value.startsWith('msg_')
573
+ ? store.db.prepare('SELECT * FROM mail_messages WHERE id = ?').get(value)
574
+ : store.db.prepare('SELECT * FROM mail_messages WHERE message_id = ? ORDER BY received_at DESC LIMIT 1').get(stripped)
575
+ if (row) rows.push(row)
576
+ }
577
+ return rows.map((row) => mapMessageRow(row, { includeBody: true }))
578
+ }
579
+ const statuses = all ? ['new', 'extracted', 'linked'] : ['new']
580
+ const params = [...statuses]
581
+ let sql = 'SELECT * FROM mail_messages WHERE body_text IS NOT NULL AND purged_at IS NULL AND status IN (' + statuses.map(() => '?').join(',') + ')'
582
+ if (accountId) { sql += ' AND account_id = ?'; params.push(accountId) }
583
+ sql += ' ORDER BY received_at ASC, uid ASC LIMIT ?'
584
+ params.push(cap)
585
+ return store.db.prepare(sql).all(...params).map((row) => mapMessageRow(row, { includeBody: true }))
586
+ }
587
+
588
+ /** Transaction-read: rows with a retained body, for retention/purge planning. */
589
+ export function listMessagesWithBody(store, { accountId, before } = {}) {
590
+ const params = []
591
+ let sql = 'SELECT * FROM mail_messages WHERE body_text IS NOT NULL AND purged_at IS NULL'
592
+ if (accountId) { sql += ' AND account_id = ?'; params.push(accountId) }
593
+ if (before) { sql += ' AND received_at < ?'; params.push(before) }
594
+ sql += ' ORDER BY received_at ASC LIMIT 2000'
595
+ return store.db.prepare(sql).all(...params).map((row) => mapMessageRow(row, { includeBody: true }))
596
+ }
597
+
598
+ /** Transaction-internal: clear retained bodies (metadata + body_sha256 stay). */
599
+ export function purgeMailBodies(store, ids, { now } = {}) {
600
+ const at = isoOf(now)
601
+ let purged = 0
602
+ for (const id of ids) {
603
+ const info = store.db.prepare('UPDATE mail_messages SET body_text = NULL, purged_at = ?, updated_at = ? WHERE id = ? AND body_text IS NOT NULL').run(at, at, id)
604
+ purged += Number(info.changes)
605
+ }
606
+ return purged
607
+ }
608
+
609
+ export function nextLocalUid(store, accountId, mailbox = MAIL_LOCAL_MAILBOX) {
610
+ const row = store.db.prepare('SELECT MAX(uid) AS max_uid FROM mail_messages WHERE account_id = ? AND mailbox = ?').get(accountId, mailbox)
611
+ return Number(row?.max_uid ?? 0) + 1
612
+ }
613
+
614
+ /* ------------------------------------------------------------------ */
615
+ /* Sync runs + one-transaction batch apply */
616
+ /* ------------------------------------------------------------------ */
617
+
618
+ /** Transaction-internal: append one mail_sync_runs row. */
619
+ export function insertMailSyncRun(store, run = {}) {
620
+ const id = run.id ?? randomId('sync_')
621
+ store.db.prepare(
622
+ 'INSERT INTO mail_sync_runs(id,account_id,mailbox,started_at,finished_at,status,fetched,inserted,skipped,failed,uidvalidity,last_uid_before,last_uid_after,error_code) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
623
+ ).run(id, run.accountId, run.mailbox, run.startedAt, run.finishedAt, run.status ?? 'ok', Number(run.fetched) || 0, Number(run.inserted) || 0, Number(run.skipped) || 0, Number(run.failed) || 0, run.uidvalidity ?? null, run.lastUidBefore ?? null, run.lastUidAfter ?? null, run.errorCode ?? null)
624
+ return id
625
+ }
626
+
627
+ /**
628
+ * Transaction-internal: write a fetched batch, advance the cursor and record
629
+ * the run in one BEGIN IMMEDIATE. The caller (lib/mail.js) performs the
630
+ * network fetch BEFORE calling this, so no DB transaction is held across the
631
+ * network.
632
+ *
633
+ * Cursor rule: never advance past a UID that failed to parse/fetch. On a
634
+ * UIDVALIDITY change (only reachable with --full) the old UID space is
635
+ * discarded instead of being carried over.
636
+ */
637
+ export function applyMailSyncBatch(store, batch = {}) {
638
+ const { accountId, mailbox, uidvalidity = null, messages = [], failedUids = [], uidnext = null, startedAt, now } = batch
639
+ const at = isoOf(now)
640
+ const failed = new Set((failedUids ?? []).filter((uid) => Number.isInteger(uid)))
641
+ let inserted = 0
642
+ let skipped = 0
643
+ let duplicates = 0
644
+ let maxCommittedUid = null
645
+ for (const row of messages) {
646
+ if (failed.has(row.uid)) continue
647
+ row.uidvalidity = uidvalidity
648
+ const duplicate = findMailMessageDuplicate(store, accountId, row)
649
+ if (duplicate) {
650
+ skipped += 1
651
+ duplicates += 1
652
+ if (Number.isInteger(row.uid)) maxCommittedUid = Math.max(maxCommittedUid ?? 0, row.uid)
653
+ continue
654
+ }
655
+ if (insertMailMessage(store, row)) inserted += 1
656
+ else { skipped += 1; duplicates += 1 }
657
+ if (Number.isInteger(row.uid)) maxCommittedUid = Math.max(maxCommittedUid ?? 0, row.uid)
658
+ }
659
+ let advanceTo = maxCommittedUid
660
+ if (failed.size) {
661
+ const cap = Math.min(...failed) - 1
662
+ advanceTo = cap < 1 ? null : advanceTo == null ? null : Math.min(advanceTo, cap)
663
+ }
664
+ const previous = batch.previousCursor
665
+ const previousUidValidity = previous?.uidvalidity ?? null
666
+ const previousLast = previous?.lastUid ?? null
667
+ const uidvalidityChanged = Boolean(previousUidValidity != null && uidvalidity != null && String(previousUidValidity) !== String(uidvalidity))
668
+ let lastUid
669
+ if (uidvalidityChanged) {
670
+ // The old UID space no longer exists: never carry the old cursor over.
671
+ lastUid = advanceTo
672
+ } else if (advanceTo == null) {
673
+ lastUid = previousLast
674
+ } else if (previousLast == null) {
675
+ lastUid = advanceTo
676
+ } else {
677
+ lastUid = Math.max(previousLast, advanceTo)
678
+ }
679
+ putMailCursor(store, {
680
+ accountId, mailbox,
681
+ uidvalidity: uidvalidity ?? previousUidValidity,
682
+ lastUid,
683
+ uidnext,
684
+ status: 'ok',
685
+ now,
686
+ })
687
+ updateMailAccount(store, accountId, { lastSyncAt: at, lastError: null }, { now })
688
+ insertMailSyncRun(store, {
689
+ accountId, mailbox, startedAt: startedAt ?? at, finishedAt: at, status: 'ok',
690
+ fetched: batch.fetched ?? messages.length + failed.size,
691
+ inserted, skipped, failed: failed.size,
692
+ uidvalidity: uidvalidity ?? previousUidValidity ?? null,
693
+ lastUidBefore: previousLast,
694
+ lastUidAfter: lastUid,
695
+ errorCode: null,
696
+ })
697
+ return { inserted, skipped, duplicates, failed: failed.size, lastUidBefore: previousLast, lastUidAfter: lastUid, uidvalidityChanged }
698
+ }
699
+
700
+ /* ------------------------------------------------------------------ */
701
+ /* Retention / purge planning */
702
+ /* ------------------------------------------------------------------ */
703
+
704
+ /**
705
+ * Plan a retention purge: for every active account, bodies of messages older
706
+ * than its retention window (or `olderThanDays` when given). Messages that
707
+ * still have a pending memory candidate are skipped so an unreviewed
708
+ * candidate can never lose its quote.
709
+ */
710
+ export function planMailPurge(store, { now, olderThanDays, accountId } = {}) {
711
+ const at = nowDate(now)
712
+ const accounts = listMailAccounts(store).filter((account) => !accountId || account.id === accountId)
713
+ const targets = []
714
+ let skippedPending = 0
715
+ let bytesFreed = 0
716
+ for (const account of accounts) {
717
+ const days = olderThanDays == null ? account.retentionDays : Number(olderThanDays)
718
+ if (!Number.isFinite(days) || days < 0) continue
719
+ if (olderThanDays == null && days === 0) continue // 0 = keep bodies until an explicit purge
720
+ const cutoff = new Date(at.getTime() - days * DAY_MS).toISOString()
721
+ const rows = listMessagesWithBody(store, { accountId: account.id, before: cutoff })
722
+ for (const row of rows) {
723
+ const pending = (row.candidateIds ?? []).some((candidateId) => store.getCandidate(candidateId)?.status === 'pending')
724
+ if (pending) { skippedPending += 1; continue }
725
+ const bytes = Buffer.byteLength(row.bodyText ?? '', 'utf8')
726
+ bytesFreed += bytes
727
+ targets.push({ id: row.id, accountId: row.accountId, bytes })
728
+ }
729
+ }
730
+ return { targets, skippedPending, bytesFreed, accountsScanned: accounts.length }
731
+ }