advisers 0.3.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh.md +13 -14
- package/README.md +32 -23
- package/THIRD-PARTY-NOTICES.md +1 -1
- package/bin/advisers.js +2180 -92
- package/index.js +1 -1
- package/lib/brief-archive.js +267 -0
- package/lib/client-pack.js +1016 -0
- package/lib/consent.js +390 -0
- package/lib/credentials.js +195 -0
- package/lib/error-codes.js +516 -7
- package/lib/evidence.js +697 -113
- package/lib/kb-extract.js +626 -0
- package/lib/kb-facts.js +429 -0
- package/lib/kb-pages.js +1084 -0
- package/lib/kb.js +1036 -0
- package/lib/mail-errors.js +46 -0
- package/lib/mail-extract.js +351 -0
- package/lib/mail-imap.js +446 -0
- package/lib/mail-import.js +176 -0
- package/lib/mail-parse.js +223 -0
- package/lib/mail-store.js +731 -0
- package/lib/mail.js +506 -0
- package/lib/mcp-server.js +196 -10
- package/lib/notify.js +265 -7
- package/lib/policy/decision.js +315 -0
- package/lib/policy/egress.js +268 -0
- package/lib/policy/errors.js +71 -0
- package/lib/policy/index.js +99 -0
- package/lib/policy/rules.js +468 -0
- package/lib/policy/surfaces.js +141 -0
- package/lib/receipt-verify.js +199 -19
- package/lib/receivables.js +1653 -0
- package/lib/scheduler.js +453 -9
- package/lib/shell.js +1406 -0
- package/lib/source.js +98 -3
- package/lib/sqlite.js +366 -49
- package/lib/tos/index.js +2 -2
- package/lib/tos/manifest.json +6 -6
- package/lib/tos/privacy.zh-CN.md +33 -20
- package/lib/tos/safety.zh-CN.md +7 -6
- package/lib/tos/summary.zh-CN.md +3 -2
- package/lib/tos/terms.zh-CN.md +31 -16
- package/lib/urgency.js +467 -0
- package/package.json +4 -4
|
@@ -0,0 +1,731 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite persistence for the 0.4.0 mail entry (schema v22 tables owned by
|
|
3
|
+
* lib/sqlite.js: mail_accounts / mail_cursors / mail_messages /
|
|
4
|
+
* mail_sync_runs). This module never creates tables and never touches
|
|
5
|
+
* credentials; it only reads/writes the mail rows through an open
|
|
6
|
+
* SqliteStore (use store.mutate()/transaction() for the write helpers that
|
|
7
|
+
* are documented as transaction-internal).
|
|
8
|
+
*
|
|
9
|
+
* Invariants:
|
|
10
|
+
* - a message row keeps `body_text` only within MAIL_MAX_BODY_BYTES; clearing
|
|
11
|
+
* it (retention / purge) keeps the metadata, body_sha256 and dedupe key, so
|
|
12
|
+
* re-syncing the mailbox can never recreate the row;
|
|
13
|
+
* - dedupe key: Message-ID first, then (mailbox, UIDVALIDITY, UID), then a
|
|
14
|
+
* canonical content hash for .eml imports without a Message-ID. The body
|
|
15
|
+
* hash is NOT a dedupe key (signature/client differences would drift);
|
|
16
|
+
* - attachments are metadata only: [{ name, size, mime }]; no attachment
|
|
17
|
+
* bytes are stored anywhere under ADVISERS_HOME;
|
|
18
|
+
* - a sync batch writes messages + cursor + account last_sync_at + the run row
|
|
19
|
+
* in ONE BEGIN IMMEDIATE (see applyMailSyncBatch); the cursor only advances
|
|
20
|
+
* past successful rows, never past a failed UID.
|
|
21
|
+
*/
|
|
22
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
23
|
+
import { mailError, MAIL_ERROR_CODES } from './mail-errors.js'
|
|
24
|
+
|
|
25
|
+
export const MAIL_DEFAULT_RETENTION_DAYS = 30
|
|
26
|
+
export const MAIL_MAX_RETENTION_DAYS = 3650
|
|
27
|
+
export const MAIL_DEFAULT_SYNC_LIMIT = 50
|
|
28
|
+
export const MAIL_MAX_SYNC_LIMIT = 200
|
|
29
|
+
export const MAIL_DEFAULT_LS_LIMIT = 50
|
|
30
|
+
export const MAIL_MAX_LS_LIMIT = 500
|
|
31
|
+
export const MAIL_MAX_BODY_BYTES = 256 * 1024
|
|
32
|
+
export const MAIL_MAX_MESSAGE_BYTES = 5 * 1024 * 1024
|
|
33
|
+
export const MAIL_MAX_ATTACHMENTS = 50
|
|
34
|
+
export const MAIL_TLS_MODES = Object.freeze(['tls', 'starttls'])
|
|
35
|
+
export const MAIL_ACCOUNT_STATUSES = Object.freeze(['active', 'removed'])
|
|
36
|
+
export const MAIL_MESSAGE_STATUSES = Object.freeze(['new', 'extracted', 'linked', 'dismissed', 'oversize', 'metadata'])
|
|
37
|
+
export const MAIL_LOCAL_ACCOUNT_ID = 'acct_local_eml'
|
|
38
|
+
export const MAIL_LOCAL_MAILBOX = 'local'
|
|
39
|
+
export const MAIL_MAX_ACCOUNT_NAME_CHARS = 120
|
|
40
|
+
export const MAIL_MAX_SUBJECT_CHARS = 500
|
|
41
|
+
export const MAIL_MAX_FROM_CHARS = 320
|
|
42
|
+
|
|
43
|
+
const REQUIRED_TABLES = ['mail_accounts', 'mail_cursors', 'mail_messages', 'mail_sync_runs']
|
|
44
|
+
const DAY_MS = 86_400_000
|
|
45
|
+
const ABSOLUTE_PATH_RE = /^(?:[A-Za-z]:[\\/]|\/|\\\\)/
|
|
46
|
+
|
|
47
|
+
function nowDate(now) {
|
|
48
|
+
const value = now instanceof Date ? now : new Date(now ?? Date.now())
|
|
49
|
+
if (!Number.isFinite(value.getTime())) throw mailError(MAIL_ERROR_CODES.CONFIG, 'invalid clock value')
|
|
50
|
+
return value
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function isoOf(now) {
|
|
54
|
+
return nowDate(now).toISOString()
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function normalizeIso(value, field) {
|
|
58
|
+
if (value == null || value === '') return null
|
|
59
|
+
const parsed = value instanceof Date ? value : new Date(value)
|
|
60
|
+
if (!Number.isFinite(parsed.getTime())) throw mailError(MAIL_ERROR_CODES.CONFIG, field + ' is not a valid date')
|
|
61
|
+
return parsed.toISOString()
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export function parseJson(value, fallback) {
|
|
65
|
+
if (value == null) return fallback
|
|
66
|
+
try {
|
|
67
|
+
const parsed = JSON.parse(value)
|
|
68
|
+
return parsed === null ? fallback : parsed
|
|
69
|
+
} catch { return fallback }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function sha256(text) {
|
|
73
|
+
return createHash('sha256').update(text, 'utf8').digest('hex')
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function randomId(prefix) {
|
|
77
|
+
return prefix + randomBytes(8).toString('hex')
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function singleLine(value, max) {
|
|
81
|
+
const text = String(value == null ? '' : value).replace(/[\u0000-\u001f\u007f]+/g, ' ').replace(/\s+/g, ' ').trim()
|
|
82
|
+
return text.length > max ? text.slice(0, max) : text
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function normalizeMessageId(value) {
|
|
86
|
+
let text = String(value == null ? '' : value).trim()
|
|
87
|
+
if (!text) return ''
|
|
88
|
+
const wrapped = /^<([^>]+)>$/.exec(text)
|
|
89
|
+
if (wrapped) text = wrapped[1].trim()
|
|
90
|
+
if (!text || /\s/.test(text) || text.length > 512) return ''
|
|
91
|
+
return text
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function canonicalText(value) {
|
|
95
|
+
return String(value == null ? '' : value)
|
|
96
|
+
.replace(/\r\n/g, '\n').replace(/\r/g, '\n')
|
|
97
|
+
.replace(/[ \t]+\n/g, '\n')
|
|
98
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
99
|
+
.trim()
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function truncateUtf8(text, maxBytes) {
|
|
103
|
+
const value = String(text == null ? '' : text)
|
|
104
|
+
if (Buffer.byteLength(value, 'utf8') <= maxBytes) return value
|
|
105
|
+
let low = 0
|
|
106
|
+
let high = value.length
|
|
107
|
+
while (low < high) {
|
|
108
|
+
const middle = Math.ceil((low + high) / 2)
|
|
109
|
+
if (Buffer.byteLength(value.slice(0, middle), 'utf8') <= maxBytes - 64) low = middle
|
|
110
|
+
else high = middle - 1
|
|
111
|
+
}
|
|
112
|
+
return value.slice(0, low) + '\n[body truncated by advisers]'
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Fail with a stable code when the v22 mail tables are missing. */
|
|
116
|
+
export function assertMailTables(store) {
|
|
117
|
+
if (!store || typeof store.db?.prepare !== 'function') {
|
|
118
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail commands require the SQLite ledger (JSON store is not supported)')
|
|
119
|
+
}
|
|
120
|
+
let rows
|
|
121
|
+
try {
|
|
122
|
+
rows = store.db.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all()
|
|
123
|
+
} catch (error) {
|
|
124
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'the ledger database is not readable: ' + String(error?.message ?? error).slice(0, 200))
|
|
125
|
+
}
|
|
126
|
+
const present = new Set(rows.map((row) => row.name))
|
|
127
|
+
const missing = REQUIRED_TABLES.filter((name) => !present.has(name))
|
|
128
|
+
if (missing.length) {
|
|
129
|
+
throw mailError('E_SCHEMA_STALE', 'mail requires the v22 schema but this ledger is missing: ' + missing.join(', '), 'upgrade advisers and run `advisers init` to migrate the ledger')
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function normalizeAttachments(attachments) {
|
|
134
|
+
if (!Array.isArray(attachments)) return []
|
|
135
|
+
const out = []
|
|
136
|
+
for (const item of attachments.slice(0, 500)) {
|
|
137
|
+
if (out.length >= MAIL_MAX_ATTACHMENTS) break
|
|
138
|
+
if (!item || typeof item !== 'object') continue
|
|
139
|
+
const name = singleLine(item.name, 120)
|
|
140
|
+
const size = Number(item.size)
|
|
141
|
+
const mime = singleLine(item.mime, 120)
|
|
142
|
+
if (!name) continue
|
|
143
|
+
out.push({ name, size: Number.isFinite(size) && size >= 0 ? Math.floor(size) : 0, mime: mime || 'application/octet-stream' })
|
|
144
|
+
}
|
|
145
|
+
return out
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/* ------------------------------------------------------------------ */
|
|
149
|
+
/* Accounts */
|
|
150
|
+
/* ------------------------------------------------------------------ */
|
|
151
|
+
|
|
152
|
+
function validateAccountText(value, field, max) {
|
|
153
|
+
if (typeof value !== 'string') throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account ' + field + ' is required')
|
|
154
|
+
const text = singleLine(value, max + 1)
|
|
155
|
+
if (!text || text.length > max) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account ' + field + ' must be 1-' + max + ' characters')
|
|
156
|
+
return text
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Validate and normalise a mail account config. Rejects cleartext IMAP
|
|
161
|
+
* (`secure: false` / `tls: 'none'`) with E_MAIL_CONFIG: RFC 8314 removed
|
|
162
|
+
* cleartext mail access and this module has no plaintext product path.
|
|
163
|
+
*/
|
|
164
|
+
export function prepareMailAccount(config = {}, { now, id } = {}) {
|
|
165
|
+
if (!config || typeof config !== 'object') throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account config is required')
|
|
166
|
+
if (config.secure === false || config.secure === 0 || config.insecure === true) {
|
|
167
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'cleartext IMAP is not supported; mail sync requires implicit TLS (993) or STARTTLS', 'drop --insecure/secure=0 and use --port 993')
|
|
168
|
+
}
|
|
169
|
+
const name = validateAccountText(config.name, 'name', MAIL_MAX_ACCOUNT_NAME_CHARS)
|
|
170
|
+
if (typeof config.host !== 'string' || !config.host.trim()) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account host is required')
|
|
171
|
+
const host = singleLine(config.host, 256)
|
|
172
|
+
if (!host || /[\s/\\]/.test(host)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account host is not a valid hostname')
|
|
173
|
+
const requestedTls = config.tls == null ? undefined : String(config.tls).trim().toLowerCase()
|
|
174
|
+
let tls
|
|
175
|
+
if (requestedTls === undefined || requestedTls === '') tls = Number(config.port) === 143 ? 'starttls' : 'tls'
|
|
176
|
+
else if (requestedTls === 'ssl' || requestedTls === 'implicit' || requestedTls === 'tls') tls = 'tls'
|
|
177
|
+
else if (requestedTls === 'starttls') tls = 'starttls'
|
|
178
|
+
else if (['none', 'plain', 'insecure', 'cleartext', '0', 'false'].includes(requestedTls)) {
|
|
179
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'cleartext IMAP is not supported (tls=' + requestedTls + ')', 'use tls (implicit TLS, 993) or starttls (143)')
|
|
180
|
+
} else {
|
|
181
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'unknown mail account tls mode: ' + String(config.tls), 'use tls or starttls')
|
|
182
|
+
}
|
|
183
|
+
const defaultPort = tls === 'starttls' ? 143 : 993
|
|
184
|
+
const port = config.port == null ? defaultPort : Number(config.port)
|
|
185
|
+
if (!Number.isInteger(port) || port < 1 || port > 65535) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account port must be an integer 1-65535')
|
|
186
|
+
const username = validateAccountText(config.username, 'username', 320)
|
|
187
|
+
const mailbox = config.mailbox == null ? 'INBOX' : singleLine(config.mailbox, 200)
|
|
188
|
+
if (!mailbox) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account mailbox must not be empty')
|
|
189
|
+
let retentionDays = config.retentionDays == null ? MAIL_DEFAULT_RETENTION_DAYS : Number(config.retentionDays)
|
|
190
|
+
if (!Number.isInteger(retentionDays) || retentionDays < 0 || retentionDays > MAIL_MAX_RETENTION_DAYS) {
|
|
191
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'retention-days must be an integer 0-' + MAIL_MAX_RETENTION_DAYS, '0 keeps bodies until an explicit mail purge')
|
|
192
|
+
}
|
|
193
|
+
const entityId = (value, field) => {
|
|
194
|
+
if (value == null || value === '') return null
|
|
195
|
+
if (typeof value !== 'string' || value.length > 200 || ABSOLUTE_PATH_RE.test(value.trim())) {
|
|
196
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account ' + field + ' must be an entity id string')
|
|
197
|
+
}
|
|
198
|
+
return value.trim()
|
|
199
|
+
}
|
|
200
|
+
const at = isoOf(now)
|
|
201
|
+
return {
|
|
202
|
+
id: id ?? randomId('acct_'),
|
|
203
|
+
name,
|
|
204
|
+
host,
|
|
205
|
+
port,
|
|
206
|
+
tls,
|
|
207
|
+
username,
|
|
208
|
+
mailbox,
|
|
209
|
+
status: 'active',
|
|
210
|
+
defaultMatterId: entityId(config.defaultMatterId, 'default-matter'),
|
|
211
|
+
clientId: entityId(config.clientId, 'client'),
|
|
212
|
+
retentionDays,
|
|
213
|
+
lastSyncAt: null,
|
|
214
|
+
lastError: null,
|
|
215
|
+
createdAt: at,
|
|
216
|
+
updatedAt: at,
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const mapAccountRow = (row) => ({
|
|
221
|
+
id: row.id,
|
|
222
|
+
name: row.name,
|
|
223
|
+
host: row.host,
|
|
224
|
+
port: Number(row.port),
|
|
225
|
+
tls: row.tls,
|
|
226
|
+
username: row.username,
|
|
227
|
+
mailbox: row.mailbox,
|
|
228
|
+
status: row.status,
|
|
229
|
+
defaultMatterId: row.default_matter_id ?? undefined,
|
|
230
|
+
clientId: row.client_id ?? undefined,
|
|
231
|
+
retentionDays: Number(row.retention_days),
|
|
232
|
+
lastSyncAt: row.last_sync_at ?? undefined,
|
|
233
|
+
lastError: row.last_error ?? undefined,
|
|
234
|
+
createdAt: row.created_at,
|
|
235
|
+
updatedAt: row.updated_at,
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
export function insertMailAccount(store, account) {
|
|
239
|
+
const duplicate = store.db.prepare("SELECT id FROM mail_accounts WHERE lower(name) = lower(?) AND status = 'active' LIMIT 1").get(account.name)
|
|
240
|
+
if (duplicate) throw mailError(MAIL_ERROR_CODES.CONFIG, 'an active mail account named ' + JSON.stringify(account.name) + ' already exists', 'use a different --name or remove the old account first')
|
|
241
|
+
store.db.prepare(
|
|
242
|
+
'INSERT INTO mail_accounts(id,name,host,port,tls,username,mailbox,status,default_matter_id,client_id,retention_days,last_sync_at,last_error,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
|
|
243
|
+
).run(account.id, account.name, account.host, account.port, account.tls, account.username, account.mailbox, account.status, account.defaultMatterId, account.clientId, account.retentionDays, account.lastSyncAt, account.lastError, account.createdAt, account.updatedAt)
|
|
244
|
+
return account
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
export function getMailAccount(store, ref, { includeRemoved = false } = {}) {
|
|
248
|
+
if (typeof ref !== 'string' || !ref.trim()) throw mailError(MAIL_ERROR_CODES.ACCOUNT_NOT_FOUND, 'mail account is required', 'run `advisers mail account ls` to list accounts')
|
|
249
|
+
const value = ref.trim()
|
|
250
|
+
const row = store.db.prepare('SELECT * FROM mail_accounts WHERE id = ? OR lower(name) = lower(?) ORDER BY CASE WHEN id = ? THEN 0 ELSE 1 END LIMIT 1').get(value, value, value)
|
|
251
|
+
if (!row || (!includeRemoved && row.status === 'removed')) {
|
|
252
|
+
throw mailError(MAIL_ERROR_CODES.ACCOUNT_NOT_FOUND, 'mail account not found: ' + value, 'run `advisers mail account ls` to list accounts')
|
|
253
|
+
}
|
|
254
|
+
return mapAccountRow(row)
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
export function listMailAccounts(store, { includeRemoved = false, limit = 100 } = {}) {
|
|
258
|
+
const cap = Math.max(1, Math.min(500, Number(limit) || 100))
|
|
259
|
+
const rows = includeRemoved
|
|
260
|
+
? store.db.prepare('SELECT * FROM mail_accounts ORDER BY created_at ASC, id ASC LIMIT ?').all(cap)
|
|
261
|
+
: store.db.prepare("SELECT * FROM mail_accounts WHERE status = 'active' ORDER BY created_at ASC, id ASC LIMIT ?").all(cap)
|
|
262
|
+
return rows.map(mapAccountRow)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const ACCOUNT_PATCHES = {
|
|
266
|
+
name: 'name',
|
|
267
|
+
mailbox: 'mailbox',
|
|
268
|
+
status: 'status',
|
|
269
|
+
defaultMatterId: 'default_matter_id',
|
|
270
|
+
clientId: 'client_id',
|
|
271
|
+
retentionDays: 'retention_days',
|
|
272
|
+
lastSyncAt: 'last_sync_at',
|
|
273
|
+
lastError: 'last_error',
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
export function updateMailAccount(store, accountId, patch = {}, { now } = {}) {
|
|
277
|
+
const sets = []
|
|
278
|
+
const values = []
|
|
279
|
+
for (const [key, column] of Object.entries(ACCOUNT_PATCHES)) {
|
|
280
|
+
if (!(key in patch)) continue
|
|
281
|
+
let value = patch[key]
|
|
282
|
+
if (key === 'status') {
|
|
283
|
+
if (!MAIL_ACCOUNT_STATUSES.includes(value)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'unknown mail account status: ' + String(value))
|
|
284
|
+
} else if (key === 'retentionDays') {
|
|
285
|
+
value = Number(value)
|
|
286
|
+
if (!Number.isInteger(value) || value < 0 || value > MAIL_MAX_RETENTION_DAYS) throw mailError(MAIL_ERROR_CODES.CONFIG, 'retention-days must be an integer 0-' + MAIL_MAX_RETENTION_DAYS)
|
|
287
|
+
} else if (key === 'lastSyncAt') {
|
|
288
|
+
value = value == null ? null : normalizeIso(value, 'last-sync-at')
|
|
289
|
+
} else if (key === 'lastError') {
|
|
290
|
+
value = value == null ? null : singleLine(value, 300)
|
|
291
|
+
} else if (key === 'name') {
|
|
292
|
+
value = validateAccountText(value, 'name', MAIL_MAX_ACCOUNT_NAME_CHARS)
|
|
293
|
+
} else if (key === 'mailbox') {
|
|
294
|
+
value = singleLine(value, 200)
|
|
295
|
+
if (!value) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account mailbox must not be empty')
|
|
296
|
+
} else if (value != null && (typeof value !== 'string' || value.length > 200)) {
|
|
297
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail account ' + key + ' must be an entity id string')
|
|
298
|
+
}
|
|
299
|
+
sets.push(column + ' = ?')
|
|
300
|
+
values.push(value)
|
|
301
|
+
}
|
|
302
|
+
if (!sets.length) return
|
|
303
|
+
sets.push('updated_at = ?')
|
|
304
|
+
values.push(isoOf(now))
|
|
305
|
+
values.push(accountId)
|
|
306
|
+
store.db.prepare('UPDATE mail_accounts SET ' + sets.join(', ') + ' WHERE id = ?').run(...values)
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/** Transaction-internal: tombstone the account and delete its sync state. */
|
|
310
|
+
export function removeMailAccountData(store, accountId, { purgeMessages = false, now } = {}) {
|
|
311
|
+
const at = isoOf(now)
|
|
312
|
+
let messagesDeleted = 0
|
|
313
|
+
if (purgeMessages) {
|
|
314
|
+
const count = store.db.prepare('SELECT COUNT(*) AS n FROM mail_messages WHERE account_id = ?').get(accountId)
|
|
315
|
+
messagesDeleted = Number(count?.n ?? 0)
|
|
316
|
+
store.db.prepare('DELETE FROM mail_messages WHERE account_id = ?').run(accountId)
|
|
317
|
+
}
|
|
318
|
+
store.db.prepare('DELETE FROM mail_cursors WHERE account_id = ?').run(accountId)
|
|
319
|
+
store.db.prepare('DELETE FROM mail_sync_runs WHERE account_id = ?').run(accountId)
|
|
320
|
+
store.db.prepare("UPDATE mail_accounts SET status = 'removed', updated_at = ? WHERE id = ?").run(at, accountId)
|
|
321
|
+
return { messagesDeleted }
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/** Built-in account for the no-network .eml/.mbox fallback import. */
|
|
325
|
+
export function ensureLocalMailAccount(store, { now } = {}) {
|
|
326
|
+
const existing = store.db.prepare('SELECT * FROM mail_accounts WHERE id = ?').get(MAIL_LOCAL_ACCOUNT_ID)
|
|
327
|
+
if (existing) return mapAccountRow(existing)
|
|
328
|
+
const at = isoOf(now)
|
|
329
|
+
store.db.prepare(
|
|
330
|
+
"INSERT OR IGNORE INTO mail_accounts(id,name,host,port,tls,username,mailbox,status,default_matter_id,client_id,retention_days,last_sync_at,last_error,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
331
|
+
).run(MAIL_LOCAL_ACCOUNT_ID, 'Local .eml import', '(local)', 0, 'local', '(local)', MAIL_LOCAL_MAILBOX, 'active', null, null, MAIL_DEFAULT_RETENTION_DAYS, null, null, at, at)
|
|
332
|
+
return getMailAccount(store, MAIL_LOCAL_ACCOUNT_ID, { includeRemoved: true })
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/* ------------------------------------------------------------------ */
|
|
336
|
+
/* Cursors */
|
|
337
|
+
/* ------------------------------------------------------------------ */
|
|
338
|
+
|
|
339
|
+
const mapCursorRow = (row) => ({
|
|
340
|
+
accountId: row.account_id,
|
|
341
|
+
mailbox: row.mailbox,
|
|
342
|
+
uidvalidity: row.uidvalidity ?? undefined,
|
|
343
|
+
lastUid: row.last_uid == null ? undefined : Number(row.last_uid),
|
|
344
|
+
uidnext: row.uidnext ?? undefined,
|
|
345
|
+
status: row.status,
|
|
346
|
+
updatedAt: row.updated_at,
|
|
347
|
+
})
|
|
348
|
+
|
|
349
|
+
export function getMailCursor(store, accountId, mailbox) {
|
|
350
|
+
const row = store.db.prepare('SELECT * FROM mail_cursors WHERE account_id = ? AND mailbox = ?').get(accountId, mailbox)
|
|
351
|
+
return row ? mapCursorRow(row) : undefined
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Transaction-internal: upsert one mailbox cursor. */
|
|
355
|
+
export function putMailCursor(store, { accountId, mailbox, uidvalidity = null, lastUid = null, uidnext = null, status = 'ok', now } = {}) {
|
|
356
|
+
store.db.prepare(
|
|
357
|
+
'INSERT INTO mail_cursors(account_id,mailbox,uidvalidity,last_uid,uidnext,status,updated_at) VALUES(?,?,?,?,?,?,?) ' +
|
|
358
|
+
'ON CONFLICT(account_id,mailbox) DO UPDATE SET uidvalidity=excluded.uidvalidity,last_uid=excluded.last_uid,uidnext=excluded.uidnext,status=excluded.status,updated_at=excluded.updated_at',
|
|
359
|
+
).run(accountId, mailbox, uidvalidity, lastUid, uidnext, status, isoOf(now))
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/** Transaction-internal: mark the cursor as needing `mail sync --full`. */
|
|
363
|
+
export function markMailCursorNeedsResync(store, { accountId, mailbox, uidvalidity = null, uidnext = null, now } = {}) {
|
|
364
|
+
putMailCursor(store, { accountId, mailbox, uidvalidity, lastUid: null, uidnext, status: 'needs_resync', now })
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/* ------------------------------------------------------------------ */
|
|
368
|
+
/* Messages */
|
|
369
|
+
/* ------------------------------------------------------------------ */
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* Stable dedupe key for one message (see file header).
|
|
373
|
+
* @returns {string} 64-char lowercase sha256 hex
|
|
374
|
+
*/
|
|
375
|
+
export function mailDedupeKey({ accountId, mailbox = 'INBOX', uidvalidity = null, uid = null, messageId = '', from = '', subject = '', date = '', text = '' } = {}) {
|
|
376
|
+
const messageKey = normalizeMessageId(messageId)
|
|
377
|
+
if (messageKey) return sha256('message-id\u0000' + accountId + '\u0000' + messageKey)
|
|
378
|
+
if (Number.isInteger(uid) && uid > 0) return sha256('uid\u0000' + accountId + '\u0000' + mailbox + '\u0000' + String(uidvalidity ?? '') + '\u0000' + uid)
|
|
379
|
+
const bodyHash = sha256(canonicalText(text))
|
|
380
|
+
return sha256('content\u0000' + accountId + '\u0000' + canonicalText(from) + '\u0000' + canonicalText(subject) + '\u0000' + canonicalText(date) + '\u0000' + bodyHash)
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
export function mailMessageRowId(accountId, dedupeKey) {
|
|
384
|
+
return 'msg_' + sha256(accountId + '\u0000' + dedupeKey).slice(0, 32)
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* Build one mail_messages row from parsed mail. Does not touch the database.
|
|
389
|
+
* `status` defaults to 'new' when a body is retained, 'metadata' otherwise;
|
|
390
|
+
* callers pass 'oversize' for header-only rows.
|
|
391
|
+
*/
|
|
392
|
+
export function buildMailMessageRow({
|
|
393
|
+
accountId, mailbox = 'INBOX', uid = null, uidvalidity = null, sizeBytes = null,
|
|
394
|
+
receivedAt, sentAt = null, from = '', subject = '', messageId = '', text = null,
|
|
395
|
+
attachments = [], status = null, clientId = null, projectId = null, now,
|
|
396
|
+
} = {}) {
|
|
397
|
+
if (typeof accountId !== 'string' || !accountId) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail message requires an account id')
|
|
398
|
+
const received = normalizeIso(receivedAt, 'received-at')
|
|
399
|
+
if (!received) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail message requires received-at')
|
|
400
|
+
const sent = normalizeIso(sentAt, 'sent-at')
|
|
401
|
+
const bodyText = text == null ? null : truncateUtf8(String(text), MAIL_MAX_BODY_BYTES)
|
|
402
|
+
const retained = bodyText != null && bodyText.trim() !== ''
|
|
403
|
+
const dedupeKey = mailDedupeKey({ accountId, mailbox, uidvalidity, uid, messageId, from, subject, date: sent ?? received, text: retained ? bodyText : '' })
|
|
404
|
+
const at = isoOf(now)
|
|
405
|
+
return {
|
|
406
|
+
id: mailMessageRowId(accountId, dedupeKey),
|
|
407
|
+
accountId,
|
|
408
|
+
mailbox,
|
|
409
|
+
uid: Number.isInteger(uid) && uid > 0 ? uid : null,
|
|
410
|
+
uidvalidity: uidvalidity == null ? null : String(uidvalidity),
|
|
411
|
+
messageId: normalizeMessageId(messageId) || null,
|
|
412
|
+
dedupeKey,
|
|
413
|
+
fromAddr: singleLine(from, MAIL_MAX_FROM_CHARS) || null,
|
|
414
|
+
subject: singleLine(subject, MAIL_MAX_SUBJECT_CHARS) || null,
|
|
415
|
+
sentAt: sent,
|
|
416
|
+
receivedAt: received,
|
|
417
|
+
sizeBytes: Number.isFinite(Number(sizeBytes)) && Number(sizeBytes) >= 0 ? Math.floor(Number(sizeBytes)) : null,
|
|
418
|
+
bodyText: retained ? bodyText : null,
|
|
419
|
+
bodySha256: retained ? sha256(bodyText) : null,
|
|
420
|
+
attachments: normalizeAttachments(attachments),
|
|
421
|
+
clientId: clientId ?? null,
|
|
422
|
+
projectId: projectId ?? null,
|
|
423
|
+
status: status ?? (retained ? 'new' : 'metadata'),
|
|
424
|
+
candidateIds: [],
|
|
425
|
+
purgedAt: null,
|
|
426
|
+
createdAt: at,
|
|
427
|
+
updatedAt: at,
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
export function findMailMessageDuplicate(store, accountId, row) {
|
|
432
|
+
const byKey = store.db.prepare('SELECT id, uid, dedupe_key FROM mail_messages WHERE account_id = ? AND dedupe_key = ? LIMIT 1').get(accountId, row.dedupeKey)
|
|
433
|
+
if (byKey) return byKey
|
|
434
|
+
if (Number.isInteger(row.uid) && row.uid > 0) {
|
|
435
|
+
const byUid = store.db.prepare('SELECT id, uid, dedupe_key FROM mail_messages WHERE account_id = ? AND mailbox = ? AND uidvalidity IS ? AND uid = ? LIMIT 1').get(accountId, row.mailbox, row.uidvalidity, row.uid)
|
|
436
|
+
if (byUid) return byUid
|
|
437
|
+
}
|
|
438
|
+
return undefined
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/** Transaction-internal: insert one row; returns false when a unique index rejected it. */
|
|
442
|
+
export function insertMailMessage(store, row) {
|
|
443
|
+
const info = store.db.prepare(
|
|
444
|
+
'INSERT OR IGNORE INTO mail_messages(id,account_id,mailbox,uid,uidvalidity,message_id,dedupe_key,from_addr,subject,sent_at,received_at,size_bytes,body_text,body_sha256,attachments,client_id,project_id,status,candidate_ids,purged_at,created_at,updated_at) ' +
|
|
445
|
+
'VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
|
|
446
|
+
).run(row.id, row.accountId, row.mailbox, row.uid, row.uidvalidity, row.messageId, row.dedupeKey, row.fromAddr, row.subject, row.sentAt, row.receivedAt, row.sizeBytes, row.bodyText, row.bodySha256, JSON.stringify(row.attachments ?? []), row.clientId, row.projectId, row.status, JSON.stringify(row.candidateIds ?? []), row.purgedAt, row.createdAt, row.updatedAt)
|
|
447
|
+
return Number(info.changes) > 0
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
const mapMessageRow = (row, { includeBody = false } = {}) => ({
|
|
451
|
+
id: row.id,
|
|
452
|
+
accountId: row.account_id,
|
|
453
|
+
mailbox: row.mailbox,
|
|
454
|
+
uid: Number(row.uid),
|
|
455
|
+
uidvalidity: row.uidvalidity ?? undefined,
|
|
456
|
+
messageId: row.message_id ?? undefined,
|
|
457
|
+
dedupeKey: row.dedupe_key,
|
|
458
|
+
from: row.from_addr ?? '',
|
|
459
|
+
subject: row.subject ?? '',
|
|
460
|
+
sentAt: row.sent_at ?? undefined,
|
|
461
|
+
receivedAt: row.received_at,
|
|
462
|
+
sizeBytes: row.size_bytes == null ? undefined : Number(row.size_bytes),
|
|
463
|
+
retained: row.body_text != null && row.purged_at == null,
|
|
464
|
+
purgedAt: row.purged_at ?? undefined,
|
|
465
|
+
bodySha256: row.body_sha256 ?? undefined,
|
|
466
|
+
attachments: parseJson(row.attachments, []),
|
|
467
|
+
clientId: row.client_id ?? undefined,
|
|
468
|
+
projectId: row.project_id ?? undefined,
|
|
469
|
+
status: row.status,
|
|
470
|
+
candidateIds: parseJson(row.candidate_ids, []),
|
|
471
|
+
createdAt: row.created_at,
|
|
472
|
+
updatedAt: row.updated_at,
|
|
473
|
+
...(includeBody ? { bodyText: row.body_text ?? null } : {}),
|
|
474
|
+
})
|
|
475
|
+
|
|
476
|
+
export function getMailMessage(store, ref, { includeBody = false } = {}) {
|
|
477
|
+
if (typeof ref !== 'string' || !ref.trim()) throw mailError(MAIL_ERROR_CODES.MESSAGE_NOT_FOUND, 'mail message id is required')
|
|
478
|
+
const value = ref.trim()
|
|
479
|
+
const stripped = normalizeMessageId(value) || value
|
|
480
|
+
const row = value.startsWith('msg_')
|
|
481
|
+
? store.db.prepare('SELECT * FROM mail_messages WHERE id = ?').get(value)
|
|
482
|
+
: store.db.prepare('SELECT * FROM mail_messages WHERE message_id = ? ORDER BY received_at DESC LIMIT 1').get(stripped)
|
|
483
|
+
if (!row) throw mailError(MAIL_ERROR_CODES.MESSAGE_NOT_FOUND, 'mail message not found: ' + value, 'run `advisers mail ls` to list messages')
|
|
484
|
+
return mapMessageRow(row, { includeBody })
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
const MESSAGE_PATCHES = {
|
|
488
|
+
status: 'status',
|
|
489
|
+
clientId: 'client_id',
|
|
490
|
+
projectId: 'project_id',
|
|
491
|
+
bodyText: 'body_text',
|
|
492
|
+
bodySha256: 'body_sha256',
|
|
493
|
+
purgedAt: 'purged_at',
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
/** Transaction-internal: update one message row. */
|
|
497
|
+
export function updateMailMessage(store, id, patch = {}, { now } = {}) {
|
|
498
|
+
const sets = []
|
|
499
|
+
const values = []
|
|
500
|
+
for (const [key, column] of Object.entries(MESSAGE_PATCHES)) {
|
|
501
|
+
if (!(key in patch)) continue
|
|
502
|
+
let value = patch[key]
|
|
503
|
+
if (key === 'status') {
|
|
504
|
+
if (!MAIL_MESSAGE_STATUSES.includes(value)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'unknown mail message status: ' + String(value))
|
|
505
|
+
} else if (key === 'bodyText') {
|
|
506
|
+
value = value == null ? null : truncateUtf8(String(value), MAIL_MAX_BODY_BYTES)
|
|
507
|
+
}
|
|
508
|
+
sets.push(column + ' = ?')
|
|
509
|
+
values.push(value)
|
|
510
|
+
}
|
|
511
|
+
if ('candidateIds' in patch && Array.isArray(patch.candidateIds)) {
|
|
512
|
+
sets.push('candidate_ids = ?')
|
|
513
|
+
values.push(JSON.stringify([...new Set(patch.candidateIds.filter((candidateId) => typeof candidateId === 'string' && candidateId))]))
|
|
514
|
+
}
|
|
515
|
+
if (!sets.length) return getMailMessage(store, id)
|
|
516
|
+
sets.push('updated_at = ?')
|
|
517
|
+
values.push(isoOf(now))
|
|
518
|
+
values.push(id)
|
|
519
|
+
store.db.prepare('UPDATE mail_messages SET ' + sets.join(', ') + ' WHERE id = ?').run(...values)
|
|
520
|
+
return getMailMessage(store, id)
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/** Transaction-internal: append candidate ids to a message's reverse pointer. */
|
|
524
|
+
export function addMailCandidateIds(store, id, candidateIds, { now } = {}) {
|
|
525
|
+
const current = store.db.prepare('SELECT candidate_ids FROM mail_messages WHERE id = ?').get(id)
|
|
526
|
+
if (!current) return
|
|
527
|
+
const merged = [...new Set([...parseJson(current.candidate_ids, []), ...candidateIds])]
|
|
528
|
+
store.db.prepare('UPDATE mail_messages SET candidate_ids = ?, updated_at = ? WHERE id = ?').run(JSON.stringify(merged), isoOf(now), id)
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
function messageWhere(filters = {}) {
|
|
532
|
+
const where = []
|
|
533
|
+
const params = []
|
|
534
|
+
if (filters.accountId) { where.push('account_id = ?'); params.push(filters.accountId) }
|
|
535
|
+
if (filters.clientId) { where.push('client_id = ?'); params.push(filters.clientId) }
|
|
536
|
+
if (filters.projectId) { where.push('project_id = ?'); params.push(filters.projectId) }
|
|
537
|
+
if (filters.status) {
|
|
538
|
+
const statuses = Array.isArray(filters.status) ? filters.status : [filters.status]
|
|
539
|
+
where.push('status IN (' + statuses.map(() => '?').join(',') + ')')
|
|
540
|
+
params.push(...statuses)
|
|
541
|
+
}
|
|
542
|
+
if (filters.retained === true) where.push('body_text IS NOT NULL')
|
|
543
|
+
if (filters.retained === false) where.push('body_text IS NULL')
|
|
544
|
+
if (filters.sinceDays != null && Number(filters.sinceDays) >= 0) {
|
|
545
|
+
const cutoff = new Date(nowDate(filters.now).getTime() - Number(filters.sinceDays) * DAY_MS).toISOString()
|
|
546
|
+
where.push('received_at >= ?')
|
|
547
|
+
params.push(cutoff)
|
|
548
|
+
}
|
|
549
|
+
return { sql: where.length ? ' WHERE ' + where.join(' AND ') : '', params }
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/**
|
|
553
|
+
* List message summaries (never the body). `newest first`, capped page.
|
|
554
|
+
*/
|
|
555
|
+
export function listMailMessages(store, filters = {}) {
|
|
556
|
+
const limit = Math.max(1, Math.min(MAIL_MAX_LS_LIMIT, Number(filters.limit) || MAIL_DEFAULT_LS_LIMIT))
|
|
557
|
+
const offset = Math.max(0, Math.floor(Number(filters.offset) || 0))
|
|
558
|
+
const where = messageWhere(filters)
|
|
559
|
+
const total = Number(store.db.prepare('SELECT COUNT(*) AS n FROM mail_messages' + where.sql).get(...where.params)?.n ?? 0)
|
|
560
|
+
const rows = store.db.prepare('SELECT * FROM mail_messages' + where.sql + ' ORDER BY received_at DESC, uid DESC LIMIT ? OFFSET ?').all(...where.params, limit, offset)
|
|
561
|
+
return { messages: rows.map((row) => mapMessageRow(row)), total, limit, offset, truncated: offset + rows.length < total }
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
/** Transaction-read: rows a `mail extract` run should scan (body retained). */
|
|
565
|
+
export function listMessagesForExtraction(store, { accountId, ids, all = false, limit = 20, now } = {}) {
|
|
566
|
+
const cap = Math.max(1, Math.min(MAIL_MAX_SYNC_LIMIT, Number(limit) || 20))
|
|
567
|
+
if (Array.isArray(ids) && ids.length) {
|
|
568
|
+
const rows = []
|
|
569
|
+
for (const ref of ids.slice(0, cap)) {
|
|
570
|
+
const value = String(ref).trim()
|
|
571
|
+
const stripped = normalizeMessageId(value) || value
|
|
572
|
+
const row = value.startsWith('msg_')
|
|
573
|
+
? store.db.prepare('SELECT * FROM mail_messages WHERE id = ?').get(value)
|
|
574
|
+
: store.db.prepare('SELECT * FROM mail_messages WHERE message_id = ? ORDER BY received_at DESC LIMIT 1').get(stripped)
|
|
575
|
+
if (row) rows.push(row)
|
|
576
|
+
}
|
|
577
|
+
return rows.map((row) => mapMessageRow(row, { includeBody: true }))
|
|
578
|
+
}
|
|
579
|
+
const statuses = all ? ['new', 'extracted', 'linked'] : ['new']
|
|
580
|
+
const params = [...statuses]
|
|
581
|
+
let sql = 'SELECT * FROM mail_messages WHERE body_text IS NOT NULL AND purged_at IS NULL AND status IN (' + statuses.map(() => '?').join(',') + ')'
|
|
582
|
+
if (accountId) { sql += ' AND account_id = ?'; params.push(accountId) }
|
|
583
|
+
sql += ' ORDER BY received_at ASC, uid ASC LIMIT ?'
|
|
584
|
+
params.push(cap)
|
|
585
|
+
return store.db.prepare(sql).all(...params).map((row) => mapMessageRow(row, { includeBody: true }))
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/** Transaction-read: rows with a retained body, for retention/purge planning. */
|
|
589
|
+
export function listMessagesWithBody(store, { accountId, before } = {}) {
|
|
590
|
+
const params = []
|
|
591
|
+
let sql = 'SELECT * FROM mail_messages WHERE body_text IS NOT NULL AND purged_at IS NULL'
|
|
592
|
+
if (accountId) { sql += ' AND account_id = ?'; params.push(accountId) }
|
|
593
|
+
if (before) { sql += ' AND received_at < ?'; params.push(before) }
|
|
594
|
+
sql += ' ORDER BY received_at ASC LIMIT 2000'
|
|
595
|
+
return store.db.prepare(sql).all(...params).map((row) => mapMessageRow(row, { includeBody: true }))
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
/** Transaction-internal: clear retained bodies (metadata + body_sha256 stay). */
|
|
599
|
+
export function purgeMailBodies(store, ids, { now } = {}) {
|
|
600
|
+
const at = isoOf(now)
|
|
601
|
+
let purged = 0
|
|
602
|
+
for (const id of ids) {
|
|
603
|
+
const info = store.db.prepare('UPDATE mail_messages SET body_text = NULL, purged_at = ?, updated_at = ? WHERE id = ? AND body_text IS NOT NULL').run(at, at, id)
|
|
604
|
+
purged += Number(info.changes)
|
|
605
|
+
}
|
|
606
|
+
return purged
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
export function nextLocalUid(store, accountId, mailbox = MAIL_LOCAL_MAILBOX) {
|
|
610
|
+
const row = store.db.prepare('SELECT MAX(uid) AS max_uid FROM mail_messages WHERE account_id = ? AND mailbox = ?').get(accountId, mailbox)
|
|
611
|
+
return Number(row?.max_uid ?? 0) + 1
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/* ------------------------------------------------------------------ */
|
|
615
|
+
/* Sync runs + one-transaction batch apply */
|
|
616
|
+
/* ------------------------------------------------------------------ */
|
|
617
|
+
|
|
618
|
+
/** Transaction-internal: append one mail_sync_runs row. */
|
|
619
|
+
export function insertMailSyncRun(store, run = {}) {
|
|
620
|
+
const id = run.id ?? randomId('sync_')
|
|
621
|
+
store.db.prepare(
|
|
622
|
+
'INSERT INTO mail_sync_runs(id,account_id,mailbox,started_at,finished_at,status,fetched,inserted,skipped,failed,uidvalidity,last_uid_before,last_uid_after,error_code) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)',
|
|
623
|
+
).run(id, run.accountId, run.mailbox, run.startedAt, run.finishedAt, run.status ?? 'ok', Number(run.fetched) || 0, Number(run.inserted) || 0, Number(run.skipped) || 0, Number(run.failed) || 0, run.uidvalidity ?? null, run.lastUidBefore ?? null, run.lastUidAfter ?? null, run.errorCode ?? null)
|
|
624
|
+
return id
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/**
|
|
628
|
+
* Transaction-internal: write a fetched batch, advance the cursor and record
|
|
629
|
+
* the run in one BEGIN IMMEDIATE. The caller (lib/mail.js) performs the
|
|
630
|
+
* network fetch BEFORE calling this, so no DB transaction is held across the
|
|
631
|
+
* network.
|
|
632
|
+
*
|
|
633
|
+
* Cursor rule: never advance past a UID that failed to parse/fetch. On a
|
|
634
|
+
* UIDVALIDITY change (only reachable with --full) the old UID space is
|
|
635
|
+
* discarded instead of being carried over.
|
|
636
|
+
*/
|
|
637
|
+
export function applyMailSyncBatch(store, batch = {}) {
|
|
638
|
+
const { accountId, mailbox, uidvalidity = null, messages = [], failedUids = [], uidnext = null, startedAt, now } = batch
|
|
639
|
+
const at = isoOf(now)
|
|
640
|
+
const failed = new Set((failedUids ?? []).filter((uid) => Number.isInteger(uid)))
|
|
641
|
+
let inserted = 0
|
|
642
|
+
let skipped = 0
|
|
643
|
+
let duplicates = 0
|
|
644
|
+
let maxCommittedUid = null
|
|
645
|
+
for (const row of messages) {
|
|
646
|
+
if (failed.has(row.uid)) continue
|
|
647
|
+
row.uidvalidity = uidvalidity
|
|
648
|
+
const duplicate = findMailMessageDuplicate(store, accountId, row)
|
|
649
|
+
if (duplicate) {
|
|
650
|
+
skipped += 1
|
|
651
|
+
duplicates += 1
|
|
652
|
+
if (Number.isInteger(row.uid)) maxCommittedUid = Math.max(maxCommittedUid ?? 0, row.uid)
|
|
653
|
+
continue
|
|
654
|
+
}
|
|
655
|
+
if (insertMailMessage(store, row)) inserted += 1
|
|
656
|
+
else { skipped += 1; duplicates += 1 }
|
|
657
|
+
if (Number.isInteger(row.uid)) maxCommittedUid = Math.max(maxCommittedUid ?? 0, row.uid)
|
|
658
|
+
}
|
|
659
|
+
let advanceTo = maxCommittedUid
|
|
660
|
+
if (failed.size) {
|
|
661
|
+
const cap = Math.min(...failed) - 1
|
|
662
|
+
advanceTo = cap < 1 ? null : advanceTo == null ? null : Math.min(advanceTo, cap)
|
|
663
|
+
}
|
|
664
|
+
const previous = batch.previousCursor
|
|
665
|
+
const previousUidValidity = previous?.uidvalidity ?? null
|
|
666
|
+
const previousLast = previous?.lastUid ?? null
|
|
667
|
+
const uidvalidityChanged = Boolean(previousUidValidity != null && uidvalidity != null && String(previousUidValidity) !== String(uidvalidity))
|
|
668
|
+
let lastUid
|
|
669
|
+
if (uidvalidityChanged) {
|
|
670
|
+
// The old UID space no longer exists: never carry the old cursor over.
|
|
671
|
+
lastUid = advanceTo
|
|
672
|
+
} else if (advanceTo == null) {
|
|
673
|
+
lastUid = previousLast
|
|
674
|
+
} else if (previousLast == null) {
|
|
675
|
+
lastUid = advanceTo
|
|
676
|
+
} else {
|
|
677
|
+
lastUid = Math.max(previousLast, advanceTo)
|
|
678
|
+
}
|
|
679
|
+
putMailCursor(store, {
|
|
680
|
+
accountId, mailbox,
|
|
681
|
+
uidvalidity: uidvalidity ?? previousUidValidity,
|
|
682
|
+
lastUid,
|
|
683
|
+
uidnext,
|
|
684
|
+
status: 'ok',
|
|
685
|
+
now,
|
|
686
|
+
})
|
|
687
|
+
updateMailAccount(store, accountId, { lastSyncAt: at, lastError: null }, { now })
|
|
688
|
+
insertMailSyncRun(store, {
|
|
689
|
+
accountId, mailbox, startedAt: startedAt ?? at, finishedAt: at, status: 'ok',
|
|
690
|
+
fetched: batch.fetched ?? messages.length + failed.size,
|
|
691
|
+
inserted, skipped, failed: failed.size,
|
|
692
|
+
uidvalidity: uidvalidity ?? previousUidValidity ?? null,
|
|
693
|
+
lastUidBefore: previousLast,
|
|
694
|
+
lastUidAfter: lastUid,
|
|
695
|
+
errorCode: null,
|
|
696
|
+
})
|
|
697
|
+
return { inserted, skipped, duplicates, failed: failed.size, lastUidBefore: previousLast, lastUidAfter: lastUid, uidvalidityChanged }
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
/* ------------------------------------------------------------------ */
|
|
701
|
+
/* Retention / purge planning */
|
|
702
|
+
/* ------------------------------------------------------------------ */
|
|
703
|
+
|
|
704
|
+
/**
|
|
705
|
+
* Plan a retention purge: for every active account, bodies of messages older
|
|
706
|
+
* than its retention window (or `olderThanDays` when given). Messages that
|
|
707
|
+
* still have a pending memory candidate are skipped so an unreviewed
|
|
708
|
+
* candidate can never lose its quote.
|
|
709
|
+
*/
|
|
710
|
+
export function planMailPurge(store, { now, olderThanDays, accountId } = {}) {
|
|
711
|
+
const at = nowDate(now)
|
|
712
|
+
const accounts = listMailAccounts(store).filter((account) => !accountId || account.id === accountId)
|
|
713
|
+
const targets = []
|
|
714
|
+
let skippedPending = 0
|
|
715
|
+
let bytesFreed = 0
|
|
716
|
+
for (const account of accounts) {
|
|
717
|
+
const days = olderThanDays == null ? account.retentionDays : Number(olderThanDays)
|
|
718
|
+
if (!Number.isFinite(days) || days < 0) continue
|
|
719
|
+
if (olderThanDays == null && days === 0) continue // 0 = keep bodies until an explicit purge
|
|
720
|
+
const cutoff = new Date(at.getTime() - days * DAY_MS).toISOString()
|
|
721
|
+
const rows = listMessagesWithBody(store, { accountId: account.id, before: cutoff })
|
|
722
|
+
for (const row of rows) {
|
|
723
|
+
const pending = (row.candidateIds ?? []).some((candidateId) => store.getCandidate(candidateId)?.status === 'pending')
|
|
724
|
+
if (pending) { skippedPending += 1; continue }
|
|
725
|
+
const bytes = Buffer.byteLength(row.bodyText ?? '', 'utf8')
|
|
726
|
+
bytesFreed += bytes
|
|
727
|
+
targets.push({ id: row.id, accountId: row.accountId, bytes })
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
return { targets, skippedPending, bytesFreed, accountsScanned: accounts.length }
|
|
731
|
+
}
|