advisers 0.3.2 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README-zh.md +13 -14
  2. package/README.md +37 -23
  3. package/THIRD-PARTY-NOTICES.md +1 -1
  4. package/bin/advisers.js +2302 -95
  5. package/index.js +1 -1
  6. package/lib/brief-archive.js +267 -0
  7. package/lib/client-pack.js +1016 -0
  8. package/lib/consent.js +390 -0
  9. package/lib/credentials.js +195 -0
  10. package/lib/error-codes.js +516 -7
  11. package/lib/evidence.js +697 -113
  12. package/lib/kb-extract.js +626 -0
  13. package/lib/kb-facts.js +429 -0
  14. package/lib/kb-pages.js +1084 -0
  15. package/lib/kb.js +1036 -0
  16. package/lib/mail-errors.js +46 -0
  17. package/lib/mail-extract.js +351 -0
  18. package/lib/mail-imap.js +446 -0
  19. package/lib/mail-import.js +176 -0
  20. package/lib/mail-parse.js +223 -0
  21. package/lib/mail-store.js +731 -0
  22. package/lib/mail.js +506 -0
  23. package/lib/mcp-server.js +196 -10
  24. package/lib/notify.js +265 -7
  25. package/lib/policy/decision.js +315 -0
  26. package/lib/policy/egress.js +268 -0
  27. package/lib/policy/errors.js +71 -0
  28. package/lib/policy/index.js +99 -0
  29. package/lib/policy/rules.js +468 -0
  30. package/lib/policy/surfaces.js +141 -0
  31. package/lib/receipt-verify.js +199 -19
  32. package/lib/receivables.js +1653 -0
  33. package/lib/scheduler.js +453 -9
  34. package/lib/shell.js +1406 -0
  35. package/lib/source.js +98 -3
  36. package/lib/sqlite.js +366 -49
  37. package/lib/tos/index.js +2 -2
  38. package/lib/tos/manifest.json +6 -6
  39. package/lib/tos/privacy.zh-CN.md +33 -20
  40. package/lib/tos/safety.zh-CN.md +7 -6
  41. package/lib/tos/summary.zh-CN.md +3 -2
  42. package/lib/tos/terms.zh-CN.md +31 -16
  43. package/lib/urgency.js +467 -0
  44. package/package.json +4 -4
@@ -0,0 +1,446 @@
1
+ /**
2
+ * Read-only IMAP session for the 0.4.0 mail entry.
3
+ *
4
+ * This is deliberately a separate, stricter client from lib/imap.js (which
5
+ * stays untouched for the deprecated `capture --from-imap` path):
6
+ *
7
+ * - TLS ONLY. The product path is implicit TLS on 993 (`tls.connect`,
8
+ * rejectUnauthorized: true, servername = host) or an opt-in STARTTLS
9
+ * upgrade on 143. There is no `secure:false` flag and no cleartext
10
+ * fallback: a server that does not advertise STARTTLS is refused with
11
+ * E_IMAP_TLS and the session is torn down before LOGIN.
12
+ * - READ-ONLY command allowlist: CAPABILITY, LOGIN, STARTTLS, EXAMINE,
13
+ * UID SEARCH, UID FETCH (BODY.PEEK[...]), LOGOUT. No SELECT, STORE,
14
+ * APPEND, EXPUNGE, COPY, MOVE, DELETE. The command log never contains the
15
+ * password (LOGIN is recorded redacted).
16
+ * - UIDVALIDITY/UIDNEXT are parsed from the EXAMINE response; sync cursor
17
+ * policy lives in lib/mail.js.
18
+ * - Two-phase fetch: first `UID RFC822.SIZE BODY.PEEK[HEADER.FIELDS ...]` for
19
+ * every candidate, then full `BODY.PEEK[]` only for messages at or below
20
+ * maxMessageBytes. Oversize messages are returned header-only; attachment
21
+ * bytes are never written to disk.
22
+ * - Every remote string in an error is sanitised (no ANSI/OSC/bidi, one line)
23
+ * and the server banner/certificate details are never echoed.
24
+ *
25
+ * `connect` is a TEST-ONLY injection point (a plain loopback socket factory,
26
+ * the same pattern lib/imap.js already exposes). When it is supplied the
27
+ * module talks the mock protocol without TLS; lib/mail.js never supplies it.
28
+ */
29
+ import net from 'node:net'
30
+ import tls from 'node:tls'
31
+ import { imapQuote, imapSinceDate } from './imap.js'
32
+ import { sanitizeConsoleText } from './text-safe.js'
33
+ import { isMailError, mailError, MAIL_ERROR_CODES } from './mail-errors.js'
34
+
35
+ export const DEFAULT_TIMEOUT_MS = 30_000
36
+ export const MAX_LITERAL_BYTES = 25 * 1024 * 1024
37
+ export const MAX_FETCH_CHUNK = 25
38
+ export const READ_ONLY_COMMANDS = Object.freeze(['CAPABILITY', 'LOGIN', 'STARTTLS', 'EXAMINE', 'LOGOUT'])
39
+ const FORBIDDEN_COMMANDS = ['SELECT', 'STORE', 'APPEND', 'EXPUNGE', 'COPY', 'MOVE', 'DELETE', 'CREATE', 'RENAME', 'SUBSCRIBE']
40
+ const CRLF = '\r\n'
41
+
42
+ function timeoutError() {
43
+ const error = new Error('IMAP request timed out')
44
+ error.imapTimeout = true
45
+ return error
46
+ }
47
+
48
+ function literalTooLargeError(size) {
49
+ const error = new Error('IMAP literal is larger than the configured limit (' + size + ' bytes)')
50
+ error.imapTooLarge = true
51
+ return error
52
+ }
53
+
54
+ function createReader(socket, { idleTimeoutMs = DEFAULT_TIMEOUT_MS, maxLiteralBytes = MAX_LITERAL_BYTES } = {}) {
55
+ let buffer = Buffer.alloc(0)
56
+ let ended = false
57
+ let error
58
+ let wake = null
59
+ let timer = null
60
+ let deadline = Number.POSITIVE_INFINITY
61
+
62
+ const clearTimer = () => { if (timer) { clearTimeout(timer); timer = null } }
63
+ const wakeUp = () => { clearTimer(); if (wake) { const w = wake; wake = null; w() } }
64
+ const onData = (chunk) => { buffer = Buffer.concat([buffer, chunk]); wakeUp() }
65
+ const onEnd = () => { ended = true; wakeUp() }
66
+ const onError = (e) => { error = e; wakeUp() }
67
+ socket.on('data', onData)
68
+ socket.on('end', onEnd)
69
+ socket.on('error', onError)
70
+
71
+ const waitData = () => new Promise((resolve) => {
72
+ if (buffer.length || ended || error) return resolve()
73
+ const remaining = deadline - Date.now()
74
+ if (remaining <= 0) { error = timeoutError(); return resolve() }
75
+ wake = resolve
76
+ timer = setTimeout(() => {
77
+ timer = null
78
+ error = timeoutError()
79
+ wakeUp()
80
+ }, Math.max(1, Math.min(idleTimeoutMs, remaining)))
81
+ })
82
+
83
+ async function readLine() {
84
+ for (;;) {
85
+ const index = buffer.indexOf('\r\n')
86
+ if (index >= 0) { const line = buffer.subarray(0, index).toString('latin1'); buffer = buffer.subarray(index + 2); return line }
87
+ if (error) throw error
88
+ if (ended) throw new Error('IMAP connection ended before a complete line')
89
+ await waitData()
90
+ }
91
+ }
92
+
93
+ async function readBytes(size) {
94
+ if (size > maxLiteralBytes) throw literalTooLargeError(size)
95
+ while (buffer.length < size) {
96
+ if (error) throw error
97
+ if (ended) throw new Error('IMAP connection ended inside a literal')
98
+ await waitData()
99
+ }
100
+ const data = buffer.subarray(0, size); buffer = buffer.subarray(size); return data
101
+ }
102
+
103
+ async function response(tag) {
104
+ const records = []
105
+ for (;;) {
106
+ const line = await readLine()
107
+ if (line.startsWith(tag + ' ')) return { records, statusLine: line }
108
+ const record = { line, literal: undefined }
109
+ records.push(record)
110
+ const match = /\{(\d+)\}$/.exec(line)
111
+ if (match) record.literal = await readBytes(Number(match[1]))
112
+ }
113
+ }
114
+
115
+ return {
116
+ begin(timeoutMs) { deadline = Date.now() + Math.max(1, timeoutMs) },
117
+ readLine,
118
+ response,
119
+ clearTimer,
120
+ dispose() {
121
+ clearTimer()
122
+ try { socket.off('data', onData) } catch {}
123
+ try { socket.off('end', onEnd) } catch {}
124
+ try { socket.off('error', onError) } catch {}
125
+ },
126
+ }
127
+ }
128
+
129
+ function mapSessionError(error, phase) {
130
+ if (error && typeof error.code === 'string' && error.code.startsWith('E_')) return error
131
+ const message = String(error?.message ?? error)
132
+ if (error?.imapTimeout || /timed out/i.test(message)) {
133
+ return mailError(MAIL_ERROR_CODES.IMAP_TIMEOUT, 'IMAP request timed out', 'check the network and the server; advisors never retries a cleartext connection')
134
+ }
135
+ if (error?.imapTooLarge) {
136
+ return mailError(MAIL_ERROR_CODES.MAIL_TOO_LARGE, 'an IMAP response exceeded the literal size limit', 'increase --max-message-bytes only if you trust the server and have memory to spare')
137
+ }
138
+ const safe = sanitizeConsoleText(message, { max: 300 })
139
+ if (phase === 'tls' || /certificate|handshake|ssl|tls|eproto/i.test(message)) {
140
+ return mailError(MAIL_ERROR_CODES.IMAP_TLS, 'IMAP TLS connection failed: ' + safe, 'the server certificate must be valid for the configured host; advisers never falls back to cleartext')
141
+ }
142
+ return mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'IMAP protocol error: ' + safe)
143
+ }
144
+
145
+ function openTlsSocket({ host, port, timeoutMs, rejectUnauthorized = true }) {
146
+ return new Promise((resolve, reject) => {
147
+ let settled = false
148
+ let socket
149
+ const timer = setTimeout(() => {
150
+ if (settled) return
151
+ settled = true
152
+ try { socket?.destroy?.() } catch {}
153
+ reject(mailError(MAIL_ERROR_CODES.IMAP_TIMEOUT, 'IMAP TLS connect timed out after ' + timeoutMs + 'ms'))
154
+ }, timeoutMs)
155
+ const finish = (fn, value) => {
156
+ if (settled) return
157
+ settled = true
158
+ clearTimeout(timer)
159
+ fn(value)
160
+ }
161
+ try {
162
+ socket = tls.connect({ host, port, servername: net.isIP(host) ? undefined : host, rejectUnauthorized })
163
+ } catch (error) {
164
+ finish(reject, mapSessionError(error, 'tls'))
165
+ return
166
+ }
167
+ socket.once('secureConnect', () => finish(resolve, socket))
168
+ socket.once('error', (error) => finish(reject, mapSessionError(error, 'tls')))
169
+ })
170
+ }
171
+
172
+ function openPlainSocket({ host, port, timeoutMs }) {
173
+ return new Promise((resolve, reject) => {
174
+ let settled = false
175
+ let socket
176
+ const timer = setTimeout(() => {
177
+ if (settled) return
178
+ settled = true
179
+ try { socket?.destroy?.() } catch {}
180
+ reject(mailError(MAIL_ERROR_CODES.IMAP_TIMEOUT, 'IMAP connect timed out after ' + timeoutMs + 'ms'))
181
+ }, timeoutMs)
182
+ const finish = (fn, value) => {
183
+ if (settled) return
184
+ settled = true
185
+ clearTimeout(timer)
186
+ fn(value)
187
+ }
188
+ try {
189
+ socket = net.connect({ host, port })
190
+ } catch (error) {
191
+ finish(reject, mapSessionError(error, 'connect'))
192
+ return
193
+ }
194
+ socket.once('connect', () => finish(resolve, socket))
195
+ socket.once('error', (error) => finish(reject, mapSessionError(error, 'connect')))
196
+ })
197
+ }
198
+
199
+ function upgradeToTls(socket, { host, timeoutMs }) {
200
+ return new Promise((resolve, reject) => {
201
+ let settled = false
202
+ let secure
203
+ const timer = setTimeout(() => {
204
+ if (settled) return
205
+ settled = true
206
+ try { secure?.destroy?.() } catch {}
207
+ reject(mailError(MAIL_ERROR_CODES.IMAP_TLS, 'STARTTLS handshake timed out'))
208
+ }, timeoutMs)
209
+ const finish = (fn, value) => {
210
+ if (settled) return
211
+ settled = true
212
+ clearTimeout(timer)
213
+ fn(value)
214
+ }
215
+ try {
216
+ secure = tls.connect({ socket, servername: net.isIP(host) ? undefined : host, rejectUnauthorized: true })
217
+ } catch (error) {
218
+ finish(reject, mapSessionError(error, 'tls'))
219
+ return
220
+ }
221
+ secure.once('secureConnect', () => finish(resolve, secure))
222
+ secure.once('error', (error) => finish(reject, mapSessionError(error, 'tls')))
223
+ })
224
+ }
225
+
226
+ function parseCapabilities(records) {
227
+ const capabilities = []
228
+ for (const record of records) {
229
+ const match = /^\* CAPABILITY (.+)$/i.exec(record.line)
230
+ if (match) capabilities.push(...match[1].trim().split(/\s+/))
231
+ }
232
+ return capabilities
233
+ }
234
+
235
+ function parseSearch(records) {
236
+ const uids = []
237
+ for (const record of records) {
238
+ const match = /^\* SEARCH(?: (.+))?$/i.exec(record.line)
239
+ if (!match) continue
240
+ for (const token of String(match[1] ?? '').trim().split(/\s+/)) {
241
+ if (/^\d+$/.test(token)) uids.push(Number(token))
242
+ }
243
+ }
244
+ return uids
245
+ }
246
+
247
+ function parseFetchRecords(records) {
248
+ const out = []
249
+ for (const record of records) {
250
+ if (!/^\* \d+ FETCH/i.test(record.line)) continue
251
+ const uid = /\bUID (\d+)\b/i.exec(record.line)
252
+ const size = /\bRFC822\.SIZE (\d+)\b/i.exec(record.line)
253
+ out.push({
254
+ uid: uid ? Number(uid[1]) : null,
255
+ size: size ? Number(size[1]) : null,
256
+ literal: record.literal,
257
+ })
258
+ }
259
+ return out
260
+ }
261
+
262
+ function assertNotForbidden(commandText) {
263
+ const verb = String(commandText).trim().split(/\s+/)[0]?.toUpperCase()
264
+ const top = verb === 'UID' ? String(commandText).trim().split(/\s+/)[1]?.toUpperCase() : verb
265
+ if (FORBIDDEN_COMMANDS.includes(top) || FORBIDDEN_COMMANDS.includes(verb)) {
266
+ throw mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'refusing non-read-only IMAP command: ' + top)
267
+ }
268
+ }
269
+
270
+ /**
271
+ * Open one read-only IMAP session (implicit TLS or STARTTLS) and LOGIN.
272
+ * The returned session exposes capability/examine/search/fetch/logout.
273
+ */
274
+ export async function openMailImapSession(options = {}) {
275
+ const {
276
+ host, port, tls: mode = 'tls', user, pass, connect,
277
+ timeoutMs = DEFAULT_TIMEOUT_MS, idleTimeoutMs = timeoutMs,
278
+ maxLiteralBytes = MAX_LITERAL_BYTES,
279
+ } = options
280
+ if (!host || !user || !pass) throw mailError(MAIL_ERROR_CODES.CONFIG, 'IMAP host/user/password are required')
281
+ if (!['tls', 'starttls'].includes(mode)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'unsupported IMAP transport: ' + String(mode), 'use tls (993) or starttls (143)')
282
+ const testSocket = typeof connect === 'function'
283
+ let socket
284
+ let reader
285
+ const commands = []
286
+ let tagNo = 0
287
+ let authenticated = false
288
+
289
+ const closeSocket = () => {
290
+ try { reader?.dispose?.() } catch {}
291
+ try { socket?.end?.() } catch {}
292
+ try { socket?.destroy?.() } catch {}
293
+ }
294
+
295
+ const begin = () => { reader.begin(timeoutMs) }
296
+
297
+ async function rawCommand(text) {
298
+ assertNotForbidden(text)
299
+ const tag = 'A' + (++tagNo)
300
+ const verb = String(text).trim().split(/\s+/)[0]?.toUpperCase()
301
+ commands.push(verb === 'LOGIN' ? 'LOGIN [redacted]' : String(text))
302
+ begin()
303
+ socket.write(tag + ' ' + text + CRLF)
304
+ const response = await reader.response(tag)
305
+ const match = /^\S+ (OK|NO|BAD)\b(.*)$/i.exec(response.statusLine)
306
+ if (!match) throw mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'IMAP server sent an unparseable status line')
307
+ if (match[1].toUpperCase() !== 'OK') {
308
+ const statusText = sanitizeConsoleText(String(match[2] ?? '').trim(), { max: 300 })
309
+ if (verb === 'LOGIN') throw mailError(MAIL_ERROR_CODES.IMAP_AUTH, 'IMAP authentication failed', 'check the account username and app password; modern providers may require OAuth (not supported)')
310
+ if (verb === 'STARTTLS') throw mailError(MAIL_ERROR_CODES.IMAP_TLS, 'the server refused STARTTLS', 'advisers never falls back to cleartext; use implicit TLS on 993')
311
+ throw mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'IMAP ' + verb + ' was rejected by the server' + (statusText ? ': ' + statusText : ''))
312
+ }
313
+ return response
314
+ }
315
+
316
+ const readGreeting = async () => {
317
+ begin()
318
+ const greeting = await reader.readLine()
319
+ const match = /^\* (OK|PREAUTH)\b/i.exec(greeting)
320
+ if (!match) throw mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'IMAP server did not greet with OK/PREAUTH')
321
+ const capabilities = []
322
+ const capabilityResponse = await rawCommand('CAPABILITY')
323
+ capabilities.push(...parseCapabilities(capabilityResponse.records))
324
+ return { preauthenticated: match[1].toUpperCase() === 'PREAUTH', capabilities }
325
+ }
326
+
327
+ try {
328
+ if (testSocket) {
329
+ socket = await connect({ host, port, tls: mode })
330
+ } else if (mode === 'tls') {
331
+ socket = await openTlsSocket({ host, port, timeoutMs })
332
+ } else {
333
+ socket = await openPlainSocket({ host, port, timeoutMs })
334
+ }
335
+ reader = createReader(socket, { idleTimeoutMs, maxLiteralBytes })
336
+ const greeting = await readGreeting()
337
+ if (mode === 'starttls' && !testSocket) {
338
+ if (!greeting.capabilities.map((item) => item.toUpperCase()).includes('STARTTLS')) {
339
+ throw mailError(MAIL_ERROR_CODES.IMAP_TLS, 'the server does not advertise STARTTLS', 'use implicit TLS on 993; advisors never sends credentials in cleartext')
340
+ }
341
+ await rawCommand('STARTTLS')
342
+ reader.dispose()
343
+ try { socket = await upgradeToTls(socket, { host, timeoutMs }) } catch (error) {
344
+ throw isMailError(error) ? error : mapSessionError(error, 'tls')
345
+ }
346
+ reader = createReader(socket, { idleTimeoutMs, maxLiteralBytes })
347
+ const postTls = await rawCommand('CAPABILITY')
348
+ greeting.capabilities.push(...parseCapabilities(postTls.records))
349
+ }
350
+ if (!greeting.preauthenticated) await rawCommand('LOGIN ' + imapQuote(user) + ' ' + imapQuote(pass))
351
+ authenticated = true
352
+ } catch (error) {
353
+ closeSocket()
354
+ if (isMailError(error)) throw error
355
+ throw mapSessionError(error, mode === 'tls' ? 'tls' : 'connect')
356
+ }
357
+
358
+ const session = {
359
+ host,
360
+ transport: mode,
361
+ commands,
362
+ get authenticated() { return authenticated },
363
+
364
+ async capability() {
365
+ const response = await rawCommand('CAPABILITY')
366
+ return parseCapabilities(response.records)
367
+ },
368
+
369
+ async examine(mailbox = 'INBOX') {
370
+ const response = await rawCommand('EXAMINE ' + imapQuote(mailbox))
371
+ const lines = response.records.map((record) => record.line).concat([response.statusLine])
372
+ let exists = null
373
+ let uidValidity = null
374
+ let uidNext = null
375
+ for (const line of lines) {
376
+ const existsMatch = /^\* (\d+) EXISTS/i.exec(line)
377
+ if (existsMatch) exists = Number(existsMatch[1])
378
+ const validity = /\[UIDVALIDITY (\d+)\]/i.exec(line)
379
+ if (validity) uidValidity = validity[1]
380
+ const next = /\[UIDNEXT (\d+)\]/i.exec(line)
381
+ if (next) uidNext = next[1]
382
+ }
383
+ return { mailbox, exists, uidValidity, uidNext }
384
+ },
385
+
386
+ async searchSince(days = 30) {
387
+ const response = await rawCommand('UID SEARCH SINCE ' + imapSinceDate(days))
388
+ return [...new Set(parseSearch(response.records))].sort((a, b) => a - b)
389
+ },
390
+
391
+ async searchUidRange(fromUid) {
392
+ const start = Math.max(1, Math.floor(Number(fromUid) || 1))
393
+ const response = await rawCommand('UID SEARCH UID ' + start + ':*')
394
+ return [...new Set(parseSearch(response.records).filter((uid) => uid >= start))].sort((a, b) => a - b)
395
+ },
396
+
397
+ async fetchMessages(uids = [], { maxMessageBytes = 5 * 1024 * 1024 } = {}) {
398
+ const wanted = [...new Set(uids.filter((uid) => Number.isInteger(uid) && uid > 0))].sort((a, b) => a - b)
399
+ if (!wanted.length) return { messages: [], failedUids: [], requested: 0 }
400
+ const headResponse = await rawCommand('UID FETCH ' + wanted.join(',') + ' (UID RFC822.SIZE BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE MESSAGE-ID)])')
401
+ const metas = parseFetchRecords(headResponse.records)
402
+ const byUid = new Map()
403
+ for (const meta of metas) if (meta.uid != null) byUid.set(meta.uid, meta)
404
+ const failedUids = []
405
+ const messages = []
406
+ const needBody = []
407
+ for (const uid of wanted) {
408
+ const meta = byUid.get(uid)
409
+ if (!meta) { failedUids.push(uid); continue }
410
+ if (meta.size != null && meta.size > maxMessageBytes) {
411
+ messages.push({ uid, size: meta.size, headerRaw: meta.literal ?? null, raw: null, oversize: true })
412
+ } else {
413
+ needBody.push(uid)
414
+ }
415
+ }
416
+ const bodies = new Map()
417
+ for (let index = 0; index < needBody.length; index += MAX_FETCH_CHUNK) {
418
+ const chunk = needBody.slice(index, index + MAX_FETCH_CHUNK)
419
+ const response = await rawCommand('UID FETCH ' + chunk.join(',') + ' (UID BODY.PEEK[])')
420
+ for (const record of parseFetchRecords(response.records)) {
421
+ if (record.uid == null || !record.literal) continue
422
+ bodies.set(record.uid, record.literal)
423
+ }
424
+ }
425
+ for (const uid of needBody) {
426
+ const raw = bodies.get(uid)
427
+ if (!raw) { failedUids.push(uid); continue }
428
+ const meta = byUid.get(uid)
429
+ messages.push({ uid, size: meta?.size ?? raw.length, headerRaw: meta?.literal ?? null, raw, oversize: false })
430
+ }
431
+ return { messages, failedUids, requested: wanted.length }
432
+ },
433
+
434
+ async logout() {
435
+ try { await rawCommand('LOGOUT') } catch {}
436
+ authenticated = false
437
+ closeSocket()
438
+ },
439
+
440
+ destroy() {
441
+ authenticated = false
442
+ closeSocket()
443
+ },
444
+ }
445
+ return session
446
+ }
@@ -0,0 +1,176 @@
1
+ /**
2
+ * Offline .eml/.mbox directory import for the 0.4.0 mail entry.
3
+ *
4
+ * This is the fallback for users whose provider disabled password IMAP
5
+ * (Google Workspace / Microsoft 365): no network, no credentials, no IMAP
6
+ * code path. Imported messages land in mail_messages under a built-in local
7
+ * account (or a caller-supplied account) and then flow through the same
8
+ * `mail extract` / candidate review pipeline.
9
+ *
10
+ * Attachments are metadata only (name/size/mime); no attachment bytes are
11
+ * written anywhere. Empty-body messages are skipped, not silently stored.
12
+ */
13
+ import { readFileSync, readdirSync, statSync } from 'node:fs'
14
+ import { extname, join } from 'node:path'
15
+ import { parseMailMessage } from './mail-parse.js'
16
+ import { mailError, MAIL_ERROR_CODES } from './mail-errors.js'
17
+ import {
18
+ assertMailTables, buildMailMessageRow, ensureLocalMailAccount, findMailMessageDuplicate,
19
+ getMailAccount, insertMailMessage, nextLocalUid, isoOf, MAIL_LOCAL_ACCOUNT_ID, MAIL_LOCAL_MAILBOX,
20
+ } from './mail-store.js'
21
+
22
+ export const EML_EXTENSIONS = Object.freeze(['.eml', '.mbox'])
23
+ export const DEFAULT_IMPORT_FILE_LIMIT = 100
24
+ export const MAX_IMPORT_FILE_LIMIT = 500
25
+ export const MAX_IMPORT_MESSAGES_PER_FILE = 500
26
+
27
+ function isEmlPath(path) {
28
+ return EML_EXTENSIONS.includes(extname(String(path)).toLowerCase())
29
+ }
30
+
31
+ function listImportFiles(target) {
32
+ if (typeof target !== 'string' || !target.trim()) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail import requires a .eml file or a directory')
33
+ let stat
34
+ try { stat = statSync(target) } catch (error) {
35
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'cannot read mail path: ' + String(error?.message ?? error).slice(0, 200))
36
+ }
37
+ if (stat.isFile()) {
38
+ if (!isEmlPath(target)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'not a .eml/.mbox file: ' + target)
39
+ return [target]
40
+ }
41
+ if (!stat.isDirectory()) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail import path is not a file or directory: ' + target)
42
+ try {
43
+ return readdirSync(target, { withFileTypes: true })
44
+ .filter((entry) => entry.isFile() && isEmlPath(entry.name))
45
+ .map((entry) => join(target, entry.name))
46
+ .sort()
47
+ } catch (error) {
48
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'cannot list mail directory: ' + String(error?.message ?? error).slice(0, 200))
49
+ }
50
+ }
51
+
52
+ /** Split an mbox stream into raw RFC 822 messages (keeping each raw). */
53
+ export function splitMboxRaw(raw) {
54
+ const source = String(raw == null ? '' : raw).replace(/^\uFEFF/, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n')
55
+ if (!source.trim()) return []
56
+ const chunks = []
57
+ let current = []
58
+ for (const line of source.split('\n')) {
59
+ if (line.startsWith('From ') && current.length) {
60
+ chunks.push(current.join('\n'))
61
+ current = []
62
+ }
63
+ current.push(line)
64
+ }
65
+ if (current.length) chunks.push(current.join('\n'))
66
+ return chunks
67
+ .map((chunk) => chunk.replace(/^From [^\n]*\n/, ''))
68
+ .filter((chunk) => chunk.trim())
69
+ }
70
+
71
+ function parseDateHeader(value) {
72
+ if (!value) return undefined
73
+ const parsed = Date.parse(value)
74
+ return Number.isFinite(parsed) ? new Date(parsed).toISOString() : undefined
75
+ }
76
+
77
+ function readMessages(file) {
78
+ let raw
79
+ try { raw = readFileSync(file, 'utf8') } catch (error) {
80
+ throw mailError(MAIL_ERROR_CODES.CONFIG, 'cannot read ' + file + ': ' + String(error?.message ?? error).slice(0, 200))
81
+ }
82
+ const isMbox = extname(file).toLowerCase() === '.mbox'
83
+ const raws = isMbox ? splitMboxRaw(raw) : [raw]
84
+ return raws.slice(0, MAX_IMPORT_MESSAGES_PER_FILE)
85
+ }
86
+
87
+ /**
88
+ * Import .eml/.mbox file(s) into mail_messages. No network, no credentials.
89
+ *
90
+ * @param {object} store open SqliteStore
91
+ * @param {{ path?: string, files?: string[], accountId?: string, clientId?: string,
92
+ * projectId?: string, limit?: number, dryRun?: boolean }} options
93
+ * @param {{ now?: Date }} [context]
94
+ * @returns {{ ok: true, command: 'mail import', accountId: string, dryRun: boolean,
95
+ * files: number, scanned: number, inserted: number, skipped: number, failed: number }}
96
+ */
97
+ export async function importMailFiles(store, options = {}, context = {}) {
98
+ assertMailTables(store)
99
+ const now = context.now instanceof Date ? context.now : new Date(context.now ?? Date.now())
100
+ const dryRun = options.dryRun === true
101
+ const limit = Math.max(1, Math.min(MAX_IMPORT_FILE_LIMIT, Number(options.limit) || DEFAULT_IMPORT_FILE_LIMIT))
102
+ const requested = Array.isArray(options.files) && options.files.length
103
+ ? options.files.filter(isEmlPath).slice(0, limit)
104
+ : listImportFiles(options.path ?? options.dir ?? options.target).slice(0, limit)
105
+ if (!requested.length) throw mailError(MAIL_ERROR_CODES.CONFIG, 'no .eml/.mbox files found to import')
106
+ let account
107
+ if (options.accountId) {
108
+ account = getMailAccount(store, options.accountId)
109
+ if (!['tls', 'starttls', 'local'].includes(account.tls)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail import requires a local or IMAP account')
110
+ } else if (dryRun) {
111
+ // A dry run must not even create the built-in local account row.
112
+ account = { id: MAIL_LOCAL_ACCOUNT_ID, mailbox: MAIL_LOCAL_MAILBOX, tls: 'local' }
113
+ } else {
114
+ account = store.mutate(() => ensureLocalMailAccount(store, { now })).result
115
+ }
116
+ const rows = []
117
+ let failed = 0
118
+ let scanned = 0
119
+ for (const file of requested) {
120
+ let raws
121
+ try { raws = readMessages(file) } catch { failed += 1; continue }
122
+ for (const raw of raws) {
123
+ scanned += 1
124
+ const parsed = parseMailMessage(raw)
125
+ const text = String(parsed.text ?? '')
126
+ if (!text.trim()) continue
127
+ const sentAt = parseDateHeader(parsed.date)
128
+ rows.push(buildMailMessageRow({
129
+ accountId: account.id,
130
+ mailbox: account.mailbox,
131
+ uid: null,
132
+ uidvalidity: 'local',
133
+ sizeBytes: parsed.sizeBytes,
134
+ receivedAt: sentAt ?? isoOf(now),
135
+ sentAt: sentAt ?? null,
136
+ from: parsed.from,
137
+ subject: parsed.subject,
138
+ messageId: parsed.messageId,
139
+ text,
140
+ attachments: parsed.attachments,
141
+ status: 'new',
142
+ clientId: options.clientId ?? null,
143
+ projectId: options.projectId ?? null,
144
+ now,
145
+ }))
146
+ }
147
+ }
148
+ const skippedEmpty = scanned - rows.length
149
+ if (dryRun) {
150
+ let duplicates = 0
151
+ for (const row of rows) if (findMailMessageDuplicate(store, account.id, row)) duplicates += 1
152
+ return {
153
+ ok: true, command: 'mail import', accountId: account.id, dryRun: true,
154
+ files: requested.length, scanned, inserted: 0,
155
+ skipped: skippedEmpty + duplicates, failed,
156
+ }
157
+ }
158
+ const audit = { actor: 'local', action: 'mail.import', allowed: true, reason: '' }
159
+ const result = store.mutate(() => {
160
+ let uid = nextLocalUid(store, account.id, account.mailbox)
161
+ let inserted = 0
162
+ let skipped = skippedEmpty
163
+ for (const row of rows) {
164
+ if (findMailMessageDuplicate(store, account.id, row)) { skipped += 1; continue }
165
+ row.uid = uid
166
+ if (insertMailMessage(store, row)) { inserted += 1; uid += 1 } else { skipped += 1 }
167
+ }
168
+ audit.reason = inserted + ' imported, ' + skipped + ' skipped, ' + failed + ' failed'
169
+ return { inserted, skipped }
170
+ }, audit)
171
+ return {
172
+ ok: true, command: 'mail import', accountId: account.id, dryRun: false,
173
+ files: requested.length, scanned, inserted: result.result.inserted,
174
+ skipped: result.result.skipped, failed,
175
+ }
176
+ }