advisers 0.3.2 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh.md +13 -14
- package/README.md +37 -23
- package/THIRD-PARTY-NOTICES.md +1 -1
- package/bin/advisers.js +2302 -95
- package/index.js +1 -1
- package/lib/brief-archive.js +267 -0
- package/lib/client-pack.js +1016 -0
- package/lib/consent.js +390 -0
- package/lib/credentials.js +195 -0
- package/lib/error-codes.js +516 -7
- package/lib/evidence.js +697 -113
- package/lib/kb-extract.js +626 -0
- package/lib/kb-facts.js +429 -0
- package/lib/kb-pages.js +1084 -0
- package/lib/kb.js +1036 -0
- package/lib/mail-errors.js +46 -0
- package/lib/mail-extract.js +351 -0
- package/lib/mail-imap.js +446 -0
- package/lib/mail-import.js +176 -0
- package/lib/mail-parse.js +223 -0
- package/lib/mail-store.js +731 -0
- package/lib/mail.js +506 -0
- package/lib/mcp-server.js +196 -10
- package/lib/notify.js +265 -7
- package/lib/policy/decision.js +315 -0
- package/lib/policy/egress.js +268 -0
- package/lib/policy/errors.js +71 -0
- package/lib/policy/index.js +99 -0
- package/lib/policy/rules.js +468 -0
- package/lib/policy/surfaces.js +141 -0
- package/lib/receipt-verify.js +199 -19
- package/lib/receivables.js +1653 -0
- package/lib/scheduler.js +453 -9
- package/lib/shell.js +1406 -0
- package/lib/source.js +98 -3
- package/lib/sqlite.js +366 -49
- package/lib/tos/index.js +2 -2
- package/lib/tos/manifest.json +6 -6
- package/lib/tos/privacy.zh-CN.md +33 -20
- package/lib/tos/safety.zh-CN.md +7 -6
- package/lib/tos/summary.zh-CN.md +3 -2
- package/lib/tos/terms.zh-CN.md +31 -16
- package/lib/urgency.js +467 -0
- package/package.json +4 -4
package/lib/mail-imap.js
ADDED
|
@@ -0,0 +1,446 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Read-only IMAP session for the 0.4.0 mail entry.
|
|
3
|
+
*
|
|
4
|
+
* This is deliberately a separate, stricter client from lib/imap.js (which
|
|
5
|
+
* stays untouched for the deprecated `capture --from-imap` path):
|
|
6
|
+
*
|
|
7
|
+
* - TLS ONLY. The product path is implicit TLS on 993 (`tls.connect`,
|
|
8
|
+
* rejectUnauthorized: true, servername = host) or an opt-in STARTTLS
|
|
9
|
+
* upgrade on 143. There is no `secure:false` flag and no cleartext
|
|
10
|
+
* fallback: a server that does not advertise STARTTLS is refused with
|
|
11
|
+
* E_IMAP_TLS and the session is torn down before LOGIN.
|
|
12
|
+
* - READ-ONLY command allowlist: CAPABILITY, LOGIN, STARTTLS, EXAMINE,
|
|
13
|
+
* UID SEARCH, UID FETCH (BODY.PEEK[...]), LOGOUT. No SELECT, STORE,
|
|
14
|
+
* APPEND, EXPUNGE, COPY, MOVE, DELETE. The command log never contains the
|
|
15
|
+
* password (LOGIN is recorded redacted).
|
|
16
|
+
* - UIDVALIDITY/UIDNEXT are parsed from the EXAMINE response; sync cursor
|
|
17
|
+
* policy lives in lib/mail.js.
|
|
18
|
+
* - Two-phase fetch: first `UID RFC822.SIZE BODY.PEEK[HEADER.FIELDS ...]` for
|
|
19
|
+
* every candidate, then full `BODY.PEEK[]` only for messages at or below
|
|
20
|
+
* maxMessageBytes. Oversize messages are returned header-only; attachment
|
|
21
|
+
* bytes are never written to disk.
|
|
22
|
+
* - Every remote string in an error is sanitised (no ANSI/OSC/bidi, one line)
|
|
23
|
+
* and the server banner/certificate details are never echoed.
|
|
24
|
+
*
|
|
25
|
+
* `connect` is a TEST-ONLY injection point (a plain loopback socket factory,
|
|
26
|
+
* the same pattern lib/imap.js already exposes). When it is supplied the
|
|
27
|
+
* module talks the mock protocol without TLS; lib/mail.js never supplies it.
|
|
28
|
+
*/
|
|
29
|
+
import net from 'node:net'
|
|
30
|
+
import tls from 'node:tls'
|
|
31
|
+
import { imapQuote, imapSinceDate } from './imap.js'
|
|
32
|
+
import { sanitizeConsoleText } from './text-safe.js'
|
|
33
|
+
import { isMailError, mailError, MAIL_ERROR_CODES } from './mail-errors.js'
|
|
34
|
+
|
|
35
|
+
export const DEFAULT_TIMEOUT_MS = 30_000
|
|
36
|
+
export const MAX_LITERAL_BYTES = 25 * 1024 * 1024
|
|
37
|
+
export const MAX_FETCH_CHUNK = 25
|
|
38
|
+
export const READ_ONLY_COMMANDS = Object.freeze(['CAPABILITY', 'LOGIN', 'STARTTLS', 'EXAMINE', 'LOGOUT'])
|
|
39
|
+
const FORBIDDEN_COMMANDS = ['SELECT', 'STORE', 'APPEND', 'EXPUNGE', 'COPY', 'MOVE', 'DELETE', 'CREATE', 'RENAME', 'SUBSCRIBE']
|
|
40
|
+
const CRLF = '\r\n'
|
|
41
|
+
|
|
42
|
+
function timeoutError() {
|
|
43
|
+
const error = new Error('IMAP request timed out')
|
|
44
|
+
error.imapTimeout = true
|
|
45
|
+
return error
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function literalTooLargeError(size) {
|
|
49
|
+
const error = new Error('IMAP literal is larger than the configured limit (' + size + ' bytes)')
|
|
50
|
+
error.imapTooLarge = true
|
|
51
|
+
return error
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function createReader(socket, { idleTimeoutMs = DEFAULT_TIMEOUT_MS, maxLiteralBytes = MAX_LITERAL_BYTES } = {}) {
|
|
55
|
+
let buffer = Buffer.alloc(0)
|
|
56
|
+
let ended = false
|
|
57
|
+
let error
|
|
58
|
+
let wake = null
|
|
59
|
+
let timer = null
|
|
60
|
+
let deadline = Number.POSITIVE_INFINITY
|
|
61
|
+
|
|
62
|
+
const clearTimer = () => { if (timer) { clearTimeout(timer); timer = null } }
|
|
63
|
+
const wakeUp = () => { clearTimer(); if (wake) { const w = wake; wake = null; w() } }
|
|
64
|
+
const onData = (chunk) => { buffer = Buffer.concat([buffer, chunk]); wakeUp() }
|
|
65
|
+
const onEnd = () => { ended = true; wakeUp() }
|
|
66
|
+
const onError = (e) => { error = e; wakeUp() }
|
|
67
|
+
socket.on('data', onData)
|
|
68
|
+
socket.on('end', onEnd)
|
|
69
|
+
socket.on('error', onError)
|
|
70
|
+
|
|
71
|
+
const waitData = () => new Promise((resolve) => {
|
|
72
|
+
if (buffer.length || ended || error) return resolve()
|
|
73
|
+
const remaining = deadline - Date.now()
|
|
74
|
+
if (remaining <= 0) { error = timeoutError(); return resolve() }
|
|
75
|
+
wake = resolve
|
|
76
|
+
timer = setTimeout(() => {
|
|
77
|
+
timer = null
|
|
78
|
+
error = timeoutError()
|
|
79
|
+
wakeUp()
|
|
80
|
+
}, Math.max(1, Math.min(idleTimeoutMs, remaining)))
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
async function readLine() {
|
|
84
|
+
for (;;) {
|
|
85
|
+
const index = buffer.indexOf('\r\n')
|
|
86
|
+
if (index >= 0) { const line = buffer.subarray(0, index).toString('latin1'); buffer = buffer.subarray(index + 2); return line }
|
|
87
|
+
if (error) throw error
|
|
88
|
+
if (ended) throw new Error('IMAP connection ended before a complete line')
|
|
89
|
+
await waitData()
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function readBytes(size) {
|
|
94
|
+
if (size > maxLiteralBytes) throw literalTooLargeError(size)
|
|
95
|
+
while (buffer.length < size) {
|
|
96
|
+
if (error) throw error
|
|
97
|
+
if (ended) throw new Error('IMAP connection ended inside a literal')
|
|
98
|
+
await waitData()
|
|
99
|
+
}
|
|
100
|
+
const data = buffer.subarray(0, size); buffer = buffer.subarray(size); return data
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async function response(tag) {
|
|
104
|
+
const records = []
|
|
105
|
+
for (;;) {
|
|
106
|
+
const line = await readLine()
|
|
107
|
+
if (line.startsWith(tag + ' ')) return { records, statusLine: line }
|
|
108
|
+
const record = { line, literal: undefined }
|
|
109
|
+
records.push(record)
|
|
110
|
+
const match = /\{(\d+)\}$/.exec(line)
|
|
111
|
+
if (match) record.literal = await readBytes(Number(match[1]))
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return {
|
|
116
|
+
begin(timeoutMs) { deadline = Date.now() + Math.max(1, timeoutMs) },
|
|
117
|
+
readLine,
|
|
118
|
+
response,
|
|
119
|
+
clearTimer,
|
|
120
|
+
dispose() {
|
|
121
|
+
clearTimer()
|
|
122
|
+
try { socket.off('data', onData) } catch {}
|
|
123
|
+
try { socket.off('end', onEnd) } catch {}
|
|
124
|
+
try { socket.off('error', onError) } catch {}
|
|
125
|
+
},
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function mapSessionError(error, phase) {
|
|
130
|
+
if (error && typeof error.code === 'string' && error.code.startsWith('E_')) return error
|
|
131
|
+
const message = String(error?.message ?? error)
|
|
132
|
+
if (error?.imapTimeout || /timed out/i.test(message)) {
|
|
133
|
+
return mailError(MAIL_ERROR_CODES.IMAP_TIMEOUT, 'IMAP request timed out', 'check the network and the server; advisors never retries a cleartext connection')
|
|
134
|
+
}
|
|
135
|
+
if (error?.imapTooLarge) {
|
|
136
|
+
return mailError(MAIL_ERROR_CODES.MAIL_TOO_LARGE, 'an IMAP response exceeded the literal size limit', 'increase --max-message-bytes only if you trust the server and have memory to spare')
|
|
137
|
+
}
|
|
138
|
+
const safe = sanitizeConsoleText(message, { max: 300 })
|
|
139
|
+
if (phase === 'tls' || /certificate|handshake|ssl|tls|eproto/i.test(message)) {
|
|
140
|
+
return mailError(MAIL_ERROR_CODES.IMAP_TLS, 'IMAP TLS connection failed: ' + safe, 'the server certificate must be valid for the configured host; advisers never falls back to cleartext')
|
|
141
|
+
}
|
|
142
|
+
return mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'IMAP protocol error: ' + safe)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function openTlsSocket({ host, port, timeoutMs, rejectUnauthorized = true }) {
|
|
146
|
+
return new Promise((resolve, reject) => {
|
|
147
|
+
let settled = false
|
|
148
|
+
let socket
|
|
149
|
+
const timer = setTimeout(() => {
|
|
150
|
+
if (settled) return
|
|
151
|
+
settled = true
|
|
152
|
+
try { socket?.destroy?.() } catch {}
|
|
153
|
+
reject(mailError(MAIL_ERROR_CODES.IMAP_TIMEOUT, 'IMAP TLS connect timed out after ' + timeoutMs + 'ms'))
|
|
154
|
+
}, timeoutMs)
|
|
155
|
+
const finish = (fn, value) => {
|
|
156
|
+
if (settled) return
|
|
157
|
+
settled = true
|
|
158
|
+
clearTimeout(timer)
|
|
159
|
+
fn(value)
|
|
160
|
+
}
|
|
161
|
+
try {
|
|
162
|
+
socket = tls.connect({ host, port, servername: net.isIP(host) ? undefined : host, rejectUnauthorized })
|
|
163
|
+
} catch (error) {
|
|
164
|
+
finish(reject, mapSessionError(error, 'tls'))
|
|
165
|
+
return
|
|
166
|
+
}
|
|
167
|
+
socket.once('secureConnect', () => finish(resolve, socket))
|
|
168
|
+
socket.once('error', (error) => finish(reject, mapSessionError(error, 'tls')))
|
|
169
|
+
})
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function openPlainSocket({ host, port, timeoutMs }) {
|
|
173
|
+
return new Promise((resolve, reject) => {
|
|
174
|
+
let settled = false
|
|
175
|
+
let socket
|
|
176
|
+
const timer = setTimeout(() => {
|
|
177
|
+
if (settled) return
|
|
178
|
+
settled = true
|
|
179
|
+
try { socket?.destroy?.() } catch {}
|
|
180
|
+
reject(mailError(MAIL_ERROR_CODES.IMAP_TIMEOUT, 'IMAP connect timed out after ' + timeoutMs + 'ms'))
|
|
181
|
+
}, timeoutMs)
|
|
182
|
+
const finish = (fn, value) => {
|
|
183
|
+
if (settled) return
|
|
184
|
+
settled = true
|
|
185
|
+
clearTimeout(timer)
|
|
186
|
+
fn(value)
|
|
187
|
+
}
|
|
188
|
+
try {
|
|
189
|
+
socket = net.connect({ host, port })
|
|
190
|
+
} catch (error) {
|
|
191
|
+
finish(reject, mapSessionError(error, 'connect'))
|
|
192
|
+
return
|
|
193
|
+
}
|
|
194
|
+
socket.once('connect', () => finish(resolve, socket))
|
|
195
|
+
socket.once('error', (error) => finish(reject, mapSessionError(error, 'connect')))
|
|
196
|
+
})
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function upgradeToTls(socket, { host, timeoutMs }) {
|
|
200
|
+
return new Promise((resolve, reject) => {
|
|
201
|
+
let settled = false
|
|
202
|
+
let secure
|
|
203
|
+
const timer = setTimeout(() => {
|
|
204
|
+
if (settled) return
|
|
205
|
+
settled = true
|
|
206
|
+
try { secure?.destroy?.() } catch {}
|
|
207
|
+
reject(mailError(MAIL_ERROR_CODES.IMAP_TLS, 'STARTTLS handshake timed out'))
|
|
208
|
+
}, timeoutMs)
|
|
209
|
+
const finish = (fn, value) => {
|
|
210
|
+
if (settled) return
|
|
211
|
+
settled = true
|
|
212
|
+
clearTimeout(timer)
|
|
213
|
+
fn(value)
|
|
214
|
+
}
|
|
215
|
+
try {
|
|
216
|
+
secure = tls.connect({ socket, servername: net.isIP(host) ? undefined : host, rejectUnauthorized: true })
|
|
217
|
+
} catch (error) {
|
|
218
|
+
finish(reject, mapSessionError(error, 'tls'))
|
|
219
|
+
return
|
|
220
|
+
}
|
|
221
|
+
secure.once('secureConnect', () => finish(resolve, secure))
|
|
222
|
+
secure.once('error', (error) => finish(reject, mapSessionError(error, 'tls')))
|
|
223
|
+
})
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function parseCapabilities(records) {
|
|
227
|
+
const capabilities = []
|
|
228
|
+
for (const record of records) {
|
|
229
|
+
const match = /^\* CAPABILITY (.+)$/i.exec(record.line)
|
|
230
|
+
if (match) capabilities.push(...match[1].trim().split(/\s+/))
|
|
231
|
+
}
|
|
232
|
+
return capabilities
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function parseSearch(records) {
|
|
236
|
+
const uids = []
|
|
237
|
+
for (const record of records) {
|
|
238
|
+
const match = /^\* SEARCH(?: (.+))?$/i.exec(record.line)
|
|
239
|
+
if (!match) continue
|
|
240
|
+
for (const token of String(match[1] ?? '').trim().split(/\s+/)) {
|
|
241
|
+
if (/^\d+$/.test(token)) uids.push(Number(token))
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return uids
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function parseFetchRecords(records) {
|
|
248
|
+
const out = []
|
|
249
|
+
for (const record of records) {
|
|
250
|
+
if (!/^\* \d+ FETCH/i.test(record.line)) continue
|
|
251
|
+
const uid = /\bUID (\d+)\b/i.exec(record.line)
|
|
252
|
+
const size = /\bRFC822\.SIZE (\d+)\b/i.exec(record.line)
|
|
253
|
+
out.push({
|
|
254
|
+
uid: uid ? Number(uid[1]) : null,
|
|
255
|
+
size: size ? Number(size[1]) : null,
|
|
256
|
+
literal: record.literal,
|
|
257
|
+
})
|
|
258
|
+
}
|
|
259
|
+
return out
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function assertNotForbidden(commandText) {
|
|
263
|
+
const verb = String(commandText).trim().split(/\s+/)[0]?.toUpperCase()
|
|
264
|
+
const top = verb === 'UID' ? String(commandText).trim().split(/\s+/)[1]?.toUpperCase() : verb
|
|
265
|
+
if (FORBIDDEN_COMMANDS.includes(top) || FORBIDDEN_COMMANDS.includes(verb)) {
|
|
266
|
+
throw mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'refusing non-read-only IMAP command: ' + top)
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Open one read-only IMAP session (implicit TLS or STARTTLS) and LOGIN.
|
|
272
|
+
* The returned session exposes capability/examine/search/fetch/logout.
|
|
273
|
+
*/
|
|
274
|
+
export async function openMailImapSession(options = {}) {
|
|
275
|
+
const {
|
|
276
|
+
host, port, tls: mode = 'tls', user, pass, connect,
|
|
277
|
+
timeoutMs = DEFAULT_TIMEOUT_MS, idleTimeoutMs = timeoutMs,
|
|
278
|
+
maxLiteralBytes = MAX_LITERAL_BYTES,
|
|
279
|
+
} = options
|
|
280
|
+
if (!host || !user || !pass) throw mailError(MAIL_ERROR_CODES.CONFIG, 'IMAP host/user/password are required')
|
|
281
|
+
if (!['tls', 'starttls'].includes(mode)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'unsupported IMAP transport: ' + String(mode), 'use tls (993) or starttls (143)')
|
|
282
|
+
const testSocket = typeof connect === 'function'
|
|
283
|
+
let socket
|
|
284
|
+
let reader
|
|
285
|
+
const commands = []
|
|
286
|
+
let tagNo = 0
|
|
287
|
+
let authenticated = false
|
|
288
|
+
|
|
289
|
+
const closeSocket = () => {
|
|
290
|
+
try { reader?.dispose?.() } catch {}
|
|
291
|
+
try { socket?.end?.() } catch {}
|
|
292
|
+
try { socket?.destroy?.() } catch {}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
const begin = () => { reader.begin(timeoutMs) }
|
|
296
|
+
|
|
297
|
+
async function rawCommand(text) {
|
|
298
|
+
assertNotForbidden(text)
|
|
299
|
+
const tag = 'A' + (++tagNo)
|
|
300
|
+
const verb = String(text).trim().split(/\s+/)[0]?.toUpperCase()
|
|
301
|
+
commands.push(verb === 'LOGIN' ? 'LOGIN [redacted]' : String(text))
|
|
302
|
+
begin()
|
|
303
|
+
socket.write(tag + ' ' + text + CRLF)
|
|
304
|
+
const response = await reader.response(tag)
|
|
305
|
+
const match = /^\S+ (OK|NO|BAD)\b(.*)$/i.exec(response.statusLine)
|
|
306
|
+
if (!match) throw mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'IMAP server sent an unparseable status line')
|
|
307
|
+
if (match[1].toUpperCase() !== 'OK') {
|
|
308
|
+
const statusText = sanitizeConsoleText(String(match[2] ?? '').trim(), { max: 300 })
|
|
309
|
+
if (verb === 'LOGIN') throw mailError(MAIL_ERROR_CODES.IMAP_AUTH, 'IMAP authentication failed', 'check the account username and app password; modern providers may require OAuth (not supported)')
|
|
310
|
+
if (verb === 'STARTTLS') throw mailError(MAIL_ERROR_CODES.IMAP_TLS, 'the server refused STARTTLS', 'advisers never falls back to cleartext; use implicit TLS on 993')
|
|
311
|
+
throw mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'IMAP ' + verb + ' was rejected by the server' + (statusText ? ': ' + statusText : ''))
|
|
312
|
+
}
|
|
313
|
+
return response
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
const readGreeting = async () => {
|
|
317
|
+
begin()
|
|
318
|
+
const greeting = await reader.readLine()
|
|
319
|
+
const match = /^\* (OK|PREAUTH)\b/i.exec(greeting)
|
|
320
|
+
if (!match) throw mailError(MAIL_ERROR_CODES.IMAP_PROTOCOL, 'IMAP server did not greet with OK/PREAUTH')
|
|
321
|
+
const capabilities = []
|
|
322
|
+
const capabilityResponse = await rawCommand('CAPABILITY')
|
|
323
|
+
capabilities.push(...parseCapabilities(capabilityResponse.records))
|
|
324
|
+
return { preauthenticated: match[1].toUpperCase() === 'PREAUTH', capabilities }
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
try {
|
|
328
|
+
if (testSocket) {
|
|
329
|
+
socket = await connect({ host, port, tls: mode })
|
|
330
|
+
} else if (mode === 'tls') {
|
|
331
|
+
socket = await openTlsSocket({ host, port, timeoutMs })
|
|
332
|
+
} else {
|
|
333
|
+
socket = await openPlainSocket({ host, port, timeoutMs })
|
|
334
|
+
}
|
|
335
|
+
reader = createReader(socket, { idleTimeoutMs, maxLiteralBytes })
|
|
336
|
+
const greeting = await readGreeting()
|
|
337
|
+
if (mode === 'starttls' && !testSocket) {
|
|
338
|
+
if (!greeting.capabilities.map((item) => item.toUpperCase()).includes('STARTTLS')) {
|
|
339
|
+
throw mailError(MAIL_ERROR_CODES.IMAP_TLS, 'the server does not advertise STARTTLS', 'use implicit TLS on 993; advisors never sends credentials in cleartext')
|
|
340
|
+
}
|
|
341
|
+
await rawCommand('STARTTLS')
|
|
342
|
+
reader.dispose()
|
|
343
|
+
try { socket = await upgradeToTls(socket, { host, timeoutMs }) } catch (error) {
|
|
344
|
+
throw isMailError(error) ? error : mapSessionError(error, 'tls')
|
|
345
|
+
}
|
|
346
|
+
reader = createReader(socket, { idleTimeoutMs, maxLiteralBytes })
|
|
347
|
+
const postTls = await rawCommand('CAPABILITY')
|
|
348
|
+
greeting.capabilities.push(...parseCapabilities(postTls.records))
|
|
349
|
+
}
|
|
350
|
+
if (!greeting.preauthenticated) await rawCommand('LOGIN ' + imapQuote(user) + ' ' + imapQuote(pass))
|
|
351
|
+
authenticated = true
|
|
352
|
+
} catch (error) {
|
|
353
|
+
closeSocket()
|
|
354
|
+
if (isMailError(error)) throw error
|
|
355
|
+
throw mapSessionError(error, mode === 'tls' ? 'tls' : 'connect')
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const session = {
|
|
359
|
+
host,
|
|
360
|
+
transport: mode,
|
|
361
|
+
commands,
|
|
362
|
+
get authenticated() { return authenticated },
|
|
363
|
+
|
|
364
|
+
async capability() {
|
|
365
|
+
const response = await rawCommand('CAPABILITY')
|
|
366
|
+
return parseCapabilities(response.records)
|
|
367
|
+
},
|
|
368
|
+
|
|
369
|
+
async examine(mailbox = 'INBOX') {
|
|
370
|
+
const response = await rawCommand('EXAMINE ' + imapQuote(mailbox))
|
|
371
|
+
const lines = response.records.map((record) => record.line).concat([response.statusLine])
|
|
372
|
+
let exists = null
|
|
373
|
+
let uidValidity = null
|
|
374
|
+
let uidNext = null
|
|
375
|
+
for (const line of lines) {
|
|
376
|
+
const existsMatch = /^\* (\d+) EXISTS/i.exec(line)
|
|
377
|
+
if (existsMatch) exists = Number(existsMatch[1])
|
|
378
|
+
const validity = /\[UIDVALIDITY (\d+)\]/i.exec(line)
|
|
379
|
+
if (validity) uidValidity = validity[1]
|
|
380
|
+
const next = /\[UIDNEXT (\d+)\]/i.exec(line)
|
|
381
|
+
if (next) uidNext = next[1]
|
|
382
|
+
}
|
|
383
|
+
return { mailbox, exists, uidValidity, uidNext }
|
|
384
|
+
},
|
|
385
|
+
|
|
386
|
+
async searchSince(days = 30) {
|
|
387
|
+
const response = await rawCommand('UID SEARCH SINCE ' + imapSinceDate(days))
|
|
388
|
+
return [...new Set(parseSearch(response.records))].sort((a, b) => a - b)
|
|
389
|
+
},
|
|
390
|
+
|
|
391
|
+
async searchUidRange(fromUid) {
|
|
392
|
+
const start = Math.max(1, Math.floor(Number(fromUid) || 1))
|
|
393
|
+
const response = await rawCommand('UID SEARCH UID ' + start + ':*')
|
|
394
|
+
return [...new Set(parseSearch(response.records).filter((uid) => uid >= start))].sort((a, b) => a - b)
|
|
395
|
+
},
|
|
396
|
+
|
|
397
|
+
async fetchMessages(uids = [], { maxMessageBytes = 5 * 1024 * 1024 } = {}) {
|
|
398
|
+
const wanted = [...new Set(uids.filter((uid) => Number.isInteger(uid) && uid > 0))].sort((a, b) => a - b)
|
|
399
|
+
if (!wanted.length) return { messages: [], failedUids: [], requested: 0 }
|
|
400
|
+
const headResponse = await rawCommand('UID FETCH ' + wanted.join(',') + ' (UID RFC822.SIZE BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE MESSAGE-ID)])')
|
|
401
|
+
const metas = parseFetchRecords(headResponse.records)
|
|
402
|
+
const byUid = new Map()
|
|
403
|
+
for (const meta of metas) if (meta.uid != null) byUid.set(meta.uid, meta)
|
|
404
|
+
const failedUids = []
|
|
405
|
+
const messages = []
|
|
406
|
+
const needBody = []
|
|
407
|
+
for (const uid of wanted) {
|
|
408
|
+
const meta = byUid.get(uid)
|
|
409
|
+
if (!meta) { failedUids.push(uid); continue }
|
|
410
|
+
if (meta.size != null && meta.size > maxMessageBytes) {
|
|
411
|
+
messages.push({ uid, size: meta.size, headerRaw: meta.literal ?? null, raw: null, oversize: true })
|
|
412
|
+
} else {
|
|
413
|
+
needBody.push(uid)
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
const bodies = new Map()
|
|
417
|
+
for (let index = 0; index < needBody.length; index += MAX_FETCH_CHUNK) {
|
|
418
|
+
const chunk = needBody.slice(index, index + MAX_FETCH_CHUNK)
|
|
419
|
+
const response = await rawCommand('UID FETCH ' + chunk.join(',') + ' (UID BODY.PEEK[])')
|
|
420
|
+
for (const record of parseFetchRecords(response.records)) {
|
|
421
|
+
if (record.uid == null || !record.literal) continue
|
|
422
|
+
bodies.set(record.uid, record.literal)
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
for (const uid of needBody) {
|
|
426
|
+
const raw = bodies.get(uid)
|
|
427
|
+
if (!raw) { failedUids.push(uid); continue }
|
|
428
|
+
const meta = byUid.get(uid)
|
|
429
|
+
messages.push({ uid, size: meta?.size ?? raw.length, headerRaw: meta?.literal ?? null, raw, oversize: false })
|
|
430
|
+
}
|
|
431
|
+
return { messages, failedUids, requested: wanted.length }
|
|
432
|
+
},
|
|
433
|
+
|
|
434
|
+
async logout() {
|
|
435
|
+
try { await rawCommand('LOGOUT') } catch {}
|
|
436
|
+
authenticated = false
|
|
437
|
+
closeSocket()
|
|
438
|
+
},
|
|
439
|
+
|
|
440
|
+
destroy() {
|
|
441
|
+
authenticated = false
|
|
442
|
+
closeSocket()
|
|
443
|
+
},
|
|
444
|
+
}
|
|
445
|
+
return session
|
|
446
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Offline .eml/.mbox directory import for the 0.4.0 mail entry.
|
|
3
|
+
*
|
|
4
|
+
* This is the fallback for users whose provider disabled password IMAP
|
|
5
|
+
* (Google Workspace / Microsoft 365): no network, no credentials, no IMAP
|
|
6
|
+
* code path. Imported messages land in mail_messages under a built-in local
|
|
7
|
+
* account (or a caller-supplied account) and then flow through the same
|
|
8
|
+
* `mail extract` / candidate review pipeline.
|
|
9
|
+
*
|
|
10
|
+
* Attachments are metadata only (name/size/mime); no attachment bytes are
|
|
11
|
+
* written anywhere. Empty-body messages are skipped, not silently stored.
|
|
12
|
+
*/
|
|
13
|
+
import { readFileSync, readdirSync, statSync } from 'node:fs'
|
|
14
|
+
import { extname, join } from 'node:path'
|
|
15
|
+
import { parseMailMessage } from './mail-parse.js'
|
|
16
|
+
import { mailError, MAIL_ERROR_CODES } from './mail-errors.js'
|
|
17
|
+
import {
|
|
18
|
+
assertMailTables, buildMailMessageRow, ensureLocalMailAccount, findMailMessageDuplicate,
|
|
19
|
+
getMailAccount, insertMailMessage, nextLocalUid, isoOf, MAIL_LOCAL_ACCOUNT_ID, MAIL_LOCAL_MAILBOX,
|
|
20
|
+
} from './mail-store.js'
|
|
21
|
+
|
|
22
|
+
export const EML_EXTENSIONS = Object.freeze(['.eml', '.mbox'])
|
|
23
|
+
export const DEFAULT_IMPORT_FILE_LIMIT = 100
|
|
24
|
+
export const MAX_IMPORT_FILE_LIMIT = 500
|
|
25
|
+
export const MAX_IMPORT_MESSAGES_PER_FILE = 500
|
|
26
|
+
|
|
27
|
+
function isEmlPath(path) {
|
|
28
|
+
return EML_EXTENSIONS.includes(extname(String(path)).toLowerCase())
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function listImportFiles(target) {
|
|
32
|
+
if (typeof target !== 'string' || !target.trim()) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail import requires a .eml file or a directory')
|
|
33
|
+
let stat
|
|
34
|
+
try { stat = statSync(target) } catch (error) {
|
|
35
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'cannot read mail path: ' + String(error?.message ?? error).slice(0, 200))
|
|
36
|
+
}
|
|
37
|
+
if (stat.isFile()) {
|
|
38
|
+
if (!isEmlPath(target)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'not a .eml/.mbox file: ' + target)
|
|
39
|
+
return [target]
|
|
40
|
+
}
|
|
41
|
+
if (!stat.isDirectory()) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail import path is not a file or directory: ' + target)
|
|
42
|
+
try {
|
|
43
|
+
return readdirSync(target, { withFileTypes: true })
|
|
44
|
+
.filter((entry) => entry.isFile() && isEmlPath(entry.name))
|
|
45
|
+
.map((entry) => join(target, entry.name))
|
|
46
|
+
.sort()
|
|
47
|
+
} catch (error) {
|
|
48
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'cannot list mail directory: ' + String(error?.message ?? error).slice(0, 200))
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Split an mbox stream into raw RFC 822 messages (keeping each raw). */
|
|
53
|
+
export function splitMboxRaw(raw) {
|
|
54
|
+
const source = String(raw == null ? '' : raw).replace(/^\uFEFF/, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n')
|
|
55
|
+
if (!source.trim()) return []
|
|
56
|
+
const chunks = []
|
|
57
|
+
let current = []
|
|
58
|
+
for (const line of source.split('\n')) {
|
|
59
|
+
if (line.startsWith('From ') && current.length) {
|
|
60
|
+
chunks.push(current.join('\n'))
|
|
61
|
+
current = []
|
|
62
|
+
}
|
|
63
|
+
current.push(line)
|
|
64
|
+
}
|
|
65
|
+
if (current.length) chunks.push(current.join('\n'))
|
|
66
|
+
return chunks
|
|
67
|
+
.map((chunk) => chunk.replace(/^From [^\n]*\n/, ''))
|
|
68
|
+
.filter((chunk) => chunk.trim())
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function parseDateHeader(value) {
|
|
72
|
+
if (!value) return undefined
|
|
73
|
+
const parsed = Date.parse(value)
|
|
74
|
+
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : undefined
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function readMessages(file) {
|
|
78
|
+
let raw
|
|
79
|
+
try { raw = readFileSync(file, 'utf8') } catch (error) {
|
|
80
|
+
throw mailError(MAIL_ERROR_CODES.CONFIG, 'cannot read ' + file + ': ' + String(error?.message ?? error).slice(0, 200))
|
|
81
|
+
}
|
|
82
|
+
const isMbox = extname(file).toLowerCase() === '.mbox'
|
|
83
|
+
const raws = isMbox ? splitMboxRaw(raw) : [raw]
|
|
84
|
+
return raws.slice(0, MAX_IMPORT_MESSAGES_PER_FILE)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Import .eml/.mbox file(s) into mail_messages. No network, no credentials.
|
|
89
|
+
*
|
|
90
|
+
* @param {object} store open SqliteStore
|
|
91
|
+
* @param {{ path?: string, files?: string[], accountId?: string, clientId?: string,
|
|
92
|
+
* projectId?: string, limit?: number, dryRun?: boolean }} options
|
|
93
|
+
* @param {{ now?: Date }} [context]
|
|
94
|
+
* @returns {{ ok: true, command: 'mail import', accountId: string, dryRun: boolean,
|
|
95
|
+
* files: number, scanned: number, inserted: number, skipped: number, failed: number }}
|
|
96
|
+
*/
|
|
97
|
+
export async function importMailFiles(store, options = {}, context = {}) {
|
|
98
|
+
assertMailTables(store)
|
|
99
|
+
const now = context.now instanceof Date ? context.now : new Date(context.now ?? Date.now())
|
|
100
|
+
const dryRun = options.dryRun === true
|
|
101
|
+
const limit = Math.max(1, Math.min(MAX_IMPORT_FILE_LIMIT, Number(options.limit) || DEFAULT_IMPORT_FILE_LIMIT))
|
|
102
|
+
const requested = Array.isArray(options.files) && options.files.length
|
|
103
|
+
? options.files.filter(isEmlPath).slice(0, limit)
|
|
104
|
+
: listImportFiles(options.path ?? options.dir ?? options.target).slice(0, limit)
|
|
105
|
+
if (!requested.length) throw mailError(MAIL_ERROR_CODES.CONFIG, 'no .eml/.mbox files found to import')
|
|
106
|
+
let account
|
|
107
|
+
if (options.accountId) {
|
|
108
|
+
account = getMailAccount(store, options.accountId)
|
|
109
|
+
if (!['tls', 'starttls', 'local'].includes(account.tls)) throw mailError(MAIL_ERROR_CODES.CONFIG, 'mail import requires a local or IMAP account')
|
|
110
|
+
} else if (dryRun) {
|
|
111
|
+
// A dry run must not even create the built-in local account row.
|
|
112
|
+
account = { id: MAIL_LOCAL_ACCOUNT_ID, mailbox: MAIL_LOCAL_MAILBOX, tls: 'local' }
|
|
113
|
+
} else {
|
|
114
|
+
account = store.mutate(() => ensureLocalMailAccount(store, { now })).result
|
|
115
|
+
}
|
|
116
|
+
const rows = []
|
|
117
|
+
let failed = 0
|
|
118
|
+
let scanned = 0
|
|
119
|
+
for (const file of requested) {
|
|
120
|
+
let raws
|
|
121
|
+
try { raws = readMessages(file) } catch { failed += 1; continue }
|
|
122
|
+
for (const raw of raws) {
|
|
123
|
+
scanned += 1
|
|
124
|
+
const parsed = parseMailMessage(raw)
|
|
125
|
+
const text = String(parsed.text ?? '')
|
|
126
|
+
if (!text.trim()) continue
|
|
127
|
+
const sentAt = parseDateHeader(parsed.date)
|
|
128
|
+
rows.push(buildMailMessageRow({
|
|
129
|
+
accountId: account.id,
|
|
130
|
+
mailbox: account.mailbox,
|
|
131
|
+
uid: null,
|
|
132
|
+
uidvalidity: 'local',
|
|
133
|
+
sizeBytes: parsed.sizeBytes,
|
|
134
|
+
receivedAt: sentAt ?? isoOf(now),
|
|
135
|
+
sentAt: sentAt ?? null,
|
|
136
|
+
from: parsed.from,
|
|
137
|
+
subject: parsed.subject,
|
|
138
|
+
messageId: parsed.messageId,
|
|
139
|
+
text,
|
|
140
|
+
attachments: parsed.attachments,
|
|
141
|
+
status: 'new',
|
|
142
|
+
clientId: options.clientId ?? null,
|
|
143
|
+
projectId: options.projectId ?? null,
|
|
144
|
+
now,
|
|
145
|
+
}))
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const skippedEmpty = scanned - rows.length
|
|
149
|
+
if (dryRun) {
|
|
150
|
+
let duplicates = 0
|
|
151
|
+
for (const row of rows) if (findMailMessageDuplicate(store, account.id, row)) duplicates += 1
|
|
152
|
+
return {
|
|
153
|
+
ok: true, command: 'mail import', accountId: account.id, dryRun: true,
|
|
154
|
+
files: requested.length, scanned, inserted: 0,
|
|
155
|
+
skipped: skippedEmpty + duplicates, failed,
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
const audit = { actor: 'local', action: 'mail.import', allowed: true, reason: '' }
|
|
159
|
+
const result = store.mutate(() => {
|
|
160
|
+
let uid = nextLocalUid(store, account.id, account.mailbox)
|
|
161
|
+
let inserted = 0
|
|
162
|
+
let skipped = skippedEmpty
|
|
163
|
+
for (const row of rows) {
|
|
164
|
+
if (findMailMessageDuplicate(store, account.id, row)) { skipped += 1; continue }
|
|
165
|
+
row.uid = uid
|
|
166
|
+
if (insertMailMessage(store, row)) { inserted += 1; uid += 1 } else { skipped += 1 }
|
|
167
|
+
}
|
|
168
|
+
audit.reason = inserted + ' imported, ' + skipped + ' skipped, ' + failed + ' failed'
|
|
169
|
+
return { inserted, skipped }
|
|
170
|
+
}, audit)
|
|
171
|
+
return {
|
|
172
|
+
ok: true, command: 'mail import', accountId: account.id, dryRun: false,
|
|
173
|
+
files: requested.length, scanned, inserted: result.result.inserted,
|
|
174
|
+
skipped: result.result.skipped, failed,
|
|
175
|
+
}
|
|
176
|
+
}
|