advisers 0.1.0-preview.4 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/LICENSE +131 -21
  2. package/NOTICE +34 -0
  3. package/README-zh.md +40 -9
  4. package/README.md +53 -10
  5. package/THIRD-PARTY-NOTICES.md +277 -0
  6. package/bin/advisers.js +4230 -122
  7. package/index.js +1 -1
  8. package/lib/agent.js +414 -0
  9. package/lib/attachments.js +283 -0
  10. package/lib/capture.js +168 -0
  11. package/lib/context-window.js +123 -0
  12. package/lib/digest.js +27 -0
  13. package/lib/doc-review.js +30 -0
  14. package/lib/egress.js +27 -0
  15. package/lib/evidence.js +1995 -0
  16. package/lib/ical.js +542 -0
  17. package/lib/imap.js +181 -0
  18. package/lib/importers.js +1001 -0
  19. package/lib/learning.js +46 -0
  20. package/lib/mcp-server.js +869 -0
  21. package/lib/model.js +307 -0
  22. package/lib/notify.js +189 -0
  23. package/lib/nudge.js +10 -0
  24. package/lib/ocr.js +47 -0
  25. package/lib/pdf.js +105 -0
  26. package/lib/pet-brief-store.js +51 -0
  27. package/lib/pet-brief.js +644 -0
  28. package/lib/receipt-verify.js +624 -0
  29. package/lib/relative-date.js +393 -0
  30. package/lib/remind.js +690 -0
  31. package/lib/research.js +193 -0
  32. package/lib/review.js +88 -0
  33. package/lib/routine.js +22 -0
  34. package/lib/safety/classifier.js +392 -0
  35. package/lib/safety/explain.js +157 -0
  36. package/lib/safety/index.js +333 -0
  37. package/lib/safety/redact.js +212 -0
  38. package/lib/safety/rules.js +824 -0
  39. package/lib/scheduler.js +399 -0
  40. package/lib/source.js +294 -0
  41. package/lib/sqlite.js +1022 -0
  42. package/lib/sync.js +129 -0
  43. package/lib/timestamp.js +45 -0
  44. package/lib/tos/index.js +332 -0
  45. package/lib/tos/manifest.json +32 -0
  46. package/lib/tos/privacy.zh-CN.md +136 -0
  47. package/lib/tos/safety.zh-CN.md +101 -0
  48. package/lib/tos/summary.zh-CN.md +12 -0
  49. package/lib/tos/terms.zh-CN.md +439 -0
  50. package/lib/ui.js +49 -0
  51. package/lib/undo.js +83 -0
  52. package/lib/vault.js +51 -0
  53. package/package.json +26 -4
@@ -0,0 +1,1001 @@
1
+ /**
2
+ * E2 read-only import parsers.
3
+ *
4
+ * Pure, offline, dependency-free parsers for the file formats a read-only
5
+ * import entry point is expected to accept:
6
+ *
7
+ * parseEml(raw) -> one RFC 822 message (minimal MIME subset)
8
+ * parseMbox(raw) -> parseEml() for every `From ` separated block
9
+ * parseIcs(raw) -> array of VEVENT records
10
+ * parseTranscript(raw, opts?) -> { text, segments } from VTT/SRT/plain text
11
+ * toCaptureInput(parsed, opts) -> { text, source, meta } ready for capture
12
+ * dedupeKey(source, parsed) -> stable sha256 hex key
13
+ *
14
+ * Design constraints:
15
+ * - no network, no filesystem, no third-party dependencies (node:crypto only);
16
+ * - every function is deterministic for the same input;
17
+ * - malformed email/transcript input never throws: the affected text is
18
+ * dropped and explained in `warnings` (calendar input is the exception:
19
+ * structurally invalid iCalendar throws Error, as required by the E2 spec).
20
+ *
21
+ * Deliberate limits (documented, not bugs):
22
+ * - emails: no address rewriting, no attachment extraction, no MIME
23
+ * parameter continuation (RFC 2231), no S/MIME, no header charset guessing
24
+ * for 8bit bodies (the caller already supplied a JS string);
25
+ * - mbox: no `>From ` unquoting, no Content-Length handling;
26
+ * - iCalendar: no time zone database (TZID is kept as a literal marker), no
27
+ * RRULE expansion, no recurrence overrides / EXDATE resolution;
28
+ * - transcripts: cue timing only (no word-level timestamps), no speaker
29
+ * diarization beyond `<v Name>` / `Name:` prefixes.
30
+ */
31
+
32
+ import { createHash } from 'node:crypto'
33
+
34
+ const HTML_ENTITIES = {
35
+ amp: '&',
36
+ apos: "'",
37
+ bull: '•',
38
+ copy: '©',
39
+ deg: '°',
40
+ divide: '÷',
41
+ emsp: '\u2003',
42
+ ensp: '\u2002',
43
+ euro: '€',
44
+ gt: '>',
45
+ hellip: '…',
46
+ ldquo: '“',
47
+ lsquo: '‘',
48
+ lt: '<',
49
+ mdash: '—',
50
+ middot: '·',
51
+ nbsp: '\u00a0',
52
+ ndash: '–',
53
+ para: '¶',
54
+ pound: '£',
55
+ quot: '"',
56
+ rdquo: '”',
57
+ reg: '®',
58
+ rsquo: '’',
59
+ sect: '§',
60
+ thinsp: '\u2009',
61
+ times: '×',
62
+ yen: '¥',
63
+ }
64
+
65
+ const CHARSET_ALIASES = {
66
+ ascii: 'windows-1252',
67
+ 'us-ascii': 'windows-1252',
68
+ 'iso-8859-1': 'windows-1252',
69
+ 'iso8859-1': 'windows-1252',
70
+ 'latin1': 'windows-1252',
71
+ utf8: 'utf-8',
72
+ 'utf-8': 'utf-8',
73
+ 'gb2312': 'gbk',
74
+ 'gb-2312': 'gbk',
75
+ 'gb_2312': 'gbk',
76
+ 'x-gbk': 'gbk',
77
+ }
78
+
79
+ const TRANSFER_IDENTITY = new Set(['', '7bit', '8bit', 'binary'])
80
+ const TRANSFER_DECODED = new Set(['base64', 'quoted-printable'])
81
+
82
+ function asString(value) {
83
+ if (value == null) return ''
84
+ return typeof value === 'string' ? value : String(value)
85
+ }
86
+
87
+ function normalizeNewlines(value) {
88
+ return asString(value).replace(/^\uFEFF/, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n')
89
+ }
90
+
91
+ function pushWarning(warnings, message) {
92
+ if (warnings && !warnings.includes(message)) warnings.push(message)
93
+ }
94
+
95
+ function cleanText(value) {
96
+ return asString(value)
97
+ .replace(/\r\n/g, '\n')
98
+ .replace(/\r/g, '\n')
99
+ .replace(/[ \t]+\n/g, '\n')
100
+ .replace(/\n{3,}/g, '\n\n')
101
+ .trim()
102
+ }
103
+
104
+ function stripTrailing(value) {
105
+ return asString(value).replace(/[ \t]+$/gm, '')
106
+ }
107
+
108
+ /* ------------------------------------------------------------------ *
109
+ * Header helpers
110
+ * ------------------------------------------------------------------ */
111
+
112
+ function splitHeadersAndBody(text) {
113
+ const separator = text.indexOf('\n\n')
114
+ if (separator === -1) return { head: text, body: '', separated: false }
115
+ return { head: text.slice(0, separator), body: text.slice(separator + 2), separated: true }
116
+ }
117
+
118
+ function unfoldHeaderBlock(block) {
119
+ const lines = []
120
+ for (const line of normalizeNewlines(block).split('\n')) {
121
+ if (/^[ \t]/.test(line) && lines.length) lines[lines.length - 1] += ` ${line.trim()}`
122
+ else lines.push(line)
123
+ }
124
+ return lines
125
+ }
126
+
127
+ function parseHeaderBlock(block) {
128
+ const headers = {}
129
+ for (const line of unfoldHeaderBlock(block)) {
130
+ const index = line.indexOf(':')
131
+ if (index <= 0) continue
132
+ const name = line.slice(0, index).trim().toLowerCase()
133
+ if (!name || name in headers) continue
134
+ headers[name] = line.slice(index + 1).trim()
135
+ }
136
+ return headers
137
+ }
138
+
139
+ function splitQuoted(value, separator) {
140
+ const out = []
141
+ let current = ''
142
+ let quoted = false
143
+ for (let i = 0; i < value.length; i++) {
144
+ const char = value[i]
145
+ if (quoted) {
146
+ current += char
147
+ if (char === '\\' && i + 1 < value.length) current += value[++i]
148
+ else if (char === '"') quoted = false
149
+ } else if (char === '"') {
150
+ quoted = true
151
+ current += char
152
+ } else if (char === separator) {
153
+ out.push(current)
154
+ current = ''
155
+ } else {
156
+ current += char
157
+ }
158
+ }
159
+ out.push(current)
160
+ return out
161
+ }
162
+
163
+ function unquote(value) {
164
+ const text = asString(value).trim()
165
+ if (text.length >= 2 && text.startsWith('"') && text.endsWith('"')) {
166
+ return text.slice(1, -1).replace(/\\(.)/g, '$1')
167
+ }
168
+ return text
169
+ }
170
+
171
+ function parseParameterList(raw) {
172
+ const params = {}
173
+ for (const segment of splitQuoted(asString(raw), ';').slice(1)) {
174
+ const equals = segment.indexOf('=')
175
+ if (equals <= 0) continue
176
+ const key = segment.slice(0, equals).trim().toLowerCase()
177
+ if (!key) continue
178
+ params[key] = unquote(segment.slice(equals + 1))
179
+ }
180
+ return params
181
+ }
182
+
183
+ function parseContentType(value) {
184
+ const raw = asString(value).trim()
185
+ if (!raw) return { type: '', params: {} }
186
+ const segments = splitQuoted(raw, ';')
187
+ return { type: segments[0].trim().toLowerCase(), params: parseParameterList(raw) }
188
+ }
189
+
190
+ function parseDisposition(value) {
191
+ const raw = asString(value).trim()
192
+ if (!raw) return ''
193
+ return splitQuoted(raw, ';')[0].trim().toLowerCase()
194
+ }
195
+
196
+ function normalizeCharset(value) {
197
+ const label = unquote(value).trim().toLowerCase()
198
+ if (!label) return 'utf-8'
199
+ return CHARSET_ALIASES[label] || label
200
+ }
201
+
202
+ function decodeBytes(bytes, charset, warnings) {
203
+ const label = normalizeCharset(charset)
204
+ if (!bytes || !bytes.length) return ''
205
+ try {
206
+ return new TextDecoder(label, { fatal: false }).decode(bytes)
207
+ } catch {
208
+ pushWarning(warnings, `unsupported charset: ${asString(charset).trim() || label}`)
209
+ return null
210
+ }
211
+ }
212
+
213
+ /* ------------------------------------------------------------------ *
214
+ * RFC 2047 encoded words (headers)
215
+ * ------------------------------------------------------------------ */
216
+
217
+ function decodeQuotedPrintableWord(payload) {
218
+ const bytes = []
219
+ for (let i = 0; i < payload.length; i++) {
220
+ const char = payload[i]
221
+ if (char === '_') {
222
+ bytes.push(0x20)
223
+ continue
224
+ }
225
+ if (char === '=' && /^[0-9A-Fa-f]{2}$/.test(payload.slice(i + 1, i + 3))) {
226
+ bytes.push(Number.parseInt(payload.slice(i + 1, i + 3), 16))
227
+ i += 2
228
+ continue
229
+ }
230
+ const code = char.charCodeAt(0)
231
+ bytes.push(code <= 0xff ? code : 0x3f)
232
+ }
233
+ return Buffer.from(bytes)
234
+ }
235
+
236
+ function decodeEncodedWord(charset, encoding, payload, warnings) {
237
+ let bytes
238
+ if (encoding.toUpperCase() === 'B') {
239
+ const compact = payload.replace(/\s+/g, '')
240
+ if (!/^[A-Za-z0-9+/]*={0,2}$/.test(compact) || compact.length % 4 === 1) {
241
+ pushWarning(warnings, 'invalid base64 encoded-word')
242
+ return null
243
+ }
244
+ bytes = Buffer.from(compact, 'base64')
245
+ } else {
246
+ bytes = decodeQuotedPrintableWord(payload)
247
+ }
248
+ return decodeBytes(bytes, charset.split('*')[0], warnings)
249
+ }
250
+
251
+ function decodeHeaderValue(value, warnings) {
252
+ const input = asString(value)
253
+ if (!input.includes('=?')) return input
254
+ const pattern = /=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g
255
+ const tokens = []
256
+ let last = 0
257
+ let match
258
+ while ((match = pattern.exec(input))) {
259
+ if (match.index > last) tokens.push({ encoded: false, raw: input.slice(last, match.index) })
260
+ tokens.push({ encoded: true, raw: match[0], charset: match[1], encoding: match[2], payload: match[3] })
261
+ last = match.index + match[0].length
262
+ }
263
+ if (!tokens.length) return input
264
+ if (last < input.length) tokens.push({ encoded: false, raw: input.slice(last) })
265
+ let out = ''
266
+ for (let i = 0; i < tokens.length; i++) {
267
+ const token = tokens[i]
268
+ if (token.encoded) {
269
+ const decoded = decodeEncodedWord(token.charset, token.encoding, token.payload, warnings)
270
+ out += decoded == null ? token.raw : decoded
271
+ } else if (tokens[i - 1]?.encoded && tokens[i + 1]?.encoded && /^[ \t\r\n]*$/.test(token.raw)) {
272
+ // RFC 2047: whitespace between two adjacent encoded-words is not content.
273
+ } else {
274
+ out += token.raw
275
+ }
276
+ }
277
+ return out
278
+ }
279
+
280
+ /* ------------------------------------------------------------------ *
281
+ * Bodies: transfer encodings, charsets, HTML
282
+ * ------------------------------------------------------------------ */
283
+
284
+ function decodeQuotedPrintableBody(value) {
285
+ const source = asString(value)
286
+ const bytes = []
287
+ for (let i = 0; i < source.length; i++) {
288
+ const char = source[i]
289
+ if (char === '=') {
290
+ if (source[i + 1] === '\n') {
291
+ i += 1 // soft line break
292
+ continue
293
+ }
294
+ if (/^[0-9A-Fa-f]{2}$/.test(source.slice(i + 1, i + 3))) {
295
+ bytes.push(Number.parseInt(source.slice(i + 1, i + 3), 16))
296
+ i += 2
297
+ continue
298
+ }
299
+ bytes.push(0x3d) // lone '=': keep literal rather than corrupt the body
300
+ continue
301
+ }
302
+ const code = char.charCodeAt(0)
303
+ if (code <= 0x7f) {
304
+ bytes.push(code)
305
+ } else {
306
+ bytes.push(...Buffer.from(char, 'utf8'))
307
+ }
308
+ }
309
+ return Buffer.from(bytes)
310
+ }
311
+
312
+ function decodeTransferBody(body, transferEncoding, warnings) {
313
+ const encoding = asString(transferEncoding).trim().toLowerCase()
314
+ if (TRANSFER_IDENTITY.has(encoding)) return { text: asString(body) }
315
+ if (TRANSFER_DECODED.has(encoding)) {
316
+ if (encoding === 'quoted-printable') return { bytes: decodeQuotedPrintableBody(body) }
317
+ const compact = asString(body).replace(/\s+/g, '')
318
+ if (!compact) return { bytes: Buffer.alloc(0) }
319
+ if (!/^[A-Za-z0-9+/]*={0,2}$/.test(compact) || compact.length % 4 === 1) {
320
+ pushWarning(warnings, 'invalid base64 body')
321
+ return null
322
+ }
323
+ return { bytes: Buffer.from(compact, 'base64') }
324
+ }
325
+ pushWarning(warnings, `unsupported content-transfer-encoding: ${encoding}`)
326
+ return null
327
+ }
328
+
329
+ function decodeEntityContent(headers, body, warnings) {
330
+ const contentType = parseContentType(headers['content-type'])
331
+ const decoded = decodeTransferBody(body, headers['content-transfer-encoding'], warnings)
332
+ if (!decoded) return { text: null, contentType }
333
+ if (decoded.text !== undefined) return { text: decoded.text, contentType }
334
+ const charset = contentType.params.charset || 'utf-8'
335
+ return { text: decodeBytes(decoded.bytes, charset, warnings), contentType }
336
+ }
337
+
338
+ function decodeNumericEntity(digits, radix) {
339
+ const codePoint = Number.parseInt(digits, radix)
340
+ if (!Number.isFinite(codePoint) || codePoint <= 0 || codePoint > 0x10ffff) return null
341
+ if (codePoint >= 0xd800 && codePoint <= 0xdfff) return null
342
+ try {
343
+ return String.fromCodePoint(codePoint)
344
+ } catch {
345
+ return null
346
+ }
347
+ }
348
+
349
+ function decodeHtmlEntities(value) {
350
+ return asString(value).replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);/gi, (entity, body) => {
351
+ if (body[0] === '#') {
352
+ const hex = body[1] === 'x' || body[1] === 'X'
353
+ const decoded = decodeNumericEntity(body.slice(hex ? 2 : 1), hex ? 16 : 10)
354
+ return decoded == null ? entity : decoded
355
+ }
356
+ const named = HTML_ENTITIES[body.toLowerCase()]
357
+ return named == null ? entity : named
358
+ })
359
+ }
360
+
361
+ function htmlToText(value) {
362
+ const html = asString(value)
363
+ if (!html) return ''
364
+ let text = html
365
+ .replace(/<!--[\s\S]*?-->/g, '')
366
+ .replace(/<(script|style|head|title)\b[^>]*>[\s\S]*?<\/\1>/gi, '')
367
+ .replace(/<br\s*\/?>/gi, '\n')
368
+ .replace(/<li\b[^>]*>/gi, '\n- ')
369
+ .replace(/<\/(p|div|li|tr|h[1-6]|blockquote|table|section|article|pre)>/gi, '\n')
370
+ .replace(/<[^>]+>/g, '')
371
+ text = decodeHtmlEntities(text)
372
+ return text
373
+ .replace(/\u00a0/g, ' ')
374
+ .split('\n')
375
+ .map((line) => line.replace(/[ \t]+/g, ' ').trim())
376
+ .join('\n')
377
+ .replace(/\n{3,}/g, '\n\n')
378
+ .trim()
379
+ }
380
+
381
+ /* ------------------------------------------------------------------ *
382
+ * MIME multipart walking
383
+ * ------------------------------------------------------------------ */
384
+
385
+ function splitMultipart(body, boundary, warnings) {
386
+ const marker = `--${asString(boundary)}`
387
+ if (!asString(boundary)) {
388
+ pushWarning(warnings, 'multipart entity without boundary parameter')
389
+ return []
390
+ }
391
+ const parts = []
392
+ let current = null
393
+ let closed = false
394
+ for (const line of normalizeNewlines(body).split('\n')) {
395
+ const trimmed = line.replace(/[ \t]+$/, '')
396
+ if (trimmed === marker || trimmed === `${marker}--`) {
397
+ if (current) parts.push(current.join('\n'))
398
+ current = null
399
+ if (trimmed === `${marker}--`) {
400
+ closed = true
401
+ break
402
+ }
403
+ current = []
404
+ continue
405
+ }
406
+ if (current) current.push(line)
407
+ }
408
+ if (current) parts.push(current.join('\n'))
409
+ if (!closed) pushWarning(warnings, 'multipart entity is missing its closing boundary')
410
+ return parts.map((part) => part.replace(/^\n+/, '')).filter((part) => part.trim() !== '')
411
+ }
412
+
413
+ function collectMimeContent(entity, warnings, collected) {
414
+ const { head, body } = splitHeadersAndBody(normalizeNewlines(entity))
415
+ const headers = parseHeaderBlock(head)
416
+ const contentType = parseContentType(headers['content-type'])
417
+ if (parseDisposition(headers['content-disposition']) === 'attachment') return
418
+
419
+ if (contentType.type.startsWith('multipart/')) {
420
+ const decoded = decodeTransferBody(body, headers['content-transfer-encoding'], warnings)
421
+ if (!decoded) return
422
+ let raw = decoded.text
423
+ if (raw === undefined) raw = decodeBytes(decoded.bytes, 'utf-8', warnings)
424
+ if (raw == null) return
425
+ const parts = splitMultipart(raw, contentType.params.boundary, warnings)
426
+ if (!parts.length) pushWarning(warnings, 'multipart entity has no parts')
427
+ for (const part of parts) collectMimeContent(part, warnings, collected)
428
+ return
429
+ }
430
+
431
+ const { text } = decodeEntityContent(headers, body, warnings)
432
+ if (text == null) return
433
+ if (contentType.type === 'text/html') collected.html.push(text)
434
+ else if (contentType.type === 'text/plain' || !contentType.type) collected.plain.push(text)
435
+ // other leaf types (images, application/pdf, message/rfc822, ...) are ignored
436
+ }
437
+
438
+ /* ------------------------------------------------------------------ *
439
+ * parseEml
440
+ * ------------------------------------------------------------------ */
441
+
442
+ function cleanMessageId(value) {
443
+ const text = asString(value).trim()
444
+ const wrapped = /^<([^>]+)>$/.exec(text)
445
+ return (wrapped ? wrapped[1] : text).trim()
446
+ }
447
+
448
+ /**
449
+ * Parse one RFC 822 message using a minimal MIME subset.
450
+ *
451
+ * `text` is always the readable rendering: the first `text/plain` part, or the
452
+ * tag-stripped / entity-decoded `text/html` part when no plain part exists.
453
+ * `html` (only when a `text/html` part exists) keeps the decoded HTML source so
454
+ * callers can re-render or archive it; it is never returned as capture text.
455
+ * Encoding failures and missing bodies yield `text: ''` plus a non-empty
456
+ * `warnings` array instead of throwing.
457
+ *
458
+ * @param {string} raw
459
+ * @returns {{ from: string, to: string, subject: string, date: string,
460
+ * messageId: string, text: string, html?: string, warnings: string[] }}
461
+ */
462
+ export function parseEml(raw) {
463
+ const warnings = []
464
+ const source = normalizeNewlines(raw)
465
+ if (!source.trim()) {
466
+ return { from: '', to: '', subject: '', date: '', messageId: '', text: '', warnings: ['empty message'] }
467
+ }
468
+
469
+ const split = splitHeadersAndBody(source)
470
+ let headers = parseHeaderBlock(split.head)
471
+ let body = split.body
472
+ if (!split.separated && !Object.keys(headers).length) {
473
+ // Tolerate a body-only input instead of silently returning nothing.
474
+ headers = {}
475
+ body = source
476
+ pushWarning(warnings, 'message has no header/body separator')
477
+ }
478
+
479
+ const from = cleanText(decodeHeaderValue(headers.from, warnings))
480
+ const to = cleanText(decodeHeaderValue(headers.to, warnings))
481
+ const subject = cleanText(decodeHeaderValue(headers.subject, warnings))
482
+ const date = cleanText(decodeHeaderValue(headers.date, warnings))
483
+ const messageId = cleanMessageId(decodeHeaderValue(headers['message-id'], warnings))
484
+
485
+ const collected = { plain: [], html: [] }
486
+ collectMimeContent(source, warnings, collected)
487
+
488
+ const plain = collected.plain.map(cleanText).find(Boolean) || ''
489
+ const html = collected.html.map(cleanText).find(Boolean) || ''
490
+ const text = plain || (html ? htmlToText(html) : '')
491
+ if (!text) pushWarning(warnings, 'message has no readable text body')
492
+
493
+ const result = { from, to, subject, date, messageId, text, warnings }
494
+ if (html) result.html = html
495
+ return result
496
+ }
497
+
498
+ /* ------------------------------------------------------------------ *
499
+ * parseMbox
500
+ * ------------------------------------------------------------------ */
501
+
502
+ /**
503
+ * Split an mbox stream on `^From ` separator lines and parse every message.
504
+ *
505
+ * @param {string} raw
506
+ * @returns {Array<ReturnType<typeof parseEml>>}
507
+ */
508
+ export function parseMbox(raw) {
509
+ const source = normalizeNewlines(raw)
510
+ if (!source.trim()) return []
511
+ const results = []
512
+ const leading = []
513
+ let current = null
514
+ for (const line of source.split('\n')) {
515
+ if (line.startsWith('From ')) {
516
+ if (current) results.push(parseEml(current.join('\n')))
517
+ current = []
518
+ continue
519
+ }
520
+ if (current) current.push(line)
521
+ else leading.push(line)
522
+ }
523
+ if (current) {
524
+ const chunk = current.join('\n')
525
+ if (chunk.trim()) results.push(parseEml(chunk))
526
+ }
527
+ const preamble = leading.join('\n').trim()
528
+ if (preamble) results.unshift(parseEml(preamble))
529
+ return results
530
+ }
531
+
532
+ /* ------------------------------------------------------------------ *
533
+ * parseIcs
534
+ * ------------------------------------------------------------------ */
535
+
536
+ function unfoldIcs(raw) {
537
+ const out = []
538
+ for (const line of normalizeNewlines(raw).split('\n')) {
539
+ if (/^[ \t]/.test(line) && out.length) out[out.length - 1] += line.slice(1)
540
+ else out.push(line)
541
+ }
542
+ return out
543
+ }
544
+
545
+ function findPropertyColon(line) {
546
+ let quoted = false
547
+ for (let i = 0; i < line.length; i++) {
548
+ const char = line[i]
549
+ if (char === '"') quoted = !quoted
550
+ else if (char === ':' && !quoted) return i
551
+ }
552
+ return -1
553
+ }
554
+
555
+ function parseIcsProperty(line) {
556
+ const colon = findPropertyColon(line)
557
+ if (colon <= 0) return null
558
+ const left = line.slice(0, colon)
559
+ const value = line.slice(colon + 1)
560
+ const segments = splitQuoted(left, ';')
561
+ const name = segments[0].trim().toUpperCase()
562
+ if (!name) return null
563
+ const params = {}
564
+ for (const segment of segments.slice(1)) {
565
+ const equals = segment.indexOf('=')
566
+ if (equals <= 0) continue
567
+ params[segment.slice(0, equals).trim().toUpperCase()] = unquote(segment.slice(equals + 1))
568
+ }
569
+ return { name, params, value }
570
+ }
571
+
572
+ function unescapeIcsText(value) {
573
+ const source = asString(value)
574
+ let out = ''
575
+ for (let i = 0; i < source.length; i++) {
576
+ const char = source[i]
577
+ if (char === '\\' && i + 1 < source.length) {
578
+ const next = source[i + 1]
579
+ if (next === 'n' || next === 'N') {
580
+ out += '\n'
581
+ i += 1
582
+ } else if (next === ',' || next === ';' || next === '\\') {
583
+ out += next
584
+ i += 1
585
+ } else {
586
+ out += char
587
+ }
588
+ } else {
589
+ out += char
590
+ }
591
+ }
592
+ return out
593
+ }
594
+
595
+ function parseIcsDate(property) {
596
+ if (!property) return { value: null, allDay: false, tzid: null }
597
+ const raw = asString(property.value).trim()
598
+ const tzid = property.params.TZID ? property.params.TZID.trim() : null
599
+ let match = /^(\d{4})(\d{2})(\d{2})$/.exec(raw)
600
+ if (match) return { value: `${match[1]}-${match[2]}-${match[3]}`, allDay: true, tzid }
601
+ match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(Z)?$/i.exec(raw)
602
+ if (match) {
603
+ const value = `${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}${match[7] ? 'Z' : ''}`
604
+ return { value, allDay: false, tzid: match[7] ? null : tzid }
605
+ }
606
+ // Unknown / extended forms are kept verbatim so no information is lost.
607
+ return { value: raw, allDay: false, tzid }
608
+ }
609
+
610
+ function buildIcsEvent(lines) {
611
+ const properties = new Map()
612
+ let skipDepth = 0
613
+ for (const line of lines) {
614
+ const trimmed = line.trim()
615
+ if (!trimmed) continue
616
+ const upper = trimmed.toUpperCase()
617
+ if (skipDepth > 0) {
618
+ if (upper.startsWith('BEGIN:')) skipDepth += 1
619
+ else if (upper.startsWith('END:')) skipDepth -= 1
620
+ continue
621
+ }
622
+ if (upper.startsWith('BEGIN:')) {
623
+ // Nested components (VALARM) may carry their own DESCRIPTION; skip them.
624
+ skipDepth = 1
625
+ continue
626
+ }
627
+ if (upper.startsWith('END:')) continue
628
+ const property = parseIcsProperty(line)
629
+ if (property && !properties.has(property.name)) properties.set(property.name, property)
630
+ }
631
+
632
+ const get = (name) => properties.get(name)
633
+ const textValue = (name) => {
634
+ const property = get(name)
635
+ return property ? unescapeIcsText(property.value).trim() : ''
636
+ }
637
+ const start = parseIcsDate(get('DTSTART'))
638
+ const end = parseIcsDate(get('DTEND'))
639
+ const uidProperty = get('UID')
640
+ const rruleProperty = get('RRULE')
641
+
642
+ return {
643
+ uid: uidProperty ? unescapeIcsText(uidProperty.value).trim() : '',
644
+ summary: textValue('SUMMARY'),
645
+ start: start.value,
646
+ end: end ? end.value : null,
647
+ allDay: start.allDay,
648
+ tzid: start.tzid ?? null,
649
+ description: textValue('DESCRIPTION'),
650
+ location: textValue('LOCATION'),
651
+ // Kept as the original rule text: it is evidence, not something to expand.
652
+ rrule: rruleProperty ? asString(rruleProperty.value).trim() : null,
653
+ }
654
+ }
655
+
656
+ /**
657
+ * Parse an iCalendar stream and return its VEVENT records.
658
+ *
659
+ * Dates are normalised to `YYYY-MM-DD` / `YYYY-MM-DDTHH:MM:SS[Z]`. Floating
660
+ * times with `TZID=...` are kept literally and the zone is exposed as `tzid`
661
+ * (there is no time zone database in this module). RRULE is never expanded.
662
+ *
663
+ * @param {string} raw
664
+ * @returns {Array<{ uid: string, summary: string, start: string|null,
665
+ * end: string|null, allDay: boolean, tzid: string|null,
666
+ * description: string, location: string, rrule: string|null }>}
667
+ * @throws {Error} on structurally invalid iCalendar input
668
+ */
669
+ export function parseIcs(raw) {
670
+ if (typeof raw !== 'string') throw new Error('parseIcs: expected a string')
671
+ const source = normalizeNewlines(raw)
672
+ if (!source.trim()) throw new Error('parseIcs: empty input')
673
+ const lines = unfoldIcs(source)
674
+ const hasCalendar = lines.some((line) => line.trim().toUpperCase() === 'BEGIN:VCALENDAR')
675
+ if (!hasCalendar) throw new Error('parseIcs: missing BEGIN:VCALENDAR')
676
+
677
+ const events = []
678
+ let calendarOpen = false
679
+ let calendarClosed = false
680
+ let inEvent = false
681
+ let eventLines = []
682
+
683
+ for (const line of lines) {
684
+ const trimmed = line.trim()
685
+ if (!trimmed) continue
686
+ const upper = trimmed.toUpperCase()
687
+ if (upper === 'BEGIN:VCALENDAR') {
688
+ if (calendarOpen) throw new Error('parseIcs: nested VCALENDAR is not supported')
689
+ calendarOpen = true
690
+ continue
691
+ }
692
+ if (upper === 'END:VCALENDAR') {
693
+ if (!calendarOpen) throw new Error('parseIcs: END:VCALENDAR without BEGIN:VCALENDAR')
694
+ if (inEvent) throw new Error('parseIcs: unterminated VEVENT')
695
+ calendarOpen = false
696
+ calendarClosed = true
697
+ continue
698
+ }
699
+ if (upper === 'BEGIN:VEVENT') {
700
+ if (inEvent) throw new Error('parseIcs: nested VEVENT is not supported')
701
+ inEvent = true
702
+ eventLines = []
703
+ continue
704
+ }
705
+ if (upper === 'END:VEVENT') {
706
+ if (!inEvent) throw new Error('parseIcs: END:VEVENT without BEGIN:VEVENT')
707
+ events.push(buildIcsEvent(eventLines))
708
+ inEvent = false
709
+ eventLines = []
710
+ continue
711
+ }
712
+ if (inEvent) eventLines.push(line)
713
+ }
714
+
715
+ if (inEvent) throw new Error('parseIcs: unterminated VEVENT')
716
+ if (!calendarClosed) throw new Error('parseIcs: unterminated VCALENDAR')
717
+ return events
718
+ }
719
+
720
+ /* ------------------------------------------------------------------ *
721
+ * parseTranscript
722
+ * ------------------------------------------------------------------ */
723
+
724
+ const CUE_TIMING = /^(?:(\d{1,3}):)?(\d{1,2}):(\d{2})(?:[,.](\d{1,3}))?[ \t]*-->[ \t]*(?:(\d{1,3}):)?(\d{1,2}):(\d{2})(?:[,.](\d{1,3}))?/
725
+
726
+ function secondsFromCue(hours, minutes, seconds, fraction) {
727
+ const whole = Number(hours || 0) * 3600 + Number(minutes) * 60 + Number(seconds)
728
+ const millis = fraction ? Number(`${fraction}000`.slice(0, 3)) : 0
729
+ return Math.round((whole + millis / 1000) * 1000) / 1000
730
+ }
731
+
732
+ function parseCueTiming(line) {
733
+ const match = CUE_TIMING.exec(asString(line).trim())
734
+ if (!match) return null
735
+ return {
736
+ start: secondsFromCue(match[1], match[2], match[3], match[4]),
737
+ end: secondsFromCue(match[5], match[6], match[7], match[8]),
738
+ }
739
+ }
740
+
741
+ function detectTranscriptFormat(text) {
742
+ const head = asString(text).replace(/^\uFEFF/, '').trimStart()
743
+ if (/^WEBVTT(?:[ \t]|\n|$)/i.test(head)) return 'vtt'
744
+ if (/^\d{1,2}:\d{2}:\d{2},\d{1,3}[ \t]*-->/m.test(text)) return 'srt'
745
+ if (/-->/.test(text)) return 'vtt'
746
+ return 'text'
747
+ }
748
+
749
+ function speakerFromPrefix(line) {
750
+ const match = /^([^::\n]{1,24})[::][ \t]*(.*)$/.exec(asString(line).trim())
751
+ if (!match) return null
752
+ const speaker = match[1]
753
+ .replace(/^[*_`~\[\]()()【】]+|[*_`~\[\]()()【】]+$/g, '')
754
+ .trim()
755
+ if (!speaker || /^\d+$/.test(speaker)) return null
756
+ return { speaker, text: match[2].trim() }
757
+ }
758
+
759
+ function normaliseCueText(raw) {
760
+ let speaker
761
+ let text = asString(raw)
762
+ text = text.replace(/<v(?:\.[^ >]+)?[ \t]+([^>]+)>/gi, (_tag, name) => {
763
+ if (!speaker) speaker = asString(name).trim()
764
+ return ''
765
+ })
766
+ text = text.replace(/<\/v>/gi, '')
767
+ text = text.replace(/<[^>]*>/g, '')
768
+ text = decodeHtmlEntities(text)
769
+ const lines = text.split('\n').map((line) => line.trim()).filter(Boolean)
770
+ let body = lines.join('\n')
771
+ if (!speaker) {
772
+ const prefixed = speakerFromPrefix(lines[0] || '')
773
+ if (prefixed) {
774
+ speaker = prefixed.speaker
775
+ body = [prefixed.text, ...lines.slice(1)].filter(Boolean).join('\n')
776
+ }
777
+ }
778
+ return { speaker, text: body.trim() }
779
+ }
780
+
781
+ function parseCueTranscript(text) {
782
+ const segments = []
783
+ for (const block of asString(text).split(/\n[ \t]*\n/)) {
784
+ const lines = block.split('\n').map((line) => line.trimEnd()).filter((line) => line.trim() !== '')
785
+ if (!lines.length) continue
786
+ const first = lines[0].trim()
787
+ if (/^WEBVTT(?:[ \t]|$)/i.test(first) || /^NOTE(?:[ \t]|$)/i.test(first) || /^STYLE(?:[ \t]|$)/i.test(first) || /^REGION(?:[ \t]|$)/i.test(first)) continue
788
+ let timingIndex = -1
789
+ let timing = null
790
+ for (let i = 0; i < Math.min(lines.length, 3); i++) {
791
+ timing = parseCueTiming(lines[i])
792
+ if (timing) {
793
+ timingIndex = i
794
+ break
795
+ }
796
+ }
797
+ if (!timing) continue
798
+ const cue = normaliseCueText(lines.slice(timingIndex + 1).join('\n'))
799
+ if (!cue.text) continue
800
+ segments.push({
801
+ start: timing.start,
802
+ end: timing.end,
803
+ ...(cue.speaker ? { speaker: cue.speaker } : {}),
804
+ text: cue.text,
805
+ })
806
+ }
807
+ return { text: segments.map((segment) => segment.text).join('\n'), segments }
808
+ }
809
+
810
+ function parsePlainTranscript(text) {
811
+ const source = stripTrailing(asString(text)).trim()
812
+ if (!source) return { text: '', segments: [] }
813
+ const lines = source.split('\n').map((line) => line.trim()).filter(Boolean)
814
+ let hasSpeaker = false
815
+ const segments = []
816
+ for (const line of lines) {
817
+ const prefixed = speakerFromPrefix(line)
818
+ if (prefixed) {
819
+ hasSpeaker = true
820
+ segments.push({ speaker: prefixed.speaker, text: prefixed.text })
821
+ } else {
822
+ segments.push({ text: line })
823
+ }
824
+ }
825
+ if (!hasSpeaker) return { text: source, segments: [{ text: source }] }
826
+ return { text: source, segments }
827
+ }
828
+
829
+ /**
830
+ * Parse a transcript in WebVTT, SubRip or plain-text form.
831
+ *
832
+ * Format is auto-detected from the content (VTT header, `-->` timing lines and
833
+ * SRT comma milliseconds); plain `.txt`/`.md` files fall back to text mode.
834
+ * Cue times are returned as seconds (numbers). Plain text without timestamps
835
+ * yields one segment covering the whole text.
836
+ *
837
+ * @param {string} raw
838
+ * @param {{ format?: 'auto'|'vtt'|'srt'|'text' }} [options]
839
+ * @returns {{ text: string, segments: Array<{ start?: number, end?: number,
840
+ * speaker?: string, text: string }> }}
841
+ */
842
+ export function parseTranscript(raw, options = {}) {
843
+ const source = normalizeNewlines(raw)
844
+ if (!source.trim()) return { text: '', segments: [] }
845
+ const requested = asString(options?.format).trim().toLowerCase()
846
+ const format = requested && requested !== 'auto' ? requested : detectTranscriptFormat(source)
847
+ if (format === 'vtt' || format === 'webvtt' || format === 'srt') return parseCueTranscript(source)
848
+ return parsePlainTranscript(source)
849
+ }
850
+
851
+ /* ------------------------------------------------------------------ *
852
+ * toCaptureInput
853
+ * ------------------------------------------------------------------ */
854
+
855
+ function isTranscriptResult(parsed) {
856
+ return Boolean(parsed && typeof parsed === 'object' && Array.isArray(parsed.segments) && typeof parsed.text === 'string')
857
+ }
858
+
859
+ function isEmailResult(parsed) {
860
+ return Boolean(parsed && typeof parsed === 'object' && !Array.isArray(parsed.segments) &&
861
+ ('messageId' in parsed || 'subject' in parsed || ('from' in parsed && 'to' in parsed)))
862
+ }
863
+
864
+ function isCalendarResult(parsed) {
865
+ return Boolean(parsed && typeof parsed === 'object' && ('uid' in parsed || ('summary' in parsed && 'start' in parsed)))
866
+ }
867
+
868
+ function uniqueSpeakers(segments) {
869
+ const speakers = []
870
+ for (const segment of Array.isArray(segments) ? segments : []) {
871
+ const speaker = asString(segment?.speaker).trim()
872
+ if (speaker && !speakers.includes(speaker)) speakers.push(speaker)
873
+ }
874
+ return speakers
875
+ }
876
+
877
+ function emailCaptureInput(parsed, source) {
878
+ const text = cleanText(parsed.text) || cleanText(parsed.subject)
879
+ const meta = {
880
+ subject: asString(parsed.subject).trim(),
881
+ from: asString(parsed.from).trim(),
882
+ date: asString(parsed.date).trim(),
883
+ messageId: asString(parsed.messageId).trim(),
884
+ }
885
+ if (Array.isArray(parsed.warnings) && parsed.warnings.length) meta.warnings = [...parsed.warnings]
886
+ return { text, source, meta }
887
+ }
888
+
889
+ function calendarCaptureInput(parsed, source) {
890
+ const summary = asString(parsed.summary).trim()
891
+ const start = parsed.start == null ? null : asString(parsed.start)
892
+ const end = parsed.end == null ? null : asString(parsed.end)
893
+ const tzid = parsed.tzid == null ? null : asString(parsed.tzid)
894
+ const location = asString(parsed.location).trim()
895
+ const description = asString(parsed.description).trim()
896
+ const rrule = parsed.rrule == null ? null : asString(parsed.rrule)
897
+ const lines = []
898
+ if (summary) lines.push(`摘要:${summary}`)
899
+ const when = [start, end].filter(Boolean).join(' ~ ')
900
+ if (when) lines.push(`时间:${when}${tzid ? ` (${tzid})` : ''}`)
901
+ if (location) lines.push(`地点:${location}`)
902
+ if (description) lines.push(`详情:${description}`)
903
+ if (rrule) lines.push(`重复:${rrule}`)
904
+ return {
905
+ text: lines.join('\n'),
906
+ source,
907
+ meta: {
908
+ uid: asString(parsed.uid).trim(),
909
+ summary,
910
+ start,
911
+ end,
912
+ allDay: Boolean(parsed.allDay),
913
+ tzid,
914
+ location,
915
+ rrule,
916
+ },
917
+ }
918
+ }
919
+
920
+ function transcriptCaptureInput(parsed, source) {
921
+ const segments = Array.isArray(parsed.segments) ? parsed.segments : []
922
+ return {
923
+ text: cleanText(parsed.text),
924
+ source,
925
+ meta: { speakers: uniqueSpeakers(segments), segments },
926
+ }
927
+ }
928
+
929
+ /**
930
+ * Normalise a parser result into the `{ text, source, meta }` shape that the
931
+ * capture pipeline consumes.
932
+ *
933
+ * Source defaults are inferred from the result shape (`email`, `calendar`,
934
+ * `transcript`) and can be overridden with `{ source }`. Arrays (from
935
+ * parseMbox / parseIcs) are collapsed into one input; their per-record meta is
936
+ * exposed as `meta.items` plus `meta.count`.
937
+ *
938
+ * @param {object|object[]} parsed
939
+ * @param {{ source?: string }} [options]
940
+ * @returns {{ text: string, source: string, meta: object }}
941
+ */
942
+ export function toCaptureInput(parsed, { source } = {}) {
943
+ const explicitSource = source == null || source === '' ? undefined : asString(source)
944
+ if (Array.isArray(parsed)) {
945
+ const items = parsed.map((item) => toCaptureInput(item, {}))
946
+ const text = items.map((item) => item.text).filter(Boolean).join('\n\n')
947
+ return {
948
+ text,
949
+ source: explicitSource || items[0]?.source || 'import',
950
+ meta: { count: items.length, items: items.map((item) => item.meta) },
951
+ }
952
+ }
953
+ if (!parsed || typeof parsed !== 'object') {
954
+ return { text: cleanText(parsed), source: explicitSource || 'import', meta: {} }
955
+ }
956
+ if (isTranscriptResult(parsed)) return transcriptCaptureInput(parsed, explicitSource || 'transcript')
957
+ if (isEmailResult(parsed)) return emailCaptureInput(parsed, explicitSource || 'email')
958
+ if (isCalendarResult(parsed)) return calendarCaptureInput(parsed, explicitSource || 'calendar')
959
+ return { text: cleanText(parsed.text), source: explicitSource || 'import', meta: {} }
960
+ }
961
+
962
+ /* ------------------------------------------------------------------ *
963
+ * dedupeKey
964
+ * ------------------------------------------------------------------ */
965
+
966
+ function canonicalKeyText(value) {
967
+ return cleanText(asString(value))
968
+ .replace(/[ \t]+/g, ' ')
969
+ .replace(/[ \t]*\n[ \t]*/g, '\n')
970
+ }
971
+
972
+ function keyMaterial(parsed) {
973
+ if (Array.isArray(parsed)) return `array:${parsed.map(keyMaterial).join('\u0001')}`
974
+ if (parsed == null) return 'empty:'
975
+ if (typeof parsed === 'string') return `text:${canonicalKeyText(parsed)}`
976
+ if (typeof parsed !== 'object') return `value:${asString(parsed)}`
977
+ const messageId = cleanMessageId(parsed.messageId)
978
+ if (messageId) return `message-id:${messageId}`
979
+ const uid = cleanMessageId(parsed.uid)
980
+ if (uid) return `uid:${uid}`
981
+ let text = typeof parsed.text === 'string' ? parsed.text : ''
982
+ if (!text && typeof parsed.html === 'string') text = htmlToText(parsed.html)
983
+ return `text:${canonicalKeyText(text)}`
984
+ }
985
+
986
+ /**
987
+ * Stable sha256 key for idempotent imports. `messageId` wins, then `uid`,
988
+ * then the normalised text. The source string is part of the hash input so the
989
+ * same body imported from two channels does not collapse into one record.
990
+ *
991
+ * @param {string} source
992
+ * @param {object|object[]|string} parsed
993
+ * @returns {string} 64-character lowercase sha256 hex digest
994
+ */
995
+ export function dedupeKey(source, parsed) {
996
+ const hash = createHash('sha256')
997
+ hash.update(asString(source))
998
+ hash.update('\u0000')
999
+ hash.update(keyMaterial(parsed))
1000
+ return hash.digest('hex')
1001
+ }