advisers 0.1.0-preview.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +131 -21
- package/NOTICE +34 -0
- package/README-zh.md +40 -9
- package/README.md +53 -10
- package/THIRD-PARTY-NOTICES.md +277 -0
- package/bin/advisers.js +4230 -122
- package/index.js +1 -1
- package/lib/agent.js +414 -0
- package/lib/attachments.js +283 -0
- package/lib/capture.js +168 -0
- package/lib/context-window.js +123 -0
- package/lib/digest.js +27 -0
- package/lib/doc-review.js +30 -0
- package/lib/egress.js +27 -0
- package/lib/evidence.js +1995 -0
- package/lib/ical.js +542 -0
- package/lib/imap.js +181 -0
- package/lib/importers.js +1001 -0
- package/lib/learning.js +46 -0
- package/lib/mcp-server.js +869 -0
- package/lib/model.js +307 -0
- package/lib/notify.js +189 -0
- package/lib/nudge.js +10 -0
- package/lib/ocr.js +47 -0
- package/lib/pdf.js +105 -0
- package/lib/pet-brief-store.js +51 -0
- package/lib/pet-brief.js +644 -0
- package/lib/receipt-verify.js +624 -0
- package/lib/relative-date.js +393 -0
- package/lib/remind.js +690 -0
- package/lib/research.js +193 -0
- package/lib/review.js +88 -0
- package/lib/routine.js +22 -0
- package/lib/safety/classifier.js +392 -0
- package/lib/safety/explain.js +157 -0
- package/lib/safety/index.js +333 -0
- package/lib/safety/redact.js +212 -0
- package/lib/safety/rules.js +824 -0
- package/lib/scheduler.js +399 -0
- package/lib/source.js +294 -0
- package/lib/sqlite.js +1022 -0
- package/lib/sync.js +129 -0
- package/lib/timestamp.js +45 -0
- package/lib/tos/index.js +332 -0
- package/lib/tos/manifest.json +32 -0
- package/lib/tos/privacy.zh-CN.md +136 -0
- package/lib/tos/safety.zh-CN.md +101 -0
- package/lib/tos/summary.zh-CN.md +12 -0
- package/lib/tos/terms.zh-CN.md +439 -0
- package/lib/ui.js +49 -0
- package/lib/undo.js +83 -0
- package/lib/vault.js +51 -0
- package/package.json +26 -4
package/lib/importers.js
ADDED
|
@@ -0,0 +1,1001 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* E2 read-only import parsers.
|
|
3
|
+
*
|
|
4
|
+
* Pure, offline, dependency-free parsers for the file formats a read-only
|
|
5
|
+
* import entry point is expected to accept:
|
|
6
|
+
*
|
|
7
|
+
* parseEml(raw) -> one RFC 822 message (minimal MIME subset)
|
|
8
|
+
* parseMbox(raw) -> parseEml() for every `From ` separated block
|
|
9
|
+
* parseIcs(raw) -> array of VEVENT records
|
|
10
|
+
* parseTranscript(raw, opts?) -> { text, segments } from VTT/SRT/plain text
|
|
11
|
+
* toCaptureInput(parsed, opts) -> { text, source, meta } ready for capture
|
|
12
|
+
* dedupeKey(source, parsed) -> stable sha256 hex key
|
|
13
|
+
*
|
|
14
|
+
* Design constraints:
|
|
15
|
+
* - no network, no filesystem, no third-party dependencies (node:crypto only);
|
|
16
|
+
* - every function is deterministic for the same input;
|
|
17
|
+
* - malformed email/transcript input never throws: the affected text is
|
|
18
|
+
* dropped and explained in `warnings` (calendar input is the exception:
|
|
19
|
+
* structurally invalid iCalendar throws Error, as required by the E2 spec).
|
|
20
|
+
*
|
|
21
|
+
* Deliberate limits (documented, not bugs):
|
|
22
|
+
* - emails: no address rewriting, no attachment extraction, no MIME
|
|
23
|
+
* parameter continuation (RFC 2231), no S/MIME, no header charset guessing
|
|
24
|
+
* for 8bit bodies (the caller already supplied a JS string);
|
|
25
|
+
* - mbox: no `>From ` unquoting, no Content-Length handling;
|
|
26
|
+
* - iCalendar: no time zone database (TZID is kept as a literal marker), no
|
|
27
|
+
* RRULE expansion, no recurrence overrides / EXDATE resolution;
|
|
28
|
+
* - transcripts: cue timing only (no word-level timestamps), no speaker
|
|
29
|
+
* diarization beyond `<v Name>` / `Name:` prefixes.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { createHash } from 'node:crypto'
|
|
33
|
+
|
|
34
|
+
const HTML_ENTITIES = {
|
|
35
|
+
amp: '&',
|
|
36
|
+
apos: "'",
|
|
37
|
+
bull: '•',
|
|
38
|
+
copy: '©',
|
|
39
|
+
deg: '°',
|
|
40
|
+
divide: '÷',
|
|
41
|
+
emsp: '\u2003',
|
|
42
|
+
ensp: '\u2002',
|
|
43
|
+
euro: '€',
|
|
44
|
+
gt: '>',
|
|
45
|
+
hellip: '…',
|
|
46
|
+
ldquo: '“',
|
|
47
|
+
lsquo: '‘',
|
|
48
|
+
lt: '<',
|
|
49
|
+
mdash: '—',
|
|
50
|
+
middot: '·',
|
|
51
|
+
nbsp: '\u00a0',
|
|
52
|
+
ndash: '–',
|
|
53
|
+
para: '¶',
|
|
54
|
+
pound: '£',
|
|
55
|
+
quot: '"',
|
|
56
|
+
rdquo: '”',
|
|
57
|
+
reg: '®',
|
|
58
|
+
rsquo: '’',
|
|
59
|
+
sect: '§',
|
|
60
|
+
thinsp: '\u2009',
|
|
61
|
+
times: '×',
|
|
62
|
+
yen: '¥',
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const CHARSET_ALIASES = {
|
|
66
|
+
ascii: 'windows-1252',
|
|
67
|
+
'us-ascii': 'windows-1252',
|
|
68
|
+
'iso-8859-1': 'windows-1252',
|
|
69
|
+
'iso8859-1': 'windows-1252',
|
|
70
|
+
'latin1': 'windows-1252',
|
|
71
|
+
utf8: 'utf-8',
|
|
72
|
+
'utf-8': 'utf-8',
|
|
73
|
+
'gb2312': 'gbk',
|
|
74
|
+
'gb-2312': 'gbk',
|
|
75
|
+
'gb_2312': 'gbk',
|
|
76
|
+
'x-gbk': 'gbk',
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const TRANSFER_IDENTITY = new Set(['', '7bit', '8bit', 'binary'])
|
|
80
|
+
const TRANSFER_DECODED = new Set(['base64', 'quoted-printable'])
|
|
81
|
+
|
|
82
|
+
function asString(value) {
|
|
83
|
+
if (value == null) return ''
|
|
84
|
+
return typeof value === 'string' ? value : String(value)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function normalizeNewlines(value) {
|
|
88
|
+
return asString(value).replace(/^\uFEFF/, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n')
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function pushWarning(warnings, message) {
|
|
92
|
+
if (warnings && !warnings.includes(message)) warnings.push(message)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function cleanText(value) {
|
|
96
|
+
return asString(value)
|
|
97
|
+
.replace(/\r\n/g, '\n')
|
|
98
|
+
.replace(/\r/g, '\n')
|
|
99
|
+
.replace(/[ \t]+\n/g, '\n')
|
|
100
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
101
|
+
.trim()
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function stripTrailing(value) {
|
|
105
|
+
return asString(value).replace(/[ \t]+$/gm, '')
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/* ------------------------------------------------------------------ *
|
|
109
|
+
* Header helpers
|
|
110
|
+
* ------------------------------------------------------------------ */
|
|
111
|
+
|
|
112
|
+
function splitHeadersAndBody(text) {
|
|
113
|
+
const separator = text.indexOf('\n\n')
|
|
114
|
+
if (separator === -1) return { head: text, body: '', separated: false }
|
|
115
|
+
return { head: text.slice(0, separator), body: text.slice(separator + 2), separated: true }
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function unfoldHeaderBlock(block) {
|
|
119
|
+
const lines = []
|
|
120
|
+
for (const line of normalizeNewlines(block).split('\n')) {
|
|
121
|
+
if (/^[ \t]/.test(line) && lines.length) lines[lines.length - 1] += ` ${line.trim()}`
|
|
122
|
+
else lines.push(line)
|
|
123
|
+
}
|
|
124
|
+
return lines
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function parseHeaderBlock(block) {
|
|
128
|
+
const headers = {}
|
|
129
|
+
for (const line of unfoldHeaderBlock(block)) {
|
|
130
|
+
const index = line.indexOf(':')
|
|
131
|
+
if (index <= 0) continue
|
|
132
|
+
const name = line.slice(0, index).trim().toLowerCase()
|
|
133
|
+
if (!name || name in headers) continue
|
|
134
|
+
headers[name] = line.slice(index + 1).trim()
|
|
135
|
+
}
|
|
136
|
+
return headers
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function splitQuoted(value, separator) {
|
|
140
|
+
const out = []
|
|
141
|
+
let current = ''
|
|
142
|
+
let quoted = false
|
|
143
|
+
for (let i = 0; i < value.length; i++) {
|
|
144
|
+
const char = value[i]
|
|
145
|
+
if (quoted) {
|
|
146
|
+
current += char
|
|
147
|
+
if (char === '\\' && i + 1 < value.length) current += value[++i]
|
|
148
|
+
else if (char === '"') quoted = false
|
|
149
|
+
} else if (char === '"') {
|
|
150
|
+
quoted = true
|
|
151
|
+
current += char
|
|
152
|
+
} else if (char === separator) {
|
|
153
|
+
out.push(current)
|
|
154
|
+
current = ''
|
|
155
|
+
} else {
|
|
156
|
+
current += char
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
out.push(current)
|
|
160
|
+
return out
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function unquote(value) {
|
|
164
|
+
const text = asString(value).trim()
|
|
165
|
+
if (text.length >= 2 && text.startsWith('"') && text.endsWith('"')) {
|
|
166
|
+
return text.slice(1, -1).replace(/\\(.)/g, '$1')
|
|
167
|
+
}
|
|
168
|
+
return text
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function parseParameterList(raw) {
|
|
172
|
+
const params = {}
|
|
173
|
+
for (const segment of splitQuoted(asString(raw), ';').slice(1)) {
|
|
174
|
+
const equals = segment.indexOf('=')
|
|
175
|
+
if (equals <= 0) continue
|
|
176
|
+
const key = segment.slice(0, equals).trim().toLowerCase()
|
|
177
|
+
if (!key) continue
|
|
178
|
+
params[key] = unquote(segment.slice(equals + 1))
|
|
179
|
+
}
|
|
180
|
+
return params
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function parseContentType(value) {
|
|
184
|
+
const raw = asString(value).trim()
|
|
185
|
+
if (!raw) return { type: '', params: {} }
|
|
186
|
+
const segments = splitQuoted(raw, ';')
|
|
187
|
+
return { type: segments[0].trim().toLowerCase(), params: parseParameterList(raw) }
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function parseDisposition(value) {
|
|
191
|
+
const raw = asString(value).trim()
|
|
192
|
+
if (!raw) return ''
|
|
193
|
+
return splitQuoted(raw, ';')[0].trim().toLowerCase()
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function normalizeCharset(value) {
|
|
197
|
+
const label = unquote(value).trim().toLowerCase()
|
|
198
|
+
if (!label) return 'utf-8'
|
|
199
|
+
return CHARSET_ALIASES[label] || label
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function decodeBytes(bytes, charset, warnings) {
|
|
203
|
+
const label = normalizeCharset(charset)
|
|
204
|
+
if (!bytes || !bytes.length) return ''
|
|
205
|
+
try {
|
|
206
|
+
return new TextDecoder(label, { fatal: false }).decode(bytes)
|
|
207
|
+
} catch {
|
|
208
|
+
pushWarning(warnings, `unsupported charset: ${asString(charset).trim() || label}`)
|
|
209
|
+
return null
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/* ------------------------------------------------------------------ *
|
|
214
|
+
* RFC 2047 encoded words (headers)
|
|
215
|
+
* ------------------------------------------------------------------ */
|
|
216
|
+
|
|
217
|
+
function decodeQuotedPrintableWord(payload) {
|
|
218
|
+
const bytes = []
|
|
219
|
+
for (let i = 0; i < payload.length; i++) {
|
|
220
|
+
const char = payload[i]
|
|
221
|
+
if (char === '_') {
|
|
222
|
+
bytes.push(0x20)
|
|
223
|
+
continue
|
|
224
|
+
}
|
|
225
|
+
if (char === '=' && /^[0-9A-Fa-f]{2}$/.test(payload.slice(i + 1, i + 3))) {
|
|
226
|
+
bytes.push(Number.parseInt(payload.slice(i + 1, i + 3), 16))
|
|
227
|
+
i += 2
|
|
228
|
+
continue
|
|
229
|
+
}
|
|
230
|
+
const code = char.charCodeAt(0)
|
|
231
|
+
bytes.push(code <= 0xff ? code : 0x3f)
|
|
232
|
+
}
|
|
233
|
+
return Buffer.from(bytes)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function decodeEncodedWord(charset, encoding, payload, warnings) {
|
|
237
|
+
let bytes
|
|
238
|
+
if (encoding.toUpperCase() === 'B') {
|
|
239
|
+
const compact = payload.replace(/\s+/g, '')
|
|
240
|
+
if (!/^[A-Za-z0-9+/]*={0,2}$/.test(compact) || compact.length % 4 === 1) {
|
|
241
|
+
pushWarning(warnings, 'invalid base64 encoded-word')
|
|
242
|
+
return null
|
|
243
|
+
}
|
|
244
|
+
bytes = Buffer.from(compact, 'base64')
|
|
245
|
+
} else {
|
|
246
|
+
bytes = decodeQuotedPrintableWord(payload)
|
|
247
|
+
}
|
|
248
|
+
return decodeBytes(bytes, charset.split('*')[0], warnings)
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function decodeHeaderValue(value, warnings) {
|
|
252
|
+
const input = asString(value)
|
|
253
|
+
if (!input.includes('=?')) return input
|
|
254
|
+
const pattern = /=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g
|
|
255
|
+
const tokens = []
|
|
256
|
+
let last = 0
|
|
257
|
+
let match
|
|
258
|
+
while ((match = pattern.exec(input))) {
|
|
259
|
+
if (match.index > last) tokens.push({ encoded: false, raw: input.slice(last, match.index) })
|
|
260
|
+
tokens.push({ encoded: true, raw: match[0], charset: match[1], encoding: match[2], payload: match[3] })
|
|
261
|
+
last = match.index + match[0].length
|
|
262
|
+
}
|
|
263
|
+
if (!tokens.length) return input
|
|
264
|
+
if (last < input.length) tokens.push({ encoded: false, raw: input.slice(last) })
|
|
265
|
+
let out = ''
|
|
266
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
267
|
+
const token = tokens[i]
|
|
268
|
+
if (token.encoded) {
|
|
269
|
+
const decoded = decodeEncodedWord(token.charset, token.encoding, token.payload, warnings)
|
|
270
|
+
out += decoded == null ? token.raw : decoded
|
|
271
|
+
} else if (tokens[i - 1]?.encoded && tokens[i + 1]?.encoded && /^[ \t\r\n]*$/.test(token.raw)) {
|
|
272
|
+
// RFC 2047: whitespace between two adjacent encoded-words is not content.
|
|
273
|
+
} else {
|
|
274
|
+
out += token.raw
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
return out
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/* ------------------------------------------------------------------ *
|
|
281
|
+
* Bodies: transfer encodings, charsets, HTML
|
|
282
|
+
* ------------------------------------------------------------------ */
|
|
283
|
+
|
|
284
|
+
function decodeQuotedPrintableBody(value) {
|
|
285
|
+
const source = asString(value)
|
|
286
|
+
const bytes = []
|
|
287
|
+
for (let i = 0; i < source.length; i++) {
|
|
288
|
+
const char = source[i]
|
|
289
|
+
if (char === '=') {
|
|
290
|
+
if (source[i + 1] === '\n') {
|
|
291
|
+
i += 1 // soft line break
|
|
292
|
+
continue
|
|
293
|
+
}
|
|
294
|
+
if (/^[0-9A-Fa-f]{2}$/.test(source.slice(i + 1, i + 3))) {
|
|
295
|
+
bytes.push(Number.parseInt(source.slice(i + 1, i + 3), 16))
|
|
296
|
+
i += 2
|
|
297
|
+
continue
|
|
298
|
+
}
|
|
299
|
+
bytes.push(0x3d) // lone '=': keep literal rather than corrupt the body
|
|
300
|
+
continue
|
|
301
|
+
}
|
|
302
|
+
const code = char.charCodeAt(0)
|
|
303
|
+
if (code <= 0x7f) {
|
|
304
|
+
bytes.push(code)
|
|
305
|
+
} else {
|
|
306
|
+
bytes.push(...Buffer.from(char, 'utf8'))
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return Buffer.from(bytes)
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function decodeTransferBody(body, transferEncoding, warnings) {
|
|
313
|
+
const encoding = asString(transferEncoding).trim().toLowerCase()
|
|
314
|
+
if (TRANSFER_IDENTITY.has(encoding)) return { text: asString(body) }
|
|
315
|
+
if (TRANSFER_DECODED.has(encoding)) {
|
|
316
|
+
if (encoding === 'quoted-printable') return { bytes: decodeQuotedPrintableBody(body) }
|
|
317
|
+
const compact = asString(body).replace(/\s+/g, '')
|
|
318
|
+
if (!compact) return { bytes: Buffer.alloc(0) }
|
|
319
|
+
if (!/^[A-Za-z0-9+/]*={0,2}$/.test(compact) || compact.length % 4 === 1) {
|
|
320
|
+
pushWarning(warnings, 'invalid base64 body')
|
|
321
|
+
return null
|
|
322
|
+
}
|
|
323
|
+
return { bytes: Buffer.from(compact, 'base64') }
|
|
324
|
+
}
|
|
325
|
+
pushWarning(warnings, `unsupported content-transfer-encoding: ${encoding}`)
|
|
326
|
+
return null
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function decodeEntityContent(headers, body, warnings) {
|
|
330
|
+
const contentType = parseContentType(headers['content-type'])
|
|
331
|
+
const decoded = decodeTransferBody(body, headers['content-transfer-encoding'], warnings)
|
|
332
|
+
if (!decoded) return { text: null, contentType }
|
|
333
|
+
if (decoded.text !== undefined) return { text: decoded.text, contentType }
|
|
334
|
+
const charset = contentType.params.charset || 'utf-8'
|
|
335
|
+
return { text: decodeBytes(decoded.bytes, charset, warnings), contentType }
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function decodeNumericEntity(digits, radix) {
|
|
339
|
+
const codePoint = Number.parseInt(digits, radix)
|
|
340
|
+
if (!Number.isFinite(codePoint) || codePoint <= 0 || codePoint > 0x10ffff) return null
|
|
341
|
+
if (codePoint >= 0xd800 && codePoint <= 0xdfff) return null
|
|
342
|
+
try {
|
|
343
|
+
return String.fromCodePoint(codePoint)
|
|
344
|
+
} catch {
|
|
345
|
+
return null
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function decodeHtmlEntities(value) {
|
|
350
|
+
return asString(value).replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);/gi, (entity, body) => {
|
|
351
|
+
if (body[0] === '#') {
|
|
352
|
+
const hex = body[1] === 'x' || body[1] === 'X'
|
|
353
|
+
const decoded = decodeNumericEntity(body.slice(hex ? 2 : 1), hex ? 16 : 10)
|
|
354
|
+
return decoded == null ? entity : decoded
|
|
355
|
+
}
|
|
356
|
+
const named = HTML_ENTITIES[body.toLowerCase()]
|
|
357
|
+
return named == null ? entity : named
|
|
358
|
+
})
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function htmlToText(value) {
|
|
362
|
+
const html = asString(value)
|
|
363
|
+
if (!html) return ''
|
|
364
|
+
let text = html
|
|
365
|
+
.replace(/<!--[\s\S]*?-->/g, '')
|
|
366
|
+
.replace(/<(script|style|head|title)\b[^>]*>[\s\S]*?<\/\1>/gi, '')
|
|
367
|
+
.replace(/<br\s*\/?>/gi, '\n')
|
|
368
|
+
.replace(/<li\b[^>]*>/gi, '\n- ')
|
|
369
|
+
.replace(/<\/(p|div|li|tr|h[1-6]|blockquote|table|section|article|pre)>/gi, '\n')
|
|
370
|
+
.replace(/<[^>]+>/g, '')
|
|
371
|
+
text = decodeHtmlEntities(text)
|
|
372
|
+
return text
|
|
373
|
+
.replace(/\u00a0/g, ' ')
|
|
374
|
+
.split('\n')
|
|
375
|
+
.map((line) => line.replace(/[ \t]+/g, ' ').trim())
|
|
376
|
+
.join('\n')
|
|
377
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
378
|
+
.trim()
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/* ------------------------------------------------------------------ *
|
|
382
|
+
* MIME multipart walking
|
|
383
|
+
* ------------------------------------------------------------------ */
|
|
384
|
+
|
|
385
|
+
function splitMultipart(body, boundary, warnings) {
|
|
386
|
+
const marker = `--${asString(boundary)}`
|
|
387
|
+
if (!asString(boundary)) {
|
|
388
|
+
pushWarning(warnings, 'multipart entity without boundary parameter')
|
|
389
|
+
return []
|
|
390
|
+
}
|
|
391
|
+
const parts = []
|
|
392
|
+
let current = null
|
|
393
|
+
let closed = false
|
|
394
|
+
for (const line of normalizeNewlines(body).split('\n')) {
|
|
395
|
+
const trimmed = line.replace(/[ \t]+$/, '')
|
|
396
|
+
if (trimmed === marker || trimmed === `${marker}--`) {
|
|
397
|
+
if (current) parts.push(current.join('\n'))
|
|
398
|
+
current = null
|
|
399
|
+
if (trimmed === `${marker}--`) {
|
|
400
|
+
closed = true
|
|
401
|
+
break
|
|
402
|
+
}
|
|
403
|
+
current = []
|
|
404
|
+
continue
|
|
405
|
+
}
|
|
406
|
+
if (current) current.push(line)
|
|
407
|
+
}
|
|
408
|
+
if (current) parts.push(current.join('\n'))
|
|
409
|
+
if (!closed) pushWarning(warnings, 'multipart entity is missing its closing boundary')
|
|
410
|
+
return parts.map((part) => part.replace(/^\n+/, '')).filter((part) => part.trim() !== '')
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
function collectMimeContent(entity, warnings, collected) {
|
|
414
|
+
const { head, body } = splitHeadersAndBody(normalizeNewlines(entity))
|
|
415
|
+
const headers = parseHeaderBlock(head)
|
|
416
|
+
const contentType = parseContentType(headers['content-type'])
|
|
417
|
+
if (parseDisposition(headers['content-disposition']) === 'attachment') return
|
|
418
|
+
|
|
419
|
+
if (contentType.type.startsWith('multipart/')) {
|
|
420
|
+
const decoded = decodeTransferBody(body, headers['content-transfer-encoding'], warnings)
|
|
421
|
+
if (!decoded) return
|
|
422
|
+
let raw = decoded.text
|
|
423
|
+
if (raw === undefined) raw = decodeBytes(decoded.bytes, 'utf-8', warnings)
|
|
424
|
+
if (raw == null) return
|
|
425
|
+
const parts = splitMultipart(raw, contentType.params.boundary, warnings)
|
|
426
|
+
if (!parts.length) pushWarning(warnings, 'multipart entity has no parts')
|
|
427
|
+
for (const part of parts) collectMimeContent(part, warnings, collected)
|
|
428
|
+
return
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
const { text } = decodeEntityContent(headers, body, warnings)
|
|
432
|
+
if (text == null) return
|
|
433
|
+
if (contentType.type === 'text/html') collected.html.push(text)
|
|
434
|
+
else if (contentType.type === 'text/plain' || !contentType.type) collected.plain.push(text)
|
|
435
|
+
// other leaf types (images, application/pdf, message/rfc822, ...) are ignored
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/* ------------------------------------------------------------------ *
|
|
439
|
+
* parseEml
|
|
440
|
+
* ------------------------------------------------------------------ */
|
|
441
|
+
|
|
442
|
+
function cleanMessageId(value) {
|
|
443
|
+
const text = asString(value).trim()
|
|
444
|
+
const wrapped = /^<([^>]+)>$/.exec(text)
|
|
445
|
+
return (wrapped ? wrapped[1] : text).trim()
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* Parse one RFC 822 message using a minimal MIME subset.
|
|
450
|
+
*
|
|
451
|
+
* `text` is always the readable rendering: the first `text/plain` part, or the
|
|
452
|
+
* tag-stripped / entity-decoded `text/html` part when no plain part exists.
|
|
453
|
+
* `html` (only when a `text/html` part exists) keeps the decoded HTML source so
|
|
454
|
+
* callers can re-render or archive it; it is never returned as capture text.
|
|
455
|
+
* Encoding failures and missing bodies yield `text: ''` plus a non-empty
|
|
456
|
+
* `warnings` array instead of throwing.
|
|
457
|
+
*
|
|
458
|
+
* @param {string} raw
|
|
459
|
+
* @returns {{ from: string, to: string, subject: string, date: string,
|
|
460
|
+
* messageId: string, text: string, html?: string, warnings: string[] }}
|
|
461
|
+
*/
|
|
462
|
+
export function parseEml(raw) {
|
|
463
|
+
const warnings = []
|
|
464
|
+
const source = normalizeNewlines(raw)
|
|
465
|
+
if (!source.trim()) {
|
|
466
|
+
return { from: '', to: '', subject: '', date: '', messageId: '', text: '', warnings: ['empty message'] }
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const split = splitHeadersAndBody(source)
|
|
470
|
+
let headers = parseHeaderBlock(split.head)
|
|
471
|
+
let body = split.body
|
|
472
|
+
if (!split.separated && !Object.keys(headers).length) {
|
|
473
|
+
// Tolerate a body-only input instead of silently returning nothing.
|
|
474
|
+
headers = {}
|
|
475
|
+
body = source
|
|
476
|
+
pushWarning(warnings, 'message has no header/body separator')
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
const from = cleanText(decodeHeaderValue(headers.from, warnings))
|
|
480
|
+
const to = cleanText(decodeHeaderValue(headers.to, warnings))
|
|
481
|
+
const subject = cleanText(decodeHeaderValue(headers.subject, warnings))
|
|
482
|
+
const date = cleanText(decodeHeaderValue(headers.date, warnings))
|
|
483
|
+
const messageId = cleanMessageId(decodeHeaderValue(headers['message-id'], warnings))
|
|
484
|
+
|
|
485
|
+
const collected = { plain: [], html: [] }
|
|
486
|
+
collectMimeContent(source, warnings, collected)
|
|
487
|
+
|
|
488
|
+
const plain = collected.plain.map(cleanText).find(Boolean) || ''
|
|
489
|
+
const html = collected.html.map(cleanText).find(Boolean) || ''
|
|
490
|
+
const text = plain || (html ? htmlToText(html) : '')
|
|
491
|
+
if (!text) pushWarning(warnings, 'message has no readable text body')
|
|
492
|
+
|
|
493
|
+
const result = { from, to, subject, date, messageId, text, warnings }
|
|
494
|
+
if (html) result.html = html
|
|
495
|
+
return result
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/* ------------------------------------------------------------------ *
|
|
499
|
+
* parseMbox
|
|
500
|
+
* ------------------------------------------------------------------ */
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* Split an mbox stream on `^From ` separator lines and parse every message.
|
|
504
|
+
*
|
|
505
|
+
* @param {string} raw
|
|
506
|
+
* @returns {Array<ReturnType<typeof parseEml>>}
|
|
507
|
+
*/
|
|
508
|
+
export function parseMbox(raw) {
|
|
509
|
+
const source = normalizeNewlines(raw)
|
|
510
|
+
if (!source.trim()) return []
|
|
511
|
+
const results = []
|
|
512
|
+
const leading = []
|
|
513
|
+
let current = null
|
|
514
|
+
for (const line of source.split('\n')) {
|
|
515
|
+
if (line.startsWith('From ')) {
|
|
516
|
+
if (current) results.push(parseEml(current.join('\n')))
|
|
517
|
+
current = []
|
|
518
|
+
continue
|
|
519
|
+
}
|
|
520
|
+
if (current) current.push(line)
|
|
521
|
+
else leading.push(line)
|
|
522
|
+
}
|
|
523
|
+
if (current) {
|
|
524
|
+
const chunk = current.join('\n')
|
|
525
|
+
if (chunk.trim()) results.push(parseEml(chunk))
|
|
526
|
+
}
|
|
527
|
+
const preamble = leading.join('\n').trim()
|
|
528
|
+
if (preamble) results.unshift(parseEml(preamble))
|
|
529
|
+
return results
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/* ------------------------------------------------------------------ *
|
|
533
|
+
* parseIcs
|
|
534
|
+
* ------------------------------------------------------------------ */
|
|
535
|
+
|
|
536
|
+
function unfoldIcs(raw) {
|
|
537
|
+
const out = []
|
|
538
|
+
for (const line of normalizeNewlines(raw).split('\n')) {
|
|
539
|
+
if (/^[ \t]/.test(line) && out.length) out[out.length - 1] += line.slice(1)
|
|
540
|
+
else out.push(line)
|
|
541
|
+
}
|
|
542
|
+
return out
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
function findPropertyColon(line) {
|
|
546
|
+
let quoted = false
|
|
547
|
+
for (let i = 0; i < line.length; i++) {
|
|
548
|
+
const char = line[i]
|
|
549
|
+
if (char === '"') quoted = !quoted
|
|
550
|
+
else if (char === ':' && !quoted) return i
|
|
551
|
+
}
|
|
552
|
+
return -1
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
function parseIcsProperty(line) {
|
|
556
|
+
const colon = findPropertyColon(line)
|
|
557
|
+
if (colon <= 0) return null
|
|
558
|
+
const left = line.slice(0, colon)
|
|
559
|
+
const value = line.slice(colon + 1)
|
|
560
|
+
const segments = splitQuoted(left, ';')
|
|
561
|
+
const name = segments[0].trim().toUpperCase()
|
|
562
|
+
if (!name) return null
|
|
563
|
+
const params = {}
|
|
564
|
+
for (const segment of segments.slice(1)) {
|
|
565
|
+
const equals = segment.indexOf('=')
|
|
566
|
+
if (equals <= 0) continue
|
|
567
|
+
params[segment.slice(0, equals).trim().toUpperCase()] = unquote(segment.slice(equals + 1))
|
|
568
|
+
}
|
|
569
|
+
return { name, params, value }
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function unescapeIcsText(value) {
|
|
573
|
+
const source = asString(value)
|
|
574
|
+
let out = ''
|
|
575
|
+
for (let i = 0; i < source.length; i++) {
|
|
576
|
+
const char = source[i]
|
|
577
|
+
if (char === '\\' && i + 1 < source.length) {
|
|
578
|
+
const next = source[i + 1]
|
|
579
|
+
if (next === 'n' || next === 'N') {
|
|
580
|
+
out += '\n'
|
|
581
|
+
i += 1
|
|
582
|
+
} else if (next === ',' || next === ';' || next === '\\') {
|
|
583
|
+
out += next
|
|
584
|
+
i += 1
|
|
585
|
+
} else {
|
|
586
|
+
out += char
|
|
587
|
+
}
|
|
588
|
+
} else {
|
|
589
|
+
out += char
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
return out
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
function parseIcsDate(property) {
|
|
596
|
+
if (!property) return { value: null, allDay: false, tzid: null }
|
|
597
|
+
const raw = asString(property.value).trim()
|
|
598
|
+
const tzid = property.params.TZID ? property.params.TZID.trim() : null
|
|
599
|
+
let match = /^(\d{4})(\d{2})(\d{2})$/.exec(raw)
|
|
600
|
+
if (match) return { value: `${match[1]}-${match[2]}-${match[3]}`, allDay: true, tzid }
|
|
601
|
+
match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})(Z)?$/i.exec(raw)
|
|
602
|
+
if (match) {
|
|
603
|
+
const value = `${match[1]}-${match[2]}-${match[3]}T${match[4]}:${match[5]}:${match[6]}${match[7] ? 'Z' : ''}`
|
|
604
|
+
return { value, allDay: false, tzid: match[7] ? null : tzid }
|
|
605
|
+
}
|
|
606
|
+
// Unknown / extended forms are kept verbatim so no information is lost.
|
|
607
|
+
return { value: raw, allDay: false, tzid }
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
function buildIcsEvent(lines) {
|
|
611
|
+
const properties = new Map()
|
|
612
|
+
let skipDepth = 0
|
|
613
|
+
for (const line of lines) {
|
|
614
|
+
const trimmed = line.trim()
|
|
615
|
+
if (!trimmed) continue
|
|
616
|
+
const upper = trimmed.toUpperCase()
|
|
617
|
+
if (skipDepth > 0) {
|
|
618
|
+
if (upper.startsWith('BEGIN:')) skipDepth += 1
|
|
619
|
+
else if (upper.startsWith('END:')) skipDepth -= 1
|
|
620
|
+
continue
|
|
621
|
+
}
|
|
622
|
+
if (upper.startsWith('BEGIN:')) {
|
|
623
|
+
// Nested components (VALARM) may carry their own DESCRIPTION; skip them.
|
|
624
|
+
skipDepth = 1
|
|
625
|
+
continue
|
|
626
|
+
}
|
|
627
|
+
if (upper.startsWith('END:')) continue
|
|
628
|
+
const property = parseIcsProperty(line)
|
|
629
|
+
if (property && !properties.has(property.name)) properties.set(property.name, property)
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
const get = (name) => properties.get(name)
|
|
633
|
+
const textValue = (name) => {
|
|
634
|
+
const property = get(name)
|
|
635
|
+
return property ? unescapeIcsText(property.value).trim() : ''
|
|
636
|
+
}
|
|
637
|
+
const start = parseIcsDate(get('DTSTART'))
|
|
638
|
+
const end = parseIcsDate(get('DTEND'))
|
|
639
|
+
const uidProperty = get('UID')
|
|
640
|
+
const rruleProperty = get('RRULE')
|
|
641
|
+
|
|
642
|
+
return {
|
|
643
|
+
uid: uidProperty ? unescapeIcsText(uidProperty.value).trim() : '',
|
|
644
|
+
summary: textValue('SUMMARY'),
|
|
645
|
+
start: start.value,
|
|
646
|
+
end: end ? end.value : null,
|
|
647
|
+
allDay: start.allDay,
|
|
648
|
+
tzid: start.tzid ?? null,
|
|
649
|
+
description: textValue('DESCRIPTION'),
|
|
650
|
+
location: textValue('LOCATION'),
|
|
651
|
+
// Kept as the original rule text: it is evidence, not something to expand.
|
|
652
|
+
rrule: rruleProperty ? asString(rruleProperty.value).trim() : null,
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/**
|
|
657
|
+
* Parse an iCalendar stream and return its VEVENT records.
|
|
658
|
+
*
|
|
659
|
+
* Dates are normalised to `YYYY-MM-DD` / `YYYY-MM-DDTHH:MM:SS[Z]`. Floating
|
|
660
|
+
* times with `TZID=...` are kept literally and the zone is exposed as `tzid`
|
|
661
|
+
* (there is no time zone database in this module). RRULE is never expanded.
|
|
662
|
+
*
|
|
663
|
+
* @param {string} raw
|
|
664
|
+
* @returns {Array<{ uid: string, summary: string, start: string|null,
|
|
665
|
+
* end: string|null, allDay: boolean, tzid: string|null,
|
|
666
|
+
* description: string, location: string, rrule: string|null }>}
|
|
667
|
+
* @throws {Error} on structurally invalid iCalendar input
|
|
668
|
+
*/
|
|
669
|
+
export function parseIcs(raw) {
|
|
670
|
+
if (typeof raw !== 'string') throw new Error('parseIcs: expected a string')
|
|
671
|
+
const source = normalizeNewlines(raw)
|
|
672
|
+
if (!source.trim()) throw new Error('parseIcs: empty input')
|
|
673
|
+
const lines = unfoldIcs(source)
|
|
674
|
+
const hasCalendar = lines.some((line) => line.trim().toUpperCase() === 'BEGIN:VCALENDAR')
|
|
675
|
+
if (!hasCalendar) throw new Error('parseIcs: missing BEGIN:VCALENDAR')
|
|
676
|
+
|
|
677
|
+
const events = []
|
|
678
|
+
let calendarOpen = false
|
|
679
|
+
let calendarClosed = false
|
|
680
|
+
let inEvent = false
|
|
681
|
+
let eventLines = []
|
|
682
|
+
|
|
683
|
+
for (const line of lines) {
|
|
684
|
+
const trimmed = line.trim()
|
|
685
|
+
if (!trimmed) continue
|
|
686
|
+
const upper = trimmed.toUpperCase()
|
|
687
|
+
if (upper === 'BEGIN:VCALENDAR') {
|
|
688
|
+
if (calendarOpen) throw new Error('parseIcs: nested VCALENDAR is not supported')
|
|
689
|
+
calendarOpen = true
|
|
690
|
+
continue
|
|
691
|
+
}
|
|
692
|
+
if (upper === 'END:VCALENDAR') {
|
|
693
|
+
if (!calendarOpen) throw new Error('parseIcs: END:VCALENDAR without BEGIN:VCALENDAR')
|
|
694
|
+
if (inEvent) throw new Error('parseIcs: unterminated VEVENT')
|
|
695
|
+
calendarOpen = false
|
|
696
|
+
calendarClosed = true
|
|
697
|
+
continue
|
|
698
|
+
}
|
|
699
|
+
if (upper === 'BEGIN:VEVENT') {
|
|
700
|
+
if (inEvent) throw new Error('parseIcs: nested VEVENT is not supported')
|
|
701
|
+
inEvent = true
|
|
702
|
+
eventLines = []
|
|
703
|
+
continue
|
|
704
|
+
}
|
|
705
|
+
if (upper === 'END:VEVENT') {
|
|
706
|
+
if (!inEvent) throw new Error('parseIcs: END:VEVENT without BEGIN:VEVENT')
|
|
707
|
+
events.push(buildIcsEvent(eventLines))
|
|
708
|
+
inEvent = false
|
|
709
|
+
eventLines = []
|
|
710
|
+
continue
|
|
711
|
+
}
|
|
712
|
+
if (inEvent) eventLines.push(line)
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
if (inEvent) throw new Error('parseIcs: unterminated VEVENT')
|
|
716
|
+
if (!calendarClosed) throw new Error('parseIcs: unterminated VCALENDAR')
|
|
717
|
+
return events
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/* ------------------------------------------------------------------ *
|
|
721
|
+
* parseTranscript
|
|
722
|
+
* ------------------------------------------------------------------ */
|
|
723
|
+
|
|
724
|
+
const CUE_TIMING = /^(?:(\d{1,3}):)?(\d{1,2}):(\d{2})(?:[,.](\d{1,3}))?[ \t]*-->[ \t]*(?:(\d{1,3}):)?(\d{1,2}):(\d{2})(?:[,.](\d{1,3}))?/
|
|
725
|
+
|
|
726
|
+
function secondsFromCue(hours, minutes, seconds, fraction) {
|
|
727
|
+
const whole = Number(hours || 0) * 3600 + Number(minutes) * 60 + Number(seconds)
|
|
728
|
+
const millis = fraction ? Number(`${fraction}000`.slice(0, 3)) : 0
|
|
729
|
+
return Math.round((whole + millis / 1000) * 1000) / 1000
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
function parseCueTiming(line) {
|
|
733
|
+
const match = CUE_TIMING.exec(asString(line).trim())
|
|
734
|
+
if (!match) return null
|
|
735
|
+
return {
|
|
736
|
+
start: secondsFromCue(match[1], match[2], match[3], match[4]),
|
|
737
|
+
end: secondsFromCue(match[5], match[6], match[7], match[8]),
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
function detectTranscriptFormat(text) {
|
|
742
|
+
const head = asString(text).replace(/^\uFEFF/, '').trimStart()
|
|
743
|
+
if (/^WEBVTT(?:[ \t]|\n|$)/i.test(head)) return 'vtt'
|
|
744
|
+
if (/^\d{1,2}:\d{2}:\d{2},\d{1,3}[ \t]*-->/m.test(text)) return 'srt'
|
|
745
|
+
if (/-->/.test(text)) return 'vtt'
|
|
746
|
+
return 'text'
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
function speakerFromPrefix(line) {
|
|
750
|
+
const match = /^([^::\n]{1,24})[::][ \t]*(.*)$/.exec(asString(line).trim())
|
|
751
|
+
if (!match) return null
|
|
752
|
+
const speaker = match[1]
|
|
753
|
+
.replace(/^[*_`~\[\]()()【】]+|[*_`~\[\]()()【】]+$/g, '')
|
|
754
|
+
.trim()
|
|
755
|
+
if (!speaker || /^\d+$/.test(speaker)) return null
|
|
756
|
+
return { speaker, text: match[2].trim() }
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
function normaliseCueText(raw) {
|
|
760
|
+
let speaker
|
|
761
|
+
let text = asString(raw)
|
|
762
|
+
text = text.replace(/<v(?:\.[^ >]+)?[ \t]+([^>]+)>/gi, (_tag, name) => {
|
|
763
|
+
if (!speaker) speaker = asString(name).trim()
|
|
764
|
+
return ''
|
|
765
|
+
})
|
|
766
|
+
text = text.replace(/<\/v>/gi, '')
|
|
767
|
+
text = text.replace(/<[^>]*>/g, '')
|
|
768
|
+
text = decodeHtmlEntities(text)
|
|
769
|
+
const lines = text.split('\n').map((line) => line.trim()).filter(Boolean)
|
|
770
|
+
let body = lines.join('\n')
|
|
771
|
+
if (!speaker) {
|
|
772
|
+
const prefixed = speakerFromPrefix(lines[0] || '')
|
|
773
|
+
if (prefixed) {
|
|
774
|
+
speaker = prefixed.speaker
|
|
775
|
+
body = [prefixed.text, ...lines.slice(1)].filter(Boolean).join('\n')
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
return { speaker, text: body.trim() }
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
function parseCueTranscript(text) {
|
|
782
|
+
const segments = []
|
|
783
|
+
for (const block of asString(text).split(/\n[ \t]*\n/)) {
|
|
784
|
+
const lines = block.split('\n').map((line) => line.trimEnd()).filter((line) => line.trim() !== '')
|
|
785
|
+
if (!lines.length) continue
|
|
786
|
+
const first = lines[0].trim()
|
|
787
|
+
if (/^WEBVTT(?:[ \t]|$)/i.test(first) || /^NOTE(?:[ \t]|$)/i.test(first) || /^STYLE(?:[ \t]|$)/i.test(first) || /^REGION(?:[ \t]|$)/i.test(first)) continue
|
|
788
|
+
let timingIndex = -1
|
|
789
|
+
let timing = null
|
|
790
|
+
for (let i = 0; i < Math.min(lines.length, 3); i++) {
|
|
791
|
+
timing = parseCueTiming(lines[i])
|
|
792
|
+
if (timing) {
|
|
793
|
+
timingIndex = i
|
|
794
|
+
break
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
if (!timing) continue
|
|
798
|
+
const cue = normaliseCueText(lines.slice(timingIndex + 1).join('\n'))
|
|
799
|
+
if (!cue.text) continue
|
|
800
|
+
segments.push({
|
|
801
|
+
start: timing.start,
|
|
802
|
+
end: timing.end,
|
|
803
|
+
...(cue.speaker ? { speaker: cue.speaker } : {}),
|
|
804
|
+
text: cue.text,
|
|
805
|
+
})
|
|
806
|
+
}
|
|
807
|
+
return { text: segments.map((segment) => segment.text).join('\n'), segments }
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
function parsePlainTranscript(text) {
|
|
811
|
+
const source = stripTrailing(asString(text)).trim()
|
|
812
|
+
if (!source) return { text: '', segments: [] }
|
|
813
|
+
const lines = source.split('\n').map((line) => line.trim()).filter(Boolean)
|
|
814
|
+
let hasSpeaker = false
|
|
815
|
+
const segments = []
|
|
816
|
+
for (const line of lines) {
|
|
817
|
+
const prefixed = speakerFromPrefix(line)
|
|
818
|
+
if (prefixed) {
|
|
819
|
+
hasSpeaker = true
|
|
820
|
+
segments.push({ speaker: prefixed.speaker, text: prefixed.text })
|
|
821
|
+
} else {
|
|
822
|
+
segments.push({ text: line })
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
if (!hasSpeaker) return { text: source, segments: [{ text: source }] }
|
|
826
|
+
return { text: source, segments }
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
/**
|
|
830
|
+
* Parse a transcript in WebVTT, SubRip or plain-text form.
|
|
831
|
+
*
|
|
832
|
+
* Format is auto-detected from the content (VTT header, `-->` timing lines and
|
|
833
|
+
* SRT comma milliseconds); plain `.txt`/`.md` files fall back to text mode.
|
|
834
|
+
* Cue times are returned as seconds (numbers). Plain text without timestamps
|
|
835
|
+
* yields one segment covering the whole text.
|
|
836
|
+
*
|
|
837
|
+
* @param {string} raw
|
|
838
|
+
* @param {{ format?: 'auto'|'vtt'|'srt'|'text' }} [options]
|
|
839
|
+
* @returns {{ text: string, segments: Array<{ start?: number, end?: number,
|
|
840
|
+
* speaker?: string, text: string }> }}
|
|
841
|
+
*/
|
|
842
|
+
export function parseTranscript(raw, options = {}) {
|
|
843
|
+
const source = normalizeNewlines(raw)
|
|
844
|
+
if (!source.trim()) return { text: '', segments: [] }
|
|
845
|
+
const requested = asString(options?.format).trim().toLowerCase()
|
|
846
|
+
const format = requested && requested !== 'auto' ? requested : detectTranscriptFormat(source)
|
|
847
|
+
if (format === 'vtt' || format === 'webvtt' || format === 'srt') return parseCueTranscript(source)
|
|
848
|
+
return parsePlainTranscript(source)
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/* ------------------------------------------------------------------ *
|
|
852
|
+
* toCaptureInput
|
|
853
|
+
* ------------------------------------------------------------------ */
|
|
854
|
+
|
|
855
|
+
function isTranscriptResult(parsed) {
|
|
856
|
+
return Boolean(parsed && typeof parsed === 'object' && Array.isArray(parsed.segments) && typeof parsed.text === 'string')
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
function isEmailResult(parsed) {
|
|
860
|
+
return Boolean(parsed && typeof parsed === 'object' && !Array.isArray(parsed.segments) &&
|
|
861
|
+
('messageId' in parsed || 'subject' in parsed || ('from' in parsed && 'to' in parsed)))
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
function isCalendarResult(parsed) {
|
|
865
|
+
return Boolean(parsed && typeof parsed === 'object' && ('uid' in parsed || ('summary' in parsed && 'start' in parsed)))
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
function uniqueSpeakers(segments) {
|
|
869
|
+
const speakers = []
|
|
870
|
+
for (const segment of Array.isArray(segments) ? segments : []) {
|
|
871
|
+
const speaker = asString(segment?.speaker).trim()
|
|
872
|
+
if (speaker && !speakers.includes(speaker)) speakers.push(speaker)
|
|
873
|
+
}
|
|
874
|
+
return speakers
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
function emailCaptureInput(parsed, source) {
|
|
878
|
+
const text = cleanText(parsed.text) || cleanText(parsed.subject)
|
|
879
|
+
const meta = {
|
|
880
|
+
subject: asString(parsed.subject).trim(),
|
|
881
|
+
from: asString(parsed.from).trim(),
|
|
882
|
+
date: asString(parsed.date).trim(),
|
|
883
|
+
messageId: asString(parsed.messageId).trim(),
|
|
884
|
+
}
|
|
885
|
+
if (Array.isArray(parsed.warnings) && parsed.warnings.length) meta.warnings = [...parsed.warnings]
|
|
886
|
+
return { text, source, meta }
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
function calendarCaptureInput(parsed, source) {
|
|
890
|
+
const summary = asString(parsed.summary).trim()
|
|
891
|
+
const start = parsed.start == null ? null : asString(parsed.start)
|
|
892
|
+
const end = parsed.end == null ? null : asString(parsed.end)
|
|
893
|
+
const tzid = parsed.tzid == null ? null : asString(parsed.tzid)
|
|
894
|
+
const location = asString(parsed.location).trim()
|
|
895
|
+
const description = asString(parsed.description).trim()
|
|
896
|
+
const rrule = parsed.rrule == null ? null : asString(parsed.rrule)
|
|
897
|
+
const lines = []
|
|
898
|
+
if (summary) lines.push(`摘要:${summary}`)
|
|
899
|
+
const when = [start, end].filter(Boolean).join(' ~ ')
|
|
900
|
+
if (when) lines.push(`时间:${when}${tzid ? ` (${tzid})` : ''}`)
|
|
901
|
+
if (location) lines.push(`地点:${location}`)
|
|
902
|
+
if (description) lines.push(`详情:${description}`)
|
|
903
|
+
if (rrule) lines.push(`重复:${rrule}`)
|
|
904
|
+
return {
|
|
905
|
+
text: lines.join('\n'),
|
|
906
|
+
source,
|
|
907
|
+
meta: {
|
|
908
|
+
uid: asString(parsed.uid).trim(),
|
|
909
|
+
summary,
|
|
910
|
+
start,
|
|
911
|
+
end,
|
|
912
|
+
allDay: Boolean(parsed.allDay),
|
|
913
|
+
tzid,
|
|
914
|
+
location,
|
|
915
|
+
rrule,
|
|
916
|
+
},
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
function transcriptCaptureInput(parsed, source) {
|
|
921
|
+
const segments = Array.isArray(parsed.segments) ? parsed.segments : []
|
|
922
|
+
return {
|
|
923
|
+
text: cleanText(parsed.text),
|
|
924
|
+
source,
|
|
925
|
+
meta: { speakers: uniqueSpeakers(segments), segments },
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
/**
|
|
930
|
+
* Normalise a parser result into the `{ text, source, meta }` shape that the
|
|
931
|
+
* capture pipeline consumes.
|
|
932
|
+
*
|
|
933
|
+
* Source defaults are inferred from the result shape (`email`, `calendar`,
|
|
934
|
+
* `transcript`) and can be overridden with `{ source }`. Arrays (from
|
|
935
|
+
* parseMbox / parseIcs) are collapsed into one input; their per-record meta is
|
|
936
|
+
* exposed as `meta.items` plus `meta.count`.
|
|
937
|
+
*
|
|
938
|
+
* @param {object|object[]} parsed
|
|
939
|
+
* @param {{ source?: string }} [options]
|
|
940
|
+
* @returns {{ text: string, source: string, meta: object }}
|
|
941
|
+
*/
|
|
942
|
+
export function toCaptureInput(parsed, { source } = {}) {
|
|
943
|
+
const explicitSource = source == null || source === '' ? undefined : asString(source)
|
|
944
|
+
if (Array.isArray(parsed)) {
|
|
945
|
+
const items = parsed.map((item) => toCaptureInput(item, {}))
|
|
946
|
+
const text = items.map((item) => item.text).filter(Boolean).join('\n\n')
|
|
947
|
+
return {
|
|
948
|
+
text,
|
|
949
|
+
source: explicitSource || items[0]?.source || 'import',
|
|
950
|
+
meta: { count: items.length, items: items.map((item) => item.meta) },
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
if (!parsed || typeof parsed !== 'object') {
|
|
954
|
+
return { text: cleanText(parsed), source: explicitSource || 'import', meta: {} }
|
|
955
|
+
}
|
|
956
|
+
if (isTranscriptResult(parsed)) return transcriptCaptureInput(parsed, explicitSource || 'transcript')
|
|
957
|
+
if (isEmailResult(parsed)) return emailCaptureInput(parsed, explicitSource || 'email')
|
|
958
|
+
if (isCalendarResult(parsed)) return calendarCaptureInput(parsed, explicitSource || 'calendar')
|
|
959
|
+
return { text: cleanText(parsed.text), source: explicitSource || 'import', meta: {} }
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
/* ------------------------------------------------------------------ *
|
|
963
|
+
* dedupeKey
|
|
964
|
+
* ------------------------------------------------------------------ */
|
|
965
|
+
|
|
966
|
+
function canonicalKeyText(value) {
|
|
967
|
+
return cleanText(asString(value))
|
|
968
|
+
.replace(/[ \t]+/g, ' ')
|
|
969
|
+
.replace(/[ \t]*\n[ \t]*/g, '\n')
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
function keyMaterial(parsed) {
|
|
973
|
+
if (Array.isArray(parsed)) return `array:${parsed.map(keyMaterial).join('\u0001')}`
|
|
974
|
+
if (parsed == null) return 'empty:'
|
|
975
|
+
if (typeof parsed === 'string') return `text:${canonicalKeyText(parsed)}`
|
|
976
|
+
if (typeof parsed !== 'object') return `value:${asString(parsed)}`
|
|
977
|
+
const messageId = cleanMessageId(parsed.messageId)
|
|
978
|
+
if (messageId) return `message-id:${messageId}`
|
|
979
|
+
const uid = cleanMessageId(parsed.uid)
|
|
980
|
+
if (uid) return `uid:${uid}`
|
|
981
|
+
let text = typeof parsed.text === 'string' ? parsed.text : ''
|
|
982
|
+
if (!text && typeof parsed.html === 'string') text = htmlToText(parsed.html)
|
|
983
|
+
return `text:${canonicalKeyText(text)}`
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
/**
|
|
987
|
+
* Stable sha256 key for idempotent imports. `messageId` wins, then `uid`,
|
|
988
|
+
* then the normalised text. The source string is part of the hash input so the
|
|
989
|
+
* same body imported from two channels does not collapse into one record.
|
|
990
|
+
*
|
|
991
|
+
* @param {string} source
|
|
992
|
+
* @param {object|object[]|string} parsed
|
|
993
|
+
* @returns {string} 64-character lowercase sha256 hex digest
|
|
994
|
+
*/
|
|
995
|
+
export function dedupeKey(source, parsed) {
|
|
996
|
+
const hash = createHash('sha256')
|
|
997
|
+
hash.update(asString(source))
|
|
998
|
+
hash.update('\u0000')
|
|
999
|
+
hash.update(keyMaterial(parsed))
|
|
1000
|
+
return hash.digest('hex')
|
|
1001
|
+
}
|