advisers 0.3.2 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README-zh.md +13 -14
  2. package/README.md +37 -23
  3. package/THIRD-PARTY-NOTICES.md +1 -1
  4. package/bin/advisers.js +2302 -95
  5. package/index.js +1 -1
  6. package/lib/brief-archive.js +267 -0
  7. package/lib/client-pack.js +1016 -0
  8. package/lib/consent.js +390 -0
  9. package/lib/credentials.js +195 -0
  10. package/lib/error-codes.js +516 -7
  11. package/lib/evidence.js +697 -113
  12. package/lib/kb-extract.js +626 -0
  13. package/lib/kb-facts.js +429 -0
  14. package/lib/kb-pages.js +1084 -0
  15. package/lib/kb.js +1036 -0
  16. package/lib/mail-errors.js +46 -0
  17. package/lib/mail-extract.js +351 -0
  18. package/lib/mail-imap.js +446 -0
  19. package/lib/mail-import.js +176 -0
  20. package/lib/mail-parse.js +223 -0
  21. package/lib/mail-store.js +731 -0
  22. package/lib/mail.js +506 -0
  23. package/lib/mcp-server.js +196 -10
  24. package/lib/notify.js +265 -7
  25. package/lib/policy/decision.js +315 -0
  26. package/lib/policy/egress.js +268 -0
  27. package/lib/policy/errors.js +71 -0
  28. package/lib/policy/index.js +99 -0
  29. package/lib/policy/rules.js +468 -0
  30. package/lib/policy/surfaces.js +141 -0
  31. package/lib/receipt-verify.js +199 -19
  32. package/lib/receivables.js +1653 -0
  33. package/lib/scheduler.js +453 -9
  34. package/lib/shell.js +1406 -0
  35. package/lib/source.js +98 -3
  36. package/lib/sqlite.js +366 -49
  37. package/lib/tos/index.js +2 -2
  38. package/lib/tos/manifest.json +6 -6
  39. package/lib/tos/privacy.zh-CN.md +33 -20
  40. package/lib/tos/safety.zh-CN.md +7 -6
  41. package/lib/tos/summary.zh-CN.md +3 -2
  42. package/lib/tos/terms.zh-CN.md +31 -16
  43. package/lib/urgency.js +467 -0
  44. package/package.json +4 -4
@@ -0,0 +1,626 @@
1
+ /**
2
+ * KB text extraction (0.4.0, schema v22).
3
+ *
4
+ * One bounded, local-only entry point for turning a user file into the text the
5
+ * KB indexes. This is deliberately NOT extractAttachment(): attachments are a
6
+ * model-context budget (10 MB file, 20k chars, 50 PDF pages, silent PDF page
7
+ * cap) while the KB is a durable local index (50 MB file, 2 MB text, 500 PDF
8
+ * pages, explicit truncation). The two callers keep their own limits and the KB
9
+ * module must never silently drop half a document.
10
+ *
11
+ * Guarantees:
12
+ * - read-only: the source file is opened for reading only and is never copied;
13
+ * the returned object never contains an absolute path, only the basename;
14
+ * - no network, no OCR fallback, no model: a scanned/image file is reported as
15
+ * needsOcr=true with empty text (the caller records a warning), it is not
16
+ * silently treated as empty content;
17
+ * - deterministic: the same bytes + options produce the same extraction result;
18
+ * - all limits are explicit: file bytes, extracted text bytes, PDF pages, ZIP
19
+ * expansion budget and per-entry XML size. A limit hit sets truncated=true and
20
+ * names the reason instead of silently discarding data;
21
+ * - text encoding: BOM (UTF-8/UTF-16LE/UTF-16BE) -> strict UTF-8 -> GB18030
22
+ * fallback (the common Chinese-Windows legacy encoding). Undecodable bytes
23
+ * throw E_KB_ENCODING rather than storing mojibake.
24
+ *
25
+ * Pure module boundaries: no store, no HOME handling, no console output. The
26
+ * only environment interaction is the imported read-only EML parser.
27
+ */
28
+ import { lstatSync, readFileSync } from 'node:fs'
29
+ import { basename, extname } from 'node:path'
30
+ import JSZip from 'jszip'
31
+ import { parseEml } from './importers.js'
32
+
33
+ /** Hard per-file cap for KB ingestion (larger than the 10 MB attachment cap). */
34
+ export const KB_MAX_FILE_BYTES = 50 * 1024 * 1024
35
+ /** Hard cap for the extracted text stored in kb_chunks (UTF-8 bytes). */
36
+ export const KB_MAX_TEXT_BYTES = 2 * 1024 * 1024
37
+ /** Hard cap for indexed PDF pages (the attachment path only reads 50). */
38
+ export const KB_MAX_PDF_PAGES = 500
39
+ /** Extraction pipeline version recorded next to a document. */
40
+ export const KB_EXTRACTOR_VERSION = 1
41
+
42
+ const MAX_ZIP_ENTRIES = 4000
43
+ const MAX_ZIP_TOTAL_UNCOMPRESSED = 256 * 1024 * 1024
44
+ const MAX_ZIP_ENTRY_UNCOMPRESSED = 64 * 1024 * 1024
45
+ const MAX_XML_CHARS = 16 * 1024 * 1024
46
+
47
+ const TEXT_EXTENSIONS = Object.freeze([
48
+ '.txt', '.md', '.markdown', '.csv', '.json', '.jsonl', '.log', '.yaml', '.yml', '.xml', '.html', '.htm',
49
+ ])
50
+ const OFFICE_EXTENSIONS = Object.freeze(['.docx', '.xlsx', '.pptx'])
51
+ const IMAGE_EXTENSIONS = Object.freeze(['.png', '.jpg', '.jpeg', '.webp', '.bmp', '.tif', '.tiff'])
52
+ const MIME_BY_EXT = Object.freeze({
53
+ '.txt': 'text/plain', '.md': 'text/markdown', '.markdown': 'text/markdown',
54
+ '.csv': 'text/csv', '.json': 'application/json', '.jsonl': 'application/jsonl',
55
+ '.log': 'text/plain', '.yaml': 'application/yaml', '.yml': 'application/yaml',
56
+ '.xml': 'application/xml', '.html': 'text/html', '.htm': 'text/html',
57
+ '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
58
+ '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
59
+ '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
60
+ '.pdf': 'application/pdf', '.ofd': 'application/ofd', '.eml': 'message/rfc822',
61
+ })
62
+
63
+ // KB errors carry stable codes; the message keeps only the basename so a log or
64
+ // an audit reason can never leak the operator's directory layout.
65
+ function kbError(code, message, hint) {
66
+ const error = new Error(code + ': ' + message + (hint ? ' (hint: ' + hint + ')' : ''))
67
+ error.code = code
68
+ if (hint) error.hint = hint
69
+ return error
70
+ }
71
+
72
+ function byteLength(text) {
73
+ return Buffer.byteLength(text, 'utf8')
74
+ }
75
+
76
+ /** Truncate a JS string to at most maxBytes UTF-8 bytes without splitting a code point. */
77
+ export function truncateUtf8(text, maxBytes) {
78
+ const raw = String(text ?? '')
79
+ if (byteLength(raw) <= maxBytes) return { text: raw, truncated: false }
80
+ const buffer = Buffer.from(raw, 'utf8').subarray(0, maxBytes)
81
+ let truncated = buffer.toString('utf8')
82
+ // Buffer.subarray can cut a multi-byte sequence; drop the replacement char(s).
83
+ while (truncated.endsWith('\uFFFD')) truncated = truncated.slice(0, -1)
84
+ return { text: truncated, truncated: true }
85
+ }
86
+
87
+ export function extensionOf(file) {
88
+ return extname(String(file ?? '')).toLowerCase()
89
+ }
90
+
91
+ function kindOfExtension(ext) {
92
+ if (TEXT_EXTENSIONS.includes(ext)) return 'text'
93
+ if (OFFICE_EXTENSIONS.includes(ext)) return 'office'
94
+ if (ext === '.pdf') return 'pdf'
95
+ if (ext === '.ofd') return 'ofd'
96
+ if (ext === '.eml') return 'eml'
97
+ if (IMAGE_EXTENSIONS.includes(ext)) return 'image'
98
+ return null
99
+ }
100
+
101
+ /** True when the file type can be indexed (used by callers before hashing). */
102
+ export function isIndexableFile(file) {
103
+ return kindOfExtension(extensionOf(file)) !== null
104
+ }
105
+
106
+ export function mimeForFile(file) {
107
+ return MIME_BY_EXT[extensionOf(file)] ?? 'application/octet-stream'
108
+ }
109
+
110
+ /* -------------------------------------------------------------------------- */
111
+ /* text decoding (BOM -> strict UTF-8 -> GB18030) */
112
+ /* -------------------------------------------------------------------------- */
113
+
114
+ function decodeWith(label, buffer) {
115
+ return new TextDecoder(label, { fatal: true }).decode(buffer)
116
+ }
117
+
118
+ /**
119
+ * Decode a plain-text file buffer.
120
+ *
121
+ * Order: a BOM (UTF-8/UTF-16LE/UTF-16BE) wins; otherwise strict UTF-8; then
122
+ * GB18030 (the Windows Chinese legacy encoding, a superset of GBK/GB2312).
123
+ * Throws E_KB_ENCODING when even GB18030 yields U+FFFD (never store mojibake).
124
+ *
125
+ * @param {Buffer} buffer
126
+ * @returns {{text: string, encoding: string}}
127
+ */
128
+ export function decodeTextBuffer(buffer) {
129
+ const bytes = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer ?? [])
130
+ let text
131
+ if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
132
+ text = decodeWith('utf-8', bytes.subarray(3))
133
+ return { text, encoding: 'utf-8' }
134
+ }
135
+ if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
136
+ text = decodeWith('utf-16le', bytes.subarray(2))
137
+ return { text, encoding: 'utf-16le' }
138
+ }
139
+ if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
140
+ text = decodeWith('utf-16be', bytes.subarray(2))
141
+ return { text, encoding: 'utf-16be' }
142
+ }
143
+ try {
144
+ return { text: decodeWith('utf-8', bytes), encoding: 'utf-8' }
145
+ } catch {
146
+ // Not UTF-8: fall through to the legacy Chinese encoding.
147
+ }
148
+ let legacy
149
+ try {
150
+ legacy = new TextDecoder('gb18030', { fatal: false }).decode(bytes)
151
+ } catch (error) {
152
+ throw kbError('E_KB_ENCODING', 'file bytes are neither valid UTF-8 nor decodable as GB18030 (' + String(error?.message ?? error).slice(0, 120) + ')')
153
+ }
154
+ if (legacy.includes('\uFFFD')) {
155
+ throw kbError('E_KB_ENCODING', 'file bytes are neither valid UTF-8 nor GB18030; convert the file to UTF-8 first')
156
+ }
157
+ return { text: legacy, encoding: 'gb18030' }
158
+ }
159
+
160
+ /* -------------------------------------------------------------------------- */
161
+ /* HTML to text (minimal, tag-stripping only; never executes anything) */
162
+ /* -------------------------------------------------------------------------- */
163
+
164
+ const HTML_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' }
165
+
166
+ export function htmlToIndexText(html) {
167
+ const source = String(html ?? '')
168
+ const withoutHidden = source
169
+ .replace(/<script\b[\s\S]*?<\/script\s*>/gi, ' ')
170
+ .replace(/<style\b[\s\S]*?<\/style\s*>/gi, ' ')
171
+ .replace(/<!--[\s\S]*?-->/g, ' ')
172
+ const blocked = withoutHidden.replace(/<(?:br|\/p|\/div|\/li|\/h[1-6]|\/tr|\/section)\s*>/gi, '\n').replace(/<[^>]+>/g, ' ')
173
+ return blocked.replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);/gi, (entity, body) => {
174
+ if (body[0] === '#') {
175
+ const hex = body[1] === 'x' || body[1] === 'X'
176
+ const code = Number.parseInt(hex ? body.slice(2) : body.slice(1), hex ? 16 : 10)
177
+ if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return entity
178
+ try { return String.fromCodePoint(code) } catch { return entity }
179
+ }
180
+ return HTML_ENTITIES[body.toLowerCase()] ?? entity
181
+ })
182
+ }
183
+
184
+ export function normalizeNewlines(text) {
185
+ return String(text ?? '').replace(/\r\n/g, '\n').replace(/\r/g, '\n')
186
+ }
187
+
188
+ /* -------------------------------------------------------------------------- */
189
+ /* bounded ZIP reading (docx/xlsx/pptx/ofd) */
190
+ /* -------------------------------------------------------------------------- */
191
+
192
+ function declaredUncompressedSize(entry) {
193
+ const size = entry?._data?.uncompressedSize
194
+ return typeof size === 'number' && Number.isFinite(size) && size >= 0 ? size : undefined
195
+ }
196
+
197
+ class KbZipReader {
198
+ constructor(zip) {
199
+ this.zip = zip
200
+ this.decompressedBytes = 0
201
+ }
202
+
203
+ names() {
204
+ return Object.keys(this.zip.files)
205
+ }
206
+
207
+ async readText(name) {
208
+ const entry = this.zip.files[name]
209
+ if (!entry || entry.dir) return undefined
210
+ const buffer = await this.readBuffer(name)
211
+ const text = buffer.toString('utf8')
212
+ if (text.length > MAX_XML_CHARS) {
213
+ throw kbError('E_KB_TOO_LARGE', 'archive text entry decodes to more than ' + MAX_XML_CHARS + ' characters')
214
+ }
215
+ return text
216
+ }
217
+
218
+ async readBuffer(name) {
219
+ const entry = this.zip.files[name]
220
+ if (!entry || entry.dir) throw kbError('E_KB_INVALID', 'archive entry is missing: ' + name)
221
+ const declared = declaredUncompressedSize(entry)
222
+ if (declared !== undefined && declared > MAX_ZIP_ENTRY_UNCOMPRESSED) {
223
+ throw kbError('E_KB_TOO_LARGE', 'archive entry declares more than ' + MAX_ZIP_ENTRY_UNCOMPRESSED + ' uncompressed bytes')
224
+ }
225
+ const stream = entry.nodeStream('nodebuffer')
226
+ const chunks = []
227
+ const self = this
228
+ let entryBytes = 0
229
+ await new Promise((resolvePromise, reject) => {
230
+ let settled = false
231
+ const fail = (error) => {
232
+ if (settled) return
233
+ settled = true
234
+ reject(error)
235
+ }
236
+ const stop = () => { if (typeof stream.destroy === 'function') stream.destroy() }
237
+ stream.on('data', (chunk) => {
238
+ if (settled) return
239
+ entryBytes += chunk.length
240
+ self.decompressedBytes += chunk.length
241
+ if (entryBytes > MAX_ZIP_ENTRY_UNCOMPRESSED) {
242
+ fail(kbError('E_KB_TOO_LARGE', 'archive entry expands beyond the single-entry budget'))
243
+ stop()
244
+ return
245
+ }
246
+ if (self.decompressedBytes > MAX_ZIP_TOTAL_UNCOMPRESSED) {
247
+ fail(kbError('E_KB_TOO_LARGE', 'archive expands beyond the unzip budget'))
248
+ stop()
249
+ return
250
+ }
251
+ chunks.push(chunk)
252
+ })
253
+ stream.on('end', () => {
254
+ if (settled) return
255
+ settled = true
256
+ resolvePromise()
257
+ })
258
+ stream.on('error', (error) => fail(error))
259
+ })
260
+ return Buffer.concat(chunks, entryBytes)
261
+ }
262
+ }
263
+
264
+ async function loadZip(buffer) {
265
+ let zip
266
+ try {
267
+ zip = await JSZip.loadAsync(buffer)
268
+ } catch (error) {
269
+ throw kbError('E_KB_INVALID', 'file is not a readable ZIP/OOXML archive (' + String(error?.message ?? error).slice(0, 120) + ')')
270
+ }
271
+ const names = Object.keys(zip.files)
272
+ if (names.length > MAX_ZIP_ENTRIES) {
273
+ throw kbError('E_KB_TOO_LARGE', 'archive has ' + names.length + ' entries (limit ' + MAX_ZIP_ENTRIES + ')')
274
+ }
275
+ let declaredTotal = 0
276
+ for (const name of names) {
277
+ const entry = zip.files[name]
278
+ if (!entry || entry.dir) continue
279
+ const declared = declaredUncompressedSize(entry) ?? 0
280
+ if (declared > MAX_ZIP_ENTRY_UNCOMPRESSED) throw kbError('E_KB_TOO_LARGE', 'archive entry declares more than the single-entry budget')
281
+ declaredTotal += declared
282
+ if (declaredTotal > MAX_ZIP_TOTAL_UNCOMPRESSED) throw kbError('E_KB_TOO_LARGE', 'archive declares more than the unzip budget')
283
+ }
284
+ return new KbZipReader(zip)
285
+ }
286
+
287
+ function decodeXml(text) {
288
+ return String(text ?? '')
289
+ .replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&amp;/g, '&')
290
+ }
291
+
292
+ function docxText(xml) {
293
+ return decodeXml(String(xml ?? '')
294
+ .replace(/<w:tab[^>]*\/>/g, '\t')
295
+ .replace(/<\/w:p>/g, '\n')
296
+ .replace(/<w:br[^>]*\/>/g, '\n')
297
+ .replace(/<[^>]+>/g, ''))
298
+ .replace(/\n{3,}/g, '\n\n').trim()
299
+ }
300
+
301
+ async function pptxText(reader) {
302
+ const slides = reader.names().filter((name) => /^ppt\/slides\/slide\d+\.xml$/.test(name))
303
+ .sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
304
+ const out = []
305
+ let index = 0
306
+ for (const name of slides) {
307
+ index += 1
308
+ const xml = await reader.readText(name)
309
+ if (!xml) continue
310
+ const texts = [...xml.matchAll(/<a:t>([\s\S]*?)<\/a:t>/g)].map((match) => decodeXml(match[1]))
311
+ if (texts.length) out.push('# slide ' + index + '\n' + texts.join('\n'))
312
+ }
313
+ return out.join('\n\n').trim()
314
+ }
315
+
316
+ async function xlsxText(reader) {
317
+ const out = []
318
+ const shared = await reader.readText('xl/sharedStrings.xml')
319
+ if (shared) out.push(...[...shared.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((match) => decodeXml(match[1])))
320
+ if (!out.length) {
321
+ const sheets = reader.names().filter((name) => /^xl\/worksheets\/sheet\d+\.xml$/.test(name)).sort()
322
+ for (const name of sheets) {
323
+ const xml = await reader.readText(name)
324
+ if (xml) out.push(...[...xml.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((match) => decodeXml(match[1])))
325
+ }
326
+ }
327
+ return out.filter(Boolean).join('\n').trim()
328
+ }
329
+
330
+ async function ofdText(reader) {
331
+ const names = reader.names()
332
+ .filter((name) => /(?:^|\/)Content\.xml$/i.test(name) || /(?:^|\/)Pages\/[^/]+\.xml$/i.test(name))
333
+ .sort()
334
+ const out = []
335
+ for (const name of names) {
336
+ const xml = await reader.readText(name)
337
+ if (!xml) continue
338
+ const texts = [...xml.matchAll(/<(?:ofd:)?TextCode[^>]*>([\s\S]*?)<\/(?:ofd:)?TextCode>/gi)].map((match) => decodeXml(match[1]))
339
+ if (texts.length) out.push(texts.join('\n'))
340
+ }
341
+ return out.join('\n\n').replace(/\n{3,}/g, '\n\n').trim()
342
+ }
343
+
344
+ /* -------------------------------------------------------------------------- */
345
+ /* PDF extraction (local pdfjs; explicit page cap) */
346
+ /* -------------------------------------------------------------------------- */
347
+
348
+ async function pdfText(file, { maxPdfPages = KB_MAX_PDF_PAGES } = {}) {
349
+ const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs')
350
+ const data = new Uint8Array(readFileSync(file))
351
+ const task = pdfjs.getDocument({ data, isEvalSupported: false, useSystemFonts: false, disableFontFace: true, verbosity: 0 })
352
+ const doc = await task.promise
353
+ try {
354
+ const totalPages = doc.numPages
355
+ const count = Math.min(totalPages, maxPdfPages)
356
+ const pages = []
357
+ for (let pageNo = 1; pageNo <= count; pageNo += 1) {
358
+ const page = await doc.getPage(pageNo)
359
+ const pageContent = await page.getTextContent()
360
+ pages.push(pageContent.items.map((item) => item.str ?? '').join(' '))
361
+ }
362
+ return { text: pages.join('\n').trim(), totalPages, pages: count, truncated: totalPages > count }
363
+ } finally {
364
+ await task.destroy()
365
+ }
366
+ }
367
+
368
+ /* -------------------------------------------------------------------------- */
369
+ /* EML extraction (reuses the read-only importers parser) */
370
+ /* -------------------------------------------------------------------------- */
371
+
372
+ function emlText(raw) {
373
+ const parsed = parseEml(raw)
374
+ const header = [
375
+ parsed.subject ? 'Subject: ' + parsed.subject : '',
376
+ parsed.from ? 'From: ' + parsed.from : '',
377
+ parsed.date ? 'Date: ' + parsed.date : '',
378
+ ].filter(Boolean).join('\n')
379
+ const body = parsed.text ?? ''
380
+ return { text: [header, body].filter(Boolean).join('\n\n').trim(), parsed }
381
+ }
382
+
383
+ /* -------------------------------------------------------------------------- */
384
+ /* chunking */
385
+ /* -------------------------------------------------------------------------- */
386
+
387
+ /** Slice by code points: codePointSlice('abc', 0, 2) === 'ab'. */
388
+ export function codePointSlice(text, startCp, endCp) {
389
+ const chars = Array.from(String(text ?? ''))
390
+ return chars.slice(startCp, endCp).join('')
391
+ }
392
+
393
+ /**
394
+ * Split extracted text into deterministic chunks.
395
+ *
396
+ * Rule (45 section 3.3): normalize CRLF, split on blank lines into paragraphs,
397
+ * pack paragraphs up to targetChars, hard-split a paragraph above maxChars at
398
+ * sentence boundaries (or a code-point boundary as the last resort), no
399
+ * overlap. maxChunks caps one document; the caller records truncated=true.
400
+ *
401
+ * @param {string} text
402
+ * @param {{targetChars?: number, maxChars?: number, maxChunks?: number}} [options]
403
+ * @returns {{chunks: Array<{seq: number, text: string, charCount: number}>, truncated: boolean, charCount: number}}
404
+ */
405
+ export function chunkText(text, { targetChars = 800, maxChars = 1500, maxChunks = 2000 } = {}) {
406
+ const normalized = normalizeNewlines(text).trim()
407
+ if (!normalized) return { chunks: [], truncated: false, charCount: 0 }
408
+ const target = Math.max(1, Math.floor(Number(targetChars) || 800))
409
+ const hardMax = Math.max(target, Math.floor(Number(maxChars) || 1500))
410
+ const cap = Math.max(1, Math.floor(Number(maxChunks) || 2000))
411
+
412
+ const pieces = []
413
+ const paragraphs = normalized.split(/\n{2,}/)
414
+ for (const rawParagraph of paragraphs) {
415
+ const paragraph = rawParagraph.trim()
416
+ if (!paragraph) continue
417
+ if (paragraph.length <= hardMax) {
418
+ pieces.push(paragraph)
419
+ continue
420
+ }
421
+ // Long paragraph: prefer sentence boundaries, then a hard code-point cut.
422
+ const sentences = paragraph.match(/[^\n。!?!?;;\.]+[\n。!?!?;;\.]*/gu) ?? [paragraph]
423
+ let current = ''
424
+ const flush = () => {
425
+ if (current.trim()) pieces.push(current.trim())
426
+ current = ''
427
+ }
428
+ for (const sentence of sentences) {
429
+ let rest = sentence
430
+ while (rest.length) {
431
+ const room = hardMax - current.length
432
+ if (rest.length <= room) {
433
+ current += rest
434
+ rest = ''
435
+ } else if (room > 0) {
436
+ const cut = codePointSlice(rest, 0, room)
437
+ current += cut
438
+ rest = rest.slice(cut.length)
439
+ flush()
440
+ } else {
441
+ flush()
442
+ }
443
+ }
444
+ }
445
+ flush()
446
+ }
447
+
448
+ const chunks = []
449
+ let current = ''
450
+ const flushChunk = () => {
451
+ const value = current.trim()
452
+ if (value) chunks.push(value)
453
+ current = ''
454
+ }
455
+ for (const piece of pieces) {
456
+ if (!current) {
457
+ current = piece
458
+ } else if (current.length + 2 + piece.length <= target) {
459
+ current += '\n\n' + piece
460
+ } else {
461
+ flushChunk()
462
+ current = piece
463
+ }
464
+ }
465
+ flushChunk()
466
+
467
+ const kept = []
468
+ let charCount = 0
469
+ for (const value of chunks) {
470
+ if (kept.length >= cap) break
471
+ const clean = value.trim()
472
+ if (!clean) continue
473
+ const codePoints = [...clean].length
474
+ kept.push({ seq: kept.length, text: clean, charCount: codePoints })
475
+ charCount += codePoints
476
+ }
477
+ return { chunks: kept, truncated: chunks.length > kept.length, charCount }
478
+ }
479
+
480
+ /* -------------------------------------------------------------------------- */
481
+ /* public entry point */
482
+ /* -------------------------------------------------------------------------- */
483
+
484
+ /**
485
+ * Extract indexable text from one local file.
486
+ *
487
+ * @param {string} file caller-resolved path (never stored)
488
+ * @param {{maxFileBytes?: number, maxTextBytes?: number, maxPdfPages?: number}} [options]
489
+ * @returns {Promise<object>} JSON-ready extraction result (never contains the path)
490
+ */
491
+ export async function extractForIndex(file, options = {}) {
492
+ if (typeof file !== 'string' || !file.trim()) throw kbError('E_KB_INVALID', 'file path is required')
493
+ const maxFileBytes = Number(options.maxFileBytes) > 0 ? Number(options.maxFileBytes) : KB_MAX_FILE_BYTES
494
+ const maxTextBytes = Number(options.maxTextBytes) > 0 ? Number(options.maxTextBytes) : KB_MAX_TEXT_BYTES
495
+ const maxPdfPages = Number(options.maxPdfPages) > 0 ? Number(options.maxPdfPages) : KB_MAX_PDF_PAGES
496
+ const name = basename(file)
497
+ const ext = extensionOf(file)
498
+ const type = kindOfExtension(ext)
499
+ if (!type) throw kbError('E_KB_UNSUPPORTED', 'unsupported file type: ' + (ext || '(none)'), 'index md/txt/docx/xlsx/pptx/ofd/pdf/eml files')
500
+
501
+ let info
502
+ try {
503
+ info = lstatSync(file)
504
+ } catch (error) {
505
+ throw kbError('E_KB_READ', 'cannot read file ' + name + ' (' + String(error?.code ?? 'ENOENT') + ')', 'check that the path exists and is readable')
506
+ }
507
+ if (info.isSymbolicLink()) throw kbError('E_KB_INVALID', 'symbolic links are not indexed: ' + name)
508
+ if (!info.isFile()) throw kbError('E_KB_INVALID', 'not a regular file: ' + name)
509
+ if (info.size > maxFileBytes) {
510
+ throw kbError('E_KB_TOO_LARGE', 'file is ' + info.size + ' bytes, above the ' + maxFileBytes + '-byte limit: ' + name)
511
+ }
512
+
513
+ const warnings = []
514
+ const truncatedBy = []
515
+ let text = ''
516
+ let extractor = (ext || 'text').replace(/^\./, '')
517
+ let encoding = null
518
+ let pages
519
+ let totalPages
520
+ let needsOcr = false
521
+ let truncated = false
522
+
523
+ if (type === 'image') {
524
+ // KB never spends CPU on OCR automatically: the user explicitly sets
525
+ // ADVISERS_OCR_CMD for the attachment path. A scanned KB file is recorded
526
+ // with needsOcr=1 and no chunks so the CLI can warn honestly.
527
+ needsOcr = true
528
+ extractor = 'image'
529
+ warnings.push('image files are not OCR-indexed by KB; set ADVISERS_OCR_CMD and index a text/PDF source instead')
530
+ } else if (type === 'text') {
531
+ const buffer = readFileSync(file)
532
+ const decoded = decodeTextBuffer(buffer)
533
+ encoding = decoded.encoding
534
+ extractor = 'plain'
535
+ text = ext === '.html' || ext === '.htm' ? htmlToIndexText(decoded.text) : decoded.text
536
+ } else if (type === 'office') {
537
+ const reader = await loadZip(readFileSync(file))
538
+ if (ext === '.docx') {
539
+ extractor = 'docx'
540
+ text = docxText(await reader.readText('word/document.xml') ?? '')
541
+ } else if (ext === '.pptx') {
542
+ extractor = 'pptx'
543
+ text = await pptxText(reader)
544
+ } else {
545
+ extractor = 'xlsx'
546
+ text = await xlsxText(reader)
547
+ }
548
+ if (!text) throw kbError('E_KB_EMPTY', 'no extractable text in ' + name)
549
+ } else if (type === 'ofd') {
550
+ extractor = 'ofd'
551
+ const reader = await loadZip(readFileSync(file))
552
+ text = await ofdText(reader)
553
+ if (!text) {
554
+ needsOcr = true
555
+ warnings.push('OFD has no text layer (image-only); not indexed')
556
+ }
557
+ } else if (type === 'pdf') {
558
+ extractor = 'pdf'
559
+ const result = await pdfText(file, { maxPdfPages })
560
+ text = result.text
561
+ pages = result.pages
562
+ totalPages = result.totalPages
563
+ if (result.truncated) {
564
+ truncated = true
565
+ truncatedBy.push('pdf_pages')
566
+ warnings.push('PDF has ' + result.totalPages + ' pages; only the first ' + result.pages + ' were indexed')
567
+ }
568
+ if (!text) {
569
+ needsOcr = true
570
+ warnings.push('PDF has no extractable text layer (scanned?); not indexed')
571
+ }
572
+ } else if (type === 'eml') {
573
+ extractor = 'eml'
574
+ const raw = readFileSync(file).toString('utf8')
575
+ const result = emlText(raw)
576
+ text = result.text
577
+ encoding = 'utf-8'
578
+ for (const warning of result.parsed?.warnings ?? []) warnings.push(String(warning).slice(0, 200))
579
+ if (!text) throw kbError('E_KB_EMPTY', 'no readable text in ' + name)
580
+ }
581
+
582
+ text = normalizeNewlines(text).trim()
583
+ if (!needsOcr && !text && type !== 'image') throw kbError('E_KB_EMPTY', 'no extractable text in ' + name)
584
+
585
+ const limited = truncateUtf8(text, maxTextBytes)
586
+ text = limited.text
587
+ if (limited.truncated) {
588
+ truncated = true
589
+ truncatedBy.push('text_bytes')
590
+ warnings.push('extracted text exceeded ' + maxTextBytes + ' bytes; only the prefix was indexed')
591
+ }
592
+
593
+ return {
594
+ text,
595
+ extractor,
596
+ encoding,
597
+ mime: mimeForFile(file),
598
+ sizeBytes: info.size,
599
+ sourceName: name,
600
+ ...(pages === undefined ? {} : { pages }),
601
+ ...(totalPages === undefined ? {} : { totalPages }),
602
+ truncated,
603
+ truncatedBy,
604
+ needsOcr,
605
+ warnings,
606
+ charCount: [...text].length,
607
+ textBytes: byteLength(text),
608
+ }
609
+ }
610
+
611
+ export default Object.freeze({
612
+ KB_MAX_FILE_BYTES,
613
+ KB_MAX_TEXT_BYTES,
614
+ KB_MAX_PDF_PAGES,
615
+ KB_EXTRACTOR_VERSION,
616
+ isIndexableFile,
617
+ mimeForFile,
618
+ extensionOf,
619
+ decodeTextBuffer,
620
+ truncateUtf8,
621
+ normalizeNewlines,
622
+ htmlToIndexText,
623
+ chunkText,
624
+ codePointSlice,
625
+ extractForIndex,
626
+ })