advisers 0.3.2 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh.md +13 -14
- package/README.md +37 -23
- package/THIRD-PARTY-NOTICES.md +1 -1
- package/bin/advisers.js +2302 -95
- package/index.js +1 -1
- package/lib/brief-archive.js +267 -0
- package/lib/client-pack.js +1016 -0
- package/lib/consent.js +390 -0
- package/lib/credentials.js +195 -0
- package/lib/error-codes.js +516 -7
- package/lib/evidence.js +697 -113
- package/lib/kb-extract.js +626 -0
- package/lib/kb-facts.js +429 -0
- package/lib/kb-pages.js +1084 -0
- package/lib/kb.js +1036 -0
- package/lib/mail-errors.js +46 -0
- package/lib/mail-extract.js +351 -0
- package/lib/mail-imap.js +446 -0
- package/lib/mail-import.js +176 -0
- package/lib/mail-parse.js +223 -0
- package/lib/mail-store.js +731 -0
- package/lib/mail.js +506 -0
- package/lib/mcp-server.js +196 -10
- package/lib/notify.js +265 -7
- package/lib/policy/decision.js +315 -0
- package/lib/policy/egress.js +268 -0
- package/lib/policy/errors.js +71 -0
- package/lib/policy/index.js +99 -0
- package/lib/policy/rules.js +468 -0
- package/lib/policy/surfaces.js +141 -0
- package/lib/receipt-verify.js +199 -19
- package/lib/receivables.js +1653 -0
- package/lib/scheduler.js +453 -9
- package/lib/shell.js +1406 -0
- package/lib/source.js +98 -3
- package/lib/sqlite.js +366 -49
- package/lib/tos/index.js +2 -2
- package/lib/tos/manifest.json +6 -6
- package/lib/tos/privacy.zh-CN.md +33 -20
- package/lib/tos/safety.zh-CN.md +7 -6
- package/lib/tos/summary.zh-CN.md +3 -2
- package/lib/tos/terms.zh-CN.md +31 -16
- package/lib/urgency.js +467 -0
- package/package.json +4 -4
|
@@ -0,0 +1,626 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* KB text extraction (0.4.0, schema v22).
|
|
3
|
+
*
|
|
4
|
+
* One bounded, local-only entry point for turning a user file into the text the
|
|
5
|
+
* KB indexes. This is deliberately NOT extractAttachment(): attachments are a
|
|
6
|
+
* model-context budget (10 MB file, 20k chars, 50 PDF pages, silent PDF page
|
|
7
|
+
* cap) while the KB is a durable local index (50 MB file, 2 MB text, 500 PDF
|
|
8
|
+
* pages, explicit truncation). The two callers keep their own limits and the KB
|
|
9
|
+
* module must never silently drop half a document.
|
|
10
|
+
*
|
|
11
|
+
* Guarantees:
|
|
12
|
+
* - read-only: the source file is opened for reading only and is never copied;
|
|
13
|
+
* the returned object never contains an absolute path, only the basename;
|
|
14
|
+
* - no network, no OCR fallback, no model: a scanned/image file is reported as
|
|
15
|
+
* needsOcr=true with empty text (the caller records a warning), it is not
|
|
16
|
+
* silently treated as empty content;
|
|
17
|
+
* - deterministic: the same bytes + options produce the same extraction result;
|
|
18
|
+
* - all limits are explicit: file bytes, extracted text bytes, PDF pages, ZIP
|
|
19
|
+
* expansion budget and per-entry XML size. A limit hit sets truncated=true and
|
|
20
|
+
* names the reason instead of silently discarding data;
|
|
21
|
+
* - text encoding: BOM (UTF-8/UTF-16LE/UTF-16BE) -> strict UTF-8 -> GB18030
|
|
22
|
+
* fallback (the common Chinese-Windows legacy encoding). Undecodable bytes
|
|
23
|
+
* throw E_KB_ENCODING rather than storing mojibake.
|
|
24
|
+
*
|
|
25
|
+
* Pure module boundaries: no store, no HOME handling, no console output. The
|
|
26
|
+
* only environment interaction is the imported read-only EML parser.
|
|
27
|
+
*/
|
|
28
|
+
import { lstatSync, readFileSync } from 'node:fs'
|
|
29
|
+
import { basename, extname } from 'node:path'
|
|
30
|
+
import JSZip from 'jszip'
|
|
31
|
+
import { parseEml } from './importers.js'
|
|
32
|
+
|
|
33
|
+
/** Hard per-file cap for KB ingestion (larger than the 10 MB attachment cap). */
|
|
34
|
+
export const KB_MAX_FILE_BYTES = 50 * 1024 * 1024
|
|
35
|
+
/** Hard cap for the extracted text stored in kb_chunks (UTF-8 bytes). */
|
|
36
|
+
export const KB_MAX_TEXT_BYTES = 2 * 1024 * 1024
|
|
37
|
+
/** Hard cap for indexed PDF pages (the attachment path only reads 50). */
|
|
38
|
+
export const KB_MAX_PDF_PAGES = 500
|
|
39
|
+
/** Extraction pipeline version recorded next to a document. */
|
|
40
|
+
export const KB_EXTRACTOR_VERSION = 1
|
|
41
|
+
|
|
42
|
+
const MAX_ZIP_ENTRIES = 4000
|
|
43
|
+
const MAX_ZIP_TOTAL_UNCOMPRESSED = 256 * 1024 * 1024
|
|
44
|
+
const MAX_ZIP_ENTRY_UNCOMPRESSED = 64 * 1024 * 1024
|
|
45
|
+
const MAX_XML_CHARS = 16 * 1024 * 1024
|
|
46
|
+
|
|
47
|
+
const TEXT_EXTENSIONS = Object.freeze([
|
|
48
|
+
'.txt', '.md', '.markdown', '.csv', '.json', '.jsonl', '.log', '.yaml', '.yml', '.xml', '.html', '.htm',
|
|
49
|
+
])
|
|
50
|
+
const OFFICE_EXTENSIONS = Object.freeze(['.docx', '.xlsx', '.pptx'])
|
|
51
|
+
const IMAGE_EXTENSIONS = Object.freeze(['.png', '.jpg', '.jpeg', '.webp', '.bmp', '.tif', '.tiff'])
|
|
52
|
+
const MIME_BY_EXT = Object.freeze({
|
|
53
|
+
'.txt': 'text/plain', '.md': 'text/markdown', '.markdown': 'text/markdown',
|
|
54
|
+
'.csv': 'text/csv', '.json': 'application/json', '.jsonl': 'application/jsonl',
|
|
55
|
+
'.log': 'text/plain', '.yaml': 'application/yaml', '.yml': 'application/yaml',
|
|
56
|
+
'.xml': 'application/xml', '.html': 'text/html', '.htm': 'text/html',
|
|
57
|
+
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
58
|
+
'.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
59
|
+
'.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
60
|
+
'.pdf': 'application/pdf', '.ofd': 'application/ofd', '.eml': 'message/rfc822',
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
// KB errors carry stable codes; the message keeps only the basename so a log or
|
|
64
|
+
// an audit reason can never leak the operator's directory layout.
|
|
65
|
+
function kbError(code, message, hint) {
|
|
66
|
+
const error = new Error(code + ': ' + message + (hint ? ' (hint: ' + hint + ')' : ''))
|
|
67
|
+
error.code = code
|
|
68
|
+
if (hint) error.hint = hint
|
|
69
|
+
return error
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function byteLength(text) {
|
|
73
|
+
return Buffer.byteLength(text, 'utf8')
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Truncate a JS string to at most maxBytes UTF-8 bytes without splitting a code point. */
|
|
77
|
+
export function truncateUtf8(text, maxBytes) {
|
|
78
|
+
const raw = String(text ?? '')
|
|
79
|
+
if (byteLength(raw) <= maxBytes) return { text: raw, truncated: false }
|
|
80
|
+
const buffer = Buffer.from(raw, 'utf8').subarray(0, maxBytes)
|
|
81
|
+
let truncated = buffer.toString('utf8')
|
|
82
|
+
// Buffer.subarray can cut a multi-byte sequence; drop the replacement char(s).
|
|
83
|
+
while (truncated.endsWith('\uFFFD')) truncated = truncated.slice(0, -1)
|
|
84
|
+
return { text: truncated, truncated: true }
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function extensionOf(file) {
|
|
88
|
+
return extname(String(file ?? '')).toLowerCase()
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function kindOfExtension(ext) {
|
|
92
|
+
if (TEXT_EXTENSIONS.includes(ext)) return 'text'
|
|
93
|
+
if (OFFICE_EXTENSIONS.includes(ext)) return 'office'
|
|
94
|
+
if (ext === '.pdf') return 'pdf'
|
|
95
|
+
if (ext === '.ofd') return 'ofd'
|
|
96
|
+
if (ext === '.eml') return 'eml'
|
|
97
|
+
if (IMAGE_EXTENSIONS.includes(ext)) return 'image'
|
|
98
|
+
return null
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** True when the file type can be indexed (used by callers before hashing). */
|
|
102
|
+
export function isIndexableFile(file) {
|
|
103
|
+
return kindOfExtension(extensionOf(file)) !== null
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function mimeForFile(file) {
|
|
107
|
+
return MIME_BY_EXT[extensionOf(file)] ?? 'application/octet-stream'
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/* -------------------------------------------------------------------------- */
|
|
111
|
+
/* text decoding (BOM -> strict UTF-8 -> GB18030) */
|
|
112
|
+
/* -------------------------------------------------------------------------- */
|
|
113
|
+
|
|
114
|
+
function decodeWith(label, buffer) {
|
|
115
|
+
return new TextDecoder(label, { fatal: true }).decode(buffer)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Decode a plain-text file buffer.
|
|
120
|
+
*
|
|
121
|
+
* Order: a BOM (UTF-8/UTF-16LE/UTF-16BE) wins; otherwise strict UTF-8; then
|
|
122
|
+
* GB18030 (the Windows Chinese legacy encoding, a superset of GBK/GB2312).
|
|
123
|
+
* Throws E_KB_ENCODING when even GB18030 yields U+FFFD (never store mojibake).
|
|
124
|
+
*
|
|
125
|
+
* @param {Buffer} buffer
|
|
126
|
+
* @returns {{text: string, encoding: string}}
|
|
127
|
+
*/
|
|
128
|
+
export function decodeTextBuffer(buffer) {
|
|
129
|
+
const bytes = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer ?? [])
|
|
130
|
+
let text
|
|
131
|
+
if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
|
|
132
|
+
text = decodeWith('utf-8', bytes.subarray(3))
|
|
133
|
+
return { text, encoding: 'utf-8' }
|
|
134
|
+
}
|
|
135
|
+
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
|
|
136
|
+
text = decodeWith('utf-16le', bytes.subarray(2))
|
|
137
|
+
return { text, encoding: 'utf-16le' }
|
|
138
|
+
}
|
|
139
|
+
if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
|
|
140
|
+
text = decodeWith('utf-16be', bytes.subarray(2))
|
|
141
|
+
return { text, encoding: 'utf-16be' }
|
|
142
|
+
}
|
|
143
|
+
try {
|
|
144
|
+
return { text: decodeWith('utf-8', bytes), encoding: 'utf-8' }
|
|
145
|
+
} catch {
|
|
146
|
+
// Not UTF-8: fall through to the legacy Chinese encoding.
|
|
147
|
+
}
|
|
148
|
+
let legacy
|
|
149
|
+
try {
|
|
150
|
+
legacy = new TextDecoder('gb18030', { fatal: false }).decode(bytes)
|
|
151
|
+
} catch (error) {
|
|
152
|
+
throw kbError('E_KB_ENCODING', 'file bytes are neither valid UTF-8 nor decodable as GB18030 (' + String(error?.message ?? error).slice(0, 120) + ')')
|
|
153
|
+
}
|
|
154
|
+
if (legacy.includes('\uFFFD')) {
|
|
155
|
+
throw kbError('E_KB_ENCODING', 'file bytes are neither valid UTF-8 nor GB18030; convert the file to UTF-8 first')
|
|
156
|
+
}
|
|
157
|
+
return { text: legacy, encoding: 'gb18030' }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/* -------------------------------------------------------------------------- */
|
|
161
|
+
/* HTML to text (minimal, tag-stripping only; never executes anything) */
|
|
162
|
+
/* -------------------------------------------------------------------------- */
|
|
163
|
+
|
|
164
|
+
const HTML_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' }
|
|
165
|
+
|
|
166
|
+
export function htmlToIndexText(html) {
|
|
167
|
+
const source = String(html ?? '')
|
|
168
|
+
const withoutHidden = source
|
|
169
|
+
.replace(/<script\b[\s\S]*?<\/script\s*>/gi, ' ')
|
|
170
|
+
.replace(/<style\b[\s\S]*?<\/style\s*>/gi, ' ')
|
|
171
|
+
.replace(/<!--[\s\S]*?-->/g, ' ')
|
|
172
|
+
const blocked = withoutHidden.replace(/<(?:br|\/p|\/div|\/li|\/h[1-6]|\/tr|\/section)\s*>/gi, '\n').replace(/<[^>]+>/g, ' ')
|
|
173
|
+
return blocked.replace(/&(#x[0-9a-f]+|#\d+|[a-z][a-z0-9]*);/gi, (entity, body) => {
|
|
174
|
+
if (body[0] === '#') {
|
|
175
|
+
const hex = body[1] === 'x' || body[1] === 'X'
|
|
176
|
+
const code = Number.parseInt(hex ? body.slice(2) : body.slice(1), hex ? 16 : 10)
|
|
177
|
+
if (!Number.isFinite(code) || code <= 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return entity
|
|
178
|
+
try { return String.fromCodePoint(code) } catch { return entity }
|
|
179
|
+
}
|
|
180
|
+
return HTML_ENTITIES[body.toLowerCase()] ?? entity
|
|
181
|
+
})
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export function normalizeNewlines(text) {
|
|
185
|
+
return String(text ?? '').replace(/\r\n/g, '\n').replace(/\r/g, '\n')
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/* -------------------------------------------------------------------------- */
|
|
189
|
+
/* bounded ZIP reading (docx/xlsx/pptx/ofd) */
|
|
190
|
+
/* -------------------------------------------------------------------------- */
|
|
191
|
+
|
|
192
|
+
function declaredUncompressedSize(entry) {
|
|
193
|
+
const size = entry?._data?.uncompressedSize
|
|
194
|
+
return typeof size === 'number' && Number.isFinite(size) && size >= 0 ? size : undefined
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
class KbZipReader {
|
|
198
|
+
constructor(zip) {
|
|
199
|
+
this.zip = zip
|
|
200
|
+
this.decompressedBytes = 0
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
names() {
|
|
204
|
+
return Object.keys(this.zip.files)
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async readText(name) {
|
|
208
|
+
const entry = this.zip.files[name]
|
|
209
|
+
if (!entry || entry.dir) return undefined
|
|
210
|
+
const buffer = await this.readBuffer(name)
|
|
211
|
+
const text = buffer.toString('utf8')
|
|
212
|
+
if (text.length > MAX_XML_CHARS) {
|
|
213
|
+
throw kbError('E_KB_TOO_LARGE', 'archive text entry decodes to more than ' + MAX_XML_CHARS + ' characters')
|
|
214
|
+
}
|
|
215
|
+
return text
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async readBuffer(name) {
|
|
219
|
+
const entry = this.zip.files[name]
|
|
220
|
+
if (!entry || entry.dir) throw kbError('E_KB_INVALID', 'archive entry is missing: ' + name)
|
|
221
|
+
const declared = declaredUncompressedSize(entry)
|
|
222
|
+
if (declared !== undefined && declared > MAX_ZIP_ENTRY_UNCOMPRESSED) {
|
|
223
|
+
throw kbError('E_KB_TOO_LARGE', 'archive entry declares more than ' + MAX_ZIP_ENTRY_UNCOMPRESSED + ' uncompressed bytes')
|
|
224
|
+
}
|
|
225
|
+
const stream = entry.nodeStream('nodebuffer')
|
|
226
|
+
const chunks = []
|
|
227
|
+
const self = this
|
|
228
|
+
let entryBytes = 0
|
|
229
|
+
await new Promise((resolvePromise, reject) => {
|
|
230
|
+
let settled = false
|
|
231
|
+
const fail = (error) => {
|
|
232
|
+
if (settled) return
|
|
233
|
+
settled = true
|
|
234
|
+
reject(error)
|
|
235
|
+
}
|
|
236
|
+
const stop = () => { if (typeof stream.destroy === 'function') stream.destroy() }
|
|
237
|
+
stream.on('data', (chunk) => {
|
|
238
|
+
if (settled) return
|
|
239
|
+
entryBytes += chunk.length
|
|
240
|
+
self.decompressedBytes += chunk.length
|
|
241
|
+
if (entryBytes > MAX_ZIP_ENTRY_UNCOMPRESSED) {
|
|
242
|
+
fail(kbError('E_KB_TOO_LARGE', 'archive entry expands beyond the single-entry budget'))
|
|
243
|
+
stop()
|
|
244
|
+
return
|
|
245
|
+
}
|
|
246
|
+
if (self.decompressedBytes > MAX_ZIP_TOTAL_UNCOMPRESSED) {
|
|
247
|
+
fail(kbError('E_KB_TOO_LARGE', 'archive expands beyond the unzip budget'))
|
|
248
|
+
stop()
|
|
249
|
+
return
|
|
250
|
+
}
|
|
251
|
+
chunks.push(chunk)
|
|
252
|
+
})
|
|
253
|
+
stream.on('end', () => {
|
|
254
|
+
if (settled) return
|
|
255
|
+
settled = true
|
|
256
|
+
resolvePromise()
|
|
257
|
+
})
|
|
258
|
+
stream.on('error', (error) => fail(error))
|
|
259
|
+
})
|
|
260
|
+
return Buffer.concat(chunks, entryBytes)
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async function loadZip(buffer) {
|
|
265
|
+
let zip
|
|
266
|
+
try {
|
|
267
|
+
zip = await JSZip.loadAsync(buffer)
|
|
268
|
+
} catch (error) {
|
|
269
|
+
throw kbError('E_KB_INVALID', 'file is not a readable ZIP/OOXML archive (' + String(error?.message ?? error).slice(0, 120) + ')')
|
|
270
|
+
}
|
|
271
|
+
const names = Object.keys(zip.files)
|
|
272
|
+
if (names.length > MAX_ZIP_ENTRIES) {
|
|
273
|
+
throw kbError('E_KB_TOO_LARGE', 'archive has ' + names.length + ' entries (limit ' + MAX_ZIP_ENTRIES + ')')
|
|
274
|
+
}
|
|
275
|
+
let declaredTotal = 0
|
|
276
|
+
for (const name of names) {
|
|
277
|
+
const entry = zip.files[name]
|
|
278
|
+
if (!entry || entry.dir) continue
|
|
279
|
+
const declared = declaredUncompressedSize(entry) ?? 0
|
|
280
|
+
if (declared > MAX_ZIP_ENTRY_UNCOMPRESSED) throw kbError('E_KB_TOO_LARGE', 'archive entry declares more than the single-entry budget')
|
|
281
|
+
declaredTotal += declared
|
|
282
|
+
if (declaredTotal > MAX_ZIP_TOTAL_UNCOMPRESSED) throw kbError('E_KB_TOO_LARGE', 'archive declares more than the unzip budget')
|
|
283
|
+
}
|
|
284
|
+
return new KbZipReader(zip)
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function decodeXml(text) {
|
|
288
|
+
return String(text ?? '')
|
|
289
|
+
.replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, '&')
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function docxText(xml) {
|
|
293
|
+
return decodeXml(String(xml ?? '')
|
|
294
|
+
.replace(/<w:tab[^>]*\/>/g, '\t')
|
|
295
|
+
.replace(/<\/w:p>/g, '\n')
|
|
296
|
+
.replace(/<w:br[^>]*\/>/g, '\n')
|
|
297
|
+
.replace(/<[^>]+>/g, ''))
|
|
298
|
+
.replace(/\n{3,}/g, '\n\n').trim()
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
async function pptxText(reader) {
|
|
302
|
+
const slides = reader.names().filter((name) => /^ppt\/slides\/slide\d+\.xml$/.test(name))
|
|
303
|
+
.sort((a, b) => a.localeCompare(b, undefined, { numeric: true }))
|
|
304
|
+
const out = []
|
|
305
|
+
let index = 0
|
|
306
|
+
for (const name of slides) {
|
|
307
|
+
index += 1
|
|
308
|
+
const xml = await reader.readText(name)
|
|
309
|
+
if (!xml) continue
|
|
310
|
+
const texts = [...xml.matchAll(/<a:t>([\s\S]*?)<\/a:t>/g)].map((match) => decodeXml(match[1]))
|
|
311
|
+
if (texts.length) out.push('# slide ' + index + '\n' + texts.join('\n'))
|
|
312
|
+
}
|
|
313
|
+
return out.join('\n\n').trim()
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
async function xlsxText(reader) {
|
|
317
|
+
const out = []
|
|
318
|
+
const shared = await reader.readText('xl/sharedStrings.xml')
|
|
319
|
+
if (shared) out.push(...[...shared.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((match) => decodeXml(match[1])))
|
|
320
|
+
if (!out.length) {
|
|
321
|
+
const sheets = reader.names().filter((name) => /^xl\/worksheets\/sheet\d+\.xml$/.test(name)).sort()
|
|
322
|
+
for (const name of sheets) {
|
|
323
|
+
const xml = await reader.readText(name)
|
|
324
|
+
if (xml) out.push(...[...xml.matchAll(/<t[^>]*>([\s\S]*?)<\/t>/g)].map((match) => decodeXml(match[1])))
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
return out.filter(Boolean).join('\n').trim()
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
async function ofdText(reader) {
|
|
331
|
+
const names = reader.names()
|
|
332
|
+
.filter((name) => /(?:^|\/)Content\.xml$/i.test(name) || /(?:^|\/)Pages\/[^/]+\.xml$/i.test(name))
|
|
333
|
+
.sort()
|
|
334
|
+
const out = []
|
|
335
|
+
for (const name of names) {
|
|
336
|
+
const xml = await reader.readText(name)
|
|
337
|
+
if (!xml) continue
|
|
338
|
+
const texts = [...xml.matchAll(/<(?:ofd:)?TextCode[^>]*>([\s\S]*?)<\/(?:ofd:)?TextCode>/gi)].map((match) => decodeXml(match[1]))
|
|
339
|
+
if (texts.length) out.push(texts.join('\n'))
|
|
340
|
+
}
|
|
341
|
+
return out.join('\n\n').replace(/\n{3,}/g, '\n\n').trim()
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/* -------------------------------------------------------------------------- */
|
|
345
|
+
/* PDF extraction (local pdfjs; explicit page cap) */
|
|
346
|
+
/* -------------------------------------------------------------------------- */
|
|
347
|
+
|
|
348
|
+
async function pdfText(file, { maxPdfPages = KB_MAX_PDF_PAGES } = {}) {
|
|
349
|
+
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs')
|
|
350
|
+
const data = new Uint8Array(readFileSync(file))
|
|
351
|
+
const task = pdfjs.getDocument({ data, isEvalSupported: false, useSystemFonts: false, disableFontFace: true, verbosity: 0 })
|
|
352
|
+
const doc = await task.promise
|
|
353
|
+
try {
|
|
354
|
+
const totalPages = doc.numPages
|
|
355
|
+
const count = Math.min(totalPages, maxPdfPages)
|
|
356
|
+
const pages = []
|
|
357
|
+
for (let pageNo = 1; pageNo <= count; pageNo += 1) {
|
|
358
|
+
const page = await doc.getPage(pageNo)
|
|
359
|
+
const pageContent = await page.getTextContent()
|
|
360
|
+
pages.push(pageContent.items.map((item) => item.str ?? '').join(' '))
|
|
361
|
+
}
|
|
362
|
+
return { text: pages.join('\n').trim(), totalPages, pages: count, truncated: totalPages > count }
|
|
363
|
+
} finally {
|
|
364
|
+
await task.destroy()
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/* -------------------------------------------------------------------------- */
|
|
369
|
+
/* EML extraction (reuses the read-only importers parser) */
|
|
370
|
+
/* -------------------------------------------------------------------------- */
|
|
371
|
+
|
|
372
|
+
function emlText(raw) {
|
|
373
|
+
const parsed = parseEml(raw)
|
|
374
|
+
const header = [
|
|
375
|
+
parsed.subject ? 'Subject: ' + parsed.subject : '',
|
|
376
|
+
parsed.from ? 'From: ' + parsed.from : '',
|
|
377
|
+
parsed.date ? 'Date: ' + parsed.date : '',
|
|
378
|
+
].filter(Boolean).join('\n')
|
|
379
|
+
const body = parsed.text ?? ''
|
|
380
|
+
return { text: [header, body].filter(Boolean).join('\n\n').trim(), parsed }
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/* -------------------------------------------------------------------------- */
|
|
384
|
+
/* chunking */
|
|
385
|
+
/* -------------------------------------------------------------------------- */
|
|
386
|
+
|
|
387
|
+
/** Slice by code points: codePointSlice('abc', 0, 2) === 'ab'. */
|
|
388
|
+
export function codePointSlice(text, startCp, endCp) {
|
|
389
|
+
const chars = Array.from(String(text ?? ''))
|
|
390
|
+
return chars.slice(startCp, endCp).join('')
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* Split extracted text into deterministic chunks.
|
|
395
|
+
*
|
|
396
|
+
* Rule (45 section 3.3): normalize CRLF, split on blank lines into paragraphs,
|
|
397
|
+
* pack paragraphs up to targetChars, hard-split a paragraph above maxChars at
|
|
398
|
+
* sentence boundaries (or a code-point boundary as the last resort), no
|
|
399
|
+
* overlap. maxChunks caps one document; the caller records truncated=true.
|
|
400
|
+
*
|
|
401
|
+
* @param {string} text
|
|
402
|
+
* @param {{targetChars?: number, maxChars?: number, maxChunks?: number}} [options]
|
|
403
|
+
* @returns {{chunks: Array<{seq: number, text: string, charCount: number}>, truncated: boolean, charCount: number}}
|
|
404
|
+
*/
|
|
405
|
+
export function chunkText(text, { targetChars = 800, maxChars = 1500, maxChunks = 2000 } = {}) {
|
|
406
|
+
const normalized = normalizeNewlines(text).trim()
|
|
407
|
+
if (!normalized) return { chunks: [], truncated: false, charCount: 0 }
|
|
408
|
+
const target = Math.max(1, Math.floor(Number(targetChars) || 800))
|
|
409
|
+
const hardMax = Math.max(target, Math.floor(Number(maxChars) || 1500))
|
|
410
|
+
const cap = Math.max(1, Math.floor(Number(maxChunks) || 2000))
|
|
411
|
+
|
|
412
|
+
const pieces = []
|
|
413
|
+
const paragraphs = normalized.split(/\n{2,}/)
|
|
414
|
+
for (const rawParagraph of paragraphs) {
|
|
415
|
+
const paragraph = rawParagraph.trim()
|
|
416
|
+
if (!paragraph) continue
|
|
417
|
+
if (paragraph.length <= hardMax) {
|
|
418
|
+
pieces.push(paragraph)
|
|
419
|
+
continue
|
|
420
|
+
}
|
|
421
|
+
// Long paragraph: prefer sentence boundaries, then a hard code-point cut.
|
|
422
|
+
const sentences = paragraph.match(/[^\n。!?!?;;\.]+[\n。!?!?;;\.]*/gu) ?? [paragraph]
|
|
423
|
+
let current = ''
|
|
424
|
+
const flush = () => {
|
|
425
|
+
if (current.trim()) pieces.push(current.trim())
|
|
426
|
+
current = ''
|
|
427
|
+
}
|
|
428
|
+
for (const sentence of sentences) {
|
|
429
|
+
let rest = sentence
|
|
430
|
+
while (rest.length) {
|
|
431
|
+
const room = hardMax - current.length
|
|
432
|
+
if (rest.length <= room) {
|
|
433
|
+
current += rest
|
|
434
|
+
rest = ''
|
|
435
|
+
} else if (room > 0) {
|
|
436
|
+
const cut = codePointSlice(rest, 0, room)
|
|
437
|
+
current += cut
|
|
438
|
+
rest = rest.slice(cut.length)
|
|
439
|
+
flush()
|
|
440
|
+
} else {
|
|
441
|
+
flush()
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
flush()
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
const chunks = []
|
|
449
|
+
let current = ''
|
|
450
|
+
const flushChunk = () => {
|
|
451
|
+
const value = current.trim()
|
|
452
|
+
if (value) chunks.push(value)
|
|
453
|
+
current = ''
|
|
454
|
+
}
|
|
455
|
+
for (const piece of pieces) {
|
|
456
|
+
if (!current) {
|
|
457
|
+
current = piece
|
|
458
|
+
} else if (current.length + 2 + piece.length <= target) {
|
|
459
|
+
current += '\n\n' + piece
|
|
460
|
+
} else {
|
|
461
|
+
flushChunk()
|
|
462
|
+
current = piece
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
flushChunk()
|
|
466
|
+
|
|
467
|
+
const kept = []
|
|
468
|
+
let charCount = 0
|
|
469
|
+
for (const value of chunks) {
|
|
470
|
+
if (kept.length >= cap) break
|
|
471
|
+
const clean = value.trim()
|
|
472
|
+
if (!clean) continue
|
|
473
|
+
const codePoints = [...clean].length
|
|
474
|
+
kept.push({ seq: kept.length, text: clean, charCount: codePoints })
|
|
475
|
+
charCount += codePoints
|
|
476
|
+
}
|
|
477
|
+
return { chunks: kept, truncated: chunks.length > kept.length, charCount }
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/* -------------------------------------------------------------------------- */
|
|
481
|
+
/* public entry point */
|
|
482
|
+
/* -------------------------------------------------------------------------- */
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* Extract indexable text from one local file.
|
|
486
|
+
*
|
|
487
|
+
* @param {string} file caller-resolved path (never stored)
|
|
488
|
+
* @param {{maxFileBytes?: number, maxTextBytes?: number, maxPdfPages?: number}} [options]
|
|
489
|
+
* @returns {Promise<object>} JSON-ready extraction result (never contains the path)
|
|
490
|
+
*/
|
|
491
|
+
export async function extractForIndex(file, options = {}) {
|
|
492
|
+
if (typeof file !== 'string' || !file.trim()) throw kbError('E_KB_INVALID', 'file path is required')
|
|
493
|
+
const maxFileBytes = Number(options.maxFileBytes) > 0 ? Number(options.maxFileBytes) : KB_MAX_FILE_BYTES
|
|
494
|
+
const maxTextBytes = Number(options.maxTextBytes) > 0 ? Number(options.maxTextBytes) : KB_MAX_TEXT_BYTES
|
|
495
|
+
const maxPdfPages = Number(options.maxPdfPages) > 0 ? Number(options.maxPdfPages) : KB_MAX_PDF_PAGES
|
|
496
|
+
const name = basename(file)
|
|
497
|
+
const ext = extensionOf(file)
|
|
498
|
+
const type = kindOfExtension(ext)
|
|
499
|
+
if (!type) throw kbError('E_KB_UNSUPPORTED', 'unsupported file type: ' + (ext || '(none)'), 'index md/txt/docx/xlsx/pptx/ofd/pdf/eml files')
|
|
500
|
+
|
|
501
|
+
let info
|
|
502
|
+
try {
|
|
503
|
+
info = lstatSync(file)
|
|
504
|
+
} catch (error) {
|
|
505
|
+
throw kbError('E_KB_READ', 'cannot read file ' + name + ' (' + String(error?.code ?? 'ENOENT') + ')', 'check that the path exists and is readable')
|
|
506
|
+
}
|
|
507
|
+
if (info.isSymbolicLink()) throw kbError('E_KB_INVALID', 'symbolic links are not indexed: ' + name)
|
|
508
|
+
if (!info.isFile()) throw kbError('E_KB_INVALID', 'not a regular file: ' + name)
|
|
509
|
+
if (info.size > maxFileBytes) {
|
|
510
|
+
throw kbError('E_KB_TOO_LARGE', 'file is ' + info.size + ' bytes, above the ' + maxFileBytes + '-byte limit: ' + name)
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const warnings = []
|
|
514
|
+
const truncatedBy = []
|
|
515
|
+
let text = ''
|
|
516
|
+
let extractor = (ext || 'text').replace(/^\./, '')
|
|
517
|
+
let encoding = null
|
|
518
|
+
let pages
|
|
519
|
+
let totalPages
|
|
520
|
+
let needsOcr = false
|
|
521
|
+
let truncated = false
|
|
522
|
+
|
|
523
|
+
if (type === 'image') {
|
|
524
|
+
// KB never spends CPU on OCR automatically: the user explicitly sets
|
|
525
|
+
// ADVISERS_OCR_CMD for the attachment path. A scanned KB file is recorded
|
|
526
|
+
// with needsOcr=1 and no chunks so the CLI can warn honestly.
|
|
527
|
+
needsOcr = true
|
|
528
|
+
extractor = 'image'
|
|
529
|
+
warnings.push('image files are not OCR-indexed by KB; set ADVISERS_OCR_CMD and index a text/PDF source instead')
|
|
530
|
+
} else if (type === 'text') {
|
|
531
|
+
const buffer = readFileSync(file)
|
|
532
|
+
const decoded = decodeTextBuffer(buffer)
|
|
533
|
+
encoding = decoded.encoding
|
|
534
|
+
extractor = 'plain'
|
|
535
|
+
text = ext === '.html' || ext === '.htm' ? htmlToIndexText(decoded.text) : decoded.text
|
|
536
|
+
} else if (type === 'office') {
|
|
537
|
+
const reader = await loadZip(readFileSync(file))
|
|
538
|
+
if (ext === '.docx') {
|
|
539
|
+
extractor = 'docx'
|
|
540
|
+
text = docxText(await reader.readText('word/document.xml') ?? '')
|
|
541
|
+
} else if (ext === '.pptx') {
|
|
542
|
+
extractor = 'pptx'
|
|
543
|
+
text = await pptxText(reader)
|
|
544
|
+
} else {
|
|
545
|
+
extractor = 'xlsx'
|
|
546
|
+
text = await xlsxText(reader)
|
|
547
|
+
}
|
|
548
|
+
if (!text) throw kbError('E_KB_EMPTY', 'no extractable text in ' + name)
|
|
549
|
+
} else if (type === 'ofd') {
|
|
550
|
+
extractor = 'ofd'
|
|
551
|
+
const reader = await loadZip(readFileSync(file))
|
|
552
|
+
text = await ofdText(reader)
|
|
553
|
+
if (!text) {
|
|
554
|
+
needsOcr = true
|
|
555
|
+
warnings.push('OFD has no text layer (image-only); not indexed')
|
|
556
|
+
}
|
|
557
|
+
} else if (type === 'pdf') {
|
|
558
|
+
extractor = 'pdf'
|
|
559
|
+
const result = await pdfText(file, { maxPdfPages })
|
|
560
|
+
text = result.text
|
|
561
|
+
pages = result.pages
|
|
562
|
+
totalPages = result.totalPages
|
|
563
|
+
if (result.truncated) {
|
|
564
|
+
truncated = true
|
|
565
|
+
truncatedBy.push('pdf_pages')
|
|
566
|
+
warnings.push('PDF has ' + result.totalPages + ' pages; only the first ' + result.pages + ' were indexed')
|
|
567
|
+
}
|
|
568
|
+
if (!text) {
|
|
569
|
+
needsOcr = true
|
|
570
|
+
warnings.push('PDF has no extractable text layer (scanned?); not indexed')
|
|
571
|
+
}
|
|
572
|
+
} else if (type === 'eml') {
|
|
573
|
+
extractor = 'eml'
|
|
574
|
+
const raw = readFileSync(file).toString('utf8')
|
|
575
|
+
const result = emlText(raw)
|
|
576
|
+
text = result.text
|
|
577
|
+
encoding = 'utf-8'
|
|
578
|
+
for (const warning of result.parsed?.warnings ?? []) warnings.push(String(warning).slice(0, 200))
|
|
579
|
+
if (!text) throw kbError('E_KB_EMPTY', 'no readable text in ' + name)
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
text = normalizeNewlines(text).trim()
|
|
583
|
+
if (!needsOcr && !text && type !== 'image') throw kbError('E_KB_EMPTY', 'no extractable text in ' + name)
|
|
584
|
+
|
|
585
|
+
const limited = truncateUtf8(text, maxTextBytes)
|
|
586
|
+
text = limited.text
|
|
587
|
+
if (limited.truncated) {
|
|
588
|
+
truncated = true
|
|
589
|
+
truncatedBy.push('text_bytes')
|
|
590
|
+
warnings.push('extracted text exceeded ' + maxTextBytes + ' bytes; only the prefix was indexed')
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
return {
|
|
594
|
+
text,
|
|
595
|
+
extractor,
|
|
596
|
+
encoding,
|
|
597
|
+
mime: mimeForFile(file),
|
|
598
|
+
sizeBytes: info.size,
|
|
599
|
+
sourceName: name,
|
|
600
|
+
...(pages === undefined ? {} : { pages }),
|
|
601
|
+
...(totalPages === undefined ? {} : { totalPages }),
|
|
602
|
+
truncated,
|
|
603
|
+
truncatedBy,
|
|
604
|
+
needsOcr,
|
|
605
|
+
warnings,
|
|
606
|
+
charCount: [...text].length,
|
|
607
|
+
textBytes: byteLength(text),
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
export default Object.freeze({
|
|
612
|
+
KB_MAX_FILE_BYTES,
|
|
613
|
+
KB_MAX_TEXT_BYTES,
|
|
614
|
+
KB_MAX_PDF_PAGES,
|
|
615
|
+
KB_EXTRACTOR_VERSION,
|
|
616
|
+
isIndexableFile,
|
|
617
|
+
mimeForFile,
|
|
618
|
+
extensionOf,
|
|
619
|
+
decodeTextBuffer,
|
|
620
|
+
truncateUtf8,
|
|
621
|
+
normalizeNewlines,
|
|
622
|
+
htmlToIndexText,
|
|
623
|
+
chunkText,
|
|
624
|
+
codePointSlice,
|
|
625
|
+
extractForIndex,
|
|
626
|
+
})
|