@xiaofanzhao/dsh-plugin-knowledge-base 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.mjs ADDED
@@ -0,0 +1,383 @@
1
+ // ============================================================================
2
+ // 检索:SQLite FTS5(关键词)+ 可选的向量混合排序(RRF 融合)。
3
+ //
4
+ // 中文的关键点:FTS5 的 unicode61 不切中文,trigram 又要求查询 ≥3 字。
5
+ // 所以入索引前把 CJK 字符逐个用空格隔开("知识库检索" → " 艾 津 农 药 "),
6
+ // 查询串做同样归一化后拼成短语("农 药"),1-2 字的中文查询也能命中。
7
+ // ============================================================================
8
+ import { createHash } from 'node:crypto';
9
+
10
+ const CJK_SOURCE = '\\u3400-\\u4dbf\\u4e00-\\u9fff\\uf900-\\ufaff';
11
+ const CJK = new RegExp(`[${CJK_SOURCE}]`);
12
+ const CJK_RUN = new RegExp(`[${CJK_SOURCE}]+`, 'g');
13
+ const CJK_CHAR = new RegExp(`[${CJK_SOURCE}]`, 'g');
14
+ const SPLIT_QUERY = /[\s,,、;;::!?!?()()[\]{}"'“”‘’]+/;
15
+
16
+ /**
17
+ * 单次 embeddings 请求最多几条文本。
18
+ * 阿里云百炼的 input 上限就是 10,一次全发会被 400 拒掉 —— 表现出来是
19
+ * "测试连接成功,但重建索引后向量还是 0"。统一按 10 发最稳,可用环境变量覆盖。
20
+ */
21
+ const EMBED_BATCH = Math.max(1, Number(process.env.DSH_KB_EMBED_BATCH) || 10);
22
+
23
+ export function normalizeCjk(text) {
24
+ return String(text ?? '')
25
+ .replace(CJK_CHAR, (char) => ` ${char} `)
26
+ .replace(/\s+/g, ' ')
27
+ .trim();
28
+ }
29
+
30
+ function escapeWord(word) {
31
+ return word.replace(/["*^:(){}[\]]/g, ' ').trim();
32
+ }
33
+
34
+ /** 把用户查询翻成 FTS5 表达式:中文按"字短语",英文按前缀词,彼此 AND。 */
35
+ export function toMatchQuery(query) {
36
+ const parts = [];
37
+ for (const raw of String(query ?? '').split(SPLIT_QUERY)) {
38
+ if (!raw) continue;
39
+ const runs = raw.match(new RegExp(`[${CJK_SOURCE}]+|[^${CJK_SOURCE}]+`, 'g')) ?? [];
40
+ for (const run of runs) {
41
+ if (CJK.test(run)) {
42
+ const chars = [...run].filter((char) => CJK.test(char));
43
+ if (chars.length) parts.push(`"${chars.join(' ')}"`);
44
+ continue;
45
+ }
46
+ for (const word of run.split(/\s+/)) {
47
+ const clean = escapeWord(word);
48
+ if (clean) parts.push(`${clean}*`);
49
+ }
50
+ }
51
+ }
52
+ return [...new Set(parts)].join(' AND ');
53
+ }
54
+
55
+ /** 从原文里截一段带查询词的窗口做摘要(FTS 的 snippet 是归一化文本,不能直接用)。 */
56
+ export function makeSnippet(text, query, width = 140) {
57
+ const source = String(text ?? '').replace(/\s+/g, ' ').trim();
58
+ if (!source) return '';
59
+ const lower = source.toLowerCase();
60
+ const words = String(query ?? '')
61
+ .split(SPLIT_QUERY)
62
+ .map((item) => item.trim().toLowerCase())
63
+ .filter(Boolean)
64
+ .sort((left, right) => right.length - left.length);
65
+ let at = -1;
66
+ for (const word of words) {
67
+ at = lower.indexOf(word);
68
+ if (at >= 0) break;
69
+ }
70
+ if (at < 0) return source.slice(0, width) + (source.length > width ? '…' : '');
71
+ const start = Math.max(0, at - Math.floor(width / 3));
72
+ const end = Math.min(source.length, start + width);
73
+ return `${start > 0 ? '…' : ''}${source.slice(start, end)}${end < source.length ? '…' : ''}`;
74
+ }
75
+
76
+ // ── 向量(可选) ──────────────────────────────────────────────────────────
77
+ function toBlob(vector) {
78
+ const array = Float32Array.from(vector);
79
+ return Buffer.from(array.buffer, array.byteOffset, array.byteLength);
80
+ }
81
+
82
+ function fromBlob(blob) {
83
+ const buffer = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
84
+ return new Float32Array(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength));
85
+ }
86
+
87
+ function cosine(left, right) {
88
+ const length = Math.min(left.length, right.length);
89
+ let dot = 0;
90
+ let leftNorm = 0;
91
+ let rightNorm = 0;
92
+ for (let index = 0; index < length; index += 1) {
93
+ dot += left[index] * right[index];
94
+ leftNorm += left[index] * left[index];
95
+ rightNorm += right[index] * right[index];
96
+ }
97
+ if (!leftNorm || !rightNorm) return 0;
98
+ return dot / Math.sqrt(leftNorm * rightNorm);
99
+ }
100
+
101
+ /**
102
+ * OpenAI 兼容的 embeddings 客户端;未配置或失败时安静降级为纯关键词。
103
+ *
104
+ * 配置来源有两种:静态的 `config`(插件 YAML),或 `resolve()` —— 后者每次调用都重新
105
+ * 读,好处是界面上改完即时生效,不用重载插件。传了 resolve 就用它。
106
+ */
107
+ export function createEmbedder({ config, resolve, logger, apiKey }) {
108
+ function effective() {
109
+ const raw = (typeof resolve === 'function' ? resolve() : config) || {};
110
+ const enabled = Boolean(raw.enabled);
111
+ const baseUrl = String(raw.baseUrl ?? '').replace(/\/+$/, '');
112
+ const model = String(raw.model ?? '');
113
+ const key = String(raw.apiKey || apiKey || process.env.DSH_KB_EMBEDDING_KEY || '');
114
+ return { enabled, baseUrl, model, key, usable: enabled && Boolean(baseUrl) && Boolean(model) };
115
+ }
116
+
117
+ return {
118
+ get enabled() {
119
+ return effective().usable;
120
+ },
121
+ describe() {
122
+ const now = effective();
123
+ return {
124
+ configured: now.enabled,
125
+ enabled: now.usable,
126
+ model: now.model,
127
+ baseUrl: now.baseUrl,
128
+ hasKey: Boolean(now.key),
129
+ missing: now.usable || !now.enabled ? [] : ['baseUrl', 'model'].filter((field) => !now[field]),
130
+ };
131
+ },
132
+ /** 拿当前生效值做一次真实请求,供界面「测试连接」用。 */
133
+ async probe(text = '知识库连接测试') {
134
+ const now = effective();
135
+ if (!now.usable) return { ok: false, error: '还没填完整(需要 baseUrl + model 且已启用)' };
136
+ try {
137
+ const response = await fetch(`${now.baseUrl}/embeddings`, {
138
+ method: 'POST',
139
+ headers: {
140
+ 'content-type': 'application/json',
141
+ ...(now.key ? { authorization: `Bearer ${now.key}` } : {}),
142
+ },
143
+ body: JSON.stringify({ model: now.model, input: [text] }),
144
+ });
145
+ if (!response.ok) {
146
+ const body = await response.text().catch(() => '');
147
+ return { ok: false, error: `HTTP ${response.status}${body ? `:${body.slice(0, 200)}` : ''}` };
148
+ }
149
+ const data = await response.json();
150
+ const vector = (data?.data ?? [])[0]?.embedding;
151
+ if (!Array.isArray(vector) || !vector.length) return { ok: false, error: '接口没有返回向量(格式不对?)' };
152
+ return { ok: true, dim: vector.length, model: now.model };
153
+ } catch (error) {
154
+ return { ok: false, error: error instanceof Error ? error.message : String(error) };
155
+ }
156
+ },
157
+ async embed(texts) {
158
+ const now = effective();
159
+ if (!now.usable || !texts.length) return null;
160
+ // 分批请求:百炼等服务的单次 input 上限只有 10 条,一次全发会被 400 拒掉,
161
+ // 表现就是"测试连接成功、但索引里一个向量都没有"。
162
+ const batches = [];
163
+ for (let index = 0; index < texts.length; index += EMBED_BATCH) {
164
+ batches.push(texts.slice(index, index + EMBED_BATCH));
165
+ }
166
+ const vectors = [];
167
+ for (const batch of batches) {
168
+ try {
169
+ const response = await fetch(`${now.baseUrl}/embeddings`, {
170
+ method: 'POST',
171
+ headers: {
172
+ 'content-type': 'application/json',
173
+ ...(now.key ? { authorization: `Bearer ${now.key}` } : {}),
174
+ },
175
+ body: JSON.stringify({ model: now.model, input: batch }),
176
+ });
177
+ if (!response.ok) {
178
+ const detail = await response.text().catch(() => '');
179
+ throw new Error(`HTTP ${response.status}${detail ? `:${detail.slice(0, 300)}` : ''}`);
180
+ }
181
+ const data = await response.json();
182
+ const items = (data?.data ?? []).filter((item) => Array.isArray(item?.embedding));
183
+ if (items.length !== batch.length) throw new Error(`返回 ${items.length} 条向量,期望 ${batch.length}`);
184
+ // 兼容没有按 index 排序的实现
185
+ const ordered = items.every((item, position) => (item.index ?? position) === position)
186
+ ? items
187
+ : [...items].sort((left, right) => (left.index ?? 0) - (right.index ?? 0));
188
+ vectors.push(...ordered.map((item) => item.embedding));
189
+ } catch (error) {
190
+ logger?.warn?.(`[knowledge-base] embedding 失败(本批 ${batch.length} 条),本次只写关键词索引:`
191
+ + `${error instanceof Error ? error.message : String(error)}`);
192
+ return null;
193
+ }
194
+ }
195
+ return vectors.length === texts.length ? vectors : null;
196
+ },
197
+ fingerprint() {
198
+ const now = effective();
199
+ return createHash('sha1').update(`${now.baseUrl}|${now.model}`).digest('hex').slice(0, 12);
200
+ },
201
+ };
202
+ }
203
+
204
+ export function createIndex({ store, embedder }) {
205
+ const { q } = store.raw;
206
+
207
+ function deleteChunks(nodeId) {
208
+ const rows = q(`SELECT id FROM chunks WHERE node_id = ?`).all(nodeId);
209
+ const delFts = q(`DELETE FROM chunks_fts WHERE rowid = ?`);
210
+ for (const row of rows) delFts.run(row.id);
211
+ q(`DELETE FROM chunks WHERE node_id = ?`).run(nodeId);
212
+ q(`DELETE FROM chunk_vec WHERE node_id = ?`).run(nodeId);
213
+ }
214
+
215
+ function insertChunks({ nodeId, kbId, chunks }) {
216
+ deleteChunks(nodeId);
217
+ const insert = q(`INSERT INTO chunks (node_id, kb_id, ord, heading, page, char_start, char_end, text)
218
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
219
+ const insertFts = q(`INSERT INTO chunks_fts (rowid, cjk, kb_id, node_id) VALUES (?, ?, ?, ?)`);
220
+ for (const chunk of chunks) {
221
+ const result = insert.run(nodeId, kbId, chunk.ord, chunk.heading ?? '', chunk.page ?? null,
222
+ chunk.charStart, chunk.charEnd, chunk.text);
223
+ insertFts.run(Number(result.lastInsertRowid), normalizeCjk(`${chunk.heading ?? ''} ${chunk.text}`), kbId, nodeId);
224
+ }
225
+ return chunks.length;
226
+ }
227
+
228
+ async function insertVectors({ nodeId, chunks }) {
229
+ q(`DELETE FROM chunk_vec WHERE node_id = ?`).run(nodeId);
230
+ if (!embedder?.enabled || !chunks.length) return 0;
231
+ const vectors = await embedder.embed(chunks.map((chunk) => `${chunk.heading ?? ''}\n${chunk.text}`.slice(0, 4000)));
232
+ if (!vectors) return 0;
233
+ const insert = q(`INSERT INTO chunk_vec (node_id, ord, dim, vec) VALUES (?, ?, ?, ?)`);
234
+ vectors.forEach((vector, index) => insert.run(nodeId, chunks[index].ord, vector.length, toBlob(vector)));
235
+ return vectors.length;
236
+ }
237
+
238
+ function keywordRows({ match, kbIds, limit }) {
239
+ const params = [match];
240
+ let sql = `SELECT rowid AS chunkId, node_id AS nodeId, bm25(chunks_fts) AS rank
241
+ FROM chunks_fts WHERE chunks_fts MATCH ?`;
242
+ if (kbIds.length) {
243
+ sql += ` AND kb_id IN (${kbIds.map(() => '?').join(',')})`;
244
+ params.push(...kbIds);
245
+ }
246
+ sql += ' ORDER BY rank LIMIT ?';
247
+ params.push(limit);
248
+ try {
249
+ return q(sql).all(...params);
250
+ } catch {
251
+ return [];
252
+ }
253
+ }
254
+
255
+ function vectorRows(kbIds) {
256
+ let sql = `SELECT v.node_id AS nodeId, c.id AS chunkId, v.vec AS vec
257
+ FROM chunk_vec v JOIN chunks c ON c.node_id = v.node_id AND c.ord = v.ord`;
258
+ const params = [];
259
+ if (kbIds.length) {
260
+ sql += ` WHERE c.kb_id IN (${kbIds.map(() => '?').join(',')})`;
261
+ params.push(...kbIds);
262
+ }
263
+ sql += ' LIMIT 20000';
264
+ return q(sql).all(...params);
265
+ }
266
+
267
+ function hydrate(entries, query) {
268
+ if (!entries.length) return [];
269
+ const ids = entries.map((entry) => Number(entry.chunkId));
270
+ const chunkRows = q(`SELECT id, node_id AS nodeId, kb_id AS kbId, ord, heading, page,
271
+ char_start AS charStart, char_end AS charEnd, text FROM chunks
272
+ WHERE id IN (${ids.map(() => '?').join(',')})`).all(...ids);
273
+ const byId = new Map(chunkRows.map((row) => [Number(row.id), row]));
274
+ const nodeIds = [...new Set(chunkRows.map((row) => row.nodeId))];
275
+ const nodes = store.mapNodeByIds(nodeIds);
276
+ const out = [];
277
+ for (const entry of entries) {
278
+ const chunk = byId.get(Number(entry.chunkId));
279
+ if (!chunk) continue;
280
+ const node = nodes.get(chunk.nodeId);
281
+ out.push({
282
+ chunkId: chunk.id,
283
+ nodeId: chunk.nodeId,
284
+ kbId: chunk.kbId,
285
+ ord: chunk.ord,
286
+ heading: chunk.heading ?? '',
287
+ page: chunk.page ?? null,
288
+ charStart: chunk.charStart,
289
+ charEnd: chunk.charEnd,
290
+ text: chunk.text,
291
+ name: node?.name ?? '(已删除)',
292
+ ext: node?.ext ?? '',
293
+ breadcrumb: node ? store.breadcrumb(node.id) : (node?.name ?? ''),
294
+ locator: `知识库 → ${node ? store.breadcrumb(node.id) : '未知'}`,
295
+ snippet: makeSnippet(chunk.text, query),
296
+ score: entry.rrf,
297
+ ftsRank: entry.ftsRank ?? null,
298
+ vecRank: entry.vecRank ?? null,
299
+ });
300
+ }
301
+ return out;
302
+ }
303
+
304
+ async function search({ query, kbIds = [], limit = 10, mode = 'keyword' }) {
305
+ const startedAt = Date.now();
306
+ const text = String(query ?? '').trim();
307
+ const took = { keyword: 0, vector: 0 };
308
+ if (!text) return { hits: [], mode: 'keyword', total: 0, truncated: false };
309
+ const ids = Array.isArray(kbIds) ? kbIds.filter(Boolean) : [];
310
+ const match = toMatchQuery(text);
311
+
312
+ const entries = new Map();
313
+ const ftsRows = match ? keywordRows({ match, kbIds: ids, limit: Math.max(limit * 5, 50) }) : [];
314
+ ftsRows.forEach((row, index) => {
315
+ entries.set(Number(row.chunkId), { chunkId: row.chunkId, nodeId: row.nodeId, ftsRank: index, rrf: 0 });
316
+ });
317
+ took.keyword = ftsRows.length;
318
+
319
+ let usedMode = 'keyword';
320
+ if (mode === 'hybrid' && embedder?.enabled) {
321
+ const queryVector = (await embedder.embed([text]))?.[0];
322
+ if (queryVector) {
323
+ const rows = vectorRows(ids);
324
+ const scored = rows
325
+ .map((row) => ({ row, value: cosine(queryVector, fromBlob(row.vec)) }))
326
+ .filter((item) => item.value > 0.2)
327
+ .sort((left, right) => right.value - left.value)
328
+ .slice(0, Math.max(limit * 5, 50));
329
+ scored.forEach((item, index) => {
330
+ const chunkId = Number(item.row.chunkId);
331
+ const existing = entries.get(chunkId);
332
+ if (existing) existing.vecRank = index;
333
+ else entries.set(chunkId, { chunkId, nodeId: item.row.nodeId, vecRank: index, rrf: 0 });
334
+ });
335
+ took.vector = scored.length;
336
+ usedMode = 'hybrid';
337
+ }
338
+ }
339
+
340
+ for (const entry of entries.values()) {
341
+ entry.rrf = (entry.ftsRank === undefined ? 0 : 1 / (60 + entry.ftsRank))
342
+ + (entry.vecRank === undefined ? 0 : 1 / (60 + entry.vecRank));
343
+ }
344
+ const ranked = [...entries.values()].sort((left, right) => right.rrf - left.rrf).slice(0, limit);
345
+ const hits = hydrate(ranked, text);
346
+
347
+ // 标题命中加权:文件名里出现查询词,通常比正文偶然命中更相关
348
+ const words = text.toLowerCase().split(/\s+/).filter(Boolean);
349
+ for (const hit of hits) {
350
+ const name = hit.name.toLowerCase();
351
+ if (words.some((word) => name.includes(word.toLowerCase()))) hit.score += 0.4;
352
+ }
353
+ hits.sort((left, right) => right.score - left.score);
354
+
355
+ // 看板埋点:记下这次检索(失败不影响返回)
356
+ store.logSearch?.({
357
+ now: startedAt,
358
+ query: text,
359
+ mode: usedMode,
360
+ hits: hits.length,
361
+ tookMs: Date.now() - startedAt,
362
+ kbId: ids[0] ?? null,
363
+ });
364
+
365
+ return { hits, mode: usedMode, total: hits.length, truncated: hits.length >= limit, took };
366
+ }
367
+
368
+ /** 给"重建索引"用:把所有文档重排一遍。 */
369
+ function resetAll() {
370
+ q(`DELETE FROM chunks_fts`).run();
371
+ q(`DELETE FROM chunks`).run();
372
+ q(`DELETE FROM chunk_vec`).run();
373
+ }
374
+
375
+ return {
376
+ insertChunks,
377
+ insertVectors,
378
+ deleteChunks,
379
+ search,
380
+ resetAll,
381
+ normalizeCjk,
382
+ };
383
+ }
package/lib/pdf.mjs ADDED
@@ -0,0 +1,88 @@
1
+ // ============================================================================
2
+ // PDF 正文抽取:pdfjs-dist(legacy build,纯 JS,Node 里只做文字层,不渲染)
3
+ //
4
+ // 依赖是可选的:装不上 or 加载失败时抛一个能给人看的原因,流水线会把文档标成
5
+ // 「解析失败」,但不影响插件其余部分。
6
+ // ============================================================================
7
+ let pdfjsPromise = null;
8
+
9
+ function loadPdfjs() {
10
+ if (!pdfjsPromise) {
11
+ pdfjsPromise = import('pdfjs-dist/legacy/build/pdf.mjs').catch((error) => {
12
+ pdfjsPromise = null;
13
+ const reason = error instanceof Error ? error.message : String(error);
14
+ throw new Error(`PDF 解析器不可用(pdfjs-dist 加载失败:${reason})`);
15
+ });
16
+ }
17
+ return pdfjsPromise;
18
+ }
19
+
20
+ /** 把一页的 text item 按物理行拼起来。 */
21
+ function pageLines(items) {
22
+ const lines = [];
23
+ let current = '';
24
+ let lastY = null;
25
+ for (const item of items) {
26
+ if (typeof item.str !== 'string') continue;
27
+ const y = Array.isArray(item.transform) ? Math.round(item.transform[5]) : null;
28
+ if (lastY !== null && y !== null && Math.abs(y - lastY) > 3 && current.trim()) {
29
+ lines.push(current.replace(/\s+/g, ' ').trim());
30
+ current = '';
31
+ }
32
+ current += item.str;
33
+ if (item.hasEOL) {
34
+ if (current.trim()) lines.push(current.replace(/\s+/g, ' ').trim());
35
+ current = '';
36
+ }
37
+ lastY = y;
38
+ }
39
+ if (current.trim()) lines.push(current.replace(/\s+/g, ' ').trim());
40
+ return lines.filter(Boolean);
41
+ }
42
+
43
+ export async function extractPdf({ buffer }) {
44
+ const pdfjs = await loadPdfjs();
45
+ const data = new Uint8Array(buffer);
46
+ let doc;
47
+ try {
48
+ doc = await pdfjs.getDocument({
49
+ data,
50
+ useSystemFonts: false,
51
+ disableFontFace: true,
52
+ isEvalSupported: false,
53
+ useWorkerFetch: false,
54
+ verbosity: 0,
55
+ }).promise;
56
+ } catch (error) {
57
+ const reason = error instanceof Error ? error.message : String(error);
58
+ if (/password|encrypt/i.test(reason)) throw new Error('PDF 已加密,无法抽取正文');
59
+ throw new Error(`PDF 打不开:${reason}`);
60
+ }
61
+
62
+ const blocks = [];
63
+ let textPages = 0;
64
+ try {
65
+ for (let pageNumber = 1; pageNumber <= doc.numPages; pageNumber += 1) {
66
+ const page = await doc.getPage(pageNumber);
67
+ let content;
68
+ try {
69
+ content = await page.getTextContent();
70
+ } finally {
71
+ // 页面对象用完即弃,避免大文档累积内存
72
+ page.cleanup?.();
73
+ }
74
+ const lines = pageLines(content.items ?? []);
75
+ if (!lines.length) continue;
76
+ textPages += 1;
77
+ blocks.push({ kind: 'page', page: pageNumber, text: `第 ${pageNumber} 页` });
78
+ for (const line of lines) blocks.push({ kind: 'text', page: pageNumber, text: line });
79
+ }
80
+ } finally {
81
+ try { await doc.destroy(); } catch { /* ignore */ }
82
+ }
83
+
84
+ if (!blocks.length || textPages === 0) {
85
+ throw new Error('这个 PDF 没有可抽取的文字层(可能是扫描件)');
86
+ }
87
+ return { blocks, pageCount: doc.numPages };
88
+ }
@@ -0,0 +1,164 @@
1
+ // ============================================================================
2
+ // 解析流水线:抽取 → 渲染正文 → 分块 → 写索引。
3
+ //
4
+ // 单飞队列(可配并发 1-2):上传请求立刻返回,解析在后台跑,前端轮询状态。
5
+ // 失败一定留下人能看懂的原因(扫描件没有文字层 / 加密 / 格式不支持 …)。
6
+ // ============================================================================
7
+ import fs from 'node:fs';
8
+ import { extract } from './extract.mjs';
9
+ import { renderDocument, chunkDocument } from './chunker.mjs';
10
+ import { describeImageFile, isVisionExtension } from './vision.mjs';
11
+
12
+ function friendly(error) {
13
+ const message = error instanceof Error ? error.message : String(error ?? '');
14
+ if (!message) return '解析失败';
15
+ if (/没有可抽取的文字层/.test(message)) return '这个文件没有可抽取的文字层(可能是扫描件或纯图片)';
16
+ if (/已加密/.test(message)) return '文件已加密,无法抽取正文';
17
+ if (/暂不支持/.test(message)) return message;
18
+ if (/过大/.test(message)) return message;
19
+ if (/ENOENT/.test(message)) return '文件不见了(可能被外部删除)';
20
+ if (/图片理解未启用/.test(message)) return '图片理解未启用:设置里打开「图片理解」后重建索引即可';
21
+ if (/HTTP 4|HTTP 5/.test(message)) return `图片理解接口报错:${message.slice(0, 200)}`;
22
+ return `解析失败:${message.slice(0, 300)}`;
23
+ }
24
+
25
+ export function createPipeline({ store, index, embedder, vision, logger, config, previewer }) {
26
+ const queue = [];
27
+ const queued = new Set();
28
+ let active = 0;
29
+ const concurrency = Math.min(2, Math.max(1, Number(config?.parseConcurrency) || 1));
30
+ const maxFileBytes = Number(config?.maxFileBytes) || 64 * 1024 * 1024;
31
+
32
+ function kick() {
33
+ while (active < concurrency && queue.length) {
34
+ const nodeId = queue.shift();
35
+ active += 1;
36
+ ingest(nodeId)
37
+ .catch((error) => logger?.warn?.(`[knowledge-base] ingest ${nodeId} crashed: ${error}`))
38
+ .finally(() => {
39
+ active -= 1;
40
+ queued.delete(nodeId);
41
+ kick();
42
+ });
43
+ }
44
+ }
45
+
46
+ function enqueue(nodeId) {
47
+ if (!nodeId || queued.has(nodeId)) return;
48
+ queued.add(nodeId);
49
+ queue.push(nodeId);
50
+ kick();
51
+ }
52
+
53
+ async function ingest(nodeId) {
54
+ const node = store.getNode(nodeId);
55
+ if (!node || node.kind !== 'doc') return;
56
+ const filePath = store.raw.q(`SELECT file_path AS filePath FROM nodes WHERE id = ?`).get(nodeId)?.filePath;
57
+ if (!filePath) {
58
+ store.setDocStatus(nodeId, 'failed', { error: '缺少原始文件路径', updatedAt: Date.now() });
59
+ return;
60
+ }
61
+ if (isVisionExtension(node.ext)) {
62
+ await ingestImage(node, filePath);
63
+ return;
64
+ }
65
+ store.setDocStatus(nodeId, 'parsing', { error: null, updatedAt: Date.now() });
66
+ try {
67
+ const stat = await fs.promises.stat(filePath);
68
+ if (stat.size > maxFileBytes) throw new Error(`文件过大(上限 ${Math.round(maxFileBytes / 1048576)} MB)`);
69
+ const buffer = await fs.promises.readFile(filePath);
70
+ const { blocks, pageCount } = await extract({ ext: node.ext, buffer, filePath });
71
+ const { text, lines } = renderDocument(blocks ?? []);
72
+ const chunks = chunkDocument(lines);
73
+
74
+ store.saveText(nodeId, text, Date.now());
75
+ index.insertChunks({ nodeId, kbId: node.kbId, chunks });
76
+ const vectorCount = await index.insertVectors({ nodeId, chunks });
77
+ store.setDocStatus(nodeId, 'ready', {
78
+ error: null,
79
+ pageCount: pageCount ?? 0,
80
+ chars: text.length,
81
+ indexedAt: Date.now(),
82
+ updatedAt: Date.now(),
83
+ });
84
+ logger?.info?.(`[knowledge-base] indexed ${node.name}: ${chunks.length} chunks, ${text.length} chars`
85
+ + `${vectorCount ? `, ${vectorCount} vectors` : ''}`);
86
+ // Office 文档顺手把原版式预览也生成好(背景优先级,别抢占前台请求)
87
+ if (previewer?.supports?.(node.ext) && previewer.available?.()) {
88
+ previewer.ensure(node, { priority: 'background' })
89
+ .catch((error) => logger?.warn?.(`[knowledge-base] preview prewarm failed for ${node.name}: ${error?.message || error}`));
90
+ }
91
+ } catch (error) {
92
+ logger?.warn?.(`[knowledge-base] parse failed for ${node.name}: ${error instanceof Error ? error.message : String(error)}`);
93
+ store.setDocStatus(nodeId, 'failed', { error: friendly(error), updatedAt: Date.now() });
94
+ }
95
+ }
96
+
97
+ /** 图片:交给视觉模型描述成文字,再走和文档一样的索引流程。 */
98
+ async function ingestImage(node, filePath) {
99
+ const nodeId = node.id;
100
+ if (!vision?.enabled) {
101
+ // 没开图片理解:保持「可看」,但不假装索引过
102
+ store.setDocStatus(nodeId, 'ready', {
103
+ error: null, pageCount: 1, chars: 0, indexedAt: Date.now(), updatedAt: Date.now(),
104
+ });
105
+ return;
106
+ }
107
+ store.setDocStatus(nodeId, 'parsing', { error: null, updatedAt: Date.now() });
108
+ try {
109
+ const stat = await fs.promises.stat(filePath);
110
+ if (stat.size > maxFileBytes) throw new Error(`文件过大(上限 ${Math.round(maxFileBytes / 1048576)} MB)`);
111
+ const description = await describeImageFile(vision, filePath, node.ext);
112
+ const blocks = [
113
+ { kind: 'heading', level: 1, text: node.name },
114
+ ...description.split('\n').map((line) => line.trim()).filter(Boolean)
115
+ .map((text) => ({ kind: 'paragraph', text })),
116
+ ];
117
+ const { text, lines } = renderDocument(blocks);
118
+ const chunks = chunkDocument(lines);
119
+
120
+ store.saveText(nodeId, text, Date.now());
121
+ index.insertChunks({ nodeId, kbId: node.kbId, chunks });
122
+ const vectorCount = await index.insertVectors({ nodeId, chunks });
123
+ store.setDocStatus(nodeId, 'ready', {
124
+ error: null,
125
+ pageCount: 1,
126
+ chars: text.length,
127
+ indexedAt: Date.now(),
128
+ updatedAt: Date.now(),
129
+ });
130
+ logger?.info?.(`[knowledge-base] described image ${node.name}: ${text.length} chars`
131
+ + `${vectorCount ? `, ${vectorCount} vectors` : ''}`);
132
+ } catch (error) {
133
+ logger?.warn?.(`[knowledge-base] image describe failed for ${node.name}: ${error instanceof Error ? error.message : String(error)}`);
134
+ store.setDocStatus(nodeId, 'failed', { error: friendly(error), updatedAt: Date.now() });
135
+ }
136
+ }
137
+
138
+ /** 启动时把上次没跑完的任务捡回来。 */
139
+ function recover() {
140
+ const rows = store.raw.q(`SELECT id FROM nodes WHERE kind = 'doc' AND status IN ('pending', 'parsing')`).all();
141
+ for (const row of rows) {
142
+ store.setDocStatus(row.id, 'pending', { error: null, updatedAt: Date.now() });
143
+ enqueue(row.id);
144
+ }
145
+ return rows.length;
146
+ }
147
+
148
+ function reindexAll() {
149
+ const rows = store.raw.q(`SELECT id FROM nodes WHERE kind = 'doc'`).all();
150
+ index.resetAll();
151
+ for (const row of rows) {
152
+ store.setDocStatus(row.id, 'pending', { error: null, updatedAt: Date.now() });
153
+ enqueue(row.id);
154
+ }
155
+ return rows.length;
156
+ }
157
+
158
+ /** 队列现状,给界面显示进度用。 */
159
+ function pendingInfo() {
160
+ return { queued: queue.length, active, concurrency };
161
+ }
162
+
163
+ return { enqueue, ingest, recover, reindexAll, concurrency, pendingInfo };
164
+ }