@xiaofanzhao/dsh-plugin-knowledge-base 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +329 -2
- package/client.js +1934 -0
- package/cordis.patch.yml +20 -0
- package/docs/DESIGN.md +662 -0
- package/docs/DETAIL.md +73 -0
- package/docs/images/architecture.svg +111 -0
- package/docs/images/ui-bigscreen.svg +59 -0
- package/docs/images/ui-overview.svg +152 -0
- package/icon.svg +13 -0
- package/index.js +146 -0
- package/lib/chunker.mjs +121 -0
- package/lib/extract.mjs +434 -0
- package/lib/index.mjs +383 -0
- package/lib/pdf.mjs +88 -0
- package/lib/pipeline.mjs +164 -0
- package/lib/preview.mjs +217 -0
- package/lib/routes.mjs +592 -0
- package/lib/settings.mjs +148 -0
- package/lib/store.mjs +445 -0
- package/lib/tools.mjs +246 -0
- package/lib/util.mjs +129 -0
- package/lib/vision.mjs +140 -0
- package/locale/en.json +33 -0
- package/locale/zh.json +33 -0
- package/package.json +63 -4
package/lib/routes.mjs
ADDED
|
@@ -0,0 +1,592 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Host HTTP 路由(浏览器端只用这一条通道)
|
|
3
|
+
//
|
|
4
|
+
// 约定:
|
|
5
|
+
// · 所有接口都要求自定义头 x-kb-client: 1(跨站简单请求过不了预检,防 CSRF)
|
|
6
|
+
// · 带 Origin 的请求必须同源
|
|
7
|
+
// · 落盘文件名永远是 <docId>.<ext>,用户文件名只存数据库
|
|
8
|
+
// ============================================================================
|
|
9
|
+
import fs from 'node:fs';
|
|
10
|
+
import path from 'node:path';
|
|
11
|
+
import { createWriteStream } from 'node:fs';
|
|
12
|
+
import { pipeline as streamPipeline } from 'node:stream/promises';
|
|
13
|
+
import { createHash } from 'node:crypto';
|
|
14
|
+
import {
|
|
15
|
+
API_BASE, extOf, filenameFromHeader, guard, newId, nowMs, readBody, sanitizeName,
|
|
16
|
+
sendBytes, sendJson, sendText, clampInt,
|
|
17
|
+
} from './util.mjs';
|
|
18
|
+
import { MIME_BY_EXT, SUPPORTED_EXTENSIONS, isIndexable } from './extract.mjs';
|
|
19
|
+
import { IMAGE_EXTENSIONS } from './preview.mjs';
|
|
20
|
+
import { isVisionExtension, createVisionClient } from './vision.mjs';
|
|
21
|
+
import { createEmbedder } from './index.mjs';
|
|
22
|
+
|
|
23
|
+
function fail(res, status, error) {
|
|
24
|
+
return sendJson(res, status, { ok: false, error });
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function safeOrigin(webServer) {
|
|
28
|
+
try {
|
|
29
|
+
return webServer?.getOrigin?.() ?? '';
|
|
30
|
+
} catch {
|
|
31
|
+
return '';
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function createRoutes({ store, index, pipeline, embedder, vision, settings, config, logger, webServer, token, previewer }) {
|
|
36
|
+
const origin = safeOrigin(webServer);
|
|
37
|
+
const maxFileBytes = Number(config?.maxFileBytes) || 64 * 1024 * 1024;
|
|
38
|
+
|
|
39
|
+
const capabilities = () => ({
|
|
40
|
+
embedding: embedder?.describe?.() ?? { configured: false, enabled: false, model: '' },
|
|
41
|
+
vision: vision?.describe?.() ?? { configured: false, enabled: false, model: '' },
|
|
42
|
+
officePreview: previewer?.available?.() ?? false,
|
|
43
|
+
maxFileBytes,
|
|
44
|
+
formats: [...SUPPORTED_EXTENSIONS],
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
/** 把一个文件直接写给浏览器。 */
|
|
48
|
+
function serveFile(res, filePath, type, { download = false, name = '', cache = 'private, max-age=60' } = {}) {
|
|
49
|
+
let stat;
|
|
50
|
+
try {
|
|
51
|
+
stat = fs.statSync(filePath);
|
|
52
|
+
} catch {
|
|
53
|
+
return fail(res, 404, '文件不见了');
|
|
54
|
+
}
|
|
55
|
+
const headers = {
|
|
56
|
+
'content-type': type,
|
|
57
|
+
'content-length': stat.size,
|
|
58
|
+
'cache-control': cache,
|
|
59
|
+
'content-disposition': download
|
|
60
|
+
? `attachment; filename*=UTF-8''${encodeURIComponent(name || 'file')}`
|
|
61
|
+
: 'inline',
|
|
62
|
+
};
|
|
63
|
+
res.writeHead(200, headers);
|
|
64
|
+
fs.createReadStream(filePath).pipe(res);
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** 全库解析进度(重建索引时界面靠它显示进度条)。 */
|
|
69
|
+
function progressOf() {
|
|
70
|
+
const rows = store.raw.q(`SELECT status, COUNT(*) AS n FROM nodes WHERE kind = 'doc' GROUP BY status`).all();
|
|
71
|
+
const counts = { pending: 0, parsing: 0, ready: 0, failed: 0 };
|
|
72
|
+
for (const row of rows) {
|
|
73
|
+
if (row.status in counts) counts[row.status] = Number(row.n);
|
|
74
|
+
else counts.pending += Number(row.n);
|
|
75
|
+
}
|
|
76
|
+
const total = counts.pending + counts.parsing + counts.ready + counts.failed;
|
|
77
|
+
const info = pipeline?.pendingInfo?.() ?? { queued: 0, active: 0, concurrency: 1 };
|
|
78
|
+
return { ...counts, total, done: counts.ready + counts.failed, queued: info.queued, active: info.active };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function stateBody(kbId) {
|
|
82
|
+
const kbs = store.listKbs();
|
|
83
|
+
const active = kbId || kbs[0]?.id || '';
|
|
84
|
+
return {
|
|
85
|
+
ok: true,
|
|
86
|
+
root: store.root,
|
|
87
|
+
token,
|
|
88
|
+
kbs,
|
|
89
|
+
kbId: active,
|
|
90
|
+
nodes: active ? store.listNodes(active) : [],
|
|
91
|
+
stats: store.stats(),
|
|
92
|
+
progress: progressOf(),
|
|
93
|
+
capabilities: capabilities(),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async function handleUpload(req, res) {
|
|
98
|
+
const url = new URL(req.url ?? '/', 'http://local');
|
|
99
|
+
const kbId = url.searchParams.get('kb') || '';
|
|
100
|
+
const parentId = url.searchParams.get('parent') || '';
|
|
101
|
+
const kb = kbId ? store.getKb(kbId) : null;
|
|
102
|
+
if (!kb) return fail(res, 400, '知识库不存在');
|
|
103
|
+
const filename = filenameFromHeader(req) || 'file';
|
|
104
|
+
const ext = extOf(filename);
|
|
105
|
+
if (!SUPPORTED_EXTENSIONS.has(ext)) return fail(res, 415, `不支持的文件类型:.${ext || '?'}`);
|
|
106
|
+
|
|
107
|
+
const declared = Number(req.headers['content-length']);
|
|
108
|
+
if (Number.isFinite(declared) && declared > maxFileBytes) {
|
|
109
|
+
return fail(res, 413, `文件超过上限(${Math.round(maxFileBytes / 1048576)} MB)`);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const docId = newId('d');
|
|
113
|
+
const tmpPath = path.join(store.tmpDir, `${docId}.part`);
|
|
114
|
+
const hash = createHash('sha256');
|
|
115
|
+
let size = 0;
|
|
116
|
+
let aborted = false;
|
|
117
|
+
req.on('data', (chunk) => {
|
|
118
|
+
size += chunk.length;
|
|
119
|
+
if (size > maxFileBytes) aborted = true;
|
|
120
|
+
hash.update(chunk);
|
|
121
|
+
});
|
|
122
|
+
try {
|
|
123
|
+
await streamPipeline(req, createWriteStream(tmpPath));
|
|
124
|
+
} catch (error) {
|
|
125
|
+
try { fs.rmSync(tmpPath, { force: true }); } catch { /* ignore */ }
|
|
126
|
+
return fail(res, 400, `上传中断:${error instanceof Error ? error.message : String(error)}`);
|
|
127
|
+
}
|
|
128
|
+
if (aborted) {
|
|
129
|
+
try { fs.rmSync(tmpPath, { force: true }); } catch { /* ignore */ }
|
|
130
|
+
return fail(res, 413, `文件超过上限(${Math.round(maxFileBytes / 1048576)} MB)`);
|
|
131
|
+
}
|
|
132
|
+
if (size === 0) {
|
|
133
|
+
try { fs.rmSync(tmpPath, { force: true }); } catch { /* ignore */ }
|
|
134
|
+
return fail(res, 400, '文件是空的');
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const finalPath = path.join(store.filesDir, `${docId}.${ext}`);
|
|
138
|
+
fs.renameSync(tmpPath, finalPath);
|
|
139
|
+
// 图片没有文字层,只有在开了「图片理解」时才需要进队列
|
|
140
|
+
const indexed = isIndexable(ext);
|
|
141
|
+
const needsVision = isVisionExtension(ext) && Boolean(vision?.enabled);
|
|
142
|
+
const indexable = indexed || needsVision;
|
|
143
|
+
const node = store.createNode({
|
|
144
|
+
id: docId,
|
|
145
|
+
kbId,
|
|
146
|
+
parentId: parentId || null,
|
|
147
|
+
kind: 'doc',
|
|
148
|
+
name: sanitizeName(filename, `未命名.${ext}`),
|
|
149
|
+
now: nowMs(),
|
|
150
|
+
ext,
|
|
151
|
+
mime: MIME_BY_EXT[ext] ?? 'application/octet-stream',
|
|
152
|
+
bytes: size,
|
|
153
|
+
filePath: finalPath,
|
|
154
|
+
status: indexable ? 'pending' : 'ready',
|
|
155
|
+
});
|
|
156
|
+
store.raw.q(`UPDATE nodes SET sha256 = ? WHERE id = ?`).run(hash.digest('hex'), docId);
|
|
157
|
+
if (indexable && pipeline) pipeline.enqueue(docId);
|
|
158
|
+
// Office 文件先把原版式预览排在后台(失败不影响上传本身)
|
|
159
|
+
if (previewer?.supports?.(ext) && previewer.available?.()) {
|
|
160
|
+
previewer.ensure(node, { priority: 'background' })
|
|
161
|
+
.catch((error) => logger?.warn?.(`[knowledge-base] preview prewarm failed for ${filename}: ${error?.message || error}`));
|
|
162
|
+
}
|
|
163
|
+
return sendJson(res, 200, {
|
|
164
|
+
ok: true,
|
|
165
|
+
node: store.getNode(docId),
|
|
166
|
+
indexable,
|
|
167
|
+
notice: indexable ? '' : '这是一张图片,只做预览,不进正文索引。',
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const routes = [];
|
|
172
|
+
|
|
173
|
+
routes.push({
|
|
174
|
+
kind: 'exact',
|
|
175
|
+
path: `${API_BASE}/state`,
|
|
176
|
+
handler: (req, res) => {
|
|
177
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
178
|
+
const url = new URL(req.url ?? '/', 'http://local');
|
|
179
|
+
return sendJson(res, 200, stateBody(url.searchParams.get('kb') || ''));
|
|
180
|
+
},
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
routes.push({
|
|
184
|
+
kind: 'exact',
|
|
185
|
+
path: `${API_BASE}/node`,
|
|
186
|
+
handler: async (req, res) => {
|
|
187
|
+
if (req.method !== 'POST') return fail(res, 405, '只接受 POST');
|
|
188
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
189
|
+
const body = await readBody(req);
|
|
190
|
+
if (!body.ok) return fail(res, 400, '请求体不是合法 JSON');
|
|
191
|
+
const { action } = body.value;
|
|
192
|
+
const now = nowMs();
|
|
193
|
+
try {
|
|
194
|
+
if (action === 'create-kb') {
|
|
195
|
+
const kb = store.createKb({ id: newId('k'), name: sanitizeName(body.value.name, '新知识库'), now });
|
|
196
|
+
return sendJson(res, 200, { ...stateBody(kb.id), created: kb });
|
|
197
|
+
}
|
|
198
|
+
if (action === 'rename-kb') {
|
|
199
|
+
store.renameKb(String(body.value.id), sanitizeName(body.value.name, '知识库'), now);
|
|
200
|
+
return sendJson(res, 200, stateBody(String(body.value.id)));
|
|
201
|
+
}
|
|
202
|
+
if (action === 'delete-kb') {
|
|
203
|
+
store.deleteKb(String(body.value.id));
|
|
204
|
+
return sendJson(res, 200, stateBody(''));
|
|
205
|
+
}
|
|
206
|
+
if (action === 'create') {
|
|
207
|
+
const kbId = String(body.value.kbId ?? '');
|
|
208
|
+
const parentId = body.value.parentId ? String(body.value.parentId) : null;
|
|
209
|
+
const kind = body.value.kind === 'folder' ? 'folder' : 'doc';
|
|
210
|
+
const node = store.createNode({
|
|
211
|
+
id: newId(kind === 'folder' ? 'f' : 'd'),
|
|
212
|
+
kbId,
|
|
213
|
+
parentId,
|
|
214
|
+
kind,
|
|
215
|
+
name: sanitizeName(body.value.name, kind === 'folder' ? '新建文件夹' : '新建文档'),
|
|
216
|
+
now,
|
|
217
|
+
status: 'ready',
|
|
218
|
+
});
|
|
219
|
+
return sendJson(res, 200, { ok: true, node, nodes: store.listNodes(kbId) });
|
|
220
|
+
}
|
|
221
|
+
if (action === 'rename') {
|
|
222
|
+
const node = store.getNode(String(body.value.id));
|
|
223
|
+
if (!node) return fail(res, 404, '节点不存在');
|
|
224
|
+
store.renameNode(node.id, sanitizeName(body.value.name, node.name), now);
|
|
225
|
+
return sendJson(res, 200, { ok: true, nodes: store.listNodes(node.kbId) });
|
|
226
|
+
}
|
|
227
|
+
if (action === 'move') {
|
|
228
|
+
const node = store.getNode(String(body.value.id));
|
|
229
|
+
if (!node) return fail(res, 404, '节点不存在');
|
|
230
|
+
store.moveNode(node.id, body.value.parentId ? String(body.value.parentId) : null,
|
|
231
|
+
Number(body.value.index), now);
|
|
232
|
+
return sendJson(res, 200, { ok: true, nodes: store.listNodes(node.kbId) });
|
|
233
|
+
}
|
|
234
|
+
if (action === 'delete') {
|
|
235
|
+
const node = store.getNode(String(body.value.id));
|
|
236
|
+
if (!node) return fail(res, 404, '节点不存在');
|
|
237
|
+
const result = store.deleteNode(node.id);
|
|
238
|
+
return sendJson(res, 200, { ok: true, removed: result.removed, nodes: store.listNodes(node.kbId) });
|
|
239
|
+
}
|
|
240
|
+
return fail(res, 400, `未知操作 ${action}`);
|
|
241
|
+
} catch (error) {
|
|
242
|
+
logger?.warn?.(`[knowledge-base] node action ${action} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
243
|
+
return fail(res, 400, error instanceof Error ? error.message : '操作失败');
|
|
244
|
+
}
|
|
245
|
+
},
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
routes.push({
|
|
249
|
+
kind: 'exact',
|
|
250
|
+
path: `${API_BASE}/upload`,
|
|
251
|
+
handler: (req, res) => {
|
|
252
|
+
if (req.method !== 'POST') return fail(res, 405, '只接受 POST');
|
|
253
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
254
|
+
return handleUpload(req, res).catch((error) => {
|
|
255
|
+
logger?.error?.('[knowledge-base] upload failed', error);
|
|
256
|
+
if (!res.headersSent) fail(res, 500, '上传失败');
|
|
257
|
+
});
|
|
258
|
+
},
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
routes.push({
|
|
262
|
+
kind: 'exact',
|
|
263
|
+
path: `${API_BASE}/search`,
|
|
264
|
+
handler: async (req, res) => {
|
|
265
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
266
|
+
const url = new URL(req.url ?? '/', 'http://local');
|
|
267
|
+
const query = (url.searchParams.get('q') ?? '').trim();
|
|
268
|
+
const kb = url.searchParams.get('kb') ?? '';
|
|
269
|
+
const mode = url.searchParams.get('mode') === 'hybrid' ? 'hybrid' : 'keyword';
|
|
270
|
+
const limit = clampInt(url.searchParams.get('limit'), 1, 50, 20);
|
|
271
|
+
if (!query) return sendJson(res, 200, { ok: true, hits: [], mode, total: 0 });
|
|
272
|
+
try {
|
|
273
|
+
const result = await index.search({ query, kbIds: kb ? [kb] : [], limit, mode });
|
|
274
|
+
return sendJson(res, 200, { ok: true, ...result });
|
|
275
|
+
} catch (error) {
|
|
276
|
+
return fail(res, 500, `检索失败:${error instanceof Error ? error.message : String(error)}`);
|
|
277
|
+
}
|
|
278
|
+
},
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
routes.push({
|
|
282
|
+
kind: 'exact',
|
|
283
|
+
path: `${API_BASE}/dashboard`,
|
|
284
|
+
handler: (req, res) => {
|
|
285
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
286
|
+
try {
|
|
287
|
+
const data = store.dashboard(nowMs());
|
|
288
|
+
return sendJson(res, 200, { ok: true, ...data, capabilities: capabilities() });
|
|
289
|
+
} catch (error) {
|
|
290
|
+
return fail(res, 500, `读取统计失败:${error instanceof Error ? error.message : String(error)}`);
|
|
291
|
+
}
|
|
292
|
+
},
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
routes.push({
|
|
296
|
+
kind: 'exact',
|
|
297
|
+
path: `${API_BASE}/settings`,
|
|
298
|
+
handler: async (req, res) => {
|
|
299
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
300
|
+
const snapshot = () => ({
|
|
301
|
+
ok: true,
|
|
302
|
+
embedding: settings.publicEmbedding(config?.embedding),
|
|
303
|
+
vision: settings.publicVision(config?.vision),
|
|
304
|
+
status: embedder.describe(),
|
|
305
|
+
visionStatus: vision?.describe?.() ?? { configured: false, enabled: false, model: '' },
|
|
306
|
+
stats: store.stats(),
|
|
307
|
+
});
|
|
308
|
+
if (req.method === 'GET') return sendJson(res, 200, snapshot());
|
|
309
|
+
if (req.method !== 'POST') return fail(res, 405, '只接受 GET / POST');
|
|
310
|
+
const body = await readBody(req);
|
|
311
|
+
if (!body.ok) return fail(res, 400, '请求体不是合法 JSON');
|
|
312
|
+
const value = body.value && typeof body.value === 'object' ? body.value : {};
|
|
313
|
+
// 支持 { embedding: {...} } / { vision: {...} };也兼容老的「直接把字段放在顶层」
|
|
314
|
+
const jobs = [];
|
|
315
|
+
if (value.embedding && typeof value.embedding === 'object') jobs.push(['embedding', value.embedding]);
|
|
316
|
+
if (value.vision && typeof value.vision === 'object') jobs.push(['vision', value.vision]);
|
|
317
|
+
if (!jobs.length) jobs.push(['embedding', value]);
|
|
318
|
+
|
|
319
|
+
for (const [section, patch] of jobs) {
|
|
320
|
+
if (patch.enabled) {
|
|
321
|
+
const baseUrl = String(patch.baseUrl ?? '').trim();
|
|
322
|
+
if (!/^https?:\/\//i.test(baseUrl)) return fail(res, 400, '接口地址要以 http:// 或 https:// 开头');
|
|
323
|
+
if (!String(patch.model ?? '').trim()) return fail(res, 400, '要填模型名');
|
|
324
|
+
}
|
|
325
|
+
const saved = section === 'vision' ? settings.saveVision(patch) : settings.saveEmbedding(patch);
|
|
326
|
+
if (!saved.ok) return fail(res, 500, '写入设置失败,检查一下磁盘权限');
|
|
327
|
+
}
|
|
328
|
+
return sendJson(res, 200, snapshot());
|
|
329
|
+
},
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
routes.push({
|
|
333
|
+
kind: 'exact',
|
|
334
|
+
path: `${API_BASE}/vision-test`,
|
|
335
|
+
handler: async (req, res) => {
|
|
336
|
+
if (req.method !== 'POST') return fail(res, 405, '只接受 POST');
|
|
337
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
338
|
+
const body = await readBody(req);
|
|
339
|
+
if (!body.ok) return fail(res, 400, '请求体不是合法 JSON');
|
|
340
|
+
const form = body.value && typeof body.value === 'object' ? body.value : {};
|
|
341
|
+
// 用表单里正在填的值测,不必先保存
|
|
342
|
+
const saved = settings.vision();
|
|
343
|
+
// 图片理解常和向量检索同一个服务商:没单独填 key 就复用嵌入那把
|
|
344
|
+
const fallbackKey = settings.embedding().apiKey || process.env.DSH_KB_EMBEDDING_KEY || '';
|
|
345
|
+
const probeClient = createVisionClient({
|
|
346
|
+
resolve: () => ({
|
|
347
|
+
enabled: true,
|
|
348
|
+
baseUrl: typeof form.baseUrl === 'string' && form.baseUrl.trim() ? form.baseUrl : saved.baseUrl,
|
|
349
|
+
model: typeof form.model === 'string' && form.model.trim() ? form.model : saved.model,
|
|
350
|
+
apiKey: (typeof form.apiKey === 'string' && form.apiKey.trim()) ? form.apiKey.trim() : (saved.apiKey || fallbackKey),
|
|
351
|
+
}),
|
|
352
|
+
logger,
|
|
353
|
+
});
|
|
354
|
+
try {
|
|
355
|
+
return sendJson(res, 200, { ok: true, probe: await probeClient.probe() });
|
|
356
|
+
} catch (error) {
|
|
357
|
+
return sendJson(res, 200, { ok: true, probe: { ok: false, error: error instanceof Error ? error.message : String(error) } });
|
|
358
|
+
}
|
|
359
|
+
},
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
routes.push({
|
|
363
|
+
kind: 'exact',
|
|
364
|
+
path: `${API_BASE}/embedding-test`,
|
|
365
|
+
handler: async (req, res) => {
|
|
366
|
+
if (req.method !== 'POST') return fail(res, 405, '只接受 POST');
|
|
367
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
368
|
+
const body = await readBody(req);
|
|
369
|
+
if (!body.ok) return fail(res, 400, '请求体不是合法 JSON');
|
|
370
|
+
const form = body.value && typeof body.value === 'object' ? body.value : {};
|
|
371
|
+
// 用「表单里正在填的值」测,不必先保存;key 没填就沿用已保存的
|
|
372
|
+
const saved = settings.embedding();
|
|
373
|
+
const probe = createEmbedder({
|
|
374
|
+
config: {
|
|
375
|
+
enabled: true,
|
|
376
|
+
baseUrl: typeof form.baseUrl === 'string' ? form.baseUrl : saved.baseUrl,
|
|
377
|
+
model: typeof form.model === 'string' ? form.model : saved.model,
|
|
378
|
+
apiKey: (typeof form.apiKey === 'string' && form.apiKey.trim()) ? form.apiKey.trim() : saved.apiKey,
|
|
379
|
+
},
|
|
380
|
+
logger,
|
|
381
|
+
});
|
|
382
|
+
try {
|
|
383
|
+
const result = await probe.probe();
|
|
384
|
+
// 探测失败也走 200:外层 ok 表示"请求处理成功",probe.ok 表示"接口通不通"
|
|
385
|
+
return sendJson(res, 200, { ok: true, probe: result });
|
|
386
|
+
} catch (error) {
|
|
387
|
+
return sendJson(res, 200, { ok: true, probe: { ok: false, error: error instanceof Error ? error.message : String(error) } });
|
|
388
|
+
}
|
|
389
|
+
},
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
routes.push({
|
|
393
|
+
kind: 'exact',
|
|
394
|
+
path: `${API_BASE}/purge-previews`,
|
|
395
|
+
handler: (req, res) => {
|
|
396
|
+
if (req.method !== 'POST') return fail(res, 405, '只接受 POST');
|
|
397
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
398
|
+
let cleared = 0;
|
|
399
|
+
for (const row of store.raw.q(`SELECT id FROM nodes WHERE kind = 'doc'`).all()) {
|
|
400
|
+
const node = store.getNode(row.id);
|
|
401
|
+
if (node && previewer?.supports?.(node.ext)) {
|
|
402
|
+
previewer.invalidate(node);
|
|
403
|
+
cleared += 1;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
return sendJson(res, 200, { ok: true, cleared });
|
|
407
|
+
},
|
|
408
|
+
});
|
|
409
|
+
|
|
410
|
+
routes.push({
|
|
411
|
+
kind: 'exact',
|
|
412
|
+
path: `${API_BASE}/reindex-all`,
|
|
413
|
+
handler: (req, res) => {
|
|
414
|
+
if (req.method !== 'POST') return fail(res, 405, '只接受 POST');
|
|
415
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
416
|
+
const count = pipeline.reindexAll();
|
|
417
|
+
return sendJson(res, 200, { ok: true, queued: count });
|
|
418
|
+
},
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
routes.push({
|
|
422
|
+
kind: 'exact',
|
|
423
|
+
path: `${API_BASE}/ask`,
|
|
424
|
+
handler: async (req, res) => {
|
|
425
|
+
if (req.method !== 'POST') return fail(res, 405, '只接受 POST');
|
|
426
|
+
if (!guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
427
|
+
const body = await readBody(req);
|
|
428
|
+
if (!body.ok) return fail(res, 400, '请求体不是合法 JSON');
|
|
429
|
+
const nodeId = String(body.value.docId ?? '');
|
|
430
|
+
const question = typeof body.value.question === 'string' ? body.value.question.trim() : '';
|
|
431
|
+
const node = nodeId ? store.getNode(nodeId) : null;
|
|
432
|
+
if (node && node.kind === 'doc') {
|
|
433
|
+
const crumb = store.breadcrumb(node.id);
|
|
434
|
+
const prompt = [
|
|
435
|
+
'请基于我的个人知识库回答。',
|
|
436
|
+
`文档:《${node.name}》(${crumb})`,
|
|
437
|
+
`docId: ${node.id}`,
|
|
438
|
+
'先用 kb_read 读这篇文档里相关的部分(必要时用 kb_search 定位),再回答;',
|
|
439
|
+
'回答里标注出处(文件名 + 章节或页码),不要编造文档里没有的内容。',
|
|
440
|
+
question ? `\n我的问题:${question}` : '\n我的问题:',
|
|
441
|
+
].join('\n');
|
|
442
|
+
return sendJson(res, 200, { ok: true, prompt });
|
|
443
|
+
}
|
|
444
|
+
const kbId = String(body.value.kbId ?? '');
|
|
445
|
+
const kb = kbId ? store.getKb(kbId) : null;
|
|
446
|
+
const prompt = [
|
|
447
|
+
kb ? `请基于我的个人知识库「${kb.name}」回答。` : '请基于我的个人知识库回答。',
|
|
448
|
+
kb ? `kbId: ${kb.id}(在 kb_search 的 kbIds 里传它)` : '先用 kb_list 看看有哪些知识库。',
|
|
449
|
+
'先用 kb_search 检索,再用 kb_read 读片段周围的正文,然后回答;标注出处,不要编造。',
|
|
450
|
+
question ? `\n我的问题:${question}` : '\n我的问题:',
|
|
451
|
+
].join('\n');
|
|
452
|
+
return sendJson(res, 200, { ok: true, prompt });
|
|
453
|
+
},
|
|
454
|
+
});
|
|
455
|
+
|
|
456
|
+
// ── /doc?id=<id>&action=meta|text|raw|preview|reindex ─────────────────
|
|
457
|
+
// 说明:这一版 webServer 的前缀路由没有接管 /api 下的自定义前缀(会落到 API
|
|
458
|
+
// 网关并拿到 401),所以文档相关操作统一走一条 exact 路由 + 查询参数。
|
|
459
|
+
routes.push({
|
|
460
|
+
kind: 'exact',
|
|
461
|
+
path: `${API_BASE}/doc`,
|
|
462
|
+
handler: async (req, res) => {
|
|
463
|
+
const url = new URL(req.url ?? '/', 'http://local');
|
|
464
|
+
const action = url.searchParams.get('action') ?? '';
|
|
465
|
+
const id = url.searchParams.get('id') ?? '';
|
|
466
|
+
// 二进制预览要能被 <img>/<iframe> 直接取,而它们带不了自定义头:
|
|
467
|
+
// 这时改用一次性启动令牌(令牌只通过需要自定义头的 /state 下发)。
|
|
468
|
+
const byToken = (action === 'raw' || action === 'preview' || action === 'page' || action === 'thumb')
|
|
469
|
+
&& Boolean(token) && url.searchParams.get('t') === token;
|
|
470
|
+
if (!byToken && !guard(origin, req).ok) return fail(res, 403, 'forbidden');
|
|
471
|
+
const node = id ? store.getNode(id) : null;
|
|
472
|
+
if (!node || node.kind !== 'doc') return fail(res, 404, '文档不存在');
|
|
473
|
+
|
|
474
|
+
if (action === 'text') {
|
|
475
|
+
const offset = clampInt(url.searchParams.get('offset'), 0, 100 * 1024 * 1024, 0);
|
|
476
|
+
const limit = clampInt(url.searchParams.get('limit'), 200, 40000, 8000);
|
|
477
|
+
const full = store.getText(node.id);
|
|
478
|
+
const end = Math.min(full.length, offset + limit);
|
|
479
|
+
return sendJson(res, 200, {
|
|
480
|
+
ok: true,
|
|
481
|
+
node,
|
|
482
|
+
breadcrumb: store.breadcrumb(node.id),
|
|
483
|
+
totalChars: full.length,
|
|
484
|
+
offset,
|
|
485
|
+
end,
|
|
486
|
+
text: full.slice(offset, end),
|
|
487
|
+
hasMore: end < full.length,
|
|
488
|
+
});
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
if (action === 'raw') {
|
|
492
|
+
const filePath = store.raw.q(`SELECT file_path AS filePath FROM nodes WHERE id = ?`).get(node.id)?.filePath;
|
|
493
|
+
if (!filePath || !fs.existsSync(filePath)) return fail(res, 404, '原始文件不见了');
|
|
494
|
+
const download = url.searchParams.get('download') === '1';
|
|
495
|
+
return serveFile(res, filePath, node.mime || 'application/octet-stream', {
|
|
496
|
+
download,
|
|
497
|
+
name: node.name,
|
|
498
|
+
cache: 'private, max-age=60',
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// 逐页图片:界面自己画阅读器(不起浏览器 PDF 阅读器)
|
|
503
|
+
if (action === 'previewinfo') {
|
|
504
|
+
if (!previewer?.supports?.(node.ext)) return fail(res, 400, '这类文件没有逐页预览');
|
|
505
|
+
try {
|
|
506
|
+
const manifest = await previewer.ensurePages(node);
|
|
507
|
+
// 缩略图栏要用的低分辨率版本,顺手在后台准备好
|
|
508
|
+
previewer.ensureThumbs(node).catch(() => { /* 缩略图失败不影响正文 */ });
|
|
509
|
+
return sendJson(res, 200, {
|
|
510
|
+
ok: true,
|
|
511
|
+
pages: manifest.pages,
|
|
512
|
+
pageCount: manifest.pageCount,
|
|
513
|
+
truncated: Boolean(manifest.truncated),
|
|
514
|
+
dpi: manifest.dpi,
|
|
515
|
+
width: manifest.width,
|
|
516
|
+
height: manifest.height,
|
|
517
|
+
});
|
|
518
|
+
} catch (error) {
|
|
519
|
+
const code = error?.code === 'unavailable' ? 503 : 500;
|
|
520
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
521
|
+
logger?.warn?.(`[knowledge-base] page render failed for ${node.name}: ${reason}`);
|
|
522
|
+
return fail(res, code, reason);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
if (action === 'page' || action === 'thumb') {
|
|
527
|
+
if (!previewer?.supports?.(node.ext)) return fail(res, 400, '这类文件没有逐页预览');
|
|
528
|
+
const page = clampInt(url.searchParams.get('n'), 1, 2000, 1);
|
|
529
|
+
const isThumb = action === 'thumb';
|
|
530
|
+
try {
|
|
531
|
+
let file = isThumb ? previewer.thumbFile(node, page) : previewer.pageFile(node, page);
|
|
532
|
+
if (!fs.existsSync(file)) {
|
|
533
|
+
await (isThumb ? previewer.ensureThumbs(node) : previewer.ensurePages(node));
|
|
534
|
+
file = isThumb ? previewer.thumbFile(node, page) : previewer.pageFile(node, page);
|
|
535
|
+
}
|
|
536
|
+
if (!fs.existsSync(file)) return fail(res, 404, `没有第 ${page} 页`);
|
|
537
|
+
return serveFile(res, file, 'image/png', { cache: 'private, max-age=3600' });
|
|
538
|
+
} catch (error) {
|
|
539
|
+
return fail(res, 500, error instanceof Error ? error.message : '渲染失败');
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// 原版式预览:PDF/图片直接给原文件;Office 用宿主的 officeKit 转成 PDF 再看
|
|
544
|
+
if (action === 'preview') {
|
|
545
|
+
if (node.ext === 'pdf' || IMAGE_EXTENSIONS.has(node.ext)) {
|
|
546
|
+
const filePath = store.raw.q(`SELECT file_path AS filePath FROM nodes WHERE id = ?`).get(node.id)?.filePath;
|
|
547
|
+
if (!filePath || !fs.existsSync(filePath)) return fail(res, 404, '原始文件不见了');
|
|
548
|
+
return serveFile(res, filePath, node.mime || 'application/octet-stream', { cache: 'private, max-age=60' });
|
|
549
|
+
}
|
|
550
|
+
if (!previewer?.supports?.(node.ext)) {
|
|
551
|
+
return fail(res, 400, '这类文件没有原版式预览,请用文字阅读');
|
|
552
|
+
}
|
|
553
|
+
try {
|
|
554
|
+
const pdfPath = await previewer.ensure(node, { priority: 'foreground' });
|
|
555
|
+
return serveFile(res, pdfPath, 'application/pdf', { cache: 'no-store' });
|
|
556
|
+
} catch (error) {
|
|
557
|
+
const code = error?.code === 'unavailable' ? 503 : 500;
|
|
558
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
559
|
+
logger?.warn?.(`[knowledge-base] preview failed for ${node.name}: ${reason}`);
|
|
560
|
+
return fail(res, code, reason);
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
if (action === 'reindex' && req.method === 'POST') {
|
|
565
|
+
// 图片在有「图片理解」时也能重建(会重新描述一遍)
|
|
566
|
+
const canIndex = isIndexable(node.ext) || (isVisionExtension(node.ext) && Boolean(vision?.enabled));
|
|
567
|
+
if (!canIndex) {
|
|
568
|
+
return fail(res, 400, isVisionExtension(node.ext)
|
|
569
|
+
? '图片没有正文索引;在设置里打开「图片理解」后可以重建'
|
|
570
|
+
: '这个格式没有正文索引');
|
|
571
|
+
}
|
|
572
|
+
store.setDocStatus(node.id, 'pending', { error: null, updatedAt: nowMs() });
|
|
573
|
+
pipeline.enqueue(node.id);
|
|
574
|
+
return sendJson(res, 200, { ok: true, node: store.getNode(node.id) });
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
if (!action || action === 'meta') {
|
|
578
|
+
return sendJson(res, 200, {
|
|
579
|
+
ok: true,
|
|
580
|
+
node,
|
|
581
|
+
breadcrumb: store.breadcrumb(node.id),
|
|
582
|
+
stats: store.stats(),
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
return fail(res, 404, '未知的文档操作');
|
|
586
|
+
},
|
|
587
|
+
});
|
|
588
|
+
|
|
589
|
+
return { routes, origin };
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
export { API_BASE, sendText, sendBytes, safeOrigin };
|