@xiaofanzhao/dsh-plugin-knowledge-base 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +300 -2
- package/client.js +1809 -0
- package/cordis.patch.yml +20 -0
- package/docs/DESIGN.md +662 -0
- package/docs/DETAIL.md +60 -0
- package/docs/images/architecture.svg +111 -0
- package/docs/images/ui-bigscreen.svg +59 -0
- package/docs/images/ui-overview.svg +152 -0
- package/icon.svg +13 -0
- package/index.js +131 -0
- package/lib/chunker.mjs +121 -0
- package/lib/extract.mjs +434 -0
- package/lib/index.mjs +383 -0
- package/lib/pdf.mjs +88 -0
- package/lib/pipeline.mjs +116 -0
- package/lib/preview.mjs +217 -0
- package/lib/routes.mjs +546 -0
- package/lib/settings.mjs +107 -0
- package/lib/store.mjs +445 -0
- package/lib/tools.mjs +246 -0
- package/lib/util.mjs +129 -0
- package/locale/en.json +33 -0
- package/locale/zh.json +33 -0
- package/package.json +63 -4
package/lib/store.mjs
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// 库存储:SQLite(node:sqlite,随 Node 24 内置,无第三方依赖)
|
|
3
|
+
//
|
|
4
|
+
// 三张业务表 + 一张全文索引表:
|
|
5
|
+
// kbs 知识库
|
|
6
|
+
// nodes 树:folder / doc 两种节点,adjacency + 物化 id 路径(移动子树只需一次前缀更新)
|
|
7
|
+
// doc_text 文档抽取出的正文(阅读器按字符偏移分页读)
|
|
8
|
+
// chunks 检索分块;chunks_fts 是它的 CJK 归一化全文索引(rowid 对齐 chunks.id)
|
|
9
|
+
// ============================================================================
|
|
10
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
11
|
+
import fs from 'node:fs';
|
|
12
|
+
import path from 'node:path';
|
|
13
|
+
|
|
14
|
+
const SCHEMA = `
|
|
15
|
+
CREATE TABLE IF NOT EXISTS kbs (
|
|
16
|
+
id TEXT PRIMARY KEY,
|
|
17
|
+
name TEXT NOT NULL,
|
|
18
|
+
created_at INTEGER NOT NULL,
|
|
19
|
+
updated_at INTEGER NOT NULL,
|
|
20
|
+
sort INTEGER NOT NULL DEFAULT 0
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
CREATE TABLE IF NOT EXISTS nodes (
|
|
24
|
+
id TEXT PRIMARY KEY,
|
|
25
|
+
kb_id TEXT NOT NULL,
|
|
26
|
+
parent_id TEXT,
|
|
27
|
+
kind TEXT NOT NULL,
|
|
28
|
+
name TEXT NOT NULL,
|
|
29
|
+
path TEXT NOT NULL,
|
|
30
|
+
sort INTEGER NOT NULL DEFAULT 0,
|
|
31
|
+
file_path TEXT,
|
|
32
|
+
ext TEXT,
|
|
33
|
+
mime TEXT,
|
|
34
|
+
bytes INTEGER,
|
|
35
|
+
sha256 TEXT,
|
|
36
|
+
status TEXT NOT NULL DEFAULT 'ready',
|
|
37
|
+
error TEXT,
|
|
38
|
+
page_count INTEGER,
|
|
39
|
+
chars INTEGER,
|
|
40
|
+
indexed_at INTEGER,
|
|
41
|
+
created_at INTEGER NOT NULL,
|
|
42
|
+
updated_at INTEGER NOT NULL
|
|
43
|
+
);
|
|
44
|
+
CREATE INDEX IF NOT EXISTS nodes_by_parent ON nodes(kb_id, parent_id, sort);
|
|
45
|
+
CREATE INDEX IF NOT EXISTS nodes_by_path ON nodes(kb_id, path);
|
|
46
|
+
|
|
47
|
+
CREATE TABLE IF NOT EXISTS doc_text (
|
|
48
|
+
node_id TEXT PRIMARY KEY,
|
|
49
|
+
text TEXT NOT NULL,
|
|
50
|
+
updated_at INTEGER NOT NULL
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
54
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
55
|
+
node_id TEXT NOT NULL,
|
|
56
|
+
kb_id TEXT NOT NULL,
|
|
57
|
+
ord INTEGER NOT NULL,
|
|
58
|
+
heading TEXT,
|
|
59
|
+
page INTEGER,
|
|
60
|
+
char_start INTEGER NOT NULL,
|
|
61
|
+
char_end INTEGER NOT NULL,
|
|
62
|
+
text TEXT NOT NULL
|
|
63
|
+
);
|
|
64
|
+
CREATE INDEX IF NOT EXISTS chunks_by_doc ON chunks(node_id, ord);
|
|
65
|
+
|
|
66
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
67
|
+
cjk,
|
|
68
|
+
kb_id UNINDEXED,
|
|
69
|
+
node_id UNINDEXED,
|
|
70
|
+
tokenize='unicode61'
|
|
71
|
+
);
|
|
72
|
+
|
|
73
|
+
CREATE TABLE IF NOT EXISTS chunk_vec (
|
|
74
|
+
node_id TEXT NOT NULL,
|
|
75
|
+
ord INTEGER NOT NULL,
|
|
76
|
+
dim INTEGER NOT NULL,
|
|
77
|
+
vec BLOB NOT NULL,
|
|
78
|
+
PRIMARY KEY (node_id, ord)
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
82
|
+
key TEXT PRIMARY KEY,
|
|
83
|
+
value TEXT
|
|
84
|
+
);
|
|
85
|
+
|
|
86
|
+
CREATE TABLE IF NOT EXISTS search_log (
|
|
87
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
88
|
+
ts INTEGER NOT NULL,
|
|
89
|
+
query TEXT NOT NULL,
|
|
90
|
+
mode TEXT NOT NULL,
|
|
91
|
+
hits INTEGER NOT NULL,
|
|
92
|
+
took_ms INTEGER NOT NULL,
|
|
93
|
+
kb_id TEXT
|
|
94
|
+
);
|
|
95
|
+
CREATE INDEX IF NOT EXISTS search_log_by_ts ON search_log(ts);
|
|
96
|
+
`;
|
|
97
|
+
|
|
98
|
+
export function openStore({ root, logger }) {
|
|
99
|
+
const filesDir = path.join(root, 'files');
|
|
100
|
+
const tmpDir = path.join(root, 'tmp');
|
|
101
|
+
fs.mkdirSync(filesDir, { recursive: true });
|
|
102
|
+
fs.mkdirSync(tmpDir, { recursive: true });
|
|
103
|
+
// 上次进程留下的半成品上传与临时文件,启动时清掉
|
|
104
|
+
for (const name of fs.readdirSync(tmpDir)) {
|
|
105
|
+
try { fs.rmSync(path.join(tmpDir, name), { recursive: true, force: true }); } catch { /* ignore */ }
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const db = new DatabaseSync(path.join(root, 'kb.db'));
|
|
109
|
+
db.exec('PRAGMA journal_mode = WAL;');
|
|
110
|
+
db.exec('PRAGMA synchronous = NORMAL;');
|
|
111
|
+
db.exec(SCHEMA);
|
|
112
|
+
|
|
113
|
+
const q = (sql) => db.prepare(sql);
|
|
114
|
+
|
|
115
|
+
// ── 知识库 ────────────────────────────────────────────────────────────
|
|
116
|
+
function listKbs() {
|
|
117
|
+
const rows = q(`SELECT id, name, created_at AS createdAt, updated_at AS updatedAt
|
|
118
|
+
FROM kbs ORDER BY sort, created_at`).all();
|
|
119
|
+
const counts = q(`SELECT kb_id AS kbId, COUNT(*) AS n FROM nodes WHERE kind = 'doc' GROUP BY kb_id`).all();
|
|
120
|
+
const byId = new Map(counts.map((row) => [row.kbId, row.n]));
|
|
121
|
+
return rows.map((row) => ({ ...row, docs: byId.get(row.id) ?? 0 }));
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function getKb(id) {
|
|
125
|
+
return q(`SELECT id, name, created_at AS createdAt, updated_at AS updatedAt FROM kbs WHERE id = ?`).get(id) ?? null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function createKb({ id, name, now }) {
|
|
129
|
+
const sort = (q(`SELECT COALESCE(MAX(sort), -1) AS s FROM kbs`).get()?.s ?? -1) + 1;
|
|
130
|
+
q(`INSERT INTO kbs (id, name, created_at, updated_at, sort) VALUES (?, ?, ?, ?, ?)`).run(id, name, now, now, sort);
|
|
131
|
+
return getKb(id);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function renameKb(id, name, now) {
|
|
135
|
+
q(`UPDATE kbs SET name = ?, updated_at = ? WHERE id = ?`).run(name, now, id);
|
|
136
|
+
return getKb(id);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function deleteKb(id) {
|
|
140
|
+
const nodes = q(`SELECT id FROM nodes WHERE kb_id = ?`).all(id).map((row) => row.id);
|
|
141
|
+
for (const nodeId of nodes) removeDocData(nodeId);
|
|
142
|
+
q(`DELETE FROM nodes WHERE kb_id = ?`).run(id);
|
|
143
|
+
q(`DELETE FROM kbs WHERE id = ?`).run(id);
|
|
144
|
+
return { removed: nodes.length };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// ── 节点 ──────────────────────────────────────────────────────────────
|
|
148
|
+
function nodeRow(row) {
|
|
149
|
+
if (!row) return null;
|
|
150
|
+
return {
|
|
151
|
+
id: row.id,
|
|
152
|
+
kbId: row.kb_id,
|
|
153
|
+
parentId: row.parent_id ?? null,
|
|
154
|
+
kind: row.kind,
|
|
155
|
+
name: row.name,
|
|
156
|
+
path: row.path,
|
|
157
|
+
sort: row.sort,
|
|
158
|
+
ext: row.ext ?? '',
|
|
159
|
+
mime: row.mime ?? '',
|
|
160
|
+
bytes: row.bytes ?? 0,
|
|
161
|
+
status: row.status,
|
|
162
|
+
error: row.error ?? '',
|
|
163
|
+
pageCount: row.page_count ?? 0,
|
|
164
|
+
chars: row.chars ?? 0,
|
|
165
|
+
indexedAt: row.indexed_at ?? 0,
|
|
166
|
+
createdAt: row.created_at,
|
|
167
|
+
updatedAt: row.updated_at,
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function getNode(id) {
|
|
172
|
+
return nodeRow(q(`SELECT * FROM nodes WHERE id = ?`).get(id));
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function listNodes(kbId) {
|
|
176
|
+
return q(`SELECT * FROM nodes WHERE kb_id = ? ORDER BY parent_id, sort`).all(kbId).map(nodeRow);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function children(parentId, kbId) {
|
|
180
|
+
return q(`SELECT * FROM nodes WHERE kb_id = ? AND parent_id IS ? ORDER BY sort`).all(kbId, parentId).map(nodeRow);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function nextSort(kbId, parentId) {
|
|
184
|
+
const row = q(`SELECT COALESCE(MAX(sort), -1) AS s FROM nodes WHERE kb_id = ? AND parent_id IS ?`).get(kbId, parentId);
|
|
185
|
+
return (row?.s ?? -1) + 1;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function createNode({ id, kbId, parentId, kind, name, now, ext = '', mime = '', bytes = 0, filePath = null, status = 'ready' }) {
|
|
189
|
+
const parent = parentId ? getNode(parentId) : null;
|
|
190
|
+
if (parentId && (!parent || parent.kbId !== kbId)) throw new Error('父节点不存在');
|
|
191
|
+
const parentPath = parent ? parent.path : '';
|
|
192
|
+
const sort = nextSort(kbId, parentId);
|
|
193
|
+
q(`INSERT INTO nodes (id, kb_id, parent_id, kind, name, path, sort, file_path, ext, mime, bytes, status,
|
|
194
|
+
created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
195
|
+
.run(id, kbId, parentId, kind, name, `${parentPath}/${id}`, sort, filePath, ext, mime, bytes, status, now, now);
|
|
196
|
+
return getNode(id);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function renameNode(id, name, now) {
|
|
200
|
+
q(`UPDATE nodes SET name = ?, updated_at = ? WHERE id = ?`).run(name, now, id);
|
|
201
|
+
return getNode(id);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function moveNode(id, parentId, index, now) {
|
|
205
|
+
const node = getNode(id);
|
|
206
|
+
if (!node) throw new Error('节点不存在');
|
|
207
|
+
if (parentId === id) throw new Error('不能移动到自身');
|
|
208
|
+
const target = parentId ? getNode(parentId) : null;
|
|
209
|
+
if (parentId && (!target || target.kbId !== node.kbId)) throw new Error('目标目录不存在');
|
|
210
|
+
if (parentId && (target.path === node.path || target.path.startsWith(`${node.path}/`))) {
|
|
211
|
+
throw new Error('不能移动到自己的子目录');
|
|
212
|
+
}
|
|
213
|
+
const newPath = `${target ? target.path : ''}/${id}`;
|
|
214
|
+
db.exec('BEGIN');
|
|
215
|
+
try {
|
|
216
|
+
q(`UPDATE nodes SET path = replace(path, ?, ?) WHERE kb_id = ? AND (path = ? OR path LIKE ?)`)
|
|
217
|
+
.run(node.path, newPath, node.kbId, node.path, `${node.path}/%`);
|
|
218
|
+
q(`UPDATE nodes SET parent_id = ?, sort = ?, updated_at = ? WHERE id = ?`)
|
|
219
|
+
.run(parentId, 1e9, now, id);
|
|
220
|
+
const siblings = q(`SELECT id FROM nodes WHERE kb_id = ? AND parent_id IS ? AND id <> ? ORDER BY sort`)
|
|
221
|
+
.all(node.kbId, parentId, id).map((row) => row.id);
|
|
222
|
+
const at = Number.isInteger(index) && index >= 0 ? Math.min(index, siblings.length) : siblings.length;
|
|
223
|
+
siblings.splice(at, 0, id);
|
|
224
|
+
const stmt = q(`UPDATE nodes SET sort = ? WHERE id = ?`);
|
|
225
|
+
siblings.forEach((siblingId, position) => stmt.run(position, siblingId));
|
|
226
|
+
db.exec('COMMIT');
|
|
227
|
+
} catch (error) {
|
|
228
|
+
db.exec('ROLLBACK');
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
231
|
+
return getNode(id);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function subtreeIds(node) {
|
|
235
|
+
const rows = q(`SELECT id, kind FROM nodes WHERE path = ? OR path LIKE ?`).all(node.path, `${node.path}/%`);
|
|
236
|
+
return rows;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
function removeDocData(nodeId) {
|
|
240
|
+
const chunkRows = q(`SELECT id FROM chunks WHERE node_id = ?`).all(nodeId);
|
|
241
|
+
const delFts = q(`DELETE FROM chunks_fts WHERE rowid = ?`);
|
|
242
|
+
for (const row of chunkRows) delFts.run(row.id);
|
|
243
|
+
q(`DELETE FROM chunks WHERE node_id = ?`).run(nodeId);
|
|
244
|
+
q(`DELETE FROM chunk_vec WHERE node_id = ?`).run(nodeId);
|
|
245
|
+
q(`DELETE FROM doc_text WHERE node_id = ?`).run(nodeId);
|
|
246
|
+
const doc = q(`SELECT file_path AS filePath FROM nodes WHERE id = ?`).get(nodeId);
|
|
247
|
+
if (doc?.filePath) {
|
|
248
|
+
try { fs.rmSync(doc.filePath, { force: true }); } catch { /* ignore */ }
|
|
249
|
+
}
|
|
250
|
+
// 转换出来的预览 PDF 也一并清掉
|
|
251
|
+
if (doc?.filePath) {
|
|
252
|
+
try { fs.rmSync(path.join(path.dirname(doc.filePath), `${nodeId}.preview.pdf`), { force: true }); } catch { /* ignore */ }
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function deleteNode(id) {
|
|
257
|
+
const node = getNode(id);
|
|
258
|
+
if (!node) throw new Error('节点不存在');
|
|
259
|
+
const rows = subtreeIds(node);
|
|
260
|
+
db.exec('BEGIN');
|
|
261
|
+
try {
|
|
262
|
+
for (const row of rows) removeDocData(row.id);
|
|
263
|
+
q(`DELETE FROM nodes WHERE id = ?`).run(id);
|
|
264
|
+
const rest = rows.filter((row) => row.id !== id).map((row) => row.id);
|
|
265
|
+
for (const restId of rest) q(`DELETE FROM nodes WHERE id = ?`).run(restId);
|
|
266
|
+
db.exec('COMMIT');
|
|
267
|
+
} catch (error) {
|
|
268
|
+
db.exec('ROLLBACK');
|
|
269
|
+
throw error;
|
|
270
|
+
}
|
|
271
|
+
return { removed: rows.length };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function setDocStatus(id, status, extra = {}) {
|
|
275
|
+
q(`UPDATE nodes SET status = ?, error = ?, page_count = ?, chars = ?, indexed_at = ?, updated_at = ?
|
|
276
|
+
WHERE id = ?`).run(
|
|
277
|
+
status,
|
|
278
|
+
extra.error ?? null,
|
|
279
|
+
extra.pageCount ?? null,
|
|
280
|
+
extra.chars ?? null,
|
|
281
|
+
extra.indexedAt ?? null,
|
|
282
|
+
extra.updatedAt ?? Date.now(),
|
|
283
|
+
id,
|
|
284
|
+
);
|
|
285
|
+
return getNode(id);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function saveText(nodeId, text, now) { q(`INSERT INTO doc_text (node_id, text, updated_at) VALUES (?, ?, ?)
|
|
289
|
+
ON CONFLICT(node_id) DO UPDATE SET text = excluded.text, updated_at = excluded.updated_at`)
|
|
290
|
+
.run(nodeId, text, now);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function getText(nodeId) {
|
|
294
|
+
const row = q(`SELECT text FROM doc_text WHERE node_id = ?`).get(nodeId);
|
|
295
|
+
return row ? row.text : '';
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function breadcrumb(nodeId) {
|
|
299
|
+
const names = [];
|
|
300
|
+
let current = getNode(nodeId);
|
|
301
|
+
let guardCount = 0;
|
|
302
|
+
while (current && guardCount < 32) {
|
|
303
|
+
names.unshift(current.name);
|
|
304
|
+
current = current.parentId ? getNode(current.parentId) : null;
|
|
305
|
+
guardCount += 1;
|
|
306
|
+
}
|
|
307
|
+
return names.join(' / ');
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function stats() {
|
|
311
|
+
return {
|
|
312
|
+
kbs: q(`SELECT COUNT(*) AS n FROM kbs`).get()?.n ?? 0,
|
|
313
|
+
docs: q(`SELECT COUNT(*) AS n FROM nodes WHERE kind = 'doc'`).get()?.n ?? 0,
|
|
314
|
+
chunks: q(`SELECT COUNT(*) AS n FROM chunks`).get()?.n ?? 0,
|
|
315
|
+
bytes: q(`SELECT COALESCE(SUM(bytes), 0) AS n FROM nodes WHERE kind = 'doc'`).get()?.n ?? 0,
|
|
316
|
+
vectors: q(`SELECT COUNT(*) AS n FROM chunk_vec`).get()?.n ?? 0,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function mapNodeByIds(ids) {
|
|
321
|
+
const out = new Map();
|
|
322
|
+
const stmt = q(`SELECT * FROM nodes WHERE id = ?`);
|
|
323
|
+
for (const id of ids) {
|
|
324
|
+
const row = stmt.get(id);
|
|
325
|
+
if (row) out.set(id, nodeRow(row));
|
|
326
|
+
}
|
|
327
|
+
return out;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// ── 检索日志(看板用)──────────────────────────────────────────────────
|
|
331
|
+
const SEARCH_LOG_KEEP = 2000;
|
|
332
|
+
|
|
333
|
+
function logSearch({ now, query, mode, hits, tookMs, kbId }) {
|
|
334
|
+
try {
|
|
335
|
+
q(`INSERT INTO search_log (ts, query, mode, hits, took_ms, kb_id) VALUES (?, ?, ?, ?, ?, ?)`)
|
|
336
|
+
.run(now, String(query ?? '').slice(0, 200), String(mode ?? 'keyword'), Number(hits) || 0,
|
|
337
|
+
Number(tookMs) || 0, kbId || null);
|
|
338
|
+
// 只留最近 SEARCH_LOG_KEEP 条,别让日志把库撑大
|
|
339
|
+
q(`DELETE FROM search_log WHERE id <= (SELECT COALESCE(MAX(id), 0) - ? FROM search_log)`).run(SEARCH_LOG_KEEP);
|
|
340
|
+
} catch { /* 埋点失败不能影响检索 */ }
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/** 看板用的聚合:总量、类型分布、状态分布、检索概览、最近活动。 */
|
|
344
|
+
function dashboard(now = Date.now()) {
|
|
345
|
+
const dayStart = new Date(now);
|
|
346
|
+
dayStart.setHours(0, 0, 0, 0);
|
|
347
|
+
const dayStartMs = dayStart.getTime();
|
|
348
|
+
|
|
349
|
+
const byExt = q(`SELECT ext, COUNT(*) AS docs, COALESCE(SUM(bytes), 0) AS bytes
|
|
350
|
+
FROM nodes WHERE kind = 'doc' GROUP BY ext ORDER BY docs DESC, bytes DESC`).all()
|
|
351
|
+
.map((row) => ({ ext: row.ext || '?', docs: Number(row.docs), bytes: Number(row.bytes) }));
|
|
352
|
+
|
|
353
|
+
const byStatus = { ready: 0, pending: 0, parsing: 0, failed: 0 };
|
|
354
|
+
for (const row of q(`SELECT status, COUNT(*) AS n FROM nodes WHERE kind = 'doc' GROUP BY status`).all()) {
|
|
355
|
+
if (row.status in byStatus) byStatus[row.status] = Number(row.n);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const byKb = q(`SELECT k.id, k.name, COUNT(n.id) AS docs, COALESCE(SUM(n.bytes), 0) AS bytes
|
|
359
|
+
FROM kbs k LEFT JOIN nodes n ON n.kb_id = k.id AND n.kind = 'doc'
|
|
360
|
+
GROUP BY k.id ORDER BY docs DESC`).all()
|
|
361
|
+
.map((row) => ({ id: row.id, name: row.name, docs: Number(row.docs), bytes: Number(row.bytes) }));
|
|
362
|
+
|
|
363
|
+
const search = {
|
|
364
|
+
total: Number(q(`SELECT COUNT(*) AS n FROM search_log`).get()?.n ?? 0),
|
|
365
|
+
today: Number(q(`SELECT COUNT(*) AS n FROM search_log WHERE ts >= ?`).get(dayStartMs)?.n ?? 0),
|
|
366
|
+
zeroHit: Number(q(`SELECT COUNT(*) AS n FROM search_log WHERE hits = 0`).get()?.n ?? 0),
|
|
367
|
+
hybrid: Number(q(`SELECT COUNT(*) AS n FROM search_log WHERE mode = 'hybrid'`).get()?.n ?? 0),
|
|
368
|
+
avgMs: Math.round(Number(q(`SELECT COALESCE(AVG(took_ms), 0) AS n FROM search_log`).get()?.n ?? 0)),
|
|
369
|
+
lastAt: Number(q(`SELECT COALESCE(MAX(ts), 0) AS n FROM search_log`).get()?.n ?? 0),
|
|
370
|
+
};
|
|
371
|
+
search.zeroRate = search.total ? Math.round((search.zeroHit / search.total) * 100) : 0;
|
|
372
|
+
|
|
373
|
+
const trend = q(`SELECT date(ts / 1000, 'unixepoch', 'localtime') AS day, COUNT(*) AS n
|
|
374
|
+
FROM search_log GROUP BY day ORDER BY day DESC LIMIT 7`).all()
|
|
375
|
+
.map((row) => ({ day: row.day, count: Number(row.n) }))
|
|
376
|
+
.reverse();
|
|
377
|
+
|
|
378
|
+
const topQueries = q(`SELECT query, COUNT(*) AS n FROM search_log
|
|
379
|
+
WHERE length(trim(query)) > 0 GROUP BY lower(trim(query)) ORDER BY n DESC, MAX(ts) DESC LIMIT 8`).all()
|
|
380
|
+
.map((row) => ({ query: row.query, count: Number(row.n) }));
|
|
381
|
+
|
|
382
|
+
const recent = q(`SELECT ts, query, mode, hits, took_ms AS tookMs FROM search_log
|
|
383
|
+
ORDER BY id DESC LIMIT 10`).all()
|
|
384
|
+
.map((row) => ({ ts: Number(row.ts), query: row.query, mode: row.mode, hits: Number(row.hits), tookMs: Number(row.tookMs) }));
|
|
385
|
+
|
|
386
|
+
const recentDocs = q(`SELECT id, name, ext, bytes, status, created_at AS createdAt, indexed_at AS indexedAt
|
|
387
|
+
FROM nodes WHERE kind = 'doc' ORDER BY created_at DESC LIMIT 6`).all()
|
|
388
|
+
.map((row) => ({
|
|
389
|
+
id: row.id,
|
|
390
|
+
name: row.name,
|
|
391
|
+
ext: row.ext ?? '',
|
|
392
|
+
bytes: Number(row.bytes ?? 0),
|
|
393
|
+
status: row.status,
|
|
394
|
+
createdAt: Number(row.createdAt ?? 0),
|
|
395
|
+
indexedAt: Number(row.indexedAt ?? 0),
|
|
396
|
+
}));
|
|
397
|
+
|
|
398
|
+
return {
|
|
399
|
+
now,
|
|
400
|
+
stats: stats(),
|
|
401
|
+
byExt,
|
|
402
|
+
byStatus,
|
|
403
|
+
byKb,
|
|
404
|
+
search,
|
|
405
|
+
trend,
|
|
406
|
+
topQueries,
|
|
407
|
+
recent,
|
|
408
|
+
recentDocs,
|
|
409
|
+
root: root,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
return {
|
|
414
|
+
db,
|
|
415
|
+
root,
|
|
416
|
+
filesDir,
|
|
417
|
+
tmpDir,
|
|
418
|
+
listKbs,
|
|
419
|
+
getKb,
|
|
420
|
+
createKb,
|
|
421
|
+
renameKb,
|
|
422
|
+
deleteKb,
|
|
423
|
+
getNode,
|
|
424
|
+
listNodes,
|
|
425
|
+
children,
|
|
426
|
+
createNode,
|
|
427
|
+
renameNode,
|
|
428
|
+
moveNode,
|
|
429
|
+
deleteNode,
|
|
430
|
+
setDocStatus,
|
|
431
|
+
saveText,
|
|
432
|
+
getText,
|
|
433
|
+
breadcrumb,
|
|
434
|
+
stats,
|
|
435
|
+
mapNodeByIds,
|
|
436
|
+
removeDocData,
|
|
437
|
+
logSearch,
|
|
438
|
+
dashboard,
|
|
439
|
+
close() {
|
|
440
|
+
try { db.close(); } catch { /* ignore */ }
|
|
441
|
+
},
|
|
442
|
+
/** 供 index.mjs 直接使用的表操作。 */
|
|
443
|
+
raw: { q },
|
|
444
|
+
};
|
|
445
|
+
}
|
package/lib/tools.mjs
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// 模型工具:kb_list / kb_search / kb_read(全部只读)
|
|
3
|
+
//
|
|
4
|
+
// 只依赖 ctx.tools.register 约定,不 import 任何 @deepseek-ai 包:
|
|
5
|
+
// { name, description, parameters, output: { schema, render }, execute, presentCall }
|
|
6
|
+
// parameters 只能使用 DSH 支持的 JSON Schema 子集。
|
|
7
|
+
// ============================================================================
|
|
8
|
+
|
|
9
|
+
const MAX_READ_CHARS = 6000;
|
|
10
|
+
const MAX_SEARCH_LIMIT = 20;
|
|
11
|
+
|
|
12
|
+
const HIT_SCHEMA = {
|
|
13
|
+
type: 'object',
|
|
14
|
+
additionalProperties: false,
|
|
15
|
+
required: ['docId', 'name', 'locator', 'snippet', 'charStart'],
|
|
16
|
+
properties: {
|
|
17
|
+
docId: { type: 'string', description: '文档 id,可直接传给 kb_read' },
|
|
18
|
+
name: { type: 'string', description: '文件名' },
|
|
19
|
+
locator: { type: 'string', description: '给人看的定位:知识库 → 目录 → 文件' },
|
|
20
|
+
heading: { type: 'string', description: '所在章节(可能为空)' },
|
|
21
|
+
snippet: { type: 'string', description: '命中片段' },
|
|
22
|
+
charStart: { type: 'number', description: '片段在正文里的起始字符偏移,传给 kb_read 的 offset' },
|
|
23
|
+
page: { type: 'number', description: '页码(PDF/PPT),没有则为 0' },
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
function formatBytes(bytes) {
|
|
28
|
+
const value = Number(bytes) || 0;
|
|
29
|
+
if (value < 1024) return `${value} B`;
|
|
30
|
+
if (value < 1048576) return `${(value / 1024).toFixed(1)} KB`;
|
|
31
|
+
return `${(value / 1048576).toFixed(1)} MB`;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function createKnowledgeBaseTools({ store, index, pipeline }) {
|
|
35
|
+
// ── kb_list ────────────────────────────────────────────────────────────
|
|
36
|
+
const listTool = {
|
|
37
|
+
name: 'kb_list',
|
|
38
|
+
description:
|
|
39
|
+
'列出用户的「个人知识库」:有哪些库、每个库里的目录结构。'
|
|
40
|
+
+ '用户提到"知识库""我的资料""我之前存的那份文档"时,先调用它确认有什么,再用 kb_search 检索。',
|
|
41
|
+
parameters: {
|
|
42
|
+
type: 'object',
|
|
43
|
+
additionalProperties: false,
|
|
44
|
+
properties: {
|
|
45
|
+
kbId: { type: 'string', description: '只列这个知识库;省略则列出全部库的概览' },
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
output: {
|
|
49
|
+
schema: {
|
|
50
|
+
type: 'object',
|
|
51
|
+
additionalProperties: false,
|
|
52
|
+
required: ['summary', 'libraries'],
|
|
53
|
+
properties: {
|
|
54
|
+
summary: { type: 'string', description: '一句话概览' },
|
|
55
|
+
libraries: {
|
|
56
|
+
type: 'array',
|
|
57
|
+
description: '知识库清单',
|
|
58
|
+
items: {
|
|
59
|
+
type: 'object',
|
|
60
|
+
additionalProperties: false,
|
|
61
|
+
required: ['id', 'name', 'docs'],
|
|
62
|
+
properties: {
|
|
63
|
+
id: { type: 'string', description: '知识库 id,可传给 kb_search 的 kbIds' },
|
|
64
|
+
name: { type: 'string', description: '知识库名称' },
|
|
65
|
+
docs: { type: 'number', description: '文档数' },
|
|
66
|
+
tree: { type: 'string', description: '缩进的目录树文本(只在该库被指定时返回)' },
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
render: (_args, value) => [{ type: 'text', text: value.summary + (value.libraries.length ? `\n${value.libraries.map((item) => item.tree || `- ${item.name}(${item.docs} 篇)`).join('\n')}` : '') }],
|
|
73
|
+
},
|
|
74
|
+
async execute(args) {
|
|
75
|
+
const only = typeof args.kbId === 'string' ? args.kbId.trim() : '';
|
|
76
|
+
const libraries = store.listKbs().filter((kb) => !only || kb.id === only);
|
|
77
|
+
if (!libraries.length) {
|
|
78
|
+
return { summary: '还没有任何知识库(用户可以在左下角「知识库」页面新建)。', libraries: [] };
|
|
79
|
+
}
|
|
80
|
+
const out = libraries.map((kb) => {
|
|
81
|
+
const base = { id: kb.id, name: kb.name, docs: kb.docs, tree: '' };
|
|
82
|
+
if (!only) return base;
|
|
83
|
+
const nodes = store.listNodes(kb.id);
|
|
84
|
+
const byParent = new Map();
|
|
85
|
+
for (const node of nodes) {
|
|
86
|
+
const key = node.parentId ?? '';
|
|
87
|
+
if (!byParent.has(key)) byParent.set(key, []);
|
|
88
|
+
byParent.get(key).push(node);
|
|
89
|
+
}
|
|
90
|
+
const lines = [`- ${kb.name}(${kb.docs} 篇)`];
|
|
91
|
+
const walk = (parentKey, depth) => {
|
|
92
|
+
if (depth > 6) return;
|
|
93
|
+
for (const node of byParent.get(parentKey) ?? []) {
|
|
94
|
+
const mark = node.kind === 'folder' ? '📁' : '📄';
|
|
95
|
+
const size = node.kind === 'doc' ? ` ${formatBytes(node.bytes)}` : '';
|
|
96
|
+
const bad = node.status === 'failed' ? '(解析失败)' : '';
|
|
97
|
+
lines.push(`${' '.repeat(depth)}${mark} ${node.name}${size}${bad}`);
|
|
98
|
+
if (node.kind === 'folder') walk(node.id, depth + 1);
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
walk('', 1);
|
|
102
|
+
return { ...base, tree: lines.join('\n') };
|
|
103
|
+
});
|
|
104
|
+
const total = out.reduce((sum, item) => sum + item.docs, 0);
|
|
105
|
+
return { summary: `共 ${out.length} 个知识库、${total} 篇文档。`, libraries: out };
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
// ── kb_search ──────────────────────────────────────────────────────────
|
|
110
|
+
const searchTool = {
|
|
111
|
+
name: 'kb_search',
|
|
112
|
+
description:
|
|
113
|
+
'在用户的个人知识库里做全文检索(中文、英文、编号都行),返回命中片段与定位。'
|
|
114
|
+
+ '回答本地资料相关的问题前先用它;命中后可用 kb_read 读片段周围的正文。',
|
|
115
|
+
parameters: {
|
|
116
|
+
type: 'object',
|
|
117
|
+
additionalProperties: false,
|
|
118
|
+
required: ['query'],
|
|
119
|
+
properties: {
|
|
120
|
+
query: { type: 'string', description: '检索词,可以是关键词、短语或一句话' },
|
|
121
|
+
kbIds: {
|
|
122
|
+
type: 'array',
|
|
123
|
+
description: '只在这些知识库里搜(可选)',
|
|
124
|
+
items: { type: 'string' },
|
|
125
|
+
},
|
|
126
|
+
limit: { type: 'number', description: `最多返回几条(默认 8,上限 ${MAX_SEARCH_LIMIT})` },
|
|
127
|
+
mode: { type: 'string', enum: ['keyword', 'hybrid'], description: 'keyword=关键词;hybrid=关键词+向量(需要用户配置了向量服务)' },
|
|
128
|
+
},
|
|
129
|
+
},
|
|
130
|
+
output: {
|
|
131
|
+
schema: {
|
|
132
|
+
type: 'object',
|
|
133
|
+
additionalProperties: false,
|
|
134
|
+
required: ['summary', 'hits'],
|
|
135
|
+
properties: {
|
|
136
|
+
summary: { type: 'string', description: '一句话结论:命中几条、用了什么模式' },
|
|
137
|
+
hits: { type: 'array', description: '命中片段', items: HIT_SCHEMA },
|
|
138
|
+
},
|
|
139
|
+
},
|
|
140
|
+
render: (_args, value) => {
|
|
141
|
+
const lines = [value.summary];
|
|
142
|
+
value.hits.forEach((hit, index) => {
|
|
143
|
+
lines.push(`${index + 1}. ${hit.locator}${hit.heading ? ` · ${hit.heading}` : ''}${hit.page ? `(第 ${hit.page} 页)` : ''}`);
|
|
144
|
+
lines.push(` ${hit.snippet}`);
|
|
145
|
+
lines.push(` docId=${hit.docId} offset=${hit.charStart}`);
|
|
146
|
+
});
|
|
147
|
+
return [{ type: 'text', text: lines.join('\n') }];
|
|
148
|
+
},
|
|
149
|
+
},
|
|
150
|
+
async execute(args) {
|
|
151
|
+
const query = String(args.query ?? '').trim();
|
|
152
|
+
const limit = Math.min(MAX_SEARCH_LIMIT, Math.max(1, Number(args.limit) || 8));
|
|
153
|
+
const kbIds = Array.isArray(args.kbIds) ? args.kbIds.filter((item) => typeof item === 'string') : [];
|
|
154
|
+
if (!query) return { summary: '检索词为空。', hits: [] };
|
|
155
|
+
const result = await index.search({
|
|
156
|
+
query,
|
|
157
|
+
kbIds,
|
|
158
|
+
limit,
|
|
159
|
+
mode: args.mode === 'hybrid' ? 'hybrid' : 'keyword',
|
|
160
|
+
});
|
|
161
|
+
if (!result.hits.length) {
|
|
162
|
+
return {
|
|
163
|
+
summary: `在知识库里没有找到与「${query}」相关的内容(模式:${result.mode})。可以先 kb_list 看看有哪些库和文档。`,
|
|
164
|
+
hits: [],
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
return {
|
|
168
|
+
summary: `在知识库里命中 ${result.hits.length} 段与「${query}」相关的内容(模式:${result.mode})。`,
|
|
169
|
+
hits: result.hits.map((hit) => ({
|
|
170
|
+
docId: hit.nodeId,
|
|
171
|
+
name: hit.name,
|
|
172
|
+
locator: hit.locator,
|
|
173
|
+
heading: hit.heading || '',
|
|
174
|
+
snippet: hit.snippet,
|
|
175
|
+
charStart: hit.charStart,
|
|
176
|
+
page: hit.page ?? 0,
|
|
177
|
+
})),
|
|
178
|
+
};
|
|
179
|
+
},
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
// ── kb_read ────────────────────────────────────────────────────────────
|
|
183
|
+
const readTool = {
|
|
184
|
+
name: 'kb_read',
|
|
185
|
+
description:
|
|
186
|
+
'读取知识库里某篇文档的正文(按字符偏移分页,单次最多 6000 字)。'
|
|
187
|
+
+ '先用 kb_search 拿到 docId 和 charStart,再用 offset 读命中位置附近的内容。',
|
|
188
|
+
parameters: {
|
|
189
|
+
type: 'object',
|
|
190
|
+
additionalProperties: false,
|
|
191
|
+
required: ['docId'],
|
|
192
|
+
properties: {
|
|
193
|
+
docId: { type: 'string', description: 'kb_search / kb_list 返回的文档 id' },
|
|
194
|
+
offset: { type: 'number', description: '起始字符偏移,默认 0' },
|
|
195
|
+
limit: { type: 'number', description: `本次最多读多少字(默认 ${MAX_READ_CHARS},上限 ${MAX_READ_CHARS})` },
|
|
196
|
+
},
|
|
197
|
+
},
|
|
198
|
+
output: {
|
|
199
|
+
schema: {
|
|
200
|
+
type: 'object',
|
|
201
|
+
additionalProperties: false,
|
|
202
|
+
required: ['ok', 'summary', 'text'],
|
|
203
|
+
properties: {
|
|
204
|
+
ok: { type: 'boolean', description: '是否读取成功' },
|
|
205
|
+
summary: { type: 'string', description: '一句话说明:哪个文件的哪一段' },
|
|
206
|
+
text: { type: 'string', description: '正文片段' },
|
|
207
|
+
next: { type: 'number', description: '下一页的 offset;为 0 表示已读完' },
|
|
208
|
+
totalChars: { type: 'number', description: '全文总字数' },
|
|
209
|
+
},
|
|
210
|
+
},
|
|
211
|
+
render: (_args, value) => [{ type: 'text', text: `${value.summary}\n\n${value.text}${value.next ? `\n\n(未完,next offset=${value.next})` : ''}` }],
|
|
212
|
+
},
|
|
213
|
+
async execute(args) {
|
|
214
|
+
const docId = String(args.docId ?? '').trim();
|
|
215
|
+
const node = docId ? store.getNode(docId) : null;
|
|
216
|
+
if (!node || node.kind !== 'doc') {
|
|
217
|
+
return { ok: false, summary: `找不到文档 ${docId || '(空)'};先用 kb_search 检索。`, text: '', next: 0, totalChars: 0 };
|
|
218
|
+
}
|
|
219
|
+
if (node.status === 'failed') {
|
|
220
|
+
return { ok: false, summary: `《${node.name}》解析失败:${node.error || '未知原因'}`, text: '', next: 0, totalChars: 0 };
|
|
221
|
+
}
|
|
222
|
+
if (node.status !== 'ready') {
|
|
223
|
+
return { ok: false, summary: `《${node.name}》还在解析中,稍后再试。`, text: '', next: 0, totalChars: 0 };
|
|
224
|
+
}
|
|
225
|
+
const full = store.getText(node.id);
|
|
226
|
+
if (!full) {
|
|
227
|
+
return { ok: false, summary: `《${node.name}》没有可读的正文。`, text: '', next: 0, totalChars: 0 };
|
|
228
|
+
}
|
|
229
|
+
const offset = Math.max(0, Math.min(full.length, Math.trunc(Number(args.offset) || 0)));
|
|
230
|
+
const limit = Math.min(MAX_READ_CHARS, Math.max(200, Number(args.limit) || MAX_READ_CHARS));
|
|
231
|
+
const end = Math.min(full.length, offset + limit);
|
|
232
|
+
const text = full.slice(offset, end);
|
|
233
|
+
const next = end < full.length ? end : 0;
|
|
234
|
+
const crumb = store.breadcrumb(node.id);
|
|
235
|
+
return {
|
|
236
|
+
ok: true,
|
|
237
|
+
summary: `《${node.name}》(${crumb})第 ${offset}-${end} 字,共 ${full.length} 字。`,
|
|
238
|
+
text,
|
|
239
|
+
next,
|
|
240
|
+
totalChars: full.length,
|
|
241
|
+
};
|
|
242
|
+
},
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
return [listTool, searchTool, readTool];
|
|
246
|
+
}
|