dsh-shared-kb 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +241 -2
- package/cordis.patch.yml +11 -0
- package/lib/client.js +61 -0
- package/lib/core/atomic.js +87 -0
- package/lib/core/errors.js +39 -0
- package/lib/core/hubconfig.js +151 -0
- package/lib/core/paths.js +129 -0
- package/lib/icon.svg +7 -0
- package/lib/index.js +202 -0
- package/lib/indexer/embedder.js +325 -0
- package/lib/indexer/sqlite-index.js +431 -0
- package/lib/indexer/tokenize.js +151 -0
- package/lib/locale/zh-CN.json +4 -0
- package/lib/parse/chunk.js +202 -0
- package/lib/parse/parser.js +643 -0
- package/lib/search/search.js +194 -0
- package/lib/service/hub.js +791 -0
- package/lib/store/docs.js +906 -0
- package/lib/store/lease.js +251 -0
- package/lib/store/manifest.js +263 -0
- package/lib/tools/index.js +759 -0
- package/lib/ui/app.js +682 -0
- package/lib/ui/icon.svg +7 -0
- package/lib/ui/index.html +133 -0
- package/lib/ui/style.css +292 -0
- package/lib/web/api.js +225 -0
- package/lib/web/http.js +114 -0
- package/lib/web/server.js +149 -0
- package/package.json +83 -4
|
@@ -0,0 +1,791 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 服务层:把解析、存储、索引、检索串成一条可被工具与 HTTP 调用的链。
|
|
3
|
+
*
|
|
4
|
+
* 多人协作的核心约束都在这里落地:
|
|
5
|
+
*
|
|
6
|
+
* 1. 索引**只在本机**建。任何一台机器都能直接读共享盘上的资料并重建自己的
|
|
7
|
+
* 索引,彼此不干扰;索引坏了删掉重建,不影响别人。
|
|
8
|
+
*
|
|
9
|
+
* 2. 索引刷新是**增量**的,判据是 manifest 里每篇文档的 version。
|
|
10
|
+
* 别人在另一台机器上导了 3 篇,我这边第一次检索时只补这 3 篇,
|
|
11
|
+
* 不会重建全库(重建 900 篇要几十秒,增量只要几百毫秒)。
|
|
12
|
+
*
|
|
13
|
+
* 3. 写操作走 withManifest 的 CAS。两个人在同一秒改同一个库时,
|
|
14
|
+
* 后提交的那个会收到 REV_CONFLICT 而不是静默覆盖。
|
|
15
|
+
*
|
|
16
|
+
* 4. provider 身份(名字 + 维度)写进索引 meta。换模型后索引自动整体重建,
|
|
17
|
+
* 绝不把两套不同模型的向量混在一个索引里。
|
|
18
|
+
*/
|
|
19
|
+
import { mkdir, readdir, stat } from "node:fs/promises";
|
|
20
|
+
import { join } from "node:path";
|
|
21
|
+
import { HubError, badRequest, notFound } from "../core/errors.js";
|
|
22
|
+
import { ensureHub, heartbeat, listMachines, localIndexCacheRoot, readHub } from "../core/hubconfig.js";
|
|
23
|
+
import {
|
|
24
|
+
docDir,
|
|
25
|
+
docIdFor,
|
|
26
|
+
kbRootDir,
|
|
27
|
+
normalizeRelKey,
|
|
28
|
+
safeFileName,
|
|
29
|
+
sanitizeKbName,
|
|
30
|
+
sourcesDir,
|
|
31
|
+
} from "../core/paths.js";
|
|
32
|
+
import { packBlocks, outlineOf } from "../parse/chunk.js";
|
|
33
|
+
import { parseFile, renderPdfPage, SUPPORTED_EXTS } from "../parse/parser.js";
|
|
34
|
+
import { createBuiltinProvider, resolveProvider } from "../indexer/embedder.js";
|
|
35
|
+
import { indexFileInfo, indexPathFor, openIndex } from "../indexer/sqlite-index.js";
|
|
36
|
+
import { groupByDoc, search as runSearch } from "../search/search.js";
|
|
37
|
+
import {
|
|
38
|
+
createKb,
|
|
39
|
+
createSnapshot,
|
|
40
|
+
keepSourceCopy,
|
|
41
|
+
listKbs,
|
|
42
|
+
listNotes,
|
|
43
|
+
listPartFiles,
|
|
44
|
+
listSnapshots,
|
|
45
|
+
purgeDoc,
|
|
46
|
+
readDocMerged,
|
|
47
|
+
readNote,
|
|
48
|
+
readPartFromDisk,
|
|
49
|
+
refreshCatalog,
|
|
50
|
+
removeKb,
|
|
51
|
+
renameKb,
|
|
52
|
+
requireKb,
|
|
53
|
+
restoreDeletedDoc,
|
|
54
|
+
restoreDocVersion,
|
|
55
|
+
listDocHistory,
|
|
56
|
+
softDeleteDoc,
|
|
57
|
+
updateDocMeta,
|
|
58
|
+
updateKbMeta,
|
|
59
|
+
writeDoc,
|
|
60
|
+
writeNote,
|
|
61
|
+
} from "../store/docs.js";
|
|
62
|
+
import { appendEvent, readEvents, readManifest, withManifest } from "../store/manifest.js";
|
|
63
|
+
import { inspectLeases, pruneStaleLeases } from "../store/lease.js";
|
|
64
|
+
|
|
65
|
+
export const DEFAULT_CONFIG = Object.freeze({
|
|
66
|
+
kbRoot: "~/.dsh/shared-kb",
|
|
67
|
+
defaultKb: "default",
|
|
68
|
+
semanticProvider: "auto",
|
|
69
|
+
semanticModelDir: "",
|
|
70
|
+
remoteBaseUrl: "",
|
|
71
|
+
remoteApiKey: "",
|
|
72
|
+
remoteModel: "",
|
|
73
|
+
remoteDim: 1024,
|
|
74
|
+
builtinDim: 512,
|
|
75
|
+
partChars: 1200,
|
|
76
|
+
overlapChars: 150,
|
|
77
|
+
indexCacheRoot: "",
|
|
78
|
+
historyKeep: 10,
|
|
79
|
+
maxImportBytes: 512 * 1024 * 1024,
|
|
80
|
+
autoIndex: true,
|
|
81
|
+
logger: null,
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Hub 门面:一个实例对应一个 kbRoot。
|
|
86
|
+
* 用工厂函数而不是 class,方便测试注入。
|
|
87
|
+
*/
|
|
88
|
+
export async function createHub(options = {}) {
|
|
89
|
+
const config = { ...DEFAULT_CONFIG, ...options };
|
|
90
|
+
const root = config.root ?? config.kbRoot;
|
|
91
|
+
if (!root) throw badRequest("createHub 需要 root(知识库物理地址)");
|
|
92
|
+
const ensured = await ensureHub(root);
|
|
93
|
+
const indexCacheRoot = config.indexCacheRoot || localIndexCacheRoot(join(homeOf(), ".dsh", "shared-kb"), root);
|
|
94
|
+
await mkdir(indexCacheRoot, { recursive: true });
|
|
95
|
+
|
|
96
|
+
let providerPromise = null;
|
|
97
|
+
let providerInfo = { name: "builtin-hash-v1", dim: config.builtinDim, kind: "builtin", notes: [] };
|
|
98
|
+
|
|
99
|
+
/** 解析 embedding provider(懒加载 + 缓存探针结果)。 */
|
|
100
|
+
async function getProvider() {
|
|
101
|
+
if (!providerPromise) {
|
|
102
|
+
providerPromise = resolveProvider({
|
|
103
|
+
semanticProvider: config.semanticProvider,
|
|
104
|
+
semanticModelDir: config.semanticModelDir,
|
|
105
|
+
remoteBaseUrl: config.remoteBaseUrl,
|
|
106
|
+
remoteApiKey: config.remoteApiKey,
|
|
107
|
+
remoteModel: config.remoteModel,
|
|
108
|
+
remoteDim: config.remoteDim,
|
|
109
|
+
builtinDim: config.builtinDim,
|
|
110
|
+
logger: config.logger,
|
|
111
|
+
})
|
|
112
|
+
.then((resolved) => {
|
|
113
|
+
providerInfo = {
|
|
114
|
+
name: resolved.provider.name,
|
|
115
|
+
dim: resolved.probe?.dim || resolved.provider.dim || config.builtinDim,
|
|
116
|
+
kind: resolved.provider.kind,
|
|
117
|
+
notes: resolved.notes ?? [],
|
|
118
|
+
};
|
|
119
|
+
return resolved.provider;
|
|
120
|
+
})
|
|
121
|
+
.catch((error) => {
|
|
122
|
+
// 任何意外都退回 builtin:语义检索可以降级,检索本身不能没有
|
|
123
|
+
providerInfo = {
|
|
124
|
+
name: "builtin-hash-v1",
|
|
125
|
+
dim: config.builtinDim,
|
|
126
|
+
kind: "builtin",
|
|
127
|
+
notes: [`provider 初始化失败:${String(error?.message ?? error)}`],
|
|
128
|
+
};
|
|
129
|
+
return createBuiltinProvider({ dim: config.builtinDim });
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
return providerPromise;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const indexHandles = new Map();
|
|
136
|
+
async function getIndex(kb) {
|
|
137
|
+
if (indexHandles.has(kb)) return indexHandles.get(kb);
|
|
138
|
+
const provider = await getProvider();
|
|
139
|
+
const path = indexPathFor(indexCacheRoot, kb);
|
|
140
|
+
const handle = await openIndex(path, { provider: provider.name, dim: providerInfo.dim });
|
|
141
|
+
indexHandles.set(kb, handle);
|
|
142
|
+
return handle;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function closeIndexes() {
|
|
146
|
+
for (const handle of indexHandles.values()) handle.close();
|
|
147
|
+
indexHandles.clear();
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// -------------------------------------------------------------------------
|
|
151
|
+
// 索引同步
|
|
152
|
+
// -------------------------------------------------------------------------
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* 把索引与本机视角下的 manifest 对齐(增量)。
|
|
156
|
+
* @returns {Promise<{added: number, removed: number, total: number, rev: number, rebuilt: boolean}>}
|
|
157
|
+
*/
|
|
158
|
+
async function syncIndex(kb, { force = false } = {}) {
|
|
159
|
+
const { dir } = await requireKb(root, kb);
|
|
160
|
+
const manifest = await readManifest(dir, kb);
|
|
161
|
+
const index = await getIndex(kb);
|
|
162
|
+
const live = (manifest.docs ?? []).filter((doc) => !doc.deletedAt && doc.status === "ready");
|
|
163
|
+
const indexed = index.indexedVersions();
|
|
164
|
+
|
|
165
|
+
const toWrite = [];
|
|
166
|
+
const toRemove = [];
|
|
167
|
+
for (const doc of live) {
|
|
168
|
+
const known = indexed.get(doc.id);
|
|
169
|
+
if (force || known === undefined || Number(known) !== Number(doc.version)) toWrite.push(doc);
|
|
170
|
+
}
|
|
171
|
+
for (const docId of indexed.keys()) {
|
|
172
|
+
if (!live.some((doc) => doc.id === docId)) toRemove.push(docId);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
let rebuilt = false;
|
|
176
|
+
if (toWrite.length === 0 && toRemove.length === 0) {
|
|
177
|
+
return { added: 0, removed: 0, total: index.partCount(), rev: manifest.rev, rebuilt, unchanged: true };
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const provider = await getProvider();
|
|
181
|
+
const payloads = [];
|
|
182
|
+
for (const doc of toWrite) {
|
|
183
|
+
const parts = await loadPartsForIndex(dir, doc);
|
|
184
|
+
if (parts.length === 0) continue;
|
|
185
|
+
let vectors = [];
|
|
186
|
+
try {
|
|
187
|
+
vectors = await provider.embed(parts.map((part) => `${doc.title}\n${part.heading}\n${part.text}`));
|
|
188
|
+
} catch (error) {
|
|
189
|
+
// 向量失败不能阻断全文索引:语义是增强,不是前提
|
|
190
|
+
vectors = [];
|
|
191
|
+
config.logger?.warn?.(`[shared-kb] ${kb}/${doc.id} 向量化失败,仅建全文索引:${String(error?.message ?? error)}`);
|
|
192
|
+
}
|
|
193
|
+
payloads.push({
|
|
194
|
+
docId: doc.id,
|
|
195
|
+
version: doc.version,
|
|
196
|
+
title: doc.title,
|
|
197
|
+
updatedAt: doc.updatedAt ?? "",
|
|
198
|
+
chars: doc.chars ?? 0,
|
|
199
|
+
parts,
|
|
200
|
+
vectors,
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if (toRemove.length > 0) index.removeDocs(toRemove);
|
|
205
|
+
if (payloads.length > 0) index.upsertDocs(payloads, { manifestRev: manifest.rev, builtAt: new Date().toISOString() });
|
|
206
|
+
else index.upsertDocs([], { manifestRev: manifest.rev });
|
|
207
|
+
|
|
208
|
+
return {
|
|
209
|
+
added: payloads.length,
|
|
210
|
+
removed: toRemove.length,
|
|
211
|
+
total: index.partCount(),
|
|
212
|
+
rev: manifest.rev,
|
|
213
|
+
rebuilt,
|
|
214
|
+
skipped: toWrite.length - payloads.length,
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** 从磁盘重新解析一篇文档的所有分块(索引不存原文来源,只存分块)。 */
|
|
219
|
+
async function loadPartsForIndex(dir, doc) {
|
|
220
|
+
const merged = await readDocMerged(dir, doc.id);
|
|
221
|
+
if (merged === null) return [];
|
|
222
|
+
const indices = await listPartFiles(dir, doc.id);
|
|
223
|
+
if (indices.length > 0) {
|
|
224
|
+
const parts = [];
|
|
225
|
+
for (const index of indices) {
|
|
226
|
+
const text = await readPartFromDisk(dir, doc.id, index);
|
|
227
|
+
if (text === null || text === "") continue;
|
|
228
|
+
parts.push({ index, text, heading: "", page: null, chars: text.length });
|
|
229
|
+
}
|
|
230
|
+
if (parts.length > 0) {
|
|
231
|
+
// heading / page 从 meta 的 outline 补齐(拿不到就留空,不影响检索)
|
|
232
|
+
const outline = Array.isArray(doc.outline) ? doc.outline : [];
|
|
233
|
+
for (const part of parts) {
|
|
234
|
+
const nearest = [...outline].reverse().find((item) => item.part != null && item.part <= part.index);
|
|
235
|
+
if (nearest) part.heading = nearest.title ?? "";
|
|
236
|
+
}
|
|
237
|
+
return parts;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
// 没有分块文件(老数据 / 手工放入的资料):现场切一遍
|
|
241
|
+
const blocks = merged.split(/\n{2,}/).map((text) => ({ text, heading: "", level: 0 }));
|
|
242
|
+
return packBlocks(blocks, { partChars: config.partChars, overlapChars: config.overlapChars });
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// -------------------------------------------------------------------------
|
|
246
|
+
// 建库 / 列库
|
|
247
|
+
// -------------------------------------------------------------------------
|
|
248
|
+
|
|
249
|
+
async function ensureDefaultKb() {
|
|
250
|
+
const name = sanitizeKbName(config.defaultKb ?? "default");
|
|
251
|
+
const result = await createKb(root, name, { existOk: true, description: "默认知识库" });
|
|
252
|
+
return result;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
async function list(options = {}) {
|
|
256
|
+
await ensureDefaultKb();
|
|
257
|
+
const kbs = await listKbs(root);
|
|
258
|
+
if (!options.withMachines) return kbs;
|
|
259
|
+
return { kbs, machines: await listMachines(root) };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async function status() {
|
|
263
|
+
await heartbeat(root, { kbCount: (await listKbs(root)).length });
|
|
264
|
+
const hub = await readHub(root);
|
|
265
|
+
const machines = await listMachines(root);
|
|
266
|
+
const provider = await getProvider();
|
|
267
|
+
const kbs = await listKbs(root);
|
|
268
|
+
const indexes = [];
|
|
269
|
+
for (const kb of kbs) {
|
|
270
|
+
if (kb.broken) {
|
|
271
|
+
indexes.push({ kb: kb.name, broken: true });
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
const path = indexPathFor(indexCacheRoot, kb.name);
|
|
275
|
+
const info = await indexFileInfo(path);
|
|
276
|
+
let indexedDocs = null;
|
|
277
|
+
let indexedParts = null;
|
|
278
|
+
try {
|
|
279
|
+
const handle = await getIndex(kb.name);
|
|
280
|
+
indexedDocs = handle.docCount();
|
|
281
|
+
indexedParts = handle.partCount();
|
|
282
|
+
} catch (error) {
|
|
283
|
+
indexes.push({ kb: kb.name, ...info, error: String(error?.message ?? error) });
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
indexes.push({
|
|
287
|
+
kb: kb.name,
|
|
288
|
+
...info,
|
|
289
|
+
indexedDocs,
|
|
290
|
+
indexedParts,
|
|
291
|
+
manifestDocs: kb.docs,
|
|
292
|
+
manifestRev: kb.rev,
|
|
293
|
+
stale: indexedDocs !== kb.docs,
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
return {
|
|
297
|
+
root,
|
|
298
|
+
indexCacheRoot,
|
|
299
|
+
indexLocation: "local",
|
|
300
|
+
hub,
|
|
301
|
+
provider: { ...providerInfo, instance: provider.name },
|
|
302
|
+
machines,
|
|
303
|
+
kbs,
|
|
304
|
+
indexes,
|
|
305
|
+
leases: await inspectLeases(await kbDirForLeases(kbs, root)),
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
async function kbDirForLeases(kbs, base) {
|
|
310
|
+
const first = kbs.find((kb) => !kb.broken);
|
|
311
|
+
return first ? kbRootDir(base, first.name) : base;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// -------------------------------------------------------------------------
|
|
315
|
+
// 导入
|
|
316
|
+
// -------------------------------------------------------------------------
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* 导入一个文件或目录。
|
|
320
|
+
*
|
|
321
|
+
* @param {object} params
|
|
322
|
+
* @param {string} params.path 源文件或目录
|
|
323
|
+
* @param {string} [params.kb] 库名(默认 defaultKb)
|
|
324
|
+
* @param {boolean} [params.recursive] 目录是否递归(默认 true)
|
|
325
|
+
* @param {object} [params.meta] 额外的元数据(title/tags/trust...),单文件时生效
|
|
326
|
+
* @param {number} [params.expectedRev] 并发保护
|
|
327
|
+
*/
|
|
328
|
+
async function importPath(params) {
|
|
329
|
+
const kbName = sanitizeKbName(params.kb ?? config.defaultKb ?? "default");
|
|
330
|
+
await createKb(root, kbName, { existOk: true });
|
|
331
|
+
const { dir } = await requireKb(root, kbName);
|
|
332
|
+
const target = String(params.path ?? "").trim();
|
|
333
|
+
if (target === "") throw badRequest("path 不能为空");
|
|
334
|
+
|
|
335
|
+
const info = await stat(target).catch(() => null);
|
|
336
|
+
if (!info) throw notFound(`路径 ${target}`);
|
|
337
|
+
const files = [];
|
|
338
|
+
if (info.isFile()) {
|
|
339
|
+
files.push({ path: target, relKey: safeFileName(target) });
|
|
340
|
+
} else if (info.isDirectory()) {
|
|
341
|
+
const recursive = params.recursive !== false;
|
|
342
|
+
const collected = await collectFiles(target, { recursive });
|
|
343
|
+
for (const filePath of collected) {
|
|
344
|
+
const rel = normalizeRelKey(filePath.slice(target.length));
|
|
345
|
+
files.push({ path: filePath, relKey: rel || safeFileName(filePath) });
|
|
346
|
+
}
|
|
347
|
+
} else {
|
|
348
|
+
throw badRequest("path 既不是文件也不是目录");
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
if (files.length === 0) {
|
|
352
|
+
return { kb: kbName, imported: 0, skipped: 0, failed: 0, results: [], note: "目录下没有受支持的文件" };
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
const results = [];
|
|
356
|
+
let imported = 0;
|
|
357
|
+
let skipped = 0;
|
|
358
|
+
let failed = 0;
|
|
359
|
+
|
|
360
|
+
for (const file of files) {
|
|
361
|
+
try {
|
|
362
|
+
const size = (await stat(file.path)).size;
|
|
363
|
+
if (size > config.maxImportBytes) {
|
|
364
|
+
results.push({ relKey: file.relKey, status: "skipped", reason: `文件超过上限 ${config.maxImportBytes} 字节` });
|
|
365
|
+
skipped += 1;
|
|
366
|
+
continue;
|
|
367
|
+
}
|
|
368
|
+
const parsed = await parseFile(file.path);
|
|
369
|
+
const parts = packBlocks(
|
|
370
|
+
parsed.blocks.map((block) => ({
|
|
371
|
+
...block,
|
|
372
|
+
page: block.page ?? null,
|
|
373
|
+
})),
|
|
374
|
+
{ partChars: config.partChars, overlapChars: config.overlapChars }
|
|
375
|
+
);
|
|
376
|
+
if (parts.length === 0) {
|
|
377
|
+
results.push({ relKey: file.relKey, status: "skipped", reason: "没有解析出任何正文(可能是空白或纯图片文档)" });
|
|
378
|
+
skipped += 1;
|
|
379
|
+
continue;
|
|
380
|
+
}
|
|
381
|
+
const title = params.meta?.title && files.length === 1 ? params.meta.title : defaultTitle(file.relKey);
|
|
382
|
+
// 原始文件留一份:库要能整包搬走,不能依赖导入时那台机器的路径
|
|
383
|
+
await keepSourceCopy(dir, file.path, file.relKey);
|
|
384
|
+
const written = await writeDoc({
|
|
385
|
+
dir,
|
|
386
|
+
kb: kbName,
|
|
387
|
+
relKey: file.relKey,
|
|
388
|
+
parsed: { ...parsed, outline: outlineOf(parts) },
|
|
389
|
+
parts,
|
|
390
|
+
meta: { ...(files.length === 1 ? params.meta ?? {} : {}), title },
|
|
391
|
+
expectedRev: params.expectedRev,
|
|
392
|
+
sourcePath: file.path,
|
|
393
|
+
historyKeep: config.historyKeep,
|
|
394
|
+
});
|
|
395
|
+
if (written.status === "unchanged") skipped += 1;
|
|
396
|
+
else imported += 1;
|
|
397
|
+
results.push({ relKey: file.relKey, ...written, warnings: parsed.warnings });
|
|
398
|
+
} catch (error) {
|
|
399
|
+
failed += 1;
|
|
400
|
+
results.push({
|
|
401
|
+
relKey: file.relKey,
|
|
402
|
+
status: "failed",
|
|
403
|
+
reason: String(error?.message ?? error),
|
|
404
|
+
code: error?.code ?? "INTERNAL",
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
await refreshCatalog(dir, kbName);
|
|
410
|
+
let index = null;
|
|
411
|
+
if (config.autoIndex && imported > 0) {
|
|
412
|
+
index = await syncIndex(kbName).catch((error) => ({ error: String(error?.message ?? error) }));
|
|
413
|
+
}
|
|
414
|
+
return { kb: kbName, imported, skipped, failed, total: files.length, results, index };
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
async function collectFiles(dir, { recursive, out = [] } = {}) {
|
|
418
|
+
let entries = [];
|
|
419
|
+
try {
|
|
420
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
421
|
+
} catch {
|
|
422
|
+
return out;
|
|
423
|
+
}
|
|
424
|
+
for (const entry of entries) {
|
|
425
|
+
if (entry.name.startsWith(".")) continue;
|
|
426
|
+
const full = join(dir, entry.name);
|
|
427
|
+
if (entry.isDirectory()) {
|
|
428
|
+
if (recursive) await collectFiles(full, { recursive, out });
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
if (!entry.isFile()) continue;
|
|
432
|
+
const ext = entry.name.slice(entry.name.lastIndexOf(".")).toLowerCase();
|
|
433
|
+
if (!SUPPORTED_EXTS.includes(ext)) continue;
|
|
434
|
+
out.push(full);
|
|
435
|
+
}
|
|
436
|
+
out.sort();
|
|
437
|
+
return out;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// -------------------------------------------------------------------------
|
|
441
|
+
// 检索
|
|
442
|
+
// -------------------------------------------------------------------------
|
|
443
|
+
|
|
444
|
+
async function query(params) {
|
|
445
|
+
const kbName = sanitizeKbName(params.kb ?? config.defaultKb ?? "default");
|
|
446
|
+
// 检索绝不隐式建库:库名打错时应该立刻报 NOT_FOUND,
|
|
447
|
+
// 而不是悄悄建一个空库然后返回「没有命中」,让人以为资料丢了。
|
|
448
|
+
await requireKb(root, kbName);
|
|
449
|
+
const text = String(params.query ?? "").trim();
|
|
450
|
+
if (text === "") throw badRequest("query 不能为空");
|
|
451
|
+
|
|
452
|
+
// 检索前自动增量同步:别人在别的机器上加的资料,这里第一次问就能看到
|
|
453
|
+
let sync = null;
|
|
454
|
+
if (config.autoIndex) {
|
|
455
|
+
sync = await syncIndex(kbName).catch((error) => ({ error: String(error?.message ?? error) }));
|
|
456
|
+
}
|
|
457
|
+
const provider = await getProvider();
|
|
458
|
+
const index = await getIndex(kbName);
|
|
459
|
+
const { hits, diagnostics } = await runSearch({
|
|
460
|
+
index,
|
|
461
|
+
query: text,
|
|
462
|
+
embedder: provider,
|
|
463
|
+
options: {
|
|
464
|
+
mode: params.mode,
|
|
465
|
+
limit: params.limit,
|
|
466
|
+
minVectorScore: params.minVectorScore,
|
|
467
|
+
},
|
|
468
|
+
});
|
|
469
|
+
|
|
470
|
+
const { dir } = await requireKb(root, kbName);
|
|
471
|
+
const manifest = await readManifest(dir, kbName);
|
|
472
|
+
const byId = new Map((manifest.docs ?? []).map((doc) => [doc.id, doc]));
|
|
473
|
+
// 过滤器:被软删的文档、以及清单里已经不存在的文档,都不能出现在检索结果里。
|
|
474
|
+
// 索引是异步增量刷的,删除后到下一次 sync 之间会有窗口期 —— 靠这一层兜住,
|
|
475
|
+
// 而不是要求调用方「删完记得重建索引」。
|
|
476
|
+
const enriched = hits
|
|
477
|
+
.filter((hit) => {
|
|
478
|
+
const doc = byId.get(hit.docId);
|
|
479
|
+
if (!doc) return false; // 清单里没有 = 已彻底删除
|
|
480
|
+
return !doc.deletedAt && doc.status === "ready";
|
|
481
|
+
})
|
|
482
|
+
.map((hit) => {
|
|
483
|
+
const doc = byId.get(hit.docId);
|
|
484
|
+
return {
|
|
485
|
+
...hit,
|
|
486
|
+
title: doc.title,
|
|
487
|
+
standardNo: doc.standardNo ?? "",
|
|
488
|
+
trust: doc.trust ?? "unknown",
|
|
489
|
+
tags: doc.tags ?? [],
|
|
490
|
+
type: doc.type ?? "",
|
|
491
|
+
status: "ready",
|
|
492
|
+
};
|
|
493
|
+
});
|
|
494
|
+
const grouped = params.groupByDoc === false ? [] : groupByDoc(enriched).map((group) => {
|
|
495
|
+
const doc = byId.get(group.docId);
|
|
496
|
+
return { ...group, title: doc?.title ?? group.title, standardNo: doc?.standardNo ?? "" };
|
|
497
|
+
});
|
|
498
|
+
|
|
499
|
+
return {
|
|
500
|
+
kb: kbName,
|
|
501
|
+
query: text,
|
|
502
|
+
total: enriched.length,
|
|
503
|
+
hits: enriched,
|
|
504
|
+
docs: grouped,
|
|
505
|
+
diagnostics: { ...diagnostics, sync, provider: providerInfo },
|
|
506
|
+
};
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
// -------------------------------------------------------------------------
|
|
510
|
+
// 精读
|
|
511
|
+
// -------------------------------------------------------------------------
|
|
512
|
+
|
|
513
|
+
async function readDocContent(params) {
|
|
514
|
+
const kbName = sanitizeKbName(params.kb ?? config.defaultKb ?? "default");
|
|
515
|
+
const { dir } = await requireKb(root, kbName);
|
|
516
|
+
const manifest = await readManifest(dir, kbName);
|
|
517
|
+
const doc = resolveDocRef(manifest, params.doc);
|
|
518
|
+
if (!doc) throw notFound(`文档 ${params.doc}`);
|
|
519
|
+
|
|
520
|
+
if (params.note) {
|
|
521
|
+
const content = await readNote(dir, doc.id);
|
|
522
|
+
if (content === null) throw notFound(`笔记 ${doc.id}`);
|
|
523
|
+
return { kb: kbName, docId: doc.id, title: doc.title, kind: "note", content, chars: content.length };
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
if (params.part === "all") {
|
|
527
|
+
const merged = (await readDocMerged(dir, doc.id)) ?? "";
|
|
528
|
+
const limit = Math.max(500, Number(params.maxChars) || 20000);
|
|
529
|
+
return {
|
|
530
|
+
kb: kbName,
|
|
531
|
+
docId: doc.id,
|
|
532
|
+
title: doc.title,
|
|
533
|
+
part: "all",
|
|
534
|
+
content: merged.slice(0, limit),
|
|
535
|
+
chars: merged.length,
|
|
536
|
+
truncated: merged.length > limit,
|
|
537
|
+
outline: doc.outline ?? [],
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
if (params.part === "outline") {
|
|
542
|
+
return {
|
|
543
|
+
kb: kbName,
|
|
544
|
+
docId: doc.id,
|
|
545
|
+
title: doc.title,
|
|
546
|
+
part: "outline",
|
|
547
|
+
outline: doc.outline ?? [],
|
|
548
|
+
parts: doc.parts ?? (await listPartFiles(dir, doc.id)).length,
|
|
549
|
+
pages: doc.pages ?? null,
|
|
550
|
+
warnings: doc.warnings ?? [],
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
const index = Math.max(1, Math.round(Number(params.part) || 1));
|
|
555
|
+
const fromDisk = await readPartFromDisk(dir, doc.id, index);
|
|
556
|
+
if (fromDisk === null) {
|
|
557
|
+
throw notFound(`文档 ${doc.title} 的第 ${index} 块(共 ${doc.parts ?? "?"} 块)`);
|
|
558
|
+
}
|
|
559
|
+
return {
|
|
560
|
+
kb: kbName,
|
|
561
|
+
docId: doc.id,
|
|
562
|
+
title: doc.title,
|
|
563
|
+
part: index,
|
|
564
|
+
total: doc.parts ?? (await listPartFiles(dir, doc.id)).length,
|
|
565
|
+
content: fromDisk,
|
|
566
|
+
chars: fromDisk.length,
|
|
567
|
+
warnings: doc.warnings ?? [],
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
function resolveDocRef(manifest, ref) {
|
|
572
|
+
const key = String(ref ?? "").trim();
|
|
573
|
+
if (key === "") return null;
|
|
574
|
+
const docs = manifest.docs ?? [];
|
|
575
|
+
const byId = docs.find((doc) => doc.id === key);
|
|
576
|
+
if (byId) return byId;
|
|
577
|
+
const lowered = key.toLowerCase();
|
|
578
|
+
const exact = docs.find((doc) => String(doc.title ?? "").toLowerCase() === lowered);
|
|
579
|
+
if (exact) return exact;
|
|
580
|
+
const partial = docs.filter((doc) => String(doc.title ?? "").toLowerCase().includes(lowered));
|
|
581
|
+
if (partial.length === 1) return partial[0];
|
|
582
|
+
if (partial.length > 1) {
|
|
583
|
+
throw new HubError("DOC_AMBIGUOUS", `标题 "${key}" 匹配到 ${partial.length} 篇文档,请用更精确的标题或 docId`, {
|
|
584
|
+
status: 409,
|
|
585
|
+
details: { candidates: partial.slice(0, 10).map((doc) => ({ id: doc.id, title: doc.title })) },
|
|
586
|
+
});
|
|
587
|
+
}
|
|
588
|
+
return null;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
// -------------------------------------------------------------------------
|
|
592
|
+
// 维护
|
|
593
|
+
// -------------------------------------------------------------------------
|
|
594
|
+
|
|
595
|
+
async function rebuildIndex(kb) {
|
|
596
|
+
const kbName = sanitizeKbName(kb ?? config.defaultKb);
|
|
597
|
+
await requireKb(root, kbName);
|
|
598
|
+
const handle = indexHandles.get(kbName);
|
|
599
|
+
if (handle) {
|
|
600
|
+
handle.close();
|
|
601
|
+
indexHandles.delete(kbName);
|
|
602
|
+
}
|
|
603
|
+
const result = await syncIndex(kbName, { force: true });
|
|
604
|
+
return { kb: kbName, ...result };
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/** 列出与检索相关的诊断信息(给 skb_status / 工作台用)。 */
|
|
608
|
+
async function inspect(kb) {
|
|
609
|
+
const kbName = sanitizeKbName(kb ?? config.defaultKb);
|
|
610
|
+
const { dir, meta } = await requireKb(root, kbName);
|
|
611
|
+
const manifest = await readManifest(dir, kbName);
|
|
612
|
+
const paths = {
|
|
613
|
+
root,
|
|
614
|
+
kbDir: dir,
|
|
615
|
+
sources: sourcesDir(dir),
|
|
616
|
+
markdown: join(dir, "markdown"),
|
|
617
|
+
notes: join(dir, "notes"),
|
|
618
|
+
manifest: join(dir, "manifest.json"),
|
|
619
|
+
events: join(dir, "events.jsonl"),
|
|
620
|
+
schema: join(dir, "schema.md"),
|
|
621
|
+
index: indexPathFor(indexCacheRoot, kbName),
|
|
622
|
+
};
|
|
623
|
+
return {
|
|
624
|
+
kb: kbName,
|
|
625
|
+
meta,
|
|
626
|
+
rev: manifest.rev,
|
|
627
|
+
docs: manifest.stats,
|
|
628
|
+
paths,
|
|
629
|
+
leases: await inspectLeases(dir),
|
|
630
|
+
history: await listDocHistory(dir, (manifest.docs ?? [])[0]?.id ?? "").catch(() => []),
|
|
631
|
+
};
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
return {
|
|
635
|
+
config,
|
|
636
|
+
root,
|
|
637
|
+
indexCacheRoot,
|
|
638
|
+
// 库
|
|
639
|
+
list,
|
|
640
|
+
status,
|
|
641
|
+
createKb: (name, opts) => createKb(root, name, opts),
|
|
642
|
+
removeKb: (name, opts) => removeKb(root, name, opts),
|
|
643
|
+
renameKb: (name, next) => renameKb(root, name, next),
|
|
644
|
+
updateKbMeta: (name, patch) => updateKbMeta(root, name, patch),
|
|
645
|
+
// 导入
|
|
646
|
+
importPath,
|
|
647
|
+
// 检索
|
|
648
|
+
query,
|
|
649
|
+
readDoc: readDocContent,
|
|
650
|
+
// 写
|
|
651
|
+
writeNote: async (params) => {
|
|
652
|
+
const kbName = sanitizeKbName(params.kb ?? config.defaultKb);
|
|
653
|
+
await createKb(root, kbName, { existOk: true });
|
|
654
|
+
const { dir } = await requireKb(root, kbName);
|
|
655
|
+
const result = await writeNote(dir, kbName, params);
|
|
656
|
+
await refreshCatalog(dir, kbName);
|
|
657
|
+
return result;
|
|
658
|
+
},
|
|
659
|
+
listNotes: async (kb) => {
|
|
660
|
+
const { dir } = await requireKb(root, sanitizeKbName(kb ?? config.defaultKb));
|
|
661
|
+
return listNotes(dir);
|
|
662
|
+
},
|
|
663
|
+
updateDocMeta: (kb, docId, patch, opts) => updateDocMeta(root, kb, docId, patch, opts),
|
|
664
|
+
// 删除/恢复都顺手同步索引:查询侧虽然有过滤器兜底,但索引里留着
|
|
665
|
+
// 已删文档的向量会让每次检索多扫无用的行,也让 status 报的数字对不上。
|
|
666
|
+
softDelete: async (kb, docId, opts) => {
|
|
667
|
+
const result = await softDeleteDoc(root, kb, docId, opts);
|
|
668
|
+
await syncIndex(sanitizeKbName(kb)).catch(() => {});
|
|
669
|
+
return result;
|
|
670
|
+
},
|
|
671
|
+
undelete: async (kb, docId, opts) => {
|
|
672
|
+
const result = await restoreDeletedDoc(root, kb, docId, opts);
|
|
673
|
+
await syncIndex(sanitizeKbName(kb)).catch(() => {});
|
|
674
|
+
return result;
|
|
675
|
+
},
|
|
676
|
+
purge: async (kb, docId, opts) => {
|
|
677
|
+
const result = await purgeDoc(root, kb, docId, opts);
|
|
678
|
+
await syncIndex(sanitizeKbName(kb)).catch(() => {});
|
|
679
|
+
return result;
|
|
680
|
+
},
|
|
681
|
+
docHistory: async (kb, docId) => {
|
|
682
|
+
const { dir } = await requireKb(root, sanitizeKbName(kb ?? config.defaultKb));
|
|
683
|
+
return listDocHistory(dir, docId);
|
|
684
|
+
},
|
|
685
|
+
restoreVersion: async (kb, docId, historyName) => {
|
|
686
|
+
const kbName = sanitizeKbName(kb ?? config.defaultKb);
|
|
687
|
+
const { dir } = await requireKb(root, kbName);
|
|
688
|
+
const result = await restoreDocVersion(dir, kbName, docId, historyName);
|
|
689
|
+
await syncIndex(kbName, { force: true }).catch(() => {});
|
|
690
|
+
return result;
|
|
691
|
+
},
|
|
692
|
+
// 清单/文档查询
|
|
693
|
+
docs: async (kb, options = {}) => {
|
|
694
|
+
const { dir } = await requireKb(root, sanitizeKbName(kb ?? config.defaultKb));
|
|
695
|
+
const manifest = await readManifest(dir, sanitizeKbName(kb ?? config.defaultKb));
|
|
696
|
+
let docs = manifest.docs ?? [];
|
|
697
|
+
if (!options.includeDeleted) docs = docs.filter((doc) => !doc.deletedAt && doc.status !== "deleted");
|
|
698
|
+
if (options.status) docs = docs.filter((doc) => doc.status === options.status);
|
|
699
|
+
if (options.kind) {
|
|
700
|
+
docs = options.kind === "note" ? docs.filter((doc) => doc.type === "note") : docs.filter((doc) => doc.type !== "note");
|
|
701
|
+
}
|
|
702
|
+
if (options.query) {
|
|
703
|
+
const q = String(options.query).toLowerCase();
|
|
704
|
+
docs = docs.filter((doc) => String(doc.title ?? "").toLowerCase().includes(q));
|
|
705
|
+
}
|
|
706
|
+
if (options.trust) docs = docs.filter((doc) => (doc.trust ?? "unknown") === options.trust);
|
|
707
|
+
docs = [...docs].sort((a, b) => String(a.title).localeCompare(String(b.title), "zh-Hans-CN"));
|
|
708
|
+
const offset = Math.max(0, Number(options.offset) || 0);
|
|
709
|
+
const limit = Math.max(1, Math.min(500, Number(options.limit) || 50));
|
|
710
|
+
return {
|
|
711
|
+
kb: sanitizeKbName(kb ?? config.defaultKb),
|
|
712
|
+
rev: manifest.rev,
|
|
713
|
+
total: docs.length,
|
|
714
|
+
offset,
|
|
715
|
+
limit,
|
|
716
|
+
hasMore: offset + limit < docs.length,
|
|
717
|
+
docs: docs.slice(offset, offset + limit),
|
|
718
|
+
};
|
|
719
|
+
},
|
|
720
|
+
// 索引
|
|
721
|
+
syncIndex,
|
|
722
|
+
rebuildIndex,
|
|
723
|
+
inspect,
|
|
724
|
+
// 快照
|
|
725
|
+
snapshot: async (kb, opts) => {
|
|
726
|
+
const kbName = sanitizeKbName(kb ?? config.defaultKb);
|
|
727
|
+
const { dir } = await requireKb(root, kbName);
|
|
728
|
+
return createSnapshot(dir, kbName, opts);
|
|
729
|
+
},
|
|
730
|
+
snapshots: async (kb) => {
|
|
731
|
+
const { dir } = await requireKb(root, sanitizeKbName(kb ?? config.defaultKb));
|
|
732
|
+
return listSnapshots(dir);
|
|
733
|
+
},
|
|
734
|
+
// 审计
|
|
735
|
+
events: async (kb, opts) => {
|
|
736
|
+
const { dir } = await requireKb(root, sanitizeKbName(kb ?? config.defaultKb));
|
|
737
|
+
return readEvents(dir, opts);
|
|
738
|
+
},
|
|
739
|
+
catalog: async (kb) => {
|
|
740
|
+
const kbName = sanitizeKbName(kb ?? config.defaultKb);
|
|
741
|
+
const { dir } = await requireKb(root, kbName);
|
|
742
|
+
return refreshCatalog(dir, kbName);
|
|
743
|
+
},
|
|
744
|
+
// 锁
|
|
745
|
+
leases: async (kb) => {
|
|
746
|
+
const { dir } = await requireKb(root, sanitizeKbName(kb ?? config.defaultKb));
|
|
747
|
+
return inspectLeases(dir);
|
|
748
|
+
},
|
|
749
|
+
pruneLeases: async (kb) => {
|
|
750
|
+
const { dir } = await requireKb(root, sanitizeKbName(kb ?? config.defaultKb));
|
|
751
|
+
return pruneStaleLeases(dir);
|
|
752
|
+
},
|
|
753
|
+
// 原件
|
|
754
|
+
renderPage: async (kb, docRef, pageNo, opts) => {
|
|
755
|
+
const kbName = sanitizeKbName(kb ?? config.defaultKb);
|
|
756
|
+
const { dir } = await requireKb(root, kbName);
|
|
757
|
+
const manifest = await readManifest(dir, kbName);
|
|
758
|
+
const doc = resolveDocRef(manifest, docRef);
|
|
759
|
+
if (!doc) throw notFound(`文档 ${docRef}`);
|
|
760
|
+
if ((doc.type ?? "") !== "pdf") throw badRequest(`只有 PDF 支持按页渲染(当前类型 ${doc.type})`);
|
|
761
|
+
const sourcePath = join(sourcesDir(dir), doc.relKey ?? "");
|
|
762
|
+
await stat(sourcePath).catch(() => {
|
|
763
|
+
throw notFound(`原始文件不在库内:${doc.relKey}(可能导入时未保留副本)`);
|
|
764
|
+
});
|
|
765
|
+
return renderPdfPage(sourcePath, pageNo, opts);
|
|
766
|
+
},
|
|
767
|
+
provider: getProvider,
|
|
768
|
+
providerInfo: () => providerInfo,
|
|
769
|
+
heartbeat: () => heartbeat(root),
|
|
770
|
+
machines: () => listMachines(root),
|
|
771
|
+
close: closeIndexes,
|
|
772
|
+
appendEvent: (event) => appendEvent(kbRootDir(root, event.kb ?? config.defaultKb), event),
|
|
773
|
+
withManifest,
|
|
774
|
+
};
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
function homeOf() {
|
|
778
|
+
try {
|
|
779
|
+
return process.env.USERPROFILE || process.env.HOME || process.cwd();
|
|
780
|
+
} catch {
|
|
781
|
+
return process.cwd();
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
/** 从 relKey 推标题:去扩展名 + 把下划线换空格。 */
|
|
786
|
+
export function defaultTitle(relKey) {
|
|
787
|
+
const base = String(relKey ?? "").split("/").pop() ?? "";
|
|
788
|
+
const dot = base.lastIndexOf(".");
|
|
789
|
+
const stem = dot > 0 ? base.slice(0, dot) : base;
|
|
790
|
+
return stem.replace(/[_]+/g, " ").trim() || "未命名文档";
|
|
791
|
+
}
|