@dsh-libing/knowledge-base 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/README.md +4 -0
- package/lib/agent-access.d.ts +23 -0
- package/lib/agent-access.js +28 -0
- package/lib/chunks.js +27 -5
- package/lib/client.js +9 -9
- package/lib/config.js +9 -1
- package/lib/document/convert.js +52 -16
- package/lib/document/pdf-layout.js +45 -10
- package/lib/document/pdf-session.js +10 -4
- package/lib/document/pdf-vision.js +8 -3
- package/lib/document/pdf-worker.js +76 -21
- package/lib/document/pdf.js +39 -9
- package/lib/document/zip.js +14 -6
- package/lib/figures/image-client.js +9 -4
- package/lib/figures/image-worker.js +15 -6
- package/lib/figures/process.js +13 -3
- package/lib/index.js +6 -2
- package/lib/indexes.js +8 -4
- package/lib/job-storage.js +6 -3
- package/lib/jobs.js +18 -9
- package/lib/models.js +7 -2
- package/lib/retrieval.js +36 -9
- package/lib/routes.js +58 -15
- package/lib/runtime.js +9 -3
- package/lib/skills.js +6 -3
- package/lib/store.js +29 -14
- package/lib/tools.js +12 -4
- package/lib/util.js +2 -1
- package/package.json +1 -1
package/lib/store.js
CHANGED
|
@@ -32,7 +32,8 @@ export class Store {
|
|
|
32
32
|
}
|
|
33
33
|
async trash(kbId, expectedName) {
|
|
34
34
|
return this.serialized(async () => {
|
|
35
|
-
const registry = await readJson(join(this.root, 'registry.json'), registrySchema)
|
|
35
|
+
const registry = await readJson(join(this.root, 'registry.json'), registrySchema);
|
|
36
|
+
const kb = registry.kbs.find(k => k.kbId === id(kbId) && !k.deletedAt);
|
|
36
37
|
if (!kb)
|
|
37
38
|
throw new Error('知识库不存在,请刷新');
|
|
38
39
|
if (kb.name !== expectedName)
|
|
@@ -46,7 +47,8 @@ export class Store {
|
|
|
46
47
|
}
|
|
47
48
|
async restore(kbId) {
|
|
48
49
|
return this.serialized(async () => {
|
|
49
|
-
const registry = await readJson(join(this.root, 'registry.json'), registrySchema)
|
|
50
|
+
const registry = await readJson(join(this.root, 'registry.json'), registrySchema);
|
|
51
|
+
const kb = registry.kbs.find(k => k.kbId === id(kbId) && k.deletedAt);
|
|
50
52
|
if (!kb)
|
|
51
53
|
throw new Error('知识库不在回收站,请刷新');
|
|
52
54
|
if (kb.purge)
|
|
@@ -80,13 +82,15 @@ export class Store {
|
|
|
80
82
|
if (confirmed !== true)
|
|
81
83
|
throw new Error('必须明确确认彻底删除不可恢复');
|
|
82
84
|
return this.serialized(async () => {
|
|
83
|
-
const registry = await readJson(join(this.root, 'registry.json'), registrySchema)
|
|
85
|
+
const registry = await readJson(join(this.root, 'registry.json'), registrySchema);
|
|
86
|
+
const kb = registry.kbs.find(k => k.kbId === id(kbId) && k.deletedAt);
|
|
84
87
|
if (!kb)
|
|
85
88
|
throw new Error('只能彻底删除回收站中的知识库');
|
|
86
89
|
if (kb.name !== expectedName)
|
|
87
90
|
throw new Error('知识库名称已变化,请重新确认彻底删除');
|
|
88
91
|
const targets = [...new Set([...(kb.purge?.jobIds || []), ...jobIds])];
|
|
89
|
-
const kbPath = await this.purgeTarget('kbs', kb.kbId)
|
|
92
|
+
const kbPath = await this.purgeTarget('kbs', kb.kbId);
|
|
93
|
+
const jobPaths = await Promise.all(targets.map(key => this.purgeTarget('jobs', key)));
|
|
90
94
|
// Persist intent before removing files: a crash or partial failure must not allow restoration.
|
|
91
95
|
kb.purge = { startedAt: kb.purge?.startedAt || now(), jobIds: targets };
|
|
92
96
|
await atomicWrite(join(this.root, 'registry.json'), registry);
|
|
@@ -110,7 +114,8 @@ export class Store {
|
|
|
110
114
|
});
|
|
111
115
|
}
|
|
112
116
|
async kb(selector) {
|
|
113
|
-
const kbs = await this.list()
|
|
117
|
+
const kbs = await this.list();
|
|
118
|
+
const matches = kbs.filter(k => k.kbId === selector || k.name === selector);
|
|
114
119
|
if (matches.length !== 1)
|
|
115
120
|
throw new Error(matches.length ? '知识库名称重复,请使用 kbId' : '知识库不存在,请先创建或选择');
|
|
116
121
|
await safePath(this.root, join('kbs', id(required(matches[0]).kbId)), true);
|
|
@@ -136,7 +141,8 @@ export class Store {
|
|
|
136
141
|
kbRoot(kbId) { return join(this.root, 'kbs', id(kbId, 'kbId')); }
|
|
137
142
|
async inspect(kbId, revision) { return this.inspectKnown(await this.kb(kbId), revision); }
|
|
138
143
|
async inspectKnown(kb, revision) {
|
|
139
|
-
const kbPath = await safePath(this.root, join('kbs', kb.kbId))
|
|
144
|
+
const kbPath = await safePath(this.root, join('kbs', kb.kbId));
|
|
145
|
+
const current = await this.current(kb.kbId);
|
|
140
146
|
if (!revision && !current)
|
|
141
147
|
return { location: { kbPath, revision: null, wikiPath: null, indexPath: null }, summary: { revision: null, documentCount: 0, chunkCount: 0, coverageIncomplete: false } };
|
|
142
148
|
const snapshot = await this.revisionKnown(kb, revision || required(current).revision);
|
|
@@ -166,7 +172,8 @@ export class Store {
|
|
|
166
172
|
filename = text(basename(filename), '文件名', 200);
|
|
167
173
|
if (!/\.(?:docx?|pdf)$/i.test(filename) || data.length > this.config.maxInputBytes)
|
|
168
174
|
throw new Error('只支持大小限制内的 .docx / .doc / .pdf 文件');
|
|
169
|
-
const receiptId = `u-${randomUUID()}
|
|
175
|
+
const receiptId = `u-${randomUUID()}`;
|
|
176
|
+
const file = join(this.root, 'uploads', receiptId + '.bin');
|
|
170
177
|
await atomicWrite(file, data);
|
|
171
178
|
await atomicWrite(join(this.root, 'uploads', receiptId + '.json'), { receiptId, filename, createdAt: now(), bytes: data.length, hash: hash(data) });
|
|
172
179
|
return { receiptId, filename, bytes: data.length };
|
|
@@ -183,7 +190,8 @@ export class Store {
|
|
|
183
190
|
async open(selector, requested) { const kb = await this.kb(selector); return { kb, snapshot: await this.revisionKnown(kb, requested) }; }
|
|
184
191
|
async revision(kbId, requested) { return (await this.open(kbId, requested)).snapshot; }
|
|
185
192
|
async revisionKnown(kb, requested) {
|
|
186
|
-
const current = requested ? null : await this.current(kb.kbId)
|
|
193
|
+
const current = requested ? null : await this.current(kb.kbId);
|
|
194
|
+
const revision = requested || current?.revision;
|
|
187
195
|
if (!revision)
|
|
188
196
|
throw new Error('知识库尚无已发布版本');
|
|
189
197
|
id(revision, 'revision');
|
|
@@ -206,7 +214,8 @@ export class Store {
|
|
|
206
214
|
return this.revisionCache.chunk(await safePath(snapshot.root, chunk.path), chunk.contentHash, fresh);
|
|
207
215
|
}
|
|
208
216
|
async asset(kbId, revision, docId, assetId) {
|
|
209
|
-
const snapshot = await this.revision(kbId, revision)
|
|
217
|
+
const snapshot = await this.revision(kbId, revision);
|
|
218
|
+
const doc = snapshot.manifest.documents.find(d => d.docId === docId);
|
|
210
219
|
const asset = doc?.assets.find(a => a.assetId === assetId);
|
|
211
220
|
if (!asset)
|
|
212
221
|
throw new Error('图片不在所选文档版本中');
|
|
@@ -267,7 +276,8 @@ export class Store {
|
|
|
267
276
|
return docs;
|
|
268
277
|
}
|
|
269
278
|
async publish(kbId, revision, stage, manifest, operation) {
|
|
270
|
-
const kbRoot = this.kbRoot(kbId)
|
|
279
|
+
const kbRoot = this.kbRoot(kbId);
|
|
280
|
+
const target = join(kbRoot, 'revisions', id(revision));
|
|
271
281
|
await atomicWrite(join(stage, 'manifest.json'), manifest);
|
|
272
282
|
await mkdir(join(kbRoot, 'revisions'), { recursive: true });
|
|
273
283
|
try {
|
|
@@ -284,7 +294,9 @@ export class Store {
|
|
|
284
294
|
// The pointer owns visibility. Replaying the same job after a crash is idempotent.
|
|
285
295
|
await atomicWrite(join(kbRoot, 'current.json'), { revision, publishedAt: now(), buildKey: manifest.buildKey, coverageIncomplete: manifest.coverageIncomplete });
|
|
286
296
|
await this.serialized(async () => {
|
|
287
|
-
const logFile = join(kbRoot, 'log.md')
|
|
297
|
+
const logFile = join(kbRoot, 'log.md');
|
|
298
|
+
const old = await readFile(logFile, 'utf8');
|
|
299
|
+
const line = `- ${operation.jobId} ${now()} ${operation.kind} ${revision} ← ${operation.docId}\n`;
|
|
288
300
|
if (!old.includes(`- ${operation.jobId} `))
|
|
289
301
|
await atomicWrite(logFile, old + line);
|
|
290
302
|
});
|
|
@@ -306,11 +318,13 @@ export class Store {
|
|
|
306
318
|
}
|
|
307
319
|
/** Cleanup runs with the sole owner before accepting uploads or starting jobs. Inputs are independent copies. */
|
|
308
320
|
async cleanUploads(at = Date.now()) {
|
|
309
|
-
const root = await safePath(this.root, 'uploads')
|
|
321
|
+
const root = await safePath(this.root, 'uploads');
|
|
322
|
+
const cutoff = at - this.config.uploadRetentionDays * 86400000;
|
|
310
323
|
for (const entry of await readdir(root, { withFileTypes: true })) {
|
|
311
324
|
if (!entry.isFile() || !/^u-[a-f0-9-]{36}\.json$/u.test(entry.name))
|
|
312
325
|
continue;
|
|
313
|
-
const file = await safePath(root, entry.name)
|
|
326
|
+
const file = await safePath(root, entry.name);
|
|
327
|
+
const receipt = await readJson(file, uploadSchema);
|
|
314
328
|
if (entry.name !== receipt.receiptId + '.json')
|
|
315
329
|
throw new Error('上传收据身份不一致');
|
|
316
330
|
const time = Date.parse(receipt.createdAt);
|
|
@@ -326,7 +340,8 @@ export class Store {
|
|
|
326
340
|
}
|
|
327
341
|
async jobDirectories() { return (await readdir(await safePath(this.root, 'jobs'), { withFileTypes: true })).filter(d => d.isDirectory()).map(d => join(this.root, 'jobs', d.name)); }
|
|
328
342
|
async acquireOwner() {
|
|
329
|
-
const dir = join(this.root, '.owner.lock')
|
|
343
|
+
const dir = join(this.root, '.owner.lock');
|
|
344
|
+
const nonce = randomUUID();
|
|
330
345
|
try {
|
|
331
346
|
await mkdir(dir);
|
|
332
347
|
}
|
package/lib/tools.js
CHANGED
|
@@ -4,10 +4,12 @@ import { z } from 'zod';
|
|
|
4
4
|
import { prepareImage } from "./figures/process.js";
|
|
5
5
|
import { buildSchema, jobActionSchema, catalogQuerySchema, readQuerySchema, searchSchema, assetQuerySchema } from "./shared.js";
|
|
6
6
|
import { checkAbort, message } from "./util.js";
|
|
7
|
+
import { selectedKnowledge } from "./agent-access.js";
|
|
7
8
|
const str = (description) => ({ type: 'string', description });
|
|
8
9
|
const bool = (description) => ({ type: 'boolean', description });
|
|
9
10
|
const int = (description) => ({ type: 'integer', description });
|
|
10
|
-
const selector = str('已登记知识库 ID 或唯一库名')
|
|
11
|
+
const selector = str('已登记知识库 ID 或唯一库名');
|
|
12
|
+
const revision = str('固定的知识库版本;首次检索可省略');
|
|
11
13
|
const canonical = (value) => z.json().parse(JSON.parse(JSON.stringify(value)));
|
|
12
14
|
function jobView(job) {
|
|
13
15
|
const { jobId, kbId, docId, revision, state, error, outputPath, progress, routes, usage, report, failures } = job;
|
|
@@ -20,7 +22,10 @@ export function tools(runtime) {
|
|
|
20
22
|
const define = (name, title, description, parameters, execute, render) => defineTool({
|
|
21
23
|
name: `libing_${name}`, description, parameters, output: { schema: { type: 'json' }, render: render ?? ((_args, value) => [{ type: 'text', text: JSON.stringify(value) }]) },
|
|
22
24
|
presentCall: args => ({ card: 'generic', title, kind: 'other', rawInput: args }),
|
|
23
|
-
execute: async (args, exec) => { await ready; checkAbort(exec.signal); return runtime.run(async (signal) =>
|
|
25
|
+
execute: async (args, exec) => { await ready; checkAbort(exec.signal); return runtime.run(async (signal) => { const policy = await selectedKnowledge(runtime, exec.agent, name, args); const scoped = { ...args, ...policy.kbId ? { kbId: policy.kbId } : {} }; const result = await execute(scoped, { ...exec, signal }); if (policy.allowed !== undefined && name === 'kb_catalog' && policy.kbId === undefined) {
|
|
26
|
+
const value = z.object({ knowledgeBases: z.array(z.object({ kbId: z.string() }).passthrough()), truncated: z.boolean() }).parse(result);
|
|
27
|
+
return canonical({ ...value, knowledgeBases: value.knowledgeBases.filter(kb => policy.allowed?.includes(kb.kbId)) });
|
|
28
|
+
} return canonical(result); }, exec.signal); },
|
|
24
29
|
});
|
|
25
30
|
return [
|
|
26
31
|
define('kb_build', '构建知识库', '将指定 Word / PDF 构建为知识库;返回后台 jobId。仅明确允许时建库、跳过图片或接受缺口。', {
|
|
@@ -55,8 +60,11 @@ export function tools(runtime) {
|
|
|
55
60
|
define('kb_read_image', '核对知识库原图', '读取所选文档版本原图及裁片。当前模型须声明图片输入能力;忽略的图片不可作为证据。', {
|
|
56
61
|
kbId: { ...selector, required: true }, revision, docId: { ...str('文档ID'), required: true }, assetId: { ...str('图片ID'), required: true },
|
|
57
62
|
}, async (raw, exec) => {
|
|
58
|
-
const args = assetQuerySchema.parse(raw)
|
|
59
|
-
const
|
|
63
|
+
const args = assetQuerySchema.parse(raw);
|
|
64
|
+
const kb = await store.kb(args.kbId);
|
|
65
|
+
const asset = await store.asset(kb.kbId, args.revision, args.docId, args.assetId);
|
|
66
|
+
const figures = asset.doc.figures.filter(f => f.assetId === args.assetId && f.status !== 'ignored');
|
|
67
|
+
const region = asset.asset.source && asset.doc.sourceFormat === 'pdf';
|
|
60
68
|
if (!figures.length && (!region || asset.doc.figures.some(f => f.assetId === args.assetId && f.status === 'ignored')))
|
|
61
69
|
throw new Error('此版本的图片已忽略,请在图片复核恢复导入');
|
|
62
70
|
const route = exec.agent?.session.requestHeader()?.config ?? exec.agent?.options;
|
package/lib/util.js
CHANGED
|
@@ -48,7 +48,8 @@ export async function atomicWrite(file, value) {
|
|
|
48
48
|
export async function safePath(root, rel = '.', existing = true) {
|
|
49
49
|
if (typeof rel !== 'string' || rel.includes('\0') || isAbsolute(rel))
|
|
50
50
|
throw new Error('无效相对路径');
|
|
51
|
-
const base = await realpath(root)
|
|
51
|
+
const base = await realpath(root);
|
|
52
|
+
const target = resolve(base, rel);
|
|
52
53
|
const inside = (value) => { const r = relative(base, value); return r !== '..' && !r.startsWith('..' + sep) && !isAbsolute(r); };
|
|
53
54
|
if (!inside(target))
|
|
54
55
|
throw new Error('路径越界');
|