@dsh-libing/knowledge-base 0.1.0 → 0.1.4

package/lib/store.js

@@ -1,3 +1,4 @@
+import { RevisionCache } from "./revision-cache.js";
 import { z } from 'zod';
 import { registrySchema, currentSchema, manifestSchema, uploadSchema, overlaySchema } from "./data.js";
 import { code, message, required } from "./util.js";
@@ -7,6 +8,7 @@ import { randomUUID } from 'node:crypto';
 import { atomicWrite, fileBytes, hash, id, now, readJson, safePath, text } from "./util.js";
 export const DEFAULT_SCHEMA = `# 知识库治理约定\n\n保留原文中的型号、管脚、单位、数值、极性和适用条件。图片描述与原文分别标注；不确定的信息保留待复核。摘要用于导航，回答依据原文和可核验图像。引用包含文档版本、章节、片段和图表编号。\n`;
 export class Store {
+    revisionCache = new RevisionCache();
     config;
     root;
     registryTail;
@@ -30,11 +32,13 @@ export class Store {
     }
     async trash(kbId, expectedName) {
         return this.serialized(async () => {
-            const registry = await readJson(join(this.root, 'registry.json'), registrySchema), kb = registry.kbs.find(k => k.kbId === id(kbId) && !k.deletedAt);
+            const registry = await readJson(join(this.root, 'registry.json'), registrySchema);
+            const kb = registry.kbs.find(k => k.kbId === id(kbId) && !k.deletedAt);
             if (!kb)
                 throw new Error('知识库不存在，请刷新');
             if (kb.name !== expectedName)
                 throw new Error('知识库名称已变化，请刷新后确认删除');
+            this.revisionCache.clear();
             kb.deletedAt = now();
             this.record(registry, 'trash', kb);
             await atomicWrite(join(this.root, 'registry.json'), registry);
@@ -43,13 +47,15 @@ export class Store {
     }
     async restore(kbId) {
         return this.serialized(async () => {
-            const registry = await readJson(join(this.root, 'registry.json'), registrySchema), kb = registry.kbs.find(k => k.kbId === id(kbId) && k.deletedAt);
+            const registry = await readJson(join(this.root, 'registry.json'), registrySchema);
+            const kb = registry.kbs.find(k => k.kbId === id(kbId) && k.deletedAt);
             if (!kb)
                 throw new Error('知识库不在回收站，请刷新');
             if (kb.purge)
                 throw new Error('知识库已开始彻底删除，不能恢复；请继续彻底删除');
             if (registry.kbs.some(k => !k.deletedAt && k.name === kb.name))
                 throw new Error('同名知识库已存在，不能恢复；请先删除同名库');
+            this.revisionCache.clear();
             delete kb.deletedAt;
             this.record(registry, 'restore', kb);
             await atomicWrite(join(this.root, 'registry.json'), registry);
@@ -72,16 +78,19 @@ export class Store {
     }
     async removePurgeTree(path) { await rm(path, { recursive: true, force: true, maxRetries: 2, retryDelay: 100 }); }
     async purge(kbId, expectedName, confirmed, jobIds = []) {
+        this.revisionCache.clear();
         if (confirmed !== true)
             throw new Error('必须明确确认彻底删除不可恢复');
         return this.serialized(async () => {
-            const registry = await readJson(join(this.root, 'registry.json'), registrySchema), kb = registry.kbs.find(k => k.kbId === id(kbId) && k.deletedAt);
+            const registry = await readJson(join(this.root, 'registry.json'), registrySchema);
+            const kb = registry.kbs.find(k => k.kbId === id(kbId) && k.deletedAt);
             if (!kb)
                 throw new Error('只能彻底删除回收站中的知识库');
             if (kb.name !== expectedName)
                 throw new Error('知识库名称已变化，请重新确认彻底删除');
             const targets = [...new Set([...(kb.purge?.jobIds || []), ...jobIds])];
-            const kbPath = await this.purgeTarget('kbs', kb.kbId), jobPaths = await Promise.all(targets.map(key => this.purgeTarget('jobs', key)));
+            const kbPath = await this.purgeTarget('kbs', kb.kbId);
+            const jobPaths = await Promise.all(targets.map(key => this.purgeTarget('jobs', key)));
             // Persist intent before removing files: a crash or partial failure must not allow restoration.
             kb.purge = { startedAt: kb.purge?.startedAt || now(), jobIds: targets };
             await atomicWrite(join(this.root, 'registry.json'), registry);
@@ -105,7 +114,8 @@ export class Store {
         });
     }
     async kb(selector) {
-        const kbs = await this.list(), matches = kbs.filter(k => k.kbId === selector || k.name === selector);
+        const kbs = await this.list();
+        const matches = kbs.filter(k => k.kbId === selector || k.name === selector);
         if (matches.length !== 1)
             throw new Error(matches.length ? '知识库名称重复，请使用 kbId' : '知识库不存在，请先创建或选择');
         await safePath(this.root, join('kbs', id(required(matches[0]).kbId)), true);
@@ -129,13 +139,21 @@ export class Store {
         });
     }
     kbRoot(kbId) { return join(this.root, 'kbs', id(kbId, 'kbId')); }
-    async inspect(kbId, revision) {
-        const kb = await this.kb(kbId), kbPath = await safePath(this.root, join('kbs', kb.kbId));
+    async inspect(kbId, revision) { return this.inspectKnown(await this.kb(kbId), revision); }
+    async inspectKnown(kb, revision) {
+        const kbPath = await safePath(this.root, join('kbs', kb.kbId));
         const current = await this.current(kb.kbId);
         if (!revision && !current)
             return { location: { kbPath, revision: null, wikiPath: null, indexPath: null }, summary: { revision: null, documentCount: 0, chunkCount: 0, coverageIncomplete: false } };
-        const snapshot = await this.revision(kb.kbId, revision || required(current).revision);
-        return { location: { kbPath, revision: snapshot.revision, wikiPath: await safePath(snapshot.root, 'wiki'), indexPath: await safePath(snapshot.root, 'index.md') }, summary: { revision: snapshot.revision, documentCount: snapshot.manifest.documents.length, chunkCount: snapshot.manifest.documents.reduce((sum, d) => sum + d.chunks.length, 0), coverageIncomplete: Boolean(snapshot.manifest.coverageIncomplete) } };
+        const snapshot = await this.revisionKnown(kb, revision || required(current).revision);
+        return { location: await this.snapshotLocation(kb.kbId, snapshot), summary: { revision: snapshot.revision, documentCount: snapshot.manifest.documents.length, chunkCount: snapshot.manifest.documents.reduce((sum, d) => sum + d.chunks.length, 0), coverageIncomplete: Boolean(snapshot.manifest.coverageIncomplete) } };
+    }
+    async overview() {
+        const registry = await readJson(join(this.root, 'registry.json'), registrySchema);
+        return { kbs: await Promise.all(registry.kbs.filter(k => !k.deletedAt).map(async (kb) => ({ ...kb, ...await this.inspectKnown(kb) }))), deletedKbs: registry.kbs.filter(k => k.deletedAt), activities: registry.activities || [] };
+    }
+    async snapshotLocation(kbId, snapshot) {
+        return { kbPath: await safePath(this.root, join('kbs', id(kbId))), revision: snapshot.revision, wikiPath: await safePath(snapshot.root, 'wiki'), indexPath: await safePath(snapshot.root, 'index.md') };
     }
     async location(kbId, revision) { return (await this.inspect(kbId, revision)).location; }
     async schema(kbId) { return readFile(await safePath(this.kbRoot(kbId), 'schema.md'), 'utf8'); }
@@ -154,7 +172,8 @@ export class Store {
         filename = text(basename(filename), '文件名', 200);
         if (!/\.(?:docx?|pdf)$/i.test(filename) || data.length > this.config.maxInputBytes)
             throw new Error('只支持大小限制内的 .docx / .doc / .pdf 文件');
-        const receiptId = `u-${randomUUID()}`, file = join(this.root, 'uploads', receiptId + '.bin');
+        const receiptId = `u-${randomUUID()}`;
+        const file = join(this.root, 'uploads', receiptId + '.bin');
         await atomicWrite(file, data);
         await atomicWrite(join(this.root, 'uploads', receiptId + '.json'), { receiptId, filename, createdAt: now(), bytes: data.length, hash: hash(data) });
         return { receiptId, filename, bytes: data.length };
@@ -167,15 +186,17 @@ export class Store {
             throw new Error('上传内容已变化');
         return { ...receipt, data };
     }
-    async current(kbId) { return readJson(join(this.kbRoot(kbId), 'current.json'), currentSchema, null); }
-    async revision(kbId, requested) {
-        await this.kb(kbId);
-        const current = await this.current(kbId), revision = requested || current?.revision;
+    async current(kbId) { return readJson(await safePath(this.root, join('kbs', id(kbId), 'current.json'), false), currentSchema, null); }
+    async open(selector, requested) { const kb = await this.kb(selector); return { kb, snapshot: await this.revisionKnown(kb, requested) }; }
+    async revision(kbId, requested) { return (await this.open(kbId, requested)).snapshot; }
+    async revisionKnown(kb, requested) {
+        const current = requested ? null : await this.current(kb.kbId);
+        const revision = requested || current?.revision;
         if (!revision)
             throw new Error('知识库尚无已发布版本');
         id(revision, 'revision');
-        const root = await safePath(this.kbRoot(kbId), join('revisions', revision));
-        const manifest = await readJson(join(root, 'manifest.json'), manifestSchema);
+        const root = await safePath(this.root, join('kbs', kb.kbId, 'revisions', revision));
+        const manifest = await this.revisionCache.manifest(await safePath(root, 'manifest.json'));
         if (manifest.revision !== revision)
             throw new Error('版本清单不一致');
         return { root, manifest, revision };
@@ -188,21 +209,21 @@ export class Store {
         const content = await this.chunkContent(snapshot, chunk);
         return { ...snapshot, chunk, content };
     }
-    async chunkContent(snapshot, chunk) {
-        const content = await readFile(await safePath(snapshot.root, chunk.path), 'utf8');
-        if (hash(content) !== chunk.contentHash)
-            throw new Error('片段内容与清单不一致，请重建');
-        return content;
+    async chunkContent(snapshot, chunk) { return (await this.chunkView(snapshot, chunk, true)).content; }
+    async chunkView(snapshot, chunk, fresh = false) {
+        return this.revisionCache.chunk(await safePath(snapshot.root, chunk.path), chunk.contentHash, fresh);
     }
     async asset(kbId, revision, docId, assetId) {
-        const snapshot = await this.revision(kbId, revision), doc = snapshot.manifest.documents.find(d => d.docId === docId);
+        const snapshot = await this.revision(kbId, revision);
+        const doc = snapshot.manifest.documents.find(d => d.docId === docId);
         const asset = doc?.assets.find(a => a.assetId === assetId);
         if (!asset)
             throw new Error('图片不在所选文档版本中');
         const file = await safePath(snapshot.root, join('sources', id(docId), asset.path));
-        if (`a-${hash(await fileBytes(file, this.config.maxUncompressedBytes)).slice(0, 20)}` !== asset.assetId)
+        const data = await fileBytes(file, this.config.maxUncompressedBytes);
+        if (`a-${hash(data).slice(0, 20)}` !== asset.assetId)
             throw new Error('图片内容与清单不一致');
-        return { ...snapshot, asset, file, doc: required(doc) };
+        return { ...snapshot, asset, file, data, doc: required(doc) };
     }
     async rawSnapshot(kbId, docId, sourceHash, data, info) {
         const root = join(this.kbRoot(kbId), 'raw', id(docId), id(sourceHash));
@@ -233,7 +254,8 @@ export class Store {
             return [];
         const { root, manifest } = await this.revision(kbId, current.revision);
         const docs = [];
-        for (const document of manifest.documents.filter(d => d.docId !== replaceDocId)) {
+        for (const saved of manifest.documents.filter(d => d.docId !== replaceDocId)) {
+            const document = structuredClone(saved);
             const chunks = [];
             for (const chunk of document.chunks) {
                 const content = await this.chunkContent({ root, manifest, revision: manifest.revision }, chunk);
@@ -254,7 +276,8 @@ export class Store {
         return docs;
     }
     async publish(kbId, revision, stage, manifest, operation) {
-        const kbRoot = this.kbRoot(kbId), target = join(kbRoot, 'revisions', id(revision));
+        const kbRoot = this.kbRoot(kbId);
+        const target = join(kbRoot, 'revisions', id(revision));
         await atomicWrite(join(stage, 'manifest.json'), manifest);
         await mkdir(join(kbRoot, 'revisions'), { recursive: true });
         try {
@@ -267,10 +290,13 @@ export class Store {
             if (existing.buildKey !== manifest.buildKey)
                 throw new Error('版本发布冲突');
         }
+        this.revisionCache.clear();
         // The pointer owns visibility. Replaying the same job after a crash is idempotent.
         await atomicWrite(join(kbRoot, 'current.json'), { revision, publishedAt: now(), buildKey: manifest.buildKey, coverageIncomplete: manifest.coverageIncomplete });
         await this.serialized(async () => {
-            const logFile = join(kbRoot, 'log.md'), old = await readFile(logFile, 'utf8'), line = `- ${operation.jobId} ${now()} ${operation.kind} ${revision} ← ${operation.docId}\n`;
+            const logFile = join(kbRoot, 'log.md');
+            const old = await readFile(logFile, 'utf8');
+            const line = `- ${operation.jobId} ${now()} ${operation.kind} ${revision} ← ${operation.docId}\n`;
             if (!old.includes(`- ${operation.jobId} `))
                 await atomicWrite(logFile, old + line);
         });
@@ -290,9 +316,32 @@ export class Store {
             return overlays[key];
         });
     }
-    async jobDirectories() { return (await readdir(join(this.root, 'jobs'), { withFileTypes: true })).filter(d => d.isDirectory()).map(d => join(this.root, 'jobs', d.name)); }
+    /** Cleanup runs with the sole owner before accepting uploads or starting jobs. Inputs are independent copies. */
+    async cleanUploads(at = Date.now()) {
+        const root = await safePath(this.root, 'uploads');
+        const cutoff = at - this.config.uploadRetentionDays * 86400000;
+        for (const entry of await readdir(root, { withFileTypes: true })) {
+            if (!entry.isFile() || !/^u-[a-f0-9-]{36}\.json$/u.test(entry.name))
+                continue;
+            const file = await safePath(root, entry.name);
+            const receipt = await readJson(file, uploadSchema);
+            if (entry.name !== receipt.receiptId + '.json')
+                throw new Error('上传收据身份不一致');
+            const time = Date.parse(receipt.createdAt);
+            if (!Number.isFinite(time))
+                throw new Error('上传收据日期无效');
+            if (time >= cutoff)
+                continue;
+            const binary = await safePath(root, receipt.receiptId + '.bin', false);
+            // Remove the binary first. If interrupted, the validated receipt makes cleanup retryable.
+            await rm(binary, { force: true });
+            await rm(file);
+        }
+    }
+    async jobDirectories() { return (await readdir(await safePath(this.root, 'jobs'), { withFileTypes: true })).filter(d => d.isDirectory()).map(d => join(this.root, 'jobs', d.name)); }
     async acquireOwner() {
-        const dir = join(this.root, '.owner.lock'), nonce = randomUUID();
+        const dir = join(this.root, '.owner.lock');
+        const nonce = randomUUID();
         try {
             await mkdir(dir);
         }

package/lib/tools.js

@@ -4,10 +4,12 @@ import { z } from 'zod';
 import { prepareImage } from "./figures/process.js";
 import { buildSchema, jobActionSchema, catalogQuerySchema, readQuerySchema, searchSchema, assetQuerySchema } from "./shared.js";
 import { checkAbort, message } from "./util.js";
+import { selectedKnowledge } from "./agent-access.js";
 const str = (description) => ({ type: 'string', description });
 const bool = (description) => ({ type: 'boolean', description });
 const int = (description) => ({ type: 'integer', description });
-const selector = str('已登记知识库 ID 或唯一库名'), revision = str('固定的知识库版本；首次检索可省略');
+const selector = str('已登记知识库 ID 或唯一库名');
+const revision = str('固定的知识库版本；首次检索可省略');
 const canonical = (value) => z.json().parse(JSON.parse(JSON.stringify(value)));
 function jobView(job) {
     const { jobId, kbId, docId, revision, state, error, outputPath, progress, routes, usage, report, failures } = job;
@@ -20,7 +22,10 @@ export function tools(runtime) {
     const define = (name, title, description, parameters, execute, render) => defineTool({
         name: `libing_${name}`, description, parameters, output: { schema: { type: 'json' }, render: render ?? ((_args, value) => [{ type: 'text', text: JSON.stringify(value) }]) },
         presentCall: args => ({ card: 'generic', title, kind: 'other', rawInput: args }),
-        execute: async (args, exec) => { await ready; checkAbort(exec.signal); return runtime.run(async (signal) => canonical(await execute(args, { ...exec, signal })), exec.signal); },
+        execute: async (args, exec) => { await ready; checkAbort(exec.signal); return runtime.run(async (signal) => { const policy = await selectedKnowledge(runtime, exec.agent, name, args); const scoped = { ...args, ...policy.kbId ? { kbId: policy.kbId } : {} }; const result = await execute(scoped, { ...exec, signal }); if (policy.allowed !== undefined && name === 'kb_catalog' && policy.kbId === undefined) {
+            const value = z.object({ knowledgeBases: z.array(z.object({ kbId: z.string() }).passthrough()), truncated: z.boolean() }).parse(result);
+            return canonical({ ...value, knowledgeBases: value.knowledgeBases.filter(kb => policy.allowed?.includes(kb.kbId)) });
+        } return canonical(result); }, exec.signal); },
     });
     return [
         define('kb_build', '构建知识库', '将指定 Word / PDF 构建为知识库；返回后台 jobId。仅明确允许时建库、跳过图片或接受缺口。', {
@@ -55,8 +60,11 @@ export function tools(runtime) {
         define('kb_read_image', '核对知识库原图', '读取所选文档版本原图及裁片。当前模型须声明图片输入能力；忽略的图片不可作为证据。', {
             kbId: { ...selector, required: true }, revision, docId: { ...str('文档ID'), required: true }, assetId: { ...str('图片ID'), required: true },
         }, async (raw, exec) => {
-            const args = assetQuerySchema.parse(raw), kb = await store.kb(args.kbId), asset = await store.asset(kb.kbId, args.revision, args.docId, args.assetId);
-            const figures = asset.doc.figures.filter(f => f.assetId === args.assetId && f.status !== 'ignored'), region = asset.asset.source && asset.doc.sourceFormat === 'pdf';
+            const args = assetQuerySchema.parse(raw);
+            const kb = await store.kb(args.kbId);
+            const asset = await store.asset(kb.kbId, args.revision, args.docId, args.assetId);
+            const figures = asset.doc.figures.filter(f => f.assetId === args.assetId && f.status !== 'ignored');
+            const region = asset.asset.source && asset.doc.sourceFormat === 'pdf';
             if (!figures.length && (!region || asset.doc.figures.some(f => f.assetId === args.assetId && f.status === 'ignored')))
                 throw new Error('此版本的图片已忽略，请在图片复核恢复导入');
             const route = exec.agent?.session.requestHeader()?.config ?? exec.agent?.options;

package/lib/util.js

@@ -48,7 +48,8 @@ export async function atomicWrite(file, value) {
 export async function safePath(root, rel = '.', existing = true) {
     if (typeof rel !== 'string' || rel.includes('\0') || isAbsolute(rel))
         throw new Error('无效相对路径');
-    const base = await realpath(root), target = resolve(base, rel);
+    const base = await realpath(root);
+    const target = resolve(base, rel);
     const inside = (value) => { const r = relative(base, value); return r !== '..' && !r.startsWith('..' + sep) && !isAbsolute(r); };
     if (!inside(target))
         throw new Error('路径越界');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dsh-libing/knowledge-base",
-  "version": "0.1.0",
+  "version": "0.1.4",
   "description": "知识治理：Word/PDF 导入、图表复核、可恢复任务与固定版本证据检索",
   "type": "module",
   "icon": "icon.svg",