@dsh-libing/knowledge-base 0.1.0 → 0.1.3

package/lib/store.d.ts

@@ -1,9 +1,11 @@
+import { RevisionCache } from './revision-cache.ts';
 import type { Settings } from './config.ts';
 import { z } from 'zod';
 import { registrySchema } from './data.ts';
 import type { Library, Manifest, Snapshot, Chunk, PublishedDocument, BuildDocument, Overlay } from './data.ts';
 export declare const DEFAULT_SCHEMA = "# \u77E5\u8BC6\u5E93\u6CBB\u7406\u7EA6\u5B9A\n\n\u4FDD\u7559\u539F\u6587\u4E2D\u7684\u578B\u53F7\u3001\u7BA1\u811A\u3001\u5355\u4F4D\u3001\u6570\u503C\u3001\u6781\u6027\u548C\u9002\u7528\u6761\u4EF6\u3002\u56FE\u7247\u63CF\u8FF0\u4E0E\u539F\u6587\u5206\u522B\u6807\u6CE8\uFF1B\u4E0D\u786E\u5B9A\u7684\u4FE1\u606F\u4FDD\u7559\u5F85\u590D\u6838\u3002\u6458\u8981\u7528\u4E8E\u5BFC\u822A\uFF0C\u56DE\u7B54\u4F9D\u636E\u539F\u6587\u548C\u53EF\u6838\u9A8C\u56FE\u50CF\u3002\u5F15\u7528\u5305\u542B\u6587\u6863\u7248\u672C\u3001\u7AE0\u8282\u3001\u7247\u6BB5\u548C\u56FE\u8868\u7F16\u53F7\u3002\n";
 export declare class Store {
+    readonly revisionCache: RevisionCache;
     readonly config: Settings;
     readonly root: string;
     private registryTail;
@@ -103,16 +105,88 @@ export declare class Store {
             coverageIncomplete: boolean;
         };
     }>;
-    location(kbId: string, revision?: string): Promise<{
+    private inspectKnown;
+    overview(): Promise<{
+        kbs: ({
+            location: {
+                kbPath: string;
+                revision: null;
+                wikiPath: null;
+                indexPath: null;
+            };
+            summary: {
+                revision: null;
+                documentCount: number;
+                chunkCount: number;
+                coverageIncomplete: boolean;
+            };
+            kbId: string;
+            name: string;
+            createdAt: string;
+            deletedAt?: string | undefined;
+            purge?: {
+                startedAt: string;
+                jobIds: string[];
+                error?: string | undefined;
+            } | undefined;
+        } | {
+            location: {
+                kbPath: string;
+                revision: string;
+                wikiPath: string;
+                indexPath: string;
+            };
+            summary: {
+                revision: string;
+                documentCount: number;
+                chunkCount: number;
+                coverageIncomplete: boolean;
+            };
+            kbId: string;
+            name: string;
+            createdAt: string;
+            deletedAt?: string | undefined;
+            purge?: {
+                startedAt: string;
+                jobIds: string[];
+                error?: string | undefined;
+            } | undefined;
+        })[];
+        deletedKbs: {
+            kbId: string;
+            name: string;
+            createdAt: string;
+            deletedAt?: string | undefined;
+            purge?: {
+                startedAt: string;
+                jobIds: string[];
+                error?: string | undefined;
+            } | undefined;
+        }[];
+        activities: {
+            eventId: string;
+            at: string;
+            action: "purge" | "create" | "trash" | "restore";
+            kbId: string;
+            name: string;
+        }[];
+    }>;
+    snapshotLocation(kbId: string, snapshot: Snapshot): Promise<{
         kbPath: string;
-        revision: null;
-        wikiPath: null;
-        indexPath: null;
-    } | {
+        revision: string;
+        wikiPath: string;
+        indexPath: string;
+    }>;
+    location(kbId: string, revision?: string): Promise<{
         kbPath: string;
         revision: string;
         wikiPath: string;
         indexPath: string;
+    } | {
+        kbPath: string;
+        revision: null;
+        wikiPath: null;
+        indexPath: null;
     }>;
     schema(kbId: string): Promise<string>;
     setSchema(kbId: string, content: string, expectedHash: string): Promise<{
@@ -138,7 +212,22 @@ export declare class Store {
         buildKey: string;
         coverageIncomplete: boolean;
     } | null>;
+    open(selector: string, requested?: string): Promise<{
+        kb: {
+            kbId: string;
+            name: string;
+            createdAt: string;
+            deletedAt?: string | undefined;
+            purge?: {
+                startedAt: string;
+                jobIds: string[];
+                error?: string | undefined;
+            } | undefined;
+        };
+        snapshot: Snapshot;
+    }>;
     revision(kbId: string, requested?: string): Promise<Snapshot>;
+    private revisionKnown;
     readChunk(kbId: string, revision: string | undefined, chunkId: string): Promise<{
         chunk: {
             chunkId: string;
@@ -187,6 +276,10 @@ export declare class Store {
         manifest: Manifest;
     }>;
     chunkContent(snapshot: Snapshot, chunk: Chunk): Promise<string>;
+    chunkView(snapshot: Snapshot, chunk: Chunk, fresh?: boolean): Promise<{
+        content: string;
+        lower: string;
+    }>;
     asset(kbId: string, revision: string | undefined, docId: string, assetId: string): Promise<{
         asset: {
             assetId: string;
@@ -197,6 +290,7 @@ export declare class Store {
             source?: import("./data.ts").Source | undefined;
         };
         file: string;
+        data: Buffer<ArrayBufferLike>;
         doc: {
             docId: string;
             title: string;
@@ -404,6 +498,8 @@ export declare class Store {
     }): Promise<string>;
     overlays(kbId: string): Promise<Record<string, Overlay>>;
     setOverlay(kbId: string, key: string, value: Overlay | null, expectedHash: string): Promise<Overlay | undefined>;
+    /** Cleanup runs with the sole owner before accepting uploads or starting jobs. Inputs are independent copies. */
+    cleanUploads(at?: number): Promise<void>;
     jobDirectories(): Promise<string[]>;
     acquireOwner(): Promise<() => Promise<void>>;
 }

package/lib/store.js

@@ -1,3 +1,4 @@
+import { RevisionCache } from "./revision-cache.js";
 import { z } from 'zod';
 import { registrySchema, currentSchema, manifestSchema, uploadSchema, overlaySchema } from "./data.js";
 import { code, message, required } from "./util.js";
@@ -7,6 +8,7 @@ import { randomUUID } from 'node:crypto';
 import { atomicWrite, fileBytes, hash, id, now, readJson, safePath, text } from "./util.js";
 export const DEFAULT_SCHEMA = `# 知识库治理约定\n\n保留原文中的型号、管脚、单位、数值、极性和适用条件。图片描述与原文分别标注；不确定的信息保留待复核。摘要用于导航，回答依据原文和可核验图像。引用包含文档版本、章节、片段和图表编号。\n`;
 export class Store {
+    revisionCache = new RevisionCache();
     config;
     root;
     registryTail;
@@ -35,6 +37,7 @@ export class Store {
                 throw new Error('知识库不存在，请刷新');
             if (kb.name !== expectedName)
                 throw new Error('知识库名称已变化，请刷新后确认删除');
+            this.revisionCache.clear();
             kb.deletedAt = now();
             this.record(registry, 'trash', kb);
             await atomicWrite(join(this.root, 'registry.json'), registry);
@@ -50,6 +53,7 @@ export class Store {
                 throw new Error('知识库已开始彻底删除，不能恢复；请继续彻底删除');
             if (registry.kbs.some(k => !k.deletedAt && k.name === kb.name))
                 throw new Error('同名知识库已存在，不能恢复；请先删除同名库');
+            this.revisionCache.clear();
             delete kb.deletedAt;
             this.record(registry, 'restore', kb);
             await atomicWrite(join(this.root, 'registry.json'), registry);
@@ -72,6 +76,7 @@ export class Store {
     }
     async removePurgeTree(path) { await rm(path, { recursive: true, force: true, maxRetries: 2, retryDelay: 100 }); }
     async purge(kbId, expectedName, confirmed, jobIds = []) {
+        this.revisionCache.clear();
         if (confirmed !== true)
             throw new Error('必须明确确认彻底删除不可恢复');
         return this.serialized(async () => {
@@ -129,13 +134,20 @@ export class Store {
         });
     }
     kbRoot(kbId) { return join(this.root, 'kbs', id(kbId, 'kbId')); }
-    async inspect(kbId, revision) {
-        const kb = await this.kb(kbId), kbPath = await safePath(this.root, join('kbs', kb.kbId));
-        const current = await this.current(kb.kbId);
+    async inspect(kbId, revision) { return this.inspectKnown(await this.kb(kbId), revision); }
+    async inspectKnown(kb, revision) {
+        const kbPath = await safePath(this.root, join('kbs', kb.kbId)), current = await this.current(kb.kbId);
         if (!revision && !current)
             return { location: { kbPath, revision: null, wikiPath: null, indexPath: null }, summary: { revision: null, documentCount: 0, chunkCount: 0, coverageIncomplete: false } };
-        const snapshot = await this.revision(kb.kbId, revision || required(current).revision);
-        return { location: { kbPath, revision: snapshot.revision, wikiPath: await safePath(snapshot.root, 'wiki'), indexPath: await safePath(snapshot.root, 'index.md') }, summary: { revision: snapshot.revision, documentCount: snapshot.manifest.documents.length, chunkCount: snapshot.manifest.documents.reduce((sum, d) => sum + d.chunks.length, 0), coverageIncomplete: Boolean(snapshot.manifest.coverageIncomplete) } };
+        const snapshot = await this.revisionKnown(kb, revision || required(current).revision);
+        return { location: await this.snapshotLocation(kb.kbId, snapshot), summary: { revision: snapshot.revision, documentCount: snapshot.manifest.documents.length, chunkCount: snapshot.manifest.documents.reduce((sum, d) => sum + d.chunks.length, 0), coverageIncomplete: Boolean(snapshot.manifest.coverageIncomplete) } };
+    }
+    async overview() {
+        const registry = await readJson(join(this.root, 'registry.json'), registrySchema);
+        return { kbs: await Promise.all(registry.kbs.filter(k => !k.deletedAt).map(async (kb) => ({ ...kb, ...await this.inspectKnown(kb) }))), deletedKbs: registry.kbs.filter(k => k.deletedAt), activities: registry.activities || [] };
+    }
+    async snapshotLocation(kbId, snapshot) {
+        return { kbPath: await safePath(this.root, join('kbs', id(kbId))), revision: snapshot.revision, wikiPath: await safePath(snapshot.root, 'wiki'), indexPath: await safePath(snapshot.root, 'index.md') };
     }
     async location(kbId, revision) { return (await this.inspect(kbId, revision)).location; }
     async schema(kbId) { return readFile(await safePath(this.kbRoot(kbId), 'schema.md'), 'utf8'); }
@@ -167,15 +179,16 @@ export class Store {
             throw new Error('上传内容已变化');
         return { ...receipt, data };
     }
-    async current(kbId) { return readJson(join(this.kbRoot(kbId), 'current.json'), currentSchema, null); }
-    async revision(kbId, requested) {
-        await this.kb(kbId);
-        const current = await this.current(kbId), revision = requested || current?.revision;
+    async current(kbId) { return readJson(await safePath(this.root, join('kbs', id(kbId), 'current.json'), false), currentSchema, null); }
+    async open(selector, requested) { const kb = await this.kb(selector); return { kb, snapshot: await this.revisionKnown(kb, requested) }; }
+    async revision(kbId, requested) { return (await this.open(kbId, requested)).snapshot; }
+    async revisionKnown(kb, requested) {
+        const current = requested ? null : await this.current(kb.kbId), revision = requested || current?.revision;
         if (!revision)
             throw new Error('知识库尚无已发布版本');
         id(revision, 'revision');
-        const root = await safePath(this.kbRoot(kbId), join('revisions', revision));
-        const manifest = await readJson(join(root, 'manifest.json'), manifestSchema);
+        const root = await safePath(this.root, join('kbs', kb.kbId, 'revisions', revision));
+        const manifest = await this.revisionCache.manifest(await safePath(root, 'manifest.json'));
         if (manifest.revision !== revision)
             throw new Error('版本清单不一致');
         return { root, manifest, revision };
@@ -188,11 +201,9 @@ export class Store {
         const content = await this.chunkContent(snapshot, chunk);
         return { ...snapshot, chunk, content };
     }
-    async chunkContent(snapshot, chunk) {
-        const content = await readFile(await safePath(snapshot.root, chunk.path), 'utf8');
-        if (hash(content) !== chunk.contentHash)
-            throw new Error('片段内容与清单不一致，请重建');
-        return content;
+    async chunkContent(snapshot, chunk) { return (await this.chunkView(snapshot, chunk, true)).content; }
+    async chunkView(snapshot, chunk, fresh = false) {
+        return this.revisionCache.chunk(await safePath(snapshot.root, chunk.path), chunk.contentHash, fresh);
     }
     async asset(kbId, revision, docId, assetId) {
         const snapshot = await this.revision(kbId, revision), doc = snapshot.manifest.documents.find(d => d.docId === docId);
@@ -200,9 +211,10 @@ export class Store {
         if (!asset)
             throw new Error('图片不在所选文档版本中');
         const file = await safePath(snapshot.root, join('sources', id(docId), asset.path));
-        if (`a-${hash(await fileBytes(file, this.config.maxUncompressedBytes)).slice(0, 20)}` !== asset.assetId)
+        const data = await fileBytes(file, this.config.maxUncompressedBytes);
+        if (`a-${hash(data).slice(0, 20)}` !== asset.assetId)
             throw new Error('图片内容与清单不一致');
-        return { ...snapshot, asset, file, doc: required(doc) };
+        return { ...snapshot, asset, file, data, doc: required(doc) };
     }
     async rawSnapshot(kbId, docId, sourceHash, data, info) {
         const root = join(this.kbRoot(kbId), 'raw', id(docId), id(sourceHash));
@@ -233,7 +245,8 @@ export class Store {
             return [];
         const { root, manifest } = await this.revision(kbId, current.revision);
         const docs = [];
-        for (const document of manifest.documents.filter(d => d.docId !== replaceDocId)) {
+        for (const saved of manifest.documents.filter(d => d.docId !== replaceDocId)) {
+            const document = structuredClone(saved);
             const chunks = [];
             for (const chunk of document.chunks) {
                 const content = await this.chunkContent({ root, manifest, revision: manifest.revision }, chunk);
@@ -267,6 +280,7 @@ export class Store {
             if (existing.buildKey !== manifest.buildKey)
                 throw new Error('版本发布冲突');
         }
+        this.revisionCache.clear();
         // The pointer owns visibility. Replaying the same job after a crash is idempotent.
         await atomicWrite(join(kbRoot, 'current.json'), { revision, publishedAt: now(), buildKey: manifest.buildKey, coverageIncomplete: manifest.coverageIncomplete });
         await this.serialized(async () => {
@@ -290,7 +304,27 @@ export class Store {
             return overlays[key];
         });
     }
-    async jobDirectories() { return (await readdir(join(this.root, 'jobs'), { withFileTypes: true })).filter(d => d.isDirectory()).map(d => join(this.root, 'jobs', d.name)); }
+    /** Cleanup runs with the sole owner before accepting uploads or starting jobs. Inputs are independent copies. */
+    async cleanUploads(at = Date.now()) {
+        const root = await safePath(this.root, 'uploads'), cutoff = at - this.config.uploadRetentionDays * 86400000;
+        for (const entry of await readdir(root, { withFileTypes: true })) {
+            if (!entry.isFile() || !/^u-[a-f0-9-]{36}\.json$/u.test(entry.name))
+                continue;
+            const file = await safePath(root, entry.name), receipt = await readJson(file, uploadSchema);
+            if (entry.name !== receipt.receiptId + '.json')
+                throw new Error('上传收据身份不一致');
+            const time = Date.parse(receipt.createdAt);
+            if (!Number.isFinite(time))
+                throw new Error('上传收据日期无效');
+            if (time >= cutoff)
+                continue;
+            const binary = await safePath(root, receipt.receiptId + '.bin', false);
+            // Remove the binary first. If interrupted, the validated receipt makes cleanup retryable.
+            await rm(binary, { force: true });
+            await rm(file);
+        }
+    }
+    async jobDirectories() { return (await readdir(await safePath(this.root, 'jobs'), { withFileTypes: true })).filter(d => d.isDirectory()).map(d => join(this.root, 'jobs', d.name)); }
     async acquireOwner() {
         const dir = join(this.root, '.owner.lock'), nonce = randomUUID();
         try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dsh-libing/knowledge-base",
-  "version": "0.1.0",
+  "version": "0.1.3",
   "description": "知识治理：Word/PDF 导入、图表复核、可恢复任务与固定版本证据检索",
   "type": "module",
   "icon": "icon.svg",