@dsh-libing/knowledge-base 0.1.0 → 0.1.4

package/lib/index.js

@@ -3,6 +3,7 @@ import { createRuntime } from "./runtime.js";
 import { routes } from "./routes.js";
 import { tools } from "./tools.js";
 import { skillProvider } from "./skills.js";
+import { registerKnowledgeCatalog } from "./agent-access.js";
 export { Config } from "./config.js";
 export { createRuntime } from "./runtime.js";
 export const name = 'libing-knowledge-base';
@@ -10,7 +11,8 @@ export const inject = ['connection', 'tools', 'skills', 'llm', 'attachments'];
 const lifetimes = new Map();
 /** Context owns all registrations and drains durable work before releasing the data root. */
 export async function apply(ctx, config = {}) {
-    const resolved = resolveConfig(config), previous = lifetimes.get(resolved.dataRoot);
+    const resolved = resolveConfig(config);
+    const previous = lifetimes.get(resolved.dataRoot);
     const scope = { disposed: false };
     // Own cancellation before awaiting a previous generation; disposal during that wait must not leak a newly created runtime.
     ctx.effect(() => () => {
@@ -28,13 +30,15 @@ export async function apply(ctx, config = {}) {
         await previous.closing;
     if (scope.disposed)
         return;
-    const runtime = createRuntime(ctx, resolved), lifetime = { runtime };
+    const runtime = createRuntime(ctx, resolved);
+    const lifetime = { runtime };
     scope.owned = lifetime;
     if (!lifetimes.has(resolved.dataRoot))
         lifetimes.set(resolved.dataRoot, lifetime);
     await runtime.ready;
     if (scope.disposed)
         return;
+    registerKnowledgeCatalog(ctx, runtime);
     for (const definition of tools(runtime))
         ctx.tools.register(definition);
     ctx.effect(() => ctx.skills.registerProvider(() => skillProvider()), 'libing-knowledge-base: bundled skills');

package/lib/indexes.js

@@ -12,7 +12,8 @@ export async function summarizeChunks(chunks, job, models, signal) {
     for (const chunk of chunks) {
         job.progress.currentChunk = { title: chunk.title, headingPath: chunk.headingPath, chunkId: chunk.chunkId };
         await job.checkpoint();
-        let summary, fallback = false;
+        let summary;
+        let fallback = false;
         if (job.routes.summary) {
             try {
                 const key = hash({ content: chunk.contentHash, route: job.routes.summary, max: job.config.summaryMaxChars, schema: job.schemaHash, prompt: 1 });
@@ -43,11 +44,13 @@ export async function summarizeChunks(chunks, job, models, signal) {
 }
 /** Every node is a bounded navigation page; overflow is split into linked pages. */
 export async function writeIndexes(dir, documents, config, revision) {
-    const catalogs = {}, entities = {};
+    const catalogs = {};
+    const entities = {};
     const writeCatalog = async (catalogId, path, title, entries) => {
         const intro = `# ${mdText(title)}\n\n版本：${revision}\n\n`;
         const pageEntries = [];
-        let group = [], length = chars(intro);
+        let group = [];
+        let length = chars(intro);
         for (const entry of entries) {
             const line = entry.line || `- [${mdText(entry.title)}](${posix.relative(posix.dirname(path), required(entry.path))}) — ${mdText(entry.summary)}；docId=${entry.docId}${entry.nodeId ? '; nodeId=' + entry.nodeId : ''}${entry.chunkId ? '; chunkId=' + entry.chunkId : ''}；${entry.fullChars || 0} 字符；${mdText((entry.keywords || []).slice(0, 12).join('、'))}；${entry.reviewStatus || ''}${entry.pdfScope ? '；' + mdText(entry.pdfScope) : ''}${entry.pageRange ? '；PDF第' + entry.pageRange.join('–') + '页' : ''}\n`;
             if (group.length && length + chars(line) > config.catalogMaxChars - 300) {
@@ -61,7 +64,8 @@ export async function writeIndexes(dir, documents, config, revision) {
         pageEntries.push(group);
         catalogs[catalogId] = { catalogId, path, title, pages: [] };
         for (let i = 0; i < pageEntries.length; i++) {
-            const pagePath = i === 0 ? path : path.replace(/\.md$/, `-${i + 1}.md`), next = i + 1 < pageEntries.length ? path.replace(/\.md$/, `-${i + 2}.md`) : null;
+            const pagePath = i === 0 ? path : path.replace(/\.md$/, `-${i + 1}.md`);
+            const next = i + 1 < pageEntries.length ? path.replace(/\.md$/, `-${i + 2}.md`) : null;
             const content = intro + required(pageEntries[i]).map(x => x.line).join('') + (next ? `\n[下一页索引](${posix.basename(next)})\n` : '');
             if (chars(content) > config.catalogMaxChars)
                 throw new Error('单个索引条目超过导航预算，请提高 catalogMaxChars 或缩短标题');
@@ -71,11 +75,18 @@ export async function writeIndexes(dir, documents, config, revision) {
     };
     for (const document of documents) {
         const base = posix.join('wiki/documents', document.docId);
+        const byId = new Map(document.nodes.map(node => [node.nodeId, node]));
+        const byNode = new Map();
+        for (const chunk of document.chunks) {
+            const group = byNode.get(chunk.nodeId) || [];
+            group.push(chunk);
+            byNode.set(chunk.nodeId, group);
+        }
         for (const node of document.nodes) {
             const parents = [];
             let current = node;
             while (current.parentId) {
-                current = required(document.nodes.find(n => n.nodeId === current.parentId));
+                current = required(byId.get(current.parentId));
                 if (current.nodeId !== 'root')
                     parents.unshift(current.nodeId);
             }
@@ -83,7 +94,7 @@ export async function writeIndexes(dir, documents, config, revision) {
             node.indexPath = path;
         }
         for (const node of document.nodes) {
-            const entries = [...node.children.map(childId => required(document.nodes.find(x => x.nodeId === childId))).map(child => ({ title: child.title, path: child.indexPath, nodeId: child.nodeId, docId: document.docId, summary: '章节导航', reviewStatus: 'navigation' })), ...document.chunks.filter(c => c.nodeId === node.nodeId).map(c => ({ title: c.title, path: c.path, chunkId: c.chunkId, docId: c.docId, summary: c.summary, fullChars: c.fullChars, keywords: c.keywords, reviewStatus: c.reviewStatus, ...c.pageRange ? { pageRange: c.pageRange } : {} }))];
+            const entries = [...node.children.map(childId => required(byId.get(childId))).map(child => ({ title: child.title, path: child.indexPath, nodeId: child.nodeId, docId: document.docId, summary: '章节导航', reviewStatus: 'navigation' })), ...(byNode.get(node.nodeId) || []).map(c => ({ title: c.title, path: c.path, chunkId: c.chunkId, docId: c.docId, summary: c.summary, fullChars: c.fullChars, keywords: c.keywords, reviewStatus: c.reviewStatus, ...c.pageRange ? { pageRange: c.pageRange } : {} }))];
             await writeCatalog(`${document.docId}-${node.nodeId}`, required(node.indexPath), node.title, document.sourceFormat === 'pdf' ? [{ line: `> ${pdfScope(document)}\n\n` }, ...entries] : entries);
         }
         for (const chunk of document.chunks) {

package/lib/job-storage.d.ts

@@ -0,0 +1,264 @@
+/** Small checkpoints own counters; existing call receipts resolve only unfinished accounting. */
+import { z } from 'zod';
+import { reportSchema } from './data.ts';
+import type { Job, JobRecord } from './data.ts';
+export declare const checkpointSchema: z.ZodObject<{
+    error: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+    title: z.ZodString;
+    updatedAt: z.ZodOptional<z.ZodString>;
+    pdf: z.ZodOptional<z.ZodObject<{
+        pageCount: z.ZodNumber;
+        selectedPages: z.ZodObject<{
+            start: z.ZodNumber;
+            end: z.ZodNumber;
+            count: z.ZodNumber;
+            total: z.ZodNumber;
+            partial: z.ZodBoolean;
+        }, z.core.$strip>;
+        outlineCount: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    sourceFormat: z.ZodDefault<z.ZodEnum<{
+        docx: "docx";
+        pdf: "pdf";
+    }>>;
+    parserVersion: z.ZodOptional<z.ZodString>;
+    sourceHash: z.ZodOptional<z.ZodString>;
+    docId: z.ZodString;
+    sourceVersion: z.ZodString;
+    filename: z.ZodString;
+    revision: z.ZodOptional<z.ZodString>;
+    buildKey: z.ZodString;
+    createdAt: z.ZodString;
+    config: z.ZodObject<{
+        dataRoot: z.ZodString;
+        chunkTargetChars: z.ZodDefault<z.ZodNumber>;
+        evidenceMaxChars: z.ZodDefault<z.ZodNumber>;
+        catalogMaxChars: z.ZodDefault<z.ZodNumber>;
+        candidateLimit: z.ZodDefault<z.ZodNumber>;
+        maxInputBytes: z.ZodDefault<z.ZodNumber>;
+        maxUncompressedBytes: z.ZodDefault<z.ZodNumber>;
+        maxArchiveEntries: z.ZodDefault<z.ZodNumber>;
+        modelTimeoutMs: z.ZodDefault<z.ZodNumber>;
+        modelAttempts: z.ZodDefault<z.ZodNumber>;
+        visionConcurrency: z.ZodDefault<z.ZodNumber>;
+        maxModelCallsLimit: z.ZodDefault<z.ZodNumber>;
+        maxModelTokensLimit: z.ZodDefault<z.ZodNumber>;
+        uploadRetentionDays: z.ZodDefault<z.ZodNumber>;
+        maxModelCalls: z.ZodDefault<z.ZodNumber>;
+        maxModelTokens: z.ZodDefault<z.ZodNumber>;
+        imageTimeoutMs: z.ZodDefault<z.ZodNumber>;
+        visionMaxEdge: z.ZodDefault<z.ZodNumber>;
+        visionTileSize: z.ZodDefault<z.ZodNumber>;
+        visionMaxTiles: z.ZodDefault<z.ZodNumber>;
+        modelMaxOutputTokens: z.ZodDefault<z.ZodNumber>;
+        summaryMaxChars: z.ZodDefault<z.ZodNumber>;
+        figureMaxChars: z.ZodDefault<z.ZodNumber>;
+        pdfMaxPages: z.ZodDefault<z.ZodNumber>;
+        pdfMaxPagePixels: z.ZodDefault<z.ZodNumber>;
+        pdfPageTimeoutMs: z.ZodDefault<z.ZodNumber>;
+        libreOfficePath: z.ZodDefault<z.ZodString>;
+        visionProvider: z.ZodDefault<z.ZodString>;
+        visionModel: z.ZodDefault<z.ZodString>;
+        summaryProvider: z.ZodDefault<z.ZodString>;
+        summaryModel: z.ZodDefault<z.ZodString>;
+        skipImages: z.ZodDefault<z.ZodBoolean>;
+        acceptPartial: z.ZodDefault<z.ZodBoolean>;
+    }, z.core.$strict>;
+    jobId: z.ZodString;
+    kbId: z.ZodString;
+    state: z.ZodEnum<{
+        failed: "failed";
+        "needs-review": "needs-review";
+        queued: "queued";
+        preflight: "preflight";
+        converting: "converting";
+        describing: "describing";
+        splitting: "splitting";
+        indexing: "indexing";
+        validating: "validating";
+        publishing: "publishing";
+        "paused-budget": "paused-budget";
+        cancelled: "cancelled";
+        ready: "ready";
+    }>;
+    sourceOriginalHash: z.ZodString;
+    routes: z.ZodObject<{
+        vision: z.ZodNullable<z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+        }, z.core.$strip>>;
+        summary: z.ZodNullable<z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+    schema: z.ZodString;
+    schemaHash: z.ZodString;
+    overlays: z.ZodRecord<z.ZodString, z.ZodType<import("./data.ts").Overlay, unknown, z.core.$ZodTypeInternals<import("./data.ts").Overlay, unknown>>>;
+    sessionId: z.ZodNullable<z.ZodString>;
+    progress: z.ZodObject<{
+        totalPages: z.ZodOptional<z.ZodNumber>;
+        parsedPages: z.ZodOptional<z.ZodNumber>;
+        ocrPages: z.ZodOptional<z.ZodNumber>;
+        tables: z.ZodOptional<z.ZodNumber>;
+        blocks: z.ZodOptional<z.ZodNumber>;
+        currentPage: z.ZodOptional<z.ZodNumber>;
+        totalFigures: z.ZodOptional<z.ZodNumber>;
+        completedFigures: z.ZodOptional<z.ZodNumber>;
+        inflightFigures: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            occurrenceId: z.ZodString;
+            caption: z.ZodString;
+        }, z.core.$strip>>>;
+        totalChunks: z.ZodOptional<z.ZodNumber>;
+        indexedChunks: z.ZodOptional<z.ZodNumber>;
+        currentChunk: z.ZodOptional<z.ZodObject<{
+            title: z.ZodString;
+            headingPath: z.ZodArray<z.ZodString>;
+            chunkId: z.ZodString;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+    pendingCalls: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
+    legacyUsageCount: z.ZodDefault<z.ZodNumber>;
+    legacyCallCount: z.ZodDefault<z.ZodNumber>;
+    reportSummary: z.ZodOptional<z.ZodObject<{
+        issues: z.ZodNumber;
+        blocking: z.ZodNumber;
+    }, z.core.$strip>>;
+    failures: z.ZodArray<z.ZodObject<{
+        at: z.ZodString;
+        stage: z.ZodEnum<{
+            failed: "failed";
+            "needs-review": "needs-review";
+            queued: "queued";
+            preflight: "preflight";
+            converting: "converting";
+            describing: "describing";
+            splitting: "splitting";
+            indexing: "indexing";
+            validating: "validating";
+            publishing: "publishing";
+            "paused-budget": "paused-budget";
+            cancelled: "cancelled";
+            ready: "ready";
+        }>;
+        error: z.ZodString;
+    }, z.core.$strip>>;
+    events: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        at: z.ZodString;
+        state: z.ZodEnum<{
+            failed: "failed";
+            "needs-review": "needs-review";
+            queued: "queued";
+            preflight: "preflight";
+            converting: "converting";
+            describing: "describing";
+            splitting: "splitting";
+            indexing: "indexing";
+            validating: "validating";
+            publishing: "publishing";
+            "paused-budget": "paused-budget";
+            cancelled: "cancelled";
+            ready: "ready";
+        }>;
+    }, z.core.$strip>>>;
+    figureRefresh: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+        occurrenceId: z.ZodString;
+        token: z.ZodString;
+    }, z.core.$strip>>>;
+    visionCapability: z.ZodOptional<z.ZodString>;
+    pageStart: z.ZodOptional<z.ZodNumber>;
+    pageEnd: z.ZodOptional<z.ZodNumber>;
+    outputPath: z.ZodOptional<z.ZodString>;
+    recoveredAt: z.ZodOptional<z.ZodString>;
+    finishedAt: z.ZodOptional<z.ZodString>;
+    cancelRequested: z.ZodOptional<z.ZodBoolean>;
+    formatVersion: z.ZodLiteral<2>;
+    usage: z.ZodObject<{
+        calls: z.ZodNumber;
+        tokens: z.ZodNumber;
+        reportedCalls: z.ZodNumber;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export declare function loadJob(directory: string): Promise<JobRecord | null>;
+export declare function summarizeReport(report: z.infer<typeof reportSchema>): {
+    issues: number;
+    blocking: number;
+};
+export declare function saveReport(directory: string, report: z.infer<typeof reportSchema>): Promise<void>;
+export declare function saveCheckpoint(directory: string, job: JobRecord): Promise<void>;
+export declare function recoverUsage(job: Job): Promise<void>;
+export declare function readReport(job: Job): Promise<{
+    revision: string;
+    sourceHash: string;
+    issues: {
+        code: string;
+        message: string;
+        blocking: boolean;
+        page?: number | undefined;
+        kind?: string | undefined;
+        snippet?: string | undefined;
+        reason?: string | undefined;
+    }[];
+    documents: number;
+    blocks: number;
+    figures: number;
+    ignoredFigures: number;
+    unreviewedFigures: number;
+    chunks: number;
+    withinTarget: number;
+    oversized: {
+        chunkId: string;
+        chars: number;
+        reason: string;
+    }[];
+    usage: {
+        calls: number;
+        tokens: number;
+        reportedCalls: number;
+        records: z.core.util.JSONType[];
+    };
+    targetChars: number;
+    pdf?: {
+        pageCount: number;
+        selectedPages: {
+            start: number;
+            end: number;
+            count: number;
+            total: number;
+            partial: boolean;
+        };
+        outlineCount: number;
+        matchedBookmarks: number;
+        ocrPages: number;
+        pages: {
+            page: number;
+            status: "failed" | "needs-review" | "navigation" | "parsed";
+            ocr: boolean;
+            tables: number;
+            figures: number;
+            headings: {
+                text: string;
+                level: number;
+                basis?: string | undefined;
+            }[];
+            width?: number | undefined;
+            height?: number | undefined;
+            pageLabel?: string | null | undefined;
+            spanCount?: number | undefined;
+            artifactCount?: number | undefined;
+            excluded?: {
+                kind: string;
+                bbox: [number, number, number, number] | null;
+                text?: string | undefined;
+            }[] | undefined;
+            previewAssetId?: string | null | undefined;
+            error?: string | undefined;
+        }[];
+    } | undefined;
+} | undefined>;
+export declare function usagePage(job: Job, offset: number, limit: number, signal?: AbortSignal): Promise<{
+    records: z.core.util.JSONType[];
+    offset: number;
+    total: number;
+    next: number | null;
+}>;

package/lib/job-storage.js

@@ -0,0 +1,118 @@
+/** Small checkpoints own counters; existing call receipts resolve only unfinished accounting. */
+import { z } from 'zod';
+import { basename, join } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { jobSchema, reportSchema, usageSchema } from "./data.js";
+import { atomicWrite, checkAbort, code, readJson, safePath, tokensUsed } from "./util.js";
+export const checkpointSchema = jobSchema.omit({ report: true, usage: true }).extend({
+    formatVersion: z.literal(2), usage: usageSchema.omit({ records: true }),
+});
+const receiptSchema = z.object({ state: z.enum(['running', 'done', 'failed']), usage: z.object({ inputTokens: z.number().nonnegative(), outputTokens: z.number().nonnegative(), cacheReadTokens: z.number().optional(), cacheWriteTokens: z.number().optional(), totalTokens: z.number().optional(), reasoningTokens: z.number().optional() }).optional() });
+export async function loadJob(directory) {
+    const file = await safePath(directory, 'job.json', false);
+    let raw;
+    try {
+        raw = JSON.parse(await readFile(file, 'utf8'));
+    }
+    catch (error) {
+        if (code(error) === 'ENOENT')
+            return null;
+        throw error;
+    }
+    const small = checkpointSchema.safeParse(raw);
+    if (small.success) {
+        if (small.data.jobId !== basename(directory))
+            throw new Error('任务目录与身份不一致');
+        return jobSchema.parse({ ...small.data, usage: { ...small.data.usage, records: [] } });
+    }
+    // Unknown checkpoint versions must not fall back to interpreting a partial record.
+    if (typeof raw === 'object' && raw !== null && 'formatVersion' in raw)
+        throw new Error('不支持的任务检查点格式');
+    const record = jobSchema.parse(raw);
+    if (record.jobId !== basename(directory))
+        throw new Error('任务目录与身份不一致');
+    if (['queued', 'preflight', 'converting', 'describing', 'splitting', 'indexing', 'validating', 'publishing'].includes(record.state)) {
+        let total = 0;
+        let reported = 0;
+        for (let number = 1; number <= record.usage.calls; number++) {
+            const receipt = await readJson(await safePath(directory, join('calls', `${String(number).padStart(6, '0')}.json`), false), receiptSchema, null);
+            if (receipt?.usage) {
+                total += usageTokens(receipt.usage);
+                reported++;
+            }
+        }
+        record.usage.tokens = Math.max(record.usage.tokens, total);
+        record.usage.reportedCalls = Math.max(record.usage.reportedCalls, reported);
+    }
+    if (record.usage.records.length) {
+        await atomicWrite(await safePath(directory, 'legacy-usage.json', false), record.usage.records);
+        record.legacyUsageCount = record.usage.records.length;
+        record.legacyCallCount = record.usage.calls;
+        record.usage.records = [];
+    }
+    if (record.report) {
+        await saveReport(directory, record.report);
+        record.reportSummary = summarizeReport(record.report);
+        delete record.report;
+    }
+    await saveCheckpoint(directory, record);
+    return record;
+}
+export function summarizeReport(report) {
+    return { issues: report.issues.length, blocking: report.issues.filter(i => i.blocking).length };
+}
+export async function saveReport(directory, report) {
+    await atomicWrite(await safePath(directory, 'quality.json', false), report);
+}
+export async function saveCheckpoint(directory, job) {
+    const { report: _report, usage, ...small } = job;
+    const { records: _records, ...counters } = usage;
+    // Strip heavy values before cloning; JSON.stringify(job) would retain the original cost.
+    const snapshot = checkpointSchema.parse({ ...small, formatVersion: 2, usage: counters });
+    await atomicWrite(await safePath(directory, 'job.json', false), snapshot);
+}
+function usageTokens(usage) {
+    return tokensUsed({ inputTokens: usage?.inputTokens ?? 0, outputTokens: usage?.outputTokens ?? 0, cacheReadTokens: usage?.cacheReadTokens ?? 0, cacheWriteTokens: usage?.cacheWriteTokens ?? 0 });
+}
+export async function recoverUsage(job) {
+    if (new Set(job.pendingCalls).size !== job.pendingCalls.length || job.pendingCalls.some(n => n > job.usage.calls))
+        throw new Error('任务待结算调用编号无效');
+    for (const number of job.pendingCalls) {
+        const file = await safePath(job.directory, join('calls', `${String(number).padStart(6, '0')}.json`), false);
+        const receipt = await readJson(file, receiptSchema, null);
+        // A reserved call remains charged even when no completion/usage reached disk.
+        if (receipt?.usage) {
+            job.usage.tokens += usageTokens(receipt.usage);
+            job.usage.reportedCalls++;
+        }
+        job.pendingCalls = job.pendingCalls.filter(n => n !== number);
+    }
+    await job.checkpoint();
+}
+export async function readReport(job) {
+    if (!job.reportSummary)
+        return undefined;
+    return readJson(await safePath(job.directory, 'quality.json'), reportSchema);
+}
+export async function usagePage(job, offset, limit, signal) {
+    if (!Number.isInteger(offset) || offset < 0 || !Number.isInteger(limit) || limit < 1 || limit > 50)
+        throw new Error('用量分页参数无效');
+    const total = job.legacyUsageCount + Math.max(0, job.usage.calls - job.legacyCallCount);
+    const records = [];
+    const end = Math.min(total, offset + limit);
+    let legacy;
+    if (offset < job.legacyUsageCount)
+        legacy = await readJson(await safePath(job.directory, 'legacy-usage.json'), z.array(z.json()));
+    for (let index = offset; index < end; index++) {
+        checkAbort(signal);
+        if (index < job.legacyUsageCount) {
+            records.push(legacy?.[index] ?? null);
+            continue;
+        }
+        const number = job.legacyCallCount + index - job.legacyUsageCount + 1;
+        const file = await safePath(job.directory, join('calls', `${String(number).padStart(6, '0')}.json`), false);
+        const receipt = await readJson(file, receiptSchema, null);
+        records.push(z.json().parse(JSON.parse(JSON.stringify({ call: number, state: receipt?.state ?? 'reserved', usage: receipt?.usage ?? null }))));
+    }
+    return { records, offset, total, next: end < total ? end : null };
+}