@dsh-libing/knowledge-base 0.1.0 → 0.1.3

package/lib/job-storage.d.ts

@@ -0,0 +1,264 @@
+/** Small checkpoints own counters; existing call receipts resolve only unfinished accounting. */
+import { z } from 'zod';
+import { reportSchema } from './data.ts';
+import type { Job, JobRecord } from './data.ts';
+export declare const checkpointSchema: z.ZodObject<{
+    error: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+    title: z.ZodString;
+    updatedAt: z.ZodOptional<z.ZodString>;
+    pdf: z.ZodOptional<z.ZodObject<{
+        pageCount: z.ZodNumber;
+        selectedPages: z.ZodObject<{
+            start: z.ZodNumber;
+            end: z.ZodNumber;
+            count: z.ZodNumber;
+            total: z.ZodNumber;
+            partial: z.ZodBoolean;
+        }, z.core.$strip>;
+        outlineCount: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
+    sourceFormat: z.ZodDefault<z.ZodEnum<{
+        docx: "docx";
+        pdf: "pdf";
+    }>>;
+    parserVersion: z.ZodOptional<z.ZodString>;
+    sourceHash: z.ZodOptional<z.ZodString>;
+    docId: z.ZodString;
+    sourceVersion: z.ZodString;
+    filename: z.ZodString;
+    revision: z.ZodOptional<z.ZodString>;
+    buildKey: z.ZodString;
+    createdAt: z.ZodString;
+    config: z.ZodObject<{
+        dataRoot: z.ZodString;
+        chunkTargetChars: z.ZodDefault<z.ZodNumber>;
+        evidenceMaxChars: z.ZodDefault<z.ZodNumber>;
+        catalogMaxChars: z.ZodDefault<z.ZodNumber>;
+        candidateLimit: z.ZodDefault<z.ZodNumber>;
+        maxInputBytes: z.ZodDefault<z.ZodNumber>;
+        maxUncompressedBytes: z.ZodDefault<z.ZodNumber>;
+        maxArchiveEntries: z.ZodDefault<z.ZodNumber>;
+        modelTimeoutMs: z.ZodDefault<z.ZodNumber>;
+        modelAttempts: z.ZodDefault<z.ZodNumber>;
+        visionConcurrency: z.ZodDefault<z.ZodNumber>;
+        maxModelCallsLimit: z.ZodDefault<z.ZodNumber>;
+        maxModelTokensLimit: z.ZodDefault<z.ZodNumber>;
+        uploadRetentionDays: z.ZodDefault<z.ZodNumber>;
+        maxModelCalls: z.ZodDefault<z.ZodNumber>;
+        maxModelTokens: z.ZodDefault<z.ZodNumber>;
+        imageTimeoutMs: z.ZodDefault<z.ZodNumber>;
+        visionMaxEdge: z.ZodDefault<z.ZodNumber>;
+        visionTileSize: z.ZodDefault<z.ZodNumber>;
+        visionMaxTiles: z.ZodDefault<z.ZodNumber>;
+        modelMaxOutputTokens: z.ZodDefault<z.ZodNumber>;
+        summaryMaxChars: z.ZodDefault<z.ZodNumber>;
+        figureMaxChars: z.ZodDefault<z.ZodNumber>;
+        pdfMaxPages: z.ZodDefault<z.ZodNumber>;
+        pdfMaxPagePixels: z.ZodDefault<z.ZodNumber>;
+        pdfPageTimeoutMs: z.ZodDefault<z.ZodNumber>;
+        libreOfficePath: z.ZodDefault<z.ZodString>;
+        visionProvider: z.ZodDefault<z.ZodString>;
+        visionModel: z.ZodDefault<z.ZodString>;
+        summaryProvider: z.ZodDefault<z.ZodString>;
+        summaryModel: z.ZodDefault<z.ZodString>;
+        skipImages: z.ZodDefault<z.ZodBoolean>;
+        acceptPartial: z.ZodDefault<z.ZodBoolean>;
+    }, z.core.$strict>;
+    jobId: z.ZodString;
+    kbId: z.ZodString;
+    state: z.ZodEnum<{
+        failed: "failed";
+        "needs-review": "needs-review";
+        queued: "queued";
+        preflight: "preflight";
+        converting: "converting";
+        describing: "describing";
+        splitting: "splitting";
+        indexing: "indexing";
+        validating: "validating";
+        publishing: "publishing";
+        "paused-budget": "paused-budget";
+        cancelled: "cancelled";
+        ready: "ready";
+    }>;
+    sourceOriginalHash: z.ZodString;
+    routes: z.ZodObject<{
+        vision: z.ZodNullable<z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+        }, z.core.$strip>>;
+        summary: z.ZodNullable<z.ZodObject<{
+            provider: z.ZodString;
+            model: z.ZodString;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+    schema: z.ZodString;
+    schemaHash: z.ZodString;
+    overlays: z.ZodRecord<z.ZodString, z.ZodType<import("./data.ts").Overlay, unknown, z.core.$ZodTypeInternals<import("./data.ts").Overlay, unknown>>>;
+    sessionId: z.ZodNullable<z.ZodString>;
+    progress: z.ZodObject<{
+        totalPages: z.ZodOptional<z.ZodNumber>;
+        parsedPages: z.ZodOptional<z.ZodNumber>;
+        ocrPages: z.ZodOptional<z.ZodNumber>;
+        tables: z.ZodOptional<z.ZodNumber>;
+        blocks: z.ZodOptional<z.ZodNumber>;
+        currentPage: z.ZodOptional<z.ZodNumber>;
+        totalFigures: z.ZodOptional<z.ZodNumber>;
+        completedFigures: z.ZodOptional<z.ZodNumber>;
+        inflightFigures: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            occurrenceId: z.ZodString;
+            caption: z.ZodString;
+        }, z.core.$strip>>>;
+        totalChunks: z.ZodOptional<z.ZodNumber>;
+        indexedChunks: z.ZodOptional<z.ZodNumber>;
+        currentChunk: z.ZodOptional<z.ZodObject<{
+            title: z.ZodString;
+            headingPath: z.ZodArray<z.ZodString>;
+            chunkId: z.ZodString;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+    pendingCalls: z.ZodDefault<z.ZodArray<z.ZodNumber>>;
+    legacyUsageCount: z.ZodDefault<z.ZodNumber>;
+    legacyCallCount: z.ZodDefault<z.ZodNumber>;
+    reportSummary: z.ZodOptional<z.ZodObject<{
+        issues: z.ZodNumber;
+        blocking: z.ZodNumber;
+    }, z.core.$strip>>;
+    failures: z.ZodArray<z.ZodObject<{
+        at: z.ZodString;
+        stage: z.ZodEnum<{
+            failed: "failed";
+            "needs-review": "needs-review";
+            queued: "queued";
+            preflight: "preflight";
+            converting: "converting";
+            describing: "describing";
+            splitting: "splitting";
+            indexing: "indexing";
+            validating: "validating";
+            publishing: "publishing";
+            "paused-budget": "paused-budget";
+            cancelled: "cancelled";
+            ready: "ready";
+        }>;
+        error: z.ZodString;
+    }, z.core.$strip>>;
+    events: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        at: z.ZodString;
+        state: z.ZodEnum<{
+            failed: "failed";
+            "needs-review": "needs-review";
+            queued: "queued";
+            preflight: "preflight";
+            converting: "converting";
+            describing: "describing";
+            splitting: "splitting";
+            indexing: "indexing";
+            validating: "validating";
+            publishing: "publishing";
+            "paused-budget": "paused-budget";
+            cancelled: "cancelled";
+            ready: "ready";
+        }>;
+    }, z.core.$strip>>>;
+    figureRefresh: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+        occurrenceId: z.ZodString;
+        token: z.ZodString;
+    }, z.core.$strip>>>;
+    visionCapability: z.ZodOptional<z.ZodString>;
+    pageStart: z.ZodOptional<z.ZodNumber>;
+    pageEnd: z.ZodOptional<z.ZodNumber>;
+    outputPath: z.ZodOptional<z.ZodString>;
+    recoveredAt: z.ZodOptional<z.ZodString>;
+    finishedAt: z.ZodOptional<z.ZodString>;
+    cancelRequested: z.ZodOptional<z.ZodBoolean>;
+    formatVersion: z.ZodLiteral<2>;
+    usage: z.ZodObject<{
+        calls: z.ZodNumber;
+        tokens: z.ZodNumber;
+        reportedCalls: z.ZodNumber;
+    }, z.core.$strip>;
+}, z.core.$strip>;
+export declare function loadJob(directory: string): Promise<JobRecord | null>;
+export declare function summarizeReport(report: z.infer<typeof reportSchema>): {
+    issues: number;
+    blocking: number;
+};
+export declare function saveReport(directory: string, report: z.infer<typeof reportSchema>): Promise<void>;
+export declare function saveCheckpoint(directory: string, job: JobRecord): Promise<void>;
+export declare function recoverUsage(job: Job): Promise<void>;
+export declare function readReport(job: Job): Promise<{
+    revision: string;
+    sourceHash: string;
+    issues: {
+        code: string;
+        message: string;
+        blocking: boolean;
+        page?: number | undefined;
+        kind?: string | undefined;
+        snippet?: string | undefined;
+        reason?: string | undefined;
+    }[];
+    documents: number;
+    blocks: number;
+    figures: number;
+    ignoredFigures: number;
+    unreviewedFigures: number;
+    chunks: number;
+    withinTarget: number;
+    oversized: {
+        chunkId: string;
+        chars: number;
+        reason: string;
+    }[];
+    usage: {
+        calls: number;
+        tokens: number;
+        reportedCalls: number;
+        records: z.core.util.JSONType[];
+    };
+    targetChars: number;
+    pdf?: {
+        pageCount: number;
+        selectedPages: {
+            start: number;
+            end: number;
+            count: number;
+            total: number;
+            partial: boolean;
+        };
+        outlineCount: number;
+        matchedBookmarks: number;
+        ocrPages: number;
+        pages: {
+            page: number;
+            status: "failed" | "needs-review" | "navigation" | "parsed";
+            ocr: boolean;
+            tables: number;
+            figures: number;
+            headings: {
+                text: string;
+                level: number;
+                basis?: string | undefined;
+            }[];
+            width?: number | undefined;
+            height?: number | undefined;
+            pageLabel?: string | null | undefined;
+            spanCount?: number | undefined;
+            artifactCount?: number | undefined;
+            excluded?: {
+                kind: string;
+                bbox: [number, number, number, number] | null;
+                text?: string | undefined;
+            }[] | undefined;
+            previewAssetId?: string | null | undefined;
+            error?: string | undefined;
+        }[];
+    } | undefined;
+} | undefined>;
+export declare function usagePage(job: Job, offset: number, limit: number, signal?: AbortSignal): Promise<{
+    records: z.core.util.JSONType[];
+    offset: number;
+    total: number;
+    next: number | null;
+}>;

package/lib/job-storage.js

@@ -0,0 +1,115 @@
+/** Small checkpoints own counters; existing call receipts resolve only unfinished accounting. */
+import { z } from 'zod';
+import { basename, join } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { jobSchema, reportSchema, usageSchema } from "./data.js";
+import { atomicWrite, checkAbort, code, readJson, safePath, tokensUsed } from "./util.js";
+export const checkpointSchema = jobSchema.omit({ report: true, usage: true }).extend({
+    formatVersion: z.literal(2), usage: usageSchema.omit({ records: true }),
+});
+const receiptSchema = z.object({ state: z.enum(['running', 'done', 'failed']), usage: z.object({ inputTokens: z.number().nonnegative(), outputTokens: z.number().nonnegative(), cacheReadTokens: z.number().optional(), cacheWriteTokens: z.number().optional(), totalTokens: z.number().optional(), reasoningTokens: z.number().optional() }).optional() });
+export async function loadJob(directory) {
+    const file = await safePath(directory, 'job.json', false);
+    let raw;
+    try {
+        raw = JSON.parse(await readFile(file, 'utf8'));
+    }
+    catch (error) {
+        if (code(error) === 'ENOENT')
+            return null;
+        throw error;
+    }
+    const small = checkpointSchema.safeParse(raw);
+    if (small.success) {
+        if (small.data.jobId !== basename(directory))
+            throw new Error('任务目录与身份不一致');
+        return jobSchema.parse({ ...small.data, usage: { ...small.data.usage, records: [] } });
+    }
+    // Unknown checkpoint versions must not fall back to interpreting a partial record.
+    if (typeof raw === 'object' && raw !== null && 'formatVersion' in raw)
+        throw new Error('不支持的任务检查点格式');
+    const record = jobSchema.parse(raw);
+    if (record.jobId !== basename(directory))
+        throw new Error('任务目录与身份不一致');
+    if (['queued', 'preflight', 'converting', 'describing', 'splitting', 'indexing', 'validating', 'publishing'].includes(record.state)) {
+        let total = 0, reported = 0;
+        for (let number = 1; number <= record.usage.calls; number++) {
+            const receipt = await readJson(await safePath(directory, join('calls', `${String(number).padStart(6, '0')}.json`), false), receiptSchema, null);
+            if (receipt?.usage) {
+                total += usageTokens(receipt.usage);
+                reported++;
+            }
+        }
+        record.usage.tokens = Math.max(record.usage.tokens, total);
+        record.usage.reportedCalls = Math.max(record.usage.reportedCalls, reported);
+    }
+    if (record.usage.records.length) {
+        await atomicWrite(await safePath(directory, 'legacy-usage.json', false), record.usage.records);
+        record.legacyUsageCount = record.usage.records.length;
+        record.legacyCallCount = record.usage.calls;
+        record.usage.records = [];
+    }
+    if (record.report) {
+        await saveReport(directory, record.report);
+        record.reportSummary = summarizeReport(record.report);
+        delete record.report;
+    }
+    await saveCheckpoint(directory, record);
+    return record;
+}
+export function summarizeReport(report) {
+    return { issues: report.issues.length, blocking: report.issues.filter(i => i.blocking).length };
+}
+export async function saveReport(directory, report) {
+    await atomicWrite(await safePath(directory, 'quality.json', false), report);
+}
+export async function saveCheckpoint(directory, job) {
+    const { report: _report, usage, ...small } = job;
+    const { records: _records, ...counters } = usage;
+    // Strip heavy values before cloning; JSON.stringify(job) would retain the original cost.
+    const snapshot = checkpointSchema.parse({ ...small, formatVersion: 2, usage: counters });
+    await atomicWrite(await safePath(directory, 'job.json', false), snapshot);
+}
+function usageTokens(usage) {
+    return tokensUsed({ inputTokens: usage?.inputTokens ?? 0, outputTokens: usage?.outputTokens ?? 0, cacheReadTokens: usage?.cacheReadTokens ?? 0, cacheWriteTokens: usage?.cacheWriteTokens ?? 0 });
+}
+export async function recoverUsage(job) {
+    if (new Set(job.pendingCalls).size !== job.pendingCalls.length || job.pendingCalls.some(n => n > job.usage.calls))
+        throw new Error('任务待结算调用编号无效');
+    for (const number of job.pendingCalls) {
+        const file = await safePath(job.directory, join('calls', `${String(number).padStart(6, '0')}.json`), false);
+        const receipt = await readJson(file, receiptSchema, null);
+        // A reserved call remains charged even when no completion/usage reached disk.
+        if (receipt?.usage) {
+            job.usage.tokens += usageTokens(receipt.usage);
+            job.usage.reportedCalls++;
+        }
+        job.pendingCalls = job.pendingCalls.filter(n => n !== number);
+    }
+    await job.checkpoint();
+}
+export async function readReport(job) {
+    if (!job.reportSummary)
+        return undefined;
+    return readJson(await safePath(job.directory, 'quality.json'), reportSchema);
+}
+export async function usagePage(job, offset, limit, signal) {
+    if (!Number.isInteger(offset) || offset < 0 || !Number.isInteger(limit) || limit < 1 || limit > 50)
+        throw new Error('用量分页参数无效');
+    const total = job.legacyUsageCount + Math.max(0, job.usage.calls - job.legacyCallCount), records = [];
+    const end = Math.min(total, offset + limit);
+    let legacy;
+    if (offset < job.legacyUsageCount)
+        legacy = await readJson(await safePath(job.directory, 'legacy-usage.json'), z.array(z.json()));
+    for (let index = offset; index < end; index++) {
+        checkAbort(signal);
+        if (index < job.legacyUsageCount) {
+            records.push(legacy?.[index] ?? null);
+            continue;
+        }
+        const number = job.legacyCallCount + index - job.legacyUsageCount + 1;
+        const file = await safePath(job.directory, join('calls', `${String(number).padStart(6, '0')}.json`), false), receipt = await readJson(file, receiptSchema, null);
+        records.push(z.json().parse(JSON.parse(JSON.stringify({ call: number, state: receipt?.state ?? 'reserved', usage: receipt?.usage ?? null }))));
+    }
+    return { records, offset, total, next: end < total ? end : null };
+}