@dsh-libing/knowledge-base 0.1.0 → 0.1.3

package/lib/jobs.js

@@ -14,6 +14,7 @@ import { convertPdf } from "./document/pdf.js";
 import { PdfSession, pageRange } from "./document/pdf-session.js";
 import { PDF_PARSER_VERSION } from "./document/pdf-layout.js";
 import { BudgetError } from "./models.js";
+import { loadJob, saveCheckpoint, saveReport, readReport, recoverUsage, summarizeReport, usagePage } from "./job-storage.js";
 const RESUMABLE = new Set(['failed', 'needs-review', 'paused-budget', 'cancelled']);
 export class Jobs {
     store;
@@ -50,33 +51,51 @@ export class Jobs {
         }
         const visible = new Set((await this.store.list()).map(k => k.kbId));
         for (const dir of await this.store.jobDirectories()) {
-            const record = await readJson(join(dir, 'job.json'), jobSchema, null);
+            const record = await loadJob(dir);
             if (!record || !visible.has(record.kbId))
                 continue;
+            if (record.jobId !== basename(dir))
+                throw new Error('任务目录与身份不一致');
             const job = this.attach(record, dir);
             this.jobs.set(job.jobId, job);
+            if (job.pendingCalls.length)
+                await recoverUsage(job);
             if (ACTIVE.has(job.state)) {
                 job.state = 'queued';
                 job.recoveredAt = now();
                 await job.checkpoint();
             }
         }
+        await this.store.cleanUploads();
         this.pump();
     }
     attach(record, directory) {
         let tail = Promise.resolve();
-        return { ...record, directory, cacheDir: join(this.store.root, 'cache'), checkpoint: async () => {
-                const snapshot = jobSchema.parse(JSON.parse(JSON.stringify(this.jobs.get(record.jobId) || record)));
-                const pending = tail.then(() => atomicWrite(join(directory, 'job.json'), snapshot));
+        const job = { ...record, directory, cacheDir: join(this.store.root, 'cache'), checkpoint: async () => {
+                const { report: _report, usage, ...value } = job;
+                const snapshot = jobSchema.parse({ ...value, usage: { ...usage, records: [] } });
+                const pending = tail.then(() => saveCheckpoint(directory, snapshot));
                 tail = pending.catch(() => { });
                 await pending;
             } };
+        return job;
     }
     public(job) {
-        const { checkpoint: _checkpoint, directory: _directory, cacheDir: _cacheDir, schema: _schema, overlays: _overlays, ...value } = job;
+        const { checkpoint: _checkpoint, directory: _directory, cacheDir: _cacheDir, schema: _schema, overlays: _overlays, report: _report, ...value } = job;
         return { ...value, outputPath: value.outputPath || null };
     }
-    list(kbId) { return [...this.jobs.values()].filter(j => !kbId || j.kbId === kbId).sort((a, b) => b.createdAt.localeCompare(a.createdAt)).map(j => this.public(j)); }
+    list(kbId, offset = 0, limit = 20) {
+        if (!Number.isInteger(offset) || offset < 0 || !Number.isInteger(limit) || limit < 1 || limit > 50)
+            throw new Error('任务分页参数无效');
+        const all = [...this.jobs.values()].filter(j => !kbId || j.kbId === kbId).sort((a, b) => Number(ACTIVE.has(b.state)) - Number(ACTIVE.has(a.state)) || b.createdAt.localeCompare(a.createdAt) || b.jobId.localeCompare(a.jobId));
+        const page = all.slice(offset, offset + limit).map(job => {
+            const { report: _report, events: _events, failures: _failures, ...value } = this.public(job);
+            return { ...value, events: [], failures: [], usage: { ...value.usage, records: [] } };
+        });
+        return { jobs: page, jobTotal: all.length, jobOffset: offset, activeJobs: [...this.jobs.values()].filter(j => ACTIVE.has(j.state)).length };
+    }
+    async details(jobId, includeReport = true) { const job = this.get(jobId), { report: _report, ...value } = this.public(job); return { ...value, ...includeReport ? { report: await readReport(job) } : {} }; }
+    async usage(jobId, offset, limit, signal) { return usagePage(this.get(jobId), offset, limit, signal); }
     get(jobId) {
         const job = this.jobs.get(id(jobId, 'jobId'));
         if (!job)
@@ -198,7 +217,7 @@ export class Jobs {
             const { nodes, chunks } = createChunks(document, info, job.config);
             for (const chunk of chunks)
                 chunk.summary = clip(chunk.segments.filter(s => ['paragraph', 'list', 'note'].includes(s.type)).map(s => s.text).join(' ').replace(/\s+/g, ' ') || chunk.title, job.config.summaryMaxChars);
-            manifest = { formatVersion: 1, revision: required(job.revision, '草稿版本'), buildKey: job.buildKey, createdAt: job.createdAt, config: { chunkTargetChars: job.config.chunkTargetChars }, catalogs: {}, entities: {}, report: job.report ?? { revision: required(job.revision), sourceHash: required(job.sourceHash), issues: document.warnings, documents: 1, blocks: document.blocks.length, figures: document.assets.length, ignoredFigures: 0, unreviewedFigures: document.assets.length, chunks: chunks.length, withinTarget: chunks.filter(c => c.fullChars <= job.config.chunkTargetChars).length, oversized: [], usage: job.usage, targetChars: job.config.chunkTargetChars }, coverageIncomplete: true, documents: [{ ...info, nodes, chunks, assets: document.assets, figures: document.blocks.filter(b => b.type === 'figure').map(b => required(b.figure)) }] };
+            manifest = { formatVersion: 1, revision: required(job.revision, '草稿版本'), buildKey: job.buildKey, createdAt: job.createdAt, config: { chunkTargetChars: job.config.chunkTargetChars }, catalogs: {}, entities: {}, report: await readReport(job) ?? { revision: required(job.revision), sourceHash: required(job.sourceHash), issues: document.warnings, documents: 1, blocks: document.blocks.length, figures: document.assets.length, ignoredFigures: 0, unreviewedFigures: document.assets.length, chunks: chunks.length, withinTarget: chunks.filter(c => c.fullChars <= job.config.chunkTargetChars).length, oversized: [], usage: job.usage, targetChars: job.config.chunkTargetChars }, coverageIncomplete: true, documents: [{ ...info, nodes, chunks, assets: document.assets, figures: document.blocks.filter(b => b.type === 'figure').map(b => required(b.figure)) }] };
         }
         return { root, manifest, revision: required(job.revision), kbId: job.kbId, jobId };
     }
@@ -249,20 +268,25 @@ export class Jobs {
             const { catalogs, entities } = await writeIndexes(stage, docs, job.config, job.revision);
             await this.phase(job, 'validating');
             checkAbort(signal);
-            for (const doc of docs)
+            const indexedChunks = new Set(Object.values(catalogs).flatMap(c => c.pages.flatMap(p => p.entries.flatMap(e => e.chunkId ? [e.chunkId] : []))));
+            for (const doc of docs) {
+                const assetIds = new Set(doc.assets.map(a => a.assetId));
                 for (const chunk of doc.chunks) {
                     if (hash(await readFile(await safePath(stage, chunk.path), 'utf8')) !== chunk.contentHash)
                         throw new Error('生成片段哈希不一致');
-                    if (!Object.values(catalogs).some(c => c.pages.some(p => p.entries.some(e => e.chunkId === chunk.chunkId))))
+                    if (!indexedChunks.has(chunk.chunkId))
                         throw new Error('索引遗漏片段');
                     for (const assetId of chunk.assetRefs)
-                        if (!doc.assets.some(a => a.assetId === assetId))
+                        if (!assetIds.has(assetId))
                             throw new Error('片段引用不存在的图片');
                 }
+            }
             const report = { revision: job.revision, sourceHash: required(job.sourceHash, '原件哈希'), issues, documents: docs.length, blocks: document.blocks.length, figures: figureResults.length, ignoredFigures: figureResults.filter(f => f.status === 'ignored').length, unreviewedFigures: figureResults.filter(f => !f.humanReviewed && f.status !== 'decoration').length, chunks: chunks.length, withinTarget: chunks.filter(c => c.fullChars <= job.config.chunkTargetChars).length, oversized: chunks.filter(c => c.oversizedReason).map(c => ({ chunkId: c.chunkId, chars: c.fullChars, reason: required(c.oversizedReason) })), usage: job.usage, targetChars: job.config.chunkTargetChars, ...document.sourceFormat === 'pdf' ? { pdf: { pageCount: required(document.pageCount), selectedPages: required(document.selectedPages), outlineCount: document.audit.outlineCount, matchedBookmarks: document.audit.matchedBookmarks, ocrPages: document.audit.pages.filter(p => p.ocr).length, pages: document.audit.pages } } : {} };
             await atomicWrite(join(stage, 'reports/quality.json'), report);
             await atomicWrite(join(stage, 'reports/quality.md'), `# 构建质量报告\n\n文档 ${report.documents}；本次片段 ${report.chunks}；目标内 ${report.withinTarget}；图片 ${report.figures}；已忽略 ${report.ignoredFigures}；未人工核验 ${report.unreviewedFigures}。\n\n` + issues.map(i => `- ${i.blocking ? '需要处理' : '提示'}：${i.message}`).join('\n') + '\n');
-            job.report = report;
+            await saveReport(job.directory, report);
+            delete job.report;
+            job.reportSummary = summarizeReport(report);
             await job.checkpoint();
             const documents = docs.map(d => ({ ...d, chunks: d.chunks.map(({ content: _content, body: _body, ...chunk }) => chunk) }));
             const manifest = { formatVersion: 1, revision: job.revision, buildKey: job.buildKey, createdAt: now(), documents, catalogs, entities, coverageIncomplete: docs.some(d => d.coverageIncomplete), config: { chunkTargetChars: job.config.chunkTargetChars }, report };
@@ -292,7 +316,7 @@ export class Jobs {
     async actionUnlocked(jobId, action, options = {}) {
         const job = this.get(jobId);
         if (action === 'status')
-            return this.public(job);
+            return this.details(jobId);
         if (action === 'cancel') {
             if (job.state === 'ready')
                 throw new Error('已发布任务不能取消');
@@ -307,18 +331,18 @@ export class Jobs {
         else if (action === 'retry') {
             if (!RESUMABLE.has(job.state) || this.controllers.has(jobId))
                 throw new Error('任务尚未退出或不能重试');
+            if (options.maxModelCalls !== undefined && (!Number.isSafeInteger(options.maxModelCalls) || options.maxModelCalls < 1 || options.maxModelCalls > this.config.maxModelCallsLimit))
+                throw new Error('模型调用预算超过宿主上限或无效');
+            if (options.maxModelTokens !== undefined && (!Number.isSafeInteger(options.maxModelTokens) || options.maxModelTokens < 1 || options.maxModelTokens > this.config.maxModelTokensLimit))
+                throw new Error('模型 token 预算超过宿主上限或无效');
+            job.config.maxModelCallsLimit = this.config.maxModelCallsLimit;
+            job.config.maxModelTokensLimit = this.config.maxModelTokensLimit;
             if (options.acceptPartial === true)
                 job.config.acceptPartial = true;
-            if (options.maxModelCalls !== undefined) {
-                if (!Number.isSafeInteger(options.maxModelCalls) || options.maxModelCalls < 1)
-                    throw new Error('模型调用预算无效');
+            if (options.maxModelCalls !== undefined)
                 job.config.maxModelCalls = options.maxModelCalls;
-            }
-            if (options.maxModelTokens !== undefined) {
-                if (!Number.isSafeInteger(options.maxModelTokens) || options.maxModelTokens < 1)
-                    throw new Error('模型 token 预算无效');
+            if (options.maxModelTokens !== undefined)
                 job.config.maxModelTokens = options.maxModelTokens;
-            }
             job.cancelRequested = false;
             job.error = null;
             await this.phase(job, 'queued');
@@ -373,7 +397,7 @@ export class Jobs {
         return this.mutate(async () => {
             const kb = await this.store.restore(kbId);
             for (const dir of await this.store.jobDirectories()) {
-                const record = await readJson(join(dir, 'job.json'), jobSchema, null);
+                const record = await loadJob(dir);
                 if (record?.kbId === kb.kbId)
                     this.jobs.set(record.jobId, this.attach(record, dir));
             }
@@ -395,7 +419,7 @@ export class Jobs {
             const jobIds = related.map(j => j.jobId);
             if (!kb.purge)
                 for (const dir of await this.store.jobDirectories()) {
-                    const record = await readJson(await safePath(dir, 'job.json', false), jobSchema, null);
+                    const record = await loadJob(dir);
                     if (record?.kbId !== kb.kbId)
                         continue;
                     const key = id(basename(dir), 'jobId');
@@ -417,7 +441,7 @@ export class Jobs {
             else
                 await new Promise(r => setTimeout(r, 0));
         }
-        return this.public(this.get(jobId));
+        return this.details(jobId);
     }
     stop() {
         this.stopped = true;

package/lib/models.d.ts

@@ -1,6 +1,6 @@
 import type { Context } from '@deepseek-ai/cordis';
 import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment';
-import { z } from 'zod';
+import type { z } from 'zod';
 import type { Settings } from './config.ts';
 import type { Job, ModelRoute } from './data.ts';
 export type PreparedImage = {

package/lib/models.js

@@ -1,5 +1,4 @@
 import { SessionId } from '@deepseek-ai/dsh-session';
-import { z } from 'zod';
 import { atomicWrite, checkAbort, chars, message, now, readJson, tokensUsed } from "./util.js";
 import { join } from 'node:path';
 export class BudgetError extends Error {
@@ -27,9 +26,11 @@ export class Models {
         const route = job.routes[kind];
         if (!route)
             throw new Error(`${kind} 模型不可用`);
-        if (job.usage.calls >= job.config.maxModelCalls || job.usage.tokens >= job.config.maxModelTokens)
+        if (job.usage.calls >= Math.min(job.config.maxModelCalls, this.config.maxModelCallsLimit) || job.usage.tokens >= Math.min(job.config.maxModelTokens, this.config.maxModelTokensLimit))
             throw new BudgetError('已达到本次构建模型调用/实际 token 预算；调整预算后重试可继续');
-        const number = ++job.usage.calls, file = join(job.directory, 'calls', `${String(number).padStart(6, '0')}.json`);
+        const number = ++job.usage.calls;
+        job.pendingCalls.push(number);
+        const file = join(job.directory, 'calls', `${String(number).padStart(6, '0')}.json`);
         const controller = new AbortController(), timeout = setTimeout(() => controller.abort(new Error('模型调用超时')), job.config.modelTimeoutMs);
         const combined = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
         const request = { kind, route, prompt, system, images, startedAt: now(), maxTokens: job.config.modelMaxOutputTokens };
@@ -41,8 +42,8 @@ export class Models {
             job.usage.tokens += tokensUsed(usage);
             if (usage) {
                 job.usage.reportedCalls++;
-                job.usage.records.push(z.json().parse(JSON.parse(JSON.stringify(usage))));
             }
+            job.pendingCalls = job.pendingCalls.filter(n => n !== number);
         };
         try {
             await job.checkpoint();
@@ -68,14 +69,14 @@ export class Models {
                 throw new Error(finish && 'failure' in finish ? finish.failure.message : `模型未正常完成：${finish?.kind ?? '缺少结束事件'}`);
             if (!response.trim())
                 throw new Error('模型返回空内容');
-            account();
             await atomicWrite(file, { request, state: 'done', response, usage, finishedAt: now() });
+            account();
             await job.checkpoint();
             return response;
         }
         catch (error) {
-            account();
             await atomicWrite(file, { request, state: 'failed', response, usage, error: message(error), finishedAt: now() });
+            account();
             await job.checkpoint();
             throw error;
         }
@@ -101,7 +102,7 @@ export class Models {
         if (job.config.skipImages)
             return;
         const info = await this.resolve(job.routes.vision, 'vision', signal);
-        const { Jimp } = await import('jimp'), data = await new Jimp({ width: 32, height: 32, color: 0xffffffff }).getBuffer('image/png');
+        const data = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAQ0lEQVR4AcXBAQEAAAiAIPP/5+qFMPsISUxiEpOYxCQmMYlJTGISk5jEJCYxiUlMYhKTmMQkJjGJSUxiEpOYxCQmsQNNiAQ8n/3A9AAAAABJRU5ErkJggg==', 'base64');
         const attachment = await this.ctx.attachments.saveImage({ data, mediaType: 'image/png', name: 'kb-vision-preflight.png' });
         const response = await this.call(job, 'vision', '这是图片接口连通性检查。能读取图像时只返回 READY。', [{ attachment, coordinates: { kind: 'preflight' } }], signal, '检查是否实际收到图片，简短回答。');
         if (response.trim() !== 'READY')

package/lib/retrieval.js

@@ -88,7 +88,7 @@ export class Retrieval {
             }
             return bounded(value, this.config.catalogMaxChars);
         }
-        const kb = await this.store.kb(args.kbId), snapshot = await this.store.revision(kb.kbId, args.revision);
+        const { kb, snapshot } = await this.store.open(args.kbId, args.revision);
         const catalogId = args.docId ? `${id(args.docId)}-${args.nodeId ? id(args.nodeId) : 'root'}` : 'root';
         const catalog = snapshot.manifest.catalogs[catalogId];
         if (!catalog)
@@ -102,13 +102,13 @@ export class Retrieval {
         return bounded(value, this.config.catalogMaxChars + 1000);
     }
     async search(args) {
-        const kb = await this.store.kb(args.kbId), snapshot = await this.store.revision(kb.kbId, args.revision);
+        const { kb, snapshot } = await this.store.open(args.kbId, args.revision);
         const terms = queryTerms(args.query, args.keywords), scored = [];
         for (const doc of snapshot.manifest.documents) {
             if (args.docId && doc.docId !== args.docId)
                 continue;
             for (const chunk of doc.chunks) {
-                const content = await this.store.chunkContent(snapshot, chunk), body = content.toLowerCase();
+                const { content, lower: body } = await this.store.chunkView(snapshot, chunk);
                 const hits = terms.filter(t => body.includes(t)), title = chunk.headingPath.join('/').toLowerCase(), keys = (chunk.keywords || []).map(k => k.toLowerCase());
                 if (!hits.length)
                     continue;
@@ -122,7 +122,9 @@ export class Retrieval {
             throw new Error('limit 必须为 1–30');
         const value = { kbId: kb.kbId, revision: snapshot.revision, query: args.query, terms, candidates: [], evidence: [], warnings: [], truncated: scored.length > limit, totalCandidates: scored.length, continuation: [] };
         const top = scored.slice(0, limit);
-        for (const { doc, chunk, score, hits, content } of top) {
+        for (const { doc, chunk, score, hits } of top) {
+            // Cached scoring is navigation only; returned evidence is always reread and hash-verified.
+            const content = await this.store.chunkContent(snapshot, chunk);
             const candidate = { chunkId: chunk.chunkId, score, matchedTerms: hits, title: chunk.title, summary: chunk.summary, fullChars: chunk.fullChars, reviewStatus: chunk.reviewStatus, source: source(required(doc), chunk, snapshot.revision) };
             value.candidates.push(candidate);
             const remaining = this.config.evidenceMaxChars - chars(JSON.stringify(value)) - 800;
@@ -145,10 +147,12 @@ export class Retrieval {
                 break;
             }
         }
+        await this.store.kb(kb.kbId); // Recheck visibility after potentially long scoring/evidence reads.
         return bounded(value, this.config.evidenceMaxChars);
     }
     async read(args) {
         const kb = await this.store.kb(args.kbId), snapshot = await this.store.readChunk(kb.kbId, args.revision, args.chunkId);
+        await this.store.kb(kb.kbId);
         return this.readSnapshot(kb.kbId, snapshot, args);
     }
     readSnapshot(kbId, snapshot, args) {

package/lib/revision-cache.d.ts

@@ -0,0 +1,24 @@
+import type { Manifest } from './data.ts';
+/** Bounded hot revision payloads. Every lookup checks file identity; evidence reads bypass reuse. */
+export declare class RevisionCache {
+    readonly maxBytes: number;
+    readonly maxEntries: number;
+    private entries;
+    bytes: number;
+    reads: number;
+    hits: number;
+    parses: number;
+    get size(): number;
+    constructor(maxBytes?: number, maxEntries?: number);
+    clear(): void;
+    private drop;
+    private put;
+    private stamp;
+    private lookup;
+    private read;
+    manifest(file: string): Promise<Manifest>;
+    chunk(file: string, digest: string, fresh?: boolean): Promise<{
+        content: string;
+        lower: string;
+    }>;
+}

package/lib/revision-cache.js

@@ -0,0 +1,100 @@
+import { stat, readFile } from 'node:fs/promises';
+import { manifestSchema } from "./data.js";
+import { hash } from "./util.js";
+function freeze(value) { if (value && typeof value === 'object') {
+    for (const child of Object.values(value))
+        freeze(child);
+    Object.freeze(value);
+} return value; }
+/** Bounded hot revision payloads. Every lookup checks file identity; evidence reads bypass reuse. */
+export class RevisionCache {
+    maxBytes;
+    maxEntries;
+    entries = new Map();
+    bytes = 0;
+    reads = 0;
+    hits = 0;
+    parses = 0;
+    get size() { return this.entries.size; }
+    constructor(maxBytes = 32 * 1024 * 1024, maxEntries = 256) {
+        this.maxBytes = maxBytes;
+        this.maxEntries = maxEntries;
+    }
+    clear() { this.entries.clear(); this.bytes = 0; }
+    drop(file) { const old = this.entries.get(file); if (old)
+        this.bytes -= old.cost; this.entries.delete(file); }
+    put(file, entry) {
+        this.drop(file);
+        if (entry.cost > this.maxBytes || this.maxEntries < 1)
+            return;
+        while (this.entries.size >= this.maxEntries || this.bytes + entry.cost > this.maxBytes) {
+            const first = this.entries.keys().next().value;
+            if (first === undefined)
+                break;
+            this.drop(first);
+        }
+        this.entries.set(file, entry);
+        this.bytes += entry.cost;
+    }
+    async stamp(file) {
+        const info = await stat(file, { bigint: true });
+        if (!info.isFile())
+            throw new Error('版本文件必须是普通文件');
+        return `${info.dev}:${info.ino}:${info.size}:${info.mtimeNs}:${info.ctimeNs}`;
+    }
+    async lookup(file) {
+        let stamp;
+        try {
+            stamp = await this.stamp(file);
+        }
+        catch (error) {
+            this.drop(file);
+            throw error;
+        }
+        const entry = this.entries.get(file);
+        if (entry?.stamp === stamp) {
+            this.drop(file);
+            this.entries.set(file, entry);
+            this.bytes += entry.cost;
+            return { stamp, entry };
+        }
+        this.drop(file);
+        return { stamp, entry: undefined };
+    }
+    async read(file, stamp) {
+        this.reads++;
+        const content = await readFile(file, 'utf8');
+        if (await this.stamp(file) !== stamp) {
+            this.drop(file);
+            throw new Error('版本文件读取期间变化，请重试');
+        }
+        return content;
+    }
+    async manifest(file) {
+        const { stamp, entry } = await this.lookup(file);
+        if (entry?.kind === 'manifest') {
+            this.hits++;
+            return entry.value;
+        }
+        const raw = await this.read(file, stamp);
+        this.parses++;
+        const value = freeze(manifestSchema.parse(JSON.parse(raw)));
+        this.put(file, { kind: 'manifest', stamp, value, cost: raw.length * 8 + 512 });
+        return value;
+    }
+    async chunk(file, digest, fresh = false) {
+        const { stamp, entry } = await this.lookup(file);
+        if (!fresh && entry?.kind === 'chunk' && entry.digest === digest) {
+            this.hits++;
+            return { content: entry.value, lower: entry.lower };
+        }
+        const content = await this.read(file, stamp);
+        if (hash(content) !== digest) {
+            this.drop(file);
+            throw new Error('片段内容与清单不一致，请重建');
+        }
+        const lower = content.toLowerCase();
+        this.put(file, { kind: 'chunk', stamp, value: content, lower, digest, cost: (content.length + lower.length) * 2 + 512 });
+        return { content, lower };
+    }
+}

package/lib/routes.js

@@ -1,7 +1,7 @@
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { z } from 'zod';
-import { API, browserBuildSchema, jobActionSchema, catalogQuerySchema, readQuerySchema, searchSchema, assetQuerySchema, revisionQuerySchema } from "./shared.js";
+import { API, jobListQuerySchema, jobDetailsQuerySchema, jobUsageQuerySchema, browserBuildSchema, jobActionSchema, catalogQuerySchema, readQuerySchema, searchSchema, assetQuerySchema, revisionQuerySchema } from "./shared.js";
 import { figureDataSchema, overlaySchema } from "./data.js";
 import { inspectPdf } from "./document/pdf-session.js";
 import { hash, safePath, message, required, fileBytes, checkAbort } from "./util.js";
@@ -69,16 +69,19 @@ export function routes(runtime, ctx) {
             throw new Error('PDF 原件不存在于所选版本中');
         return binary((await store.rawSource(snapshot.kbId, doc)).data, 'application/pdf');
     };
-    const image = async (asset, file) => {
+    const image = async (asset, file, verified) => {
         if (!['image/png', 'image/jpeg', 'image/webp', 'image/gif', 'image/bmp', 'image/tiff'].includes(asset.mime))
             throw new Error('不支持浏览此图片格式，请提供 PNG/JPEG 预览');
-        const data = await fileBytes(file, config.maxUncompressedBytes);
-        if (`a-${hash(data).slice(0, 20)}` !== asset.assetId)
+        const data = verified ?? await fileBytes(file, config.maxUncompressedBytes);
+        if (!verified && `a-${hash(data).slice(0, 20)}` !== asset.assetId)
             throw new Error('图片内容与清单不一致');
         return binary(data, asset.mime);
     };
     const tree = (snapshot, kbId) => ({ kbId, revision: snapshot.revision, coverageIncomplete: snapshot.manifest.coverageIncomplete, report: snapshot.manifest.report, documents: snapshot.manifest.documents.map(doc => ({ ...doc, chunks: doc.chunks.map(({ segments: _segments, anchors: _anchors, ...chunk }) => chunk) })) });
-    add('list', 'GET', async (request) => { z.strictObject({}).parse(query(request)); return ({ hostVersion: HOST_VERSION, kbs: await Promise.all((await store.list()).map(async (kb) => ({ ...kb, ...await store.inspect(kb.kbId) }))), deletedKbs: await store.deleted(), activities: await store.activities(), jobs: jobs.list(), dataRoot: store.root, config: { chunkTargetChars: config.chunkTargetChars, visionProvider: config.visionProvider, visionModel: config.visionModel, maxInputBytes: config.maxInputBytes } }); });
+    add('list', 'GET', async (request) => { const args = jobListQuerySchema.parse(query(request)); if (args.jobsKbId)
+        await store.kb(args.jobsKbId); return ({ hostVersion: HOST_VERSION, ...await store.overview(), ...jobs.list(args.jobsKbId, args.jobOffset, args.jobLimit), dataRoot: store.root, config: { chunkTargetChars: config.chunkTargetChars, visionProvider: config.visionProvider, visionModel: config.visionModel, maxInputBytes: config.maxInputBytes } }); });
+    add('job-details', 'GET', async (request) => { const args = jobDetailsQuerySchema.parse(query(request)); return jobs.details(args.jobId, args.section === 'report'); });
+    add('job-usage', 'GET', async (request) => { const args = jobUsageQuerySchema.parse(query(request)); return jobs.usage(args.jobId, args.offset, args.limit, request.signal); });
     add('create', 'POST', async (request) => store.create((await body(request, z.strictObject({ name }))).name));
     add('delete', 'POST', async (request) => { const args = await body(request, managementArgs); return jobs.trash(args.kbId, args.confirmName); });
     add('restore', 'POST', async (request) => jobs.restore((await body(request, libraryArgs)).kbId));
@@ -107,10 +110,10 @@ export function routes(runtime, ctx) {
     });
     add('draft-asset', 'GET', async (request) => { const args = draftAssetQuery.parse(query(request)), snapshot = await jobs.preview(args.jobId), doc = required(snapshot.manifest.documents.find(d => d.docId === args.docId), '草稿文档'), asset = required(doc.assets.find(a => a.assetId === args.assetId), '草稿图片'); return image(asset, await safePath(snapshot.root, join('sources', doc.docId, asset.path))); });
     add('catalog', 'GET', async (request) => retrieval.catalog(catalogQuerySchema.parse(query(request))));
-    add('tree', 'GET', async (request) => { const args = revisionQuerySchema.parse(query(request)), kb = await store.kb(args.kbId), snapshot = await store.revision(kb.kbId, args.revision), overlays = await store.overlays(kb.kbId), value = tree(snapshot, kb.kbId); return { ...value, location: await store.location(kb.kbId, snapshot.revision), documents: value.documents.map(doc => ({ ...doc, figures: doc.figures.map(figure => ({ ...figure, expectedHash: hash(overlays[figure.overlayKey] || null) })) })) }; });
+    add('tree', 'GET', async (request) => { const args = revisionQuerySchema.parse(query(request)), { kb, snapshot } = await store.open(args.kbId, args.revision), overlays = await store.overlays(kb.kbId), value = tree(snapshot, kb.kbId); return { ...value, location: await store.snapshotLocation(kb.kbId, snapshot), documents: value.documents.map(doc => ({ ...doc, figures: doc.figures.map(figure => ({ ...figure, expectedHash: hash(overlays[figure.overlayKey] || null) })) })) }; });
     add('read', 'GET', async (request) => retrieval.read(readQuerySchema.parse(query(request))));
     add('search', 'POST', async (request) => retrieval.search(await body(request, searchSchema)));
-    add('asset', 'GET', async (request) => { const args = assetQuerySchema.parse(query(request)), kb = await store.kb(args.kbId), asset = await store.asset(kb.kbId, args.revision, args.docId, args.assetId); return image(asset.asset, asset.file); });
+    add('asset', 'GET', async (request) => { const args = assetQuerySchema.parse(query(request)), kb = await store.kb(args.kbId), asset = await store.asset(kb.kbId, args.revision, args.docId, args.assetId); return image(asset.asset, asset.file, asset.data); });
     add('schema', 'GET', async (request) => { const kb = await store.kb(libraryArgs.parse(query(request)).kbId), content = await store.schema(kb.kbId); return { content, hash: hash(content) }; });
     add('schema', 'POST', async (request) => { const args = await body(request, libraryArgs.extend({ content: z.string().max(20000), expectedHash: digest })), kb = await store.kb(args.kbId); return store.setSchema(kb.kbId, args.content, args.expectedHash); });
     add('review', 'POST', async (request) => {

package/lib/runtime.d.ts

@@ -18,8 +18,12 @@ export declare function createRuntime(ctx: Context, raw?: Config): {
         modelTimeoutMs: number;
         modelAttempts: number;
         visionConcurrency: number;
+        maxModelCallsLimit: number;
+        maxModelTokensLimit: number;
+        uploadRetentionDays: number;
         maxModelCalls: number;
         maxModelTokens: number;
+        imageTimeoutMs: number;
         visionMaxEdge: number;
         visionTileSize: number;
         visionMaxTiles: number;

package/lib/runtime.js

@@ -32,7 +32,12 @@ export function createRuntime(ctx, raw = {}) {
             }
             finally {
                 await Promise.allSettled(pending);
-                await jobs.close();
+                try {
+                    await jobs.close();
+                }
+                finally {
+                    store.revisionCache.clear();
+                }
             }
         })();
         return closing;

package/lib/settings.d.ts

@@ -11,8 +11,12 @@ export declare const settingsSchema: z.ZodObject<{
     modelTimeoutMs: z.ZodDefault<z.ZodNumber>;
     modelAttempts: z.ZodDefault<z.ZodNumber>;
     visionConcurrency: z.ZodDefault<z.ZodNumber>;
+    maxModelCallsLimit: z.ZodDefault<z.ZodNumber>;
+    maxModelTokensLimit: z.ZodDefault<z.ZodNumber>;
+    uploadRetentionDays: z.ZodDefault<z.ZodNumber>;
     maxModelCalls: z.ZodDefault<z.ZodNumber>;
     maxModelTokens: z.ZodDefault<z.ZodNumber>;
+    imageTimeoutMs: z.ZodDefault<z.ZodNumber>;
     visionMaxEdge: z.ZodDefault<z.ZodNumber>;
     visionTileSize: z.ZodDefault<z.ZodNumber>;
     visionMaxTiles: z.ZodDefault<z.ZodNumber>;

package/lib/settings.js

@@ -6,13 +6,16 @@ export const settingsSchema = z.strictObject({
     catalogMaxChars: positive.min(512).default(6000), candidateLimit: positive.max(30).default(6),
     maxInputBytes: positive.default(64 * 1024 * 1024), maxUncompressedBytes: positive.default(256 * 1024 * 1024), maxArchiveEntries: positive.default(10000),
     modelTimeoutMs: positive.default(90000), modelAttempts: positive.default(3), visionConcurrency: positive.max(8).default(2),
-    maxModelCalls: positive.default(1000), maxModelTokens: positive.default(2000000), visionMaxEdge: positive.default(2048),
-    visionTileSize: positive.default(1536), visionMaxTiles: positive.default(12), modelMaxOutputTokens: positive.default(4096),
+    maxModelCallsLimit: positive.max(100000).default(10000), maxModelTokensLimit: positive.max(1000000000).default(20000000), uploadRetentionDays: positive.max(365).default(7),
+    maxModelCalls: positive.max(100000).default(1000), maxModelTokens: positive.max(1000000000).default(2000000), imageTimeoutMs: positive.max(120000).default(60000), visionMaxEdge: positive.max(4096).default(2048),
+    visionTileSize: positive.max(4096).default(1536), visionMaxTiles: positive.max(64).default(12), modelMaxOutputTokens: positive.default(4096),
     summaryMaxChars: positive.default(160), figureMaxChars: positive.default(6000), pdfMaxPages: positive.default(2000),
     pdfMaxPagePixels: positive.default(12000000), pdfPageTimeoutMs: positive.default(60000),
     libreOfficePath: z.string().default(''), visionProvider: z.string().default(''), visionModel: z.string().default(''),
     summaryProvider: z.string().default(''), summaryModel: z.string().default(''), skipImages: z.boolean().default(false), acceptPartial: z.boolean().default(false),
 }).superRefine((value, context) => {
+    if (value.maxModelCalls > value.maxModelCallsLimit || value.maxModelTokens > value.maxModelTokensLimit)
+        context.addIssue({ code: 'custom', message: '默认任务预算不能超过宿主预算上限' });
     for (const kind of ['vision', 'summary'])
         if (Boolean(value[`${kind}Provider`]) !== Boolean(value[`${kind}Model`]))
             context.addIssue({ code: 'custom', message: `${kind}Provider / ${kind}Model 必须成对配置` });

package/lib/shared.d.ts

@@ -93,3 +93,20 @@ export declare const revisionQuerySchema: z.ZodObject<{
     kbId: z.ZodString;
     revision: z.ZodOptional<z.ZodString>;
 }, z.core.$strict>;
+export declare const jobListQuerySchema: z.ZodObject<{
+    jobsKbId: z.ZodOptional<z.ZodString>;
+    jobOffset: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    jobLimit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+}, z.core.$strict>;
+export declare const jobDetailsQuerySchema: z.ZodObject<{
+    jobId: z.ZodString;
+    section: z.ZodDefault<z.ZodEnum<{
+        report: "report";
+        details: "details";
+    }>>;
+}, z.core.$strict>;
+export declare const jobUsageQuerySchema: z.ZodObject<{
+    jobId: z.ZodString;
+    offset: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+    limit: z.ZodDefault<z.ZodCoercedNumber<unknown>>;
+}, z.core.$strict>;

package/lib/shared.js

@@ -15,3 +15,6 @@ export const searchSchema = z.strictObject({ kbId: selector, revision: id.option
 export const readQuerySchema = z.strictObject({ kbId: selector, revision: id.optional(), chunkId: id, query: z.string().max(2000).optional(), keywords: keywordsSchema.optional(), segmentIndex: z.coerce.number().int().nonnegative().optional(), offset: z.coerce.number().int().nonnegative().optional() });
 export const assetQuerySchema = z.strictObject({ kbId: selector, revision: id.optional(), docId: id, assetId: id });
 export const revisionQuerySchema = z.strictObject({ kbId: selector, revision: id.optional() });
+export const jobListQuerySchema = z.strictObject({ jobsKbId: id.optional(), jobOffset: z.coerce.number().int().nonnegative().max(1000000).default(0), jobLimit: z.coerce.number().int().min(1).max(50).default(20) });
+export const jobDetailsQuerySchema = z.strictObject({ jobId: id, section: z.enum(['details', 'report']).default('details') });
+export const jobUsageQuerySchema = jobDetailsQuerySchema.omit({ section: true }).extend({ offset: z.coerce.number().int().nonnegative().max(1000000).default(0), limit: z.coerce.number().int().min(1).max(50).default(20) });