dsh-knowledge-library 0.1.1 → 0.3.0

package/dist/core/incremental.js

@@ -0,0 +1,52 @@
+import { createHash } from 'node:crypto';
+export const noteFingerprint = (note) => createHash('sha256').update(JSON.stringify([note.content, note.segments, note.title, note.category, note.tags, note.citations, note.revision, note.updatedAt, note.deletedAt, note.deletionId])).digest('hex');
+const tokens = (text) => [...new Set((text.toLowerCase().match(/[a-z0-9_]{2,}|[\p{Script=Han}]+/gu) || []).flatMap(term => /^[\p{Script=Han}]/u.test(term) ? Array.from({ length: Math.max(0, term.length - 1) }, (_, index) => term.slice(index, index + 2)) : [term]))].slice(0, 500);
+/** Local lexical recall, followed by full-body model comparison; no silent body truncation. */
+export function recallNotes(library, draft, topic, sources) {
+    const rows = library.db.prepare('SELECT id,meta,content FROM materials ORDER BY updated_at DESC').all();
+    const query = tokens([topic, draft.title, ...draft.tags, ...sources.flatMap(s => [s.title, ...s.tags]), draft.content].join(' '));
+    const notes = rows.map(row => ({ ...row, metadata: JSON.parse(row.meta) })).filter(row => row.metadata.kind === 'note' && !row.metadata.deletedAt);
+    const ranked = notes.map(row => {
+        const title = `${row.metadata.title} ${row.metadata.tags.join(' ')}`.toLowerCase();
+        const body = row.content.toLowerCase();
+        return { ...row, score: query.reduce((sum, term) => sum + (title.includes(term) ? 4 : body.includes(term) ? 1 : 0), 0) };
+    }).filter(row => row.score > 0).sort((a, b) => b.score - a.score);
+    let size = 0;
+    let oversized = 0;
+    const candidates = [];
+    for (const row of ranked) {
+        if (row.content.length > 50_000) {
+            oversized++;
+            continue;
+        }
+        if (candidates.length >= 5 || size + row.content.length > 60_000)
+            continue;
+        const note = library.get(row.id);
+        candidates.push(note);
+        size += note.content.length;
+    }
+    return { candidates, search: { scanned: notes.length, matched: ranked.length, compared: candidates.length, oversized } };
+}
+export function integrationValue(value, candidates, search) {
+    const data = value;
+    const bounded = (v, max, label, empty = false) => {
+        if (typeof v !== 'string' || v.length > max || (!empty && !v.trim()))
+            throw new Error(`增量整理的${label}无效`);
+        return v.trim();
+    };
+    const list = (v) => { if (!Array.isArray(v) || v.length > 12)
+        throw new Error('增量对比列表格式无效'); return v.map(item => bounded(item, 1000, '对比说明')); };
+    if (!data || !['new', 'append', 'duplicate'].includes(data.action))
+        throw new Error('增量整理建议格式无效');
+    const recommendation = data.action;
+    const target = recommendation === 'new' ? undefined : candidates.find(n => n.id === data.noteId);
+    if (recommendation !== 'new' && !target)
+        throw new Error('增量整理指定了未检索的笔记');
+    const content = bounded(data.content, 50_000, '新增正文', recommendation !== 'append');
+    if (recommendation !== 'append' && content)
+        throw new Error('新建或重复建议不应包含追加正文');
+    if (target && content && target.content.includes(content))
+        throw new Error('建议追加的正文已存在，请重新核对');
+    return { recommendation, choice: recommendation, reason: bounded(data.reason, 2000, '建议理由'), overlaps: list(data.overlaps), additions: list(data.additions), conflicts: list(data.conflicts), content, search,
+        ...(target ? { target: { id: target.id, title: target.title, category: target.category, revision: target.revision, fingerprint: noteFingerprint(target), content: target.content } } : {}) };
+}

package/dist/core/jobs.d.ts

@@ -1,16 +1,21 @@
 import type { CaptureFields, ImportJob } from '../types.js';
 import { Library } from './library.js';
 import { fetchPublicPage } from './fetch.js';
+import type { ImageCapture } from './image-capture.js';
 export declare class ImportQueue {
     private library;
     private fetchPage;
+    readonly imageCapture?: ImageCapture | undefined;
     private jobs;
     private file;
     private running;
     private stopped;
     private abort;
     private active?;
-    constructor(library: Library, fetchPage?: typeof fetchPublicPage);
+    private imageAbort?;
+    private activeJob?;
+    private retrying;
+    constructor(library: Library, fetchPage?: typeof fetchPublicPage, imageCapture?: ImageCapture | undefined);
     list(): {
         id: string;
         url: string;
@@ -20,9 +25,24 @@ export declare class ImportQueue {
         attempts: number;
         materialId?: string;
         error?: string;
+        errorCode?: "PAGE_REQUIRES_BROWSER";
+        progress?: string;
+        imageCapture?: {
+            model: string;
+            maxImages: number;
+        };
+        imageSummary?: {
+            total: number;
+            completed: number;
+            failed: number;
+            skipped: number;
+        };
     }[];
     add(url: string, fields?: CaptureFields): ImportJob;
     retry(id: string): ImportJob;
+    retryImages(id: string, expectedModel: unknown, expectedIds: unknown): Promise<ImportJob>;
+    retryImagesForMaterial(materialId: string, expectedModel: unknown, expectedIds: unknown): Promise<ImportJob>;
+    cancelImages(id: string): ImportJob;
     private persist;
     private kick;
     idle(): Promise<void>;

package/dist/core/jobs.js

@@ -3,32 +3,60 @@ import { join } from 'node:path';
 import { randomUUID } from 'node:crypto';
 import { atomicJson } from './library.js';
 import { fetchPublicPage, validateUrl } from './fetch.js';
+import { PageAccessError } from './page-access.js';
 export class ImportQueue {
     library;
     fetchPage;
+    imageCapture;
     jobs;
     file;
     running = false;
     stopped = false;
     abort = new AbortController();
     active;
-    constructor(library, fetchPage = fetchPublicPage) {
+    imageAbort;
+    activeJob;
+    retrying = new Set();
+    constructor(library, fetchPage = fetchPublicPage, imageCapture) {
         this.library = library;
         this.fetchPage = fetchPage;
+        this.imageCapture = imageCapture;
         this.file = join(library.root, '.system', 'jobs.json');
         this.jobs = existsSync(this.file) ? JSON.parse(readFileSync(this.file, 'utf8')) : [];
         if (!Array.isArray(this.jobs))
             throw new Error('任务记录损坏，请备份并检查 .system/jobs.json');
-        for (const job of this.jobs)
-            if (job.status === 'running') {
+        for (const job of this.jobs) {
+            let unfinishedImages = false;
+            if (job.materialId && job.imageCapture && library.has(job.materialId) && imageCapture) {
+                // Recover even if a storage error completed the outer job before the image
+                // state could be written, or the process stopped immediately after body save.
+                if (!library.pageImages(job.materialId) && ['queued', 'running'].includes(job.status)) {
+                    const page = library.get(job.materialId);
+                    imageCapture.initialize(job.materialId, library.original(job.materialId).bytes.toString('utf8'), page.sourceUrl || job.url, job.imageCapture);
+                }
+                unfinishedImages = !!library.pageImages(job.materialId)?.records.some(image => ['pending', 'downloading', 'recognizing'].includes(image.status));
+            }
+            if (job.materialId && job.imageCapture && (['queued', 'running'].includes(job.status) || unfinishedImages)) {
+                if (library.has(job.materialId)) {
+                    imageCapture?.interrupt(job.materialId);
+                    job.imageSummary = imageCapture?.summary(job.materialId);
+                }
+                job.status = 'completed';
+                job.progress = '正文已保存，图片处理被中断';
+                job.error = '图片需主动重试，未自动重复模型请求';
+            }
+            else if (job.status === 'running') {
                 job.status = job.attempts >= 3 ? 'failed' : 'queued';
                 job.error = '上次处理被中断，等待恢复';
             }
+        }
         this.persist();
         this.kick();
     }
     list() { return this.jobs.slice().reverse().slice(0, 100).map(({ fields: _fields, ...job }) => job); }
     add(url, fields = {}) {
+        if (this.stopped)
+            throw new Error('网页采集服务已关闭');
         const normalized = validateUrl(url).href;
         const existing = this.jobs.find(j => j.url === normalized && ['queued', 'running'].includes(j.status));
         if (existing)
@@ -36,7 +64,7 @@ export class ImportQueue {
         if (this.jobs.filter(j => ['queued', 'running'].includes(j.status)).length >= 50)
             throw new Error('最多同时排队 50 个网页');
         const timestamp = new Date().toISOString();
-        const job = { id: `job_${randomUUID()}`, url: normalized, fields, status: 'queued', createdAt: timestamp, updatedAt: timestamp, attempts: 0 };
+        const job = { id: `job_${randomUUID()}`, url: normalized, fields, status: 'queued', createdAt: timestamp, updatedAt: timestamp, attempts: 0, imageCapture: this.imageCapture?.options() };
         this.jobs.push(job);
         this.persist();
         this.kick();
@@ -48,11 +76,55 @@ export class ImportQueue {
             throw new Error('只能重试失败的任务');
         job.status = 'queued';
         job.error = undefined;
+        job.errorCode = undefined;
         job.attempts = 0;
         this.persist();
         this.kick();
         return structuredClone(job);
     }
+    async retryImages(id, expectedModel, expectedIds) {
+        const job = this.jobs.find(j => j.id === id);
+        if (!job?.materialId || !job.imageCapture || !this.imageCapture || job.status !== 'completed' || this.retrying.has(id))
+            throw new Error('图片任务尚未结束或正在重试');
+        this.retrying.add(id);
+        try {
+            const data = this.library.pageImages(job.materialId);
+            const eligible = data?.records.filter(image => image.status === 'failed' && image.attempts < 3).map(image => image.id).sort();
+            if (!Array.isArray(expectedIds) || JSON.stringify([...expectedIds].sort()) !== JSON.stringify(eligible))
+                throw new Error('失败图片已改变，请重新确认重试范围');
+            await this.imageCapture.prepareRetry(job.materialId, expectedModel);
+            job.status = 'queued';
+            job.progress = '等待重试失败图片';
+            delete job.error;
+            this.persist();
+            this.kick();
+            return structuredClone(job);
+        }
+        finally {
+            this.retrying.delete(id);
+        }
+    }
+    retryImagesForMaterial(materialId, expectedModel, expectedIds) {
+        const job = this.jobs.slice().reverse().find(job => job.materialId === materialId && job.imageCapture);
+        if (!job)
+            throw new Error('未找到该网页的图片任务');
+        return this.retryImages(job.id, expectedModel, expectedIds);
+    }
+    cancelImages(id) {
+        const job = this.jobs.find(j => j.id === id);
+        if (!job?.materialId || !job.imageCapture || !['queued', 'running'].includes(job.status))
+            throw new Error('当前没有可停止的图片处理');
+        if (this.activeJob === id)
+            this.imageAbort?.abort();
+        else {
+            this.imageCapture?.interrupt(job.materialId);
+            job.imageSummary = this.imageCapture?.summary(job.materialId);
+            job.status = 'completed';
+        }
+        job.progress = '正文已保存，图片处理已停止';
+        this.persist();
+        return structuredClone(job);
+    }
     persist() { atomicJson(this.file, this.jobs); }
     kick() {
         if (this.running || this.stopped)
@@ -67,21 +139,38 @@ export class ImportQueue {
                 job.attempts++;
                 job.updatedAt = new Date().toISOString();
                 this.persist();
+                this.activeJob = job.id;
                 try {
-                    const page = await this.fetchPage(job.url, this.abort.signal);
-                    if (this.stopped)
-                        break;
-                    const result = this.library.captureHtml(page.html, page.url, job.fields);
-                    job.materialId = result.material.id;
+                    if (!job.materialId) {
+                        const page = await this.fetchPage(job.url, this.abort.signal);
+                        if (this.stopped)
+                            break;
+                        const result = this.library.captureHtml(page.html, page.url, job.fields);
+                        job.materialId = result.material.id;
+                        job.progress = '网页正文已保存';
+                        this.persist();
+                        if (job.imageCapture && this.imageCapture)
+                            this.imageCapture.initialize(job.materialId, page.html, page.url, job.imageCapture);
+                    }
+                    if (job.imageCapture && this.imageCapture) {
+                        this.imageAbort = new AbortController();
+                        await this.imageCapture.run(job.materialId, AbortSignal.any([this.abort.signal, this.imageAbort.signal]), message => { job.progress = message; job.imageSummary = this.imageCapture.summary(job.materialId); this.persist(); });
+                        job.imageSummary = this.imageCapture.summary(job.materialId);
+                    }
                     job.status = 'completed';
-                    job.error = undefined;
+                    job.error = job.imageSummary?.failed ? `${job.imageSummary.failed} 张图片未完成，可核对后重试；正文已保留` : undefined;
+                    job.errorCode = undefined;
+                    job.progress = this.imageAbort?.signal.aborted || this.abort.signal.aborted ? '正文已保存，图片处理已停止' : job.imageCapture ? '采集已完成，正文与已识别图片文字保存在本地' : '网页正文已保存';
                 }
                 catch (error) {
-                    job.status = this.stopped ? 'queued' : 'failed';
+                    job.status = job.materialId ? 'completed' : this.stopped ? 'queued' : 'failed';
                     job.error = error.message;
+                    job.errorCode = error instanceof PageAccessError ? error.code : undefined;
                 }
                 job.updatedAt = new Date().toISOString();
                 this.persist();
+                this.imageAbort = undefined;
+                this.activeJob = undefined;
             }
         }).finally(() => {
             for (const job of this.jobs)
@@ -92,5 +181,5 @@ export class ImportQueue {
         });
     }
     async idle() { await this.active; }
-    async close() { this.stopped = true; this.abort.abort(); await this.active; }
+    async close() { this.stopped = true; this.imageCapture?.close(); this.abort.abort(); await this.active; }
 }

package/dist/core/knowledge.d.ts

@@ -0,0 +1,37 @@
+import { Library } from './library.js';
+import type { KnowledgeJob, KnowledgeOverview } from '../types.js';
+import type { KnowledgeModel } from '../knowledge-model.js';
+export declare class KnowledgeQueue {
+    private library;
+    private model;
+    private jobs;
+    private file;
+    private stopped;
+    private running?;
+    private abort?;
+    constructor(library: Library, model: KnowledgeModel);
+    overview(): KnowledgeOverview;
+    get(id: string): KnowledgeJob;
+    private find;
+    private persist;
+    private sources;
+    private checkedSources;
+    add(ids: unknown, topic?: unknown, requestedMode?: unknown): KnowledgeJob;
+    private newJob;
+    cancel(id: string): KnowledgeJob;
+    discard(id: string): KnowledgeJob;
+    editPlan(id: string, value: unknown, version: unknown): KnowledgeJob;
+    confirmPlan(id: string, version: unknown, apply?: unknown, expectedModel?: unknown): KnowledgeJob;
+    private runClassification;
+    rollbackPlan(id: string): KnowledgeJob;
+    edit(id: string, value: unknown, version: unknown, integration?: unknown): KnowledgeJob;
+    confirm(id: string, version: unknown, category?: unknown): import("../types.js").CaptureResult;
+    private progress;
+    private call;
+    private evidence;
+    private analyze;
+    private generate;
+    private kick;
+    idle(): Promise<void>;
+    close(): Promise<void>;
+}