dsh-knowledge-library 0.0.0-stage → 0.1.0

package/dist/core/extract.d.ts

@@ -0,0 +1,9 @@
+import type { Segment } from '../types.js';
+export declare function extractArticle(html: string, url: string): {
+    title: string;
+    author: string | undefined;
+    content: string;
+};
+export declare function parseSubtitles(input: string): Segment[];
+export declare function formatTime(value: number): string;
+export declare function splitSegments(content: string): Segment[];

package/dist/core/extract.js

@@ -0,0 +1,105 @@
+import { parseHTML } from 'linkedom';
+import { Readability } from '@mozilla/readability';
+import TurndownService from 'turndown';
+const markdown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+markdown.remove(['script', 'style', 'iframe', 'form', 'input', 'button']);
+function parseDocument(html, url) {
+    let { document } = parseHTML(html);
+    if (document.documentElement?.localName !== 'html') {
+        ({ document } = parseHTML(`<!doctype html><html><head></head><body>${html}</body></html>`));
+    }
+    let baseURI = url;
+    try {
+        baseURI = new URL(document.querySelector('base[href]')?.getAttribute('href') || url, url).href;
+    }
+    catch { /* Keep the source URL for invalid base tags. */ }
+    Object.defineProperties(document, { documentURI: { value: url }, baseURI: { value: baseURI } });
+    return document;
+}
+export function extractArticle(html, url) {
+    // Linkedom parses inert DOM only: no script execution, windows, or resource loading.
+    // It also avoids the punycode/ dependency that DSH 0.2.1-rc.2 misroutes in packed profiles.
+    const document = parseDocument(html, url);
+    for (const node of Array.from(document.querySelectorAll('script,style,iframe,form')))
+        node.remove();
+    const article = new Readability(document).parse();
+    if (!article?.content || (article.textContent?.trim().length ?? 0) < 40) {
+        throw new Error('未提取到有效正文。该网页可能需要登录或动态渲染，请复制正文或导入保存的 HTML。');
+    }
+    const clean = parseDocument(`<!doctype html><html><head></head><body>${article.content}</body></html>`, url);
+    for (const anchor of Array.from(clean.querySelectorAll('a'))) {
+        try {
+            const href = new URL(anchor.getAttribute('href') ?? '', url);
+            if (['http:', 'https:'].includes(href.protocol))
+                anchor.setAttribute('href', href.href);
+            else
+                anchor.removeAttribute('href');
+        }
+        catch {
+            anchor.removeAttribute('href');
+        }
+    }
+    // Preserve image references; do not silently fetch third-party resources.
+    for (const image of Array.from(clean.querySelectorAll('img'))) {
+        try {
+            const src = new URL(image.getAttribute('src') ?? '', url);
+            if (['http:', 'https:'].includes(src.protocol))
+                image.setAttribute('src', src.href);
+            else
+                image.remove();
+        }
+        catch {
+            image.remove();
+        }
+    }
+    return {
+        title: article.title?.trim() || new URL(url).hostname,
+        author: article.byline || undefined,
+        content: markdown.turndown(clean.body.innerHTML).trim(),
+    };
+}
+function seconds(value) {
+    const parts = value.replace(',', '.').split(':').map(Number);
+    if (parts.some(n => !Number.isFinite(n) || n < 0))
+        throw new Error('字幕时间戳无效');
+    return parts.reduce((sum, part) => sum * 60 + part, 0);
+}
+export function parseSubtitles(input) {
+    const blocks = input.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n').split(/\n\s*\n/);
+    const segments = [];
+    for (const block of blocks) {
+        if (/^(NOTE|STYLE|REGION)(?:\s|$)/.test(block.trim()))
+            continue;
+        const lines = block.split('\n');
+        const index = lines.findIndex(line => /-->/.test(line));
+        if (index < 0)
+            continue;
+        const match = lines[index].match(/((?:\d{1,3}:)?\d{2}:\d{2}[.,]\d{3})\s*-->\s*((?:\d{1,3}:)?\d{2}:\d{2}[.,]\d{3})/);
+        if (!match)
+            throw new Error('字幕时间戳格式不正确，请使用 SRT 或 WebVTT');
+        const text = lines.slice(index + 1).join('\n').replace(/<[^>]+>/g, '').trim();
+        if (!text)
+            continue;
+        const startSeconds = seconds(match[1]);
+        const endSeconds = seconds(match[2]);
+        if (endSeconds < startSeconds)
+            throw new Error('字幕结束时间早于开始时间');
+        segments.push({ id: `p${String(segments.length + 1).padStart(4, '0')}`, text, startSeconds, endSeconds });
+    }
+    if (!segments.length)
+        throw new Error('没有找到可读取的字幕，请检查文件是否为 SRT 或 WebVTT');
+    return segments;
+}
+export function formatTime(value) {
+    const s = Math.floor(value);
+    return `${String(Math.floor(s / 3600)).padStart(2, '0')}:${String(Math.floor(s / 60) % 60).padStart(2, '0')}:${String(s % 60).padStart(2, '0')}`;
+}
+export function splitSegments(content) {
+    const chunks = [];
+    for (const paragraph of content.split(/\n\s*\n/).map(p => p.trim()).filter(Boolean)) {
+        // Bound each readable segment without assuming spaces exist in Chinese.
+        for (let offset = 0; offset < paragraph.length; offset += 1600)
+            chunks.push(paragraph.slice(offset, offset + 1600));
+    }
+    return chunks.map((text, index) => ({ id: `p${String(index + 1).padStart(4, '0')}`, text }));
+}

package/dist/core/fetch.d.ts

@@ -0,0 +1,10 @@
+export declare function isPublicAddress(address: string): boolean;
+export declare function validateUrl(input: string): URL;
+export interface FetchPolicy {
+    allowProxyFakeIp?: boolean;
+}
+export declare function permittedAddress(hostname: string, address: string, policy: FetchPolicy): boolean;
+export declare function fetchPublicPage(input: string, signal?: AbortSignal, policy?: FetchPolicy): Promise<{
+    html: string;
+    url: string;
+}>;

package/dist/core/fetch.js

@@ -0,0 +1,105 @@
+import { request as httpRequest } from 'node:http';
+import { request as httpsRequest } from 'node:https';
+import { lookup } from 'node:dns/promises';
+import { isIP } from 'node:net';
+const MAX_BYTES = 6 * 1024 * 1024;
+export function isPublicAddress(address) {
+    if (isIP(address) === 4) {
+        const [a, b, c] = address.split('.').map(Number);
+        return !(a === 0 || a === 10 || a === 127 || a >= 224
+            || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31)
+            || (a === 192 && (b === 168 || b === 0 || (b === 88 && c === 99)))
+            || (a === 100 && b >= 64 && b <= 127) || (a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100)))
+            || (a === 203 && b === 0 && c === 113));
+    }
+    if (isIP(address) === 6) {
+        const lower = address.toLowerCase();
+        return /^[23][0-9a-f]{3}:/.test(lower) && !lower.startsWith('2001:db8:')
+            && !lower.startsWith('2001:0:') && !lower.startsWith('2002:');
+    }
+    return false;
+}
+export function validateUrl(input) {
+    let url;
+    try {
+        url = new URL(input);
+    }
+    catch {
+        throw new Error('请输入完整的 http 或 https 网页地址');
+    }
+    if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) {
+        throw new Error('仅支持不含账号密码的 http/https 地址');
+    }
+    if (url.port && !['80', '443'].includes(url.port))
+        throw new Error('网页采集仅支持标准 HTTP/HTTPS 端口');
+    url.hash = '';
+    return url;
+}
+export function permittedAddress(hostname, address, policy) {
+    if (isPublicAddress(address))
+        return true;
+    // An explicit deployment opt-in for Clash/Mihomo TUN fake DNS, not a general private-network exemption.
+    return policy.allowProxyFakeIp === true && isIP(hostname) === 0
+        && hostname.includes('.') && !/\.(localhost|local|internal|lan)$/i.test(hostname)
+        && /^198\.(18|19)\.\d{1,3}\.\d{1,3}$/.test(address);
+}
+export async function fetchPublicPage(input, signal, policy = {}) {
+    let url = validateUrl(input);
+    const timeout = AbortSignal.timeout(25_000);
+    const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
+    for (let redirect = 0; redirect <= 4; redirect++) {
+        combined.throwIfAborted();
+        const hostname = url.hostname.replace(/^\[|\]$/g, '');
+        const addresses = await lookup(hostname, { all: true });
+        if (!addresses.length || addresses.some(item => !permittedAddress(hostname, item.address, policy))) {
+            if (addresses.some(item => /^198\.(18|19)\./.test(item.address)))
+                throw new Error('检测到代理 fake-IP DNS。若本机使用 Clash/Mihomo TUN，请启用插件的 allowProxyFakeIp；否则请直接导入文本。');
+            throw new Error('网页采集不访问本机、内网或保留地址，请直接导入文件或文本');
+        }
+        const address = addresses[0];
+        const response = await new Promise((resolve, reject) => {
+            const req = (url.protocol === 'https:' ? httpsRequest : httpRequest)(url, {
+                method: 'GET', signal: combined, family: address.family,
+                headers: { 'User-Agent': 'DSH-Knowledge-Library/0.1', Accept: 'text/html,text/plain;q=0.8', 'Accept-Encoding': 'identity' },
+                // Pin the validated DNS result so a second resolution cannot reach an internal host.
+                lookup: (_host, _options, callback) => callback(null, address.address, address.family),
+            }, res => {
+                const chunks = [];
+                let size = 0;
+                res.on('data', (chunk) => {
+                    size += chunk.length;
+                    if (size > MAX_BYTES) {
+                        req.destroy(new Error('网页超过 6 MB，请下载后手动导入'));
+                        return;
+                    }
+                    chunks.push(chunk);
+                });
+                res.on('end', () => resolve({ status: res.statusCode ?? 0, location: res.headers.location, contentType: String(res.headers['content-type'] ?? ''), bytes: Buffer.concat(chunks) }));
+                res.on('error', reject);
+            });
+            req.on('error', reject);
+            req.end();
+        });
+        if ([301, 302, 303, 307, 308].includes(response.status) && response.location) {
+            url = validateUrl(new URL(response.location, url).href);
+            continue;
+        }
+        if (response.status < 200 || response.status >= 300)
+            throw new Error(`网页返回 HTTP ${response.status}，请检查链接或改为手动导入`);
+        if (!/text\/(html|plain)|application\/xhtml\+xml/i.test(response.contentType))
+            throw new Error('该地址不是可提取的网页，请下载文件后导入');
+        const encoding = /charset=["']?([\w-]+)/i.exec(response.contentType)?.[1] || 'utf-8';
+        let html;
+        try {
+            html = new TextDecoder(encoding).decode(response.bytes);
+        }
+        catch {
+            throw new Error(`暂不支持网页编码 ${encoding}`);
+        }
+        if (response.contentType.includes('text/plain')) {
+            html = `<html><body><article><p>${html.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('\n', '</p><p>')}</p></article></body></html>`;
+        }
+        return { html, url: url.href };
+    }
+    throw new Error('网页重定向次数过多');
+}

package/dist/core/jobs.d.ts

@@ -0,0 +1,30 @@
+import type { CaptureFields, ImportJob } from '../types.js';
+import { Library } from './library.js';
+import { fetchPublicPage } from './fetch.js';
+export declare class ImportQueue {
+    private library;
+    private fetchPage;
+    private jobs;
+    private file;
+    private running;
+    private stopped;
+    private abort;
+    private active?;
+    constructor(library: Library, fetchPage?: typeof fetchPublicPage);
+    list(): {
+        id: string;
+        url: string;
+        status: "queued" | "running" | "completed" | "failed";
+        createdAt: string;
+        updatedAt: string;
+        attempts: number;
+        materialId?: string;
+        error?: string;
+    }[];
+    add(url: string, fields?: CaptureFields): ImportJob;
+    retry(id: string): ImportJob;
+    private persist;
+    private kick;
+    idle(): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/core/jobs.js

@@ -0,0 +1,96 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { atomicJson } from './library.js';
+import { fetchPublicPage, validateUrl } from './fetch.js';
+export class ImportQueue {
+    library;
+    fetchPage;
+    jobs;
+    file;
+    running = false;
+    stopped = false;
+    abort = new AbortController();
+    active;
+    constructor(library, fetchPage = fetchPublicPage) {
+        this.library = library;
+        this.fetchPage = fetchPage;
+        this.file = join(library.root, '.system', 'jobs.json');
+        this.jobs = existsSync(this.file) ? JSON.parse(readFileSync(this.file, 'utf8')) : [];
+        if (!Array.isArray(this.jobs))
+            throw new Error('任务记录损坏，请备份并检查 .system/jobs.json');
+        for (const job of this.jobs)
+            if (job.status === 'running') {
+                job.status = job.attempts >= 3 ? 'failed' : 'queued';
+                job.error = '上次处理被中断，等待恢复';
+            }
+        this.persist();
+        this.kick();
+    }
+    list() { return this.jobs.slice().reverse().slice(0, 100).map(({ fields: _fields, ...job }) => job); }
+    add(url, fields = {}) {
+        const normalized = validateUrl(url).href;
+        const existing = this.jobs.find(j => j.url === normalized && ['queued', 'running'].includes(j.status));
+        if (existing)
+            return structuredClone(existing);
+        if (this.jobs.filter(j => ['queued', 'running'].includes(j.status)).length >= 50)
+            throw new Error('最多同时排队 50 个网页');
+        const timestamp = new Date().toISOString();
+        const job = { id: `job_${randomUUID()}`, url: normalized, fields, status: 'queued', createdAt: timestamp, updatedAt: timestamp, attempts: 0 };
+        this.jobs.push(job);
+        this.persist();
+        this.kick();
+        return structuredClone(job);
+    }
+    retry(id) {
+        const job = this.jobs.find(j => j.id === id);
+        if (!job || job.status !== 'failed')
+            throw new Error('只能重试失败的任务');
+        job.status = 'queued';
+        job.error = undefined;
+        job.attempts = 0;
+        this.persist();
+        this.kick();
+        return structuredClone(job);
+    }
+    persist() { atomicJson(this.file, this.jobs); }
+    kick() {
+        if (this.running || this.stopped)
+            return;
+        this.running = true;
+        this.active = new Promise(resolve => setImmediate(resolve)).then(async () => {
+            while (!this.stopped) {
+                const job = this.jobs.find(j => j.status === 'queued');
+                if (!job)
+                    break;
+                job.status = 'running';
+                job.attempts++;
+                job.updatedAt = new Date().toISOString();
+                this.persist();
+                try {
+                    const page = await this.fetchPage(job.url, this.abort.signal);
+                    if (this.stopped)
+                        break;
+                    const result = this.library.captureHtml(page.html, page.url, job.fields);
+                    job.materialId = result.material.id;
+                    job.status = 'completed';
+                    job.error = undefined;
+                }
+                catch (error) {
+                    job.status = this.stopped ? 'queued' : 'failed';
+                    job.error = error.message;
+                }
+                job.updatedAt = new Date().toISOString();
+                this.persist();
+            }
+        }).finally(() => {
+            for (const job of this.jobs)
+                if (job.status === 'running')
+                    job.status = 'queued';
+            this.persist();
+            this.running = false;
+        });
+    }
+    async idle() { await this.active; }
+    async close() { this.stopped = true; this.abort.abort(); await this.active; }
+}

package/dist/core/library.d.ts

@@ -0,0 +1,61 @@
+import { DatabaseSync } from 'node:sqlite';
+import type { CaptureFields, CaptureResult, Citation, MaterialDetail, SearchOptions, SearchResult } from '../types.js';
+export declare const MAX_FILE_BYTES: number;
+export declare const MAX_TEXT_CHARS = 2000000;
+export declare function categoryName(value: unknown): string;
+export declare function atomicJson(path: string, value: unknown): void;
+export declare class Library {
+    readonly root: string;
+    readonly db: DatabaseSync;
+    readonly warnings: string[];
+    private lockPath;
+    private closed;
+    constructor(root: string);
+    close(): void;
+    private assertContained;
+    private index;
+    rebuild(): {
+        indexed: number;
+        warnings: string[];
+    };
+    private row;
+    get(id: string): MaterialDetail;
+    original(id: string): {
+        bytes: Buffer;
+        name: string;
+    };
+    private save;
+    private tags;
+    captureText(text: string, fields?: CaptureFields): CaptureResult;
+    captureHtml(html: string, url: string, fields?: CaptureFields): CaptureResult;
+    importFile(path: string, fields?: CaptureFields): CaptureResult;
+    importBytes(name: string, bytes: Buffer, fields?: CaptureFields): CaptureResult;
+    private decode;
+    update(id: string, fields: CaptureFields): MaterialDetail;
+    saveNote(content: string, citations: Citation[], fields: CaptureFields): CaptureResult;
+    search(options?: SearchOptions): {
+        items: SearchResult[];
+        total: number;
+    };
+    stats(): {
+        total: number;
+        ready: number;
+        notes: number;
+        inbox: number;
+        categories: {
+            name: string;
+            count: number;
+        }[];
+        tags: {
+            name: string;
+            count: number;
+        }[];
+        collections: {
+            name: string;
+            count: number;
+        }[];
+        root: string;
+        warnings: string[];
+    };
+    exportMarkdown(id: string): string;
+}