@dsh-libing/knowledge-base 0.1.0

package/lib/data.js

@@ -0,0 +1,80 @@
+/** Persisted schemas validate untrusted receipts and preserve the previous format's versioned references. */
+import { z } from 'zod';
+import { settingsSchema } from "./settings.js";
+export const idSchema = z.string().regex(/^[a-z0-9][a-z0-9-]{0,95}$/u);
+export const routeSchema = z.object({ provider: z.string().min(1), model: z.string().min(1) });
+export const bboxSchema = z.tuple([z.number().finite(), z.number().finite(), z.number().finite(), z.number().finite()]);
+export const sourceSchema = z.object({ page: z.number().int().positive(), pageLabel: z.string().nullable().optional(),
+    width: z.number().nullable().optional(), height: z.number().nullable().optional(), bbox: bboxSchema.nullable(),
+    coordinateSystem: z.literal('viewport-points-top-left'), rotation: z.number().optional(), method: z.string(), caption: z.string().optional(),
+});
+export const fullSourceSchema = sourceSchema.extend({ locations: z.array(sourceSchema).optional() });
+export const rangeSchema = z.object({ start: z.number().int().positive(), end: z.number().int().positive(), count: z.number().int().positive(), total: z.number().int().positive(), partial: z.boolean() });
+export const warningSchema = z.object({ code: z.string(), message: z.string(), blocking: z.boolean(), page: z.number().optional(), kind: z.string().optional(), snippet: z.string().optional(), reason: z.string().optional() });
+export const assetSchema = z.object({ assetId: idSchema, mime: z.string(), path: z.string(), bytes: z.number().int().nonnegative(), origin: z.string(), source: fullSourceSchema.optional() });
+export const FIGURE_TYPES = ['architecture', 'pinout', 'schematic', 'timing', 'flowchart', 'table', 'curve', 'other', 'decoration'];
+export const figureDataSchema = z.object({ types: z.array(z.enum(FIGURE_TYPES)).min(1), visibleLabels: z.array(z.string()), description: z.string().min(1),
+    relations: z.array(z.object({ from: z.string(), to: z.string(), label: z.string() })), uncertainties: z.array(z.string()), needsReview: z.boolean(),
+    contextNotes: z.string().optional(), pins: z.array(z.json()).optional(), timingConstraints: z.array(z.json()).optional(), tableRows: z.array(z.json()).optional() });
+export const overlaySchema = z.lazy(() => figureDataSchema.partial().extend({ ignore: z.boolean().optional(), skip: z.boolean().optional(), updatedAt: z.string().optional(), previous: overlaySchema.nullable().optional() }));
+export const figureSchema = figureDataSchema.partial().extend({ status: z.enum(['generated', 'failed', 'skipped', 'ignored', 'decoration', 'reviewed']),
+    assetId: idSchema, occurrenceId: idSchema, caption: z.string().default(''), cacheKey: z.string(), overlayKey: z.string(), overlayHash: z.string(),
+    humanReviewed: z.boolean().default(false), needsReview: z.boolean().default(false), error: z.string().optional(),
+    classification: z.object({ types: z.array(z.enum(FIGURE_TYPES)), reason: z.string() }).optional(), width: z.number().optional(), height: z.number().optional(),
+    model: routeSchema.optional(), promptVersion: z.string().optional(), context: z.object({ headingPath: z.array(z.string()), caption: z.string().optional(), nearby: z.array(z.string()) }).optional(),
+    source: fullSourceSchema.optional(), nativeLabels: z.array(z.string()).optional(), refreshToken: z.string().optional(), ignore: z.boolean().optional(), skip: z.boolean().optional(), previous: overlaySchema.nullable().optional(), updatedAt: z.string().optional() });
+export const cellSchema = z.object({ text: z.string(), colspan: z.number().int().positive(), rowspan: z.number().int().positive(), source: fullSourceSchema.optional(), spanIds: z.array(z.string()).optional(), ocr: z.boolean().optional() });
+export const blockSchema = z.object({ type: z.enum(['heading', 'paragraph', 'table', 'figure', 'caption', 'list', 'note', 'code', 'formula']),
+    blockId: z.string().default(''), text: z.string().default(''), level: z.number().int().min(1).max(32).default(1), rows: z.array(z.array(cellSchema)).default([]),
+    caption: z.string().default(''), assetId: z.string().default(''), occurrenceId: z.string().default(''), spanIds: z.array(z.string()).default([]),
+    anchor: z.string().nullable().optional(), markdown: z.string().optional(), tableRef: z.string().optional(), figureRef: z.string().optional(),
+    source: fullSourceSchema.optional(), sourcePreviewAssetId: z.string().optional(), sourcePreviewAssetIds: z.array(z.string()).optional(),
+    nativeLabels: z.array(z.string()).default([]), figure: figureSchema.optional(), part: z.number().optional(), oversizedReason: z.string().optional(),
+    needsReview: z.boolean().optional(), headingBasis: z.string().optional(), bookmark: z.string().optional(), layoutBasis: z.string().optional(),
+    openEnd: z.boolean().optional(), tableCaption: z.boolean().optional(), ocr: z.boolean().optional(), mergeRemove: z.boolean().optional(),
+    localIndex: z.number().optional(), captionFigureIndex: z.number().optional() });
+export function block(value) { return blockSchema.parse(value); }
+export const pageAuditSchema = z.object({ page: z.number(), width: z.number().optional(), height: z.number().optional(), pageLabel: z.string().nullable().optional(),
+    status: z.enum(['needs-review', 'navigation', 'parsed', 'failed']), ocr: z.boolean(), spanCount: z.number().optional(), artifactCount: z.number().optional(),
+    excluded: z.array(z.object({ kind: z.string(), text: z.string().optional(), bbox: bboxSchema.nullable() })).optional(),
+    tables: z.number(), figures: z.number(), headings: z.array(z.object({ text: z.string(), basis: z.string().optional(), level: z.number() })),
+    previewAssetId: z.string().nullable().optional(), error: z.string().optional() });
+export const auditSchema = z.object({ pages: z.array(pageAuditSchema).default([]), outlineCount: z.number().default(0), matchedBookmarks: z.number().default(0),
+    convertedFigures: z.number(), convertedTables: z.number(), warnings: z.array(warningSchema).optional(), styleLevels: z.record(z.string(), z.number()).optional(),
+    headingLevels: z.record(z.string(), z.number()).optional(), outlineLevels: z.record(z.string(), z.number()).optional(),
+    expectedFigures: z.number().optional(), expectedTables: z.number().optional(), expectedHeadings: z.array(z.string()).optional(), textBoxes: z.number().optional(),
+    footnoteRefs: z.number().optional(), anchors: z.array(z.string()).optional(), rawMediaCount: z.number().optional() });
+export const documentSchema = z.object({ formatVersion: z.literal(1), sourceFormat: z.enum(['docx', 'pdf']).optional(), pageCount: z.number().optional(),
+    selectedPages: rangeSchema.optional(), parserVersion: z.string().optional(), sourceHash: z.string().optional(), blocks: z.array(blockSchema), assets: z.array(assetSchema),
+    internalRefs: z.array(z.object({ target: z.string(), label: z.string() })), warnings: z.array(warningSchema), audit: auditSchema });
+export const nodeSchema = z.object({ nodeId: idSchema, title: z.string(), level: z.number(), parentId: z.string().nullable(), headingPath: z.array(z.string()), children: z.array(z.string()), anchor: z.string().nullable().optional(), indexPath: z.string().optional() });
+export const segmentSchema = z.object({ text: z.string(), blockIds: z.array(z.string()), type: blockSchema.shape.type, sourceLocations: z.array(fullSourceSchema).optional(), startLine: z.number().optional(), endLine: z.number().optional() });
+export const chunkSchema = z.object({ chunkId: idSchema, docId: idSchema, sourceHash: z.string(), sourceVersion: z.string(), title: z.string(), nodeId: idSchema, headingPath: z.array(z.string()),
+    blockRange: z.tuple([z.string(), z.string()]), assetRefs: z.array(z.string()), tableRefs: z.array(z.string()), relatedChunks: z.array(z.string()), reviewStatus: z.enum(['needs-review', 'generated', 'source']),
+    path: z.string(), segments: z.array(segmentSchema), anchors: z.array(z.string()), sourceFormat: z.enum(['docx', 'pdf']).optional(), sourceLocations: z.array(fullSourceSchema).optional(),
+    pageRange: z.tuple([z.number(), z.number()]).optional(), selectedPages: rangeSchema.optional(), oversizedReason: z.string().optional(),
+    fullChars: z.number(), bodyChars: z.number(), contentHash: z.string(), summary: z.string().default(''), summaryFallback: z.boolean().optional(), keywords: z.array(z.string()).default([]) });
+export const infoSchema = z.object({ docId: idSchema, title: z.string(), filename: z.string(), sourceHash: z.string(), sourceVersion: z.string(), sourceFormat: z.enum(['docx', 'pdf']).optional(), pageCount: z.number().optional(), selectedPages: rangeSchema.optional(), parserVersion: z.string().optional(), audit: auditSchema.optional() });
+export const entrySchema = z.object({ title: z.string().optional(), path: z.string().optional(), docId: z.string().optional(), nodeId: z.string().optional(), chunkId: z.string().optional(), summary: z.string().optional(), fullChars: z.number().optional(), keywords: z.array(z.string()).optional(), reviewStatus: z.string().optional(), pdfScope: z.string().optional(), pageRange: z.tuple([z.number(), z.number()]).optional() });
+export const catalogSchema = z.object({ catalogId: z.string(), path: z.string(), title: z.string(), pages: z.array(z.object({ path: z.string(), content: z.string(), entries: z.array(entrySchema), next: z.string().nullable() })) });
+export const entitySchema = z.object({ label: z.string(), docId: z.string(), chunkId: z.string(), path: z.string(), headingPath: z.array(z.string()) });
+export const usageSchema = z.object({ calls: z.number().int().nonnegative(), tokens: z.number().nonnegative(), reportedCalls: z.number().int().nonnegative(), records: z.array(z.json()) });
+export const reportSchema = z.object({ revision: z.string(), sourceHash: z.string(), issues: z.array(warningSchema), documents: z.number(), blocks: z.number(), figures: z.number(), ignoredFigures: z.number(), unreviewedFigures: z.number(), chunks: z.number(), withinTarget: z.number(), oversized: z.array(z.object({ chunkId: z.string(), chars: z.number(), reason: z.string() })), usage: usageSchema, targetChars: z.number(), pdf: z.object({ pageCount: z.number(), selectedPages: rangeSchema, outlineCount: z.number(), matchedBookmarks: z.number(), ocrPages: z.number(), pages: z.array(pageAuditSchema) }).optional() });
+export const publishedDocumentSchema = infoSchema.extend({ chunkTargetChars: z.number().optional(), nodes: z.array(nodeSchema), chunks: z.array(chunkSchema), assets: z.array(assetSchema), figures: z.array(figureSchema), coverageIncomplete: z.boolean().optional(), warnings: z.array(warningSchema).optional() });
+export const manifestSchema = z.object({ formatVersion: z.literal(1), revision: idSchema, buildKey: z.string(), createdAt: z.string(), documents: z.array(publishedDocumentSchema), catalogs: z.record(z.string(), catalogSchema), entities: z.record(z.string(), z.array(entitySchema)), coverageIncomplete: z.boolean(), config: z.object({ chunkTargetChars: z.number() }), report: reportSchema });
+export const jobStateSchema = z.enum(['queued', 'preflight', 'converting', 'describing', 'splitting', 'indexing', 'validating', 'publishing', 'needs-review', 'paused-budget', 'cancelled', 'failed', 'ready']);
+export const jobSchema = z.object({ jobId: idSchema, kbId: idSchema, docId: idSchema, title: z.string(), sourceVersion: z.string(), filename: z.string(), sourceFormat: z.enum(['docx', 'pdf']).default('docx'),
+    sourceOriginalHash: z.string(), sourceHash: z.string().optional(), buildKey: z.string(), config: settingsSchema, routes: z.object({ vision: routeSchema.nullable(), summary: routeSchema.nullable() }),
+    schema: z.string(), schemaHash: z.string(), overlays: z.record(z.string(), overlaySchema), sessionId: z.string().nullable(), state: jobStateSchema, createdAt: z.string(), updatedAt: z.string().optional(),
+    progress: z.object({ totalPages: z.number().optional(), parsedPages: z.number().optional(), ocrPages: z.number().optional(), tables: z.number().optional(), blocks: z.number().optional(), currentPage: z.number().optional(),
+        totalFigures: z.number().optional(), completedFigures: z.number().optional(), inflightFigures: z.array(z.object({ occurrenceId: z.string(), caption: z.string() })).optional(),
+        totalChunks: z.number().optional(), indexedChunks: z.number().optional(), currentChunk: z.object({ title: z.string(), headingPath: z.array(z.string()), chunkId: z.string() }).optional() }),
+    usage: usageSchema, failures: z.array(z.object({ at: z.string(), stage: jobStateSchema, error: z.string() })), events: z.array(z.object({ at: z.string(), state: jobStateSchema })).optional(),
+    figureRefresh: z.object({ occurrenceId: z.string(), token: z.string() }).nullable().optional(), visionCapability: z.string().optional(), parserVersion: z.string().optional(),
+    pageStart: z.number().optional(), pageEnd: z.number().optional(), pdf: z.object({ pageCount: z.number(), selectedPages: rangeSchema, outlineCount: z.number().optional() }).optional(),
+    revision: z.string().optional(), report: reportSchema.optional(), outputPath: z.string().optional(), error: z.string().nullable().optional(), recoveredAt: z.string().optional(), finishedAt: z.string().optional(), cancelRequested: z.boolean().optional() });
+export const librarySchema = z.object({ kbId: idSchema, name: z.string(), createdAt: z.string(), deletedAt: z.string().optional(), purge: z.object({ startedAt: z.string(), jobIds: z.array(idSchema), error: z.string().optional() }).optional() });
+export const registrySchema = z.object({ formatVersion: z.literal(1), kbs: z.array(librarySchema), activities: z.array(z.object({ eventId: z.string(), at: z.string(), action: z.enum(['create', 'trash', 'restore', 'purge']), kbId: z.string(), name: z.string() })).default([]) });
+export const currentSchema = z.object({ revision: idSchema, publishedAt: z.string(), buildKey: z.string(), coverageIncomplete: z.boolean() });
+export const uploadSchema = z.object({ receiptId: idSchema, filename: z.string(), createdAt: z.string(), bytes: z.number(), hash: z.string() });
+export const pageReceiptSchema = z.object({ blocks: z.array(blockSchema), assets: z.array(assetSchema), warnings: z.array(warningSchema), audit: pageAuditSchema });

package/lib/document/convert.d.ts

@@ -0,0 +1,5 @@
+import type { Settings } from '../config.ts';
+import type { Block, Asset, Document } from '../data.ts';
+export declare function convertDocx(buffer: Buffer, destination: string, config: Settings, signal?: AbortSignal): Promise<Document>;
+export declare function blockMarkdown(block: Block, assets?: Asset[], assetPrefix?: string): string;
+export declare function legacyToDocx(file: string, executable: string, signal?: AbortSignal): Promise<Buffer>;

package/lib/document/convert.js

@@ -0,0 +1,325 @@
+import { z } from 'zod';
+import { block as makeBlock, documentSchema } from "../data.js";
+import { required } from "../util.js";
+import mammoth from 'mammoth';
+import { XMLParser } from 'fast-xml-parser';
+import { parseFragment } from 'parse5';
+import { atomicWrite, checkAbort, escapeHtml, hash, mdText } from "../util.js";
+import { unpackDocx } from "./zip.js";
+import { join, posix } from 'node:path';
+import { spawn } from 'node:child_process';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+const parser = new XMLParser({ ignoreAttributes: false, removeNSPrefix: true, parseTagValue: false, trimValues: false });
+const xmlSchema = z.lazy(() => z.union([z.string(), z.number(), z.boolean(), z.null(), z.array(xmlSchema), z.record(z.string(), xmlSchema)]));
+const list = (value) => value === undefined ? [] : Array.isArray(value) ? value : [value];
+const object = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
+function child(value, ...keys) {
+    for (const key of keys)
+        value = object(value)[key];
+    return value;
+}
+function xmlString(value) { return typeof value === 'string' || typeof value === 'number' ? String(value) : ''; }
+function parseXml(bytes) {
+    const xml = Buffer.from(bytes).toString('utf8');
+    if (/<!DOCTYPE|<!ENTITY/iu.test(xml))
+        throw new Error('不允许 DOCX XML 实体定义');
+    return xmlSchema.parse(parser.parse(xml));
+}
+function visit(value, fn) {
+    for (const [key, c] of Object.entries(object(value)))
+        for (const item of list(c)) {
+            fn(key, item);
+            visit(item, fn);
+        }
+}
+function wordText(value) {
+    let result = '';
+    visit(value, (key, v) => {
+        if (key === 't')
+            result += typeof v === 'object' ? xmlString(child(v, '#text')) : xmlString(v);
+    });
+    return result;
+}
+const normalize = (value) => value.replace(/\s/gu, '');
+function audit(files) {
+    const doc = parseXml(required(files['word/document.xml'])), styles = files['word/styles.xml'] ? parseXml(files['word/styles.xml']) : {};
+    const warnings = [], styleLevels = {}, headingLevels = {}, outlineLevels = {}, anchors = new Set();
+    const styleRows = list(child(styles, 'styles', 'style'));
+    const resolveLevel = (styleId, seen = new Set()) => {
+        if (seen.has(styleId) || !styleId)
+            return;
+        seen.add(styleId);
+        const row = styleRows.find(v => xmlString(child(v, '@_styleId')) === styleId);
+        if (!row)
+            return;
+        const explicit = child(row, 'pPr', 'outlineLvl', '@_val'), name = xmlString(child(row, 'name', '@_val')) || styleId, common = /^(?:heading|标题)\s*([1-9])$/iu.exec(name);
+        return explicit !== undefined && Number(explicit) < 9 ? Number(explicit) + 1 : common ? Number(common[1]) : resolveLevel(xmlString(child(row, 'basedOn', '@_val')), seen);
+    };
+    for (const row of styleRows) {
+        const styleId = xmlString(child(row, '@_styleId')), level = resolveLevel(styleId);
+        if (level)
+            styleLevels[styleId] = level;
+    }
+    let expectedFigures = 0, expectedTables = 0, textBoxes = 0, footnoteRefs = 0;
+    const expectedHeadings = [];
+    visit(doc, (key, value) => {
+        if (key === 'p') {
+            const outline = child(value, 'pPr', 'outlineLvl', '@_val'), style = xmlString(child(value, 'pPr', 'pStyle', '@_val')), level = outline !== undefined && Number(outline) < 9 ? Number(outline) + 1 : styleLevels[style];
+            if (outline !== undefined && Number(outline) < 9)
+                outlineLevels[normalize(wordText(value))] = Number(outline) + 1;
+            if (level && !/^TOC/iu.test(style)) {
+                const title = wordText(value);
+                if (title) {
+                    expectedHeadings.push(title);
+                    headingLevels[normalize(title)] = level;
+                }
+            }
+        }
+        if ((key === 'blip' && child(value, '@_embed')) || (key === 'imagedata' && child(value, '@_id')))
+            expectedFigures++;
+        if (key === 'tbl')
+            expectedTables++;
+        if (key === 'txbxContent')
+            textBoxes++;
+        if (key === 'footnoteReference' || key === 'endnoteReference')
+            footnoteRefs++;
+        if (key === 'bookmarkStart' && child(value, '@_name'))
+            anchors.add(xmlString(child(value, '@_name')));
+        if (['graphicData', 'object', 'pict', 'oMath', 'oMathPara', 'ins', 'del'].includes(key)) {
+            if (key === 'graphicData' && /\/picture$/iu.test(xmlString(child(value, '@_uri'))))
+                return;
+            if (key === 'pict' && JSON.stringify(value).includes('imagedata'))
+                return;
+            const reason = ['ins', 'del'].includes(key) ? '文档含修订，请核对转换结果并明确接受版本' : '复杂对象需要核对预览或补图';
+            warnings.push({ code: 'unsupported-object', message: `${key}：${reason}`, kind: key, snippet: wordText(value).slice(0, 120), reason, blocking: true });
+        }
+    });
+    for (const [name, bytes] of Object.entries(files))
+        if (/word\/(?:header|footer)\d*\.xml$/u.test(name)) {
+            const t = wordText(parseXml(bytes)).trim();
+            if (t)
+                warnings.push({ code: 'header-footer', message: `${name} 页眉/页脚未作为正文切分：${t.slice(0, 160)}`, blocking: false });
+        }
+    return { warnings, styleLevels, headingLevels, outlineLevels, expectedFigures, expectedTables, expectedHeadings, textBoxes, footnoteRefs, anchors: [...anchors] };
+}
+const children = (node) => 'childNodes' in node ? node.childNodes : [];
+const tag = (node) => 'tagName' in node ? node.tagName : '';
+function isMammothNode(value) {
+    if (typeof value !== 'object' || value === null || !('type' in value) || typeof value.type !== 'string')
+        return false;
+    for (const [key, field] of Object.entries(value))
+        if (['styleId', 'styleName', 'value'].includes(key) && field !== null && field !== undefined && typeof field !== 'string')
+            return false;
+    return !('children' in value) || value.children === undefined || Array.isArray(value.children) && value.children.every(isMammothNode);
+}
+function attr(node, key) { return 'attrs' in node ? node.attrs.find(a => a.name === key)?.value : undefined; }
+function htmlText(node) { return 'value' in node ? node.value : children(node).map(htmlText).join(''); }
+function safeLink(value) { return /^(?:https?:|mailto:|#)/i.test(value || '') ? value : null; }
+function inline(node, internalRefs) {
+    if ('value' in node)
+        return node.value.replace(/([\\`*[\]<>])/g, '\\$1');
+    const t = tag(node), body = children(node).map(n => inline(n, internalRefs)).join('');
+    if (['script', 'style', 'iframe', 'object'].includes(t))
+        return '';
+    if (t === 'img')
+        return `\u0001${attr(node, 'src') || ''}\u0001`;
+    if (t === 'br')
+        return '\n';
+    if (t === 'strong' || t === 'b')
+        return `**${body}**`;
+    if (t === 'em' || t === 'i')
+        return `*${body}*`;
+    if (t === 'sup' || t === 'sub')
+        return `<${t}>${body}</${t}>`;
+    if (t === 'a') {
+        const href = safeLink(attr(node, 'href'));
+        if (href?.startsWith('#'))
+            internalRefs.push({ target: href.slice(1), label: htmlText(node) });
+        return href ? `[${body}](${href.replace(/[()\s]/g, ch => encodeURIComponent(ch))})` : body;
+    }
+    return body;
+}
+function tableCells(table) {
+    const rows = [];
+    const walk = (node) => {
+        if (tag(node) === 'tr')
+            rows.push(children(node).filter(c => ['td', 'th'].includes(tag(c))).map(cell => ({
+                text: htmlText(cell).trim(), colspan: Math.max(1, Math.min(100, Number(attr(cell, 'colspan')) || 1)), rowspan: Math.max(1, Math.min(1000, Number(attr(cell, 'rowspan')) || 1)),
+            })));
+        else
+            for (const child of children(node))
+                walk(child);
+    };
+    walk(table);
+    return rows;
+}
+function tableHtml(rows) {
+    return '<table>\n' + rows.map((row, i) => '<tr>' + row.map(cell => `<${i === 0 ? 'th' : 'td'}${cell.colspan > 1 ? ` colspan="${cell.colspan}"` : ''}${cell.rowspan > 1 ? ` rowspan="${cell.rowspan}"` : ''}>${escapeHtml(cell.text).replace(/\n/g, '<br>')}</${i === 0 ? 'th' : 'td'}>`).join('') + '</tr>').join('\n') + '\n</table>';
+}
+export async function convertDocx(buffer, destination, config, signal) {
+    checkAbort(signal);
+    const files = unpackDocx(buffer, config), originalAudit = audit(files);
+    const assets = [], warnings = [...originalAudit.warnings];
+    const mimeExtension = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/gif': 'gif', 'image/webp': 'webp', 'image/bmp': 'bmp', 'image/tiff': 'tif', 'image/x-emf': 'emf', 'image/x-wmf': 'wmf', 'image/svg+xml': 'svg' };
+    const addAsset = async (data, mime, origin) => {
+        const assetId = `a-${hash(data).slice(0, 20)}`;
+        let asset = assets.find(a => a.assetId === assetId);
+        if (!asset) {
+            asset = { assetId, mime, path: `assets/${assetId}.${mimeExtension[mime] || 'bin'}`, bytes: data.length, origin };
+            assets.push(asset);
+            await atomicWrite(join(destination, asset.path), data);
+        }
+        return asset;
+    };
+    const rawMedia = Object.entries(files).filter(([name]) => name.startsWith('word/media/'));
+    for (const [name, bytes] of rawMedia) {
+        const ext = (name.split('.').at(-1) || '').toLowerCase(), mime = Object.entries(mimeExtension).find(([, x]) => x === ext || ext === 'jpeg' && x === 'jpg')?.[0] || 'application/octet-stream';
+        await addAsset(Buffer.from(bytes), mime, name);
+    }
+    const result = await mammoth.convertToHtml({ buffer }, {
+        externalFileAccess: false, includeDefaultStyleMap: true,
+        styleMap: Array.from({ length: 9 }, (_, i) => `p[style-name='DSHHeading${i + 1}'] => h${i + 1}:fresh`),
+        transformDocument(document) {
+            if (!isMammothNode(document))
+                throw new Error('Word 转换器文档节点无效');
+            const walk = (node) => {
+                if (node.type === 'paragraph' && !/^TOC/i.test(node.styleId || '')) {
+                    const level = originalAudit.styleLevels[node.styleId || ''] || originalAudit.outlineLevels[normalize((node.children || []).map(c => c.value || (c.children || []).map(x => x.value || '').join('')).join(''))];
+                    if (level) {
+                        node.styleName = `DSHHeading${level}`;
+                        node.styleId = `DSHHeading${level}`;
+                    }
+                }
+                for (const child of node.children || [])
+                    walk(child);
+            };
+            walk(document);
+            return document;
+        },
+        convertImage: mammoth.images.imgElement(async (image) => {
+            checkAbort(signal);
+            const asset = await addAsset(await image.readAsBuffer(), image.contentType, 'mammoth');
+            return { src: `asset:${asset.assetId}` };
+        }),
+    });
+    checkAbort(signal);
+    warnings.push(...result.messages.map(m => ({ code: 'converter', message: m.message,
+        // Unmapped styles lose formatting, not paragraph text; heading coverage is audited separately.
+        blocking: !/^Unrecognised (?:paragraph|run) style:/i.test(m.message) && /image|unrecognised|unsupported/i.test(m.message),
+    })));
+    const fragment = parseFragment(result.value), blocks = [], internalRefs = [];
+    let figureCount = 0;
+    const add = (value) => { blocks.push(makeBlock({ blockId: `b-${String(blocks.length + 1).padStart(6, '0')}`, ...value })); return required(blocks.at(-1)); };
+    const addInline = (node, type = 'paragraph', prefix = '') => {
+        const body = inline(node, internalRefs), pieces = body.split(String.fromCharCode(1)).filter(Boolean);
+        for (const piece of pieces) {
+            if (piece.startsWith('asset:')) {
+                figureCount++;
+                add({ type: 'figure', assetId: piece.slice(6), occurrenceId: `fig-${String(figureCount).padStart(5, '0')}`, caption: '', text: '' });
+            }
+            else if (piece.trim())
+                add({ type, text: prefix + piece.trim(), anchor: attr(node, 'id') || null });
+        }
+    };
+    const walk = (node, depth = 0) => {
+        const t = tag(node);
+        if (/^h[1-9]$/.test(t))
+            add({ type: 'heading', level: Number(t[1]), text: htmlText(node).trim(), anchor: attr(node, 'id') || null });
+        else if (t === 'table') {
+            const rows = tableCells(node), table = add({ type: 'table', rows, text: rows.map(row => row.map(c => c.text).join(' | ')).join('\n'), markdown: tableHtml(rows) });
+            const scan = (n) => {
+                if (tag(n) === 'img') {
+                    const assetId = (attr(n, 'src') || '').replace(/^asset:/, '');
+                    figureCount++;
+                    add({ type: 'figure', assetId, occurrenceId: `fig-${String(figureCount).padStart(5, '0')}`, caption: '', tableRef: table.blockId, text: '' });
+                }
+                else
+                    for (const c of children(n))
+                        scan(c);
+            };
+            scan(node);
+        }
+        else if (t === 'p') {
+            const t = htmlText(node).trim(), previous = blocks.at(-1);
+            if (/^(?:图|Figure)\s*[A-Za-z]?\d[\d.\-–]*/i.test(t) && previous?.type === 'figure') {
+                previous.caption = t;
+                add({ type: 'caption', text: t, figureRef: previous.occurrenceId });
+            }
+            else
+                addInline(node, attr(node, 'id')?.startsWith('footnote') ? 'note' : 'paragraph');
+        }
+        else if (t === 'li') {
+            for (const child of children(node)) {
+                if (['ul', 'ol'].includes(tag(child)))
+                    walk(child, depth + 1);
+                else
+                    addInline(child, 'list', '  '.repeat(depth) + '- ');
+            }
+        }
+        else if (t === 'img')
+            addInline(node);
+        else if (!['script', 'style', 'iframe', 'object'].includes(t))
+            for (const child of children(node))
+                walk(child, depth);
+    };
+    for (const node of fragment.childNodes)
+        walk(node);
+    if (!blocks.some(b => b.text || b.type === 'figure'))
+        throw new Error('未能提取正文');
+    if (!blocks.some(b => b.type === 'heading'))
+        warnings.push({ code: 'no-headings', message: '未识别可靠标题，按正文段落分组', blocking: false });
+    if (figureCount < originalAudit.expectedFigures)
+        warnings.push({ code: 'missing-figures', message: `正文引用 ${originalAudit.expectedFigures} 张图片，转换得到 ${figureCount} 个位置`, blocking: true });
+    if (blocks.filter(b => b.type === 'table').length < originalAudit.expectedTables)
+        warnings.push({ code: 'missing-tables', message: '表格数量未能覆盖原文，需核对嵌套表格', blocking: true });
+    const headings = blocks.filter(b => b.type === 'heading').map(b => normalize(b.text));
+    for (const title of originalAudit.expectedHeadings)
+        if (!headings.includes(normalize(title)))
+            warnings.push({ code: 'missing-heading', message: `未转换标题：${title}`, blocking: true });
+    const anchors = new Set(blocks.filter(b => b.anchor).map(b => b.anchor));
+    for (const ref of internalRefs)
+        if (!anchors.has(ref.target))
+            warnings.push({ code: 'unresolved-reference', message: `未定位内部引用：${ref.label} → ${ref.target}`, blocking: false });
+    const rawText = (await mammoth.extractRawText({ buffer })).value;
+    // A token multiset catches omitted paragraphs without claiming Word page positions.
+    const normalizedOutput = normalize(htmlText(fragment));
+    for (const paragraph of rawText.split(/\n\s*\n/).map(normalize).filter(x => x.length > 12)) {
+        if (!normalizedOutput.includes(paragraph))
+            warnings.push({ code: 'text-coverage', message: `需要核对未完整匹配的正文：${paragraph.slice(0, 100)}`, blocking: true });
+    }
+    const document = documentSchema.parse({ formatVersion: 1, blocks, assets, internalRefs, warnings, audit: { ...originalAudit, rawMediaCount: rawMedia.length, convertedFigures: figureCount, convertedTables: blocks.filter(b => b.type === 'table').length } });
+    await atomicWrite(join(destination, 'document.json'), document);
+    await atomicWrite(join(destination, 'converted.md'), blocks.map(b => blockMarkdown(b, assets)).join('\n\n') + '\n');
+    return document;
+}
+export function blockMarkdown(block, assets = [], assetPrefix = '') {
+    if (block.type === 'heading')
+        return '#'.repeat(Math.min(6, block.level)) + ' ' + block.text;
+    if (block.type === 'figure') {
+        const asset = assets.find(a => a.assetId === block.assetId);
+        return asset ? `![${mdText(block.caption || block.occurrenceId)}](${posix.join(assetPrefix, asset.path)})` : `> 图片未提取：${block.occurrenceId}`;
+    }
+    if (block.type === 'code')
+        return '```\n' + (block.text || '').replace(/```/g, '` ` `') + '\n```';
+    return block.markdown || block.text || '';
+}
+export async function legacyToDocx(file, executable, signal) {
+    if (!executable)
+        throw new Error('旧 .doc 需要另存为 .docx，或配置 libreOfficePath');
+    const dir = await mkdtemp(join(tmpdir(), 'dsh-kb-convert-'));
+    try {
+        await new Promise((yes, no) => {
+            const child = spawn(executable, ['-env:UserInstallation=file://' + dir.replace(/\\/g, '/'), '--headless', '--convert-to', 'docx', '--outdir', dir, file], { signal, stdio: ['ignore', 'ignore', 'pipe'] });
+            let error = '';
+            child.stderr.on('data', b => { error = (error + b).slice(-2000); });
+            child.on('error', no);
+            child.on('exit', code => code === 0 ? yes() : no(new Error(`Word 转换失败：${error}`)));
+        });
+        const { basename } = await import('node:path');
+        return await readFile(join(dir, basename(file).replace(/\.doc$/i, '.docx')));
+    }
+    finally {
+        await rm(dir, { recursive: true, force: true });
+    }
+}

package/lib/document/pdf-data.d.ts

@@ -0,0 +1,99 @@
+import { z } from 'zod';
+import { type blockSchema } from '../data.ts';
+import type { Asset, Block, Box, Source, Warning } from '../data.ts';
+export declare const spanSchema: z.ZodObject<{
+    id: z.ZodString;
+    text: z.ZodString;
+    bbox: z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodNumber], null>;
+    readingBox: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodNumber], null>>;
+    fontSize: z.ZodNumber;
+    fontName: z.ZodOptional<z.ZodString>;
+    mcid: z.ZodNullable<z.ZodString>;
+    artifact: z.ZodBoolean;
+}, z.core.$strip>;
+export type Span = z.infer<typeof spanSchema>;
+export interface Structure {
+    type?: string | undefined;
+    id?: string | undefined;
+    role?: string | undefined;
+    children?: Structure[] | undefined;
+}
+export declare const structureSchema: z.ZodType<Structure>;
+export declare const nativePageSchema: z.ZodObject<{
+    page: z.ZodNumber;
+    width: z.ZodNumber;
+    height: z.ZodNumber;
+    rotation: z.ZodNumber;
+    viewportTransform: z.ZodArray<z.ZodNumber>;
+    warnings: z.ZodArray<z.ZodString>;
+    spans: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        text: z.ZodString;
+        bbox: z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodNumber], null>;
+        readingBox: z.ZodOptional<z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodNumber], null>>;
+        fontSize: z.ZodNumber;
+        fontName: z.ZodOptional<z.ZodString>;
+        mcid: z.ZodNullable<z.ZodString>;
+        artifact: z.ZodBoolean;
+    }, z.core.$strip>>;
+    tree: z.ZodNullable<z.ZodType<Structure, unknown, z.core.$ZodTypeInternals<Structure, unknown>>>;
+    drawings: z.ZodArray<z.ZodObject<{
+        bbox: z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodNumber], null>;
+        kind: z.ZodEnum<{
+            path: "path";
+            image: "image";
+        }>;
+        shape: z.ZodNullable<z.ZodString>;
+        mcid: z.ZodNullable<z.ZodString>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type NativePage = z.infer<typeof nativePageSchema>;
+export declare const pdfInfoSchema: z.ZodObject<{
+    warnings: z.ZodArray<z.ZodString>;
+    pageCount: z.ZodNumber;
+    pageLabels: z.ZodNullable<z.ZodArray<z.ZodString>>;
+    outline: z.ZodArray<z.ZodObject<{
+        title: z.ZodString;
+        level: z.ZodNumber;
+        page: z.ZodNullable<z.ZodNumber>;
+        y: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strip>>;
+}, z.core.$strip>;
+export type PdfInfo = z.infer<typeof pdfInfoSchema>;
+export declare const renderSchema: z.ZodObject<{
+    data: z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>;
+    scale: z.ZodNumber;
+    bbox: z.ZodTuple<[z.ZodNumber, z.ZodNumber, z.ZodNumber, z.ZodNumber], null>;
+    warnings: z.ZodArray<z.ZodString>;
+}, z.core.$strip>;
+export type RenderResult = z.infer<typeof renderSchema>;
+export type PdfSource = Source & {
+    bbox: Box;
+};
+export interface LayoutBlock extends Block {
+    source: PdfSource;
+    captionFor?: LayoutBlock | undefined;
+    captionBlock?: LayoutBlock | undefined;
+    previewAsset?: Asset | undefined;
+}
+export declare const pdfBlock: (value: z.input<typeof blockSchema> & {
+    source: PdfSource;
+}) => LayoutBlock;
+export interface Layout {
+    blocks: LayoutBlock[];
+    warnings: Warning[];
+    excluded: {
+        kind: string;
+        text?: string | undefined;
+        bbox: Box;
+    }[];
+    ocr: boolean;
+    hasGarbled: boolean;
+    needsLayout: boolean;
+    navigation: boolean;
+    unexplained: Box[];
+    source: (bbox: Box, method?: string) => PdfSource;
+    rawText: string;
+    spanCount: number;
+    artifactCount: number;
+}

package/lib/document/pdf-data.js

@@ -0,0 +1,9 @@
+import { z } from 'zod';
+import { bboxSchema, block as makeBlock } from "../data.js";
+export const spanSchema = z.object({ id: z.string(), text: z.string(), bbox: bboxSchema, readingBox: bboxSchema.optional(), fontSize: z.number(), fontName: z.string().optional(), mcid: z.string().nullable(), artifact: z.boolean() });
+export const structureSchema = z.lazy(() => z.object({ type: z.string().optional(), id: z.string().optional(), role: z.string().optional(), children: z.array(structureSchema).optional() }));
+export const nativePageSchema = z.object({ page: z.number().int().positive(), width: z.number().positive(), height: z.number().positive(), rotation: z.number(), viewportTransform: z.array(z.number()).length(6),
+    warnings: z.array(z.string()), spans: z.array(spanSchema), tree: structureSchema.nullable(), drawings: z.array(z.object({ bbox: bboxSchema, kind: z.enum(['path', 'image']), shape: z.string().nullable(), mcid: z.string().nullable() })) });
+export const pdfInfoSchema = z.object({ warnings: z.array(z.string()), pageCount: z.number().int().positive(), pageLabels: z.array(z.string()).nullable(), outline: z.array(z.object({ title: z.string(), level: z.number(), page: z.number().nullable(), y: z.number().nullable() })) });
+export const renderSchema = z.object({ data: z.instanceof(Uint8Array), scale: z.number(), bbox: bboxSchema, warnings: z.array(z.string()) });
+export const pdfBlock = (value) => ({ ...makeBlock(value), source: value.source });

package/lib/document/pdf-layout.d.ts

@@ -0,0 +1,24 @@
+import type { Box, Cell } from '../data.ts';
+import type { NativePage, PdfInfo, Span, Layout } from './pdf-data.ts';
+export declare const PDF_PARSER_VERSION = "pdfjs-native-1";
+export declare const normalize: (value: unknown) => string;
+export declare const union: (boxes: Box[]) => Box | null;
+export declare const intersects: (a: Box, b: Box) => boolean;
+export declare function boundedBox(b: Box, page: Pick<NativePage, 'width' | 'height'>, pad?: number): Box;
+export declare function lines(spans: Span[]): {
+    text: string;
+    spans: {
+        id: string;
+        text: string;
+        bbox: [number, number, number, number];
+        fontSize: number;
+        mcid: string | null;
+        artifact: boolean;
+        readingBox?: [number, number, number, number] | undefined;
+        fontName?: string | undefined;
+    }[];
+    bbox: [number, number, number, number];
+}[];
+export declare const textOf: (spans: Span[]) => string;
+export declare function tableMarkdown(rows: Cell[][]): string;
+export declare function nativeLayout(page: NativePage, info: PdfInfo): Layout;