@de-otio/chaoskb-client 0.3.2 → 0.3.4

package/dist/pipeline/file-extract.js

@@ -0,0 +1,249 @@
+/**
+ * Local file content extraction.
+ *
+ * Dispatches to format-specific extractors based on file extension.
+ * Supports PDF, DOCX, PPTX, HTML, TXT, and Markdown.
+ */
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+// ===== Supported formats ===================================================
+const EXTENSION_MAP = {
+    '.pdf': 'pdf',
+    '.docx': 'docx',
+    '.pptx': 'pptx',
+    '.html': 'html',
+    '.htm': 'html',
+    '.txt': 'txt',
+    '.md': 'md',
+    '.markdown': 'md',
+};
+const SUPPORTED_EXTENSIONS = Object.keys(EXTENSION_MAP).join(', ');
+// ===== Limits ==============================================================
+/** Maximum file size in bytes (50 MB). */
+const MAX_FILE_SIZE = 50 * 1024 * 1024;
+/** Maximum total uncompressed size for ZIP-based formats (100 MB). */
+const MAX_UNCOMPRESSED_SIZE = 100 * 1024 * 1024;
+// ===== Public API ==========================================================
+/**
+ * Extract content from a local file.
+ *
+ * @param filePath - Path to the file (resolved to absolute).
+ * @returns Extracted content with title, text, and the absolute path as `url`.
+ * @throws On missing/unreadable file, unsupported format, or empty content.
+ */
+export async function extractFromFile(filePath) {
+    // Check extension first for a clear error on unsupported formats
+    const ext = path.extname(filePath).toLowerCase();
+    const format = EXTENSION_MAP[ext];
+    if (!format) {
+        throw new Error(`Unsupported file format "${ext}". Supported formats: ${SUPPORTED_EXTENSIONS}`);
+    }
+    const absPath = path.resolve(filePath);
+    // Resolve symlinks to get the real path, then re-check the extension
+    let realPath;
+    try {
+        realPath = await fs.realpath(absPath);
+    }
+    catch {
+        throw new Error(`File not found or not readable: ${path.basename(filePath)}`);
+    }
+    const realExt = path.extname(realPath).toLowerCase();
+    if (!EXTENSION_MAP[realExt]) {
+        throw new Error(`Symlink target has unsupported extension "${realExt}". Supported formats: ${SUPPORTED_EXTENSIONS}`);
+    }
+    // Check file is regular and within size limit
+    const stat = await fs.stat(realPath);
+    if (!stat.isFile()) {
+        throw new Error('Path is not a regular file.');
+    }
+    if (stat.size > MAX_FILE_SIZE) {
+        throw new Error(`File is too large (${(stat.size / 1024 / 1024).toFixed(1)} MB). Maximum supported size is ${MAX_FILE_SIZE / 1024 / 1024} MB.`);
+    }
+    switch (format) {
+        case 'pdf':
+            return extractPdf(realPath);
+        case 'docx':
+            return extractDocx(realPath);
+        case 'pptx':
+            return extractPptx(realPath);
+        case 'html':
+            return extractHtmlFile(realPath);
+        case 'txt':
+        case 'md':
+            return extractPlainText(realPath);
+        default:
+            throw new Error(`Unsupported format: ${format}`);
+    }
+}
+// ===== Format extractors ===================================================
+async function extractPdf(filePath) {
+    const { PDFParse } = await import('pdf-parse');
+    const buffer = await fs.readFile(filePath);
+    const uint8 = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+    const parser = new PDFParse({ data: uint8, isEvalSupported: false });
+    const textResult = await parser.getText();
+    // Strip the page footer markers ("-- N of M --")
+    const rawText = textResult.text.replace(/\n-- \d+ of \d+ --\n/g, '\n');
+    const content = cleanText(rawText);
+    if (content.length === 0) {
+        throw new Error('No extractable text in PDF. The file may be a scanned document without OCR.');
+    }
+    const infoResult = await parser.getInfo();
+    const title = infoResult?.info?.Title || filenameTitle(filePath);
+    parser.destroy();
+    return {
+        title,
+        content,
+        url: filePath,
+        byteLength: Buffer.byteLength(content, 'utf-8'),
+    };
+}
+async function extractDocx(filePath) {
+    const mammoth = await import('mammoth');
+    const result = await mammoth.convertToHtml({ path: filePath });
+    if (!result.value || result.value.trim().length === 0) {
+        throw new Error('No extractable content in DOCX file.');
+    }
+    // Pipe the clean HTML through existing Readability extraction
+    const { extractContent } = await import('./extract.js');
+    try {
+        const extracted = extractContent(result.value, filePath);
+        return {
+            ...extracted,
+            url: filePath,
+            title: extracted.title || filenameTitle(filePath),
+        };
+    }
+    catch {
+        // If Readability fails (e.g. very simple doc), fall back to plain text
+        const text = cleanText(result.value.replace(/<[^>]+>/g, ' '));
+        if (text.length === 0) {
+            throw new Error('No extractable content in DOCX file.');
+        }
+        return {
+            title: filenameTitle(filePath),
+            content: text,
+            url: filePath,
+            byteLength: Buffer.byteLength(text, 'utf-8'),
+        };
+    }
+}
+async function extractPptx(filePath) {
+    const JSZip = (await import('jszip')).default;
+    const buffer = await fs.readFile(filePath);
+    const zip = await JSZip.loadAsync(buffer);
+    // Find slide XML files and sort by slide number
+    const slideFiles = Object.keys(zip.files)
+        .filter((name) => /^ppt\/slides\/slide\d+\.xml$/.test(name))
+        .sort((a, b) => {
+        const numA = parseInt(a.match(/slide(\d+)/)?.[1] ?? '0');
+        const numB = parseInt(b.match(/slide(\d+)/)?.[1] ?? '0');
+        return numA - numB;
+    });
+    if (slideFiles.length === 0) {
+        throw new Error('No slides found in PPTX file.');
+    }
+    // Decompress slides incrementally, tracking actual bytes to guard against zip bombs
+    const slideTexts = [];
+    let firstSlideTitle = '';
+    let totalDecompressed = 0;
+    for (const slideFile of slideFiles) {
+        const bytes = await zip.file(slideFile).async('uint8array');
+        totalDecompressed += bytes.byteLength;
+        if (totalDecompressed > MAX_UNCOMPRESSED_SIZE) {
+            throw new Error(`PPTX uncompressed content exceeds ${MAX_UNCOMPRESSED_SIZE / 1024 / 1024} MB limit.`);
+        }
+        const xml = new TextDecoder().decode(bytes);
+        const text = extractSlideText(xml);
+        if (text) {
+            slideTexts.push(text);
+            if (!firstSlideTitle) {
+                firstSlideTitle = text.split('\n')[0].trim();
+            }
+        }
+    }
+    const content = cleanText(slideTexts.join('\n\n'));
+    if (content.length === 0) {
+        throw new Error('No extractable text in PPTX file.');
+    }
+    return {
+        title: firstSlideTitle || filenameTitle(filePath),
+        content,
+        url: filePath,
+        byteLength: Buffer.byteLength(content, 'utf-8'),
+    };
+}
+async function extractHtmlFile(filePath) {
+    const html = await fs.readFile(filePath, 'utf-8');
+    const { extractContent } = await import('./extract.js');
+    const extracted = extractContent(html, filePath);
+    return {
+        ...extracted,
+        url: filePath,
+        title: extracted.title || filenameTitle(filePath),
+    };
+}
+async function extractPlainText(filePath) {
+    const raw = await fs.readFile(filePath, 'utf-8');
+    const content = cleanText(raw);
+    if (content.length === 0) {
+        throw new Error('Empty file.');
+    }
+    // Title: first markdown heading, or first short line, or filename
+    let title = '';
+    const headingMatch = content.match(/^#\s+(.+)$/m);
+    if (headingMatch) {
+        title = headingMatch[1].trim();
+    }
+    else {
+        const firstLine = content.split('\n').find((l) => l.trim().length > 0);
+        if (firstLine && firstLine.length < 120) {
+            title = firstLine.trim();
+        }
+    }
+    return {
+        title: title || filenameTitle(filePath),
+        content,
+        url: filePath,
+        byteLength: Buffer.byteLength(content, 'utf-8'),
+    };
+}
+// ===== Utilities ===========================================================
+/**
+ * Extract text from a PPTX slide XML string.
+ * Groups by `<a:p>` paragraphs, collects `<a:t>` text runs within each.
+ */
+function extractSlideText(xml) {
+    const paragraphs = [];
+    const pRegex = /<a:p\b[^>]*>([\s\S]*?)<\/a:p>/g;
+    const tRegex = /<a:t>([\s\S]*?)<\/a:t>/g;
+    let pMatch;
+    while ((pMatch = pRegex.exec(xml)) !== null) {
+        const pContent = pMatch[1];
+        const texts = [];
+        let tMatch;
+        tRegex.lastIndex = 0;
+        while ((tMatch = tRegex.exec(pContent)) !== null) {
+            texts.push(tMatch[1]);
+        }
+        if (texts.length > 0) {
+            paragraphs.push(texts.join(''));
+        }
+    }
+    return paragraphs.join('\n');
+}
+/** Extract a readable title from a file path (filename without extension). */
+function filenameTitle(filePath) {
+    return path.basename(filePath, path.extname(filePath));
+}
+/** Clean extracted text: strip steganographic chars, collapse whitespace, trim lines. */
+function cleanText(text) {
+    return text
+        .replace(/[\u2028\u2029]/g, '\n') // Unicode line/paragraph separators → newline
+        .replace(/[\u200B-\u200F\u202A-\u202F\u2060-\u206F\uFEFF]/g, '') // strip zero-width / bidi / invisible chars
+        .replace(/[\t ]+/g, ' ')
+        .replace(/\n{3,}/g, '\n\n')
+        .replace(/^ +| +$/gm, '')
+        .trim();
+}
+//# sourceMappingURL=file-extract.js.map

package/dist/pipeline/file-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-extract.js","sourceRoot":"","sources":["../../pipeline/file-extract.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAGlC,8EAA8E;AAE9E,MAAM,aAAa,GAA2B;IAC5C,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,IAAI;IACX,WAAW,EAAE,IAAI;CAClB,CAAC;AAEF,MAAM,oBAAoB,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnE,8EAA8E;AAE9E,0CAA0C;AAC1C,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAEvC,sEAAsE;AACtE,MAAM,qBAAqB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAEhD,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACpD,iEAAiE;IACjE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACjD,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CACb,4BAA4B,GAAG,yBAAyB,oBAAoB,EAAE,CAC/E,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEvC,qEAAqE;IACrE,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACrD,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,6CAA6C,OAAO,yBAAyB,oBAAoB,EAAE,CACpG,CAAC;IACJ,CAAC;IAED,8CAA8C;IAC9C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,GAAG,aAAa,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,sBAAsB,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,mCAAmC,aAAa,GAAG,IAAI,GAAG,IAAI,MAAM,CAC/H,CAAC;IACJ,CAAC;IAED,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,KAAK;YACR,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC9B,KAAK,MAAM;YACT,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC/B,KAAK,MAAM;YACT,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC/B,KAAK,MAAM;YACT,OAAO,eAAe,CAAC,QAAQ,CAAC,CAAC;QACnC,KAAK,KAAK,CAAC;QACX,KAAK,IAAI;YACP,OAAO,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACpC;YACE,MAAM,IAAI,KAAK,CAAC,uBAAuB,MAAM,EAAE,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,8EAA8E;AAE9E,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IAElF,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IAC1C,iDAAiD;IACjD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,UAAU,EAAE,IAAI,EAAE,KAAK,IAAI,aAAa,CAAC,QAAQ,CAAC,CAAC;IACjE,MAAM,CAAC,OAAO,EAAE,CAAC;IAEjB,OAAO;QACL,KAAK;QACL,OAAO;QACP,GAAG,EAAE,QAAQ;QACb,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC;KAChD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,QAAgB;IACzC,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAE/D,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,8DAA8D;IAC9D,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IACxD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QACzD,OAAO;YACL,GAAG,SAAS;YACZ,GAAG,EAAE,QAAQ;YACb,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,aAAa,CAAC,QAAQ,CAAC;SAClD,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,uEAAuE;QACvE,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO;YACL,KAAK,EAAE,aAAa,CAAC,QAAQ,CAAC;YAC9B,OAAO,EAAE,IAAI;YACb,GAAG,EAAE,QAAQ;YACb,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;SAC7C,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,QAAgB;IACzC,MAAM,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IAC9C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAE1C,gDAAgD;IAChD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;SACtC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC3D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;QACzD,OAAO,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC,CAAC,CAAC;IAEL,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IAED,oFAAoF;IACpF,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,IAAI,eAAe,GAAG,EAAE,CAAC;IACzB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAE1B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC7D,iBAAiB,IAAI,KAAK,CAAC,UAAU,CAAC;QACtC,IAAI,iBAAiB,GAAG,qBAAqB,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CACb,qCAAqC,qBAAqB,GAAG,IAAI,GAAG,IAAI,YAAY,CACrF,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,EAAE,CAAC;YACT,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC/C,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACnD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,OAAO;QACL,KAAK,EAAE,eAAe,IAAI,aAAa,CAAC,QAAQ,CAAC;QACjD,OAAO;QACP,GAAG,EAAE,QAAQ;QACb,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC;KAChD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,QAAgB;IAC7C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACjD,OAAO;QACL,GAAG,SAAS;QACZ,GAAG,EAAE,QAAQ;QACb,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,aAAa,CAAC,QAAQ,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAgB;IAC9C,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAE/B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;IACjC,CAAC;IAED,kEAAkE;IAClE,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAClD,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACjC,CAAC;SAAM,CAAC;QACN,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvE,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACxC,KAAK,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,KAAK,IAAI,aAAa,CAAC,QAAQ,CAAC;QACvC,OAAO;QACP,GAAG,EAAE,QAAQ;QACb,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC;KAChD,CAAC;AACJ,CAAC;AAED,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,gCAAgC,CAAC;IAChD,MAAM,MAAM,GAAG,yBAAyB,CAAC;IAEzC,IAAI,MAAM,CAAC;IACX,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC3B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,IAAI,MAAM,CAAC;QACX,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;QACrB,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,8EAA8E;AAC9E,SAAS,aAAa,CAAC,QAAgB;IACrC,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,yFAAyF;AACzF,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI;SACR,OAAO,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAgC,8CAA8C;SAC9G,OAAO,CAAC,kDAAkD,EAAE,EAAE,CAAC,CAAC,4CAA4C;SAC5G,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,IAAI,EAAE,CAAC;AACZ,CAAC"}

package/dist/pipeline/index.d.ts

@@ -7,4 +7,6 @@ export { Embedder } from './embedder.js';
 export { ModelManager } from './model-manager.js';
 export { ContentPipeline } from './content-pipeline.js';
 export { cosineSimilarity, searchEmbeddings } from './search.js';
+export { validateContent, validateFileContent, type ContentIssue } from './validate.js';
+export { extractFromFile } from './file-extract.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/pipeline/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../pipeline/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../pipeline/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,KAAK,YAAY,EAAE,MAAM,eAAe,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC"}

package/dist/pipeline/index.js

@@ -7,4 +7,6 @@ export { Embedder } from './embedder.js';
 export { ModelManager } from './model-manager.js';
 export { ContentPipeline } from './content-pipeline.js';
 export { cosineSimilarity, searchEmbeddings } from './search.js';
+export { validateContent, validateFileContent } from './validate.js';
+export { extractFromFile } from './file-extract.js';
 //# sourceMappingURL=index.js.map

package/dist/pipeline/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../pipeline/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../pipeline/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAqB,MAAM,eAAe,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC"}

package/dist/pipeline/types.d.ts

@@ -12,6 +12,8 @@ export interface PipelineConfig {
     maxRedirects: number;
     /** User-Agent header */
     userAgent: string;
+    /** Skip SSRF validation — ONLY for tests against localhost servers */
+    _skipSsrfCheck?: boolean;
 }
 /** Extracted content from a URL */
 export interface ExtractedContent {
@@ -21,6 +23,8 @@ export interface ExtractedContent {
     url: string;
     /** Byte length of extracted content */
     byteLength: number;
+    /** Validation warnings (content was stored, but quality may be degraded) */
+    warnings?: string[];
 }
 /** A text chunk from extracted content */
 export interface Chunk {
@@ -59,6 +63,8 @@ export type DownloadProgressCallback = (downloaded: number, total: number) => vo
 export interface IContentPipeline {
     /** Fetch and extract content from a URL */
     fetchAndExtract(url: string): Promise<ExtractedContent>;
+    /** Extract content from a local file (PDF, DOCX, PPTX, HTML, TXT, MD) */
+    extractFromFile(filePath: string): Promise<ExtractedContent>;
     /** Split extracted text into chunks */
     chunk(text: string): Chunk[];
     /** Embed a single text string */

package/dist/pipeline/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../pipeline/types.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,MAAM,WAAW,cAAc;IAC7B,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,iDAAiD;IACjD,aAAa,EAAE,MAAM,CAAC;IACtB,kDAAkD;IAClD,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,mCAAmC;AACnC,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,0CAA0C;AAC1C,MAAM,WAAW,KAAK;IACpB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,sEAAsE;AACtE,MAAM,MAAM,eAAe,GAAG,YAAY,CAAC;AAE3C,0CAA0C;AAC1C,MAAM,WAAW,aAAc,SAAQ,KAAK;IAC1C,uCAAuC;IACvC,SAAS,EAAE,eAAe,CAAC;IAC3B,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;CACf;AAED,6CAA6C;AAC7C,MAAM,WAAW,YAAY;IAC3B,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,uCAAuC;AACvC,MAAM,MAAM,wBAAwB,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;AAEnF,yCAAyC;AACzC,MAAM,WAAW,gBAAgB;IAC/B,2CAA2C;IAC3C,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACxD,uCAAuC;IACvC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE,CAAC;IAC7B,iCAAiC;IACjC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC9C,4BAA4B;IAC5B,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACvD,4CAA4C;IAC5C,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CACvF;AAED,iEAAiE;AACjE,MAAM,WAAW,aAAa;IAC5B,gDAAgD;IAChD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,2EAA2E;IAC3E,WAAW,CAAC,UAAU,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACpE,6BAA6B;IAC7B,YAAY,IAAI,MAAM,CAAC;IACvB,kCAAkC;IAClC,YAAY,IAAI,MAAM,CAAC;IACvB,qDAAqD;IACrD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,yDAAyD;IACzD,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CAChC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../pipeline/types.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,MAAM,WAAW,cAAc;IAC7B,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,cAAc,EAAE,MAAM,CAAC;IACvB,iDAAiD;IACjD,aAAa,EAAE,MAAM,CAAC;IACtB,kDAAkD;IAClD,cAAc,EAAE,MAAM,CAAC;IACvB,+CAA+C;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,mCAAmC;AACnC,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,mBAAmB;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,0CAA0C;AAC1C,MAAM,WAAW,KAAK;IACpB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,sEAAsE;AACtE,MAAM,MAAM,eAAe,GAAG,YAAY,CAAC;AAE3C,0CAA0C;AAC1C,MAAM,WAAW,aAAc,SAAQ,KAAK;IAC1C,uCAAuC;IACvC,SAAS,EAAE,eAAe,CAAC;IAC3B,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;CACf;AAED,6CAA6C;AAC7C,MAAM,WAAW,YAAY;IAC3B,qBAAqB;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,uCAAuC;AACvC,MAAM,MAAM,wBAAwB,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;AAEnF,yCAAyC;AACzC,MAAM,WAAW,gBAAgB;IAC/B,2CAA2C;IAC3C,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACxD,yEAAyE;IACzE,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC7D,uCAAuC;IACvC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,EAAE,CAAC;IAC7B,iCAAiC;IACjC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC9C,4BAA4B;IAC5B,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACvD,4CAA4C;IAC5C,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CACvF;AAED,iEAAiE;AACjE,MAAM,WAAW,aAAa;IAC5B,gDAAgD;IAChD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,2EAA2E;IAC3E,WAAW,CAAC,UAAU,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACpE,6BAA6B;IAC7B,YAAY,IAAI,MAAM,CAAC;IACvB,kCAAkC;IAClC,YAAY,IAAI,MAAM,CAAC;IACvB,qDAAqD;IACrD,YAAY,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IACjC,yDAAyD;IACzD,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CAChC"}

package/dist/pipeline/validate.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Content validation for the ingestion pipeline.
+ *
+ * Runs quality checks on fetched HTML and extracted content, returning
+ * structured issues.  Errors block ingestion; warnings are surfaced
+ * to the user alongside the stored content.
+ *
+ * Checks are organised into three tiers:
+ *   1. Pattern matching  — known services / phrases
+ *   2. Structural HTML   — HTML characteristics regardless of service
+ *   3. Content heuristics — statistical properties of extracted text
+ *
+ * All patterns are English-only for now.
+ */
+import type { ExtractedContent } from './types.js';
+export interface ContentIssue {
+    severity: 'error' | 'warning';
+    code: string;
+    message: string;
+}
+/**
+ * Validate fetched HTML and its extracted content.
+ *
+ * Returns all detected issues (not just the first).  Callers should
+ * treat `error`-severity issues as ingestion blockers and `warning`
+ * issues as informational.
+ */
+export declare function validateContent(html: string, extracted: ExtractedContent): ContentIssue[];
+/**
+ * Validate extracted content from a local file.
+ *
+ * Runs only Tier 3 (content heuristic) checks.  Tier 1 (pattern matching)
+ * and Tier 2 (structural HTML analysis) are specific to URL-fetched content.
+ */
+export declare function validateFileContent(extracted: ExtractedContent): ContentIssue[];
+//# sourceMappingURL=validate.d.ts.map

package/dist/pipeline/validate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../pipeline/validate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAInD,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,OAAO,GAAG,SAAS,CAAC;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AA8RD;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,gBAAgB,GAC1B,YAAY,EAAE,CAgChB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,gBAAgB,GAAG,YAAY,EAAE,CAa/E"}