abb-opencode-local-rag 0.1.0

package/dist/parser/html-parser.js

@@ -0,0 +1,102 @@
+// HTML Parser using Readability and Turndown
+// Extracts main content from HTML and converts to Markdown
+import { Readability } from '@mozilla/readability';
+import { JSDOM } from 'jsdom';
+import TurndownService from 'turndown';
+import { extractHtmlTitle } from './title-extractor.js';
+// ============================================
+// Turndown Service Configuration
+// ============================================
+/**
+ * Create and configure Turndown service for HTML to Markdown conversion
+ */
+function createTurndownService() {
+    const turndownService = new TurndownService({
+        headingStyle: 'atx', // Use # style headings
+        codeBlockStyle: 'fenced', // Use ``` for code blocks
+        bulletListMarker: '-', // Use - for bullet lists
+        emDelimiter: '_', // Use _ for emphasis
+        strongDelimiter: '**', // Use ** for bold
+    });
+    // Keep code blocks intact
+    turndownService.addRule('codeBlocks', {
+        filter: ['pre'],
+        replacement: (_content, node) => {
+            const element = node;
+            const codeElement = element.querySelector('code');
+            const code = codeElement ? codeElement.textContent : element.textContent;
+            const language = codeElement?.className?.replace('language-', '') || '';
+            return `\n\`\`\`${language}\n${code?.trim() || ''}\n\`\`\`\n`;
+        },
+    });
+    return turndownService;
+}
+// ============================================
+// HTML Parser
+// ============================================
+/**
+ * Parse HTML content and extract main content as Markdown
+ *
+ * Flow:
+ * 1. HTML string → JSDOM (DOM creation)
+ * 2. JSDOM → Readability (main content extraction, noise removal)
+ * 3. Readability result → Turndown (Markdown conversion)
+ * 4. Title extracted separately via extractHtmlTitle (NOT prepended to content)
+ *
+ * @param html - Raw HTML string
+ * @param url - Source URL (used for resolving relative links)
+ * @returns Object with content (markdown) and title (extracted separately)
+ */
+export async function parseHtml(html, url) {
+    // Handle empty or whitespace-only HTML
+    if (!html || html.trim().length === 0) {
+        return { content: '', title: '' };
+    }
+    try {
+        // Create DOM from HTML string
+        const dom = new JSDOM(html, {
+            url,
+            // Enable features needed for Readability
+            runScripts: 'outside-only',
+        });
+        const document = dom.window.document;
+        // Use Readability to extract main content
+        const reader = new Readability(document, {
+            keepClasses: false,
+            debug: false,
+        });
+        const article = reader.parse();
+        // If Readability couldn't extract content, fall back to body text
+        if (!article?.content) {
+            // Try to get body content directly
+            const bodyContent = document.body?.innerHTML || '';
+            if (!bodyContent.trim()) {
+                return { content: '', title: '' };
+            }
+            // Convert raw body HTML to Markdown
+            const turndownService = createTurndownService();
+            return { content: turndownService.turndown(bodyContent).trim(), title: '' };
+        }
+        // Convert extracted HTML content to Markdown
+        const turndownService = createTurndownService();
+        const markdown = turndownService.turndown(article.content);
+        // Extract title separately (NOT prepended to markdown content)
+        // Use URL-derived filename as fallback when Readability has no title
+        let urlFileName = '';
+        try {
+            urlFileName = new URL(url).pathname.split('/').filter(Boolean).pop() || '';
+        }
+        catch {
+            // Non-URL string, empty fallback
+        }
+        const titleResult = extractHtmlTitle(article.title || '', urlFileName);
+        const title = titleResult.title;
+        return { content: markdown.trim(), title };
+    }
+    catch (error) {
+        // Log error but don't throw - return empty values for graceful degradation
+        console.error('Failed to parse HTML:', error);
+        return { content: '', title: '' };
+    }
+}
+//# sourceMappingURL=html-parser.js.map

package/dist/parser/html-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-parser.js","sourceRoot":"","sources":["../../src/parser/html-parser.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,2DAA2D;AAE3D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAA;AAC7B,OAAO,eAAe,MAAM,UAAU,CAAA;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAcvD,+CAA+C;AAC/C,iCAAiC;AACjC,+CAA+C;AAE/C;;GAEG;AACH,SAAS,qBAAqB;IAC5B,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC;QAC1C,YAAY,EAAE,KAAK,EAAE,uBAAuB;QAC5C,cAAc,EAAE,QAAQ,EAAE,0BAA0B;QACpD,gBAAgB,EAAE,GAAG,EAAE,yBAAyB;QAChD,WAAW,EAAE,GAAG,EAAE,qBAAqB;QACvC,eAAe,EAAE,IAAI,EAAE,kBAAkB;KAC1C,CAAC,CAAA;IAEF,0BAA0B;IAC1B,eAAe,CAAC,OAAO,CAAC,YAAY,EAAE;QACpC,MAAM,EAAE,CAAC,KAAK,CAAC;QACf,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,OAAO,GAAG,IAAe,CAAA;YAC/B,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAA;YACjD,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAA;YACxE,MAAM,QAAQ,GAAG,WAAW,EAAE,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,IAAI,EAAE,CAAA;YACvE,OAAO,WAAW,QAAQ,KAAK,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,CAAA;QAC/D,CAAC;KACF,CAAC,CAAA;IAEF,OAAO,eAAe,CAAA;AACxB,CAAC;AAED,+CAA+C;AAC/C,cAAc;AACd,+CAA+C;AAE/C;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,GAAW;IAEX,uCAAuC;IACvC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;IACnC,CAAC;IAED,IAAI,CAAC;QACH,8BAA8B;QAC9B,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE;YAC1B,GAAG;YACH,yCAAyC;YACzC,UAAU,EAAE,cAAc;SAC3B,CAAC,CAAA;QAEF,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAA;QAEpC,0CAA0C;QAC1C,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAQ,EAAE;YACvC,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,KAAK;SACb,CAAC,CAAA;QAEF,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAA8B,CAAA;QAE1D,kEAAkE;QAClE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;YACtB,mCAAmC;YACnC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,IAAI,EAAE,CAAA;YAClD,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,CAAC;gBACxB,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;YACnC,CAAC;YAED,oCAAoC;YACpC,MAAM,eAAe,GAAG,qBAAqB,EAAE,CAAA;YAC/C,OAAO,EAAE,OAAO,EAAE,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;QAC7E,CAAC;QAED,6CAA6C;QAC7C,MAAM,eAAe,GAAG,qBAAqB,EAAE,CAAA;QAC/C,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QAE1D,+DAA+D;QAC/D,qEAAqE;QACrE,IAAI,WAAW,GAAG,EAAE,CAAA;QACpB,IAAI,CAAC;YACH,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAA;QAC5E,CAAC;QAAC,MAAM,CAAC;YACP,iCAAiC;QACnC,CAAC;QACD,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,WAAW,CAAC,CAAA;QACtE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAA;QAE/B,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAA;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,2EAA2E;QAC3E,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,KAAK,CAAC,CAAA;QAC7C,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;IACnC,CAAC;AACH,CAAC"}

package/dist/parser/index.d.ts

@@ -0,0 +1,214 @@
+import type { Document as MupdfDocument } from 'mupdf';
+import { type AtomicTextRange } from '../chunker/index.js';
+import { AppError } from '../utils/errors.js';
+import type { EmbedderInterface, FilteredTextFragment } from './pdf-filter.js';
+/**
+ * File extensions supported by the parser module (parseFile + parsePdf).
+ * Exported so other modules (e.g. list_files) stay in sync automatically
+ * when new formats are added here.
+ */
+export declare const SUPPORTED_EXTENSIONS: Set<string>;
+/**
+ * Result from parsing a document, containing both content and extracted title.
+ * Title is display-only metadata (NOT used for search scoring).
+ */
+export interface ParseResult {
+    content: string;
+    title: string;
+    atomicRanges?: readonly AtomicTextRange[];
+    imageAnchors?: readonly ParsedImageAnchor[];
+}
+export interface ParsedImageAnchor {
+    offset: number;
+    imageIndex: number;
+    mimeType: 'image/png' | 'image/jpeg';
+    bytes: Uint8Array;
+}
+export interface ParseFileOptions {
+    images?: boolean;
+}
+/**
+ * DocumentParser configuration.
+ *
+ * Accepts either a single `baseDir` (legacy single-root shape — preserved for
+ * backward compatibility with downstream callers that have not yet migrated
+ * to the multi-root model) or a `baseDirs` array (multi-root shape produced
+ * by `resolveBaseDirs`). Exactly one of the two MUST be supplied; supplying
+ * both is rejected by the constructor so misconfiguration cannot silently
+ * pick one source over the other.
+ *
+ * Behavior under a single allowed root (`{ baseDir }` or
+ * `{ baseDirs: [oneRoot] }`) is byte-identical to the previous single-root
+ * implementation — see `validateFilePath` for the iteration contract under
+ * multiple roots.
+ */
+export type ParserConfig = {
+    /** Security: single allowed base directory (legacy shape). */
+    baseDir: string;
+    baseDirs?: undefined;
+    /** Maximum file size (100MB). */
+    maxFileSize: number;
+} | {
+    /** Security: one or more allowed base directories (multi-root shape). */
+    baseDirs: readonly string[];
+    baseDir?: undefined;
+    /** Maximum file size (100MB). */
+    maxFileSize: number;
+};
+/**
+ * Validation error (equivalent to 400)
+ */
+export declare class ValidationError extends AppError {
+    constructor(message: string, cause?: Error);
+}
+/**
+ * File operation error (equivalent to 500)
+ */
+export declare class FileOperationError extends AppError {
+    constructor(message: string, cause?: Error);
+}
+/**
+ * Document parser class (PDF/DOCX/TXT/MD support)
+ *
+ * Responsibilities:
+ * - File path validation (path traversal prevention)
+ * - File size validation (100MB limit)
+ * - Parse 4 formats (PDF/DOCX/TXT/MD)
+ */
+export declare class DocumentParser {
+    private readonly config;
+    /** Raw allowed roots in input order (pre-realpath). Always non-empty. */
+    private readonly rawBaseDirs;
+    /**
+     * Lazily cached realpath-normalized allowed roots, each with a trailing
+     * path separator so the `startsWith` check is sibling-prefix safe (e.g.
+     * `/foo/bar/` must not match `/foo/barista/x.txt`). Order is preserved
+     * from `rawBaseDirs` so the legacy single-root rejection message keeps
+     * referencing the user-configured first root. Assumes the allowed roots
+     * are stable for the process lifetime.
+     */
+    private resolvedBaseDirs;
+    constructor(config: ParserConfig);
+    /**
+     * File path validation (Absolute path requirement + Path traversal prevention).
+     *
+     * This is THE place realpath is used (with the base-dir resolver): the
+     * security/containment boundary. Following symlinks here makes prefix
+     * containment unforgeable. Stored/scanned/looked-up paths elsewhere use
+     * resolve() — see {@link BaseDirsConfig} for the path policy.
+     *
+     * Multi-root semantics: a file is accepted iff its realpath (or, for a
+     * non-symlink path that does not yet exist, its `resolve()`-normalized
+     * absolute path) is under ANY realpath-normalized allowed root using a
+     * trailing-separator prefix check. Broken symlinks are still rejected
+     * outright — the lstat-based detection mirrors the previous single-root
+     * behavior.
+     *
+     * Under a single allowed root the behavior is identical to the previous
+     * single-root implementation.
+     *
+     * @param filePath - File path to validate (must be absolute)
+     * @throws ValidationError - When path is not absolute or outside all allowed roots
+     */
+    validateFilePath(filePath: string): Promise<void>;
+    /**
+     * File size validation (100MB limit)
+     *
+     * @param filePath - File path to validate
+     * @throws ValidationError - When file size exceeds limit
+     * @throws FileOperationError - When file read fails
+     */
+    validateFileSize(filePath: string): void;
+    /**
+     * File parsing (auto format detection)
+     *
+     * @param filePath - File path to parse
+     * @returns ParseResult with content and extracted title
+     * @throws ValidationError - Path traversal, size exceeded, unsupported format
+     * @throws FileOperationError - File read failed, parse failed
+     */
+    parseFile(filePath: string, options?: ParseFileOptions): Promise<ParseResult>;
+    /**
+     * PDF parsing with header/footer filtering
+     *
+     * Features:
+     * - Extracts text with position information (x, y, fontSize)
+     * - Semantic header/footer detection using embedding similarity
+     * - Uses hasEOL for proper line break handling
+     * - Extracts document title from PDF metadata and first page font heuristic
+     *
+     * @param filePath - PDF file path
+     * @param embedder - Embedder for semantic header/footer detection
+     * @returns ParseResult with content and extracted title
+     * @throws FileOperationError - File read failed, parse failed
+     */
+    parsePdf(filePath: string, embedder: EmbedderInterface): Promise<ParseResult>;
+    /**
+     * Per-page PDF parsing for the visual-enrichment path.
+     *
+     * Opens a mupdf `Document`, delegates per-page extraction to the shared
+     * `extractPdfPages` helper with the `'preserve-whitespace,preserve-images'`
+     * stext option string so mupdf emits `block.type === 'image'` blocks for
+     * the downstream visual-candidate detector.
+     *
+     * Returns the open `Document` handle alongside the per-page records and
+     * title-resolution materials so the caller can:
+     *   - run the renderer (`page.toPixmap()`) on the same handle,
+     *   - feed `metadataTitle` + `pages[0].page1FontHint` into `extractPdfTitle`
+     *     after `buildChunksAndEmbeddings` returns.
+     *
+     * Disposal contract (asymmetric — read carefully):
+     *   - SUCCESS path: this method returns the open `doc` handle. The caller
+     *     owns disposal and MUST wrap the call site in
+     *     `try { ... } finally { doc.destroy() }`.
+     *   - ERROR path: when this method throws, `doc` has already been destroyed
+     *     internally before the exception propagates (so the caller never
+     *     receives a handle it would not know to clean up). Callers MUST NOT
+     *     call `doc.destroy()` on an error from this method.
+     * Title resolution stays here so visual captions cannot change PDF metadata.
+     * Visual candidate selection remains `pdf-visual/detector`'s responsibility.
+     *
+     * @param filePath - PDF file path (validated against BASE_DIR and size limit)
+     * @param embedder - Embedder for semantic header/footer detection
+     * @returns Open mupdf `Document`, resolved title, and per-page text/layout records.
+     * @throws ValidationError - Path traversal, size exceeded
+     * @throws FileOperationError - File read or parse failed (after destroying `doc` internally)
+     */
+    parsePdfPages(filePath: string, embedder: EmbedderInterface): Promise<{
+        doc: MupdfDocument;
+        title: string;
+        pages: Array<{
+            pageNum: number;
+            text: string;
+            textFragments: FilteredTextFragment[];
+            stextJson: unknown;
+        }>;
+    }>;
+    /**
+     * DOCX parsing (using mammoth)
+     *
+     * Uses DOCX package metadata plus one Mammoth HTML conversion for title and body extraction.
+     *
+     * @param filePath - DOCX file path
+     * @returns ParseResult with content and extracted title
+     * @throws FileOperationError - File read failed, parse failed
+     */
+    private parseDocx;
+    /**
+     * TXT parsing (using fs.readFile)
+     *
+     * @param filePath - TXT file path
+     * @returns ParseResult with content and extracted title
+     * @throws FileOperationError - File read failed
+     */
+    private parseTxt;
+    /**
+     * MD parsing (using fs.readFile)
+     *
+     * @param filePath - MD file path
+     * @returns ParseResult with content and extracted title
+     * @throws FileOperationError - File read failed
+     */
+    private parseMd;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/parser/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/parser/index.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,QAAQ,IAAI,aAAa,EAAE,MAAM,OAAO,CAAA;AACtD,OAAO,EAAE,KAAK,eAAe,EAAmB,MAAM,qBAAqB,CAAA;AAE3E,OAAO,EAAE,QAAQ,EAAc,MAAM,oBAAoB,CAAA;AAGzD,OAAO,KAAK,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AAY9E;;;;GAIG;AACH,eAAO,MAAM,oBAAoB,aAA4C,CAAA;AAM7E;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,CAAC,EAAE,SAAS,eAAe,EAAE,CAAA;IACzC,YAAY,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAA;CAC5C;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,WAAW,GAAG,YAAY,CAAA;IACpC,KAAK,EAAE,UAAU,CAAA;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB;AAwBD;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,YAAY,GACpB;IACE,8DAA8D;IAC9D,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,SAAS,CAAA;IACpB,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAA;CACpB,GACD;IACE,yEAAyE;IACzE,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAA;IAC3B,OAAO,CAAC,EAAE,SAAS,CAAA;IACnB,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAEL;;GAEG;AACH,qBAAa,eAAgB,SAAQ,QAAQ;IAC3C,YAAY,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,EAGzC;CACF;AAED;;GAEG;AACH,qBAAa,kBAAmB,SAAQ,QAAQ;IAC9C,YAAY,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,EAGzC;CACF;AAMD;;;;;;;GAOG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmB;IAC/C;;;;;;;OAOG;IACH,OAAO,CAAC,gBAAgB,CAAwB;IAEhD,YAAY,MAAM,EAAE,YAAY,EAkB/B;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAsEtD;IAED;;;;;;OAMG;IACH,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAkBvC;IAED;;;;;;;OAOG;IACG,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,WAAW,CAAC,CAiBtF;IAED;;;;;;;;;;;;;OAaG;IACG,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CA8ClF;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACG,aAAa,CACjB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC;QACT,GAAG,EAAE,aAAa,CAAA;QAClB,KAAK,EAAE,MAAM,CAAA;QACb,KAAK,EAAE,KAAK,CAAC;YACX,OAAO,EAAE,MAAM,CAAA;YACf,IAAI,EAAE,MAAM,CAAA;YACZ,aAAa,EAAE,oBAAoB,EAAE,CAAA;YACrC,SAAS,EAAE,OAAO,CAAA;SACnB,CAAC,CAAA;KACH,CAAC,CAoDD;IAED;;;;;;;;OAQG;YACW,SAAS;IA4DvB;;;;;;OAMG;YACW,QAAQ;IAYtB;;;;;;OAMG;YACW,OAAO;CAWtB"}