@alextis59/athena 1.0.0

package/dist/parsers/line-parser.js

@@ -0,0 +1,242 @@
+import { normalizeText, stripMarkdownHeadingDecorators } from "./normalize.js";
+export function parseMarkdownLikeLines(sourceText, options) {
+    const lines = splitLines(sourceText);
+    const blocks = [];
+    const warnings = [];
+    const headingStack = [];
+    let paragraphLines = [];
+    let fenceStartLine = null;
+    let codeLines = [];
+    for (const [index, line] of lines.entries()) {
+        const lineNumber = index + 1;
+        const fenceMatch = line.match(/^```|^~~~~/u);
+        if (fenceStartLine !== null) {
+            codeLines.push({ line, lineNumber });
+            if (fenceMatch !== null) {
+                blocks.push(createBlock("code", codeLines, headingStack));
+                codeLines = [];
+                fenceStartLine = null;
+            }
+            continue;
+        }
+        if (fenceMatch !== null) {
+            flushParagraph();
+            fenceStartLine = lineNumber;
+            codeLines = [{ line, lineNumber }];
+            continue;
+        }
+        const headingMatch = line.match(/^(#{1,6})\s+(.+)$/u);
+        if (headingMatch !== null) {
+            flushParagraph();
+            const depth = headingMatch[1]?.length ?? 1;
+            const heading = stripMarkdownHeadingDecorators(headingMatch[2] ?? "");
+            headingStack.splice(depth - 1);
+            headingStack[depth - 1] = heading;
+            blocks.push(createBlock("heading", [{ line: heading, lineNumber }], headingStack.filter(Boolean)));
+            continue;
+        }
+        if (line.trim() === "") {
+            flushParagraph();
+            continue;
+        }
+        paragraphLines.push({ line, lineNumber });
+    }
+    if (fenceStartLine !== null) {
+        warnings.push({
+            code: options.mdx ? "mdx.unclosed_fence" : "markdown.unclosed_fence",
+            message: "Unclosed fenced code block.",
+            startLine: fenceStartLine,
+            endLine: lines.length
+        });
+        blocks.push(createBlock("code", codeLines, headingStack));
+    }
+    flushParagraph();
+    return { blocks, warnings };
+    function flushParagraph() {
+        if (paragraphLines.length === 0) {
+            return;
+        }
+        blocks.push(createBlock(classifyParagraph(paragraphLines), paragraphLines, headingStack));
+        paragraphLines = [];
+    }
+}
+export function parsePlainTextLines(sourceText) {
+    const lines = splitLines(sourceText);
+    const blocks = [];
+    let paragraphLines = [];
+    for (const [index, line] of lines.entries()) {
+        const lineNumber = index + 1;
+        if (line.trim() === "") {
+            flushParagraph();
+            continue;
+        }
+        paragraphLines.push({ line, lineNumber });
+    }
+    flushParagraph();
+    return { blocks, warnings: [] };
+    function flushParagraph() {
+        if (paragraphLines.length === 0) {
+            return;
+        }
+        blocks.push(createBlock("paragraph", paragraphLines, []));
+        paragraphLines = [];
+    }
+}
+export function parseRstLines(sourceText) {
+    const lines = splitLines(sourceText);
+    const blocks = [];
+    const headingStack = [];
+    let paragraphLines = [];
+    for (let index = 0; index < lines.length; index += 1) {
+        const line = lines[index] ?? "";
+        const nextLine = lines[index + 1] ?? "";
+        const lineNumber = index + 1;
+        if (line.trim() !== "" &&
+            /^([=\-~^"`:#*+])\1{2,}\s*$/u.test(nextLine.trim())) {
+            flushParagraph();
+            const depth = headingDepth(nextLine.trim()[0] ?? "=");
+            headingStack.splice(depth - 1);
+            headingStack[depth - 1] = line.trim();
+            blocks.push(createBlock("heading", [{ line: line.trim(), lineNumber }], headingStack.filter(Boolean)));
+            index += 1;
+            continue;
+        }
+        if (line.trim() === "") {
+            flushParagraph();
+            continue;
+        }
+        paragraphLines.push({ line, lineNumber });
+    }
+    flushParagraph();
+    return { blocks, warnings: [] };
+    function flushParagraph() {
+        if (paragraphLines.length === 0) {
+            return;
+        }
+        blocks.push(createBlock(classifyParagraph(paragraphLines), paragraphLines, headingStack));
+        paragraphLines = [];
+    }
+}
+export function parseAsciiDocLines(sourceText) {
+    const lines = splitLines(sourceText);
+    const blocks = [];
+    const headingStack = [];
+    let paragraphLines = [];
+    for (const [index, line] of lines.entries()) {
+        const lineNumber = index + 1;
+        const headingMatch = line.match(/^(={1,6})\s+(.+)$/u);
+        if (headingMatch !== null) {
+            flushParagraph();
+            const depth = headingMatch[1]?.length ?? 1;
+            const heading = headingMatch[2]?.trim() ?? "";
+            headingStack.splice(depth - 1);
+            headingStack[depth - 1] = heading;
+            blocks.push(createBlock("heading", [{ line: heading, lineNumber }], headingStack.filter(Boolean)));
+            continue;
+        }
+        if (line.trim() === "") {
+            flushParagraph();
+            continue;
+        }
+        paragraphLines.push({ line, lineNumber });
+    }
+    flushParagraph();
+    return { blocks, warnings: [] };
+    function flushParagraph() {
+        if (paragraphLines.length === 0) {
+            return;
+        }
+        blocks.push(createBlock(classifyParagraph(paragraphLines), paragraphLines, headingStack));
+        paragraphLines = [];
+    }
+}
+export function parseHtmlLines(sourceText) {
+    const lines = splitLines(sourceText);
+    const blocks = [];
+    const headingStack = [];
+    let ignoringRaw = false;
+    for (const [index, line] of lines.entries()) {
+        const lineNumber = index + 1;
+        const lower = line.toLowerCase();
+        if (/<(script|style)\b/u.test(lower)) {
+            ignoringRaw = true;
+        }
+        if (ignoringRaw) {
+            if (/<\/(script|style)>/u.test(lower)) {
+                ignoringRaw = false;
+            }
+            continue;
+        }
+        const headingMatch = line.match(/<h([1-6])\b[^>]*>(.*?)<\/h\1>/iu);
+        if (headingMatch !== null) {
+            const depth = Number.parseInt(headingMatch[1] ?? "1", 10);
+            const heading = decodeBasicEntities(stripTags(headingMatch[2] ?? "")).trim();
+            headingStack.splice(depth - 1);
+            headingStack[depth - 1] = heading;
+            blocks.push(createBlock("heading", [{ line: heading, lineNumber }], headingStack.filter(Boolean)));
+            continue;
+        }
+        const text = decodeBasicEntities(stripTags(line)).trim();
+        if (text !== "") {
+            blocks.push(createBlock(classifyHtmlLine(line), [{ line: text, lineNumber }], headingStack.filter(Boolean)));
+        }
+    }
+    return { blocks, warnings: [] };
+}
+function createBlock(kind, lines, headingPath) {
+    const text = lines
+        .map((line) => line.line)
+        .join("\n")
+        .trim();
+    return {
+        kind,
+        text,
+        normalizedText: normalizeText(text),
+        startLine: lines[0]?.lineNumber ?? 1,
+        endLine: lines.at(-1)?.lineNumber ?? lines[0]?.lineNumber ?? 1,
+        headingPath: [...headingPath]
+    };
+}
+function classifyParagraph(lines) {
+    const firstLine = lines[0]?.line.trim() ?? "";
+    if (/^(\*|-|\+|\d+\.)\s+/u.test(firstLine)) {
+        return "list";
+    }
+    if (firstLine.includes("|")) {
+        return "table";
+    }
+    return "paragraph";
+}
+function classifyHtmlLine(line) {
+    if (/<(li)\b/iu.test(line)) {
+        return "list";
+    }
+    if (/<(td|th|tr|table)\b/iu.test(line)) {
+        return "table";
+    }
+    if (/<(pre|code)\b/iu.test(line)) {
+        return "code";
+    }
+    return "paragraph";
+}
+function headingDepth(marker) {
+    const order = ["=", "-", "~", "^", '"', "`", ":", "#", "*", "+"];
+    const index = order.indexOf(marker);
+    return index === -1 ? 1 : index + 1;
+}
+function splitLines(text) {
+    return text.replace(/\r\n?/gu, "\n").split("\n");
+}
+function stripTags(text) {
+    return text.replace(/<[^>]*>/gu, " ");
+}
+function decodeBasicEntities(text) {
+    return text
+        .replace(/&nbsp;/gu, " ")
+        .replace(/&amp;/gu, "&")
+        .replace(/&lt;/gu, "<")
+        .replace(/&gt;/gu, ">")
+        .replace(/&quot;/gu, '"')
+        .replace(/&#39;/gu, "'");
+}
+//# sourceMappingURL=line-parser.js.map

package/dist/parsers/line-parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"line-parser.js","sourceRoot":"","sources":["../../src/parsers/line-parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,8BAA8B,EAAE,MAAM,gBAAgB,CAAC;AAQ/E,MAAM,UAAU,sBAAsB,CACpC,UAAkB,EAClB,OAAyB;IAEzB,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,cAAc,GAA2C,EAAE,CAAC;IAChE,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,IAAI,SAAS,GAA2C,EAAE,CAAC;IAE3D,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,KAAK,GAAG,CAAC,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAE7C,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;YAC5B,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAErC,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;gBAC1D,SAAS,GAAG,EAAE,CAAC;gBACf,cAAc,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,SAAS;QACX,CAAC;QAED,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACxB,cAAc,EAAE,CAAC;YACjB,cAAc,GAAG,UAAU,CAAC;YAC5B,SAAS,GAAG,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YACnC,SAAS;QACX,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAEtD,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,cAAc,EAAE,CAAC;YACjB,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,8BAA8B,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACtE,YAAY,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC/B,YAAY,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC;YAClC,MAAM,CAAC,IAAI,CACT,WAAW,CACT,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAC/B,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAC7B,CACF,CAAC;YACF,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,cAAc,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,yBAAyB;YACpE,OAAO,EAAE,6BAA6B;YACtC,SAAS,EAAE,cAAc;YACzB,OAAO,EAAE,KAAK,CAAC,MAAM;SACtB,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,cAAc,EAAE,CAAC;IAEjB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IAE5B,SAAS,cAAc;QACrB,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO;QACT,CAAC;QAED,MAAM,CAAC,IAAI,CACT,WAAW,CACT,iBAAiB,CAAC,cAAc,CAAC,EACjC,cAAc,EACd,YAAY,CACb,CACF,CAAC;QACF,cAAc,GAAG,EAAE,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,UAAkB;IACpD,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,cAAc,GAA2C,EAAE,CAAC;IAEhE,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,KAAK,GAAG,CAAC,CAAC;QAE7B,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,cAAc,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,cAAc,EAAE,CAAC;IAEjB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAEhC,SAAS,cAAc;QACrB,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO;QACT,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,cAAc,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1D,cAAc,GAAG,EAAE,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,UAAkB;IAC9C,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,cAAc,GAA2C,EAAE,CAAC;IAEhE,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,UAAU,GAAG,KAAK,GAAG,CAAC,CAAC;QAE7B,IACE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAClB,6BAA6B,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,EACnD,CAAC;YACD,cAAc,EAAE,CAAC;YACjB,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;YACtD,YAAY,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC/B,YAAY,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CACT,WAAW,CACT,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,EACnC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAC7B,CACF,CAAC;YACF,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,cAAc,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,cAAc,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAEhC,SAAS,cAAc;QACrB,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO;QACT,CAAC;QAED,MAAM,CAAC,IAAI,CACT,WAAW,CACT,iBAAiB,CAAC,cAAc,CAAC,EACjC,cAAc,EACd,YAAY,CACb,CACF,CAAC;QACF,cAAc,GAAG,EAAE,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,UAAkB;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,cAAc,GAA2C,EAAE,CAAC;IAEhE,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,KAAK,GAAG,CAAC,CAAC;QAC7B,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAEtD,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,cAAc,EAAE,CAAC;YACjB,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAC9C,YAAY,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC/B,YAAY,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC;YAClC,MAAM,CAAC,IAAI,CACT,WAAW,CACT,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAC/B,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAC7B,CACF,CAAC;YACF,SAAS;QACX,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YACvB,cAAc,EAAE,CAAC;YACjB,SAAS;QACX,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,cAAc,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAEhC,SAAS,cAAc;QACrB,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,OAAO;QACT,CAAC;QAED,MAAM,CAAC,IAAI,CACT,WAAW,CACT,iBAAiB,CAAC,cAAc,CAAC,EACjC,cAAc,EACd,YAAY,CACb,CACF,CAAC;QACF,cAAc,GAAG,EAAE,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,UAAkB;IAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,KAAK,GAAG,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAEjC,IAAI,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;QAED,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtC,WAAW,GAAG,KAAK,CAAC;YACtB,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAEnE,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAG,mBAAmB,CACjC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CACjC,CAAC,IAAI,EAAE,CAAC;YACT,YAAY,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC/B,YAAY,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC;YAClC,MAAM,CAAC,IAAI,CACT,WAAW,CACT,SAAS,EACT,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAC/B,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAC7B,CACF,CAAC;YACF,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEzD,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CACT,WAAW,CACT,gBAAgB,CAAC,IAAI,CAAC,EACtB,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAC5B,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAC7B,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,WAAW,CAClB,IAAyB,EACzB,KAA6C,EAC7C,WAAqB;IAErB,MAAM,IAAI,GAAG,KAAK;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;SACxB,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,EAAE,CAAC;IAEV,OAAO;QACL,IAAI;QACJ,IAAI;QACJ,cAAc,EAAE,aAAa,CAAC,IAAI,CAAC;QACnC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC;QACpC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,CAAC;QAC9D,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,KAA6C;IAE7C,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAE9C,IAAI,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAEpC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;AAC7B,CAAC"}

package/dist/parsers/normalize.d.ts

@@ -0,0 +1,4 @@
+export declare function normalizeText(text: string): string;
+export declare function estimateTokens(text: string): number;
+export declare function stripMarkdownHeadingDecorators(text: string): string;
+//# sourceMappingURL=normalize.d.ts.map

package/dist/parsers/normalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.d.ts","sourceRoot":"","sources":["../../src/parsers/normalize.ts"],"names":[],"mappings":"AAAA,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQnD;AAED,wBAAgB,8BAA8B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKnE"}

package/dist/parsers/normalize.js

@@ -0,0 +1,17 @@
+export function normalizeText(text) {
+    return text.replace(/\s+/g, " ").trim().toLowerCase();
+}
+export function estimateTokens(text) {
+    const trimmed = text.trim();
+    if (trimmed === "") {
+        return 0;
+    }
+    return trimmed.split(/\s+/).length;
+}
+export function stripMarkdownHeadingDecorators(text) {
+    return text
+        .replace(/\s*\{\/\*.*?\*\/\}\s*$/u, "")
+        .replace(/\s+#+\s*$/u, "")
+        .trim();
+}
+//# sourceMappingURL=normalize.js.map

package/dist/parsers/normalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../src/parsers/normalize.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAE5B,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QACnB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,OAAO,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,IAAY;IACzD,OAAO,IAAI;SACR,OAAO,CAAC,yBAAyB,EAAE,EAAE,CAAC;SACtC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;SACzB,IAAI,EAAE,CAAC;AACZ,CAAC"}

package/dist/parsers/parser.d.ts

@@ -0,0 +1,14 @@
+import { type Pdf2LlmConverter } from "./pdf2llm.js";
+import type { PdfConfig } from "../config/index.js";
+import type { ParsedDocument } from "./types.js";
+import type { SourceKind } from "../shared/types.js";
+export declare const TEXT_PARSER_VERSION = "1";
+export interface ParseSourceFileOptions {
+    pdf: PdfConfig;
+    pdfConverter?: Pdf2LlmConverter;
+}
+export declare function parseSourceFile(absolutePath: string, sourceKind: SourceKind, options: ParseSourceFileOptions): Promise<ParsedDocument | null>;
+export declare function parseSourceText(sourceText: string, sourceKind: SourceKind): Pick<ParsedDocument, "blocks" | "warnings">;
+export declare function parserNameForKind(sourceKind: SourceKind): string;
+export declare function parserVersionForKind(sourceKind: SourceKind): string;
+//# sourceMappingURL=parser.d.ts.map

package/dist/parsers/parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../src/parsers/parser.ts"],"names":[],"mappings":"AAUA,OAAO,EAIL,KAAK,gBAAgB,EACtB,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,EAAe,cAAc,EAAE,MAAM,YAAY,CAAC;AAC9D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAEvC,MAAM,WAAW,sBAAsB;IACrC,GAAG,EAAE,SAAS,CAAC;IACf,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,wBAAsB,eAAe,CACnC,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CA4BhC;AAED,wBAAgB,eAAe,CAC7B,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,UAAU,GACrB,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,UAAU,CAAC,CAkB7C;AAED,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAmBhE;AAED,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAEnE"}

package/dist/parsers/parser.js

@@ -0,0 +1,77 @@
+import { readFile } from "node:fs/promises";
+import { parseAsciiDocLines, parseHtmlLines, parseMarkdownLikeLines, parsePlainTextLines, parseRstLines } from "./line-parser.js";
+import { normalizeText } from "./normalize.js";
+import { PDF2LLM_PARSER_NAME, PDF2LLM_PARSER_VERSION, parsePdfFile } from "./pdf2llm.js";
+export const TEXT_PARSER_VERSION = "1";
+export async function parseSourceFile(absolutePath, sourceKind, options) {
+    if (sourceKind === "pdf") {
+        return await parsePdfFile(absolutePath, {
+            config: options.pdf,
+            ...(options.pdfConverter === undefined
+                ? {}
+                : { converter: options.pdfConverter })
+        });
+    }
+    if (sourceKind === "unknown") {
+        return null;
+    }
+    const sourceText = await readFile(absolutePath, "utf8");
+    const parsed = parseSourceText(sourceText, sourceKind);
+    const text = parsed.blocks.map((block) => block.text).join("\n\n");
+    return {
+        parserName: parserNameForKind(sourceKind),
+        parserVersion: TEXT_PARSER_VERSION,
+        sourceKind,
+        title: firstHeading(parsed.blocks),
+        text,
+        normalizedText: normalizeText(text),
+        blocks: parsed.blocks,
+        warnings: parsed.warnings
+    };
+}
+export function parseSourceText(sourceText, sourceKind) {
+    switch (sourceKind) {
+        case "markdown":
+            return parseMarkdownLikeLines(sourceText, { mdx: false });
+        case "mdx":
+            return parseMarkdownLikeLines(sourceText, { mdx: true });
+        case "text":
+            return parsePlainTextLines(sourceText);
+        case "rst":
+            return parseRstLines(sourceText);
+        case "asciidoc":
+            return parseAsciiDocLines(sourceText);
+        case "html":
+            return parseHtmlLines(sourceText);
+        case "pdf":
+        case "unknown":
+            return { blocks: [], warnings: [] };
+    }
+}
+export function parserNameForKind(sourceKind) {
+    switch (sourceKind) {
+        case "markdown":
+            return "markdown-line-parser";
+        case "mdx":
+            return "mdx-line-parser";
+        case "text":
+            return "text-line-parser";
+        case "rst":
+            return "rst-line-parser";
+        case "asciidoc":
+            return "asciidoc-line-parser";
+        case "html":
+            return "html-line-parser";
+        case "pdf":
+            return PDF2LLM_PARSER_NAME;
+        case "unknown":
+            return "unknown";
+    }
+}
+export function parserVersionForKind(sourceKind) {
+    return sourceKind === "pdf" ? PDF2LLM_PARSER_VERSION : TEXT_PARSER_VERSION;
+}
+function firstHeading(blocks) {
+    return blocks.find((block) => block.kind === "heading")?.text ?? null;
+}
+//# sourceMappingURL=parser.js.map

package/dist/parsers/parser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../src/parsers/parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EACL,kBAAkB,EAClB,cAAc,EACd,sBAAsB,EACtB,mBAAmB,EACnB,aAAa,EACd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EACL,mBAAmB,EACnB,sBAAsB,EACtB,YAAY,EAEb,MAAM,cAAc,CAAC;AAKtB,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAOvC,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,UAAsB,EACtB,OAA+B;IAE/B,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;QACzB,OAAO,MAAM,YAAY,CAAC,YAAY,EAAE;YACtC,MAAM,EAAE,OAAO,CAAC,GAAG;YACnB,GAAG,CAAC,OAAO,CAAC,YAAY,KAAK,SAAS;gBACpC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC;SACzC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACvD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEnE,OAAO;QACL,UAAU,EAAE,iBAAiB,CAAC,UAAU,CAAC;QACzC,aAAa,EAAE,mBAAmB;QAClC,UAAU;QACV,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC;QAClC,IAAI;QACJ,cAAc,EAAE,aAAa,CAAC,IAAI,CAAC;QACnC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,UAAkB,EAClB,UAAsB;IAEtB,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,UAAU;YACb,OAAO,sBAAsB,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5D,KAAK,KAAK;YACR,OAAO,sBAAsB,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,KAAK,MAAM;YACT,OAAO,mBAAmB,CAAC,UAAU,CAAC,CAAC;QACzC,KAAK,KAAK;YACR,OAAO,aAAa,CAAC,UAAU,CAAC,CAAC;QACnC,KAAK,UAAU;YACb,OAAO,kBAAkB,CAAC,UAAU,CAAC,CAAC;QACxC,KAAK,MAAM;YACT,OAAO,cAAc,CAAC,UAAU,CAAC,CAAC;QACpC,KAAK,KAAK,CAAC;QACX,KAAK,SAAS;YACZ,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACxC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,UAAsB;IACtD,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,UAAU;YACb,OAAO,sBAAsB,CAAC;QAChC,KAAK,KAAK;YACR,OAAO,iBAAiB,CAAC;QAC3B,KAAK,MAAM;YACT,OAAO,kBAAkB,CAAC;QAC5B,KAAK,KAAK;YACR,OAAO,iBAAiB,CAAC;QAC3B,KAAK,UAAU;YACb,OAAO,sBAAsB,CAAC;QAChC,KAAK,MAAM;YACT,OAAO,kBAAkB,CAAC;QAC5B,KAAK,KAAK;YACR,OAAO,mBAAmB,CAAC;QAC7B,KAAK,SAAS;YACZ,OAAO,SAAS,CAAC;IACrB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAAsB;IACzD,OAAO,UAAU,KAAK,KAAK,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,mBAAmB,CAAC;AAC7E,CAAC;AAED,SAAS,YAAY,CAAC,MAAqB;IACzC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC;AACxE,CAAC"}

package/dist/parsers/pdf2llm.d.ts

@@ -0,0 +1,14 @@
+import { convertPdfToMarkdown } from "pdf-2-llm/node";
+import type { PdfConfig } from "../config/index.js";
+import type { ParsedDocument } from "./types.js";
+export declare const PDF2LLM_PARSER_NAME = "pdf-2-llm";
+export declare const PDF2LLM_PACKAGE_VERSION = "1.1.0";
+export declare const PDF2LLM_PARSER_VERSION = "1.1.0";
+export type Pdf2LlmConverter = typeof convertPdfToMarkdown;
+export interface ParsePdfFileOptions {
+    config: PdfConfig;
+    converter?: Pdf2LlmConverter;
+}
+export declare function parsePdfFile(absolutePath: string, options: ParsePdfFileOptions): Promise<ParsedDocument>;
+export declare function createPdfParserOptionsHash(config: PdfConfig): string;
+//# sourceMappingURL=pdf2llm.d.ts.map

package/dist/parsers/pdf2llm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf2llm.d.ts","sourceRoot":"","sources":["../../src/parsers/pdf2llm.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,oBAAoB,EAOrB,MAAM,gBAAgB,CAAC;AAKxB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,EAEV,cAAc,EAGf,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAC/C,eAAO,MAAM,uBAAuB,UAAU,CAAC;AAC/C,eAAO,MAAM,sBAAsB,UAA0B,CAAC;AAE9D,MAAM,MAAM,gBAAgB,GAAG,OAAO,oBAAoB,CAAC;AAE3D,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,SAAS,CAAC;IAClB,SAAS,CAAC,EAAE,gBAAgB,CAAC;CAC9B;AAOD,wBAAsB,YAAY,CAChC,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAmCzB;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,SAAS,GAAG,MAAM,CASpE"}

package/dist/parsers/pdf2llm.js

@@ -0,0 +1,215 @@
+import { createHash } from "node:crypto";
+import { convertPdfToMarkdown, schemaVersion } from "pdf-2-llm/node";
+import { UserFacingError } from "../errors.js";
+import { parseMarkdownLikeLines } from "./line-parser.js";
+import { normalizeText } from "./normalize.js";
+export const PDF2LLM_PARSER_NAME = "pdf-2-llm";
+export const PDF2LLM_PACKAGE_VERSION = "1.1.0";
+export const PDF2LLM_PARSER_VERSION = PDF2LLM_PACKAGE_VERSION;
+export async function parsePdfFile(absolutePath, options) {
+    const result = await convertWithTimeout(absolutePath, options);
+    const parsed = parseMarkdownLikeLines(result.markdown, { mdx: false });
+    const warnings = [
+        ...parsed.warnings,
+        ...toParserWarnings(result, options.config)
+    ];
+    const blocks = annotatePdfBlocks(parsed.blocks, result, createLineStartOffsets(result.markdown), warnings);
+    const text = blocks.map((block) => block.text).join("\n\n");
+    return {
+        parserName: PDF2LLM_PARSER_NAME,
+        parserVersion: PDF2LLM_PARSER_VERSION,
+        sourceKind: "pdf",
+        title: firstHeading(blocks),
+        text,
+        normalizedText: normalizeText(text),
+        blocks,
+        warnings,
+        metadata: {
+            pdf2llm: {
+                packageVersion: PDF2LLM_PACKAGE_VERSION,
+                schemaVersion,
+                confidence: result.confidence,
+                pageCount: result.diagnostics.pages.length,
+                sourceType: result.ir.sourceType
+            }
+        },
+        artifacts: createPdfArtifacts(result, options.config)
+    };
+}
+export function createPdfParserOptionsHash(config) {
+    return hashString(JSON.stringify({
+        parser: PDF2LLM_PARSER_NAME,
+        packageVersion: PDF2LLM_PACKAGE_VERSION,
+        schemaVersion,
+        options: createPdfConvertOptionsSummary(config)
+    }));
+}
+function convertWithTimeout(absolutePath, options) {
+    const converter = options.converter ?? convertPdfToMarkdown;
+    const controller = new AbortController();
+    const timeout = setTimeout(() => {
+        controller.abort();
+    }, options.config.timeoutMs);
+    return converter(absolutePath, {
+        ...createPdfConvertOptions(options.config),
+        signal: controller.signal
+    })
+        .catch((error) => {
+        if (controller.signal.aborted) {
+            throw new UserFacingError(`PDF parsing timed out after ${String(options.config.timeoutMs)} ms: ${absolutePath}`, error);
+        }
+        throw new UserFacingError(`PDF parsing failed: ${absolutePath}`, error);
+    })
+        .finally(() => {
+        clearTimeout(timeout);
+    });
+}
+function createPdfConvertOptions(config) {
+    return {
+        markdown: {
+            pageAnchors: config.pageAnchors
+        },
+        parser: {
+            mode: "tolerant"
+        },
+        security: {
+            timeoutMs: config.timeoutMs
+        },
+        tables: {
+            csvSidecars: config.csvSidecars
+        }
+    };
+}
+function createPdfConvertOptionsSummary(config) {
+    return {
+        markdown: {
+            pageAnchors: config.pageAnchors
+        },
+        parser: {
+            mode: "tolerant"
+        },
+        security: {
+            timeoutMs: config.timeoutMs
+        },
+        tables: {
+            csvSidecars: config.csvSidecars
+        },
+        citation: {
+            minConfidenceForAutoCitation: config.minConfidenceForAutoCitation
+        }
+    };
+}
+function annotatePdfBlocks(blocks, result, lineStartOffsets, warnings) {
+    const warningCodes = warnings.map((warning) => warning.code);
+    return blocks.map((block) => {
+        const match = findSourceMapMatch(result.sourceMap, lineStartOffsets[block.startLine - 1] ?? 0);
+        if (match === null) {
+            return {
+                ...block,
+                metadata: {
+                    ...(block.metadata ?? {}),
+                    pdfConfidence: result.confidence.overall,
+                    pdfWarningCodes: warningCodes
+                }
+            };
+        }
+        return {
+            ...block,
+            metadata: {
+                ...(block.metadata ?? {}),
+                pdfPage: match.region.pageIndex + 1,
+                pdfConfidence: result.confidence.overall,
+                pdfWarningCodes: warningCodes,
+                pdfSourceMapKind: match.entry.kind,
+                pdfRegion: {
+                    pageIndex: match.region.pageIndex,
+                    x: match.region.x,
+                    y: match.region.y,
+                    width: match.region.width,
+                    height: match.region.height,
+                    source: match.region.source
+                }
+            }
+        };
+    });
+}
+function findSourceMapMatch(sourceMap, markdownOffset) {
+    for (const entry of sourceMap.entries) {
+        if (markdownOffset < entry.markdownStart ||
+            markdownOffset > entry.markdownEnd) {
+            continue;
+        }
+        const region = entry.regions[0];
+        if (region !== undefined) {
+            return { entry, region };
+        }
+    }
+    return null;
+}
+function createLineStartOffsets(markdown) {
+    const offsets = [0];
+    for (let index = 0; index < markdown.length; index += 1) {
+        if (markdown[index] === "\n") {
+            offsets.push(index + 1);
+        }
+    }
+    return offsets;
+}
+function toParserWarnings(result, config) {
+    const warnings = result.warnings.map((warning) => ({
+        code: warning.code,
+        message: warning.message
+    }));
+    if (result.confidence.overall < config.minConfidenceForAutoCitation) {
+        warnings.push({
+            code: "pdf-low-confidence",
+            message: `PDF confidence ${String(result.confidence.overall)} is below the auto-citation threshold ${String(config.minConfidenceForAutoCitation)}.`
+        });
+    }
+    return warnings;
+}
+function createPdfArtifacts(result, config) {
+    return [
+        {
+            kind: "generated-markdown",
+            payload: {
+                markdown: result.markdown
+            }
+        },
+        {
+            kind: "source-map",
+            payload: result.sourceMap
+        },
+        {
+            kind: "document-ir",
+            payload: result.ir
+        },
+        {
+            kind: "assets",
+            payload: result.assets
+        },
+        {
+            kind: "diagnostics",
+            payload: result.diagnostics
+        },
+        {
+            kind: "warnings",
+            payload: result.warnings
+        },
+        {
+            kind: "confidence",
+            payload: result.confidence
+        },
+        {
+            kind: "parser-options",
+            payload: createPdfConvertOptionsSummary(config)
+        }
+    ];
+}
+function firstHeading(blocks) {
+    return blocks.find((block) => block.kind === "heading")?.text ?? null;
+}
+function hashString(value) {
+    return createHash("sha256").update(value).digest("hex");
+}
+//# sourceMappingURL=pdf2llm.js.map

package/dist/parsers/pdf2llm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf2llm.js","sourceRoot":"","sources":["../../src/parsers/pdf2llm.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EACL,oBAAoB,EACpB,aAAa,EAMd,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAS/C,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAC;AAC/C,MAAM,CAAC,MAAM,uBAAuB,GAAG,OAAO,CAAC;AAC/C,MAAM,CAAC,MAAM,sBAAsB,GAAG,uBAAuB,CAAC;AAc9D,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,YAAoB,EACpB,OAA4B;IAE5B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,sBAAsB,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG;QACf,GAAG,MAAM,CAAC,QAAQ;QAClB,GAAG,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC;KAC5C,CAAC;IACF,MAAM,MAAM,GAAG,iBAAiB,CAC9B,MAAM,CAAC,MAAM,EACb,MAAM,EACN,sBAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,EACvC,QAAQ,CACT,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE5D,OAAO;QACL,UAAU,EAAE,mBAAmB;QAC/B,aAAa,EAAE,sBAAsB;QACrC,UAAU,EAAE,KAAK;QACjB,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC;QAC3B,IAAI;QACJ,cAAc,EAAE,aAAa,CAAC,IAAI,CAAC;QACnC,MAAM;QACN,QAAQ;QACR,QAAQ,EAAE;YACR,OAAO,EAAE;gBACP,cAAc,EAAE,uBAAuB;gBACvC,aAAa;gBACb,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM;gBAC1C,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC,UAAU;aACjC;SACF;QACD,SAAS,EAAE,kBAAkB,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC;KACtD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAAiB;IAC1D,OAAO,UAAU,CACf,IAAI,CAAC,SAAS,CAAC;QACb,MAAM,EAAE,mBAAmB;QAC3B,cAAc,EAAE,uBAAuB;QACvC,aAAa;QACb,OAAO,EAAE,8BAA8B,CAAC,MAAM,CAAC;KAChD,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CACzB,YAAoB,EACpB,OAA4B;IAE5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,oBAAoB,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;QAC9B,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAE7B,OAAO,SAAS,CAAC,YAAY,EAAE;QAC7B,GAAG,uBAAuB,CAAC,OAAO,CAAC,MAAM,CAAC;QAC1C,MAAM,EAAE,UAAU,CAAC,MAAM;KAC1B,CAAC;SACC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QACxB,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC9B,MAAM,IAAI,eAAe,CACvB,+BAA+B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,YAAY,EAAE,EACrF,KAAK,CACN,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,eAAe,CAAC,uBAAuB,YAAY,EAAE,EAAE,KAAK,CAAC,CAAC;IAC1E,CAAC,CAAC;SACD,OAAO,CAAC,GAAG,EAAE;QACZ,YAAY,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,uBAAuB,CAAC,MAAiB;IAChD,OAAO;QACL,QAAQ,EAAE;YACR,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC;QACD,MAAM,EAAE;YACN,IAAI,EAAE,UAAU;SACjB;QACD,QAAQ,EAAE;YACR,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B;QACD,MAAM,EAAE;YACN,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,8BAA8B,CAAC,MAAiB;IACvD,OAAO;QACL,QAAQ,EAAE;YACR,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC;QACD,MAAM,EAAE;YACN,IAAI,EAAE,UAAU;SACjB;QACD,QAAQ,EAAE;YACR,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B;QACD,MAAM,EAAE;YACN,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC;QACD,QAAQ,EAAE;YACR,4BAA4B,EAAE,MAAM,CAAC,4BAA4B;SAClE;KACF,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,MAA8B,EAC9B,MAAqB,EACrB,gBAAmC,EACnC,QAAkC;IAElC,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1B,MAAM,KAAK,GAAG,kBAAkB,CAC9B,MAAM,CAAC,SAAS,EAChB,gBAAgB,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAC3C,CAAC;QAEF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,OAAO;gBACL,GAAG,KAAK;gBACR,QAAQ,EAAE;oBACR,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;oBACzB,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO;oBACxC,eAAe,EAAE,YAAY;iBAC9B;aACF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,GAAG,KAAK;YACR,QAAQ,EAAE;gBACR,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;gBACzB,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC;gBACnC,aAAa,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO;gBACxC,eAAe,EAAE,YAAY;gBAC7B,gBAAgB,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI;gBAClC,SAAS,EAAE;oBACT,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,SAAS;oBACjC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;oBACjB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;oBACjB,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK;oBACzB,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;oBAC3B,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM;iBAC5B;aACF;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CACzB,SAA4B,EAC5B,cAAsB;IAEtB,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;QACtC,IACE,cAAc,GAAG,KAAK,CAAC,aAAa;YACpC,cAAc,GAAG,KAAK,CAAC,WAAW,EAClC,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEhC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,sBAAsB,CAAC,QAAgB;IAC9C,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;IAEpB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,IAAI,QAAQ,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,gBAAgB,CACvB,MAAqB,EACrB,MAAiB;IAEjB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC,CAAC;IAEJ,IAAI,MAAM,CAAC,UAAU,CAAC,OAAO,GAAG,MAAM,CAAC,4BAA4B,EAAE,CAAC;QACpE,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,oBAAoB;YAC1B,OAAO,EAAE,kBAAkB,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,yCAAyC,MAAM,CAAC,MAAM,CAAC,4BAA4B,CAAC,GAAG;SACpJ,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAqB,EACrB,MAAiB;IAEjB,OAAO;QACL;YACE,IAAI,EAAE,oBAAoB;YAC1B,OAAO,EAAE;gBACP,QAAQ,EAAE,MAAM,CAAC,QAAQ;aAC1B;SACF;QACD;YACE,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,MAAM,CAAC,SAAS;SAC1B;QACD;YACE,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,MAAM,CAAC,EAAE;SACnB;QACD;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,MAAM,CAAC,MAAM;SACvB;QACD;YACE,IAAI,EAAE,aAAa;YACnB,OAAO,EAAE,MAAM,CAAC,WAAW;SAC5B;QACD;YACE,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,MAAM,CAAC,QAAQ;SACzB;QACD;YACE,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,MAAM,CAAC,UAAU;SAC3B;QACD;YACE,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,8BAA8B,CAAC,MAAM,CAAC;SAChD;KACF,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,MAA8B;IAClD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC;AACxE,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC"}