@jcode.labs/ragmir 2.17.2 → 2.18.0

package/dist/parsing.js

@@ -2,6 +2,7 @@ import { createRequire as _createRequire } from "module";
 const __require = _createRequire(import.meta.url);
 import { spawn } from "node:child_process";
 import { readFile } from "node:fs/promises";
+import path from "node:path";
 import { strFromU8, unzipSync } from "fflate";
 import { htmlToText } from "html-to-text";
 import readExcelFile from "read-excel-file/node";
@@ -9,11 +10,15 @@ const mammoth = __require("mammoth");
 import { getDocumentProxy } from "unpdf";
 import YAML from "yaml";
 import { OCR_IMAGE_EXTENSIONS } from "./files.js";
+import { isRecord } from "./guards.js";
 import { MAX_EXTERNAL_TEXT_STDIO_BYTES, MAX_OFFICE_TEXT_ENTRY_COUNT, MAX_OFFICE_XML_ENTRY_BYTES, MAX_OFFICE_XML_TOTAL_BYTES, MAX_PDF_PAGES, MAX_PDF_TEXT_CHARACTERS, } from "./limits.js";
+import { MAX_OCR_BATCH_PAGES, pdfOcrCommandIdentity } from "./ocr.js";
+import { PDF_OCR_PARSER_POLICY, readPdfOcrCache, writePdfOcrCache, } from "./ocr-cache.js";
 import { throwIfAborted } from "./operation.js";
 const EXTERNAL_COMMAND_KILL_GRACE_MS = 2_000;
 const LONG_BASE64_TEXT_PATTERN = /\b[A-Za-z0-9+/]{240,}={0,2}\b/gu;
-const OFFICE_TEXT_ENTRY_PATTERN = /\.(?:xml|rels|xhtml|html|htm)$/iu;
+const OFFICE_TEXT_ENTRY_PATTERN = /\.(?:xml|rels|xhtml|html|htm|opf)$/iu;
+const NATURAL_PATH_ORDER = new Intl.Collator("en", { numeric: true, sensitivity: "base" });
 const HTML_TO_TEXT_OPTIONS = {
     wordwrap: false,
     selectors: [
@@ -25,22 +30,27 @@ export async function parseFile(file, options = {}) {
     throwIfAborted(options.signal);
     let text;
     let pages;
+    let regions;
+    let ocr;
+    let sourceLineCoordinates = false;
     switch (file.extension) {
         case ".pdf":
             ;
-            ({ text, pages } = await parsePdf(file.absolutePath, options));
+            ({ text, pages, regions, ocr } = await parsePdf(file, options));
             break;
         case ".doc":
             text = await parseLegacyWord(file.absolutePath, options);
             break;
         case ".docx":
-            text = await parseDocx(file.absolutePath);
+            text = await parseDocx(file.absolutePath, options.signal);
             break;
         case ".pptx":
-            text = await parsePptx(file.absolutePath);
+            ;
+            ({ text, regions } = await parsePptx(file.absolutePath, options.signal));
             break;
         case ".xlsx":
-            text = await parseXlsx(file.absolutePath);
+            ;
+            ({ text, regions } = await parseXlsx(file.absolutePath, options.signal));
             break;
         case ".avif":
         case ".bmp":
@@ -58,61 +68,138 @@ export async function parseFile(file, options = {}) {
         case ".odt":
         case ".ods":
         case ".odp":
-            text = await parseOpenDocument(file.absolutePath);
+            text = await parseOpenDocument(file.absolutePath, options.signal);
             break;
         case ".epub":
-            text = await parseEpub(file.absolutePath);
+            ;
+            ({ text, regions } = await parseEpub(file.absolutePath, options.signal));
             break;
         case ".html":
         case ".htm":
             text = htmlToText(await readFile(file.absolutePath, "utf8"), HTML_TO_TEXT_OPTIONS);
             break;
         case ".json":
-        case ".ipynb":
+        case ".ipynb": {
             text = JSON.stringify(JSON.parse(await readFile(file.absolutePath, "utf8")), null, 2);
             break;
+        }
         case ".yaml":
-        case ".yml":
+        case ".yml": {
             text = YAML.stringify(YAML.parse(await readFile(file.absolutePath, "utf8")));
             break;
+        }
         case ".rtf":
             text = stripRtf(await readFile(file.absolutePath, "utf8"));
             break;
         default:
             text = await readFile(file.absolutePath, "utf8");
+            sourceLineCoordinates = true;
     }
     throwIfAborted(options.signal);
-    const document = { file, text: normalizeText(text) };
+    const document = {
+        file,
+        text: regions ? text : normalizeText(text, sourceLineCoordinates),
+        sourceLineCoordinates,
+    };
     if (pages) {
         document.pages = pages;
     }
+    if (regions) {
+        document.regions = regions;
+    }
+    if (ocr) {
+        document.ocr = ocr;
+    }
     return document;
 }
-async function parseDocx(filePath) {
+async function parseDocx(filePath, signal) {
     const buffer = await readFile(filePath);
-    unzipOfficeFile(buffer);
+    throwIfAborted(signal);
+    validateOfficeArchive(buffer, signal);
     const result = await mammoth.extractRawText({ buffer });
+    throwIfAborted(signal);
     return result.value;
 }
-async function parsePptx(filePath) {
-    const entries = unzipOfficeFile(await readFile(filePath));
-    return xmlEntriesToText(entries, [
-        /^ppt\/slides\/slide\d+\.xml$/u,
-        /^ppt\/notesSlides\/notesSlide\d+\.xml$/u,
-    ]);
+async function parsePptx(filePath, signal) {
+    const entries = unzipOfficeFile(await readFile(filePath), signal);
+    const slides = orderedPresentationSlides(entries, signal);
+    const notes = new Map(numberedOfficeParts(entries, /^ppt\/notesSlides\/notesSlide(\d+)\.xml$/u, signal).map((part) => [part.number, part.text]));
+    const parts = [];
+    for (const [index, slide] of slides.entries()) {
+        throwIfAborted(signal);
+        const noteText = presentationSlideNotes(entries, slide.number) ?? notes.get(slide.number) ?? "";
+        const text = [slide.text, noteText].filter(Boolean).join("\n\n");
+        const slideNumber = index + 1;
+        if (text) {
+            parts.push({
+                text,
+                contextPath: `Slide ${slideNumber}`,
+                location: {
+                    kind: "slide",
+                    start: slideNumber,
+                    end: slideNumber,
+                },
+            });
+        }
+    }
+    return joinLocatedParts(parts, signal);
+}
+function presentationSlideNotes(entries, slideNumber) {
+    const relationships = entries.get(`ppt/slides/_rels/slide${slideNumber}.xml.rels`);
+    if (!relationships) {
+        return null;
+    }
+    for (const relationship of xmlStartTags(relationships, "Relationship")) {
+        const target = xmlAttribute(relationship, "Target");
+        if (!target) {
+            continue;
+        }
+        const entry = path.posix.normalize(path.posix.join("ppt/slides", target));
+        if (!/^ppt\/notesSlides\/notesSlide\d+\.xml$/u.test(entry)) {
+            continue;
+        }
+        const xml = entries.get(entry);
+        if (xml) {
+            return xmlToText(xml);
+        }
+    }
+    return null;
 }
-async function parseXlsx(filePath) {
+async function parseXlsx(filePath, signal) {
     const buffer = await readFile(filePath);
-    unzipOfficeFile(buffer);
+    throwIfAborted(signal);
+    validateOfficeArchive(buffer, signal);
     const workbook = await readExcelFile(buffer, { trim: false });
-    const sheets = [];
-    for (const sheet of workbook) {
-        const rows = sheet.data.map(spreadsheetRowToText).filter((row) => row.some(Boolean));
-        if (rows.length > 0) {
-            sheets.push(`# ${sheet.sheet}`, rows.map((row) => row.join("\t")).join("\n"));
+    throwIfAborted(signal);
+    const parts = [];
+    for (const [sheetIndex, sheet] of workbook.entries()) {
+        throwIfAborted(signal);
+        let firstRow = true;
+        for (const [rowIndex, rawRow] of sheet.data.entries()) {
+            throwIfAborted(signal);
+            const row = spreadsheetRowToText(rawRow);
+            const firstColumn = row.findIndex(Boolean);
+            if (firstColumn < 0) {
+                continue;
+            }
+            const lastColumn = lastNonEmptyIndex(row);
+            const rowNumber = rowIndex + 1;
+            parts.push({
+                text: `${firstRow ? `# ${sheet.sheet}\n` : ""}${row.join("\t")}`,
+                contextPath: `Sheet: ${sheet.sheet}`,
+                location: {
+                    kind: "sheet",
+                    start: sheetIndex + 1,
+                    end: sheetIndex + 1,
+                    label: sheet.sheet,
+                    cellStart: `${spreadsheetColumnName(firstColumn + 1)}${rowNumber}`,
+                    cellEnd: `${spreadsheetColumnName(lastColumn + 1)}${rowNumber}`,
+                },
+            });
+            firstRow = false;
         }
     }
-    return sheets.join("\n\n");
+    return joinLocatedParts(parts, signal);
 }
 async function parseImage(filePath, extension, options) {
     if (!OCR_IMAGE_EXTENSIONS.has(extension)) {
@@ -143,59 +230,220 @@ async function parseLegacyWord(filePath, options) {
         ...(options.signal ? { signal: options.signal } : {}),
     });
 }
-async function parseOpenDocument(filePath) {
-    const entries = unzipOfficeFile(await readFile(filePath));
-    return xmlEntriesToText(entries, [/^content\.xml$/u, /^meta\.xml$/u]);
+async function parseOpenDocument(filePath, signal) {
+    const entries = unzipOfficeFile(await readFile(filePath), signal);
+    return xmlEntriesToText(entries, [/^content\.xml$/u, /^meta\.xml$/u], signal);
 }
-async function parseEpub(filePath) {
-    const entries = unzipOfficeFile(await readFile(filePath));
+async function parseEpub(filePath, signal) {
+    const entries = unzipOfficeFile(await readFile(filePath), signal);
+    const orderedEntries = epubReadingOrder(entries, signal);
     const parts = [];
-    for (const [name, content] of [...entries.entries()].sort(([a], [b]) => a.localeCompare(b))) {
-        if (!/\.(?:xhtml|html|htm|xml)$/iu.test(name)) {
+    for (const [index, name] of orderedEntries.entries()) {
+        throwIfAborted(signal);
+        const content = entries.get(name);
+        if (content === undefined) {
             continue;
         }
         const text = htmlToText(content, HTML_TO_TEXT_OPTIONS);
         if (text.trim()) {
-            parts.push(text);
+            parts.push({
+                text,
+                contextPath: `EPUB: ${name}`,
+                location: {
+                    kind: "epub",
+                    start: index + 1,
+                    end: index + 1,
+                    label: name,
+                },
+            });
         }
     }
-    return parts.join("\n\n");
+    return joinLocatedParts(parts, signal);
+}
+function numberedOfficeParts(entries, pattern, signal) {
+    const parts = [];
+    for (const [name, xml] of entries) {
+        throwIfAborted(signal);
+        const match = pattern.exec(name);
+        const number = Number(match?.[1]);
+        const text = match && Number.isSafeInteger(number) && number > 0 ? xmlToText(xml) : "";
+        if (text) {
+            parts.push({ number, text });
+        }
+    }
+    return parts.sort((left, right) => left.number - right.number);
+}
+function orderedPresentationSlides(entries, signal) {
+    const natural = numberedOfficeParts(entries, /^ppt\/slides\/slide(\d+)\.xml$/u, signal);
+    const presentation = entries.get("ppt/presentation.xml");
+    const relationships = entries.get("ppt/_rels/presentation.xml.rels");
+    if (!presentation || !relationships) {
+        return natural;
+    }
+    const targets = new Map();
+    for (const relationship of xmlStartTags(relationships, "Relationship")) {
+        throwIfAborted(signal);
+        const id = xmlAttribute(relationship, "Id");
+        const target = xmlAttribute(relationship, "Target");
+        if (id && target) {
+            targets.set(id, path.posix.normalize(path.posix.join("ppt", target)));
+        }
+    }
+    const byNumber = new Map(natural.map((slide) => [slide.number, slide]));
+    const ordered = [];
+    for (const slide of xmlStartTags(presentation, "sldId")) {
+        throwIfAborted(signal);
+        const relationshipId = xmlAttribute(slide, "r:id");
+        const target = relationshipId ? targets.get(relationshipId) : undefined;
+        const match = target ? /^ppt\/slides\/slide(\d+)\.xml$/u.exec(target) : null;
+        const number = Number(match?.[1]);
+        const part = Number.isSafeInteger(number) ? byNumber.get(number) : undefined;
+        if (!part) {
+            continue;
+        }
+        byNumber.delete(number);
+        ordered.push(part);
+    }
+    return ordered.length > 0 ? [...ordered, ...byNumber.values()] : natural;
+}
+function joinLocatedParts(parts, signal) {
+    let text = "";
+    const regions = [];
+    for (const part of parts) {
+        throwIfAborted(signal);
+        const normalized = normalizeText(part.text);
+        if (!normalized) {
+            continue;
+        }
+        if (text) {
+            text += "\n\n";
+        }
+        const charStart = text.length;
+        text += normalized;
+        regions.push({
+            charStart,
+            charEnd: text.length,
+            contextPath: part.contextPath,
+            location: part.location,
+        });
+    }
+    return { text, regions };
+}
+function epubReadingOrder(entries, signal) {
+    const htmlEntries = [];
+    for (const name of entries.keys()) {
+        throwIfAborted(signal);
+        if (/\.(?:xhtml|html|htm)$/iu.test(name)) {
+            htmlEntries.push(name);
+        }
+    }
+    htmlEntries.sort((left, right) => NATURAL_PATH_ORDER.compare(left, right));
+    const containerPath = [...entries.keys()].find((name) => name.toLowerCase() === "meta-inf/container.xml");
+    const container = containerPath ? entries.get(containerPath) : undefined;
+    const packagePath = container ? firstXmlAttribute(container, "rootfile", "full-path") : null;
+    const packageXml = packagePath ? entries.get(packagePath) : undefined;
+    if (!packagePath || !packageXml) {
+        return htmlEntries;
+    }
+    const packageDirectory = path.posix.dirname(packagePath);
+    const manifest = new Map();
+    for (const element of xmlStartTags(packageXml, "item")) {
+        throwIfAborted(signal);
+        const id = xmlAttribute(element, "id");
+        const href = xmlAttribute(element, "href");
+        if (id && href) {
+            manifest.set(id, resolveEpubEntry(packageDirectory, href));
+        }
+    }
+    const spine = [];
+    for (const element of xmlStartTags(packageXml, "itemref")) {
+        throwIfAborted(signal);
+        const id = xmlAttribute(element, "idref");
+        const entry = id ? manifest.get(id) : undefined;
+        if (entry && entries.has(entry)) {
+            spine.push(entry);
+        }
+    }
+    return spine.length > 0 ? [...new Set(spine)] : htmlEntries;
+}
+function resolveEpubEntry(packageDirectory, href) {
+    const withoutFragment = href.split("#", 1)[0] ?? href;
+    let decoded = withoutFragment;
+    try {
+        decoded = decodeURIComponent(withoutFragment);
+    }
+    catch {
+        // Keep the literal manifest path when percent encoding is malformed.
+    }
+    return path.posix.normalize(path.posix.join(packageDirectory, decoded));
+}
+function xmlStartTags(xml, localName) {
+    return [...xml.matchAll(new RegExp(`<(?:[A-Za-z_][\\w.-]*:)?${localName}\\b[^>]*>`, "giu"))].map((match) => match[0]);
+}
+function firstXmlAttribute(xml, element, attribute) {
+    const tag = xmlStartTags(xml, element)[0];
+    return tag ? xmlAttribute(tag, attribute) : null;
 }
-function unzipOfficeFile(buffer) {
-    let textEntryCount = 0;
-    let totalTextBytes = 0;
-    let rejectedForSafetyLimit = false;
+function xmlAttribute(element, name) {
+    const match = new RegExp(`\\b${name}\\s*=\\s*(["'])(.*?)\\1`, "iu").exec(element);
+    return match?.[2] ? decodeXmlEntities(match[2]) : null;
+}
+function validateOfficeArchive(buffer, signal) {
+    const state = officeArchiveLimitsState();
+    unzipSync(new Uint8Array(buffer), {
+        filter: (file) => {
+            throwIfAborted(signal);
+            includeOfficeTextEntry(file, state);
+            return false;
+        },
+    });
+    assertOfficeArchiveLimits(state);
+}
+function unzipOfficeFile(buffer, signal) {
+    const state = officeArchiveLimitsState();
     const unzipped = unzipSync(new Uint8Array(buffer), {
         filter: (file) => {
-            if (!OFFICE_TEXT_ENTRY_PATTERN.test(file.name)) {
-                return false;
-            }
-            if (file.originalSize > MAX_OFFICE_XML_ENTRY_BYTES) {
-                rejectedForSafetyLimit = true;
-                return false;
-            }
-            textEntryCount += 1;
-            totalTextBytes += file.originalSize;
-            if (textEntryCount > MAX_OFFICE_TEXT_ENTRY_COUNT ||
-                totalTextBytes > MAX_OFFICE_XML_TOTAL_BYTES) {
-                rejectedForSafetyLimit = true;
-                return false;
-            }
-            return true;
+            throwIfAborted(signal);
+            return includeOfficeTextEntry(file, state);
         },
     });
-    if (rejectedForSafetyLimit) {
-        throw new Error("Archive text payload exceeds Ragmir safety limits.");
-    }
+    assertOfficeArchiveLimits(state);
     const entries = new Map();
     for (const [name, content] of Object.entries(unzipped)) {
+        throwIfAborted(signal);
         entries.set(name, strFromU8(content));
     }
     return entries;
 }
-function xmlEntriesToText(entries, patterns) {
+function officeArchiveLimitsState() {
+    return { textEntryCount: 0, totalTextBytes: 0, rejected: false };
+}
+function includeOfficeTextEntry(file, state) {
+    if (!OFFICE_TEXT_ENTRY_PATTERN.test(file.name)) {
+        return false;
+    }
+    if (file.originalSize > MAX_OFFICE_XML_ENTRY_BYTES) {
+        state.rejected = true;
+        return false;
+    }
+    state.textEntryCount += 1;
+    state.totalTextBytes += file.originalSize;
+    if (state.textEntryCount > MAX_OFFICE_TEXT_ENTRY_COUNT ||
+        state.totalTextBytes > MAX_OFFICE_XML_TOTAL_BYTES) {
+        state.rejected = true;
+        return false;
+    }
+    return true;
+}
+function assertOfficeArchiveLimits(state) {
+    if (state.rejected) {
+        throw new Error("Archive text payload exceeds Ragmir safety limits.");
+    }
+}
+function xmlEntriesToText(entries, patterns, signal) {
     const parts = [];
     for (const [name, xml] of [...entries.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+        throwIfAborted(signal);
         if (patterns.some((pattern) => pattern.test(name))) {
             const text = xmlToText(xml);
             if (text) {
@@ -227,6 +475,24 @@ function spreadsheetCellToText(value) {
     }
     return JSON.stringify(value);
 }
+function spreadsheetColumnName(column) {
+    let current = column;
+    let name = "";
+    while (current > 0) {
+        current -= 1;
+        name = String.fromCharCode(65 + (current % 26)) + name;
+        current = Math.floor(current / 26);
+    }
+    return name;
+}
+function lastNonEmptyIndex(values) {
+    for (let index = values.length - 1; index >= 0; index -= 1) {
+        if (values[index]) {
+            return index;
+        }
+    }
+    return -1;
+}
 function xmlToText(xml) {
     return normalizeText(decodeXmlEntities(xml
         .replace(/<w:tab\/>/gu, " ")
@@ -250,8 +516,8 @@ function decodeXmlEntities(input) {
         .replace(/&apos;/gu, "'")
         .replace(/&amp;/gu, "&");
 }
-async function parsePdf(filePath, options) {
-    const buffer = await readFile(filePath);
+async function parsePdf(file, options) {
+    const buffer = await readFile(file.absolutePath);
     let pdf;
     try {
         pdf = await getDocumentProxy(new Uint8Array(buffer));
@@ -263,12 +529,11 @@ async function parsePdf(filePath, options) {
         }
         throw error;
     }
+    const pageTexts = [];
     try {
         if (pdf.numPages > MAX_PDF_PAGES) {
             throw new Error(`PDF has ${pdf.numPages} pages; the safety limit is ${MAX_PDF_PAGES}.`);
         }
-        const pageTexts = [];
-        let totalCharacters = 0;
         for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) {
             const page = await pdf.getPage(pageNumber);
             let pageText = "";
@@ -284,28 +549,148 @@ async function parsePdf(filePath, options) {
                 page.cleanup();
             }
             pageText = normalizePdfPageText(pageText);
-            if (!pageText && options.pdfOcrCommand && options.pdfOcrCommand.length > 0) {
-                pageText = normalizePdfPageText(await runExternalTextCommand(filePath, {
-                    command: options.pdfOcrCommand,
-                    cwd: options.projectRoot,
-                    label: `OCR command for PDF page ${pageNumber}`,
-                    timeoutMs: options.pdfOcrTimeoutMs ?? 120_000,
-                    pathEnvName: "RAGMIR_PDF_PATH",
-                    pageNumber,
-                    ...(options.signal ? { signal: options.signal } : {}),
-                }));
-            }
-            totalCharacters += pageText.length;
-            if (totalCharacters > MAX_PDF_TEXT_CHARACTERS) {
-                throw new Error(`PDF text exceeds the ${MAX_PDF_TEXT_CHARACTERS} character safety limit.`);
-            }
             pageTexts.push(pageText);
         }
-        return joinPdfPages(pageTexts);
     }
     finally {
         await pdf.destroy();
     }
+    const blankPages = pageTexts.flatMap((text, index) => (text ? [] : [index + 1]));
+    const ocr = blankPages.length > 0 && options.pdfOcrCommand && options.pdfOcrCommand.length > 0
+        ? await fillPdfOcrPages(file, pageTexts, blankPages, options)
+        : undefined;
+    const totalCharacters = pageTexts.reduce((total, pageText) => total + pageText.length, 0);
+    if (totalCharacters > MAX_PDF_TEXT_CHARACTERS) {
+        throw new Error(`PDF text exceeds the ${MAX_PDF_TEXT_CHARACTERS} character safety limit.`);
+    }
+    return { ...joinPdfPages(pageTexts), ...(ocr ? { ocr } : {}) };
+}
+async function fillPdfOcrPages(file, pageTexts, blankPages, options) {
+    const startedAt = performance.now();
+    const command = options.pdfOcrCommand ?? [];
+    const projectRoot = options.projectRoot ?? path.dirname(file.absolutePath);
+    const commandIdentity = await pdfOcrCommandIdentity(command, projectRoot);
+    const metrics = {
+        pages: blankPages.length,
+        cacheHits: 0,
+        cacheMisses: 0,
+        batches: 0,
+        subprocesses: commandIdentity.subprocesses,
+        durationMs: 0,
+    };
+    const missingPages = [];
+    for (const pageNumber of blankPages) {
+        const cached = await readPdfOcrCache(projectRoot, pdfOcrCacheIdentity(file.checksum, pageNumber, commandIdentity));
+        if (cached === null) {
+            metrics.cacheMisses += 1;
+            missingPages.push(pageNumber);
+        }
+        else {
+            metrics.cacheHits += 1;
+            pageTexts[pageNumber - 1] = cached;
+        }
+    }
+    if (commandIdentity.supportsBatch) {
+        for (const batch of pdfOcrPageBatches(missingPages)) {
+            const output = await runExternalTextCommand(file.absolutePath, {
+                command,
+                cwd: options.projectRoot,
+                label: `OCR command for PDF pages ${batch.join(",")}`,
+                timeoutMs: options.pdfOcrTimeoutMs ?? 120_000,
+                pathEnvName: "RAGMIR_PDF_PATH",
+                pageNumbers: batch,
+                ...(options.signal ? { signal: options.signal } : {}),
+            });
+            const result = parsePdfOcrBatchOutput(output, batch);
+            metrics.batches += 1;
+            metrics.subprocesses += 1 + result.subprocesses;
+            for (const page of result.pages) {
+                const text = normalizePdfPageText(page.text);
+                pageTexts[page.page - 1] = text;
+                await writePdfOcrCache(projectRoot, pdfOcrCacheIdentity(file.checksum, page.page, commandIdentity), text);
+            }
+        }
+    }
+    else {
+        for (const pageNumber of missingPages) {
+            const text = normalizePdfPageText(await runExternalTextCommand(file.absolutePath, {
+                command,
+                cwd: options.projectRoot,
+                label: `OCR command for PDF page ${pageNumber}`,
+                timeoutMs: options.pdfOcrTimeoutMs ?? 120_000,
+                pathEnvName: "RAGMIR_PDF_PATH",
+                pageNumber,
+                ...(options.signal ? { signal: options.signal } : {}),
+            }));
+            metrics.batches += 1;
+            metrics.subprocesses += 1;
+            pageTexts[pageNumber - 1] = text;
+            await writePdfOcrCache(projectRoot, pdfOcrCacheIdentity(file.checksum, pageNumber, commandIdentity), text);
+        }
+    }
+    metrics.durationMs = Math.round((performance.now() - startedAt) * 1_000) / 1_000;
+    return metrics;
+}
+function pdfOcrCacheIdentity(sourceChecksum, page, command) {
+    return {
+        sourceChecksum,
+        page,
+        engine: command.engine,
+        engineVersion: command.engineVersion,
+        language: command.language,
+        dpi: command.dpi,
+        parserPolicy: PDF_OCR_PARSER_POLICY,
+        commandFingerprint: command.commandFingerprint,
+    };
+}
+function pdfOcrPageBatches(pages) {
+    const batches = [];
+    for (const page of pages) {
+        const current = batches.at(-1);
+        if (!current ||
+            current.length >= MAX_OCR_BATCH_PAGES ||
+            page - (current[0] ?? page) >= MAX_OCR_BATCH_PAGES) {
+            batches.push([page]);
+        }
+        else {
+            current.push(page);
+        }
+    }
+    return batches;
+}
+function parsePdfOcrBatchOutput(output, requestedPages) {
+    let value;
+    try {
+        value = JSON.parse(output);
+    }
+    catch {
+        throw new Error("Batched PDF OCR command returned invalid JSON.");
+    }
+    if (!isRecord(value)) {
+        throw new Error("Batched PDF OCR command returned an invalid result.");
+    }
+    if (!Number.isInteger(value.subprocesses) ||
+        Number(value.subprocesses) < 0 ||
+        Number(value.subprocesses) > 100 ||
+        !Array.isArray(value.pages)) {
+        throw new Error("Batched PDF OCR command returned invalid process diagnostics.");
+    }
+    const pages = value.pages.flatMap((entry) => {
+        if (isRecord(entry) && Number.isInteger(entry.page) && typeof entry.text === "string") {
+            return [
+                {
+                    page: Number(entry.page),
+                    text: entry.text,
+                },
+            ];
+        }
+        return [];
+    });
+    if (pages.length !== requestedPages.length ||
+        pages.some((page, index) => page.page !== requestedPages[index])) {
+        throw new Error("Batched PDF OCR command returned pages outside the requested ordered batch.");
+    }
+    return { pages, subprocesses: Number(value.subprocesses) };
 }
 function isPdfTextItem(value) {
     return (typeof value === "object" && value !== null && "str" in value && typeof value.str === "string");
@@ -316,15 +701,23 @@ function normalizePdfPageText(text) {
 function joinPdfPages(pageTexts) {
     let text = "";
     const pages = [];
+    const regions = [];
     for (const [index, pageText] of pageTexts.entries()) {
         if (index > 0) {
             text += "\n\n";
         }
         const charStart = text.length;
         text += pageText;
-        pages.push({ pageNumber: index + 1, charStart, charEnd: text.length });
+        const pageNumber = index + 1;
+        pages.push({ pageNumber, charStart, charEnd: text.length });
+        regions.push({
+            charStart,
+            charEnd: text.length,
+            contextPath: `Page ${pageNumber}`,
+            location: { kind: "page", start: pageNumber, end: pageNumber },
+        });
     }
-    return { text, pages };
+    return { text, pages, regions };
 }
 async function runExternalTextCommand(filePath, options) {
     throwIfAborted(options.signal);
@@ -336,8 +729,9 @@ async function runExternalTextCommand(filePath, options) {
     const hasInputPlaceholder = command.some((part) => part.includes("{input}"));
     const args = configuredArgs.map((part) => part.replaceAll("{input}", filePath));
     const pageNumber = options.pageNumber === undefined ? "" : String(options.pageNumber);
+    const pageNumbers = options.pageNumbers?.join(",") ?? "";
     for (const [index, arg] of args.entries()) {
-        args[index] = arg.replaceAll("{page}", pageNumber);
+        args[index] = arg.replaceAll("{page}", pageNumber).replaceAll("{pages}", pageNumbers);
     }
     if (!hasInputPlaceholder) {
         args.push(filePath);
@@ -350,6 +744,7 @@ async function runExternalTextCommand(filePath, options) {
                 ...externalCommandEnvironment(),
                 [options.pathEnvName]: filePath,
                 ...(options.pageNumber === undefined ? {} : { RAGMIR_PDF_PAGE: pageNumber }),
+                ...(options.pageNumbers === undefined ? {} : { RAGMIR_PDF_PAGES: pageNumbers }),
             },
             stdio: ["ignore", "pipe", "pipe"],
         });
@@ -478,10 +873,12 @@ function externalCommandEnvironment() {
         return value === undefined ? [] : [[name, value]];
     }));
 }
-function normalizeText(input) {
-    return input
-        .replace(LONG_BASE64_TEXT_PATTERN, " ")
-        .replace(/\r\n/g, "\n")
+function normalizeText(input, preserveSourceLines = false) {
+    const normalized = input.replace(LONG_BASE64_TEXT_PATTERN, " ").replace(/\r\n?/gu, "\n");
+    if (preserveSourceLines) {
+        return normalized.replace(/[ \t]+\n/gu, "\n").trimEnd();
+    }
+    return normalized
         .replace(/[ \t]+\n/g, "\n")
         .replace(/\n{4,}/g, "\n\n\n")
         .trim();