@bli-cockpit/cli 0.2.271 → 0.2.273

package/dist/pdf-layout.js

@@ -0,0 +1,287 @@
+/**
+ * One PDF page's text items to readable text, tables kept (BLI-23347).
+ *
+ * Pure: positioned text items in, a string out. No pdf library, no I/O, so the heuristics are tested on
+ * hand-made items and measured on real PDFs without a parser in the way.
+ *
+ * What it does, in order:
+ *  1. Items are grouped into ROWS by their baseline (y), left to right inside a row.
+ *  2. A page of two justified text columns (a paper) is split at its gutter, so the left column is read
+ *     before the right one; rows that cross the gutter (title, abstract, figure captions) stay whole.
+ *  3. Inside a row, items that sit close together join into one CELL; a gap of about a em starts the next.
+ *  4. Consecutive rows with two or more cells whose cells line up in the same vertical channels become a
+ *     Markdown table (first row is the header). Rows that do not line up stay plain lines, cells apart by
+ *     two spaces, so nothing is invented and nothing is dropped.
+ *
+ * Measured against `pdftotext -layout` on public PDFs: docs/architecture/pdf-transcripts.md.
+ */
+/** A gap wider than this many font sizes starts a new cell. */
+const CELL_GAP_EM = 0.5;
+/** A gap wider than this many font sizes between two items still needs a space when joining them. */
+const WORD_GAP_EM = 0.15;
+/** A vertical jump of more than this many font sizes between rows is a paragraph break (or the end of a table). */
+const BREAK_EM = 1.9;
+/** Rows of one table sit at most this many font sizes apart (booktabs rules add space around the header). */
+const TABLE_ROW_EM = 3;
+function median(values) {
+    const sorted = [...values].sort((a, b) => a - b);
+    return sorted.length === 0 ? 0 : sorted[Math.floor(sorted.length / 2)];
+}
+function toRows(items) {
+    const live = items.filter((item) => item.str.trim() !== "").sort((a, b) => b.y - a.y || a.x - b.x);
+    const rows = [];
+    for (const item of live) {
+        const last = rows[rows.length - 1];
+        if (last && Math.abs(last.y - item.y) <= Math.max(1.5, 0.5 * Math.min(last.size, item.size || last.size))) {
+            last.items.push(item);
+            last.size = Math.max(last.size, item.size);
+        }
+        else {
+            rows.push({ y: item.y, size: item.size || 10, items: [item], cells: [] });
+        }
+    }
+    const merged = attachScripts(rows);
+    for (const row of merged) {
+        row.items.sort((a, b) => a.x - b.x);
+        row.cells = cellsOf(row);
+    }
+    return merged;
+}
+/**
+ * A row of one or two tiny, smaller items right above or below a normal row is a superscript or subscript
+ * (`n·d²`), not a line of its own: it joins that row as `^2` or `_i`. Without this a table whose cells hold
+ * exponents is cut in pieces.
+ */
+function attachScripts(rows) {
+    const out = [];
+    for (let at = 0; at < rows.length; at += 1) {
+        const row = rows[at];
+        const chars = row.items.reduce((sum, item) => sum + item.str.trim().length, 0);
+        const near = [rows[at - 1], rows[at + 1]]
+            .filter((other) => other !== undefined && row.size <= 0.85 * other.size && Math.abs(other.y - row.y) <= 0.85 * other.size)
+            .sort((a, b) => Math.abs(a.y - row.y) - Math.abs(b.y - row.y))[0];
+        if (!near || chars > 6 || row.items.length > 2) {
+            out.push(row);
+            continue;
+        }
+        for (const item of row.items)
+            near.items.push({ ...item, str: `${item.y > near.y ? "^" : "_"}${item.str.trim()}` });
+    }
+    return out.filter((row, index) => out.indexOf(row) === index);
+}
+function cellsOf(row) {
+    const cells = [];
+    let end = Number.NEGATIVE_INFINITY;
+    for (const item of row.items) {
+        const gap = item.x - end;
+        const last = cells[cells.length - 1];
+        const text = item.str.replace(/\s+/g, " ");
+        if (!last || gap > CELL_GAP_EM * row.size) {
+            cells.push({ text: text.trim(), x0: item.x, x1: item.x + item.width });
+        }
+        else {
+            const script = text.startsWith("^") || text.startsWith("_");
+            const spaced = !script && gap > WORD_GAP_EM * row.size && !last.text.endsWith(" ") && !text.startsWith(" ");
+            last.text = (last.text + (spaced ? " " : "") + text).replace(/\s+/g, " ");
+            last.x1 = item.x + item.width;
+        }
+        end = item.x + item.width;
+    }
+    for (const cell of cells)
+        cell.text = cell.text.replace(/\s*(?:\.\s?){4,}\s*/g, " … ").trim();
+    return cells.filter((cell) => cell.text !== "");
+}
+function percentile(values, fraction) {
+    const sorted = [...values].sort((a, b) => a - b);
+    return sorted.length === 0 ? 0 : sorted[Math.min(sorted.length - 1, Math.floor(fraction * sorted.length))];
+}
+const spans = (item, x) => item.x < x - 2 && item.x + item.width > x + 2;
+/**
+ * The x of a gutter that splits two justified text columns, or null. Rows that cross it (title, abstract,
+ * a wide figure or table) stay whole; the caller keeps them. Decided on items, not rows: the two columns'
+ * baselines rarely line up, so a "row" holds one side of the page at a time.
+ */
+export function findGutter(rows) {
+    const items = rows.flatMap((row) => row.items);
+    if (items.length < 40)
+        return null;
+    const left = percentile(items.map((item) => item.x), 0.05);
+    const right = percentile(items.map((item) => item.x + item.width), 0.95);
+    const width = right - left;
+    if (width < 200)
+        return null;
+    const crossing = (x) => items.filter((item) => spans(item, x)).length;
+    let best = left + 0.4 * width;
+    for (let x = best; x <= left + 0.6 * width; x += 2)
+        if (crossing(x) < crossing(best))
+            best = x;
+    const floor = crossing(best);
+    if (floor > 0.04 * items.length)
+        return null;
+    let lo = best;
+    let hi = best;
+    while (crossing(lo - 1) <= floor && lo > left + 0.3 * width)
+        lo -= 1;
+    while (crossing(hi + 1) <= floor && hi < left + 0.7 * width)
+        hi += 1;
+    if (hi - lo < 6)
+        return null;
+    const gutter = (lo + hi) / 2;
+    // Both sides carry text, and the lines of each side run most of the way across their column (justified text).
+    const leftRows = rows.map((row) => row.items.filter((item) => item.x + item.width <= gutter)).filter((side) => side.length > 0);
+    const rightRows = rows.map((row) => row.items.filter((item) => item.x >= gutter)).filter((side) => side.length > 0);
+    if (leftRows.length < 8 || rightRows.length < 8)
+        return null;
+    const leftReach = median(leftRows.map((side) => Math.max(...side.map((item) => item.x + item.width)) - left));
+    const rightReach = median(rightRows.map((side) => Math.max(...side.map((item) => item.x + item.width)) - gutter));
+    if (leftReach < 0.8 * (lo - left) || rightReach < 0.8 * (right - hi))
+        return null;
+    return gutter;
+}
+function crosses(row, gutter) {
+    return row.items.some((item) => spans(item, gutter));
+}
+/** A section of rows read in order: whole rows, or left column then right column. */
+function sections(rows, gutter) {
+    if (gutter === null)
+        return [rows];
+    const out = [];
+    let run = [];
+    const flush = () => {
+        if (run.length === 0)
+            return;
+        const leftItems = run.flatMap((row) => row.items.filter((item) => item.x + item.width / 2 < gutter));
+        const rightItems = run.flatMap((row) => row.items.filter((item) => item.x + item.width / 2 >= gutter));
+        out.push(toRows(leftItems), toRows(rightItems));
+        run = [];
+    };
+    for (const row of rows) {
+        if (crosses(row, gutter)) {
+            flush();
+            out.push([row]);
+        }
+        else {
+            run.push(row);
+        }
+    }
+    flush();
+    return out.filter((section) => section.length > 0);
+}
+/** Column channels of a block of columnar rows: the x ranges where cells of the rows with the most cells sit. */
+function columnsOf(block) {
+    const most = median(block.map((row) => row.cells.length));
+    const spans = block
+        .filter((row) => row.cells.length >= Math.max(2, most))
+        .flatMap((row) => row.cells.map((cell) => [cell.x0, cell.x1]))
+        .sort((a, b) => a[0] - b[0]);
+    const pad = 0.3 * median(block.map((row) => row.size));
+    const columns = [];
+    for (const [x0, x1] of spans) {
+        const last = columns[columns.length - 1];
+        if (last && x0 <= last[1] + pad)
+            last[1] = Math.max(last[1], x1);
+        else
+            columns.push([x0, x1]);
+    }
+    return columns;
+}
+function columnIndex(columns, cell) {
+    const center = (cell.x0 + cell.x1) / 2;
+    let best = 0;
+    let bestDistance = Number.POSITIVE_INFINITY;
+    columns.forEach(([x0, x1], index) => {
+        const distance = center >= x0 && center <= x1 ? 0 : Math.min(Math.abs(center - x0), Math.abs(center - x1));
+        if (distance < bestDistance) {
+            best = index;
+            bestDistance = distance;
+        }
+    });
+    return best;
+}
+function markdownTable(block, columns) {
+    const lines = block.map((row) => {
+        const cells = Array.from({ length: columns.length }, () => "");
+        for (const cell of row.cells) {
+            const at = columnIndex(columns, cell);
+            cells[at] = cells[at] ? `${cells[at]} ${cell.text}` : cell.text;
+        }
+        return `| ${cells.map((text) => text.replace(/\|/g, "\\|")).join(" | ")} |`;
+    });
+    lines.splice(1, 0, `| ${columns.map(() => "---").join(" | ")} |`);
+    return lines.join("\n");
+}
+const plainLine = (row) => row.cells.map((cell) => cell.text).join(row.cells.length > 1 ? "  " : " ");
+/**
+ * A short one-cell row under a columnar row, with another columnar row after it, is the wrapped second line
+ * of one of that row's cells (a table header that wrapped): it joins that cell.
+ */
+function foldContinuations(rows) {
+    const out = [];
+    for (let at = 0; at < rows.length; at += 1) {
+        const row = rows[at];
+        const above = out[out.length - 1];
+        const below = rows[at + 1];
+        if (above && above.cells.length >= 2 && below && below.cells.length >= 2 && row.cells.length === 1 && row.cells[0].text.length <= 24 && above.y - row.y <= TABLE_ROW_EM * row.size) {
+            const cell = row.cells[0];
+            const center = (cell.x0 + cell.x1) / 2;
+            const target = above.cells.find((candidate) => center >= candidate.x0 - 2 && center <= candidate.x1 + 2);
+            if (target) {
+                target.text = `${target.text} ${cell.text}`;
+                target.x0 = Math.min(target.x0, cell.x0);
+                target.x1 = Math.max(target.x1, cell.x1);
+                continue;
+            }
+        }
+        out.push(row);
+    }
+    return out;
+}
+function renderSection(input) {
+    const rows = foldContinuations(input);
+    const out = [];
+    let previous;
+    let at = 0;
+    while (at < rows.length) {
+        const row = rows[at];
+        if (previous && previous.y - row.y > BREAK_EM * Math.max(previous.size, row.size))
+            out.push("");
+        if (row.cells.length >= 2) {
+            // A block: consecutive columnar rows with no paragraph-sized jump between them.
+            let end = at + 1;
+            while (end < rows.length && rows[end].cells.length >= 2 && rows[end - 1].y - rows[end].y <= TABLE_ROW_EM * rows[end].size)
+                end += 1;
+            const block = rows.slice(at, end);
+            const columns = block.length >= 2 ? columnsOf(block) : [];
+            const fullRows = block.filter((member) => member.cells.length === columns.length).length;
+            if (columns.length >= 2 && fullRows >= 2) {
+                out.push(markdownTable(block, columns));
+            }
+            else {
+                for (const member of block)
+                    out.push(plainLine(member));
+            }
+            previous = block[block.length - 1];
+            at = end;
+            continue;
+        }
+        if (row.cells.length === 1)
+            out.push(plainLine(row));
+        previous = row;
+        at += 1;
+    }
+    return out;
+}
+/** One page's items as text. Empty string when the page has no text. */
+export function layoutPage(items) {
+    const rows = toRows(items);
+    if (rows.length === 0)
+        return "";
+    const gutter = findGutter(rows);
+    const lines = [];
+    for (const section of sections(rows, gutter)) {
+        if (lines.length > 0 && lines[lines.length - 1] !== "")
+            lines.push("");
+        lines.push(...renderSection(section));
+    }
+    return lines.join("\n").replace(/\n{3,}/g, "\n\n").trim();
+}

package/dist/pdf-page-image.js

@@ -0,0 +1,114 @@
+/**
+ * One decoded PDF page image to a small grayscale PNG the image describe door takes (BLI-23347).
+ *
+ * A scanned PDF page is one big raster (a 7.4 MB scan decodes to 2550x3300 RGB, 25 MB). The describer
+ * takes at most 3 MB and reads text fine at about 1,600 px on the long side, so the raster is box-filtered
+ * down, turned to 8-bit grey and PNG-encoded, trying smaller sizes until it fits. Pure: pixels in, bytes
+ * out, no pdf library and no native dependency, so it runs the same on macOS and Windows.
+ */
+import { createHash } from "node:crypto";
+import { deflateSync } from "node:zlib";
+/** Long-side sizes tried in order; the first PNG under the byte limit wins. */
+const LONG_SIDES = [1600, 1200, 900, 640];
+const CRC_TABLE = (() => {
+    const table = new Uint32Array(256);
+    for (let n = 0; n < 256; n += 1) {
+        let c = n;
+        for (let k = 0; k < 8; k += 1)
+            c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+        table[n] = c >>> 0;
+    }
+    return table;
+})();
+function crc32(buffer) {
+    let c = 0xffffffff;
+    for (let i = 0; i < buffer.length; i += 1)
+        c = CRC_TABLE[(c ^ buffer[i]) & 0xff] ^ (c >>> 8);
+    return (c ^ 0xffffffff) >>> 0;
+}
+function chunk(type, body) {
+    const head = Buffer.alloc(8);
+    head.writeUInt32BE(body.length, 0);
+    head.write(type, 4, "latin1");
+    const crc = Buffer.alloc(4);
+    crc.writeUInt32BE(crc32(Buffer.concat([head.subarray(4), body])), 0);
+    return Buffer.concat([head, body, crc]);
+}
+/** Grey level (0-255) of every pixel, one byte each. */
+function toGrey(raster) {
+    const { width, height, kind, data } = raster;
+    const grey = new Uint8Array(width * height);
+    if (kind === 1) {
+        const stride = Math.ceil(width / 8);
+        for (let y = 0; y < height; y += 1) {
+            for (let x = 0; x < width; x += 1)
+                grey[y * width + x] = (data[y * stride + (x >> 3)] >> (7 - (x & 7))) & 1 ? 255 : 0;
+        }
+        return grey;
+    }
+    const step = kind === 2 ? 3 : 4;
+    for (let i = 0, p = 0; i < grey.length; i += 1, p += step) {
+        const luma = (data[p] * 299 + data[p + 1] * 587 + data[p + 2] * 114) / 1000;
+        // Transparent pixels read as paper, not as black.
+        const alpha = kind === 3 ? data[p + 3] / 255 : 1;
+        grey[i] = Math.round(luma * alpha + 255 * (1 - alpha));
+    }
+    return grey;
+}
+/** Box-filter `grey` (width x height) down to `toWidth` x `toHeight`. */
+function shrink(grey, width, height, toWidth, toHeight) {
+    if (toWidth === width && toHeight === height)
+        return grey;
+    const out = new Uint8Array(toWidth * toHeight);
+    for (let y = 0; y < toHeight; y += 1) {
+        const y0 = Math.floor((y * height) / toHeight);
+        const y1 = Math.max(y0 + 1, Math.floor(((y + 1) * height) / toHeight));
+        for (let x = 0; x < toWidth; x += 1) {
+            const x0 = Math.floor((x * width) / toWidth);
+            const x1 = Math.max(x0 + 1, Math.floor(((x + 1) * width) / toWidth));
+            let sum = 0;
+            for (let yy = y0; yy < y1; yy += 1)
+                for (let xx = x0; xx < x1; xx += 1)
+                    sum += grey[yy * width + xx];
+            out[y * toWidth + x] = Math.round(sum / ((y1 - y0) * (x1 - x0)));
+        }
+    }
+    return out;
+}
+/** An 8-bit grey PNG with the Sub filter on every row. */
+export function encodeGreyPng(grey, width, height) {
+    const rows = Buffer.alloc((width + 1) * height);
+    for (let y = 0; y < height; y += 1) {
+        const at = y * (width + 1);
+        rows[at] = 1;
+        for (let x = 0; x < width; x += 1)
+            rows[at + 1 + x] = (grey[y * width + x] - (x > 0 ? grey[y * width + x - 1] : 0)) & 0xff;
+    }
+    const header = Buffer.alloc(13);
+    header.writeUInt32BE(width, 0);
+    header.writeUInt32BE(height, 4);
+    header[8] = 8; // bit depth
+    header[9] = 0; // grey
+    return Buffer.concat([
+        Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+        chunk("IHDR", header),
+        chunk("IDAT", deflateSync(rows, { level: 6 })),
+        chunk("IEND", Buffer.alloc(0)),
+    ]);
+}
+/** The raster as a PNG of at most `maxBytes`, or null when even the smallest size is too big. */
+export function rasterToPagePng(raster, maxBytes) {
+    const { width, height } = raster;
+    if (!(width > 0 && height > 0) || raster.data.length < (raster.kind === 1 ? Math.ceil(width / 8) * height : width * height * (raster.kind === 2 ? 3 : 4)))
+        return null;
+    const grey = toGrey(raster);
+    for (const longSide of LONG_SIDES) {
+        const scale = Math.min(1, longSide / Math.max(width, height));
+        const toWidth = Math.max(1, Math.round(width * scale));
+        const toHeight = Math.max(1, Math.round(height * scale));
+        const bytes = encodeGreyPng(shrink(grey, width, height, toWidth, toHeight), toWidth, toHeight);
+        if (bytes.length <= maxBytes)
+            return { bytes, sha256: createHash("sha256").update(bytes).digest("hex"), width: toWidth, height: toHeight };
+    }
+    return null;
+}

package/dist/pdf-transcript-step.js

@@ -0,0 +1,218 @@
+/**
+ * The PDF step of the transcript prepare path (BLI-23347): turn each base64 PDF into a marker block, in the
+ * same pass and under the same budgets as images. Drop mode only.
+ *
+ * The session (`image-description-session.ts`) holds the image state; this module plugs into it through ONE
+ * seam, `addPdfsToFinds`, which takes the images the session found and returns the same shape with the PDFs
+ * added:
+ *
+ *  - every PDF occurrence is a stand-in `EmbeddedImage` carrying `replacement` (its marker text, set once the
+ *    conversion is complete), so the swap in `replaceImagesWithMarkers` and `slimLineStream` needs no PDF code;
+ *  - a scanned PDF's page images join `unique`, so the existing describe door, budgets, backoff and state do
+ *    the describing; the PDF is replaced only when every page image has a description (an undescribed PDF is
+ *    kept, exactly as an undescribed image is);
+ *  - the FIRST copy of a PDF in the transcript gets the full marker block, every later copy a one-line reference.
+ *
+ * Conversion results are cached on disk by sha256 (`pdf-conversion-cache.ts`), so a transcript rebuilt every
+ * tick converts each PDF once. A PDF that cannot be converted (too big, the budget spent, unreadable) is left
+ * as it is and says why in the log; it is not counted, so it never keeps a transcript "pending" forever.
+ */
+import { createHash } from "node:crypto";
+import { formatImageMarker, formatPdfMarker, formatPdfReference } from "@bli-cockpit/telemetry-core";
+import { convertPdf } from "./pdf-convert.js";
+import { readCachedPdf, writeCachedPdf } from "./pdf-conversion-cache.js";
+import { bytesMayHoldPdf, scanPdfsInLine } from "./pdf-byte-scan.js";
+const TAG = "[pdf-step]";
+/** Largest PDF decoded for conversion: the document, its pdf.js structures and one page raster must fit comfortably. */
+export const MAX_PDF_BYTES = 25 * 1024 * 1024;
+/** Time spent converting across one sync; later PDFs wait for the next tick (the cache keeps finished ones). */
+export const MAX_PDF_RUN_MS = 60_000;
+const BLOCK_CHARS = 48 * 1024;
+const runStates = new WeakMap();
+function runStateOf(key) {
+    let state = runStates.get(key);
+    if (!state) {
+        state = { spentMs: 0, records: new Map() };
+        runStates.set(key, state);
+    }
+    return state;
+}
+/** `text` in a JSON string that sits `depth` strings deep: one escaping pass per level, plus the string's own. */
+function jsonEscaped(text, depth) {
+    let out = JSON.stringify(text).slice(1, -1);
+    if (depth === 1)
+        out = JSON.stringify(out).slice(1, -1);
+    return out;
+}
+/** The decoded bytes of a base64 run, in blocks, into one exact-size buffer. */
+function decode(payload, byteSize) {
+    const out = Buffer.alloc(byteSize);
+    let at = 0;
+    const length = payload.length;
+    for (let from = 0; from < length; from += BLOCK_CHARS) {
+        const block = Buffer.from(typeof payload === "string" ? payload.slice(from, from + BLOCK_CHARS) : payload.toString("latin1", from, Math.min(length, from + BLOCK_CHARS)), "base64");
+        if (at + block.length > byteSize)
+            return null;
+        block.copy(out, at);
+        at += block.length;
+    }
+    return at === byteSize ? new Uint8Array(out.buffer, out.byteOffset, out.length) : null;
+}
+const prefix = (sha256) => sha256.slice(0, 16);
+export async function planPdfs(found, ctx) {
+    const state = runStateOf(ctx.key);
+    const distinct = new Map();
+    const order = new Map();
+    for (const pdf of found)
+        order.set(pdf.sha256, [...(order.get(pdf.sha256) ?? []), pdf]);
+    const pageImages = new Map();
+    const payloads = new Map();
+    for (const [sha256, copies] of order) {
+        const first = copies[0];
+        const started = Date.now();
+        // A PDF given up on earlier in this sync is not tried again.
+        if (state.records.has(sha256) && state.records.get(sha256) === null)
+            continue;
+        let entry = state.records.get(sha256) ?? (await readCachedPdf(ctx.stateDir, sha256));
+        const cacheHit = entry !== null;
+        const needsPages = entry?.kind === "scanned" && entry.pageImages.some((page) => !ctx.isDescribed(page.sha256));
+        if (!entry || needsPages) {
+            const skip = first.byteSize > MAX_PDF_BYTES ? "pdf_too_large" : state.spentMs > MAX_PDF_RUN_MS ? "pdf_time_budget_spent" : null;
+            if (skip) {
+                console.error(TAG + " PDF left as it is", JSON.stringify({ reason: skip, sha256_prefix: prefix(sha256), byte_size: first.byteSize, copies: copies.length }));
+                state.records.set(sha256, null);
+                continue;
+            }
+            const bytes = decode(await ctx.readBase64(first), first.byteSize);
+            if (!bytes || createHash("sha256").update(bytes).digest("hex") !== sha256)
+                throw ctx.sourceChanged();
+            const converted = await convertPdf(bytes);
+            state.spentMs += Date.now() - started;
+            if (converted.kind === "failed") {
+                console.error(TAG + " PDF left as it is", JSON.stringify({ reason: converted.reason, sha256_prefix: prefix(sha256), byte_size: first.byteSize, pages: converted.pages, copies: copies.length }));
+                state.records.set(sha256, null);
+                continue;
+            }
+            if (converted.kind === "text") {
+                entry = { kind: "text", pages: converted.pages, notes: converted.notes, body: converted.body };
+            }
+            else if (converted.pageImages.length === 0) {
+                console.error(TAG + " PDF left as it is", JSON.stringify({ reason: "scanned_without_page_images", sha256_prefix: prefix(sha256), byte_size: first.byteSize, pages: converted.pages, copies: copies.length }));
+                state.records.set(sha256, null);
+                continue;
+            }
+            else {
+                entry = { kind: "scanned", pages: converted.pages, notes: converted.notes, pageImages: converted.pageImages.map(({ page, png }) => ({ page, sha256: png.sha256, byteSize: png.bytes.length })) };
+                for (const { png } of converted.pageImages) {
+                    if (ctx.isDescribed(png.sha256) || payloads.has(png.sha256))
+                        continue;
+                    payloads.set(png.sha256, png.bytes.toString("base64"));
+                }
+            }
+            await writeCachedPdf(ctx.stateDir, sha256, entry);
+        }
+        if (!entry)
+            continue;
+        state.records.set(sha256, entry);
+        if (entry.kind === "scanned") {
+            for (const page of entry.pageImages) {
+                if (ctx.isDescribed(page.sha256) || !payloads.has(page.sha256))
+                    continue;
+                pageImages.set(page.sha256, { replaceStart: -1, replaceEnd: -1, base64Start: -1, base64End: -1, sha256: page.sha256, mediaType: "image/png", byteSize: page.byteSize });
+            }
+        }
+        console.error(TAG + " PDF converted", JSON.stringify({ kind: entry.kind, sha256_prefix: prefix(sha256), byte_size: first.byteSize, pages: entry.pages, chars: entry.kind === "text" ? entry.body.length : null, page_images: entry.kind === "scanned" ? entry.pageImages.length : null, copies: copies.length, cache: cacheHit ? "hit" : "miss", elapsed_ms: Date.now() - started }));
+        distinct.set(sha256, { pdf: first, copies, entry });
+    }
+    const occurrences = [];
+    const standIns = new Map();
+    let counted = 0;
+    for (const pdf of found) {
+        if (!distinct.has(pdf.sha256))
+            continue;
+        const standIn = { replaceStart: pdf.replaceStart, replaceEnd: pdf.replaceEnd, base64Start: pdf.base64Start, base64End: pdf.base64End, sha256: pdf.sha256, mediaType: "image/png", byteSize: pdf.byteSize };
+        standIns.set(pdf, standIn);
+        occurrences.push(standIn);
+        counted += 1;
+    }
+    const settle = () => {
+        let replacedCount = 0;
+        let charsSaved = 0;
+        for (const { copies, entry } of distinct.values()) {
+            const ready = entry.kind === "text" || entry.pageImages.every((page) => ctx.isDescribed(page.sha256));
+            if (!ready)
+                continue;
+            const facts = { sha256: copies[0].sha256, pages: entry.pages, byteSize: copies[0].byteSize };
+            const full = entry.kind === "text"
+                ? formatPdfMarker({ ...facts, method: "text-layer", notes: entry.notes }, entry.body)
+                : formatPdfMarker({ ...facts, method: "page-images", notes: entry.notes }, entry.pageImages.map((page) => `page ${page.page}: ${formatImageMarker({ sha256: page.sha256, mediaType: "image/png", byteSize: page.byteSize })}`).join("\n"));
+            const reference = formatPdfReference(facts);
+            copies.forEach((pdf, index) => {
+                const standIn = standIns.get(pdf);
+                const text = index === 0 ? full : reference;
+                standIn.replacement = jsonEscaped(text, pdf.escapeDepth);
+                replacedCount += 1;
+                charsSaved += pdf.replaceEnd - pdf.replaceStart - standIn.replacement.length;
+            });
+        }
+        return { replacedCount, charsSaved };
+    };
+    return { occurrences, pageImages, payloads, counted, settle };
+}
+/**
+ * The session's finds with the PDFs added; `finds` itself when there are none. Call only in drop mode.
+ */
+export async function addPdfsToFinds(finds, found, ctx) {
+    if (found.length === 0)
+        return finds;
+    const step = await planPdfs(found, ctx);
+    if (step.counted === 0)
+        return finds;
+    const images = finds.images ? [...finds.images, ...step.occurrences].sort((a, b) => a.replaceStart - b.replaceStart) : undefined;
+    return {
+        ...finds,
+        total: finds.total + step.counted,
+        unique: new Map([...finds.unique, ...step.pageImages]),
+        base64For: (image) => step.payloads.get(image.sha256) ?? finds.base64For(image),
+        swapped: (replace) => {
+            const base = finds.swapped(replace);
+            const pdf = step.settle();
+            return { replacedCount: base.replacedCount + pdf.replacedCount, charsSaved: base.charsSaved + pdf.charsSaved };
+        },
+        ...(images ? { images } : {}),
+        extraOccurrences: step.occurrences,
+    };
+}
+/** The PDFs of a whole transcript's text, with character offsets (the text path; files at the streaming size use the census). */
+export function pdfsOfText(text) {
+    if (!text.includes("JVBERi0") && !text.includes("data:application/pdf"))
+        return [];
+    const bytes = Buffer.from(text, "utf8");
+    if (!bytesMayHoldPdf(bytes))
+        return [];
+    const byteOffsets = scanPdfsInLine(bytes);
+    const out = [];
+    let consumedBytes = 0;
+    let consumedChars = 0;
+    const charAt = (byteOffset) => {
+        consumedChars += bytes.toString("utf8", consumedBytes, byteOffset).length;
+        consumedBytes = byteOffset;
+        return consumedChars;
+    };
+    for (const pdf of byteOffsets) {
+        const replaceStart = charAt(pdf.replaceStart);
+        const base64Start = replaceStart + (pdf.base64Start - pdf.replaceStart);
+        const length = pdf.base64End - pdf.base64Start;
+        const replaceEnd = base64Start + length;
+        consumedBytes = pdf.replaceEnd;
+        consumedChars = replaceEnd;
+        out.push({ ...pdf, replaceStart, replaceEnd, base64Start, base64End: replaceEnd });
+    }
+    return out;
+}
+/** Merge two lists already sorted by `replaceStart`. */
+export function mergeOccurrences(a, b) {
+    if (!b || b.length === 0)
+        return a;
+    return [...a, ...b].sort((x, y) => x.replaceStart - y.replaceStart);
+}

package/dist/raw-evidence-staging.js

@@ -2,6 +2,7 @@ import { DELIVERY_BACKOFF_HOLDING } from "@bli-cockpit/telemetry-core";
 import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
+import { bookkeepingRead, bookkeepingWrite } from "./bookkeeping-session.js";
 import { describeError, isMissingFileFailure } from "./health-detail.js";
 /**
  * Durable local state for raw-evidence STAGING and DELIVERY ATTEMPTS.
@@ -67,7 +68,9 @@ export function emptyRawEvidenceStagingState() {
 export function rawEvidenceStagingStatePath(stateDir) {
     return path.join(stateDir, RAW_EVIDENCE_STAGING_FILENAME);
 }
-export async function readRawEvidenceStagingState(stateDir) {
+export const readRawEvidenceStagingState = (stateDir) => // BLI-23346: once per loop
+ bookkeepingRead(rawEvidenceStagingStatePath(stateDir), () => readStagingFromDisk(stateDir));
+async function readStagingFromDisk(stateDir) {
     try {
         const raw = JSON.parse(await fs.readFile(rawEvidenceStagingStatePath(stateDir), "utf8"));
         const state = parseStagingState(raw);
@@ -94,11 +97,12 @@ export async function readRawEvidenceStagingState(stateDir) {
  * lose every attempt count at once — which is exactly the thing that must not
  * be losable.
  */
-export async function writeRawEvidenceStagingState(stateDir, state) {
+export const writeRawEvidenceStagingState = (stateDir, state) => bookkeepingWrite(rawEvidenceStagingStatePath(stateDir), state, (held) => writeStagingToDisk(stateDir, held));
+async function writeStagingToDisk(stateDir, state) {
     const filePath = rawEvidenceStagingStatePath(stateDir);
     await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
     const pruned = pruneStagingState(state);
-    const serialized = `${JSON.stringify(pruned, null, 2)}\n`;
+    const serialized = `${JSON.stringify(pruned)}\n`; // compact (BLI-23346); readers accept either
     const tempPath = `${filePath}.tmp-${process.pid}-${crypto.randomUUID()}`;
     let handle = null;
     try {