@semiont/content 0.5.24 → 0.5.26

package/dist/index.js

@@ -202,111 +202,838 @@ function deriveStorageUri(name, format) {
   return `file://${slug}${MEDIA_TYPES[format].extension}`;
 }
 
+// src/content-extractor.ts
+import { decodeRepresentation } from "@semiont/core";
+
+// src/pdf-extractor.ts
+import { isObject as isObject4 } from "@semiont/core";
+
 // src/extract-pdf-text-layer.ts
 import * as pdfjs from "pdfjs-dist/legacy/build/pdf.mjs";
+
+// src/pdfjs-assets.ts
+import { createRequire } from "module";
+import path2 from "path";
+var require2 = createRequire(import.meta.url);
+var STANDARD_FONT_DATA_URL = `${path2.join(path2.dirname(require2.resolve("pdfjs-dist/package.json")), "standard_fonts")}${path2.sep}`;
+
+// src/extract-pdf-text-layer.ts
+import { isObject, isString, isNumber, isArray, anchorRuns, isTextRun } from "@semiont/core";
+function toFormField(entry) {
+  if (!isObject(entry)) return null;
+  const { name, value, page, rect } = entry;
+  if (!isString(name) || !isString(value) || !value.trim()) return null;
+  if (!isNumber(page) || page < 0) return null;
+  if (!isArray(rect) || rect.length < 4 || !rect.every(isNumber)) return null;
+  const [x1, y1, x2, y2] = rect;
+  return {
+    name,
+    value: value.trim(),
+    page: page + 1,
+    // pdf.js reports 0-indexed; PdfTextItem is 1-indexed
+    x: Math.min(x1, x2),
+    y: Math.min(y1, y2),
+    width: Math.abs(x2 - x1),
+    height: Math.abs(y2 - y1)
+  };
+}
+async function readFormFields(doc) {
+  const fieldObjects = await doc.getFieldObjects();
+  if (!fieldObjects) return [];
+  const byName = /* @__PURE__ */ new Map();
+  for (const entries of Object.values(fieldObjects)) {
+    if (!isArray(entries)) continue;
+    for (const entry of entries) {
+      const field = toFormField(entry);
+      if (field && !byName.has(field.name)) byName.set(field.name, field);
+    }
+  }
+  return [...byName.values()];
+}
 async function extractPdfTextLayer(bytes) {
-  const loadingTask = pdfjs.getDocument({ data: bytes });
-  const doc = await loadingTask.promise;
+  const data = new Uint8Array(bytes);
+  const loadingTask = pdfjs.getDocument({ data, standardFontDataUrl: STANDARD_FONT_DATA_URL });
   try {
+    const doc = await loadingTask.promise;
     const pages = [];
     const items = [];
     let text = "";
-    let hasAnyTextItems = false;
     for (let pageNum = 1; pageNum <= doc.numPages; pageNum++) {
       const page = await doc.getPage(pageNum);
       const viewport = page.getViewport({ scale: 1 });
       const content = await page.getTextContent();
+      const pageTextStart = text.length;
+      const page1 = anchorRuns(content.items.filter(isTextRun), pageNum);
+      for (const item of page1.items) {
+        items.push({ ...item, start: item.start + pageTextStart, end: item.end + pageTextStart });
+      }
+      text += page1.text;
+      text += "\n";
       pages.push({
         pageNumber: pageNum,
         widthPt: viewport.width,
-        heightPt: viewport.height
+        heightPt: viewport.height,
+        textStart: pageTextStart,
+        textEnd: text.length,
+        hasTextLayer: page1.items.length > 0
       });
-      for (const item of content.items) {
-        if (!("str" in item)) continue;
-        if (item.str.trim()) {
-          hasAnyTextItems = true;
+    }
+    if (!pages.some((page) => page.hasTextLayer)) return null;
+    return { pages, text, items, fields: await readFormFields(doc) };
+  } finally {
+    await loadingTask.destroy();
+  }
+}
+
+// src/pdf-tables.ts
+var MIN_ROWS = 3;
+var MIN_COLUMNS = 2;
+var ROW_TOLERANCE = 0.5;
+var CELL_GAP = 0.8;
+function median(values) {
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.floor(sorted.length / 2)] ?? 0;
+}
+function groupRows(items, tolerance) {
+  const rows = [];
+  for (const item of [...items].sort((a, b) => b.y - a.y)) {
+    const row = rows[rows.length - 1];
+    if (row && Math.abs(row[0].y - item.y) <= tolerance) row.push(item);
+    else rows.push([item]);
+  }
+  return rows;
+}
+function toCell(runs, text) {
+  const x = Math.min(...runs.map((r) => r.x));
+  const y = Math.min(...runs.map((r) => r.y));
+  const right = Math.max(...runs.map((r) => r.x + r.width));
+  const top = Math.max(...runs.map((r) => r.y + r.height));
+  return {
+    text: runs.map((r) => text.slice(r.start, r.end)).join(" ").trim(),
+    x,
+    y,
+    width: right - x,
+    height: top - y
+  };
+}
+function toCells(row, gap, text) {
+  const cells = [];
+  let current = [];
+  for (const item of [...row].sort((a, b) => a.x - b.x)) {
+    const previous = current[current.length - 1];
+    if (previous && item.x - (previous.x + previous.width) > gap) {
+      cells.push(toCell(current, text));
+      current = [];
+    }
+    current.push(item);
+  }
+  if (current.length > 0) cells.push(toCell(current, text));
+  return cells;
+}
+function detectTable(items, text) {
+  if (items.length === 0) return null;
+  const unit = median(items.map((i) => i.height).filter((h) => h > 0)) || 12;
+  const rows = groupRows(items, unit * ROW_TOLERANCE).map((row) => toCells(row, unit * CELL_GAP, text));
+  if (rows.length < MIN_ROWS) return null;
+  const columnCount = rows[0].length;
+  if (columnCount < MIN_COLUMNS) return null;
+  if (!rows.every((row) => row.length === columnCount)) return null;
+  for (let column = 0; column < columnCount; column++) {
+    const lefts = rows.map((row) => row[column].x);
+    if (Math.max(...lefts) - Math.min(...lefts) > unit) return null;
+  }
+  if (rows.some((row) => row.some((cell) => cell.text.length === 0))) return null;
+  return rows;
+}
+function renderTable(rows, page, offset) {
+  let text = "";
+  const items = [];
+  rows.forEach((row, rowIndex) => {
+    text += "|";
+    for (const cell of row) {
+      text += " ";
+      const start = offset + text.length;
+      text += cell.text;
+      items.push({
+        start,
+        end: offset + text.length,
+        page,
+        x: cell.x,
+        y: cell.y,
+        width: cell.width,
+        height: cell.height
+      });
+      text += " |";
+    }
+    text += "\n";
+    if (rowIndex === 0) text += `|${" --- |".repeat(row.length)}
+`;
+  });
+  return { text, items };
+}
+
+// src/pdf-page-images.ts
+import * as pdfjs2 from "pdfjs-dist/legacy/build/pdf.mjs";
+import { isObject as isObject2, isNumber as isNumber2, isString as isString2, isArray as isArray2 } from "@semiont/core";
+
+// src/png-encode.ts
+import zlib from "zlib";
+var CRC_TABLE = (() => {
+  const table = new Int32Array(256);
+  for (let n = 0; n < 256; n++) {
+    let c = n;
+    for (let k = 0; k < 8; k++) c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
+    table[n] = c;
+  }
+  return table;
+})();
+function crc32(buf) {
+  let c = -1;
+  for (const byte of buf) c = CRC_TABLE[(c ^ byte) & 255] ^ c >>> 8;
+  return (c ^ -1) >>> 0;
+}
+function chunk(type, data) {
+  const length = Buffer.alloc(4);
+  length.writeUInt32BE(data.length);
+  const body = Buffer.concat([Buffer.from(type, "ascii"), data]);
+  const crc = Buffer.alloc(4);
+  crc.writeUInt32BE(crc32(body));
+  return Buffer.concat([length, body, crc]);
+}
+function encodePng(width, height, rgb) {
+  const stride = width * 3 + 1;
+  const raw = Buffer.alloc(stride * height);
+  for (let y = 0; y < height; y++) {
+    raw[y * stride] = 0;
+    Buffer.from(rgb.buffer, rgb.byteOffset + y * width * 3, width * 3).copy(raw, y * stride + 1);
+  }
+  const ihdr = Buffer.alloc(13);
+  ihdr.writeUInt32BE(width, 0);
+  ihdr.writeUInt32BE(height, 4);
+  ihdr[8] = 8;
+  ihdr[9] = 2;
+  return Buffer.concat([
+    Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
+    chunk("IHDR", ihdr),
+    chunk("IDAT", zlib.deflateSync(raw, { level: 9 })),
+    chunk("IEND", Buffer.alloc(0))
+  ]);
+}
+
+// src/pdf-page-images.ts
+var GRAYSCALE_1BPP = 1;
+var RGB_24BPP = 2;
+var RGBA_32BPP = 3;
+var IDENTITY = [1, 0, 0, 1, 0, 0];
+var MAX_IMAGE_PIXELS = 48e6;
+function withinPixelBudget(width, height) {
+  if (!Number.isFinite(width) || !Number.isFinite(height)) return false;
+  if (width <= 0 || height <= 0) return false;
+  return width * height <= MAX_IMAGE_PIXELS;
+}
+function findPlacedImages(fnArray, argsArray) {
+  const placed = [];
+  const stack = [];
+  let ctm = [...IDENTITY];
+  for (let i = 0; i < fnArray.length; i++) {
+    const op = fnArray[i];
+    const args = argsArray[i];
+    if (op === pdfjs2.OPS.save) {
+      stack.push([...ctm]);
+    } else if (op === pdfjs2.OPS.restore) {
+      ctm = stack.pop() ?? [...IDENTITY];
+    } else if (op === pdfjs2.OPS.transform) {
+      if (isArray2(args) && args.length >= 6 && args.every(isNumber2)) {
+        ctm = pdfjs2.Util.transform(ctm, args);
+      }
+    } else if (op === pdfjs2.OPS.paintImageXObject) {
+      const ref = args?.[0];
+      const width = args?.[1];
+      const height = args?.[2];
+      if (isString2(ref) && isNumber2(width) && isNumber2(height)) {
+        placed.push({ ref, width, height, ctm: [...ctm] });
+      }
+    }
+  }
+  return placed;
+}
+function asBytes(data) {
+  if (data instanceof Uint8Array) return data;
+  if (data instanceof Uint8ClampedArray) return new Uint8Array(data.buffer, data.byteOffset, data.length);
+  return null;
+}
+function toRgb(image) {
+  if (!isObject2(image)) return null;
+  const { width, height, kind } = image;
+  const data = asBytes(image.data);
+  if (!isNumber2(width) || !isNumber2(height) || !data) return null;
+  if (width <= 0 || height <= 0) return null;
+  if (kind === RGB_24BPP) {
+    return data.length >= width * height * 3 ? { width, height, rgb: data } : null;
+  }
+  if (kind === RGBA_32BPP) {
+    if (data.length < width * height * 4) return null;
+    const rgb = new Uint8Array(width * height * 3);
+    for (let i = 0, o = 0; o < rgb.length; i += 4, o += 3) {
+      rgb[o] = data[i];
+      rgb[o + 1] = data[i + 1];
+      rgb[o + 2] = data[i + 2];
+    }
+    return { width, height, rgb };
+  }
+  if (kind === GRAYSCALE_1BPP) {
+    const rowBytes = Math.ceil(width / 8);
+    if (data.length < rowBytes * height) return null;
+    const rgb = new Uint8Array(width * height * 3);
+    for (let y = 0; y < height; y++) {
+      for (let x = 0; x < width; x++) {
+        const bit = data[y * rowBytes + (x >> 3)] & 128 >> (x & 7);
+        const value = bit ? 255 : 0;
+        const o = (y * width + x) * 3;
+        rgb[o] = value;
+        rgb[o + 1] = value;
+        rgb[o + 2] = value;
+      }
+    }
+    return { width, height, rgb };
+  }
+  return null;
+}
+var IMAGE_RESOLVE_TIMEOUT_MS = 3e4;
+function resolveImage(page, ref) {
+  const scope = ref.startsWith("g_") ? page.commonObjs : page.objs;
+  return new Promise((resolve) => {
+    const timer = setTimeout(() => resolve(null), IMAGE_RESOLVE_TIMEOUT_MS);
+    const settle = (value) => {
+      clearTimeout(timer);
+      resolve(value);
+    };
+    try {
+      scope.get(ref, settle);
+    } catch {
+      settle(null);
+    }
+  });
+}
+async function extractPageImages(bytes, pageNumbers) {
+  const wanted = pageNumbers ? new Set(pageNumbers) : null;
+  const loadingTask = pdfjs2.getDocument({ data: new Uint8Array(bytes), standardFontDataUrl: STANDARD_FONT_DATA_URL });
+  const byPage = /* @__PURE__ */ new Map();
+  try {
+    const doc = await loadingTask.promise;
+    for (let pageNum = 1; pageNum <= doc.numPages; pageNum++) {
+      if (wanted && !wanted.has(pageNum)) continue;
+      const page = await doc.getPage(pageNum);
+      const ops = await page.getOperatorList();
+      const images = [];
+      for (const placement of findPlacedImages(ops.fnArray, ops.argsArray)) {
+        if (!withinPixelBudget(placement.width, placement.height)) continue;
+        const rgb = toRgb(await resolveImage(page, placement.ref));
+        if (!rgb) continue;
+        images.push({
+          png: encodePng(rgb.width, rgb.height, rgb.rgb),
+          width: rgb.width,
+          height: rgb.height,
+          ctm: placement.ctm
+        });
+      }
+      if (images.length > 0) byPage.set(pageNum, images);
+    }
+    return byPage;
+  } finally {
+    await loadingTask.destroy();
+  }
+}
+
+// src/ocr.ts
+import { createRequire as createRequire2 } from "module";
+import { createWorker } from "tesseract.js";
+import { isObject as isObject3, isString as isString3 } from "@semiont/core";
+var cachedLangPath;
+function langPath() {
+  if (cachedLangPath) return cachedLangPath;
+  const data = createRequire2(import.meta.url)("@tesseract.js-data/eng");
+  if (!isObject3(data) || !isString3(data.langPath)) {
+    throw new Error(
+      "Vendored OCR language data is missing or malformed: @tesseract.js-data/eng did not export a langPath"
+    );
+  }
+  cachedLangPath = data.langPath;
+  return cachedLangPath;
+}
+function assemblePage(blocks) {
+  let text = "";
+  const words = [];
+  for (const block of blocks ?? []) {
+    for (const paragraph of block.paragraphs ?? []) {
+      for (const line of paragraph.lines ?? []) {
+        let wroteWord = false;
+        for (const word of line.words ?? []) {
+          const value = word.text.trim();
+          if (!value) continue;
+          if (wroteWord) text += " ";
           const start = text.length;
-          text += item.str;
-          const end = text.length;
-          const [, , , , x, y] = item.transform;
-          items.push({
+          text += value;
+          words.push({
+            text: value,
             start,
-            end,
-            page: pageNum,
-            x,
-            y,
-            width: item.width,
-            height: item.height
+            end: text.length,
+            // Horizontal extent from the word, vertical from the
+            // line. OCR boxes hug their glyphs, so a descender
+            // ('page') sits lower than its neighbours — and
+            // `locate()` groups items into lines by comparing `y`
+            // within a couple of points, a threshold that holds
+            // because NATIVE runs take y from the shared baseline.
+            // Passing per-word descenders through would split one
+            // visual line into several rects and draw a highlight
+            // as stacked fragments. Nothing is lost: `locate()`
+            // bounds each line anyway, so per-word vertical extent
+            // never reaches an annotation.
+            bbox: {
+              x0: word.bbox.x0,
+              x1: word.bbox.x1,
+              y0: line.bbox.y0,
+              y1: line.bbox.y1
+            },
+            confidence: word.confidence
           });
-          text += item.hasEOL ? "\n" : " ";
-        } else if (item.hasEOL) {
-          text += "\n";
+          wroteWord = true;
         }
+        if (wroteWord) text += "\n";
       }
       text += "\n";
     }
-    if (!hasAnyTextItems) return null;
-    return { pages, text, items };
+  }
+  return { text: text.trimEnd(), words };
+}
+async function recognizeImages(images) {
+  if (images.length === 0) return [];
+  const worker = await createWorker("eng", void 0, {
+    langPath: langPath(),
+    cacheMethod: "none"
+  });
+  try {
+    const results = [];
+    for (const image of images) {
+      const { data } = await worker.recognize(image, {}, { blocks: true, text: false });
+      results.push(assemblePage(data.blocks));
+    }
+    return results;
   } finally {
-    await loadingTask.destroy();
+    await worker.terminate();
   }
 }
 
-// src/locate.ts
-var SAME_LINE_THRESHOLD_PT = 2;
-function locate(layer, start, end) {
-  const overlap = layer.items.filter(
-    (item) => item.start < end && item.end > start
-  );
-  if (overlap.length === 0) return { rects: [], overlap };
-  const pages = groupItemsByPage(overlap);
-  const rects = [];
-  for (const [page, pageItems] of pages) {
-    const lines = groupItemsByLine(pageItems, SAME_LINE_THRESHOLD_PT);
-    for (const lineItems of lines) {
-      const x = Math.min(...lineItems.map((i) => i.x));
-      const right = Math.max(...lineItems.map((i) => i.x + i.width));
-      const y = Math.min(...lineItems.map((i) => i.y));
-      const top = Math.max(...lineItems.map((i) => i.y + i.height));
-      rects.push({ page, x, y, width: right - x, height: top - y });
+// src/ocr-geometry.ts
+import * as pdfjs3 from "pdfjs-dist/legacy/build/pdf.mjs";
+function toPagePoint(px, py, placement) {
+  const point = [px / placement.width, 1 - py / placement.height];
+  pdfjs3.Util.applyTransform(point, placement.ctm);
+  return point;
+}
+function mapWordsToItems(words, placement, page, textOffset) {
+  if (placement.width <= 0 || placement.height <= 0) return [];
+  return words.map((word) => {
+    const [ax, ay] = toPagePoint(word.bbox.x0, word.bbox.y0, placement);
+    const [bx, by] = toPagePoint(word.bbox.x1, word.bbox.y1, placement);
+    const x = Math.min(ax, bx);
+    const y = Math.min(ay, by);
+    return {
+      start: word.start + textOffset,
+      end: word.end + textOffset,
+      page,
+      x,
+      y,
+      width: Math.abs(bx - ax),
+      height: Math.abs(by - ay)
+    };
+  });
+}
+
+// src/pdf-extractor.ts
+var MAX_PDF_BYTES = 200 * 1024 * 1024;
+function withinByteBudget(bytes) {
+  return Number.isFinite(bytes) && bytes >= 0 && bytes <= MAX_PDF_BYTES;
+}
+var LOW_CONFIDENCE = 60;
+function summarize(confidences) {
+  if (confidences.length === 0) return void 0;
+  const total = confidences.reduce((sum, c) => sum + c, 0);
+  return {
+    mean: Math.round(total / confidences.length * 10) / 10,
+    lowConfidenceWords: confidences.filter((c) => c < LOW_CONFIDENCE).length,
+    totalWords: confidences.length
+  };
+}
+async function ocrPages(content, pageNumbers) {
+  const imagesByPage = await extractPageImages(content, pageNumbers);
+  if (imagesByPage.size === 0) return { text: "", items: [], confidences: [] };
+  const pages = [...imagesByPage.keys()].sort((a, b) => a - b);
+  const batch = pages.flatMap((page) => imagesByPage.get(page).map((image) => image.png));
+  const recognized = await recognizeImages(batch);
+  const byPage = /* @__PURE__ */ new Map();
+  let cursor = 0;
+  for (const page of pages) {
+    const images = imagesByPage.get(page);
+    let text = "";
+    const items = [];
+    const confidences = [];
+    for (const image of images) {
+      const result = recognized[cursor++];
+      if (!result?.text.trim()) continue;
+      if (text) text += "\n";
+      items.push(...mapWordsToItems(result.words, image, page, text.length));
+      confidences.push(...result.words.map((word) => word.confidence));
+      text += result.text;
     }
+    if (text) byPage.set(page, { text, items, confidences });
   }
-  return { rects, overlap };
+  return joinPages(byPage, 0);
 }
-function groupItemsByPage(items) {
-  const map = /* @__PURE__ */ new Map();
-  for (const item of items) {
-    const existing = map.get(item.page);
-    if (existing) {
-      existing.push(item);
+function joinPages(byPage, baseOffset) {
+  let text = "";
+  const items = [];
+  const confidences = [];
+  for (const [, page] of [...byPage.entries()].sort((a, b) => a[0] - b[0])) {
+    if (text) text += "\n\n";
+    const shift = baseOffset + text.length;
+    for (const item of page.items) {
+      items.push({ ...item, start: item.start + shift, end: item.end + shift });
+    }
+    confidences.push(...page.confidences);
+    text += page.text;
+  }
+  return { text, items, confidences };
+}
+function classifyPdfError(error) {
+  return isObject4(error) && error.name === "PasswordException" ? "encrypted" : "corrupt";
+}
+function foldFormFields(layer) {
+  let text = layer.text;
+  const items = [...layer.items];
+  for (const field of layer.fields) {
+    const start = text.length + `${field.name}: `.length;
+    text += `${field.name}: ${field.value}
+`;
+    items.push({
+      start,
+      end: start + field.value.length,
+      page: field.page,
+      x: field.x,
+      y: field.y,
+      width: field.width,
+      height: field.height
+    });
+  }
+  return { text, items, method: "form", pdfClass: "E" };
+}
+function shapeTables(layer) {
+  const pages = layer.pages.map((page) => {
+    const pageItems = layer.items.filter((item) => item.page === page.pageNumber);
+    return { page, pageItems, table: detectTable(pageItems, layer.text) };
+  });
+  if (!pages.some((p) => p.table)) return null;
+  let text = "";
+  const items = [];
+  for (const { page, pageItems, table } of pages) {
+    if (table) {
+      const rendered = renderTable(table, page.pageNumber, text.length);
+      text += rendered.text;
+      items.push(...rendered.items);
     } else {
-      map.set(item.page, [item]);
+      const shift = text.length - page.textStart;
+      text += layer.text.slice(page.textStart, page.textEnd);
+      for (const item of pageItems) {
+        items.push({ ...item, start: item.start + shift, end: item.end + shift });
+      }
     }
   }
-  return map;
+  return { text, items, method: "table", pdfClass: "D" };
 }
-function groupItemsByLine(items, sameLineThreshold) {
-  const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);
+var pdfExtractor = {
+  // Every non-declined PDF extraction carries positioned runs — native text
+  // layers and OCR both anchor by page geometry.
+  yieldsGeometry: true,
+  async extract(content, _mediaType, cache) {
+    const hit = await cache?.store.read(cache.key);
+    if (hit) return hit;
+    const outcome = await extractPdf(content);
+    if (cache) {
+      if ("declined" in outcome) await cache.store.write(cache.key, outcome);
+      else if (outcome.items) await cache.store.write(cache.key, { ...outcome, items: outcome.items });
+    }
+    return outcome;
+  }
+};
+async function extractPdf(content) {
+  if (!withinByteBudget(content.length)) return { declined: "too-large" };
+  let layer;
+  try {
+    layer = await extractPdfTextLayer(content);
+  } catch (error) {
+    return { declined: classifyPdfError(error) };
+  }
+  if (!layer) {
+    const ocr2 = await ocrPages(content);
+    if (!ocr2.text) return { declined: "no-text-layer" };
+    const confidence2 = summarize(ocr2.confidences);
+    return {
+      text: ocr2.text,
+      items: ocr2.items,
+      method: "ocr",
+      pdfClass: "B",
+      ...confidence2 ? { ocrConfidence: confidence2 } : {}
+    };
+  }
+  const shaped = layer.fields.length > 0 ? foldFormFields(layer) : shapeTables(layer) ?? { text: layer.text, items: layer.items, method: "pdf-text-layer", pdfClass: "A" };
+  const unreadPages = layer.pages.filter((page) => !page.hasTextLayer).map((page) => page.pageNumber);
+  if (unreadPages.length === 0) return shaped;
+  const recovered = await ocrPages(content, unreadPages);
+  const readPages = new Set(recovered.items.map((item) => item.page));
+  const stillUnread = unreadPages.filter((page) => !readPages.has(page));
+  const hybridClass = shaped.pdfClass === "A" ? "C" : shaped.pdfClass;
+  if (!recovered.text) {
+    return { ...shaped, unreadPages: stillUnread, pdfClass: hybridClass };
+  }
+  const shift = shaped.text.length;
+  const ocr = {
+    text: recovered.text,
+    items: recovered.items.map((item) => ({ ...item, start: item.start + shift, end: item.end + shift })),
+    confidences: recovered.confidences
+  };
+  const confidence = summarize(ocr.confidences);
+  return {
+    ...shaped,
+    text: `${shaped.text}${ocr.text}
+`,
+    items: [...shaped.items ?? [], ...ocr.items],
+    method: "ocr",
+    pdfClass: hybridClass,
+    ...confidence ? { ocrConfidence: confidence } : {},
+    ...stillUnread.length > 0 ? { unreadPages: stillUnread } : {}
+  };
+}
+
+// src/content-extractor.ts
+var passthroughExtractor = {
+  yieldsGeometry: false,
+  async extract(content, mediaType) {
+    return { text: decodeRepresentation(content, mediaType), method: "text-passthrough" };
+  }
+};
+var EXTRACTORS = {
+  "decode": passthroughExtractor,
+  "pdf-text-layer": pdfExtractor,
+  "none": null
+};
+
+// src/anchored-text-store.ts
+import fs2 from "fs";
+import path3 from "path";
+import { createRequire as createRequire3 } from "module";
+import { getShardPath, isObject as isObject5, isString as isString4, isNumber as isNumber3, isArray as isArray3 } from "@semiont/core";
+function buildStamp() {
+  const require3 = createRequire3(import.meta.url);
+  const version = (specifier) => {
+    try {
+      const pkg = require3(specifier);
+      return isObject5(pkg) && isString4(pkg.version) ? pkg.version : "unknown";
+    } catch {
+      return "unknown";
+    }
+  };
+  return `content-${version("../package.json")}+pdfjs-${version("pdfjs-dist/package.json")}+tesseract-${version("tesseract.js/package.json")}+eng-${version("@tesseract.js-data/eng/package.json")}`;
+}
+var STAMP = buildStamp();
+function encodeLines(items) {
   const lines = [];
-  let currentLine = [];
-  for (const item of sorted) {
-    if (currentLine.length === 0 || Math.abs(item.y - currentLine[0].y) <= sameLineThreshold) {
-      currentLine.push(item);
+  for (const item of items) {
+    const last = lines[lines.length - 1];
+    if (last && last.p === item.page && last.y === item.y && last.h === item.height) {
+      last.words.push([item.x, item.width, item.start, item.end]);
     } else {
-      lines.push(currentLine);
-      currentLine = [item];
+      lines.push({ p: item.page, y: item.y, h: item.height, words: [[item.x, item.width, item.start, item.end]] });
     }
   }
-  if (currentLine.length > 0) lines.push(currentLine);
   return lines;
 }
+function decodeLines(lines) {
+  const items = [];
+  for (const line of lines) {
+    for (const [x, width, start, end] of line.words) {
+      items.push({ start, end, page: line.p, x, y: line.y, width, height: line.h });
+    }
+  }
+  return items;
+}
+function isCached(value) {
+  if (!isObject5(value) || value.v !== 2 || !isString4(value.stamp)) return false;
+  if (isString4(value.declined)) return true;
+  if (!isString4(value.text) || !isString4(value.method) || !isArray3(value.lines)) return false;
+  return value.lines.every((line) => isObject5(line) && isNumber3(line.p) && isNumber3(line.y) && isNumber3(line.h) && isArray3(line.words) && line.words.every((w) => isArray3(w) && w.length === 4 && w.every(isNumber3)));
+}
+var VALID_KEY = /^[A-Za-z0-9_-]+$/;
+function createAnchoredTextStore(dir, logger) {
+  const fileFor = (key) => {
+    if (!VALID_KEY.test(key)) return null;
+    const [ab, cd] = getShardPath(key);
+    return path3.join(dir, ab, cd, `${key}.json`);
+  };
+  return {
+    async read(key) {
+      let hit = null;
+      try {
+        const file = fileFor(key);
+        if (file === null) throw new Error("invalid key");
+        const parsed = JSON.parse(await fs2.promises.readFile(file, "utf8"));
+        if (isCached(parsed) && parsed.stamp === STAMP) hit = parsed;
+      } catch {
+        hit = null;
+      }
+      logger?.debug("Anchored-text cache", {
+        outcome: hit ? "hit" : "miss",
+        key,
+        ...hit ? "declined" in hit ? { declined: hit.declined } : { lines: hit.lines.length } : {}
+      });
+      if (!hit) return null;
+      if ("declined" in hit) return { declined: hit.declined };
+      const { v: _v, stamp: _stamp, lines, text, ...provenance } = hit;
+      return { text, items: decodeLines(lines), ...provenance };
+    },
+    async write(key, outcome) {
+      const target = fileFor(key);
+      if (target === null) {
+        logger?.debug("Anchored-text cache: refusing invalid key", { key });
+        return;
+      }
+      const entry = "declined" in outcome ? { v: 2, stamp: STAMP, declined: outcome.declined } : (() => {
+        const { text, items, ...provenance } = outcome;
+        return { v: 2, stamp: STAMP, text, lines: encodeLines(items), ...provenance };
+      })();
+      const temp = `${target}.${process.pid}.tmp`;
+      try {
+        await fs2.promises.mkdir(path3.dirname(target), { recursive: true });
+        await fs2.promises.writeFile(temp, JSON.stringify(entry), "utf8");
+        await fs2.promises.rename(temp, target);
+      } catch {
+        await fs2.promises.rm(temp, { force: true }).catch(() => {
+        });
+      }
+    },
+    async list() {
+      const prefix = JSON.stringify({ v: 2, stamp: STAMP }).slice(0, -1) + ",";
+      let rootNames;
+      try {
+        rootNames = await fs2.promises.readdir(dir);
+      } catch {
+        return [];
+      }
+      let swept = 0;
+      for (const name of rootNames) {
+        if (!name.endsWith(".json")) continue;
+        await fs2.promises.rm(path3.join(dir, name), { force: true }).then(() => {
+          swept += 1;
+        }, () => {
+        });
+      }
+      if (swept > 0) logger?.info("Anchored-text cache: swept pre-P1 flat entries", { swept });
+      const keys = [];
+      let sweptInterim = 0;
+      for (const ab of rootNames) {
+        if (!/^[0-9a-f]{2}$/.test(ab)) continue;
+        let cdNames;
+        try {
+          cdNames = await fs2.promises.readdir(path3.join(dir, ab));
+        } catch {
+          continue;
+        }
+        for (const cd of cdNames) {
+          if (!/^[0-9a-f]{2}$/.test(cd)) continue;
+          let names;
+          try {
+            names = await fs2.promises.readdir(path3.join(dir, ab, cd));
+          } catch {
+            continue;
+          }
+          for (const name of names) {
+            if (!name.endsWith(".json")) continue;
+            const base = name.slice(0, -".json".length);
+            if (/^[0-9a-f]{32}$/.test(base)) {
+              await fs2.promises.rm(path3.join(dir, ab, cd, name), { force: true }).then(() => {
+                sweptInterim += 1;
+              }, () => {
+              });
+              continue;
+            }
+            let handle = null;
+            try {
+              handle = await fs2.promises.open(path3.join(dir, ab, cd, name), "r");
+              const buf = Buffer.alloc(prefix.length);
+              const { bytesRead } = await handle.read(buf, 0, prefix.length, 0);
+              if (bytesRead === prefix.length && buf.toString("utf8") === prefix) {
+                keys.push(base);
+              }
+            } catch {
+            } finally {
+              await handle?.close().catch(() => {
+              });
+            }
+          }
+        }
+      }
+      if (sweptInterim > 0) logger?.info("Anchored-text cache: swept interim resource-id entries", { swept: sweptInterim });
+      return keys;
+    }
+  };
+}
+
+// src/anchored-text-store-adapter.ts
+function anchoredTextStoreOverTransport(content, logger) {
+  return {
+    async read(key) {
+      try {
+        return await content.getAnchoredTextByChecksum(key);
+      } catch (error) {
+        logger?.debug("Anchored-text cache: transport read failed \u2014 treating as miss", {
+          key,
+          reason: error instanceof Error ? error.message : String(error)
+        });
+        return null;
+      }
+    },
+    async write(key, outcome) {
+      try {
+        await content.putAnchoredText(key, outcome);
+      } catch (error) {
+        logger?.debug("Anchored-text cache: transport write failed \u2014 entry not stored", {
+          key,
+          reason: error instanceof Error ? error.message : String(error)
+        });
+      }
+    },
+    async list() {
+      try {
+        return await content.listAnchoredTextKeys();
+      } catch (error) {
+        logger?.debug("Anchored-text cache: transport list failed \u2014 treating as empty", {
+          reason: error instanceof Error ? error.message : String(error)
+        });
+        return [];
+      }
+    }
+  };
+}
 export {
   ChecksumMismatchError,
+  EXTRACTORS,
+  MAX_PDF_BYTES,
   WorkingTreeStore,
+  anchoredTextStoreOverTransport,
   calculateChecksum,
+  createAnchoredTextStore,
   deriveStorageUri,
   extractPdfTextLayer,
-  locate,
-  verifyChecksum
+  verifyChecksum,
+  withinByteBudget
 };
 //# sourceMappingURL=index.js.map