@opencraw/core 0.1.0 → 0.1.1

package/README.md

@@ -20,6 +20,7 @@ npx playwright install chromium   # web recipes and browser bootstraps only
 | `loadAccessConfig(path)`, `AccessBroker`, `ACCESS_PRESETS` | Access configs: load and validate one, lease a profile outside a crawl (the cli's `probe` does), list the provider presets. |
 | `crawler.run(set)` | Runs every input recipe in sequence; returns a `CrawlReport`. |
 | `crawler.close()` | Closes the browser, if one was launched. |
+| `readPdf(bytes)`, `findTables(pdf, query)`, `pdfText(pdf)` | The PDF reader and table extractor `request as: "pdf"` and `extract kind: "table"` use, for tooling. |
 | `HttpClient`, `BrowserClient` | The same clients the engine's runners use, for tooling built on top of `@opencraw/core` (`@opencraw/cli`'s `probe` command uses both). |
 | `parseInputRecipe`, `parseOutputRecipe`, `inputRecipeJsonSchema`, `outputRecipeJsonSchema`, `accessConfigJsonSchema` | The contracts, for tooling. |
 

package/dist/index.esm.js

@@ -7,6 +7,7 @@ import { createWriteStream } from 'node:fs';
 import { once } from 'node:events';
 import { JSONPath } from 'jsonpath-plus';
 import { load } from 'cheerio';
+import { fileURLToPath } from 'node:url';
 
 /** Resource types a page may skip loading, to save bandwidth on per-GB proxies. */
 const BLOCKABLE_RESOURCES = ['image', 'media', 'font', 'stylesheet', 'script', 'texttrack', 'xhr', 'fetch', 'eventsource', 'websocket', 'manifest', 'other'];
@@ -1292,7 +1293,7 @@ function traceLine(event) {
       }
     case 'recipe:finish':
       {
-        return `■ ${event.recipeId}: ${event.emitted} emitted, ${event.rejected} rejected, ${event.duplicates} duplicates, ${event.skipped > 0 ? `${event.skipped} skipped, ` : ''}${event.pages} pages, ${event.durationMs} ms${event.error === undefined ? '' : `\n  ✖ stopped: ${event.error}`}`;
+        return `■ ${event.recipeId}: ${event.emitted} emitted, ${event.rejected} rejected, ${event.duplicates} duplicates, ${event.skipped > 0 ? `${event.skipped} skipped, ` : ''}${(event.stepsSkipped ?? 0) > 0 ? `${event.stepsSkipped} steps skipped, ` : ''}${event.pages} pages, ${event.durationMs} ms${event.error === undefined ? '' : `\n  ✖ stopped: ${event.error}`}`;
       }
     case 'access:lease':
       {
@@ -2106,10 +2107,444 @@ async function detectBlock(response, rule = DEFAULT_BLOCK_RULE) {
   return undefined;
 }
 
+/** Runs closer than this share of the font size join into one cell. */
+const JOIN_GAP = 0.35;
+/** Runs further apart than this share of the font size get a space between them when joined. */
+const SPACE_GAP = 0.1;
+/** Runs on baselines closer than this share of the font size are on one line. */
+const SAME_BASELINE = 0.2;
+/** A cell joins a row when this share of its height overlaps the row. */
+const ROW_OVERLAP = 0.4;
+/**
+ * Turns a page's text runs into rows of cells, top to bottom.
+ *
+ * Runs on one baseline that nearly touch become one cell. Cells whose vertical
+ * extents overlap become one row, even when their baselines differ: a table
+ * that centres its cells vertically puts a one-line value a few points above
+ * or below its two-line label, and a row built from equal baselines would pair
+ * the value with the wrong label.
+ *
+ * @param runs - The page's text runs, in any order.
+ * @returns The rows.
+ */
+function assembleRows(runs) {
+  const cells = joinCells(runs);
+  const ordered = [...cells].sort((a, b) => middle(b) - middle(a) || a.x - b.x);
+  const rows = [];
+  let top = 0;
+  let bottom = 0;
+  for (const cell of ordered) {
+    const current = rows.at(-1);
+    const overlap = Math.min(cell.y + cell.height, top) - Math.max(cell.y, bottom);
+    if (current !== undefined && overlap >= ROW_OVERLAP * cell.height) {
+      current.push(cell);
+      top = Math.max(top, cell.y + cell.height);
+      bottom = Math.min(bottom, cell.y);
+    } else {
+      rows.push([cell]);
+      top = cell.y + cell.height;
+      bottom = cell.y;
+    }
+  }
+  return rows.map(row => rowOf(row));
+}
+function joinCells(runs) {
+  // Whitespace runs are gaps, not text: pdf.js emits the space between two
+  // table columns as one wide " ", which would bridge the columns.
+  // Left to right within a line, lines top to bottom: a run may sit a fraction of a point off its neighbours' baseline.
+  const ordered = runs.filter(run => run.text.trim() !== '').sort((a, b) => Math.abs(a.y - b.y) <= SAME_BASELINE * Math.max(a.height, b.height, 1) ? a.x - b.x : b.y - a.y);
+  const cells = [];
+  for (const run of ordered) {
+    const previous = lastOf(cells, cell => Math.abs(cell.y - run.y) <= SAME_BASELINE * Math.max(cell.height, run.height, 1));
+    const size = Math.max(run.height, previous?.height ?? 0, 1);
+    const gap = previous === undefined ? Infinity : run.x - (previous.x + previous.width);
+    if (previous !== undefined && gap <= JOIN_GAP * size && gap > -size) {
+      const space = gap > SPACE_GAP * size && !/\s$/.test(previous.text) && !/^\s/.test(run.text) ? ' ' : '';
+      previous.text += space + run.text;
+      previous.width = run.x + run.width - previous.x;
+      previous.height = Math.max(previous.height, run.height);
+    } else {
+      cells.push({
+        ...run
+      });
+    }
+  }
+  return cells.map(cell => ({
+    ...cell,
+    text: cell.text.trim()
+  })).filter(cell => cell.text !== '');
+}
+function rowOf(cells) {
+  const ordered = [...cells].sort((a, b) => a.x - b.x || b.y - a.y);
+  return {
+    top: Math.max(...ordered.map(cell => cell.y + cell.height)),
+    bottom: Math.min(...ordered.map(cell => cell.y)),
+    cells: ordered,
+    text: ordered.map(cell => cell.text).join('\t')
+  };
+}
+function middle(cell) {
+  return cell.y + cell.height / 2;
+}
+/** `Array#findLast`, which the es2022 library does not declare. */
+function lastOf(items, test) {
+  for (let index = items.length - 1; index >= 0; index -= 1) if (test(items[index])) return items[index];
+  return undefined;
+}
+
+/** A PDF that cannot be read: not a PDF, encrypted, or with no text to read. */
+class PdfReadError extends Error {
+  name = 'PdfReadError';
+}
+/**
+ * Reads a PDF's text layer into rows of positioned cells, with pdf.js. pdf.js
+ * is imported on first use, so recipes that never read a PDF never load it.
+ * Only text is read: no page is rendered, no script runs, no font is loaded.
+ *
+ * @param bytes - The file.
+ * @param source - Where it came from, for messages.
+ * @returns The document.
+ * @throws PdfReadError when the bytes are not a readable PDF, or no page has a
+ * text layer (a scan: OCR is not supported).
+ */
+async function readPdf(bytes, source = 'PDF') {
+  const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
+  // pdf.js takes ownership of the buffer it is given, and refuses a Node
+  // Buffer: hand it a plain copy.
+  const task = pdfjs.getDocument({
+    data: new Uint8Array(bytes),
+    verbosity: 0,
+    disableFontFace: true,
+    useSystemFonts: false,
+    stopAtErrors: true
+  });
+  let loaded;
+  try {
+    loaded = await task.promise;
+  } catch (error) {
+    throw new PdfReadError(`${source}: not a readable PDF (${error.message})`, {
+      cause: error
+    });
+  }
+  try {
+    const pages = [];
+    for (let number = 1; number <= loaded.numPages; number += 1) {
+      const page = await loaded.getPage(number);
+      const {
+        width,
+        height
+      } = page.getViewport({
+        scale: 1
+      });
+      const content = await page.getTextContent();
+      const runs = content.items.flatMap(item => 'str' in item ? [runOf(item)] : []);
+      pages.push({
+        number,
+        width,
+        height,
+        rows: assembleRows(runs)
+      });
+    }
+    if (pages.every(page => page.rows.length === 0)) throw new PdfReadError(`${source}: no page has a text layer (a scanned PDF? OCR is not supported)`);
+    return {
+      kind: 'pdf',
+      pages
+    };
+  } finally {
+    await task.destroy();
+  }
+}
+function runOf(item) {
+  const [a, b, c, d, x, y] = item.transform;
+  return {
+    x,
+    y,
+    width: item.width,
+    height: item.height > 0 ? item.height : Math.hypot(c, d) || Math.hypot(a, b),
+    text: item.str
+  };
+}
+
+/**
+ * The text a `regex` extract reads: one line per row (cells separated by a
+ * tab), pages separated by a blank line.
+ *
+ * @param document - The PDF.
+ * @returns The text.
+ */
+function pdfText(document) {
+  return document.pages.map(page => page.rows.map(row => row.text).join('\n')).join('\n\n');
+}
+/**
+ * Whether a value bound in scope is a read PDF (so `extract … from` can take it).
+ *
+ * @param value - Anything.
+ * @returns Whether it is a {@link PdfDocument}.
+ */
+function isPdfDocument(value) {
+  return typeof value === 'object' && value !== null && value.kind === 'pdf' && Array.isArray(value.pages);
+}
+
+/**
+ * Finds every table whose header row matches, and reads its rows by column.
+ *
+ * Columns come from the body, not the header: a header is often centred over
+ * a column whose cells are left-aligned, so the header's position says little
+ * about where the column starts. The left edges of the body cells cluster into
+ * bands; each band belongs to the header cell that overlaps it most, or the
+ * nearest one when none does. A header spanning two columns reads both.
+ *
+ * A cell wrapped over several lines (a long name, a note, a list of versions)
+ * spreads one row over several lines, its values often centred beside it:
+ * the lines are regrouped into rows around the lines that carry values.
+ *
+ * @param document - The PDF.
+ * @param query - Which tables, and how to name their columns.
+ * @returns The tables, in page order.
+ */
+function findTables(document, query) {
+  const tables = [];
+  for (const page of document.pages) {
+    const starts = page.rows.flatMap((row, index) => query.header.test(plain(row)) ? [index] : []);
+    for (const [position, start] of starts.entries()) {
+      const body = bodyOf(page.rows.slice(start + 1, starts[position + 1] ?? page.rows.length), query.until);
+      tables.push(readTable(page.number, page.rows[start], body, query));
+    }
+  }
+  return tables;
+}
+/** The rows under a header, up to the first one `until` matches. */
+function bodyOf(rows, until) {
+  const end = until === undefined ? -1 : rows.findIndex(row => until.test(plain(row)));
+  return end === -1 ? [...rows] : rows.slice(0, end);
+}
+function readTable(page, headerRow, body, query) {
+  const headers = headerCells(headerRow);
+  const bands = bandsOf(body, headers);
+  const lines = body.map(row => ({
+    row,
+    values: valuesOf(row, bands, headers.length)
+  }));
+  const groups = groupLines(lines, query.align ?? 'auto');
+  return {
+    page,
+    title: headers[0]?.text ?? '',
+    header: headers.map(header => header.text),
+    rows: groups.map(group => named(joinLines(group, headers.length), headers, query.columns))
+  };
+}
+/**
+ * Groups the body's lines into table rows. A line anchors a row when it has a
+ * name (the first column) and a value, or, without a name, a value in the
+ * first value column: the middle line of a name wrapped over several lines,
+ * the values centred beside it. Every other line (a wrapped name, a wrapped
+ * note, a list of versions over two lines) joins an anchor:
+ *
+ * - `top`: the nearest anchor above it; `bottom`: the nearest below;
+ * - `center`: a wrapped name splits evenly around its anchor (as many lines
+ *   below as above), and other lines join the nearest anchor;
+ * - `auto`: `center` when some line carries values but no name (only a
+ *   centred table does that), else the nearest anchor, a tie going to the one
+ *   below (a wrapped cell's first line comes before its row).
+ */
+function groupLines(lines, align) {
+  const named = line => line.values[0] !== '';
+  const valued = line => line.values.slice(1).some(value => value !== '');
+  let anchors = lines.filter(line => named(line) && valued(line) || !named(line) && (line.values[1] ?? '') !== '');
+  if (anchors.length === 0) anchors = lines.filter(line => named(line));
+  if (anchors.length === 0) return [];
+  const groups = new Map(anchors.map(anchor => [anchor, [anchor]]));
+  const centred = align === 'center' || align === 'auto' && anchors.some(anchor => !named(anchor));
+  if (centred) splitNamesEvenly(lines, anchors, named, groups);
+  const placed = new Set();
+  for (const members of groups.values()) for (const line of members) placed.add(line);
+  for (const line of lines) if (!placed.has(line)) groups.get(ownerOf(line, anchors, align))?.push(line);
+  // An anchor that ended up with no name at all is a value spilling out of the row next to it.
+  const nameless = anchors.filter(anchor => (groups.get(anchor) ?? []).every(line => !named(line)));
+  const kept = anchors.filter(anchor => !nameless.includes(anchor));
+  if (kept.length === 0) return [];
+  for (const anchor of nameless) groups.get(nearest(anchor, kept))?.push(...(groups.get(anchor) ?? []));
+  return kept.map(anchor => groups.get(anchor) ?? []);
+}
+/**
+ * Hands the wrapped name lines between two anchors out evenly: the upper
+ * anchor takes as many lines below it as it took above it, the lower one the
+ * rest. Lines are in page order, top to bottom.
+ */
+function splitNamesEvenly(lines, anchors, named, groups) {
+  const positions = anchors.map(anchor => lines.indexOf(anchor));
+  const namesBetween = (from, to) => lines.slice(from, to).filter(line => named(line) && !groups.has(line));
+  let above = namesBetween(0, positions[0]);
+  groups.get(anchors[0])?.push(...above);
+  for (const [index, anchor] of anchors.entries()) {
+    const next = anchors[index + 1];
+    const run = namesBetween(positions[index] + 1, next === undefined ? lines.length : positions[index + 1]);
+    const taken = next === undefined ? run.length : Math.min(run.length, above.length);
+    groups.get(anchor)?.push(...run.slice(0, taken));
+    above = run.slice(taken);
+    if (next !== undefined) groups.get(next)?.push(...above);
+  }
+}
+function ownerOf(line, anchors, align) {
+  if (align === 'top') return lastOf(anchors, anchor => anchor.row.bottom >= line.row.bottom) ?? anchors[0];
+  if (align === 'bottom') return anchors.find(anchor => anchor.row.bottom <= line.row.bottom) ?? anchors.at(-1) ?? anchors[0];
+  return nearest(line, anchors);
+}
+/** A row's text per column: its lines top to bottom, each column's pieces joined by spaces. */
+function joinLines(group, width) {
+  const ordered = [...group].sort((a, b) => b.row.top - a.row.top);
+  return Array.from({
+    length: width
+  }, (_, column) => ordered.map(line => line.values[column]).filter(value => value !== '').join(' '));
+}
+/** Header cells that overlap horizontally (a header on two lines) are one header. */
+function headerCells(row) {
+  const merged = [];
+  for (const cell of row.cells) {
+    const previous = merged.at(-1);
+    if (previous !== undefined && cell.x < previous.x + previous.width) {
+      previous.text = `${previous.text} ${cell.text}`;
+      previous.width = Math.max(previous.x + previous.width, cell.x + cell.width) - previous.x;
+    } else {
+      merged.push({
+        ...cell
+      });
+    }
+  }
+  return merged;
+}
+function bandsOf(body, headers) {
+  const cells = body.flatMap(row => row.cells);
+  if (cells.length === 0 || headers.length === 0) return [];
+  const tolerance = Math.max(3, median(cells.map(cell => cell.height)) * 0.6);
+  const edges = cells.map(cell => cell.x).sort((a, b) => a - b);
+  const starts = [];
+  for (const edge of edges) if (starts.length === 0 || edge - (starts.at(-1) ?? 0) > tolerance) starts.push(edge);
+  const spans = starts.map((start, index) => {
+    const next = starts[index + 1] ?? Infinity;
+    // A band spans what its cells cover, not the gap up to the next band.
+    const right = Math.max(...cells.filter(cell => cell.x >= start - 0.5 && cell.x < next - 0.5).map(cell => cell.x + cell.width));
+    return {
+      start,
+      end: Math.min(next, right)
+    };
+  });
+  const columns = assignColumns(spans, headers);
+  return spans.map((span, index) => ({
+    start: span.start,
+    column: columns[index]
+  }));
+}
+/**
+ * Maps bands to headers, left to right: columns never cross, so the mapping
+ * is monotone. Among monotone mappings it first uses as many headers as it
+ * can (a table with as many bands as headers maps one to one), then prefers
+ * the one where bands overlap their header most, or sit nearest to it. The
+ * leftover choice is which neighbouring bands share a header: one header
+ * over two columns, or a column whose cells start at two edges.
+ *
+ * @param spans - The bands, left to right.
+ * @param headers - The header cells, left to right.
+ * @returns The header index of each band.
+ */
+function assignColumns(spans, headers) {
+  const affinity = (span, header) => {
+    const overlap = overlapOf(header, span);
+    return overlap > 0 ? overlap : -Math.max(0, header.x - span.end, span.start - (header.x + header.width));
+  };
+  const better = (a, b) => b === undefined || a.used > b.used || a.used === b.used && a.affinity > b.affinity;
+  const table = [headers.map(header => ({
+    used: 1,
+    affinity: affinity(spans[0], header),
+    previous: -1
+  }))];
+  for (const span of spans.slice(1)) {
+    const last = table.at(-1) ?? [];
+    table.push(headers.map((header, column) => {
+      let best;
+      for (let from = 0; from <= column; from += 1) {
+        const candidate = {
+          used: last[from].used + (from === column ? 0 : 1),
+          affinity: last[from].affinity + affinity(span, header),
+          previous: from
+        };
+        if (better(candidate, best)) best = candidate;
+      }
+      return best ?? {
+        used: 0,
+        affinity: -Infinity,
+        previous: 0
+      };
+    }));
+  }
+  const last = table.at(-1) ?? [];
+  let column = last.reduce((best, score, index) => better(score, last[best]) ? index : best, 0);
+  const columns = [];
+  for (let index = table.length - 1; index >= 0; index -= 1) {
+    columns.unshift(column);
+    column = table[index][column].previous;
+  }
+  return columns;
+}
+function valuesOf(row, bands, width) {
+  const values = Array.from({
+    length: width
+  }, () => '');
+  const ordered = [...row.cells].sort((a, b) => b.y - a.y || a.x - b.x);
+  for (const cell of ordered) {
+    const band = lastOf(bands, candidate => candidate.start <= cell.x + 0.5) ?? bands[0];
+    if (band === undefined) continue;
+    values[band.column] = joinText(values[band.column], cell.text);
+  }
+  return values;
+}
+function named(values, headers, columns) {
+  if (columns === undefined) return Object.fromEntries(headers.map((header, index) => [header.text, values[index]]));
+  const record = {};
+  for (const [key, pattern] of Object.entries(columns)) {
+    const index = headers.findIndex(header => pattern.test(header.text));
+    if (index !== -1) record[key] = values[index];
+  }
+  return record;
+}
+/** Distances closer than this, in points, are a tie. */
+const TIE = 1;
+/**
+ * The anchor a line belongs to: the nearest by vertical gap. On a tie (evenly
+ * spaced lines) the anchor below wins: text reads top down, so a wrapped
+ * cell's first line comes before the row it belongs to.
+ */
+function nearest(line, candidates) {
+  const row = line.row;
+  const gap = candidate => Math.max(0, candidate.bottom - row.top, row.bottom - candidate.top);
+  const [first, ...rest] = candidates;
+  if (first === undefined) throw new Error('no row to attach a line to');
+  let best = first;
+  for (const candidate of rest) {
+    const difference = gap(candidate.row) - gap(best.row);
+    if (difference < -TIE || Math.abs(difference) <= TIE && candidate.row.bottom < best.row.bottom) best = candidate;
+  }
+  return best;
+}
+function overlapOf(header, band) {
+  return Math.max(0, Math.min(header.x + header.width, band.end) - Math.max(header.x, band.start));
+}
+function plain(row) {
+  return row.cells.map(cell => cell.text).join(' ');
+}
+function joinText(first, second) {
+  return first === '' ? second : second === '' ? first : `${first} ${second}`;
+}
+function median(values) {
+  const ordered = [...values].sort((a, b) => a - b);
+  return ordered[Math.floor(ordered.length / 2)] ?? 0;
+}
+
 /**
  * Runs an `extract` step against a static document: the value bound under
  * `from`, else the scope's current document. `css` reads HTML, `jsonpath`
- * reads JSON; `xpath` needs a live page and is refused here.
+ * reads JSON (or a read PDF's rows), `table` reads a PDF's tables, `regex`
+ * reads any document as text; `xpath` needs a live page and is refused here.
  *
  * A `jsonpath` extract whose `from` is text parses that text as JSON, and a
  * list of texts (every `<script type="application/ld+json">` of a page) becomes
@@ -2128,8 +2563,14 @@ function extractFromDocument(step, scope) {
   switch (step.kind) {
     case 'jsonpath':
       {
-        if (document.kind !== 'json') throw new Error(`jsonpath needs a JSON document; the current document is ${document.kind}`);
-        values = selectJson(document.data, selector).map(node => takeFromJson(node, take));
+        if (document.kind !== 'json' && document.kind !== 'pdf') throw new Error(`jsonpath needs a JSON document; the current document is ${document.kind}`);
+        values = selectJson(document.kind === 'pdf' ? document : document.data, selector).map(node => takeFromJson(node, take));
+        break;
+      }
+    case 'table':
+      {
+        if (document.kind !== 'pdf') throw new Error(`table reads a PDF; the current document is ${document.kind} (request it with "as": "pdf")`);
+        values = findTables(document, tableQuery(step, selector));
         break;
       }
     case 'css':
@@ -2166,10 +2607,34 @@ function extractFromDocument(step, scope) {
 function renderSelector(selector, scope) {
   return hasPlaceholder(selector) ? renderText(selector, path => scope.lookup(path)) : selector;
 }
-/** The text a regex extract reads: markup, text, or JSON re-serialised (a list of texts joined by newlines). */
+/**
+ * A table extract's query: the selector matches the header row, the other
+ * patterns come from the step; all case-insensitive, since PDFs capitalise
+ * headings freely.
+ */
+function tableQuery(step, selector) {
+  const columns = step.columns === undefined ? undefined : Object.fromEntries(Object.entries(step.columns).map(([key, pattern]) => [key, patternOf(pattern, `columns.${key}`)]));
+  return {
+    header: patternOf(selector, 'selector'),
+    until: step.until === undefined ? undefined : patternOf(step.until, 'until'),
+    columns,
+    align: step.align
+  };
+}
+function patternOf(source, where) {
+  try {
+    return new RegExp(source, 'i');
+  } catch (error) {
+    throw new Error(`${where}: invalid pattern ${source} (${error.message})`, {
+      cause: error
+    });
+  }
+}
+/** The text a regex extract reads: markup, text, a PDF's rows, or JSON re-serialised (a list of texts joined by newlines). */
 function textOf$1(document) {
   if (document.kind === 'html') return document.html;
   if (document.kind === 'text') return document.text;
+  if (document.kind === 'pdf') return pdfText(document);
   if (Array.isArray(document.data) && document.data.every(entry => typeof entry === 'string')) return document.data.join('\n');
   return typeof document.data === 'string' ? document.data : JSON.stringify(document.data);
 }
@@ -2181,6 +2646,8 @@ function documentFor(step, scope) {
   }
   const source = scope.get(step.from);
   if (source === undefined) throw new Error(`"${step.from}" is not bound`);
+  if (isPdfDocument(source)) return source;
+  if (step.kind === 'table') throw new Error(`"${step.from}" is not a PDF; request it with "as": "pdf"`);
   if (step.kind === 'regex') {
     if (typeof source === 'string') return {
       kind: 'text',
@@ -2263,6 +2730,7 @@ class HttpClient {
    * @throws HttpError for a 4xx or 5xx status.
    */
   async send(httpRequest) {
+    if (httpRequest.url.startsWith('file:')) return readLocalFile(httpRequest);
     const response = await this.context.fetch(httpRequest.url, {
       method: httpRequest.method ?? (httpRequest.body === undefined ? 'GET' : 'POST'),
       params: httpRequest.query,
@@ -2290,7 +2758,25 @@ class HttpClient {
 }
 async function readBody(response, as) {
   const kind = as ?? kindFromContentType(response.headers()['content-type'] ?? '');
-  const text = await response.text();
+  return parseBody(kind, await response.body(), response.url());
+}
+/**
+ * A `file:` URL, read from disk: a PDF or JSON a recipe gets from a folder
+ * instead of a server. The kind is `as`, else the file extension.
+ */
+async function readLocalFile(httpRequest) {
+  const path = fileURLToPath(httpRequest.url);
+  const bytes = await readFile(path);
+  return {
+    status: 200,
+    url: httpRequest.url,
+    headers: {},
+    body: await parseBody(httpRequest.as ?? kindFromExtension(extname(path)), bytes, httpRequest.url)
+  };
+}
+async function parseBody(kind, bytes, url) {
+  if (kind === 'pdf') return readPdf(bytes, url);
+  const text = new TextDecoder().decode(bytes);
   if (kind === 'json') {
     try {
       return {
@@ -2298,7 +2784,7 @@ async function readBody(response, as) {
         data: JSON.parse(text)
       };
     } catch (error) {
-      throw new Error(`${response.url()}: body is not JSON (${error.message})`, {
+      throw new Error(`${url}: body is not JSON (${error.message})`, {
         cause: error
       });
     }
@@ -2314,9 +2800,20 @@ async function readBody(response, as) {
 function kindFromContentType(contentType) {
   const type = contentType.toLowerCase();
   if (type.includes('json')) return 'json';
+  if (type.includes('pdf')) return 'pdf';
   if (type.includes('html') || type.includes('xml')) return 'html';
   return 'text';
 }
+function kindFromExtension(extension) {
+  const kinds = {
+    '.json': 'json',
+    '.pdf': 'pdf',
+    '.html': 'html',
+    '.htm': 'html',
+    '.xml': 'html'
+  };
+  return kinds[extension.toLowerCase()] ?? 'text';
+}
 
 /**
  * Sends a `request` step: renders its templates, waits for the gate's throttle,
@@ -2384,6 +2881,7 @@ async function sendRequest(step, scope, client, recipe, gate, events) {
 }
 function bodyText(body) {
   if (body.kind === 'json') return JSON.stringify(body.data);
+  if (body.kind === 'pdf') return pdfText(body);
   return body.kind === 'html' ? body.html : body.text;
 }
 function renderMap(map, lookup) {
@@ -2404,9 +2902,10 @@ function resolveUrl(target, base) {
     throw new Error(`"${target}" is not a URL${base === undefined || base === '' ? ' and no page is known to resolve it against' : ` and cannot be resolved against ${base}`}`);
   }
 }
-/** What a step id holds for a document: parsed JSON, or the markup / text. */
+/** What a step id holds for a document: parsed JSON, the read PDF, or the markup / text. */
 function documentValue(body) {
   if (body.kind === 'json') return body.data;
+  if (body.kind === 'pdf') return body;
   return body.kind === 'html' ? body.html : body.text;
 }
 
@@ -3489,6 +3988,7 @@ async function extractFromPage(step, page, scope) {
     extractFromDocument(step, scope);
     return;
   }
+  if (step.kind === 'table') throw new Error('table reads a PDF: fetch it with a request step in api mode, or extract "from" a PDF bound earlier');
   const rendered = renderSelector(step.selector, scope);
   const take = step.take ?? 'text';
   const raw = step.kind === 'regex' ? selectRegex(await page.content(), rendered) : await page.locator(step.kind === 'xpath' ? `xpath=${rendered}` : rendered).evaluateAll(readAll, take);
@@ -4051,6 +4551,7 @@ async function runInputRecipe(input, output, deps) {
     rejected: 0,
     duplicates: 0,
     skipped: 0,
+    stepsSkipped: 0,
     pages: 0,
     durationMs: 0
   };
@@ -4061,6 +4562,7 @@ async function runInputRecipe(input, output, deps) {
   let chain = Promise.resolve();
   const unsubscribe = deps.events.subscribe(event => {
     if (event.type === 'page:visit' && event.recipeId === input.id) report.pages += 1;
+    if (event.type === 'step:skip' && event.recipeId === input.id) report.stepsSkipped += 1;
   });
   deps.events.emit({
     type: 'recipe:start',
@@ -4118,6 +4620,7 @@ async function runInputRecipe(input, output, deps) {
       rejected: report.rejected,
       duplicates: report.duplicates,
       skipped: report.skipped,
+      stepsSkipped: report.stepsSkipped,
       pages: report.pages,
       durationMs: report.durationMs,
       error: report.error
@@ -4371,10 +4874,12 @@ function createCrawler(options = {}) {
 
 /** The closed vocabularies a recipe file can use. Each is a `readonly` tuple so zod and TypeScript share it. */
 const CRAWL_MODES = ['web', 'api'];
-const SELECTOR_KINDS = ['css', 'xpath', 'jsonpath', 'regex'];
+const SELECTOR_KINDS = ['css', 'xpath', 'jsonpath', 'regex', 'table'];
 /** `take` also accepts `attr:<name>`, which is validated by pattern rather than listed. */
 const TAKE_KINDS = ['text', 'html', 'value', 'json'];
-const BODY_KINDS = ['json', 'html', 'text'];
+const BODY_KINDS = ['json', 'html', 'text', 'pdf'];
+/** How a PDF table aligns a row's values against a cell wrapped over several lines. */
+const TABLE_ALIGNS = ['auto', 'top', 'center', 'bottom'];
 const FIELD_TYPES = ['string', 'number', 'integer', 'boolean', 'date', 'datetime', 'currency', 'url', 'enum', 'array', 'object', 'json'];
 const MISSING_POLICIES = ['fail', 'skip-record', 'null', 'default'];
 const RECIPE_MISSING_POLICIES = ['fail', 'skip-record', 'null'];
@@ -4551,6 +5056,7 @@ const requestStep = z.strictObject({
   body: z.unknown().optional(),
   as: z.enum(BODY_KINDS).optional()
 });
+const tableOnly = ['columns', 'until', 'align'];
 const extractStep = z.strictObject({
   ...base,
   type: z.literal('extract'),
@@ -4558,7 +5064,20 @@ const extractStep = z.strictObject({
   kind: z.enum(SELECTOR_KINDS),
   take: takeKindSchema.optional(),
   many: z.boolean().optional(),
-  from: stepId.optional()
+  from: stepId.optional(),
+  columns: stringMap.optional(),
+  until: z.string().min(1).optional(),
+  align: z.enum(TABLE_ALIGNS).optional()
+}).check(context => {
+  if (context.value.kind === 'table') return;
+  for (const key of tableOnly) {
+    if (context.value[key] !== undefined) context.issues.push({
+      code: 'custom',
+      input: context.value,
+      path: [key],
+      message: `"${key}" belongs to kind "table"`
+    });
+  }
 });
 const assignStep = z.strictObject({
   ...base,
@@ -5136,6 +5655,7 @@ function walkStep(step, at, mode, known, report, state) {
     known.add(step.id);
   }
   if (step.type === 'extract' && step.from !== undefined && !known.has(step.from)) report(`${at}.from`, `"${step.from}" is not a known id`);
+  if (mode === 'web' && state.bootstrap !== true && step.type === 'extract' && step.kind === 'table' && step.from === undefined) report(at, 'a "table" extract reads a PDF, which a web page is not: fetch the PDF with a request step in an api recipe, or extract "from" a PDF bound earlier');
   if (step.type === 'collect' && !known.has(step.into)) report(`${at}.into`, `"${step.into}" is not a known id: set it to [] before the loop that collects into it`);
   const nested = () => ({
     ...state,
@@ -5298,5 +5818,5 @@ function isSameOutput(document, output) {
   return recipeKindOf(document.content) === 'output' && document.content.id === output.id;
 }
 
-export { ACCESS_PRESETS, AccessBroker, AccessConfigError, BrowserClient, BrowserSession, HttpClient, HttpError, MappingFailedError, RecipeBindingError, RecipeSet, RecipeValidationError, RecordRejectedError, StepFailure, TransformError, UnknownHookError, accessConfigJsonSchema, accessConfigSchema, bindRecipeSet, createCrawler, inputRecipeJsonSchema, inputRecipeSchema, jsonLinesSink, loadAccessConfig, loadRecipeSet, loadRecipes, memorySink, outputRecipeJsonSchema, outputRecipeSchema, parseInputRecipe, parseOutputRecipe, readRecipeSource, traceLine, tryParseJson, validateBinding };
+export { ACCESS_PRESETS, AccessBroker, AccessConfigError, BrowserClient, BrowserSession, HttpClient, HttpError, MappingFailedError, PdfReadError, RecipeBindingError, RecipeSet, RecipeValidationError, RecordRejectedError, StepFailure, TransformError, UnknownHookError, accessConfigJsonSchema, accessConfigSchema, bindRecipeSet, createCrawler, findTables, inputRecipeJsonSchema, inputRecipeSchema, jsonLinesSink, loadAccessConfig, loadRecipeSet, loadRecipes, memorySink, outputRecipeJsonSchema, outputRecipeSchema, parseInputRecipe, parseOutputRecipe, pdfText, readPdf, readRecipeSource, traceLine, tryParseJson, validateBinding };
 //# sourceMappingURL=index.esm.js.map

package/dist/src/api-steps/extract-from-document.use-case.d.ts

@@ -3,7 +3,8 @@ import type { ExtractStep } from '../recipe-schema/index.js';
 /**
  * Runs an `extract` step against a static document: the value bound under
  * `from`, else the scope's current document. `css` reads HTML, `jsonpath`
- * reads JSON; `xpath` needs a live page and is refused here.
+ * reads JSON (or a read PDF's rows), `table` reads a PDF's tables, `regex`
+ * reads any document as text; `xpath` needs a live page and is refused here.
  *
  * A `jsonpath` extract whose `from` is text parses that text as JSON, and a
  * list of texts (every `<script type="application/ld+json">` of a page) becomes

package/dist/src/api-steps/send-request.use-case.d.ts

@@ -17,6 +17,6 @@ import type { RunGate } from '../step-flow/index.js';
  * @throws BlockedError when the response is a block; HttpError for any other 4xx/5xx.
  */
 export declare function sendRequest(step: RequestStep, scope: ExtractionScope, client: HttpSender, recipe: InputRecipe, gate: RunGate, events: EventBus): Promise<void>;
-/** What a step id holds for a document: parsed JSON, or the markup / text. */
+/** What a step id holds for a document: parsed JSON, the read PDF, or the markup / text. */
 export declare function documentValue(body: HttpBody): unknown;
 //# sourceMappingURL=send-request.use-case.d.ts.map

package/dist/src/crawl-events/crawl-event.contract.d.ts

@@ -16,6 +16,7 @@ export type CrawlEvent = (Base & {
     rejected: number;
     duplicates: number;
     skipped: number;
+    stepsSkipped?: number;
     pages: number;
     durationMs: number;
     error?: string;

package/dist/src/crawl-execution/crawl-report.model.d.ts

@@ -8,6 +8,8 @@ export interface RecipeReport {
     duplicates: number;
     /** Records a resumed run found in the sink already. */
     skipped: number;
+    /** Steps whose `onError: skip` policy swallowed a failure: a check that found nothing, a value a page lacked. */
+    stepsSkipped: number;
     pages: number;
     durationMs: number;
     /** Set when the recipe stopped on a failure. */

package/dist/src/extraction-scope/extraction-scope.model.d.ts

@@ -7,8 +7,9 @@
  * Page state (the current URL, page number and the document `extract` reads by
  * default) is scope state too, bound in the innermost scope that navigated.
  */
+import type { PdfDocument } from '../pdf-document/index.js';
 /** A fetched or rendered document a later `extract` can read. */
-export type ScopeDocument = {
+export type ScopeDocument = PdfDocument | {
     kind: 'json';
     data: unknown;
 } | {

package/dist/src/http-session/http-response.contract.d.ts

@@ -1,3 +1,4 @@
+import type { PdfDocument } from '../pdf-document/index.js';
 import type { BodyKind, HttpMethod } from '../recipe-schema/index.js';
 /** One HTTP request as the api runner sends it, templates already rendered. */
 export interface HttpRequest {
@@ -11,7 +12,7 @@ export interface HttpRequest {
     timeoutMs?: number;
 }
 /** A parsed response body. Structurally the same as a scope document, on purpose. */
-export type HttpBody = {
+export type HttpBody = PdfDocument | {
     kind: 'json';
     data: unknown;
 } | {

package/dist/src/index.d.ts

@@ -19,6 +19,8 @@ export { BrowserClient, BrowserSession } from './browser-session/index.js';
 export { HttpClient, HttpError } from './http-session/index.js';
 export type { HttpClientOptions, HttpRequest, HttpResponse, HttpBody } from './http-session/index.js';
 export { tryParseJson } from './selection/index.js';
+export { readPdf, PdfReadError, findTables, pdfText } from './pdf-document/index.js';
+export type { PdfDocument, PdfPage, PdfRow, PdfCell, PdfTable, TableQuery, TableAlign } from './pdf-document/index.js';
 export { StepFailure } from './step-flow/index.js';
 export { TransformError } from './transformation/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/pdf-document/index.d.ts

@@ -0,0 +1,7 @@
+export { readPdf, PdfReadError } from './read-pdf.client.js';
+export { pdfText, isPdfDocument } from './pdf-document.model.js';
+export type { PdfDocument, PdfPage, PdfRow, PdfCell, PositionedText } from './pdf-document.model.js';
+export { assembleRows } from './row-assembly.algorithm.js';
+export { findTables } from './pdf-table.algorithm.js';
+export type { PdfTable, TableQuery, TableAlign } from './pdf-table.algorithm.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/pdf-document/pdf-document.model.d.ts

@@ -0,0 +1,51 @@
+/** A run of text at a position on a page, in PDF points, y growing upwards from the bottom edge. */
+export interface PositionedText {
+    x: number;
+    /** The baseline. */
+    y: number;
+    width: number;
+    /** The font size: the text spans roughly `y` to `y + height`. */
+    height: number;
+    text: string;
+}
+/** One or more text runs that sit side by side on one baseline: a table cell, a label, a value. */
+export type PdfCell = PositionedText;
+/** Cells whose vertical extents overlap: one visual line, which in a table is one row. */
+export interface PdfRow {
+    /** The top of the highest cell. */
+    top: number;
+    /** The baseline of the lowest cell. */
+    bottom: number;
+    /** The cells, left to right. */
+    cells: PdfCell[];
+    /** The cells' texts joined by a tab, left to right. */
+    text: string;
+}
+export interface PdfPage {
+    number: number;
+    width: number;
+    height: number;
+    /** Top to bottom. Empty on a page with no text layer (a scan). */
+    rows: PdfRow[];
+}
+/** A PDF read into rows of positioned text: what `extract` works on. */
+export interface PdfDocument {
+    kind: 'pdf';
+    pages: PdfPage[];
+}
+/**
+ * The text a `regex` extract reads: one line per row (cells separated by a
+ * tab), pages separated by a blank line.
+ *
+ * @param document - The PDF.
+ * @returns The text.
+ */
+export declare function pdfText(document: PdfDocument): string;
+/**
+ * Whether a value bound in scope is a read PDF (so `extract … from` can take it).
+ *
+ * @param value - Anything.
+ * @returns Whether it is a {@link PdfDocument}.
+ */
+export declare function isPdfDocument(value: unknown): value is PdfDocument;
+//# sourceMappingURL=pdf-document.model.d.ts.map

package/dist/src/pdf-document/pdf-table.algorithm.d.ts

@@ -0,0 +1,43 @@
+import type { PdfDocument } from './pdf-document.model.js';
+/** What a table extract looks for. */
+export interface TableQuery {
+    /** Matches a table's header row, its cells joined by spaces (`^MODELLI`). */
+    header: RegExp;
+    /** Matches the row that ends a table (`^NOTA BENE`); a table also ends at the next header or the page's end. */
+    until?: RegExp;
+    /** Output key -> a pattern for the header cell of that column; unmatched columns are dropped. Without it, the header texts are the keys. */
+    columns?: Record<string, RegExp>;
+    /** How the table aligns a row's values against a cell wrapped over several lines; `auto` (the default) infers it. */
+    align?: TableAlign;
+}
+/** Where a row's values sit against a cell wrapped over several lines. */
+export type TableAlign = 'auto' | 'top' | 'center' | 'bottom';
+/** One table found in a PDF. */
+export interface PdfTable {
+    page: number;
+    /** The first header cell: the table's name when it has one (`MODELLI FIAT`). */
+    title: string;
+    /** The header cells, left to right. */
+    header: string[];
+    /** One object per row, keyed by column; a cell's lines are joined by spaces. */
+    rows: Record<string, string>[];
+}
+/**
+ * Finds every table whose header row matches, and reads its rows by column.
+ *
+ * Columns come from the body, not the header: a header is often centred over
+ * a column whose cells are left-aligned, so the header's position says little
+ * about where the column starts. The left edges of the body cells cluster into
+ * bands; each band belongs to the header cell that overlaps it most, or the
+ * nearest one when none does. A header spanning two columns reads both.
+ *
+ * A cell wrapped over several lines (a long name, a note, a list of versions)
+ * spreads one row over several lines, its values often centred beside it:
+ * the lines are regrouped into rows around the lines that carry values.
+ *
+ * @param document - The PDF.
+ * @param query - Which tables, and how to name their columns.
+ * @returns The tables, in page order.
+ */
+export declare function findTables(document: PdfDocument, query: TableQuery): PdfTable[];
+//# sourceMappingURL=pdf-table.algorithm.d.ts.map

package/dist/src/pdf-document/read-pdf.client.d.ts

@@ -0,0 +1,18 @@
+import type { PdfDocument } from './pdf-document.model.js';
+/** A PDF that cannot be read: not a PDF, encrypted, or with no text to read. */
+export declare class PdfReadError extends Error {
+    readonly name = "PdfReadError";
+}
+/**
+ * Reads a PDF's text layer into rows of positioned cells, with pdf.js. pdf.js
+ * is imported on first use, so recipes that never read a PDF never load it.
+ * Only text is read: no page is rendered, no script runs, no font is loaded.
+ *
+ * @param bytes - The file.
+ * @param source - Where it came from, for messages.
+ * @returns The document.
+ * @throws PdfReadError when the bytes are not a readable PDF, or no page has a
+ * text layer (a scan: OCR is not supported).
+ */
+export declare function readPdf(bytes: Uint8Array, source?: string): Promise<PdfDocument>;
+//# sourceMappingURL=read-pdf.client.d.ts.map

package/dist/src/pdf-document/row-assembly.algorithm.d.ts

@@ -0,0 +1,17 @@
+import type { PdfRow, PositionedText } from './pdf-document.model.js';
+/**
+ * Turns a page's text runs into rows of cells, top to bottom.
+ *
+ * Runs on one baseline that nearly touch become one cell. Cells whose vertical
+ * extents overlap become one row, even when their baselines differ: a table
+ * that centres its cells vertically puts a one-line value a few points above
+ * or below its two-line label, and a row built from equal baselines would pair
+ * the value with the wrong label.
+ *
+ * @param runs - The page's text runs, in any order.
+ * @returns The rows.
+ */
+export declare function assembleRows(runs: readonly PositionedText[]): PdfRow[];
+/** `Array#findLast`, which the es2022 library does not declare. */
+export declare function lastOf<T>(items: readonly T[], test: (item: T) => boolean): T | undefined;
+//# sourceMappingURL=row-assembly.algorithm.d.ts.map

package/dist/src/recipe-schema/recipe-kind.enum.d.ts

@@ -1,10 +1,12 @@
 /** The closed vocabularies a recipe file can use. Each is a `readonly` tuple so zod and TypeScript share it. */
 export declare const RECIPE_KINDS: readonly ["input", "output"];
 export declare const CRAWL_MODES: readonly ["web", "api"];
-export declare const SELECTOR_KINDS: readonly ["css", "xpath", "jsonpath", "regex"];
+export declare const SELECTOR_KINDS: readonly ["css", "xpath", "jsonpath", "regex", "table"];
 /** `take` also accepts `attr:<name>`, which is validated by pattern rather than listed. */
 export declare const TAKE_KINDS: readonly ["text", "html", "value", "json"];
-export declare const BODY_KINDS: readonly ["json", "html", "text"];
+export declare const BODY_KINDS: readonly ["json", "html", "text", "pdf"];
+/** How a PDF table aligns a row's values against a cell wrapped over several lines. */
+export declare const TABLE_ALIGNS: readonly ["auto", "top", "center", "bottom"];
 export declare const FIELD_TYPES: readonly ["string", "number", "integer", "boolean", "date", "datetime", "currency", "url", "enum", "array", "object", "json"];
 export declare const MISSING_POLICIES: readonly ["fail", "skip-record", "null", "default"];
 export declare const RECIPE_MISSING_POLICIES: readonly ["fail", "skip-record", "null"];
@@ -21,6 +23,7 @@ export type RecipeKind = typeof RECIPE_KINDS[number];
 export type CrawlMode = typeof CRAWL_MODES[number];
 export type SelectorKind = typeof SELECTOR_KINDS[number];
 export type BodyKind = typeof BODY_KINDS[number];
+export type TableAlign = typeof TABLE_ALIGNS[number];
 export type FieldType = typeof FIELD_TYPES[number];
 export type MissingPolicy = typeof MISSING_POLICIES[number];
 export type RecipeMissingPolicy = typeof RECIPE_MISSING_POLICIES[number];

package/dist/src/recipe-schema/step.contract.d.ts

@@ -1,6 +1,6 @@
 import { z } from 'zod';
 import { TAKE_KINDS } from './recipe-kind.enum.js';
-import type { BodyKind, HttpMethod, SelectorKind, WaitUntil } from './recipe-kind.enum.js';
+import type { BodyKind, HttpMethod, SelectorKind, TableAlign, WaitUntil } from './recipe-kind.enum.js';
 /** What to do when a step fails. Resolved step -> recipe -> `fail`. */
 export type ErrorPolicy = {
     policy: 'fail';
@@ -95,6 +95,12 @@ export interface ExtractStep extends StepBaseFields {
     many?: boolean;
     /** Id of a document or fragment to read instead of the current document. */
     from?: string;
+    /** `table` only: output key -> a pattern (case-insensitive) for that column's header cell. */
+    columns?: Record<string, string>;
+    /** `table` only: a pattern (case-insensitive) for the row that ends a table. */
+    until?: string;
+    /** `table` only: how a row's values sit against a cell wrapped over several lines; default `auto`. */
+    align?: TableAlign;
 }
 export interface SetStep extends StepBaseFields {
     type: 'set';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/core",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",
@@ -23,6 +23,7 @@
     "cheerio": "^1.2.0",
     "domhandler": "^6.0.1",
     "jsonpath-plus": "^10.4.0",
+    "pdfjs-dist": "^6.3.289",
     "playwright": "^1.63.0",
     "zod": "^4.6.5"
   },