@opencraw/cli 0.1.0 → 0.1.2

package/README.md

@@ -43,7 +43,7 @@ input recipes) and reports what happened.
 | `--dry-run` | One record per input recipe, printed with the scope it was mapped from — for checking a recipe under construction without a full run. |
 | `--trace` | Print the crawl trace to stderr. |
 | `--headed` | Show the browser instead of running headless. |
-| `--hooks <file>` | A JavaScript module whose default export is `{ name: function }`: the hooks the recipes call (or `OPENCRAW_HOOKS`). It runs as your code; load only files you trust. See [§6 of the authoring guide](../../docs/recipes/authoring.md#6-hooks). |
+| `--plugins <file>` (or `--hooks`) | A plugins module: named exports `hooks` (the recipes' hook steps and transforms), `accessPlugins` (for `{ kind: "plugin" }` access profiles) and `captchaSolvers`. A module whose default export is `{ name: function }` is read as hooks alone. `OPENCRAW_PLUGINS` / `OPENCRAW_HOOKS` when not given; `probe` reads it too. It runs as your code; load only files you trust. See [§6 of the authoring guide](../../docs/recipes/authoring.md#6-hooks). |
 
 ```sh
 opencraw run recipes/ --out out/products.jsonl --trace
@@ -62,6 +62,22 @@ opencraw probe https://example.com/product/1
 opencraw probe https://example.com/configurator --browser
 ```
 
+A PDF, a spreadsheet, a CSV or a presentation (a URL served with its content type, or a local `.pdf`,
+`.xlsx`, `.csv`, `.tsv` or `.pptx` path) is read instead: `probe` lists its pages, sheets or slides, its first
+rows, and every row that looks like a table header, with the `selector` a `table` extract needs. For a CSV it
+also names the encoding and the delimiter it detected; for a presentation, its charts and the slides whose
+text boxes look like a table. JSON, JSON Lines and YAML show their structure instead, and every list of
+records with the `jsonpath` that walks it. An HTML page also lists its tables' header rows, and Markdown
+(a `.md` URL is read as Markdown even when served as `text/plain`) its sections and front matter keys.
+
+```sh
+opencraw probe https://example.com/price-list.pdf
+opencraw probe ./sheets/september.pdf
+opencraw probe ./exports/listino.csv
+opencraw probe ./exports/incentivi.xlsx
+opencraw probe ./decks/incentivi.pptx
+```
+
 Use it before writing an input recipe, to find the shape a site's data actually takes (§9 of
 [the authoring guide](../../docs/recipes/authoring.md)).
 

package/dist/index.esm.js

@@ -1,8 +1,8 @@
 import { parseArgs } from 'node:util';
-import { loadAccessConfig, AccessBroker, HttpClient, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
-import { readFileSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
+import { loadAccessConfig, htmlTableSheets, AccessBroker, HttpClient, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
+import { readFileSync } from 'node:fs';
 
 const USAGE = `opencraw <command> [options]
 
@@ -19,8 +19,6 @@ Options for run
   --dry-run           One record per input, printed with the scope it was mapped from.
   --trace             Print the crawl trace to stderr.
   --headed            Show the browser.
-  --hooks <file>      A JavaScript module whose default export is { name: function }: the hooks
-                      the recipes call (or OPENCRAW_HOOKS). It runs as your code: trust it.
 
 Options for probe
   --browser           Also render the page in a browser and list the JSON it fetches.
@@ -32,6 +30,10 @@ Options for both
   --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).
   --access-profile <name>
                       The access profile to use when a recipe names none.
+  --plugins <file>    A JavaScript module exporting hooks (the recipes' hook steps and transforms),
+                      accessPlugins (for { kind: "plugin" } access profiles) and captchaSolvers
+                      (for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /
+                      OPENCRAW_HOOKS. It runs as your code: trust it.
   --help, --version`;
 const OPTIONS = {
   'out': {
@@ -59,6 +61,9 @@ const OPTIONS = {
   'hooks': {
     type: 'string'
   },
+  'plugins': {
+    type: 'string'
+  },
   'browser': {
     type: 'boolean'
   },
@@ -116,7 +121,8 @@ function parseArguments(argv, env = {}) {
     insecureTls: values['insecure-tls'] === true || env.OPENCRAW_INSECURE_TLS === '1',
     userAgent: values['user-agent'],
     access: values.access ?? (env.OPENCRAW_ACCESS === '' ? undefined : env.OPENCRAW_ACCESS),
-    accessProfile: values['access-profile']
+    accessProfile: values['access-profile'],
+    plugins: values.plugins ?? values.hooks ?? nonEmpty(env.OPENCRAW_PLUGINS) ?? nonEmpty(env.OPENCRAW_HOOKS)
   };
   switch (name) {
     case undefined:
@@ -148,7 +154,6 @@ function parseArguments(argv, env = {}) {
           dryRun: values['dry-run'] === true,
           only: values.only ?? [],
           headed: values.headed === true,
-          hooks: values.hooks ?? (env.OPENCRAW_HOOKS === '' ? undefined : env.OPENCRAW_HOOKS),
           options
         };
       }
@@ -168,6 +173,9 @@ function parseArguments(argv, env = {}) {
       }
   }
 }
+function nonEmpty(value) {
+  return value === '' ? undefined : value;
+}
 
 /**
  * The access config a command runs with: the file `--access` (or
@@ -193,6 +201,109 @@ async function resolveAccess(options) {
   };
 }
 
+/** The named exports that mark a module as a plugins module rather than a bare hooks module. */
+const PLUGIN_EXPORTS = ['hooks', 'accessPlugins', 'captchaSolvers'];
+/**
+ * Loads a plugins module: a JavaScript module whose named exports `hooks`
+ * (`{ name: function }`), `accessPlugins` (`[{ name, lease }]`) and
+ * `captchaSolvers` (`[{ name, solve }]`) give what recipes and access profiles
+ * refer to by name:
+ *
+ * ```js
+ * export const hooks = { positive: v => v > 0 }
+ * export const accessPlugins = [{ name: 'browserbase', lease: async () => ({ cdp: { endpoint } }) }]
+ * export const captchaSolvers = [{ name: 'capsolver', solve: async (challenge, { page }) => ({ status: 'solved' }) }]
+ * ```
+ *
+ * A module with none of those names is a bare hooks module, as before: its
+ * default export when that is an object of functions, else its named function
+ * exports.
+ *
+ * The module runs with the caller's privileges, like any code the caller
+ * imports: point this only at a file you trust.
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file.
+ * @param importModule - How to import it; `import()` unless a test says otherwise.
+ * @returns The hooks and plugins.
+ * @throws Error naming the file when it cannot be imported, provides nothing,
+ * or provides something that is not what its name promises.
+ */
+async function loadPlugins(path, importModule = importByUrl) {
+  const url = pathToFileURL(resolve(path)).href;
+  let exported;
+  try {
+    exported = await importModule(url);
+  } catch (error) {
+    throw new Error(`${path}: cannot load hooks (${error.message})`, {
+      cause: error
+    });
+  }
+  if (PLUGIN_EXPORTS.every(name => !Object.hasOwn(exported, name))) {
+    const hooks = hooksOf(hookExports(exported), path);
+    if (Object.keys(hooks).length === 0) throw new Error(`${path}: exports no hooks (export default { name: function } or named functions) and no plugins (export const hooks, accessPlugins, captchaSolvers)`);
+    return {
+      hooks,
+      accessPlugins: [],
+      captchaSolvers: []
+    };
+  }
+  return {
+    hooks: hooksOf(objectExport(exported.hooks, 'hooks', path), path),
+    accessPlugins: pluginsOf(exported.accessPlugins, 'accessPlugins', 'lease', path),
+    captchaSolvers: pluginsOf(exported.captchaSolvers, 'captchaSolvers', 'solve', path)
+  };
+}
+/**
+ * Loads the hooks of a hooks or plugins module (see {@link loadPlugins}).
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file.
+ * @param importModule - How to import it.
+ * @returns The hooks by name.
+ * @throws Error naming the file when it cannot be imported or exports no hooks.
+ */
+async function loadHooks(path, importModule = importByUrl) {
+  const {
+    hooks
+  } = await loadPlugins(path, importModule);
+  if (Object.keys(hooks).length === 0) throw new Error(`${path}: exports no hooks (export default { name: function } or named functions)`);
+  return hooks;
+}
+function hooksOf(candidates, path) {
+  const hooks = {};
+  for (const [name, value] of Object.entries(candidates)) {
+    if (typeof value !== 'function') throw new Error(`${path}: hook "${name}" is ${value === null ? 'null' : typeof value}, not a function`);
+    hooks[name] = value;
+  }
+  return hooks;
+}
+function objectExport(value, name, path) {
+  if (value === undefined) return {};
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) throw new Error(`${path}: "${name}" must be an object of functions`);
+  return value;
+}
+/** A list of `{ name, <method> }` plugins, checked. */
+function pluginsOf(value, exportName, method, path) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) throw new Error(`${path}: "${exportName}" must be an array of { name, ${method} }`);
+  const names = new Set();
+  for (const [index, plugin] of value.entries()) {
+    const entry = plugin;
+    if (typeof entry?.name !== 'string' || entry.name === '' || typeof entry[method] !== 'function') throw new Error(`${path}: ${exportName}[${index}] must be { name: string, ${method}: function }`);
+    if (names.has(entry.name)) throw new Error(`${path}: ${exportName} names "${entry.name}" twice`);
+    names.add(entry.name);
+  }
+  return value;
+}
+async function importByUrl(url) {
+  return await import(url);
+}
+function hookExports(exported) {
+  const fallback = exported.default;
+  if (typeof fallback === 'object' && fallback !== null && !Array.isArray(fallback)) return fallback;
+  const named = Object.entries(exported).filter(([name]) => name !== 'default' && name !== 'module.exports');
+  return Object.fromEntries(named);
+}
+
 /**
  * Finds where a fetched page's data lives: JSON-LD blocks, inline JSON objects
  * big enough to matter, URLs that look like a data endpoint, script hosts and
@@ -304,6 +415,302 @@ function tryParse(text) {
   }
 }
 
+/** Rows shown per sheet in a probe, at most. */
+const ROW_LIMIT$1 = 20;
+/** How far below a header row a probe looks for the first data row. */
+const LOOKAHEAD = 3;
+/**
+ * Summarises a workbook for someone writing a recipe: its sheets, what their
+ * rows look like and where the tables start, so a `table` extract's `sheet`,
+ * `selector` and columns can be read off the result instead of guessed.
+ *
+ * @param document - The read workbook.
+ * @returns The findings.
+ */
+function describeWorkbook(document) {
+  const visible = document.sheets.filter(sheet => sheet.hidden !== true);
+  return {
+    ...(document.csv !== undefined && {
+      csv: document.csv
+    }),
+    sheets: document.sheets.map(sheet => ({
+      name: sheet.name,
+      hidden: sheet.hidden === true,
+      rows: sheet.rows.length,
+      columns: Math.max(0, ...sheet.rows.map(row => row.length))
+    })),
+    rows: visible.flatMap(sheet => filledRows(sheet).slice(0, ROW_LIMIT$1).map(({
+      index,
+      cells
+    }) => ({
+      sheet: sheet.name,
+      row: index + 1,
+      text: cells.join(' | ')
+    }))),
+    headers: visible.flatMap(sheet => headersOf(sheet))
+  };
+}
+function headersOf(sheet) {
+  const rows = filledRows(sheet);
+  const merged = mergedRows(sheet);
+  return rows.flatMap(({
+    index,
+    cells
+  }, position) => {
+    if (cells.length < 2 || cells.some(cell => /\d/.test(cell))) return [];
+    if (rows.slice(position + 1, position + 1 + LOOKAHEAD).every(row => !(row.cells.length >= 2 && row.cells.some(cell => /\d/.test(cell))))) return [];
+    const header = {
+      sheet: sheet.name,
+      row: index + 1,
+      text: cells.join(' | '),
+      selector: `^${escape$1(cells[0])}`
+    };
+    return [merged.has(index) ? {
+      ...header,
+      hint: 'merged header cells: try "headerRows": 2'
+    } : header];
+  });
+}
+/** The non-empty rows, as their non-empty cells, whitespace collapsed. */
+function filledRows(sheet) {
+  const hidden = new Set(sheet.hiddenRows);
+  return sheet.rows.flatMap((row, index) => {
+    const cells = row.map(cell => String(cell).replaceAll(/\s+/g, ' ').trim()).filter(cell => cell !== '');
+    return cells.length === 0 || hidden.has(index) ? [] : [{
+      index,
+      cells
+    }];
+  });
+}
+/** Rows a merged range starts on. */
+function mergedRows(sheet) {
+  const rows = new Set();
+  const merges = sheet.merges ?? [];
+  for (const reference of merges) {
+    const match = /^\$?[A-Z]+\$?(\d+)/i.exec(reference);
+    if (match !== null) rows.add(Number(match[1]) - 1);
+  }
+  return rows;
+}
+function escape$1(text) {
+  return text.replaceAll(/[$()*+.?[\\\]^{|}]/g, String.raw`\$&`);
+}
+
+/** Short text boxes on one slide from which a probe suggests reading them as a table. */
+const GRID_BOXES = 6;
+const SHORT_TEXT = 40;
+/**
+ * Summarises a deck for someone writing a recipe: its slides, where the native
+ * tables start, what the charts hold, and which slides lay text boxes out as
+ * a table.
+ *
+ * @param document - The read deck.
+ * @returns The findings.
+ */
+function describeDeck(document) {
+  return {
+    width: document.width,
+    height: document.height,
+    slides: document.slides.map(slide => ({
+      number: slide.number,
+      title: slide.title ?? '',
+      hidden: slide.hidden,
+      shapes: slide.shapes.length,
+      tables: slide.tables.length,
+      charts: slide.charts.length
+    })),
+    headers: document.slides.flatMap(slide => describeWorkbook({
+      sheets: slide.tables
+    }).headers.map(({
+      sheet: _sheet,
+      row: _row,
+      ...header
+    }) => ({
+      slide: slide.number,
+      ...header
+    }))),
+    charts: document.slides.flatMap(slide => slide.charts.map(chart => ({
+      slide: slide.number,
+      type: chart.type,
+      ...(chart.title !== undefined && {
+        title: chart.title
+      }),
+      series: chart.series.map(series => ({
+        name: series.name,
+        points: series.values.length
+      }))
+    }))),
+    grids: document.slides.flatMap(slide => {
+      const boxes = slide.shapes.filter(shape => shape.placeholder === undefined && shape.text.length <= SHORT_TEXT).length;
+      return boxes >= GRID_BOXES ? [{
+        slide: slide.number,
+        title: slide.title ?? '',
+        boxes
+      }] : [];
+    })
+  };
+}
+
+const SECTION = /<section data-heading="([^"]*)" data-level="(\d)">/g;
+const FRONT_MATTER = /<script type="application\/json" data-front-matter>([\s\S]*?)<\/script>/;
+/**
+ * Summarises an HTML document for a `table` extract, and, when it is rendered
+ * Markdown, its outline and front matter.
+ *
+ * @param html - The document.
+ * @param markdown - Whether it was rendered from Markdown.
+ * @returns The findings.
+ */
+function describeHtml(html, markdown) {
+  const tables = htmlTableSheets(html);
+  const headers = describeWorkbook({
+    sheets: tables
+  }).headers.map(({
+    sheet,
+    row: _row,
+    ...header
+  }) => ({
+    table: sheet,
+    ...header
+  }));
+  if (!markdown) return {
+    tables: headers
+  };
+  const outline = Array.from(html.matchAll(SECTION), ([, heading, level]) => ({
+    level: Number(level),
+    heading: unescape(heading),
+    selector: `section[data-heading='${unescape(heading).replaceAll("'", String.raw`\'`)}' i]`
+  }));
+  const front = FRONT_MATTER.exec(html)?.[1];
+  const data = front === undefined ? undefined : JSON.parse(front);
+  return {
+    tables: headers,
+    outline,
+    frontMatter: typeof data === 'object' && data !== null && !Array.isArray(data) ? Object.keys(data) : []
+  };
+}
+function unescape(text) {
+  return text.replaceAll('&quot;', '"').replaceAll('&lt;', '<').replaceAll('&amp;', '&');
+}
+
+/** How deep a probe describes a JSON document. */
+const DEPTH = 4;
+/** Keys shown per object, at most. */
+const KEY_LIMIT = 25;
+/** Entries of a record list whose keys are compared. */
+const SAMPLE = 50;
+/** A sample value longer than this is cut. */
+const SAMPLE_CHARS = 40;
+/**
+ * Summarises JSON data for someone writing a recipe: its shape, and where its
+ * record lists are, so a `jsonpath` selector can be read off the result.
+ *
+ * @param data - The parsed document.
+ * @param format - What it was read as (`json`, `jsonl`, `yaml`).
+ * @returns The findings.
+ */
+function describeJson(data, format) {
+  const tree = [];
+  const lists = [];
+  walk(data, '$', 0, tree, lists);
+  return {
+    format,
+    type: typeOf(data),
+    tree,
+    lists: lists.sort((first, second) => second.length - first.length)
+  };
+}
+function walk(value, path, depth, tree, lists) {
+  if (Array.isArray(value)) {
+    const objects = value.filter(entry => isObject(entry));
+    if (objects.length >= 2 && objects.length === value.length) lists.push({
+      path: `${path}[*]`,
+      length: value.length,
+      keys: sharedKeys(objects.slice(0, SAMPLE))
+    });
+    if (depth > 0) tree.push(`${path}  ${typeOf(value)}`);
+    if (depth < DEPTH && value.length > 0) walk(value[0], `${path}[*]`, depth + 1, tree, lists);
+    return;
+  }
+  if (isObject(value)) {
+    if (depth > 0) tree.push(`${path}  object`);
+    if (depth >= DEPTH) return;
+    for (const [key, child] of Object.entries(value).slice(0, KEY_LIMIT)) walk(child, childPath(path, key), depth + 1, tree, lists);
+    return;
+  }
+  tree.push(`${path}  ${typeOf(value)}${sample(value)}`);
+}
+function typeOf(value) {
+  if (Array.isArray(value)) {
+    const kinds = [...new Set(value.slice(0, SAMPLE).map(entry => typeOf(entry).split(' ', 1)[0]))];
+    return `array (${value.length})${kinds.length === 0 ? '' : ` of ${kinds.join(' | ')}`}`;
+  }
+  if (value === null) return 'null';
+  return typeof value === 'object' ? 'object' : typeof value;
+}
+function sample(value) {
+  if (value === null || value === undefined) return '';
+  const text = JSON.stringify(value);
+  return ` ${text.length > SAMPLE_CHARS ? `${text.slice(0, SAMPLE_CHARS)}…` : text}`;
+}
+function sharedKeys(objects) {
+  const [first, ...rest] = objects;
+  return Object.keys(first).filter(key => rest.every(object => Object.hasOwn(object, key)));
+}
+function childPath(path, key) {
+  return /^[$A-Z_][\w$]*$/i.test(key) ? `${path}.${key}` : `${path}['${key.replaceAll("'", String.raw`\'`)}']`;
+}
+function isObject(value) {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/** Rows shown in a probe, at most. */
+const ROW_LIMIT = 80;
+/**
+ * Summarises a PDF for someone writing a recipe: what the rows look like, and
+ * where the tables start, so a `table` extract's selector and columns can be
+ * read off the result instead of guessed.
+ *
+ * @param document - The read PDF.
+ * @returns The findings.
+ */
+function describePdf(document) {
+  const rows = document.pages.flatMap(page => page.rows.map((row, index) => ({
+    page: page.number,
+    row,
+    next: page.rows.slice(index + 1, index + 3)
+  })));
+  const headers = rows.filter(({
+    row,
+    next
+  }) => row.cells.length >= 2 && row.cells.every(cell => !/\d/.test(cell.text)) && next.some(candidate => dataRow(candidate)));
+  return {
+    pages: document.pages.length,
+    rows: rows.slice(0, ROW_LIMIT).map(({
+      page,
+      row
+    }) => ({
+      page,
+      text: row.cells.map(cell => cell.text).join(' | ')
+    })),
+    headers: headers.map(({
+      page,
+      row
+    }) => ({
+      page,
+      text: row.cells.map(cell => cell.text).join(' | '),
+      selector: `^${escape(row.cells[0].text)}`
+    }))
+  };
+}
+/** A row of at least two cells, one with a digit: data, not a heading. */
+function dataRow(row) {
+  return row.cells.length >= 2 && row.cells.some(cell => /\d/.test(cell.text));
+}
+function escape(text) {
+  return text.replaceAll(/[$()*+.?[\\\]^{|}]/g, String.raw`\$&`);
+}
+
 /**
  * Renders a probe's findings as text.
  *
@@ -311,12 +718,73 @@ function tryParse(text) {
  * @param status - The HTTP status.
  * @param findings - What `findData` found.
  * @param observed - JSON responses a browser saw, when `--browser` was used.
+ * @param html - The tables (and, for Markdown, the outline and front matter) found.
  * @returns The report.
  */
-function probeReport(url, status, findings, observed = []) {
-  const sections = [section('JSON-LD blocks', findings.jsonLd.map(block => `  type=${block.types}, ${block.keys} keys`)), section('Inline JSON (candidates)', findings.inlineJson.map(block => `  ${block.where}, ${block.size} chars, keys: ${block.keys.slice(0, 8).join(', ')}`)), section('.json URLs referenced', findings.jsonUrls.map(url_ => `  ${url_}`)), section('Script hosts', findings.scriptHosts.map(host => `  ${host}`)), section('Links that look like an API', findings.apiLinks.map(link => `  ${link}`)), observed.length > 0 ? section('JSON responses observed in the browser', observed.map(entry => `  ${entry}`)) : ''];
+function probeReport(url, status, findings, observed = [], html) {
+  const sections = [section('Markdown front matter (in script[data-front-matter])', html?.frontMatter === undefined || html.frontMatter.length === 0 ? [] : [`  keys: ${html.frontMatter.join(', ')}`]), section('Markdown sections (a "css" selector for each)', (html?.outline ?? []).map(entry => `  ${'  '.repeat(entry.level - 1)}${entry.selector}`)), section('HTML tables (a "table" extract selector for each)', (html?.tables ?? []).map(table => `  ${table.table}  ${table.selector}${table.hint === undefined ? '' : `  (${table.hint})`}\n        ${table.text}`)), section('JSON-LD blocks', findings.jsonLd.map(block => `  type=${block.types}, ${block.keys} keys`)), section('Inline JSON (candidates)', findings.inlineJson.map(block => `  ${block.where}, ${block.size} chars, keys: ${block.keys.slice(0, 8).join(', ')}`)), section('.json URLs referenced', findings.jsonUrls.map(url_ => `  ${url_}`)), section('Script hosts', findings.scriptHosts.map(host => `  ${host}`)), section('Links that look like an API', findings.apiLinks.map(link => `  ${link}`)), observed.length > 0 ? section('JSON responses observed in the browser', observed.map(entry => `  ${entry}`)) : ''];
   return [`${url} (HTTP ${status})`, '', ...sections].filter(line => line !== '').join('\n');
 }
+/**
+ * Renders what a probe found in a PDF as text.
+ *
+ * @param url - The URL or file probed.
+ * @param pdf - What `describePdf` found.
+ * @returns The report.
+ */
+function pdfReport(url, pdf) {
+  return [`${url} (PDF, ${pdf.pages} page${pdf.pages === 1 ? '' : 's'})`, '', section('Likely table headers (a "table" extract selector for each)', pdf.headers.map(header => `  p${header.page}  ${header.selector}\n        ${header.text}`)), section('First rows (cells separated by " | ")', pdf.rows.map(row => `  p${row.page}  ${row.text}`))].filter(line => line !== '').join('\n');
+}
+/**
+ * Formats what a probe found in a workbook (a CSV or a spreadsheet): its sheets,
+ * the rows that look like table headers with a selector for each, and the first
+ * rows.
+ *
+ * @param url - The workbook's URL.
+ * @param workbook - The findings.
+ * @returns The report.
+ */
+function workbookReport(url, workbook) {
+  const read = workbook.csv === undefined ? 'workbook' : `CSV, ${workbook.csv.encoding}, delimited by ${delimiterName(workbook.csv.delimiter)}`;
+  return [`${url} (${read})`, '', section('Sheets', workbook.sheets.map(sheet => `  ${sheet.name}${sheet.hidden ? ' (hidden)' : ''}: ${sheet.rows} rows × ${sheet.columns} columns`)), section('Likely table headers (a "table" extract selector for each)', workbook.headers.map(header => `  ${header.sheet} r${header.row}  ${header.selector}${header.hint === undefined ? '' : `  (${header.hint})`}\n        ${header.text}`)), section('First rows (cells separated by " | ")', workbook.rows.map(row => `  ${row.sheet} r${row.row}  ${row.text}`))].filter(line => line !== '').join('\n');
+}
+/**
+ * Formats what a probe found in a deck (a presentation): its slides, the native
+ * tables' headers with a selector for each, its charts, and the slides whose
+ * text boxes look like a table.
+ *
+ * @param url - The deck's URL.
+ * @param deck - The findings.
+ * @returns The report.
+ */
+function deckReport(url, deck) {
+  return [`${url} (presentation, ${deck.slides.length} slide${deck.slides.length === 1 ? '' : 's'}, ${deck.width} × ${deck.height} pt)`, '', section('Slides', deck.slides.map(slide => `  ${slide.number}  ${slide.title === '' ? '(no title)' : slide.title}${slide.hidden ? ' (hidden)' : ''}: ${slide.shapes} text boxes, ${slide.tables} tables, ${slide.charts} charts`)), section('Likely table headers (a "table" extract selector for each)', deck.headers.map(header => `  slide ${header.slide}  ${header.selector}${header.hint === undefined ? '' : `  (${header.hint})`}\n        ${header.text}`)), section('Text boxes laid out as a table (a "table" extract with "shapes": true)', deck.grids.map(grid => `  slide ${grid.slide}  ${grid.title}: ${grid.boxes} short boxes`)), section('Charts (read with jsonpath: $.slides[*].charts[*].series[*])', deck.charts.map(chart => `  slide ${chart.slide}  ${chart.type}${chart.title === undefined ? '' : ` "${chart.title}"`}: ${chart.series.map(series => `${series.name} (${series.points})`).join(', ')}`))].filter(line => line !== '').join('\n');
+}
+/**
+ * Formats what a probe found in JSON data: its record lists with the path a
+ * `jsonpath` extract needs, then its structure.
+ *
+ * @param url - The document's URL.
+ * @param json - The findings.
+ * @returns The report.
+ */
+function jsonReport(url, json) {
+  const read = {
+    json: 'JSON',
+    jsonl: 'JSON Lines',
+    yaml: 'YAML'
+  }[json.format] ?? json.format;
+  return [`${url} (${read}: ${json.type})`, '', section('Record lists (a "jsonpath" selector for each, largest first)', json.lists.map(list => `  ${list.path}  ${list.length} entries\n        keys: ${list.keys.join(', ')}`)), section('Structure', json.tree.map(line => `  ${line}`))].filter(line => line !== '').join('\n');
+}
+function delimiterName(delimiter) {
+  const names = {
+    '\t': 'tabs',
+    ',': 'commas',
+    ';': 'semicolons',
+    '|': 'pipes'
+  };
+  return names[delimiter] ?? `"${delimiter}"`;
+}
 function section(title, rows) {
   if (rows.length === 0) return '';
   return [`${title}:`, ...rows, ''].join('\n');
@@ -332,13 +800,17 @@ const OBSERVE_MS = 4000;
  * involved: the cli's `probePage` and `@opencraw/mcp`'s probe tool both
  * build on this, one printing the result, the other returning it as data.
  *
- * @param url - The page to probe.
+ * A PDF, a spreadsheet, a CSV or a presentation (by content type, or a local
+ * path) is read instead: its rows, and the rows that look like table headers.
+ *
+ * @param url - The page to probe, or a local file path.
  * @param options - Browser path, TLS and user agent, plus whether to render.
  * @returns What was found.
  * @throws Error when the fetch itself fails.
  */
 async function probeUrl(url, options) {
-  const lease = await new AccessBroker(await resolveAccess(options)).lease({
+  const plugins = options.plugins === undefined ? undefined : await loadPlugins(options.plugins);
+  const lease = await new AccessBroker(await resolveAccess(options), plugins?.accessPlugins).lease({
     recipeId: 'probe'
   });
   if (lease.cdp !== undefined) throw new Error(`access profile "${lease.profile}" is a remote browser; probe fetches over HTTP and needs a proxy profile`);
@@ -349,17 +821,57 @@ async function probeUrl(url, options) {
     headers: lease.headers
   });
   try {
+    const target = /^[a-z][\w+.-]+:/i.test(url) ? url : pathToFileURL(resolve(url)).href;
+    // GitHub raw and CDNs serve Markdown as text/plain: a .md URL is read as Markdown.
+    const markdown = /\.(?:md|markdown)$/i.test(new URL(target).pathname);
     const response = await client.send({
-      url,
-      as: 'html'
+      url: target,
+      ...(markdown && {
+        as: 'markdown'
+      })
     });
-    const html = response.body.kind === 'html' ? response.body.html : '';
-    const observed = options.browser ? await observeBrowserJson(url, options, lease) : [];
+    const {
+      body
+    } = response;
+    if (body.kind === 'pdf') return {
+      url: response.url,
+      status: response.status,
+      findings: findData(''),
+      observed: [],
+      pdf: describePdf(body)
+    };
+    if (body.kind === 'workbook') return {
+      url: response.url,
+      status: response.status,
+      findings: findData(''),
+      observed: [],
+      workbook: describeWorkbook(body)
+    };
+    if (body.kind === 'deck') return {
+      url: response.url,
+      status: response.status,
+      findings: findData(''),
+      observed: [],
+      deck: describeDeck(body)
+    };
+    if (body.kind === 'json') return {
+      url: response.url,
+      status: response.status,
+      findings: findData(''),
+      observed: [],
+      json: describeJson(body.data, response.format ?? 'json')
+    };
+    const text = body.kind === 'html' ? body.html : body.text;
+    const observed = options.browser && !target.startsWith('file:') ? await observeBrowserJson(url, options, lease) : [];
+    const html = body.kind === 'html' ? {
+      html: describeHtml(body.html, response.format === 'markdown')
+    } : {};
     return {
       url: response.url,
       status: response.status,
-      findings: findData(html),
-      observed
+      findings: findData(text),
+      observed,
+      ...html
     };
   } finally {
     await client.dispose();
@@ -376,7 +888,7 @@ async function probeUrl(url, options) {
 async function probePage(url, options, terminal) {
   try {
     const result = await probeUrl(url, options);
-    terminal.out(probeReport(result.url, result.status, result.findings, result.observed));
+    terminal.out(reportOf(result));
     return 0;
   } catch (error) {
     terminal.err(`probe failed: ${error instanceof Error ? error.message : String(error)}`);
@@ -413,50 +925,12 @@ async function observeBrowserJson(url, options, lease) {
     await browser.close();
   }
 }
-
-/**
- * Loads the hooks a recipe's `hook` steps and transforms call, from a
- * JavaScript module: its default export when that is an object of functions
- * (`export default { positive: v => v > 0 }`), else its named function exports
- * (`export function positive (v) { ... }`).
- *
- * The module runs with the caller's privileges, like any code the caller
- * imports: point this only at a file you trust.
- *
- * @param path - A `.mjs` / `.js` / `.cjs` file.
- * @param importModule - How to import it; `import()` unless a test says otherwise.
- * @returns The hooks by name.
- * @throws Error naming the file when it cannot be imported, exports no hooks,
- * or exports something under a hook's name that is not a function.
- */
-async function loadHooks(path, importModule = importByUrl) {
-  const url = pathToFileURL(resolve(path)).href;
-  let exported;
-  try {
-    exported = await importModule(url);
-  } catch (error) {
-    throw new Error(`${path}: cannot load hooks (${error.message})`, {
-      cause: error
-    });
-  }
-  const candidates = hookExports(exported);
-  const entries = Object.entries(candidates);
-  if (entries.length === 0) throw new Error(`${path}: exports no hooks (export default { name: function } or named functions)`);
-  const hooks = {};
-  for (const [name, value] of entries) {
-    if (typeof value !== 'function') throw new Error(`${path}: hook "${name}" is ${value === null ? 'null' : typeof value}, not a function`);
-    hooks[name] = value;
-  }
-  return hooks;
-}
-async function importByUrl(url) {
-  return await import(url);
-}
-function hookExports(exported) {
-  const fallback = exported.default;
-  if (typeof fallback === 'object' && fallback !== null && !Array.isArray(fallback)) return fallback;
-  const named = Object.entries(exported).filter(([name]) => name !== 'default' && name !== 'module.exports');
-  return Object.fromEntries(named);
+function reportOf(result) {
+  if (result.pdf !== undefined) return pdfReport(result.url, result.pdf);
+  if (result.workbook !== undefined) return workbookReport(result.url, result.workbook);
+  if (result.deck !== undefined) return deckReport(result.url, result.deck);
+  if (result.json !== undefined) return jsonReport(result.url, result.json);
+  return probeReport(result.url, result.status, result.findings, result.observed, result.html);
 }
 
 /**
@@ -530,10 +1004,10 @@ async function runRecipes(command, terminal) {
     return 1;
   }
   let access;
-  let hooks;
+  let plugins;
   try {
     access = await resolveAccess(command.options);
-    hooks = command.hooks === undefined ? undefined : await loadHooks(command.hooks);
+    plugins = command.options.plugins === undefined ? undefined : await loadPlugins(command.options.plugins);
   } catch (error) {
     terminal.err(error instanceof Error ? error.message : String(error));
     return 1;
@@ -544,7 +1018,9 @@ async function runRecipes(command, terminal) {
   const crawler = createCrawler({
     sink,
     access,
-    hooks,
+    hooks: plugins?.hooks,
+    accessPlugins: plugins?.accessPlugins,
+    captchaSolvers: plugins?.captchaSolvers,
     resume: command.resume,
     debug: command.dryRun,
     browser: {
@@ -565,7 +1041,7 @@ async function runRecipes(command, terminal) {
     const report = await crawler.run(new RecipeSet(set.output, inputs));
     if (command.out === undefined && !command.dryRun) for (const record of sink.records) terminal.out(JSON.stringify(record.data));
     for (const recipe of report.recipes) {
-      terminal.err(`${recipe.recipeId}: ${recipe.emitted} emitted, ${recipe.rejected} rejected, ${recipe.duplicates} duplicates, ${recipe.skipped} skipped, ${recipe.pages} pages, ${recipe.durationMs} ms${recipe.error === undefined ? '' : `, stopped: ${recipe.error}`}`);
+      terminal.err(`${recipe.recipeId}: ${recipe.emitted} emitted, ${recipe.rejected} rejected, ${recipe.duplicates} duplicates, ${recipe.skipped} skipped, ${recipe.stepsSkipped > 0 ? `${recipe.stepsSkipped} steps skipped (see --trace), ` : ''}${recipe.pages} pages, ${recipe.durationMs} ms${recipe.error === undefined ? '' : `, stopped: ${recipe.error}`}`);
     }
     if (!command.dryRun) terminal.err(`${report.records} records${report.sink.location === undefined ? '' : ` written to ${report.sink.location}`}`);
     return report.recipes.some(recipe => recipe.error !== undefined) ? 1 : 0;
@@ -695,5 +1171,5 @@ function processTerminal() {
   };
 }
 
-export { USAGE, findData, loadForRun, loadHooks, main, parseArguments, probeReport, probeUrl, processTerminal, readRecipeFiles, resolveAccess };
+export { USAGE, findData, loadForRun, loadHooks, loadPlugins, main, parseArguments, probeReport, probeUrl, processTerminal, readRecipeFiles, resolveAccess };
 //# sourceMappingURL=index.esm.js.map

package/dist/src/arguments/command.contract.d.ts

@@ -10,6 +10,8 @@ export interface CommonOptions {
     access?: string;
     /** The access profile to use by default, overriding the config's `default`. */
     accessProfile?: string;
+    /** A plugins module: hooks, access plugins; `--hooks` / `OPENCRAW_PLUGINS` / `OPENCRAW_HOOKS` too. */
+    plugins?: string;
 }
 export type Command = {
     name: 'help';
@@ -30,8 +32,6 @@ export type Command = {
     dryRun: boolean;
     /** Run only the input recipes with these ids. */
     only: string[];
-    /** A JavaScript module exporting the hooks the recipes call; `OPENCRAW_HOOKS` when not given. */
-    hooks?: string;
     headed: boolean;
     options: CommonOptions;
 } | {

package/dist/src/arguments/parse-arguments.algorithm.d.ts

@@ -1,5 +1,5 @@
 import type { Command } from './command.contract.js';
-export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.\n  probe <url>                                 Fetch a page and report where its data lives.\n\nOptions for run\n  --out <file>        Write records to this JSON Lines file (default: stdout).\n  --append            Keep what --out holds and add to it (each line carries _key).\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n  --hooks <file>      A JavaScript module whose default export is { name: function }: the hooks\n                      the recipes call (or OPENCRAW_HOOKS). It runs as your code: trust it.\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --help, --version";
+export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.\n  probe <url>                                 Fetch a page and report where its data lives.\n\nOptions for run\n  --out <file>        Write records to this JSON Lines file (default: stdout).\n  --append            Keep what --out holds and add to it (each line carries _key).\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --plugins <file>    A JavaScript module exporting hooks (the recipes' hook steps and transforms),\n                      accessPlugins (for { kind: \"plugin\" } access profiles) and captchaSolvers\n                      (for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /\n                      OPENCRAW_HOOKS. It runs as your code: trust it.\n  --help, --version";
 /**
  * Turns argv into a command.
  *

package/dist/src/hooks-module/index.d.ts

@@ -1,3 +1,3 @@
-export { loadHooks } from './load-hooks.repository.js';
-export type { ModuleImporter } from './load-hooks.repository.js';
+export { loadHooks, loadPlugins } from './load-hooks.repository.js';
+export type { ModuleImporter, PluginModule } from './load-hooks.repository.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/hooks-module/load-hooks.repository.d.ts

@@ -1,18 +1,45 @@
-import type { HookMap } from '@opencraw/core';
+import type { AccessPlugin, CaptchaSolver, HookMap } from '@opencraw/core';
+/** What a plugins module provides: hooks for `hook` steps and transforms, and plugins the engine calls. */
+export interface PluginModule {
+    hooks: HookMap;
+    /** Access plugins: `{ kind: "plugin", name }` profiles in the access config use them. */
+    accessPlugins: AccessPlugin[];
+    /** Captcha solvers: recipes name them in `session.captcha.solver` and `captcha` steps. */
+    captchaSolvers: CaptchaSolver[];
+}
 /**
- * Loads the hooks a recipe's `hook` steps and transforms call, from a
- * JavaScript module: its default export when that is an object of functions
- * (`export default { positive: v => v > 0 }`), else its named function exports
- * (`export function positive (v) { ... }`).
+ * Loads a plugins module: a JavaScript module whose named exports `hooks`
+ * (`{ name: function }`), `accessPlugins` (`[{ name, lease }]`) and
+ * `captchaSolvers` (`[{ name, solve }]`) give what recipes and access profiles
+ * refer to by name:
+ *
+ * ```js
+ * export const hooks = { positive: v => v > 0 }
+ * export const accessPlugins = [{ name: 'browserbase', lease: async () => ({ cdp: { endpoint } }) }]
+ * export const captchaSolvers = [{ name: 'capsolver', solve: async (challenge, { page }) => ({ status: 'solved' }) }]
+ * ```
+ *
+ * A module with none of those names is a bare hooks module, as before: its
+ * default export when that is an object of functions, else its named function
+ * exports.
  *
  * The module runs with the caller's privileges, like any code the caller
  * imports: point this only at a file you trust.
  *
  * @param path - A `.mjs` / `.js` / `.cjs` file.
  * @param importModule - How to import it; `import()` unless a test says otherwise.
+ * @returns The hooks and plugins.
+ * @throws Error naming the file when it cannot be imported, provides nothing,
+ * or provides something that is not what its name promises.
+ */
+export declare function loadPlugins(path: string, importModule?: ModuleImporter): Promise<PluginModule>;
+/**
+ * Loads the hooks of a hooks or plugins module (see {@link loadPlugins}).
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file.
+ * @param importModule - How to import it.
  * @returns The hooks by name.
- * @throws Error naming the file when it cannot be imported, exports no hooks,
- * or exports something under a hook's name that is not a function.
+ * @throws Error naming the file when it cannot be imported or exports no hooks.
  */
 export declare function loadHooks(path: string, importModule?: ModuleImporter): Promise<HookMap>;
 /** Imports a module by URL, returning its namespace. */

package/dist/src/index.d.ts

@@ -10,5 +10,6 @@ export type { RecipeFiles } from './recipe-files/index.js';
 export { loadForRun } from './run/index.js';
 export type { CommonOptions } from './arguments/index.js';
 export { resolveAccess } from './access/index.js';
-export { loadHooks } from './hooks-module/index.js';
+export { loadHooks, loadPlugins } from './hooks-module/index.js';
+export type { PluginModule } from './hooks-module/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/probe/deck-findings.mapper.d.ts

@@ -0,0 +1,46 @@
+import type { DeckDocument } from '@opencraw/core';
+/** What a probe shows of a deck (a presentation). */
+export interface DeckFindings {
+    width: number;
+    height: number;
+    slides: {
+        number: number;
+        title: string;
+        hidden: boolean;
+        shapes: number;
+        tables: number;
+        charts: number;
+    }[];
+    /** The likely header rows of native tables, with the `slide` and `selector` a `table` extract needs. */
+    headers: {
+        slide: number;
+        text: string;
+        selector: string;
+        hint?: string;
+    }[];
+    charts: {
+        slide: number;
+        type: string;
+        title?: string;
+        series: {
+            name: string;
+            points: number;
+        }[];
+    }[];
+    /** Slides whose short text boxes look like a table: read them with `"shapes": true`. */
+    grids: {
+        slide: number;
+        title: string;
+        boxes: number;
+    }[];
+}
+/**
+ * Summarises a deck for someone writing a recipe: its slides, where the native
+ * tables start, what the charts hold, and which slides lay text boxes out as
+ * a table.
+ *
+ * @param document - The read deck.
+ * @returns The findings.
+ */
+export declare function describeDeck(document: DeckDocument): DeckFindings;
+//# sourceMappingURL=deck-findings.mapper.d.ts.map

package/dist/src/probe/html-findings.mapper.d.ts

@@ -0,0 +1,28 @@
+/** What a probe shows of an HTML document's tables, and of rendered Markdown's sections and front matter. */
+export interface HtmlFindings {
+    /** The likely header rows of the `<table>`s, with the `selector` a `table` extract needs. */
+    tables: {
+        table: string;
+        text: string;
+        selector: string;
+        hint?: string;
+    }[];
+    /** Rendered Markdown: its headings, each with the selector of its section. */
+    outline?: {
+        level: number;
+        heading: string;
+        selector: string;
+    }[];
+    /** Rendered Markdown: the keys of its front matter. */
+    frontMatter?: string[];
+}
+/**
+ * Summarises an HTML document for a `table` extract, and, when it is rendered
+ * Markdown, its outline and front matter.
+ *
+ * @param html - The document.
+ * @param markdown - Whether it was rendered from Markdown.
+ * @returns The findings.
+ */
+export declare function describeHtml(html: string, markdown: boolean): HtmlFindings;
+//# sourceMappingURL=html-findings.mapper.d.ts.map

package/dist/src/probe/index.d.ts

@@ -2,5 +2,15 @@ export { probePage, probeUrl } from './probe-page.use-case.js';
 export type { ProbeResult } from './probe-page.use-case.js';
 export { findData } from './find-data.algorithm.js';
 export type { ProbeFindings, JsonLdFinding, InlineJsonFinding } from './find-data.algorithm.js';
-export { probeReport } from './probe-report.mapper.js';
+export { probeReport, pdfReport, workbookReport, deckReport, jsonReport } from './probe-report.mapper.js';
+export { describePdf } from './pdf-findings.mapper.js';
+export type { PdfFindings } from './pdf-findings.mapper.js';
+export { describeWorkbook } from './workbook-findings.mapper.js';
+export type { WorkbookFindings } from './workbook-findings.mapper.js';
+export { describeDeck } from './deck-findings.mapper.js';
+export type { DeckFindings } from './deck-findings.mapper.js';
+export { describeJson } from './json-findings.mapper.js';
+export { describeHtml } from './html-findings.mapper.js';
+export type { HtmlFindings } from './html-findings.mapper.js';
+export type { JsonFindings } from './json-findings.mapper.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/probe/json-findings.mapper.d.ts

@@ -0,0 +1,31 @@
+/** What a probe shows of JSON data (a JSON, JSON Lines or YAML body). */
+export interface JsonFindings {
+    /** What the body was read as. */
+    format: string;
+    /** `object`, `array (n)`, or a scalar type. */
+    type: string;
+    /**
+     * The document's structure down to four levels, one line per node:
+     * `$.data.products  array (1240) of object`, `$.data.products[*].name  string "Pandina"`.
+     */
+    tree: string[];
+    /**
+     * Every array of objects with two entries or more, largest first: the
+     * `forEach` or `each` a recipe needs, with the keys all its entries share.
+     */
+    lists: {
+        path: string;
+        length: number;
+        keys: string[];
+    }[];
+}
+/**
+ * Summarises JSON data for someone writing a recipe: its shape, and where its
+ * record lists are, so a `jsonpath` selector can be read off the result.
+ *
+ * @param data - The parsed document.
+ * @param format - What it was read as (`json`, `jsonl`, `yaml`).
+ * @returns The findings.
+ */
+export declare function describeJson(data: unknown, format: string): JsonFindings;
+//# sourceMappingURL=json-findings.mapper.d.ts.map

package/dist/src/probe/pdf-findings.mapper.d.ts

@@ -0,0 +1,26 @@
+import type { PdfDocument } from '@opencraw/core';
+/** What a probe shows of a PDF: its rows, and the rows that look like table headers. */
+export interface PdfFindings {
+    pages: number;
+    /** The first rows, cells separated by " | ". */
+    rows: {
+        page: number;
+        text: string;
+    }[];
+    /** Rows of text cells (no digit) with a row of data (a digit) right under them: likely table headers, with a selector that finds them. */
+    headers: {
+        page: number;
+        text: string;
+        selector: string;
+    }[];
+}
+/**
+ * Summarises a PDF for someone writing a recipe: what the rows look like, and
+ * where the tables start, so a `table` extract's selector and columns can be
+ * read off the result instead of guessed.
+ *
+ * @param document - The read PDF.
+ * @returns The findings.
+ */
+export declare function describePdf(document: PdfDocument): PdfFindings;
+//# sourceMappingURL=pdf-findings.mapper.d.ts.map

package/dist/src/probe/probe-page.use-case.d.ts

@@ -1,6 +1,11 @@
 import type { CommonOptions } from '../arguments/index.js';
 import type { Terminal } from '../terminal/index.js';
 import { findData } from './find-data.algorithm.js';
+import type { DeckFindings } from './deck-findings.mapper.js';
+import type { HtmlFindings } from './html-findings.mapper.js';
+import type { JsonFindings } from './json-findings.mapper.js';
+import type { PdfFindings } from './pdf-findings.mapper.js';
+import type { WorkbookFindings } from './workbook-findings.mapper.js';
 /** What fetching and, optionally, rendering a page found. */
 export interface ProbeResult {
     url: string;
@@ -8,6 +13,16 @@ export interface ProbeResult {
     findings: ReturnType<typeof findData>;
     /** JSON responses observed while the page rendered; empty unless `options.browser` was set. */
     observed: string[];
+    /** Present when the URL is a PDF: its rows and likely table headers. */
+    pdf?: PdfFindings;
+    /** Present when the URL is a CSV or a spreadsheet: its sheets, rows and likely table headers. */
+    workbook?: WorkbookFindings;
+    /** Present when the URL is a presentation: its slides, table headers, charts and text-box grids. */
+    deck?: DeckFindings;
+    /** Present when the URL is JSON, JSON Lines or YAML: its structure and record lists. */
+    json?: JsonFindings;
+    /** Present for HTML: its tables' headers; for rendered Markdown also its outline and front matter. */
+    html?: HtmlFindings;
 }
 /**
  * Fetches a page and finds where its data lives, through the access profile the
@@ -17,7 +32,10 @@ export interface ProbeResult {
  * involved: the cli's `probePage` and `@opencraw/mcp`'s probe tool both
  * build on this, one printing the result, the other returning it as data.
  *
- * @param url - The page to probe.
+ * A PDF, a spreadsheet, a CSV or a presentation (by content type, or a local
+ * path) is read instead: its rows, and the rows that look like table headers.
+ *
+ * @param url - The page to probe, or a local file path.
  * @param options - Browser path, TLS and user agent, plus whether to render.
  * @returns What was found.
  * @throws Error when the fetch itself fails.

package/dist/src/probe/probe-report.mapper.d.ts

@@ -1,4 +1,9 @@
 import type { ProbeFindings } from './find-data.algorithm.js';
+import type { DeckFindings } from './deck-findings.mapper.js';
+import type { HtmlFindings } from './html-findings.mapper.js';
+import type { JsonFindings } from './json-findings.mapper.js';
+import type { PdfFindings } from './pdf-findings.mapper.js';
+import type { WorkbookFindings } from './workbook-findings.mapper.js';
 /**
  * Renders a probe's findings as text.
  *
@@ -6,7 +11,45 @@ import type { ProbeFindings } from './find-data.algorithm.js';
  * @param status - The HTTP status.
  * @param findings - What `findData` found.
  * @param observed - JSON responses a browser saw, when `--browser` was used.
+ * @param html - The tables (and, for Markdown, the outline and front matter) found.
  * @returns The report.
  */
-export declare function probeReport(url: string, status: number, findings: ProbeFindings, observed?: string[]): string;
+export declare function probeReport(url: string, status: number, findings: ProbeFindings, observed?: string[], html?: HtmlFindings): string;
+/**
+ * Renders what a probe found in a PDF as text.
+ *
+ * @param url - The URL or file probed.
+ * @param pdf - What `describePdf` found.
+ * @returns The report.
+ */
+export declare function pdfReport(url: string, pdf: PdfFindings): string;
+/**
+ * Formats what a probe found in a workbook (a CSV or a spreadsheet): its sheets,
+ * the rows that look like table headers with a selector for each, and the first
+ * rows.
+ *
+ * @param url - The workbook's URL.
+ * @param workbook - The findings.
+ * @returns The report.
+ */
+export declare function workbookReport(url: string, workbook: WorkbookFindings): string;
+/**
+ * Formats what a probe found in a deck (a presentation): its slides, the native
+ * tables' headers with a selector for each, its charts, and the slides whose
+ * text boxes look like a table.
+ *
+ * @param url - The deck's URL.
+ * @param deck - The findings.
+ * @returns The report.
+ */
+export declare function deckReport(url: string, deck: DeckFindings): string;
+/**
+ * Formats what a probe found in JSON data: its record lists with the path a
+ * `jsonpath` extract needs, then its structure.
+ *
+ * @param url - The document's URL.
+ * @param json - The findings.
+ * @returns The report.
+ */
+export declare function jsonReport(url: string, json: JsonFindings): string;
 //# sourceMappingURL=probe-report.mapper.d.ts.map

package/dist/src/probe/workbook-findings.mapper.d.ts

@@ -0,0 +1,43 @@
+import type { WorkbookDocument } from '@opencraw/core';
+/** What a probe shows of a workbook (a spreadsheet or a CSV). */
+export interface WorkbookFindings {
+    /** How a CSV was read. */
+    csv?: {
+        encoding: string;
+        delimiter: string;
+    };
+    sheets: {
+        name: string;
+        hidden: boolean;
+        rows: number;
+        columns: number;
+    }[];
+    /** The first non-empty rows of each visible sheet, cells separated by " | ". */
+    rows: {
+        sheet: string;
+        row: number;
+        text: string;
+    }[];
+    /**
+     * Rows of text cells (no digit) with a data row (a digit) just below: likely
+     * table headers, with the `sheet` and `selector` a `table` extract needs, and
+     * a hint when the header has merged cells (a header over two rows).
+     */
+    headers: {
+        sheet: string;
+        row: number;
+        text: string;
+        selector: string;
+        hint?: string;
+    }[];
+}
+/**
+ * Summarises a workbook for someone writing a recipe: its sheets, what their
+ * rows look like and where the tables start, so a `table` extract's `sheet`,
+ * `selector` and columns can be read off the result instead of guessed.
+ *
+ * @param document - The read workbook.
+ * @returns The findings.
+ */
+export declare function describeWorkbook(document: WorkbookDocument): WorkbookFindings;
+//# sourceMappingURL=workbook-findings.mapper.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/cli",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",
@@ -24,7 +24,7 @@
     "!**/*.js.map"
   ],
   "dependencies": {
-    "@opencraw/core": "^0.1.0",
+    "@opencraw/core": "^0.1.2",
     "playwright": "^1.63.0"
   },
   "publishConfig": {