@opencraw/cli 0.1.1 → 0.1.3

package/README.md

@@ -43,7 +43,11 @@ input recipes) and reports what happened.
 | `--dry-run` | One record per input recipe, printed with the scope it was mapped from — for checking a recipe under construction without a full run. |
 | `--trace` | Print the crawl trace to stderr. |
 | `--headed` | Show the browser instead of running headless. |
-| `--hooks <file>` | A JavaScript module whose default export is `{ name: function }`: the hooks the recipes call (or `OPENCRAW_HOOKS`). It runs as your code; load only files you trust. See [§6 of the authoring guide](../../docs/recipes/authoring.md#6-hooks). |
+| `--parallel <n>` | Run this many input recipes at once (default 1). Iterations inside a recipe follow its `limits.concurrency`. See [§3.9 of the authoring guide](../../docs/recipes/authoring.md#39-concurrency). |
+| `--retries <n>` | Tries per request that fails in passing (a dropped connection, a timeout, a 429, a 5xx), for recipes whose `limits.retry` says nothing. Default 3; `1` turns retrying off. See [§7 of the authoring guide](../../docs/recipes/authoring.md#7-policies-what-happens-when-something-is-missing-or-fails). |
+| `--profiles <dir>` | Where the persistent browser profiles of `session.browserProfile` live (or `OPENCRAW_PROFILES`; default `.opencraw/profiles`). See [access.md](../../docs/recipes/access.md#persistent-browser-profiles). |
+| `--host-delay <ms>`, `--host-concurrency <n>` | Per-site politeness across every recipe: at least `ms` between two requests to one site, at most `n` in flight. They override the access config's `throttle` defaults. See [access.md](../../docs/recipes/access.md#throttling-per-site). |
+| `--plugins <file>` (or `--hooks`) | A plugins module: named exports `hooks` (the recipes' hook steps and transforms), `accessPlugins` (for `{ kind: "plugin" }` access profiles) and `captchaSolvers`. A module whose default export is `{ name: function }` is read as hooks alone. `OPENCRAW_PLUGINS` / `OPENCRAW_HOOKS` when not given; `probe` reads it too. It runs as your code; load only files you trust. See [§6 of the authoring guide](../../docs/recipes/authoring.md#6-hooks). |
 
 ```sh
 opencraw run recipes/ --out out/products.jsonl --trace
@@ -62,12 +66,20 @@ opencraw probe https://example.com/product/1
 opencraw probe https://example.com/configurator --browser
 ```
 
-A PDF (a URL served as `application/pdf`, or a local `.pdf` path) is read instead: `probe` lists its first
-rows and every row that looks like a table header, with the `selector` a `table` extract needs.
+A PDF, a spreadsheet, a CSV or a presentation (a URL served with its content type, or a local `.pdf`,
+`.xlsx`, `.csv`, `.tsv` or `.pptx` path) is read instead: `probe` lists its pages, sheets or slides, its first
+rows, and every row that looks like a table header, with the `selector` a `table` extract needs. For a CSV it
+also names the encoding and the delimiter it detected; for a presentation, its charts and the slides whose
+text boxes look like a table. JSON, JSON Lines and YAML show their structure instead, and every list of
+records with the `jsonpath` that walks it. An HTML page also lists its tables' header rows, and Markdown
+(a `.md` URL is read as Markdown even when served as `text/plain`) its sections and front matter keys.
 
 ```sh
 opencraw probe https://example.com/price-list.pdf
 opencraw probe ./sheets/september.pdf
+opencraw probe ./exports/listino.csv
+opencraw probe ./exports/incentivi.xlsx
+opencraw probe ./decks/incentivi.pptx
 ```
 
 Use it before writing an input recipe, to find the shape a site's data actually takes (§9 of

package/dist/index.esm.js

@@ -1,7 +1,7 @@
 import { parseArgs } from 'node:util';
 import { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
-import { loadAccessConfig, AccessBroker, HttpClient, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
+import { loadAccessConfig, htmlTableSheets, AccessBroker, HttpClient, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
 import { readFileSync } from 'node:fs';
 
 const USAGE = `opencraw <command> [options]
@@ -19,8 +19,16 @@ Options for run
   --dry-run           One record per input, printed with the scope it was mapped from.
   --trace             Print the crawl trace to stderr.
   --headed            Show the browser.
-  --hooks <file>      A JavaScript module whose default export is { name: function }: the hooks
-                      the recipes call (or OPENCRAW_HOOKS). It runs as your code: trust it.
+  --parallel <n>      Run this many input recipes at once (default 1). Iterations inside a recipe
+                      follow its limits.concurrency.
+  --retries <n>       Tries per request that fails in passing (connection, timeout, 429, 5xx), for
+                      recipes whose limits.retry says nothing. Default 3; 1 turns retrying off.
+  --profiles <dir>    Where recipes' session.browserProfile profiles live (or OPENCRAW_PROFILES;
+                      default .opencraw/profiles).
+  --host-delay <ms>   At least this long between two requests to one site, across every recipe.
+  --host-concurrency <n>
+                      At most this many requests to one site in flight, across every recipe.
+                      Both override the access config's "throttle" defaults.
 
 Options for probe
   --browser           Also render the page in a browser and list the JSON it fetches.
@@ -32,6 +40,10 @@ Options for both
   --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).
   --access-profile <name>
                       The access profile to use when a recipe names none.
+  --plugins <file>    A JavaScript module exporting hooks (the recipes' hook steps and transforms),
+                      accessPlugins (for { kind: "plugin" } access profiles) and captchaSolvers
+                      (for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /
+                      OPENCRAW_HOOKS. It runs as your code: trust it.
   --help, --version`;
 const OPTIONS = {
   'out': {
@@ -56,9 +68,27 @@ const OPTIONS = {
   'headed': {
     type: 'boolean'
   },
+  'profiles': {
+    type: 'string'
+  },
+  'retries': {
+    type: 'string'
+  },
+  'parallel': {
+    type: 'string'
+  },
+  'host-delay': {
+    type: 'string'
+  },
+  'host-concurrency': {
+    type: 'string'
+  },
   'hooks': {
     type: 'string'
   },
+  'plugins': {
+    type: 'string'
+  },
   'browser': {
     type: 'boolean'
   },
@@ -116,7 +146,8 @@ function parseArguments(argv, env = {}) {
     insecureTls: values['insecure-tls'] === true || env.OPENCRAW_INSECURE_TLS === '1',
     userAgent: values['user-agent'],
     access: values.access ?? (env.OPENCRAW_ACCESS === '' ? undefined : env.OPENCRAW_ACCESS),
-    accessProfile: values['access-profile']
+    accessProfile: values['access-profile'],
+    plugins: values.plugins ?? values.hooks ?? nonEmpty(env.OPENCRAW_PLUGINS) ?? nonEmpty(env.OPENCRAW_HOOKS)
   };
   switch (name) {
     case undefined:
@@ -148,7 +179,13 @@ function parseArguments(argv, env = {}) {
           dryRun: values['dry-run'] === true,
           only: values.only ?? [],
           headed: values.headed === true,
-          hooks: values.hooks ?? (env.OPENCRAW_HOOKS === '' ? undefined : env.OPENCRAW_HOOKS),
+          profiles: values.profiles ?? nonEmpty(env.OPENCRAW_PROFILES),
+          retries: integer(values.retries, '--retries', 1),
+          parallel: integer(values.parallel, '--parallel', 1),
+          throttle: {
+            delayMs: integer(values['host-delay'], '--host-delay', 0),
+            concurrency: integer(values['host-concurrency'], '--host-concurrency', 1)
+          },
           options
         };
       }
@@ -168,6 +205,15 @@ function parseArguments(argv, env = {}) {
       }
   }
 }
+function integer(value, name, minimum) {
+  if (value === undefined) return undefined;
+  const number = Number(value);
+  if (!Number.isSafeInteger(number) || number < minimum) throw new Error(`${name} needs a whole number of at least ${minimum}, not "${value}"`);
+  return number;
+}
+function nonEmpty(value) {
+  return value === '' ? undefined : value;
+}
 
 /**
  * The access config a command runs with: the file `--access` (or
@@ -193,6 +239,109 @@ async function resolveAccess(options) {
   };
 }
 
+/** The named exports that mark a module as a plugins module rather than a bare hooks module. */
+const PLUGIN_EXPORTS = ['hooks', 'accessPlugins', 'captchaSolvers'];
+/**
+ * Loads a plugins module: a JavaScript module whose named exports `hooks`
+ * (`{ name: function }`), `accessPlugins` (`[{ name, lease }]`) and
+ * `captchaSolvers` (`[{ name, solve }]`) give what recipes and access profiles
+ * refer to by name:
+ *
+ * ```js
+ * export const hooks = { positive: v => v > 0 }
+ * export const accessPlugins = [{ name: 'browserbase', lease: async () => ({ cdp: { endpoint } }) }]
+ * export const captchaSolvers = [{ name: 'capsolver', solve: async (challenge, { page }) => ({ status: 'solved' }) }]
+ * ```
+ *
+ * A module with none of those names is a bare hooks module, as before: its
+ * default export when that is an object of functions, else its named function
+ * exports.
+ *
+ * The module runs with the caller's privileges, like any code the caller
+ * imports: point this only at a file you trust.
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file.
+ * @param importModule - How to import it; `import()` unless a test says otherwise.
+ * @returns The hooks and plugins.
+ * @throws Error naming the file when it cannot be imported, provides nothing,
+ * or provides something that is not what its name promises.
+ */
+async function loadPlugins(path, importModule = importByUrl) {
+  const url = pathToFileURL(resolve(path)).href;
+  let exported;
+  try {
+    exported = await importModule(url);
+  } catch (error) {
+    throw new Error(`${path}: cannot load hooks (${error.message})`, {
+      cause: error
+    });
+  }
+  if (PLUGIN_EXPORTS.every(name => !Object.hasOwn(exported, name))) {
+    const hooks = hooksOf(hookExports(exported), path);
+    if (Object.keys(hooks).length === 0) throw new Error(`${path}: exports no hooks (export default { name: function } or named functions) and no plugins (export const hooks, accessPlugins, captchaSolvers)`);
+    return {
+      hooks,
+      accessPlugins: [],
+      captchaSolvers: []
+    };
+  }
+  return {
+    hooks: hooksOf(objectExport(exported.hooks, 'hooks', path), path),
+    accessPlugins: pluginsOf(exported.accessPlugins, 'accessPlugins', 'lease', path),
+    captchaSolvers: pluginsOf(exported.captchaSolvers, 'captchaSolvers', 'solve', path)
+  };
+}
+/**
+ * Loads the hooks of a hooks or plugins module (see {@link loadPlugins}).
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file.
+ * @param importModule - How to import it.
+ * @returns The hooks by name.
+ * @throws Error naming the file when it cannot be imported or exports no hooks.
+ */
+async function loadHooks(path, importModule = importByUrl) {
+  const {
+    hooks
+  } = await loadPlugins(path, importModule);
+  if (Object.keys(hooks).length === 0) throw new Error(`${path}: exports no hooks (export default { name: function } or named functions)`);
+  return hooks;
+}
+function hooksOf(candidates, path) {
+  const hooks = {};
+  for (const [name, value] of Object.entries(candidates)) {
+    if (typeof value !== 'function') throw new Error(`${path}: hook "${name}" is ${value === null ? 'null' : typeof value}, not a function`);
+    hooks[name] = value;
+  }
+  return hooks;
+}
+function objectExport(value, name, path) {
+  if (value === undefined) return {};
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) throw new Error(`${path}: "${name}" must be an object of functions`);
+  return value;
+}
+/** A list of `{ name, <method> }` plugins, checked. */
+function pluginsOf(value, exportName, method, path) {
+  if (value === undefined) return [];
+  if (!Array.isArray(value)) throw new Error(`${path}: "${exportName}" must be an array of { name, ${method} }`);
+  const names = new Set();
+  for (const [index, plugin] of value.entries()) {
+    const entry = plugin;
+    if (typeof entry?.name !== 'string' || entry.name === '' || typeof entry[method] !== 'function') throw new Error(`${path}: ${exportName}[${index}] must be { name: string, ${method}: function }`);
+    if (names.has(entry.name)) throw new Error(`${path}: ${exportName} names "${entry.name}" twice`);
+    names.add(entry.name);
+  }
+  return value;
+}
+async function importByUrl(url) {
+  return await import(url);
+}
+function hookExports(exported) {
+  const fallback = exported.default;
+  if (typeof fallback === 'object' && fallback !== null && !Array.isArray(fallback)) return fallback;
+  const named = Object.entries(exported).filter(([name]) => name !== 'default' && name !== 'module.exports');
+  return Object.fromEntries(named);
+}
+
 /**
  * Finds where a fetched page's data lives: JSON-LD blocks, inline JSON objects
  * big enough to matter, URLs that look like a data endpoint, script hosts and
@@ -304,6 +453,255 @@ function tryParse(text) {
   }
 }
 
+/** Rows shown per sheet in a probe, at most. */
+const ROW_LIMIT$1 = 20;
+/** How far below a header row a probe looks for the first data row. */
+const LOOKAHEAD = 3;
+/**
+ * Summarises a workbook for someone writing a recipe: its sheets, what their
+ * rows look like and where the tables start, so a `table` extract's `sheet`,
+ * `selector` and columns can be read off the result instead of guessed.
+ *
+ * @param document - The read workbook.
+ * @returns The findings.
+ */
+function describeWorkbook(document) {
+  const visible = document.sheets.filter(sheet => sheet.hidden !== true);
+  return {
+    ...(document.csv !== undefined && {
+      csv: document.csv
+    }),
+    sheets: document.sheets.map(sheet => ({
+      name: sheet.name,
+      hidden: sheet.hidden === true,
+      rows: sheet.rows.length,
+      columns: Math.max(0, ...sheet.rows.map(row => row.length))
+    })),
+    rows: visible.flatMap(sheet => filledRows(sheet).slice(0, ROW_LIMIT$1).map(({
+      index,
+      cells
+    }) => ({
+      sheet: sheet.name,
+      row: index + 1,
+      text: cells.join(' | ')
+    }))),
+    headers: visible.flatMap(sheet => headersOf(sheet))
+  };
+}
+function headersOf(sheet) {
+  const rows = filledRows(sheet);
+  const merged = mergedRows(sheet);
+  return rows.flatMap(({
+    index,
+    cells
+  }, position) => {
+    if (cells.length < 2 || cells.some(cell => /\d/.test(cell))) return [];
+    if (rows.slice(position + 1, position + 1 + LOOKAHEAD).every(row => !(row.cells.length >= 2 && row.cells.some(cell => /\d/.test(cell))))) return [];
+    const header = {
+      sheet: sheet.name,
+      row: index + 1,
+      text: cells.join(' | '),
+      selector: `^${escape$1(cells[0])}`
+    };
+    return [merged.has(index) ? {
+      ...header,
+      hint: 'merged header cells: try "headerRows": 2'
+    } : header];
+  });
+}
+/** The non-empty rows, as their non-empty cells, whitespace collapsed. */
+function filledRows(sheet) {
+  const hidden = new Set(sheet.hiddenRows);
+  return sheet.rows.flatMap((row, index) => {
+    const cells = row.map(cell => String(cell).replaceAll(/\s+/g, ' ').trim()).filter(cell => cell !== '');
+    return cells.length === 0 || hidden.has(index) ? [] : [{
+      index,
+      cells
+    }];
+  });
+}
+/** Rows a merged range starts on. */
+function mergedRows(sheet) {
+  const rows = new Set();
+  const merges = sheet.merges ?? [];
+  for (const reference of merges) {
+    const match = /^\$?[A-Z]+\$?(\d+)/i.exec(reference);
+    if (match !== null) rows.add(Number(match[1]) - 1);
+  }
+  return rows;
+}
+function escape$1(text) {
+  return text.replaceAll(/[$()*+.?[\\\]^{|}]/g, String.raw`\$&`);
+}
+
+/** Short text boxes on one slide from which a probe suggests reading them as a table. */
+const GRID_BOXES = 6;
+const SHORT_TEXT = 40;
+/**
+ * Summarises a deck for someone writing a recipe: its slides, where the native
+ * tables start, what the charts hold, and which slides lay text boxes out as
+ * a table.
+ *
+ * @param document - The read deck.
+ * @returns The findings.
+ */
+function describeDeck(document) {
+  return {
+    width: document.width,
+    height: document.height,
+    slides: document.slides.map(slide => ({
+      number: slide.number,
+      title: slide.title ?? '',
+      hidden: slide.hidden,
+      shapes: slide.shapes.length,
+      tables: slide.tables.length,
+      charts: slide.charts.length
+    })),
+    headers: document.slides.flatMap(slide => describeWorkbook({
+      sheets: slide.tables
+    }).headers.map(({
+      sheet: _sheet,
+      row: _row,
+      ...header
+    }) => ({
+      slide: slide.number,
+      ...header
+    }))),
+    charts: document.slides.flatMap(slide => slide.charts.map(chart => ({
+      slide: slide.number,
+      type: chart.type,
+      ...(chart.title !== undefined && {
+        title: chart.title
+      }),
+      series: chart.series.map(series => ({
+        name: series.name,
+        points: series.values.length
+      }))
+    }))),
+    grids: document.slides.flatMap(slide => {
+      const boxes = slide.shapes.filter(shape => shape.placeholder === undefined && shape.text.length <= SHORT_TEXT).length;
+      return boxes >= GRID_BOXES ? [{
+        slide: slide.number,
+        title: slide.title ?? '',
+        boxes
+      }] : [];
+    })
+  };
+}
+
+const SECTION = /<section data-heading="([^"]*)" data-level="(\d)">/g;
+const FRONT_MATTER = /<script type="application\/json" data-front-matter>([\s\S]*?)<\/script>/;
+/**
+ * Summarises an HTML document for a `table` extract, and, when it is rendered
+ * Markdown, its outline and front matter.
+ *
+ * @param html - The document.
+ * @param markdown - Whether it was rendered from Markdown.
+ * @returns The findings.
+ */
+function describeHtml(html, markdown) {
+  const tables = htmlTableSheets(html);
+  const headers = describeWorkbook({
+    sheets: tables
+  }).headers.map(({
+    sheet,
+    row: _row,
+    ...header
+  }) => ({
+    table: sheet,
+    ...header
+  }));
+  if (!markdown) return {
+    tables: headers
+  };
+  const outline = Array.from(html.matchAll(SECTION), ([, heading, level]) => ({
+    level: Number(level),
+    heading: unescape(heading),
+    selector: `section[data-heading='${unescape(heading).replaceAll("'", String.raw`\'`)}' i]`
+  }));
+  const front = FRONT_MATTER.exec(html)?.[1];
+  const data = front === undefined ? undefined : JSON.parse(front);
+  return {
+    tables: headers,
+    outline,
+    frontMatter: typeof data === 'object' && data !== null && !Array.isArray(data) ? Object.keys(data) : []
+  };
+}
+function unescape(text) {
+  return text.replaceAll('&quot;', '"').replaceAll('&lt;', '<').replaceAll('&amp;', '&');
+}
+
+/** How deep a probe describes a JSON document. */
+const DEPTH = 4;
+/** Keys shown per object, at most. */
+const KEY_LIMIT = 25;
+/** Entries of a record list whose keys are compared. */
+const SAMPLE = 50;
+/** A sample value longer than this is cut. */
+const SAMPLE_CHARS = 40;
+/**
+ * Summarises JSON data for someone writing a recipe: its shape, and where its
+ * record lists are, so a `jsonpath` selector can be read off the result.
+ *
+ * @param data - The parsed document.
+ * @param format - What it was read as (`json`, `jsonl`, `yaml`).
+ * @returns The findings.
+ */
+function describeJson(data, format) {
+  const tree = [];
+  const lists = [];
+  walk(data, '$', 0, tree, lists);
+  return {
+    format,
+    type: typeOf(data),
+    tree,
+    lists: lists.sort((first, second) => second.length - first.length)
+  };
+}
+function walk(value, path, depth, tree, lists) {
+  if (Array.isArray(value)) {
+    const objects = value.filter(entry => isObject(entry));
+    if (objects.length >= 2 && objects.length === value.length) lists.push({
+      path: `${path}[*]`,
+      length: value.length,
+      keys: sharedKeys(objects.slice(0, SAMPLE))
+    });
+    if (depth > 0) tree.push(`${path}  ${typeOf(value)}`);
+    if (depth < DEPTH && value.length > 0) walk(value[0], `${path}[*]`, depth + 1, tree, lists);
+    return;
+  }
+  if (isObject(value)) {
+    if (depth > 0) tree.push(`${path}  object`);
+    if (depth >= DEPTH) return;
+    for (const [key, child] of Object.entries(value).slice(0, KEY_LIMIT)) walk(child, childPath(path, key), depth + 1, tree, lists);
+    return;
+  }
+  tree.push(`${path}  ${typeOf(value)}${sample(value)}`);
+}
+function typeOf(value) {
+  if (Array.isArray(value)) {
+    const kinds = [...new Set(value.slice(0, SAMPLE).map(entry => typeOf(entry).split(' ', 1)[0]))];
+    return `array (${value.length})${kinds.length === 0 ? '' : ` of ${kinds.join(' | ')}`}`;
+  }
+  if (value === null) return 'null';
+  return typeof value === 'object' ? 'object' : typeof value;
+}
+function sample(value) {
+  if (value === null || value === undefined) return '';
+  const text = JSON.stringify(value);
+  return ` ${text.length > SAMPLE_CHARS ? `${text.slice(0, SAMPLE_CHARS)}…` : text}`;
+}
+function sharedKeys(objects) {
+  const [first, ...rest] = objects;
+  return Object.keys(first).filter(key => rest.every(object => Object.hasOwn(object, key)));
+}
+function childPath(path, key) {
+  return /^[$A-Z_][\w$]*$/i.test(key) ? `${path}.${key}` : `${path}['${key.replaceAll("'", String.raw`\'`)}']`;
+}
+function isObject(value) {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
 /** Rows shown in a probe, at most. */
 const ROW_LIMIT = 80;
 /**
@@ -358,10 +756,11 @@ function escape(text) {
  * @param status - The HTTP status.
  * @param findings - What `findData` found.
  * @param observed - JSON responses a browser saw, when `--browser` was used.
+ * @param html - The tables (and, for Markdown, the outline and front matter) found.
  * @returns The report.
  */
-function probeReport(url, status, findings, observed = []) {
-  const sections = [section('JSON-LD blocks', findings.jsonLd.map(block => `  type=${block.types}, ${block.keys} keys`)), section('Inline JSON (candidates)', findings.inlineJson.map(block => `  ${block.where}, ${block.size} chars, keys: ${block.keys.slice(0, 8).join(', ')}`)), section('.json URLs referenced', findings.jsonUrls.map(url_ => `  ${url_}`)), section('Script hosts', findings.scriptHosts.map(host => `  ${host}`)), section('Links that look like an API', findings.apiLinks.map(link => `  ${link}`)), observed.length > 0 ? section('JSON responses observed in the browser', observed.map(entry => `  ${entry}`)) : ''];
+function probeReport(url, status, findings, observed = [], html) {
+  const sections = [section('Markdown front matter (in script[data-front-matter])', html?.frontMatter === undefined || html.frontMatter.length === 0 ? [] : [`  keys: ${html.frontMatter.join(', ')}`]), section('Markdown sections (a "css" selector for each)', (html?.outline ?? []).map(entry => `  ${'  '.repeat(entry.level - 1)}${entry.selector}`)), section('HTML tables (a "table" extract selector for each)', (html?.tables ?? []).map(table => `  ${table.table}  ${table.selector}${table.hint === undefined ? '' : `  (${table.hint})`}\n        ${table.text}`)), section('JSON-LD blocks', findings.jsonLd.map(block => `  type=${block.types}, ${block.keys} keys`)), section('Inline JSON (candidates)', findings.inlineJson.map(block => `  ${block.where}, ${block.size} chars, keys: ${block.keys.slice(0, 8).join(', ')}`)), section('.json URLs referenced', findings.jsonUrls.map(url_ => `  ${url_}`)), section('Script hosts', findings.scriptHosts.map(host => `  ${host}`)), section('Links that look like an API', findings.apiLinks.map(link => `  ${link}`)), observed.length > 0 ? section('JSON responses observed in the browser', observed.map(entry => `  ${entry}`)) : ''];
   return [`${url} (HTTP ${status})`, '', ...sections].filter(line => line !== '').join('\n');
 }
 /**
@@ -374,6 +773,56 @@ function probeReport(url, status, findings, observed = []) {
 function pdfReport(url, pdf) {
   return [`${url} (PDF, ${pdf.pages} page${pdf.pages === 1 ? '' : 's'})`, '', section('Likely table headers (a "table" extract selector for each)', pdf.headers.map(header => `  p${header.page}  ${header.selector}\n        ${header.text}`)), section('First rows (cells separated by " | ")', pdf.rows.map(row => `  p${row.page}  ${row.text}`))].filter(line => line !== '').join('\n');
 }
+/**
+ * Formats what a probe found in a workbook (a CSV or a spreadsheet): its sheets,
+ * the rows that look like table headers with a selector for each, and the first
+ * rows.
+ *
+ * @param url - The workbook's URL.
+ * @param workbook - The findings.
+ * @returns The report.
+ */
+function workbookReport(url, workbook) {
+  const read = workbook.csv === undefined ? 'workbook' : `CSV, ${workbook.csv.encoding}, delimited by ${delimiterName(workbook.csv.delimiter)}`;
+  return [`${url} (${read})`, '', section('Sheets', workbook.sheets.map(sheet => `  ${sheet.name}${sheet.hidden ? ' (hidden)' : ''}: ${sheet.rows} rows × ${sheet.columns} columns`)), section('Likely table headers (a "table" extract selector for each)', workbook.headers.map(header => `  ${header.sheet} r${header.row}  ${header.selector}${header.hint === undefined ? '' : `  (${header.hint})`}\n        ${header.text}`)), section('First rows (cells separated by " | ")', workbook.rows.map(row => `  ${row.sheet} r${row.row}  ${row.text}`))].filter(line => line !== '').join('\n');
+}
+/**
+ * Formats what a probe found in a deck (a presentation): its slides, the native
+ * tables' headers with a selector for each, its charts, and the slides whose
+ * text boxes look like a table.
+ *
+ * @param url - The deck's URL.
+ * @param deck - The findings.
+ * @returns The report.
+ */
+function deckReport(url, deck) {
+  return [`${url} (presentation, ${deck.slides.length} slide${deck.slides.length === 1 ? '' : 's'}, ${deck.width} × ${deck.height} pt)`, '', section('Slides', deck.slides.map(slide => `  ${slide.number}  ${slide.title === '' ? '(no title)' : slide.title}${slide.hidden ? ' (hidden)' : ''}: ${slide.shapes} text boxes, ${slide.tables} tables, ${slide.charts} charts`)), section('Likely table headers (a "table" extract selector for each)', deck.headers.map(header => `  slide ${header.slide}  ${header.selector}${header.hint === undefined ? '' : `  (${header.hint})`}\n        ${header.text}`)), section('Text boxes laid out as a table (a "table" extract with "shapes": true)', deck.grids.map(grid => `  slide ${grid.slide}  ${grid.title}: ${grid.boxes} short boxes`)), section('Charts (read with jsonpath: $.slides[*].charts[*].series[*])', deck.charts.map(chart => `  slide ${chart.slide}  ${chart.type}${chart.title === undefined ? '' : ` "${chart.title}"`}: ${chart.series.map(series => `${series.name} (${series.points})`).join(', ')}`))].filter(line => line !== '').join('\n');
+}
+/**
+ * Formats what a probe found in JSON data: its record lists with the path a
+ * `jsonpath` extract needs, then its structure.
+ *
+ * @param url - The document's URL.
+ * @param json - The findings.
+ * @returns The report.
+ */
+function jsonReport(url, json) {
+  const read = {
+    json: 'JSON',
+    jsonl: 'JSON Lines',
+    yaml: 'YAML'
+  }[json.format] ?? json.format;
+  return [`${url} (${read}: ${json.type})`, '', section('Record lists (a "jsonpath" selector for each, largest first)', json.lists.map(list => `  ${list.path}  ${list.length} entries\n        keys: ${list.keys.join(', ')}`)), section('Structure', json.tree.map(line => `  ${line}`))].filter(line => line !== '').join('\n');
+}
+function delimiterName(delimiter) {
+  const names = {
+    '\t': 'tabs',
+    ',': 'commas',
+    ';': 'semicolons',
+    '|': 'pipes'
+  };
+  return names[delimiter] ?? `"${delimiter}"`;
+}
 function section(title, rows) {
   if (rows.length === 0) return '';
   return [`${title}:`, ...rows, ''].join('\n');
@@ -389,8 +838,8 @@ const OBSERVE_MS = 4000;
  * involved: the cli's `probePage` and `@opencraw/mcp`'s probe tool both
  * build on this, one printing the result, the other returning it as data.
  *
- * A PDF (by content type, or a local `.pdf` path) is read instead: its rows,
- * and the rows that look like table headers.
+ * A PDF, a spreadsheet, a CSV or a presentation (by content type, or a local
+ * path) is read instead: its rows, and the rows that look like table headers.
  *
  * @param url - The page to probe, or a local file path.
  * @param options - Browser path, TLS and user agent, plus whether to render.
@@ -398,7 +847,8 @@ const OBSERVE_MS = 4000;
  * @throws Error when the fetch itself fails.
  */
 async function probeUrl(url, options) {
-  const lease = await new AccessBroker(await resolveAccess(options)).lease({
+  const plugins = options.plugins === undefined ? undefined : await loadPlugins(options.plugins);
+  const lease = await new AccessBroker(await resolveAccess(options), plugins?.accessPlugins).lease({
     recipeId: 'probe'
   });
   if (lease.cdp !== undefined) throw new Error(`access profile "${lease.profile}" is a remote browser; probe fetches over HTTP and needs a proxy profile`);
@@ -410,8 +860,13 @@ async function probeUrl(url, options) {
   });
   try {
     const target = /^[a-z][\w+.-]+:/i.test(url) ? url : pathToFileURL(resolve(url)).href;
+    // GitHub raw and CDNs serve Markdown as text/plain: a .md URL is read as Markdown.
+    const markdown = /\.(?:md|markdown)$/i.test(new URL(target).pathname);
     const response = await client.send({
-      url: target
+      url: target,
+      ...(markdown && {
+        as: 'markdown'
+      })
     });
     const {
       body
@@ -423,13 +878,38 @@ async function probeUrl(url, options) {
       observed: [],
       pdf: describePdf(body)
     };
-    const text = body.kind === 'html' ? body.html : body.kind === 'text' ? body.text : JSON.stringify(body.data);
+    if (body.kind === 'workbook') return {
+      url: response.url,
+      status: response.status,
+      findings: findData(''),
+      observed: [],
+      workbook: describeWorkbook(body)
+    };
+    if (body.kind === 'deck') return {
+      url: response.url,
+      status: response.status,
+      findings: findData(''),
+      observed: [],
+      deck: describeDeck(body)
+    };
+    if (body.kind === 'json') return {
+      url: response.url,
+      status: response.status,
+      findings: findData(''),
+      observed: [],
+      json: describeJson(body.data, response.format ?? 'json')
+    };
+    const text = body.kind === 'html' ? body.html : body.text;
     const observed = options.browser && !target.startsWith('file:') ? await observeBrowserJson(url, options, lease) : [];
+    const html = body.kind === 'html' ? {
+      html: describeHtml(body.html, response.format === 'markdown')
+    } : {};
     return {
       url: response.url,
       status: response.status,
       findings: findData(text),
-      observed
+      observed,
+      ...html
     };
   } finally {
     await client.dispose();
@@ -446,7 +926,7 @@ async function probeUrl(url, options) {
 async function probePage(url, options, terminal) {
   try {
     const result = await probeUrl(url, options);
-    terminal.out(result.pdf === undefined ? probeReport(result.url, result.status, result.findings, result.observed) : pdfReport(result.url, result.pdf));
+    terminal.out(reportOf(result));
     return 0;
   } catch (error) {
     terminal.err(`probe failed: ${error instanceof Error ? error.message : String(error)}`);
@@ -483,50 +963,12 @@ async function observeBrowserJson(url, options, lease) {
     await browser.close();
   }
 }
-
-/**
- * Loads the hooks a recipe's `hook` steps and transforms call, from a
- * JavaScript module: its default export when that is an object of functions
- * (`export default { positive: v => v > 0 }`), else its named function exports
- * (`export function positive (v) { ... }`).
- *
- * The module runs with the caller's privileges, like any code the caller
- * imports: point this only at a file you trust.
- *
- * @param path - A `.mjs` / `.js` / `.cjs` file.
- * @param importModule - How to import it; `import()` unless a test says otherwise.
- * @returns The hooks by name.
- * @throws Error naming the file when it cannot be imported, exports no hooks,
- * or exports something under a hook's name that is not a function.
- */
-async function loadHooks(path, importModule = importByUrl) {
-  const url = pathToFileURL(resolve(path)).href;
-  let exported;
-  try {
-    exported = await importModule(url);
-  } catch (error) {
-    throw new Error(`${path}: cannot load hooks (${error.message})`, {
-      cause: error
-    });
-  }
-  const candidates = hookExports(exported);
-  const entries = Object.entries(candidates);
-  if (entries.length === 0) throw new Error(`${path}: exports no hooks (export default { name: function } or named functions)`);
-  const hooks = {};
-  for (const [name, value] of entries) {
-    if (typeof value !== 'function') throw new Error(`${path}: hook "${name}" is ${value === null ? 'null' : typeof value}, not a function`);
-    hooks[name] = value;
-  }
-  return hooks;
-}
-async function importByUrl(url) {
-  return await import(url);
-}
-function hookExports(exported) {
-  const fallback = exported.default;
-  if (typeof fallback === 'object' && fallback !== null && !Array.isArray(fallback)) return fallback;
-  const named = Object.entries(exported).filter(([name]) => name !== 'default' && name !== 'module.exports');
-  return Object.fromEntries(named);
+function reportOf(result) {
+  if (result.pdf !== undefined) return pdfReport(result.url, result.pdf);
+  if (result.workbook !== undefined) return workbookReport(result.url, result.workbook);
+  if (result.deck !== undefined) return deckReport(result.url, result.deck);
+  if (result.json !== undefined) return jsonReport(result.url, result.json);
+  return probeReport(result.url, result.status, result.findings, result.observed, result.html);
 }
 
 /**
@@ -600,10 +1042,10 @@ async function runRecipes(command, terminal) {
     return 1;
   }
   let access;
-  let hooks;
+  let plugins;
   try {
     access = await resolveAccess(command.options);
-    hooks = command.hooks === undefined ? undefined : await loadHooks(command.hooks);
+    plugins = command.options.plugins === undefined ? undefined : await loadPlugins(command.options.plugins);
   } catch (error) {
     terminal.err(error instanceof Error ? error.message : String(error));
     return 1;
@@ -614,7 +1056,15 @@ async function runRecipes(command, terminal) {
   const crawler = createCrawler({
     sink,
     access,
-    hooks,
+    throttle: throttleFor(command, access?.throttle),
+    profilesDir: command.profiles,
+    parallel: command.parallel,
+    retry: command.retries === undefined ? undefined : {
+      attempts: command.retries
+    },
+    hooks: plugins?.hooks,
+    accessPlugins: plugins?.accessPlugins,
+    captchaSolvers: plugins?.captchaSolvers,
     resume: command.resume,
     debug: command.dryRun,
     browser: {
@@ -643,6 +1093,23 @@ async function runRecipes(command, terminal) {
     await crawler.close();
   }
 }
+/** The access config's throttle, with `--host-delay` / `--host-concurrency` over its defaults. */
+function throttleFor(command, fromConfig) {
+  const {
+    delayMs,
+    concurrency
+  } = command.throttle;
+  if (delayMs === undefined && concurrency === undefined) return fromConfig;
+  return {
+    ...fromConfig,
+    ...(delayMs !== undefined && {
+      delayMs
+    }),
+    ...(concurrency !== undefined && {
+      concurrency
+    })
+  };
+}
 function onEvent(event, command, terminal) {
   if (command.trace) {
     const line = traceLine(event);
@@ -765,5 +1232,5 @@ function processTerminal() {
   };
 }
 
-export { USAGE, findData, loadForRun, loadHooks, main, parseArguments, probeReport, probeUrl, processTerminal, readRecipeFiles, resolveAccess };
+export { USAGE, findData, loadForRun, loadHooks, loadPlugins, main, parseArguments, probeReport, probeUrl, processTerminal, readRecipeFiles, resolveAccess };
 //# sourceMappingURL=index.esm.js.map

package/dist/src/arguments/command.contract.d.ts

@@ -10,6 +10,8 @@ export interface CommonOptions {
     access?: string;
     /** The access profile to use by default, overriding the config's `default`. */
     accessProfile?: string;
+    /** A plugins module: hooks, access plugins; `--hooks` / `OPENCRAW_PLUGINS` / `OPENCRAW_HOOKS` too. */
+    plugins?: string;
 }
 export type Command = {
     name: 'help';
@@ -30,9 +32,18 @@ export type Command = {
     dryRun: boolean;
     /** Run only the input recipes with these ids. */
     only: string[];
-    /** A JavaScript module exporting the hooks the recipes call; `OPENCRAW_HOOKS` when not given. */
-    hooks?: string;
     headed: boolean;
+    /** How many input recipes run at once. */
+    parallel?: number;
+    /** Tries per request for recipes whose `limits.retry` says nothing (1 turns retrying off). */
+    retries?: number;
+    /** Where `session.browserProfile` profiles live; `OPENCRAW_PROFILES` when not given. */
+    profiles?: string;
+    /** Per-site politeness across every recipe: overrides the access config's `throttle` defaults. */
+    throttle: {
+        delayMs?: number;
+        concurrency?: number;
+    };
     options: CommonOptions;
 } | {
     name: 'probe';

package/dist/src/arguments/parse-arguments.algorithm.d.ts

@@ -1,5 +1,5 @@
 import type { Command } from './command.contract.js';
-export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.\n  probe <url>                                 Fetch a page and report where its data lives.\n\nOptions for run\n  --out <file>        Write records to this JSON Lines file (default: stdout).\n  --append            Keep what --out holds and add to it (each line carries _key).\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n  --hooks <file>      A JavaScript module whose default export is { name: function }: the hooks\n                      the recipes call (or OPENCRAW_HOOKS). It runs as your code: trust it.\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --help, --version";
+export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.\n  probe <url>                                 Fetch a page and report where its data lives.\n\nOptions for run\n  --out <file>        Write records to this JSON Lines file (default: stdout).\n  --append            Keep what --out holds and add to it (each line carries _key).\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n  --parallel <n>      Run this many input recipes at once (default 1). Iterations inside a recipe\n                      follow its limits.concurrency.\n  --retries <n>       Tries per request that fails in passing (connection, timeout, 429, 5xx), for\n                      recipes whose limits.retry says nothing. Default 3; 1 turns retrying off.\n  --profiles <dir>    Where recipes' session.browserProfile profiles live (or OPENCRAW_PROFILES;\n                      default .opencraw/profiles).\n  --host-delay <ms>   At least this long between two requests to one site, across every recipe.\n  --host-concurrency <n>\n                      At most this many requests to one site in flight, across every recipe.\n                      Both override the access config's \"throttle\" defaults.\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --plugins <file>    A JavaScript module exporting hooks (the recipes' hook steps and transforms),\n                      accessPlugins (for { kind: \"plugin\" } access profiles) and captchaSolvers\n                      (for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /\n                      OPENCRAW_HOOKS. It runs as your code: trust it.\n  --help, --version";
 /**
  * Turns argv into a command.
  *

package/dist/src/hooks-module/index.d.ts

@@ -1,3 +1,3 @@
-export { loadHooks } from './load-hooks.repository.js';
-export type { ModuleImporter } from './load-hooks.repository.js';
+export { loadHooks, loadPlugins } from './load-hooks.repository.js';
+export type { ModuleImporter, PluginModule } from './load-hooks.repository.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/hooks-module/load-hooks.repository.d.ts

@@ -1,18 +1,45 @@
-import type { HookMap } from '@opencraw/core';
+import type { AccessPlugin, CaptchaSolver, HookMap } from '@opencraw/core';
+/** What a plugins module provides: hooks for `hook` steps and transforms, and plugins the engine calls. */
+export interface PluginModule {
+    hooks: HookMap;
+    /** Access plugins: `{ kind: "plugin", name }` profiles in the access config use them. */
+    accessPlugins: AccessPlugin[];
+    /** Captcha solvers: recipes name them in `session.captcha.solver` and `captcha` steps. */
+    captchaSolvers: CaptchaSolver[];
+}
 /**
- * Loads the hooks a recipe's `hook` steps and transforms call, from a
- * JavaScript module: its default export when that is an object of functions
- * (`export default { positive: v => v > 0 }`), else its named function exports
- * (`export function positive (v) { ... }`).
+ * Loads a plugins module: a JavaScript module whose named exports `hooks`
+ * (`{ name: function }`), `accessPlugins` (`[{ name, lease }]`) and
+ * `captchaSolvers` (`[{ name, solve }]`) give what recipes and access profiles
+ * refer to by name:
+ *
+ * ```js
+ * export const hooks = { positive: v => v > 0 }
+ * export const accessPlugins = [{ name: 'browserbase', lease: async () => ({ cdp: { endpoint } }) }]
+ * export const captchaSolvers = [{ name: 'capsolver', solve: async (challenge, { page }) => ({ status: 'solved' }) }]
+ * ```
+ *
+ * A module with none of those names is a bare hooks module, as before: its
+ * default export when that is an object of functions, else its named function
+ * exports.
  *
  * The module runs with the caller's privileges, like any code the caller
  * imports: point this only at a file you trust.
  *
  * @param path - A `.mjs` / `.js` / `.cjs` file.
  * @param importModule - How to import it; `import()` unless a test says otherwise.
+ * @returns The hooks and plugins.
+ * @throws Error naming the file when it cannot be imported, provides nothing,
+ * or provides something that is not what its name promises.
+ */
+export declare function loadPlugins(path: string, importModule?: ModuleImporter): Promise<PluginModule>;
+/**
+ * Loads the hooks of a hooks or plugins module (see {@link loadPlugins}).
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file.
+ * @param importModule - How to import it.
  * @returns The hooks by name.
- * @throws Error naming the file when it cannot be imported, exports no hooks,
- * or exports something under a hook's name that is not a function.
+ * @throws Error naming the file when it cannot be imported or exports no hooks.
  */
 export declare function loadHooks(path: string, importModule?: ModuleImporter): Promise<HookMap>;
 /** Imports a module by URL, returning its namespace. */

package/dist/src/index.d.ts

@@ -10,5 +10,6 @@ export type { RecipeFiles } from './recipe-files/index.js';
 export { loadForRun } from './run/index.js';
 export type { CommonOptions } from './arguments/index.js';
 export { resolveAccess } from './access/index.js';
-export { loadHooks } from './hooks-module/index.js';
+export { loadHooks, loadPlugins } from './hooks-module/index.js';
+export type { PluginModule } from './hooks-module/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/probe/deck-findings.mapper.d.ts

@@ -0,0 +1,46 @@
+import type { DeckDocument } from '@opencraw/core';
+/** What a probe shows of a deck (a presentation). */
+export interface DeckFindings {
+    width: number;
+    height: number;
+    slides: {
+        number: number;
+        title: string;
+        hidden: boolean;
+        shapes: number;
+        tables: number;
+        charts: number;
+    }[];
+    /** The likely header rows of native tables, with the `slide` and `selector` a `table` extract needs. */
+    headers: {
+        slide: number;
+        text: string;
+        selector: string;
+        hint?: string;
+    }[];
+    charts: {
+        slide: number;
+        type: string;
+        title?: string;
+        series: {
+            name: string;
+            points: number;
+        }[];
+    }[];
+    /** Slides whose short text boxes look like a table: read them with `"shapes": true`. */
+    grids: {
+        slide: number;
+        title: string;
+        boxes: number;
+    }[];
+}
+/**
+ * Summarises a deck for someone writing a recipe: its slides, where the native
+ * tables start, what the charts hold, and which slides lay text boxes out as
+ * a table.
+ *
+ * @param document - The read deck.
+ * @returns The findings.
+ */
+export declare function describeDeck(document: DeckDocument): DeckFindings;
+//# sourceMappingURL=deck-findings.mapper.d.ts.map

package/dist/src/probe/html-findings.mapper.d.ts

@@ -0,0 +1,28 @@
+/** What a probe shows of an HTML document's tables, and of rendered Markdown's sections and front matter. */
+export interface HtmlFindings {
+    /** The likely header rows of the `<table>`s, with the `selector` a `table` extract needs. */
+    tables: {
+        table: string;
+        text: string;
+        selector: string;
+        hint?: string;
+    }[];
+    /** Rendered Markdown: its headings, each with the selector of its section. */
+    outline?: {
+        level: number;
+        heading: string;
+        selector: string;
+    }[];
+    /** Rendered Markdown: the keys of its front matter. */
+    frontMatter?: string[];
+}
+/**
+ * Summarises an HTML document for a `table` extract, and, when it is rendered
+ * Markdown, its outline and front matter.
+ *
+ * @param html - The document.
+ * @param markdown - Whether it was rendered from Markdown.
+ * @returns The findings.
+ */
+export declare function describeHtml(html: string, markdown: boolean): HtmlFindings;
+//# sourceMappingURL=html-findings.mapper.d.ts.map

package/dist/src/probe/index.d.ts

@@ -2,7 +2,15 @@ export { probePage, probeUrl } from './probe-page.use-case.js';
 export type { ProbeResult } from './probe-page.use-case.js';
 export { findData } from './find-data.algorithm.js';
 export type { ProbeFindings, JsonLdFinding, InlineJsonFinding } from './find-data.algorithm.js';
-export { probeReport, pdfReport } from './probe-report.mapper.js';
+export { probeReport, pdfReport, workbookReport, deckReport, jsonReport } from './probe-report.mapper.js';
 export { describePdf } from './pdf-findings.mapper.js';
 export type { PdfFindings } from './pdf-findings.mapper.js';
+export { describeWorkbook } from './workbook-findings.mapper.js';
+export type { WorkbookFindings } from './workbook-findings.mapper.js';
+export { describeDeck } from './deck-findings.mapper.js';
+export type { DeckFindings } from './deck-findings.mapper.js';
+export { describeJson } from './json-findings.mapper.js';
+export { describeHtml } from './html-findings.mapper.js';
+export type { HtmlFindings } from './html-findings.mapper.js';
+export type { JsonFindings } from './json-findings.mapper.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/probe/json-findings.mapper.d.ts

@@ -0,0 +1,31 @@
+/** What a probe shows of JSON data (a JSON, JSON Lines or YAML body). */
+export interface JsonFindings {
+    /** What the body was read as. */
+    format: string;
+    /** `object`, `array (n)`, or a scalar type. */
+    type: string;
+    /**
+     * The document's structure down to four levels, one line per node:
+     * `$.data.products  array (1240) of object`, `$.data.products[*].name  string "Pandina"`.
+     */
+    tree: string[];
+    /**
+     * Every array of objects with two entries or more, largest first: the
+     * `forEach` or `each` a recipe needs, with the keys all its entries share.
+     */
+    lists: {
+        path: string;
+        length: number;
+        keys: string[];
+    }[];
+}
+/**
+ * Summarises JSON data for someone writing a recipe: its shape, and where its
+ * record lists are, so a `jsonpath` selector can be read off the result.
+ *
+ * @param data - The parsed document.
+ * @param format - What it was read as (`json`, `jsonl`, `yaml`).
+ * @returns The findings.
+ */
+export declare function describeJson(data: unknown, format: string): JsonFindings;
+//# sourceMappingURL=json-findings.mapper.d.ts.map

package/dist/src/probe/probe-page.use-case.d.ts

@@ -1,7 +1,11 @@
 import type { CommonOptions } from '../arguments/index.js';
 import type { Terminal } from '../terminal/index.js';
 import { findData } from './find-data.algorithm.js';
+import type { DeckFindings } from './deck-findings.mapper.js';
+import type { HtmlFindings } from './html-findings.mapper.js';
+import type { JsonFindings } from './json-findings.mapper.js';
 import type { PdfFindings } from './pdf-findings.mapper.js';
+import type { WorkbookFindings } from './workbook-findings.mapper.js';
 /** What fetching and, optionally, rendering a page found. */
 export interface ProbeResult {
     url: string;
@@ -11,6 +15,14 @@ export interface ProbeResult {
     observed: string[];
     /** Present when the URL is a PDF: its rows and likely table headers. */
     pdf?: PdfFindings;
+    /** Present when the URL is a CSV or a spreadsheet: its sheets, rows and likely table headers. */
+    workbook?: WorkbookFindings;
+    /** Present when the URL is a presentation: its slides, table headers, charts and text-box grids. */
+    deck?: DeckFindings;
+    /** Present when the URL is JSON, JSON Lines or YAML: its structure and record lists. */
+    json?: JsonFindings;
+    /** Present for HTML: its tables' headers; for rendered Markdown also its outline and front matter. */
+    html?: HtmlFindings;
 }
 /**
  * Fetches a page and finds where its data lives, through the access profile the
@@ -20,8 +32,8 @@ export interface ProbeResult {
  * involved: the cli's `probePage` and `@opencraw/mcp`'s probe tool both
  * build on this, one printing the result, the other returning it as data.
  *
- * A PDF (by content type, or a local `.pdf` path) is read instead: its rows,
- * and the rows that look like table headers.
+ * A PDF, a spreadsheet, a CSV or a presentation (by content type, or a local
+ * path) is read instead: its rows, and the rows that look like table headers.
  *
  * @param url - The page to probe, or a local file path.
  * @param options - Browser path, TLS and user agent, plus whether to render.

package/dist/src/probe/probe-report.mapper.d.ts

@@ -1,5 +1,9 @@
 import type { ProbeFindings } from './find-data.algorithm.js';
+import type { DeckFindings } from './deck-findings.mapper.js';
+import type { HtmlFindings } from './html-findings.mapper.js';
+import type { JsonFindings } from './json-findings.mapper.js';
 import type { PdfFindings } from './pdf-findings.mapper.js';
+import type { WorkbookFindings } from './workbook-findings.mapper.js';
 /**
  * Renders a probe's findings as text.
  *
@@ -7,9 +11,10 @@ import type { PdfFindings } from './pdf-findings.mapper.js';
  * @param status - The HTTP status.
  * @param findings - What `findData` found.
  * @param observed - JSON responses a browser saw, when `--browser` was used.
+ * @param html - The tables (and, for Markdown, the outline and front matter) found.
  * @returns The report.
  */
-export declare function probeReport(url: string, status: number, findings: ProbeFindings, observed?: string[]): string;
+export declare function probeReport(url: string, status: number, findings: ProbeFindings, observed?: string[], html?: HtmlFindings): string;
 /**
  * Renders what a probe found in a PDF as text.
  *
@@ -18,4 +23,33 @@ export declare function probeReport(url: string, status: number, findings: Probe
  * @returns The report.
  */
 export declare function pdfReport(url: string, pdf: PdfFindings): string;
+/**
+ * Formats what a probe found in a workbook (a CSV or a spreadsheet): its sheets,
+ * the rows that look like table headers with a selector for each, and the first
+ * rows.
+ *
+ * @param url - The workbook's URL.
+ * @param workbook - The findings.
+ * @returns The report.
+ */
+export declare function workbookReport(url: string, workbook: WorkbookFindings): string;
+/**
+ * Formats what a probe found in a deck (a presentation): its slides, the native
+ * tables' headers with a selector for each, its charts, and the slides whose
+ * text boxes look like a table.
+ *
+ * @param url - The deck's URL.
+ * @param deck - The findings.
+ * @returns The report.
+ */
+export declare function deckReport(url: string, deck: DeckFindings): string;
+/**
+ * Formats what a probe found in JSON data: its record lists with the path a
+ * `jsonpath` extract needs, then its structure.
+ *
+ * @param url - The document's URL.
+ * @param json - The findings.
+ * @returns The report.
+ */
+export declare function jsonReport(url: string, json: JsonFindings): string;
 //# sourceMappingURL=probe-report.mapper.d.ts.map

package/dist/src/probe/workbook-findings.mapper.d.ts

@@ -0,0 +1,43 @@
+import type { WorkbookDocument } from '@opencraw/core';
+/** What a probe shows of a workbook (a spreadsheet or a CSV). */
+export interface WorkbookFindings {
+    /** How a CSV was read. */
+    csv?: {
+        encoding: string;
+        delimiter: string;
+    };
+    sheets: {
+        name: string;
+        hidden: boolean;
+        rows: number;
+        columns: number;
+    }[];
+    /** The first non-empty rows of each visible sheet, cells separated by " | ". */
+    rows: {
+        sheet: string;
+        row: number;
+        text: string;
+    }[];
+    /**
+     * Rows of text cells (no digit) with a data row (a digit) just below: likely
+     * table headers, with the `sheet` and `selector` a `table` extract needs, and
+     * a hint when the header has merged cells (a header over two rows).
+     */
+    headers: {
+        sheet: string;
+        row: number;
+        text: string;
+        selector: string;
+        hint?: string;
+    }[];
+}
+/**
+ * Summarises a workbook for someone writing a recipe: its sheets, what their
+ * rows look like and where the tables start, so a `table` extract's `sheet`,
+ * `selector` and columns can be read off the result instead of guessed.
+ *
+ * @param document - The read workbook.
+ * @returns The findings.
+ */
+export declare function describeWorkbook(document: WorkbookDocument): WorkbookFindings;
+//# sourceMappingURL=workbook-findings.mapper.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/cli",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",
@@ -24,7 +24,7 @@
     "!**/*.js.map"
   ],
   "dependencies": {
-    "@opencraw/core": "^0.1.1",
+    "@opencraw/core": "^0.1.3",
     "playwright": "^1.63.0"
   },
   "publishConfig": {