@opencraw/cli 0.1.2 → 0.1.4

package/README.md

@@ -43,6 +43,10 @@ input recipes) and reports what happened.
 | `--dry-run` | One record per input recipe, printed with the scope it was mapped from — for checking a recipe under construction without a full run. |
 | `--trace` | Print the crawl trace to stderr. |
 | `--headed` | Show the browser instead of running headless. |
+| `--parallel <n>` | Run this many input recipes at once (default 1). Iterations inside a recipe follow its `limits.concurrency`. See [§3.9 of the authoring guide](../../docs/recipes/authoring.md#39-concurrency). |
+| `--retries <n>` | Tries per request that fails in passing (a dropped connection, a timeout, a 429, a 5xx), for recipes whose `limits.retry` says nothing. Default 3; `1` turns retrying off. See [§7 of the authoring guide](../../docs/recipes/authoring.md#7-policies-what-happens-when-something-is-missing-or-fails). |
+| `--profiles <dir>` | Where the persistent browser profiles of `session.browserProfile` live (or `OPENCRAW_PROFILES`; default `.opencraw/profiles`). See [access.md](../../docs/recipes/access.md#persistent-browser-profiles). |
+| `--host-delay <ms>`, `--host-concurrency <n>` | Per-site politeness across every recipe: at least `ms` between two requests to one site, at most `n` in flight. They override the access config's `throttle` defaults. See [access.md](../../docs/recipes/access.md#throttling-per-site). |
 | `--plugins <file>` (or `--hooks`) | A plugins module: named exports `hooks` (the recipes' hook steps and transforms), `accessPlugins` (for `{ kind: "plugin" }` access profiles) and `captchaSolvers`. A module whose default export is `{ name: function }` is read as hooks alone. `OPENCRAW_PLUGINS` / `OPENCRAW_HOOKS` when not given; `probe` reads it too. It runs as your code; load only files you trust. See [§6 of the authoring guide](../../docs/recipes/authoring.md#6-hooks). |
 
 ```sh
@@ -50,6 +54,33 @@ opencraw run recipes/ --out out/products.jsonl --trace
 opencraw run recipes/movie.output.json recipes/tmdb.input.json --dry-run
 ```
 
+### `diff`
+
+```sh
+opencraw diff last-week.jsonl today.jsonl --key model,trim --ignore scrapedAt
+opencraw run recipes/ --out today.jsonl --diff last-week.jsonl --changes changes.jsonl
+```
+
+Compares two runs' records **by key**, not line by line, so a reordered file is no change:
+
+```text
+1 added, 1 removed, 2 changed, 38 unchanged (41 records before, 41 now)
+- 600e · La Prima
+~ Avenger · Summit  price.amount: 24950 → 23950; inStock: true → false
+~ Pandina · Hybrid  title: "Pandina" → "Pandina Hybrid"
++ Grande Panda · Icon
+```
+
+- `diff <previous> <current>`: `--key` names the fields that identify a record (default: each line's `_key`,
+  which `--append` writes); `--ignore` leaves fields out; `--changes <file>` also writes every change as a
+  JSON line (`{ change, key, before?, after?, fields? }`).
+- `run --diff <previous>` compares the records this run emits with a previous file, taking the key and the
+  fields to ignore (`generated: now` / `uuid`) from the output recipe. The file is read before the crawl, so it
+  may be `--out` itself: `run … --out prices.jsonl --diff prices.jsonl` keeps one file and reports each change.
+  Not with `--resume`, which skips records.
+- A run that holds under half the records the previous one did gets a warning: when the source did not
+  shrink, the site changed and the recipe quietly stopped finding everything.
+
 ### `probe`
 
 Fetches a page and reports where its data lives: JSON-LD blocks, inline JSON objects, `.json` URLs
@@ -68,7 +99,10 @@ rows, and every row that looks like a table header, with the `selector` a `table
 also names the encoding and the delimiter it detected; for a presentation, its charts and the slides whose
 text boxes look like a table. JSON, JSON Lines and YAML show their structure instead, and every list of
 records with the `jsonpath` that walks it. An HTML page also lists its tables' header rows, and Markdown
-(a `.md` URL is read as Markdown even when served as `text/plain`) its sections and front matter keys.
+(a `.md` URL is read as Markdown even when served as `text/plain`) its sections and front matter keys. XML
+(a feed, a sitemap, an export) shows its namespaces with the `namespaces` line an `xpath` extract needs, the
+elements that repeat (its records), and its structure; a sitemap says how many pages it lists. A Word document
+(`.docx`) lists its sections and tables, like Markdown.
 
 ```sh
 opencraw probe https://example.com/price-list.pdf

package/dist/index.esm.js

@@ -1,7 +1,8 @@
 import { parseArgs } from 'node:util';
+import { writeFile } from 'node:fs/promises';
+import { readableKey, readRecordsFile, diffRecords, loadAccessConfig, htmlTableSheets, AccessBroker, HttpClient, parseXml, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, diffOptionsFor, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
 import { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
-import { loadAccessConfig, htmlTableSheets, AccessBroker, HttpClient, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
 import { readFileSync } from 'node:fs';
 
 const USAGE = `opencraw <command> [options]
@@ -10,6 +11,7 @@ Commands
   validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.
   run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.
   probe <url>                                 Fetch a page and report where its data lives.
+  diff <previous.jsonl> <current.jsonl>       Compare two runs' records by key: added, removed, changed.
 
 Options for run
   --out <file>        Write records to this JSON Lines file (default: stdout).
@@ -19,6 +21,23 @@ Options for run
   --dry-run           One record per input, printed with the scope it was mapped from.
   --trace             Print the crawl trace to stderr.
   --headed            Show the browser.
+  --parallel <n>      Run this many input recipes at once (default 1). Iterations inside a recipe
+                      follow its limits.concurrency.
+  --retries <n>       Tries per request that fails in passing (connection, timeout, 429, 5xx), for
+                      recipes whose limits.retry says nothing. Default 3; 1 turns retrying off.
+  --profiles <dir>    Where recipes' session.browserProfile profiles live (or OPENCRAW_PROFILES;
+                      default .opencraw/profiles).
+  --diff <file>       Compare this run's records with a previous run's JSON Lines file (it may be
+                      --out itself: it is read before the crawl). Keys come from the output recipe.
+  --host-delay <ms>   At least this long between two requests to one site, across every recipe.
+  --host-concurrency <n>
+                      At most this many requests to one site in flight, across every recipe.
+                      Both override the access config's "throttle" defaults.
+
+Options for diff
+  --key <a,b>         The fields that identify a record (default: each line's _key).
+  --ignore <a,b>      Fields left out of the comparison (a date the crawl stamps, say).
+  --changes <file>    Also write every change as a JSON line (run --diff takes it too).
 
 Options for probe
   --browser           Also render the page in a browser and list the JSON it fetches.
@@ -58,6 +77,33 @@ const OPTIONS = {
   'headed': {
     type: 'boolean'
   },
+  'profiles': {
+    type: 'string'
+  },
+  'retries': {
+    type: 'string'
+  },
+  'parallel': {
+    type: 'string'
+  },
+  'diff': {
+    type: 'string'
+  },
+  'changes': {
+    type: 'string'
+  },
+  'key': {
+    type: 'string'
+  },
+  'ignore': {
+    type: 'string'
+  },
+  'host-delay': {
+    type: 'string'
+  },
+  'host-concurrency': {
+    type: 'string'
+  },
   'hooks': {
     type: 'string'
   },
@@ -139,9 +185,23 @@ function parseArguments(argv, env = {}) {
           paths: rest
         };
       }
+    case 'diff':
+      {
+        if (rest.length !== 2) throw new Error('diff needs two JSON Lines files: the previous run, then the current one');
+        return {
+          name: 'diff',
+          previous: rest[0],
+          current: rest[1],
+          key: list(values.key),
+          ignore: list(values.ignore),
+          changes: values.changes
+        };
+      }
     case 'run':
       {
         if (rest.length === 0) throw new Error('run needs at least one recipe file or directory');
+        if (values.diff !== undefined && values.resume === true) throw new Error('--diff compares the records a run emits, and --resume skips some: use one or the other');
+        if (values.changes !== undefined && values.diff === undefined) throw new Error('--changes needs --diff (or the diff command)');
         if (values.resume === true && values.append !== true) throw new Error('--resume needs --append (and --out)');
         if ((values.append === true || values.resume === true) && values.out === undefined) throw new Error('--append and --resume need --out');
         return {
@@ -154,6 +214,15 @@ function parseArguments(argv, env = {}) {
           dryRun: values['dry-run'] === true,
           only: values.only ?? [],
           headed: values.headed === true,
+          profiles: values.profiles ?? nonEmpty(env.OPENCRAW_PROFILES),
+          retries: integer(values.retries, '--retries', 1),
+          parallel: integer(values.parallel, '--parallel', 1),
+          diff: values.diff,
+          changes: values.changes,
+          throttle: {
+            delayMs: integer(values['host-delay'], '--host-delay', 0),
+            concurrency: integer(values['host-concurrency'], '--host-concurrency', 1)
+          },
           options
         };
       }
@@ -173,10 +242,84 @@ function parseArguments(argv, env = {}) {
       }
   }
 }
+function integer(value, name, minimum) {
+  if (value === undefined) return undefined;
+  const number = Number(value);
+  if (!Number.isSafeInteger(number) || number < minimum) throw new Error(`${name} needs a whole number of at least ${minimum}, not "${value}"`);
+  return number;
+}
+function list(value) {
+  return value === undefined ? [] : value.split(',').map(entry => entry.trim()).filter(entry => entry !== '');
+}
 function nonEmpty(value) {
   return value === '' ? undefined : value;
 }
 
+/** Changes listed one by one, at most; the rest are counted. */
+const LIST_LIMIT = 50;
+/** A value longer than this is cut in the report. */
+const VALUE_CHARS = 60;
+/**
+ * A diff as people read it: the counts, a warning when the run lost most of
+ * its records, then one line per change, `-` removed, `~` changed (with each
+ * field's before → after), `+` added.
+ *
+ * @param diff - The diff.
+ * @param limit - Changes listed at most.
+ * @returns The lines.
+ */
+function diffReport(diff, limit = LIST_LIMIT) {
+  const lines = [`${diff.added} added, ${diff.removed} removed, ${diff.changed} changed, ${diff.unchanged} unchanged (${diff.counts.previous} records before, ${diff.counts.current} now)`];
+  if (diff.shrunk !== undefined) lines.push(`! ${diff.shrunk.current} records where the previous run had ${diff.shrunk.previous}: if the source did not shrink, the site may have changed and the recipe no longer finds everything`);
+  if (diff.repeated > 0) lines.push(`! ${diff.repeated} records repeat a key already seen and were not compared: is the key complete?`);
+  for (const change of diff.changes.slice(0, limit)) {
+    const key = readableKey(change.key);
+    if (change.change === 'added') lines.push(`+ ${key}`);else if (change.change === 'removed') lines.push(`- ${key}`);else lines.push(`~ ${key}  ${change.fields.map(field => fieldLine(field)).join('; ')}`);
+  }
+  if (diff.changes.length > limit) lines.push(`… and ${diff.changes.length - limit} more`);
+  return lines;
+}
+function fieldLine(change) {
+  return `${change.field}: ${shown(change.before)} → ${shown(change.after)}`;
+}
+function shown(value) {
+  const text = JSON.stringify(value) ?? 'null';
+  return text.length > VALUE_CHARS ? `${text.slice(0, VALUE_CHARS)}…` : text;
+}
+
+/**
+ * Compares two runs' JSON Lines files and prints what changed; with
+ * `--changes`, writes every change as a JSON line too.
+ *
+ * @param command - The parsed `diff` command.
+ * @param terminal - Where the report goes.
+ * @returns The exit code: 0, or 1 when a file cannot be read or has no key.
+ */
+async function diffFiles(command, terminal) {
+  try {
+    const [previous, current] = await Promise.all([readRecordsFile(command.previous), readRecordsFile(command.current)]);
+    const diff = diffRecords(previous, current, {
+      key: command.key.length === 0 ? undefined : command.key,
+      ignore: command.ignore
+    });
+    for (const line of diffReport(diff)) terminal.out(line);
+    if (command.changes !== undefined) await writeChanges(command.changes, diff);
+    return 0;
+  } catch (error) {
+    terminal.err(error instanceof Error ? error.message : String(error));
+    return 1;
+  }
+}
+/**
+ * Writes a diff's changes as JSON Lines: `{ change, key, before?, after?, fields? }`.
+ *
+ * @param path - The file.
+ * @param diff - The diff.
+ */
+async function writeChanges(path, diff) {
+  await writeFile(path, diff.changes.map(change => `${JSON.stringify(change)}\n`).join(''));
+}
+
 /**
  * The access config a command runs with: the file `--access` (or
  * `OPENCRAW_ACCESS`) names, with `--access-profile` as its default.
@@ -555,10 +698,10 @@ const SECTION = /<section data-heading="([^"]*)" data-level="(\d)">/g;
 const FRONT_MATTER = /<script type="application\/json" data-front-matter>([\s\S]*?)<\/script>/;
 /**
  * Summarises an HTML document for a `table` extract, and, when it is rendered
- * Markdown, its outline and front matter.
+ * Markdown or a Word document, its outline (and Markdown's front matter).
  *
  * @param html - The document.
- * @param markdown - Whether it was rendered from Markdown.
+ * @param markdown - Whether it was rendered from Markdown or Word, and so has sections.
  * @returns The findings.
  */
 function describeHtml(html, markdown) {
@@ -594,13 +737,13 @@ function unescape(text) {
 }
 
 /** How deep a probe describes a JSON document. */
-const DEPTH = 4;
+const DEPTH$1 = 4;
 /** Keys shown per object, at most. */
 const KEY_LIMIT = 25;
 /** Entries of a record list whose keys are compared. */
 const SAMPLE = 50;
 /** A sample value longer than this is cut. */
-const SAMPLE_CHARS = 40;
+const SAMPLE_CHARS$1 = 40;
 /**
  * Summarises JSON data for someone writing a recipe: its shape, and where its
  * record lists are, so a `jsonpath` selector can be read off the result.
@@ -612,7 +755,7 @@ const SAMPLE_CHARS = 40;
 function describeJson(data, format) {
   const tree = [];
   const lists = [];
-  walk(data, '$', 0, tree, lists);
+  walk$1(data, '$', 0, tree, lists);
   return {
     format,
     type: typeOf(data),
@@ -620,7 +763,7 @@ function describeJson(data, format) {
     lists: lists.sort((first, second) => second.length - first.length)
   };
 }
-function walk(value, path, depth, tree, lists) {
+function walk$1(value, path, depth, tree, lists) {
   if (Array.isArray(value)) {
     const objects = value.filter(entry => isObject(entry));
     if (objects.length >= 2 && objects.length === value.length) lists.push({
@@ -629,13 +772,13 @@ function walk(value, path, depth, tree, lists) {
       keys: sharedKeys(objects.slice(0, SAMPLE))
     });
     if (depth > 0) tree.push(`${path}  ${typeOf(value)}`);
-    if (depth < DEPTH && value.length > 0) walk(value[0], `${path}[*]`, depth + 1, tree, lists);
+    if (depth < DEPTH$1 && value.length > 0) walk$1(value[0], `${path}[*]`, depth + 1, tree, lists);
     return;
   }
   if (isObject(value)) {
     if (depth > 0) tree.push(`${path}  object`);
-    if (depth >= DEPTH) return;
-    for (const [key, child] of Object.entries(value).slice(0, KEY_LIMIT)) walk(child, childPath(path, key), depth + 1, tree, lists);
+    if (depth >= DEPTH$1) return;
+    for (const [key, child] of Object.entries(value).slice(0, KEY_LIMIT)) walk$1(child, childPath(path, key), depth + 1, tree, lists);
     return;
   }
   tree.push(`${path}  ${typeOf(value)}${sample(value)}`);
@@ -651,7 +794,7 @@ function typeOf(value) {
 function sample(value) {
   if (value === null || value === undefined) return '';
   const text = JSON.stringify(value);
-  return ` ${text.length > SAMPLE_CHARS ? `${text.slice(0, SAMPLE_CHARS)}…` : text}`;
+  return ` ${text.length > SAMPLE_CHARS$1 ? `${text.slice(0, SAMPLE_CHARS$1)}…` : text}`;
 }
 function sharedKeys(objects) {
   const [first, ...rest] = objects;
@@ -722,7 +865,7 @@ function escape(text) {
  * @returns The report.
  */
 function probeReport(url, status, findings, observed = [], html) {
-  const sections = [section('Markdown front matter (in script[data-front-matter])', html?.frontMatter === undefined || html.frontMatter.length === 0 ? [] : [`  keys: ${html.frontMatter.join(', ')}`]), section('Markdown sections (a "css" selector for each)', (html?.outline ?? []).map(entry => `  ${'  '.repeat(entry.level - 1)}${entry.selector}`)), section('HTML tables (a "table" extract selector for each)', (html?.tables ?? []).map(table => `  ${table.table}  ${table.selector}${table.hint === undefined ? '' : `  (${table.hint})`}\n        ${table.text}`)), section('JSON-LD blocks', findings.jsonLd.map(block => `  type=${block.types}, ${block.keys} keys`)), section('Inline JSON (candidates)', findings.inlineJson.map(block => `  ${block.where}, ${block.size} chars, keys: ${block.keys.slice(0, 8).join(', ')}`)), section('.json URLs referenced', findings.jsonUrls.map(url_ => `  ${url_}`)), section('Script hosts', findings.scriptHosts.map(host => `  ${host}`)), section('Links that look like an API', findings.apiLinks.map(link => `  ${link}`)), observed.length > 0 ? section('JSON responses observed in the browser', observed.map(entry => `  ${entry}`)) : ''];
+  const sections = [section('Markdown front matter (in script[data-front-matter])', html?.frontMatter === undefined || html.frontMatter.length === 0 ? [] : [`  keys: ${html.frontMatter.join(', ')}`]), section('Sections (a "css" selector for each)', (html?.outline ?? []).map(entry => `  ${'  '.repeat(entry.level - 1)}${entry.selector}`)), section('HTML tables (a "table" extract selector for each)', (html?.tables ?? []).map(table => `  ${table.table}  ${table.selector}${table.hint === undefined ? '' : `  (${table.hint})`}\n        ${table.text}`)), section('JSON-LD blocks', findings.jsonLd.map(block => `  type=${block.types}, ${block.keys} keys`)), section('Inline JSON (candidates)', findings.inlineJson.map(block => `  ${block.where}, ${block.size} chars, keys: ${block.keys.slice(0, 8).join(', ')}`)), section('.json URLs referenced', findings.jsonUrls.map(url_ => `  ${url_}`)), section('Script hosts', findings.scriptHosts.map(host => `  ${host}`)), section('Links that look like an API', findings.apiLinks.map(link => `  ${link}`)), observed.length > 0 ? section('JSON responses observed in the browser', observed.map(entry => `  ${entry}`)) : ''];
   return [`${url} (HTTP ${status})`, '', ...sections].filter(line => line !== '').join('\n');
 }
 /**
@@ -776,6 +919,20 @@ function jsonReport(url, json) {
   }[json.format] ?? json.format;
   return [`${url} (${read}: ${json.type})`, '', section('Record lists (a "jsonpath" selector for each, largest first)', json.lists.map(list => `  ${list.path}  ${list.length} entries\n        keys: ${list.keys.join(', ')}`)), section('Structure', json.tree.map(line => `  ${line}`))].filter(line => line !== '').join('\n');
 }
+/**
+ * Formats what a probe found in an XML document: its namespaces and the
+ * `namespaces` an `xpath` extract needs, the elements that repeat, and its structure.
+ *
+ * @param url - The document's URL.
+ * @param xml - The findings.
+ * @returns The report.
+ */
+function xmlReport(url, xml) {
+  const defaultNamespace = xml.namespaces[''];
+  const namespaceRows = Object.entries(xml.namespaces).map(([prefix, uri]) => `  ${prefix === '' ? '(default)' : prefix}  ${uri}`);
+  const hint = defaultNamespace === undefined ? [] : ['', `  The elements are in a default namespace: query with "namespaces": { "x": "${defaultNamespace}" } and //x:${xml.root},`, '  or with "ignoreNamespaces": true and plain names, as below.'];
+  return [`${url} (XML: ${xml.root}${xml.sitemap === undefined ? '' : `, a sitemap of ${xml.sitemap.locations} ${xml.sitemap.kind === 'urlset' ? 'pages' : 'sitemaps'}`})`, '', section('Namespaces', [...namespaceRows, ...hint]), section('Repeated elements (an "xpath" selector for each, largest first)', xml.lists.map(list => `  ${list.path}  ${list.count} entries${list.children.length > 0 ? `\n        children: ${list.children.join(', ')}` : ''}`)), section('Structure', xml.tree.map(line => `  ${line}`))].filter(line => line !== '').join('\n');
+}
 function delimiterName(delimiter) {
   const names = {
     '\t': 'tabs',
@@ -790,6 +947,86 @@ function section(title, rows) {
   return [`${title}:`, ...rows, ''].join('\n');
 }
 
+/** How deep a probe describes an XML document. */
+const DEPTH = 5;
+/** Distinct paths shown, at most. */
+const PATH_LIMIT = 60;
+/** A sample value longer than this is cut. */
+const SAMPLE_CHARS = 40;
+const ELEMENT = 1;
+/**
+ * Summarises an XML document for someone writing a recipe: its namespaces
+ * (and the `namespaces` a query needs), the elements that repeat (the
+ * records a `forEach` walks), and its structure.
+ *
+ * @param document - A parsed document (`parseXml` from the core), read through the W3C DOM properties.
+ * @returns The findings.
+ */
+function describeXml(document) {
+  const root = document.documentElement;
+  if (root === null) return {
+    root: '',
+    namespaces: {},
+    lists: [],
+    tree: []
+  };
+  const paths = new Map();
+  walk(root, `/${root.localName ?? ''}`, 1, paths);
+  const namespaces = Object.fromEntries(Array.from(root.attributes ?? [], attribute => attribute).filter(attribute => attribute.name === 'xmlns' || attribute.prefix === 'xmlns').map(attribute => [attribute.name === 'xmlns' ? '' : attribute.localName, attribute.value]));
+  const lists = [...paths].filter(([, info]) => info.maxSiblings >= 2).map(([path, info]) => ({
+    path: `//${path.split('/').slice(-2).join('/')}`.replace(/^\/\/\//, '//'),
+    count: info.count,
+    children: [...info.children]
+  })).sort((first, second) => second.count - first.count);
+  const tree = [...paths].slice(0, PATH_LIMIT).map(([path, info]) => `${path}${info.count > 1 ? `  ×${info.count}` : ''}${info.attributes.size > 0 ? `  ${[...info.attributes].map(name => `@${name}`).join(' ')}` : ''}${info.sample === undefined ? '' : `  ${JSON.stringify(info.sample)}`}`);
+  const name = root.localName ?? '';
+  const sitemap = name === 'urlset' || name === 'sitemapindex' ? {
+    kind: name,
+    locations: paths.get(`/${name}/${name === 'urlset' ? 'url' : 'sitemap'}/loc`)?.count ?? 0
+  } : undefined;
+  return {
+    root: name,
+    namespaces,
+    lists,
+    tree,
+    ...(sitemap !== undefined && {
+      sitemap
+    })
+  };
+}
+function walk(element, path, depth, paths) {
+  const info = paths.get(path) ?? {
+    count: 0,
+    attributes: new Set(),
+    children: new Set(),
+    maxSiblings: 1
+  };
+  paths.set(path, info);
+  info.count += 1;
+  const attributes = element.attributes ?? [];
+  for (const attribute of attributes) {
+    if (attribute.name !== 'xmlns' && attribute.prefix !== 'xmlns') info.attributes.add(attribute.name);
+  }
+  const children = [...element.childNodes].filter(child => child.nodeType === ELEMENT);
+  if (children.length === 0) {
+    const text = (element.textContent ?? '').replaceAll(/\s+/g, ' ').trim();
+    if (text !== '' && info.sample === undefined) info.sample = text.length > SAMPLE_CHARS ? `${text.slice(0, SAMPLE_CHARS)}…` : text;
+    return;
+  }
+  const siblings = new Map();
+  for (const child of children) {
+    const name = child.localName ?? '';
+    info.children.add(name);
+    siblings.set(name, (siblings.get(name) ?? 0) + 1);
+  }
+  for (const child of children) {
+    const childPath = `${path}/${child.localName ?? ''}`;
+    if (depth < DEPTH) walk(child, childPath, depth + 1, paths);
+    const childInfo = paths.get(childPath);
+    if (childInfo !== undefined) childInfo.maxSiblings = Math.max(childInfo.maxSiblings, siblings.get(child.localName ?? '') ?? 1);
+  }
+}
+
 const BROWSER_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36';
 const OBSERVE_MS = 4000;
 /**
@@ -861,10 +1098,17 @@ async function probeUrl(url, options) {
       observed: [],
       json: describeJson(body.data, response.format ?? 'json')
     };
+    if (body.kind === 'xml') return {
+      url: response.url,
+      status: response.status,
+      findings: findData(''),
+      observed: [],
+      xml: describeXml(parseXml(body.xml, response.url))
+    };
     const text = body.kind === 'html' ? body.html : body.text;
     const observed = options.browser && !target.startsWith('file:') ? await observeBrowserJson(url, options, lease) : [];
     const html = body.kind === 'html' ? {
-      html: describeHtml(body.html, response.format === 'markdown')
+      html: describeHtml(body.html, response.format === 'markdown' || response.format === 'docx')
     } : {};
     return {
       url: response.url,
@@ -930,6 +1174,7 @@ function reportOf(result) {
   if (result.workbook !== undefined) return workbookReport(result.url, result.workbook);
   if (result.deck !== undefined) return deckReport(result.url, result.deck);
   if (result.json !== undefined) return jsonReport(result.url, result.json);
+  if (result.xml !== undefined) return xmlReport(result.url, result.xml);
   return probeReport(result.url, result.status, result.findings, result.observed, result.html);
 }
 
@@ -1005,9 +1250,13 @@ async function runRecipes(command, terminal) {
   }
   let access;
   let plugins;
+  let previous;
+  const emitted = [];
   try {
     access = await resolveAccess(command.options);
     plugins = command.options.plugins === undefined ? undefined : await loadPlugins(command.options.plugins);
+    // Read before the crawl: --diff may name the very file --out is about to overwrite.
+    previous = command.diff === undefined ? undefined : await readRecordsFile(command.diff);
   } catch (error) {
     terminal.err(error instanceof Error ? error.message : String(error));
     return 1;
@@ -1018,6 +1267,12 @@ async function runRecipes(command, terminal) {
   const crawler = createCrawler({
     sink,
     access,
+    throttle: throttleFor(command, access?.throttle),
+    profilesDir: command.profiles,
+    parallel: command.parallel,
+    retry: command.retries === undefined ? undefined : {
+      attempts: command.retries
+    },
     hooks: plugins?.hooks,
     accessPlugins: plugins?.accessPlugins,
     captchaSolvers: plugins?.captchaSolvers,
@@ -1028,7 +1283,10 @@ async function runRecipes(command, terminal) {
       executablePath: command.options.browserPath,
       ignoreHTTPSErrors: command.options.insecureTls
     },
-    onEvent: event => onEvent(event, command, terminal)
+    onEvent: event => {
+      if (previous !== undefined && event.type === 'record:emit') emitted.push(event.data);
+      onEvent(event, command, terminal);
+    }
   });
   try {
     const inputs = command.dryRun ? set.inputs.map(input => ({
@@ -1044,11 +1302,34 @@ async function runRecipes(command, terminal) {
       terminal.err(`${recipe.recipeId}: ${recipe.emitted} emitted, ${recipe.rejected} rejected, ${recipe.duplicates} duplicates, ${recipe.skipped} skipped, ${recipe.stepsSkipped > 0 ? `${recipe.stepsSkipped} steps skipped (see --trace), ` : ''}${recipe.pages} pages, ${recipe.durationMs} ms${recipe.error === undefined ? '' : `, stopped: ${recipe.error}`}`);
     }
     if (!command.dryRun) terminal.err(`${report.records} records${report.sink.location === undefined ? '' : ` written to ${report.sink.location}`}`);
+    if (previous !== undefined) {
+      const diff = diffRecords(previous, emitted, diffOptionsFor(set.output));
+      terminal.err(`compared with ${command.diff ?? ''}:`);
+      for (const line of diffReport(diff)) terminal.err(`  ${line}`);
+      if (command.changes !== undefined) await writeChanges(command.changes, diff);
+    }
     return report.recipes.some(recipe => recipe.error !== undefined) ? 1 : 0;
   } finally {
     await crawler.close();
   }
 }
+/** The access config's throttle, with `--host-delay` / `--host-concurrency` over its defaults. */
+function throttleFor(command, fromConfig) {
+  const {
+    delayMs,
+    concurrency
+  } = command.throttle;
+  if (delayMs === undefined && concurrency === undefined) return fromConfig;
+  return {
+    ...fromConfig,
+    ...(delayMs !== undefined && {
+      delayMs
+    }),
+    ...(concurrency !== undefined && {
+      concurrency
+    })
+  };
+}
 function onEvent(event, command, terminal) {
   if (command.trace) {
     const line = traceLine(event);
@@ -1156,6 +1437,10 @@ async function main(argv, terminal) {
           ...command.options
         }, terminal);
       }
+    case 'diff':
+      {
+        return diffFiles(command, terminal);
+      }
   }
 }
 
@@ -1171,5 +1456,5 @@ function processTerminal() {
   };
 }
 
-export { USAGE, findData, loadForRun, loadHooks, loadPlugins, main, parseArguments, probeReport, probeUrl, processTerminal, readRecipeFiles, resolveAccess };
+export { USAGE, diffReport, findData, loadForRun, loadHooks, loadPlugins, main, parseArguments, probeReport, probeUrl, processTerminal, readRecipeFiles, resolveAccess, writeChanges };
 //# sourceMappingURL=index.esm.js.map

package/dist/src/arguments/command.contract.d.ts

@@ -20,6 +20,16 @@ export type Command = {
 } | {
     name: 'validate';
     paths: string[];
+} | {
+    name: 'diff';
+    previous: string;
+    current: string;
+    /** The fields that identify a record; empty uses each line's `_key`. */
+    key: string[];
+    /** Fields left out of the comparison. */
+    ignore: string[];
+    /** Where to write the changes as JSON Lines. */
+    changes?: string;
 } | {
     name: 'run';
     paths: string[];
@@ -33,6 +43,21 @@ export type Command = {
     /** Run only the input recipes with these ids. */
     only: string[];
     headed: boolean;
+    /** A previous run's JSON Lines file to compare this run's records with. */
+    diff?: string;
+    /** Where to write the changes `diff` finds, as JSON Lines. */
+    changes?: string;
+    /** How many input recipes run at once. */
+    parallel?: number;
+    /** Tries per request for recipes whose `limits.retry` says nothing (1 turns retrying off). */
+    retries?: number;
+    /** Where `session.browserProfile` profiles live; `OPENCRAW_PROFILES` when not given. */
+    profiles?: string;
+    /** Per-site politeness across every recipe: overrides the access config's `throttle` defaults. */
+    throttle: {
+        delayMs?: number;
+        concurrency?: number;
+    };
     options: CommonOptions;
 } | {
     name: 'probe';

package/dist/src/arguments/parse-arguments.algorithm.d.ts

@@ -1,5 +1,5 @@
 import type { Command } from './command.contract.js';
-export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.\n  probe <url>                                 Fetch a page and report where its data lives.\n\nOptions for run\n  --out <file>        Write records to this JSON Lines file (default: stdout).\n  --append            Keep what --out holds and add to it (each line carries _key).\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --plugins <file>    A JavaScript module exporting hooks (the recipes' hook steps and transforms),\n                      accessPlugins (for { kind: \"plugin\" } access profiles) and captchaSolvers\n                      (for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /\n                      OPENCRAW_HOOKS. It runs as your code: trust it.\n  --help, --version";
+export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.\n  probe <url>                                 Fetch a page and report where its data lives.\n  diff <previous.jsonl> <current.jsonl>       Compare two runs' records by key: added, removed, changed.\n\nOptions for run\n  --out <file>        Write records to this JSON Lines file (default: stdout).\n  --append            Keep what --out holds and add to it (each line carries _key).\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n  --parallel <n>      Run this many input recipes at once (default 1). Iterations inside a recipe\n                      follow its limits.concurrency.\n  --retries <n>       Tries per request that fails in passing (connection, timeout, 429, 5xx), for\n                      recipes whose limits.retry says nothing. Default 3; 1 turns retrying off.\n  --profiles <dir>    Where recipes' session.browserProfile profiles live (or OPENCRAW_PROFILES;\n                      default .opencraw/profiles).\n  --diff <file>       Compare this run's records with a previous run's JSON Lines file (it may be\n                      --out itself: it is read before the crawl). Keys come from the output recipe.\n  --host-delay <ms>   At least this long between two requests to one site, across every recipe.\n  --host-concurrency <n>\n                      At most this many requests to one site in flight, across every recipe.\n                      Both override the access config's \"throttle\" defaults.\n\nOptions for diff\n  --key <a,b>         The fields that identify a record (default: each line's _key).\n  --ignore <a,b>      Fields left out of the comparison (a date the crawl stamps, say).\n  --changes <file>    Also write every change as a JSON line (run --diff takes it too).\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --plugins <file>    A JavaScript module exporting hooks (the recipes' hook steps and transforms),\n                      accessPlugins (for { kind: \"plugin\" } access profiles) and captchaSolvers\n                      (for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /\n                      OPENCRAW_HOOKS. It runs as your code: trust it.\n  --help, --version";
 /**
  * Turns argv into a command.
  *

package/dist/src/diff/diff-files.use-case.d.ts

@@ -0,0 +1,22 @@
+import type { RecordDiff } from '@opencraw/core';
+import type { Command } from '../arguments/index.js';
+import type { Terminal } from '../terminal/index.js';
+/**
+ * Compares two runs' JSON Lines files and prints what changed; with
+ * `--changes`, writes every change as a JSON line too.
+ *
+ * @param command - The parsed `diff` command.
+ * @param terminal - Where the report goes.
+ * @returns The exit code: 0, or 1 when a file cannot be read or has no key.
+ */
+export declare function diffFiles(command: Extract<Command, {
+    name: 'diff';
+}>, terminal: Terminal): Promise<number>;
+/**
+ * Writes a diff's changes as JSON Lines: `{ change, key, before?, after?, fields? }`.
+ *
+ * @param path - The file.
+ * @param diff - The diff.
+ */
+export declare function writeChanges(path: string, diff: RecordDiff): Promise<void>;
+//# sourceMappingURL=diff-files.use-case.d.ts.map

package/dist/src/diff/diff-report.mapper.d.ts

@@ -0,0 +1,12 @@
+import type { RecordDiff } from '@opencraw/core';
+/**
+ * A diff as people read it: the counts, a warning when the run lost most of
+ * its records, then one line per change, `-` removed, `~` changed (with each
+ * field's before → after), `+` added.
+ *
+ * @param diff - The diff.
+ * @param limit - Changes listed at most.
+ * @returns The lines.
+ */
+export declare function diffReport(diff: RecordDiff, limit?: number): string[];
+//# sourceMappingURL=diff-report.mapper.d.ts.map

package/dist/src/diff/index.d.ts

@@ -0,0 +1,3 @@
+export { diffFiles, writeChanges } from './diff-files.use-case.js';
+export { diffReport } from './diff-report.mapper.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts

@@ -8,6 +8,7 @@ export type { ProbeFindings, ProbeResult } from './probe/index.js';
 export { readRecipeFiles } from './recipe-files/index.js';
 export type { RecipeFiles } from './recipe-files/index.js';
 export { loadForRun } from './run/index.js';
+export { diffReport, writeChanges } from './diff/index.js';
 export type { CommonOptions } from './arguments/index.js';
 export { resolveAccess } from './access/index.js';
 export { loadHooks, loadPlugins } from './hooks-module/index.js';

package/dist/src/probe/html-findings.mapper.d.ts

@@ -18,10 +18,10 @@ export interface HtmlFindings {
 }
 /**
  * Summarises an HTML document for a `table` extract, and, when it is rendered
- * Markdown, its outline and front matter.
+ * Markdown or a Word document, its outline (and Markdown's front matter).
  *
  * @param html - The document.
- * @param markdown - Whether it was rendered from Markdown.
+ * @param markdown - Whether it was rendered from Markdown or Word, and so has sections.
  * @returns The findings.
  */
 export declare function describeHtml(html: string, markdown: boolean): HtmlFindings;

package/dist/src/probe/index.d.ts

@@ -2,7 +2,9 @@ export { probePage, probeUrl } from './probe-page.use-case.js';
 export type { ProbeResult } from './probe-page.use-case.js';
 export { findData } from './find-data.algorithm.js';
 export type { ProbeFindings, JsonLdFinding, InlineJsonFinding } from './find-data.algorithm.js';
-export { probeReport, pdfReport, workbookReport, deckReport, jsonReport } from './probe-report.mapper.js';
+export { probeReport, pdfReport, workbookReport, deckReport, jsonReport, xmlReport } from './probe-report.mapper.js';
+export { describeXml } from './xml-findings.mapper.js';
+export type { XmlFindings } from './xml-findings.mapper.js';
 export { describePdf } from './pdf-findings.mapper.js';
 export type { PdfFindings } from './pdf-findings.mapper.js';
 export { describeWorkbook } from './workbook-findings.mapper.js';

package/dist/src/probe/probe-page.use-case.d.ts

@@ -5,6 +5,7 @@ import type { DeckFindings } from './deck-findings.mapper.js';
 import type { HtmlFindings } from './html-findings.mapper.js';
 import type { JsonFindings } from './json-findings.mapper.js';
 import type { PdfFindings } from './pdf-findings.mapper.js';
+import type { XmlFindings } from './xml-findings.mapper.js';
 import type { WorkbookFindings } from './workbook-findings.mapper.js';
 /** What fetching and, optionally, rendering a page found. */
 export interface ProbeResult {
@@ -23,6 +24,8 @@ export interface ProbeResult {
     json?: JsonFindings;
     /** Present for HTML: its tables' headers; for rendered Markdown also its outline and front matter. */
     html?: HtmlFindings;
+    /** Present for XML: its namespaces, repeated elements and structure. */
+    xml?: XmlFindings;
 }
 /**
  * Fetches a page and finds where its data lives, through the access profile the

package/dist/src/probe/probe-report.mapper.d.ts

@@ -2,6 +2,7 @@ import type { ProbeFindings } from './find-data.algorithm.js';
 import type { DeckFindings } from './deck-findings.mapper.js';
 import type { HtmlFindings } from './html-findings.mapper.js';
 import type { JsonFindings } from './json-findings.mapper.js';
+import type { XmlFindings } from './xml-findings.mapper.js';
 import type { PdfFindings } from './pdf-findings.mapper.js';
 import type { WorkbookFindings } from './workbook-findings.mapper.js';
 /**
@@ -52,4 +53,13 @@ export declare function deckReport(url: string, deck: DeckFindings): string;
  * @returns The report.
  */
 export declare function jsonReport(url: string, json: JsonFindings): string;
+/**
+ * Formats what a probe found in an XML document: its namespaces and the
+ * `namespaces` an `xpath` extract needs, the elements that repeat, and its structure.
+ *
+ * @param url - The document's URL.
+ * @param xml - The findings.
+ * @returns The report.
+ */
+export declare function xmlReport(url: string, xml: XmlFindings): string;
 //# sourceMappingURL=probe-report.mapper.d.ts.map

package/dist/src/probe/xml-findings.mapper.d.ts

@@ -0,0 +1,30 @@
+/** What a probe shows of an XML document. */
+export interface XmlFindings {
+    /** The root element's local name. */
+    root: string;
+    /** Declared on the root: prefix (`''` for the default namespace) to URI. */
+    namespaces: Record<string, string>;
+    /** Elements that repeat under one parent, largest first: the records, with the child names they share. */
+    lists: {
+        path: string;
+        count: number;
+        children: string[];
+    }[];
+    /** One line per distinct element path (local names), with its count, attributes and a sample text. */
+    tree: string[];
+    /** For a sitemap (`urlset`, `sitemapindex`): how many locations it lists. */
+    sitemap?: {
+        kind: 'urlset' | 'sitemapindex';
+        locations: number;
+    };
+}
+/**
+ * Summarises an XML document for someone writing a recipe: its namespaces
+ * (and the `namespaces` a query needs), the elements that repeat (the
+ * records a `forEach` walks), and its structure.
+ *
+ * @param document - A parsed document (`parseXml` from the core), read through the W3C DOM properties.
+ * @returns The findings.
+ */
+export declare function describeXml(document: unknown): XmlFindings;
+//# sourceMappingURL=xml-findings.mapper.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/cli",
-  "version": "0.1.2",
+  "version": "0.1.4",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",
@@ -24,7 +24,7 @@
     "!**/*.js.map"
   ],
   "dependencies": {
-    "@opencraw/core": "^0.1.2",
+    "@opencraw/core": "^0.1.4",
     "playwright": "^1.63.0"
   },
   "publishConfig": {