@opencraw/cli 0.1.0

package/README.md

@@ -0,0 +1,82 @@
+# @opencraw/cli
+
+Command-line tools for [`@opencraw/core`](../core): probe a page for scrapable data, validate recipes,
+and run a crawl from the terminal.
+
+## Install
+
+```sh
+npm install -g @opencraw/cli
+# or, inside this workspace: node packages/cli/bin/opencraw.mjs <command>
+```
+
+## Commands
+
+```
+opencraw validate <recipe files or directories...>
+opencraw run <recipe files or directories...> [options]
+opencraw probe <url> [options]
+```
+
+### `validate`
+
+Loads every recipe file (or every `.json` and `.jsonl` file in a directory; a `.jsonl` file holds one
+recipe per line), parses it against its schema and binds
+the input recipes to the output recipe, printing every problem with its JSON path. Exit code 1 if anything
+fails.
+
+```sh
+opencraw validate recipes/
+```
+
+### `run`
+
+Crawls the recipes at the given paths (files or directories; exactly one output recipe, any number of
+input recipes) and reports what happened.
+
+| Flag | Meaning |
+|---|---|
+| `--out <file>` | Write records to this JSON Lines file. Without it, records print to stdout as JSON Lines. |
+| `--append` | Keep what `--out` holds and add to it; each line carries `_key`. |
+| `--resume` | Skip records `--out` already has (needs `--append`). |
+| `--only <id>` | Run only this input recipe; repeat for more than one. |
+| `--dry-run` | One record per input recipe, printed with the scope it was mapped from — for checking a recipe under construction without a full run. |
+| `--trace` | Print the crawl trace to stderr. |
+| `--headed` | Show the browser instead of running headless. |
+| `--hooks <file>` | A JavaScript module whose default export is `{ name: function }`: the hooks the recipes call (or `OPENCRAW_HOOKS`). It runs as your code; load only files you trust. See [§6 of the authoring guide](../../docs/recipes/authoring.md#6-hooks). |
+
+```sh
+opencraw run recipes/ --out out/products.jsonl --trace
+opencraw run recipes/movie.output.json recipes/tmdb.input.json --dry-run
+```
+
+### `probe`
+
+Fetches a page and reports where its data lives: JSON-LD blocks, inline JSON objects, `.json` URLs
+referenced in the markup, script hosts, and links that look like an API. With `--browser`, it also renders
+the page in a browser and lists the JSON responses it observes while the page settles — useful for
+endpoints only a script fetches after load.
+
+```sh
+opencraw probe https://example.com/product/1
+opencraw probe https://example.com/configurator --browser
+```
+
+Use it before writing an input recipe, to find the shape a site's data actually takes (§9 of
+[the authoring guide](../../docs/recipes/authoring.md)).
+
+### Options common to `run` and `probe`
+
+| Flag | Meaning |
+|---|---|
+| `--browser-path <path>` | A browser binary other than the one Playwright installed (or `OPENCRAW_CHROMIUM`). |
+| `--insecure-tls` | Accept an intercepting proxy's certificate (or `OPENCRAW_INSECURE_TLS=1`). |
+| `--access <file>` | An access config: proxy profiles, with credentials as `{{env.NAME}}` (or `OPENCRAW_ACCESS`). See [access.md](../../docs/recipes/access.md). |
+| `--access-profile <name>` | The profile used by recipes that name none, overriding the config's `default`. `probe` uses it for its fetch. |
+| `--user-agent <ua>` | The user agent to send. |
+
+## Building
+
+Run `nx build @opencraw/cli` to build this project, `nx test @opencraw/cli` for its unit tests, and
+`nx run @opencraw/cli:e2e` for the spawn-driven suite against the fixture shop `@opencraw/core` ships
+(needs a browser: `npm run playwright:install` once, or `OPENCRAW_CHROMIUM=/path/to/chrome`).

package/bin/opencraw.mjs

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { main, processTerminal } from '../dist/index.esm.js'
+
+process.exitCode = await main(process.argv.slice(2), processTerminal())

package/dist/index.d.ts

@@ -0,0 +1 @@
+export * from "./src/index.js";

package/dist/index.esm.js

@@ -0,0 +1,699 @@
+import { parseArgs } from 'node:util';
+import { loadAccessConfig, AccessBroker, HttpClient, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+
+const USAGE = `opencraw <command> [options]
+
+Commands
+  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.
+  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.
+  probe <url>                                 Fetch a page and report where its data lives.
+
+Options for run
+  --out <file>        Write records to this JSON Lines file (default: stdout).
+  --append            Keep what --out holds and add to it (each line carries _key).
+  --resume            Skip records --out already has (needs --append).
+  --only <id>         Run only this input recipe (repeatable).
+  --dry-run           One record per input, printed with the scope it was mapped from.
+  --trace             Print the crawl trace to stderr.
+  --headed            Show the browser.
+  --hooks <file>      A JavaScript module whose default export is { name: function }: the hooks
+                      the recipes call (or OPENCRAW_HOOKS). It runs as your code: trust it.
+
+Options for probe
+  --browser           Also render the page in a browser and list the JSON it fetches.
+
+Options for both
+  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).
+  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).
+  --user-agent <ua>   The user agent to send.
+  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).
+  --access-profile <name>
+                      The access profile to use when a recipe names none.
+  --help, --version`;
+const OPTIONS = {
+  'out': {
+    type: 'string'
+  },
+  'append': {
+    type: 'boolean'
+  },
+  'resume': {
+    type: 'boolean'
+  },
+  'only': {
+    type: 'string',
+    multiple: true
+  },
+  'dry-run': {
+    type: 'boolean'
+  },
+  'trace': {
+    type: 'boolean'
+  },
+  'headed': {
+    type: 'boolean'
+  },
+  'hooks': {
+    type: 'string'
+  },
+  'browser': {
+    type: 'boolean'
+  },
+  'browser-path': {
+    type: 'string'
+  },
+  'insecure-tls': {
+    type: 'boolean'
+  },
+  'user-agent': {
+    type: 'string'
+  },
+  'access': {
+    type: 'string'
+  },
+  'access-profile': {
+    type: 'string'
+  },
+  'help': {
+    type: 'boolean',
+    short: 'h'
+  },
+  'version': {
+    type: 'boolean',
+    short: 'v'
+  }
+};
+/**
+ * Turns argv into a command.
+ *
+ * @param argv - The arguments after the program name.
+ * @param env - The environment, for the defaults some options have.
+ * @returns The command.
+ * @throws Error with a usage message on bad input.
+ */
+function parseArguments(argv, env = {}) {
+  const {
+    values,
+    positionals
+  } = parseArgs({
+    args: [...argv],
+    options: OPTIONS,
+    allowPositionals: true,
+    strict: true
+  });
+  if (values.help === true) return {
+    name: 'help'
+  };
+  if (values.version === true) return {
+    name: 'version'
+  };
+  const [name, ...rest] = positionals;
+  const options = {
+    browserPath: values['browser-path'] ?? env.OPENCRAW_CHROMIUM,
+    insecureTls: values['insecure-tls'] === true || env.OPENCRAW_INSECURE_TLS === '1',
+    userAgent: values['user-agent'],
+    access: values.access ?? (env.OPENCRAW_ACCESS === '' ? undefined : env.OPENCRAW_ACCESS),
+    accessProfile: values['access-profile']
+  };
+  switch (name) {
+    case undefined:
+      {
+        return {
+          name: 'help'
+        };
+      }
+    case 'validate':
+      {
+        if (rest.length === 0) throw new Error('validate needs at least one recipe file or directory');
+        return {
+          name: 'validate',
+          paths: rest
+        };
+      }
+    case 'run':
+      {
+        if (rest.length === 0) throw new Error('run needs at least one recipe file or directory');
+        if (values.resume === true && values.append !== true) throw new Error('--resume needs --append (and --out)');
+        if ((values.append === true || values.resume === true) && values.out === undefined) throw new Error('--append and --resume need --out');
+        return {
+          name: 'run',
+          paths: rest,
+          out: values.out,
+          append: values.append === true,
+          resume: values.resume === true,
+          trace: values.trace === true,
+          dryRun: values['dry-run'] === true,
+          only: values.only ?? [],
+          headed: values.headed === true,
+          hooks: values.hooks ?? (env.OPENCRAW_HOOKS === '' ? undefined : env.OPENCRAW_HOOKS),
+          options
+        };
+      }
+    case 'probe':
+      {
+        if (rest.length !== 1) throw new Error('probe needs exactly one URL');
+        return {
+          name: 'probe',
+          url: rest[0],
+          browser: values.browser === true,
+          options
+        };
+      }
+    default:
+      {
+        throw new Error(`unknown command "${name}"`);
+      }
+  }
+}
+
+/**
+ * The access config a command runs with: the file `--access` (or
+ * `OPENCRAW_ACCESS`) names, with `--access-profile` as its default.
+ *
+ * @param options - The command's shared options.
+ * @returns The config, or `undefined` for direct access.
+ * @throws Error when a profile is asked for without a config, or is not in it.
+ */
+async function resolveAccess(options) {
+  if (options.access === undefined) {
+    if (options.accessProfile !== undefined) throw new Error('--access-profile needs an access config: --access <file> or OPENCRAW_ACCESS');
+    return undefined;
+  }
+  const config = await loadAccessConfig(options.access);
+  if (options.accessProfile === undefined) return config;
+  if (!Object.hasOwn(config.profiles, options.accessProfile)) {
+    throw new Error(`access profile "${options.accessProfile}" is not in ${options.access} (profiles: ${Object.keys(config.profiles).join(', ') || 'none'})`);
+  }
+  return {
+    ...config,
+    default: options.accessProfile
+  };
+}
+
+/**
+ * Finds where a fetched page's data lives: JSON-LD blocks, inline JSON objects
+ * big enough to matter, URLs that look like a data endpoint, script hosts and
+ * links whose path suggests an API. Pattern-matching only; it never fetches.
+ */
+const MIN_INLINE_SIZE = 200;
+const JSON_LD = /<script[^\s>]*\s+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+const INLINE_SCRIPT = /<script(?![^>]*\ssrc=)(?![^>]*\stype=["']application\/ld\+json["'])[^>]*>([\s\S]*?)<\/script>/gi;
+const DATA_ATTR = /data-[\w-]+=["'](\{[\s\S]*?\})["']/g;
+const SCRIPT_SRC = /<script[^\s>]*\s+src=["']([^"']+)["']/gi;
+const HREF_OR_JSON_URL = /["'](https?:\/\/[^"'\s]+\.json[^"'\s]*|\/[^"'\s]*\.json[^"'\s]*)["']/g;
+const LINK = /<a[^\s>]*\s+href=["']([^"']+)["']/gi;
+/**
+ * Scans a page's HTML for data-shaped things.
+ *
+ * @param html - The document.
+ * @returns What was found, each list capped at a handful of entries.
+ */
+function findData(html) {
+  return {
+    jsonLd: jsonLdBlocks(html),
+    inlineJson: inlineJsonBlocks(html),
+    jsonUrls: unique(matchesOf(HREF_OR_JSON_URL, html)).slice(0, 10),
+    scriptHosts: scriptHosts(html),
+    apiLinks: apiLinks(html)
+  };
+}
+function jsonLdBlocks(html) {
+  const findings = [];
+  for (const text of matchesOf(JSON_LD, html)) {
+    const parsed = tryParse(text);
+    if (parsed === undefined) continue;
+    findings.push(...entriesOf(parsed));
+  }
+  return findings.slice(0, 10);
+}
+/** Every plain object in a JSON-LD block (it may be one object or an array of them), as a finding. */
+function entriesOf(parsed) {
+  const list = Array.isArray(parsed) ? parsed : [parsed];
+  return list.flatMap(entry => {
+    if (typeof entry !== 'object' || entry === null) return [];
+    const record = entry;
+    const type = record['@type'];
+    return [{
+      types: Array.isArray(type) ? type.join(',') : String(type ?? '?'),
+      keys: Object.keys(record).length
+    }];
+  });
+}
+function inlineJsonBlocks(html) {
+  const findings = [];
+  for (const text of matchesOf(INLINE_SCRIPT, html)) findings.push(...objectsIn(text, 'inline script'));
+  for (const text of matchesOf(DATA_ATTR, html, 1)) findings.push(...objectsIn(text, 'data-* attribute'));
+  return findings.filter(finding => finding.size >= MIN_INLINE_SIZE).sort((a, b) => b.size - a.size).slice(0, 10);
+}
+/** Every `{...}` in a text that parses as JSON, by scanning for balanced braces (regex cannot nest). */
+function objectsIn(text, where) {
+  const found = [];
+  for (let at = text.indexOf('{'); at !== -1; at = text.indexOf('{', at + 1)) {
+    const end = matchingBrace(text, at);
+    if (end === -1) continue;
+    const candidate = text.slice(at, end + 1);
+    const parsed = tryParse(candidate);
+    if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
+      found.push({
+        where,
+        keys: Object.keys(parsed),
+        size: candidate.length
+      });
+      at = end;
+    }
+  }
+  return found;
+}
+function matchingBrace(text, open) {
+  let depth = 0;
+  for (let at = open; at < text.length; at += 1) {
+    if (text[at] === '{') depth += 1;else if (text[at] === '}') {
+      depth -= 1;
+      if (depth === 0) return at;
+    }
+  }
+  return -1;
+}
+function scriptHosts(html) {
+  const hosts = matchesOf(SCRIPT_SRC, html).map(src => {
+    try {
+      return new URL(src).host;
+    } catch {
+      return;
+    }
+  }).filter(host => host !== undefined);
+  return unique(hosts).slice(0, 10);
+}
+function apiLinks(html) {
+  return unique(matchesOf(LINK, html).filter(href => /\bapi\b/i.test(href))).slice(0, 10);
+}
+function matchesOf(pattern, text, group = 1) {
+  return Array.from(text.matchAll(pattern), match => match[group]);
+}
+function unique(values) {
+  return [...new Set(values)];
+}
+function tryParse(text) {
+  try {
+    return JSON.parse(text.trim());
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Renders a probe's findings as text.
+ *
+ * @param url - The URL probed.
+ * @param status - The HTTP status.
+ * @param findings - What `findData` found.
+ * @param observed - JSON responses a browser saw, when `--browser` was used.
+ * @returns The report.
+ */
+function probeReport(url, status, findings, observed = []) {
+  const sections = [section('JSON-LD blocks', findings.jsonLd.map(block => `  type=${block.types}, ${block.keys} keys`)), section('Inline JSON (candidates)', findings.inlineJson.map(block => `  ${block.where}, ${block.size} chars, keys: ${block.keys.slice(0, 8).join(', ')}`)), section('.json URLs referenced', findings.jsonUrls.map(url_ => `  ${url_}`)), section('Script hosts', findings.scriptHosts.map(host => `  ${host}`)), section('Links that look like an API', findings.apiLinks.map(link => `  ${link}`)), observed.length > 0 ? section('JSON responses observed in the browser', observed.map(entry => `  ${entry}`)) : ''];
+  return [`${url} (HTTP ${status})`, '', ...sections].filter(line => line !== '').join('\n');
+}
+function section(title, rows) {
+  if (rows.length === 0) return '';
+  return [`${title}:`, ...rows, ''].join('\n');
+}
+
+const BROWSER_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36';
+const OBSERVE_MS = 4000;
+/**
+ * Fetches a page and finds where its data lives, through the access profile the
+ * options name (direct without one). With `browser: true` it also
+ * renders the page and lists the JSON responses seen while it settles, which
+ * finds endpoints a plain fetch of the initial HTML cannot. No `Terminal`
+ * involved: the cli's `probePage` and `@opencraw/mcp`'s probe tool both
+ * build on this, one printing the result, the other returning it as data.
+ *
+ * @param url - The page to probe.
+ * @param options - Browser path, TLS and user agent, plus whether to render.
+ * @returns What was found.
+ * @throws Error when the fetch itself fails.
+ */
+async function probeUrl(url, options) {
+  const lease = await new AccessBroker(await resolveAccess(options)).lease({
+    recipeId: 'probe'
+  });
+  if (lease.cdp !== undefined) throw new Error(`access profile "${lease.profile}" is a remote browser; probe fetches over HTTP and needs a proxy profile`);
+  const client = await HttpClient.open({
+    userAgent: options.userAgent ?? BROWSER_USER_AGENT,
+    ignoreHTTPSErrors: options.insecureTls || lease.ignoreHTTPSErrors === true,
+    proxy: lease.proxy,
+    headers: lease.headers
+  });
+  try {
+    const response = await client.send({
+      url,
+      as: 'html'
+    });
+    const html = response.body.kind === 'html' ? response.body.html : '';
+    const observed = options.browser ? await observeBrowserJson(url, options, lease) : [];
+    return {
+      url: response.url,
+      status: response.status,
+      findings: findData(html),
+      observed
+    };
+  } finally {
+    await client.dispose();
+  }
+}
+/**
+ * Fetches a page and reports where its data lives, to a `Terminal`.
+ *
+ * @param url - The page to probe.
+ * @param options - Browser path, TLS and user agent, plus whether to render.
+ * @param terminal - Where the report goes.
+ * @returns The exit code: 0, or 1 on a fetch failure.
+ */
+async function probePage(url, options, terminal) {
+  try {
+    const result = await probeUrl(url, options);
+    terminal.out(probeReport(result.url, result.status, result.findings, result.observed));
+    return 0;
+  } catch (error) {
+    terminal.err(`probe failed: ${error instanceof Error ? error.message : String(error)}`);
+    return 1;
+  }
+}
+async function observeBrowserJson(url, options, lease) {
+  const browser = await BrowserClient.launch({
+    executablePath: options.browserPath,
+    ignoreHTTPSErrors: options.insecureTls
+  });
+  try {
+    const session = await browser.newSession({
+      userAgent: options.userAgent,
+      proxy: lease.proxy,
+      headers: lease.headers,
+      ignoreHTTPSErrors: lease.ignoreHTTPSErrors
+    });
+    const seen = [];
+    session.page.on('response', response => {
+      if ((response.headers()['content-type'] ?? '').includes('json')) seen.push(`${response.status()} ${response.url()}`);
+    });
+    try {
+      await session.page.goto(url, {
+        waitUntil: 'networkidle'
+      });
+    } catch {
+      // a page that never idles (long-poll, websocket) still yields whatever the browser observed
+    }
+    await session.page.waitForTimeout(OBSERVE_MS);
+    await session.close();
+    return [...new Set(seen)].slice(0, 15);
+  } finally {
+    await browser.close();
+  }
+}
+
+/**
+ * Loads the hooks a recipe's `hook` steps and transforms call, from a
+ * JavaScript module: its default export when that is an object of functions
+ * (`export default { positive: v => v > 0 }`), else its named function exports
+ * (`export function positive (v) { ... }`).
+ *
+ * The module runs with the caller's privileges, like any code the caller
+ * imports: point this only at a file you trust.
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file.
+ * @param importModule - How to import it; `import()` unless a test says otherwise.
+ * @returns The hooks by name.
+ * @throws Error naming the file when it cannot be imported, exports no hooks,
+ * or exports something under a hook's name that is not a function.
+ */
+async function loadHooks(path, importModule = importByUrl) {
+  const url = pathToFileURL(resolve(path)).href;
+  let exported;
+  try {
+    exported = await importModule(url);
+  } catch (error) {
+    throw new Error(`${path}: cannot load hooks (${error.message})`, {
+      cause: error
+    });
+  }
+  const candidates = hookExports(exported);
+  const entries = Object.entries(candidates);
+  if (entries.length === 0) throw new Error(`${path}: exports no hooks (export default { name: function } or named functions)`);
+  const hooks = {};
+  for (const [name, value] of entries) {
+    if (typeof value !== 'function') throw new Error(`${path}: hook "${name}" is ${value === null ? 'null' : typeof value}, not a function`);
+    hooks[name] = value;
+  }
+  return hooks;
+}
+async function importByUrl(url) {
+  return await import(url);
+}
+function hookExports(exported) {
+  const fallback = exported.default;
+  if (typeof fallback === 'object' && fallback !== null && !Array.isArray(fallback)) return fallback;
+  const named = Object.entries(exported).filter(([name]) => name !== 'default' && name !== 'module.exports');
+  return Object.fromEntries(named);
+}
+
+/**
+ * Reads recipes, expanding directories to their `.json` and `.jsonl` files
+ * (sorted, not recursive), and sorts them by `kind`. The command line passes
+ * paths; the MCP server also passes recipes it was handed inline.
+ *
+ * @param sources - Files or directories, or any other recipe source core reads.
+ * @returns The decoded recipes.
+ * @throws Error naming a file that is not JSON or JSON Lines.
+ */
+async function readRecipeFiles(sources) {
+  const result = {
+    outputs: [],
+    inputs: [],
+    others: []
+  };
+  const documents = await readRecipeSource(sources);
+  for (const document of documents) {
+    const kind = document.content?.kind;
+    if (kind === 'output') result.outputs.push({
+      path: document.source,
+      recipe: document.content
+    });else if (kind === 'input') result.inputs.push({
+      path: document.source,
+      recipe: document.content
+    });else result.others.push(document.source);
+  }
+  return result;
+}
+
+/**
+ * Loads and binds the recipes `run` needs, filtered to the input ids in `only`
+ * when given.
+ *
+ * @param sources - Recipe files or directories (or recipes the MCP server was handed inline).
+ * @param only - Input recipe ids to keep; empty keeps all.
+ * @returns The bound set.
+ * @throws Error when there is not exactly one output recipe.
+ * @throws RecipeValidationError, RecipeBindingError on a bad recipe.
+ */
+async function loadForRun(sources, only) {
+  const files = await readRecipeFiles(sources);
+  if (files.outputs.length !== 1) throw new Error(`expected exactly one output recipe, found ${files.outputs.length}`);
+  const output = parseOutputRecipe(files.outputs[0].recipe, files.outputs[0].path);
+  const inputs = files.inputs.map(file => parseInputRecipe(file.recipe, file.path)).filter(input => only.length === 0 || only.includes(input.id));
+  return bindRecipeSet(output, inputs);
+}
+
+/**
+ * Crawls the recipes at `paths` and reports what happened.
+ *
+ * `--dry-run` caps every input at one record, attaches the scope each record
+ * was mapped from, and prints them side by side instead of writing them, so a
+ * recipe under construction can be checked without a full run.
+ *
+ * @param command - The parsed `run` command.
+ * @param terminal - Where records, the trace and errors go.
+ * @returns The exit code: 0, or 1 on a load failure or a recipe error.
+ */
+async function runRecipes(command, terminal) {
+  let set;
+  try {
+    set = await loadForRun(command.paths, command.only);
+  } catch (error) {
+    for (const line of loadErrorLines(error)) terminal.err(line);
+    return 1;
+  }
+  if (set.inputs.length === 0) {
+    terminal.err(command.only.length > 0 ? `no input recipe matches --only ${command.only.join(', ')}` : 'no input recipes found');
+    return 1;
+  }
+  let access;
+  let hooks;
+  try {
+    access = await resolveAccess(command.options);
+    hooks = command.hooks === undefined ? undefined : await loadHooks(command.hooks);
+  } catch (error) {
+    terminal.err(error instanceof Error ? error.message : String(error));
+    return 1;
+  }
+  const sink = command.dryRun || command.out === undefined ? memorySink() : jsonLinesSink(command.out, {
+    append: command.append
+  });
+  const crawler = createCrawler({
+    sink,
+    access,
+    hooks,
+    resume: command.resume,
+    debug: command.dryRun,
+    browser: {
+      headless: !command.headed,
+      executablePath: command.options.browserPath,
+      ignoreHTTPSErrors: command.options.insecureTls
+    },
+    onEvent: event => onEvent(event, command, terminal)
+  });
+  try {
+    const inputs = command.dryRun ? set.inputs.map(input => ({
+      ...input,
+      limits: {
+        ...input.limits,
+        maxRecords: 1
+      }
+    })) : set.inputs;
+    const report = await crawler.run(new RecipeSet(set.output, inputs));
+    if (command.out === undefined && !command.dryRun) for (const record of sink.records) terminal.out(JSON.stringify(record.data));
+    for (const recipe of report.recipes) {
+      terminal.err(`${recipe.recipeId}: ${recipe.emitted} emitted, ${recipe.rejected} rejected, ${recipe.duplicates} duplicates, ${recipe.skipped} skipped, ${recipe.pages} pages, ${recipe.durationMs} ms${recipe.error === undefined ? '' : `, stopped: ${recipe.error}`}`);
+    }
+    if (!command.dryRun) terminal.err(`${report.records} records${report.sink.location === undefined ? '' : ` written to ${report.sink.location}`}`);
+    return report.recipes.some(recipe => recipe.error !== undefined) ? 1 : 0;
+  } finally {
+    await crawler.close();
+  }
+}
+function onEvent(event, command, terminal) {
+  if (command.trace) {
+    const line = traceLine(event);
+    if (line !== undefined) terminal.err(line);
+  }
+  if (command.dryRun && event.type === 'record:emit') {
+    terminal.out(`--- ${event.recipeId} ---`);
+    terminal.out(`scope: ${JSON.stringify(event.scope, null, 2)}`);
+    terminal.out(`record: ${JSON.stringify(event.data, null, 2)}`);
+  }
+  if (command.dryRun && event.type === 'record:reject') terminal.out(`--- ${event.recipeId}: rejected (${event.field}: ${event.reason}) ---\nscope: ${JSON.stringify(event.scope, null, 2)}`);
+}
+function loadErrorLines(error) {
+  if (error instanceof RecipeValidationError) return error.issues.map(issue => `${error.source}: ${issue.path === '' ? '(root)' : issue.path}: ${issue.message}`);
+  if (error instanceof RecipeBindingError) return error.issues.map(issue => `${issue.recipeId}: ${issue.path}: ${issue.message}`);
+  return [error instanceof Error ? error.message : String(error)];
+}
+
+/**
+ * Loads and binds recipe files, printing every problem with its JSON path.
+ *
+ * @param paths - Recipe files or directories.
+ * @param terminal - Where results go.
+ * @returns The exit code: 0 when the recipes bind cleanly, 1 otherwise.
+ */
+async function validateRecipes(paths, terminal) {
+  const files = await readRecipeFiles(paths);
+  for (const path of files.others) terminal.err(`${path}: not a recipe ("kind" is missing or not "input"/"output")`);
+  if (files.outputs.length === 0) {
+    terminal.err('no output recipe found');
+    return 1;
+  }
+  if (files.outputs.length > 1) {
+    terminal.err(`more than one output recipe: ${files.outputs.map(file => file.path).join(', ')}`);
+    return 1;
+  }
+  let ok = true;
+  const output = parseOne(files.outputs[0].path, files.outputs[0].recipe, parseOutputRecipe, terminal);
+  const inputs = files.inputs.flatMap(file => {
+    const parsed = parseOne(file.path, file.recipe, parseInputRecipe, terminal);
+    return parsed === undefined ? [] : [parsed];
+  });
+  ok &&= output !== undefined && inputs.length === files.inputs.length;
+  if (output === undefined) return 1;
+  try {
+    const set = bindRecipeSet(output, inputs);
+    terminal.out(`ok: ${set.output.id} <- ${set.inputs.map(input => input.id).join(', ') || '(no inputs)'}`);
+  } catch (error) {
+    ok = false;
+    if (error instanceof RecipeBindingError) for (const issue of error.issues) terminal.err(`${issue.recipeId}: ${issue.path}: ${issue.message}`);else terminal.err(String(error));
+  }
+  return ok ? 0 : 1;
+}
+function parseOne(path, recipe, parse, terminal) {
+  try {
+    return parse(recipe, path);
+  } catch (error) {
+    if (error instanceof RecipeValidationError) for (const issue of error.issues) terminal.err(`${path}: ${issue.path}: ${issue.message}`);else terminal.err(`${path}: ${String(error)}`);
+    return undefined;
+  }
+}
+
+const VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version;
+/**
+ * Runs one cli invocation: parses argv, dispatches to a use case, writes
+ * through `terminal`. Never throws; a bad command or a use case failure both
+ * become a non-zero return.
+ *
+ * @param argv - Arguments after the program name.
+ * @param terminal - Where output goes.
+ * @returns The process exit code.
+ */
+async function main(argv, terminal) {
+  let command;
+  try {
+    command = parseArguments(argv, process.env);
+  } catch (error) {
+    terminal.err(error instanceof Error ? error.message : String(error));
+    terminal.err(USAGE);
+    return 1;
+  }
+  switch (command.name) {
+    case 'help':
+      {
+        terminal.out(USAGE);
+        return 0;
+      }
+    case 'version':
+      {
+        terminal.out(VERSION);
+        return 0;
+      }
+    case 'validate':
+      {
+        return validateRecipes(command.paths, terminal);
+      }
+    case 'run':
+      {
+        return runRecipes(command, terminal);
+      }
+    case 'probe':
+      {
+        return probePage(command.url, {
+          browser: command.browser,
+          ...command.options
+        }, terminal);
+      }
+  }
+}
+
+/** The process's stdout and stderr. */
+function processTerminal() {
+  return {
+    out: line => {
+      process.stdout.write(`${line}\n`);
+    },
+    err: line => {
+      process.stderr.write(`${line}\n`);
+    }
+  };
+}
+
+export { USAGE, findData, loadForRun, loadHooks, main, parseArguments, probeReport, probeUrl, processTerminal, readRecipeFiles, resolveAccess };
+//# sourceMappingURL=index.esm.js.map

package/dist/src/access/index.d.ts

@@ -0,0 +1,2 @@
+export { resolveAccess } from './resolve-access.use-case.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/access/resolve-access.use-case.d.ts

@@ -0,0 +1,12 @@
+import type { AccessConfig } from '@opencraw/core';
+import type { CommonOptions } from '../arguments/index.js';
+/**
+ * The access config a command runs with: the file `--access` (or
+ * `OPENCRAW_ACCESS`) names, with `--access-profile` as its default.
+ *
+ * @param options - The command's shared options.
+ * @returns The config, or `undefined` for direct access.
+ * @throws Error when a profile is asked for without a config, or is not in it.
+ */
+export declare function resolveAccess(options: CommonOptions): Promise<AccessConfig | undefined>;
+//# sourceMappingURL=resolve-access.use-case.d.ts.map

package/dist/src/arguments/command.contract.d.ts

@@ -0,0 +1,44 @@
+/** Options every command shares. */
+export interface CommonOptions {
+    /** A browser binary other than the one Playwright installed. */
+    browserPath?: string;
+    /** Accept an intercepting proxy's certificate. */
+    insecureTls: boolean;
+    /** A user agent for requests and the browser. */
+    userAgent?: string;
+    /** An access config file (proxy profiles); `OPENCRAW_ACCESS` when not given. */
+    access?: string;
+    /** The access profile to use by default, overriding the config's `default`. */
+    accessProfile?: string;
+}
+export type Command = {
+    name: 'help';
+} | {
+    name: 'version';
+} | {
+    name: 'validate';
+    paths: string[];
+} | {
+    name: 'run';
+    paths: string[];
+    /** A JSON Lines file; records go to stdout without it. */
+    out?: string;
+    append: boolean;
+    resume: boolean;
+    trace: boolean;
+    /** One record per input, with the scope it was mapped from. */
+    dryRun: boolean;
+    /** Run only the input recipes with these ids. */
+    only: string[];
+    /** A JavaScript module exporting the hooks the recipes call; `OPENCRAW_HOOKS` when not given. */
+    hooks?: string;
+    headed: boolean;
+    options: CommonOptions;
+} | {
+    name: 'probe';
+    url: string;
+    /** Also render the page in a browser and watch the JSON it fetches. */
+    browser: boolean;
+    options: CommonOptions;
+};
+//# sourceMappingURL=command.contract.d.ts.map

package/dist/src/arguments/index.d.ts

@@ -0,0 +1,3 @@
+export type { Command, CommonOptions } from './command.contract.js';
+export { parseArguments, USAGE } from './parse-arguments.algorithm.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/arguments/parse-arguments.algorithm.d.ts

@@ -0,0 +1,12 @@
+import type { Command } from './command.contract.js';
+export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.\n  probe <url>                                 Fetch a page and report where its data lives.\n\nOptions for run\n  --out <file>        Write records to this JSON Lines file (default: stdout).\n  --append            Keep what --out holds and add to it (each line carries _key).\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n  --hooks <file>      A JavaScript module whose default export is { name: function }: the hooks\n                      the recipes call (or OPENCRAW_HOOKS). It runs as your code: trust it.\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --help, --version";
+/**
+ * Turns argv into a command.
+ *
+ * @param argv - The arguments after the program name.
+ * @param env - The environment, for the defaults some options have.
+ * @returns The command.
+ * @throws Error with a usage message on bad input.
+ */
+export declare function parseArguments(argv: readonly string[], env?: Record<string, string | undefined>): Command;
+//# sourceMappingURL=parse-arguments.algorithm.d.ts.map

package/dist/src/hooks-module/index.d.ts

@@ -0,0 +1,3 @@
+export { loadHooks } from './load-hooks.repository.js';
+export type { ModuleImporter } from './load-hooks.repository.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/hooks-module/load-hooks.repository.d.ts

@@ -0,0 +1,20 @@
+import type { HookMap } from '@opencraw/core';
+/**
+ * Loads the hooks a recipe's `hook` steps and transforms call, from a
+ * JavaScript module: its default export when that is an object of functions
+ * (`export default { positive: v => v > 0 }`), else its named function exports
+ * (`export function positive (v) { ... }`).
+ *
+ * The module runs with the caller's privileges, like any code the caller
+ * imports: point this only at a file you trust.
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file.
+ * @param importModule - How to import it; `import()` unless a test says otherwise.
+ * @returns The hooks by name.
+ * @throws Error naming the file when it cannot be imported, exports no hooks,
+ * or exports something under a hook's name that is not a function.
+ */
+export declare function loadHooks(path: string, importModule?: ModuleImporter): Promise<HookMap>;
+/** Imports a module by URL, returning its namespace. */
+export type ModuleImporter = (url: string) => Promise<Record<string, unknown>>;
+//# sourceMappingURL=load-hooks.repository.d.ts.map

package/dist/src/index.d.ts

@@ -0,0 +1,14 @@
+export { main } from './main.js';
+export { parseArguments, USAGE } from './arguments/index.js';
+export type { Command } from './arguments/index.js';
+export { processTerminal } from './terminal/index.js';
+export type { Terminal } from './terminal/index.js';
+export { findData, probeReport, probeUrl } from './probe/index.js';
+export type { ProbeFindings, ProbeResult } from './probe/index.js';
+export { readRecipeFiles } from './recipe-files/index.js';
+export type { RecipeFiles } from './recipe-files/index.js';
+export { loadForRun } from './run/index.js';
+export type { CommonOptions } from './arguments/index.js';
+export { resolveAccess } from './access/index.js';
+export { loadHooks } from './hooks-module/index.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/main.d.ts

@@ -0,0 +1,12 @@
+import type { Terminal } from './terminal/index.js';
+/**
+ * Runs one cli invocation: parses argv, dispatches to a use case, writes
+ * through `terminal`. Never throws; a bad command or a use case failure both
+ * become a non-zero return.
+ *
+ * @param argv - Arguments after the program name.
+ * @param terminal - Where output goes.
+ * @returns The process exit code.
+ */
+export declare function main(argv: readonly string[], terminal: Terminal): Promise<number>;
+//# sourceMappingURL=main.d.ts.map

package/dist/src/probe/find-data.algorithm.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Finds where a fetched page's data lives: JSON-LD blocks, inline JSON objects
+ * big enough to matter, URLs that look like a data endpoint, script hosts and
+ * links whose path suggests an API. Pattern-matching only; it never fetches.
+ */
+export interface JsonLdFinding {
+    /** The `@type` value(s), or `'?'` when the block has none. */
+    types: string;
+    /** How many top-level keys the block has. */
+    keys: number;
+}
+export interface InlineJsonFinding {
+    /** Where it was found: a `<script>` with no `type`, or an attribute. */
+    where: string;
+    keys: string[];
+    /** Serialized size, for ranking. */
+    size: number;
+}
+export interface ProbeFindings {
+    jsonLd: JsonLdFinding[];
+    inlineJson: InlineJsonFinding[];
+    /** URLs referenced in the page whose path ends in `.json`. */
+    jsonUrls: string[];
+    /** Distinct hosts the page loads a `<script src>` from. */
+    scriptHosts: string[];
+    /** Links whose path contains `api`. */
+    apiLinks: string[];
+}
+/**
+ * Scans a page's HTML for data-shaped things.
+ *
+ * @param html - The document.
+ * @returns What was found, each list capped at a handful of entries.
+ */
+export declare function findData(html: string): ProbeFindings;
+//# sourceMappingURL=find-data.algorithm.d.ts.map

package/dist/src/probe/index.d.ts

@@ -0,0 +1,6 @@
+export { probePage, probeUrl } from './probe-page.use-case.js';
+export type { ProbeResult } from './probe-page.use-case.js';
+export { findData } from './find-data.algorithm.js';
+export type { ProbeFindings, JsonLdFinding, InlineJsonFinding } from './find-data.algorithm.js';
+export { probeReport } from './probe-report.mapper.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/probe/probe-page.use-case.d.ts

@@ -0,0 +1,39 @@
+import type { CommonOptions } from '../arguments/index.js';
+import type { Terminal } from '../terminal/index.js';
+import { findData } from './find-data.algorithm.js';
+/** What fetching and, optionally, rendering a page found. */
+export interface ProbeResult {
+    url: string;
+    status: number;
+    findings: ReturnType<typeof findData>;
+    /** JSON responses observed while the page rendered; empty unless `options.browser` was set. */
+    observed: string[];
+}
+/**
+ * Fetches a page and finds where its data lives, through the access profile the
+ * options name (direct without one). With `browser: true` it also
+ * renders the page and lists the JSON responses seen while it settles, which
+ * finds endpoints a plain fetch of the initial HTML cannot. No `Terminal`
+ * involved: the cli's `probePage` and `@opencraw/mcp`'s probe tool both
+ * build on this, one printing the result, the other returning it as data.
+ *
+ * @param url - The page to probe.
+ * @param options - Browser path, TLS and user agent, plus whether to render.
+ * @returns What was found.
+ * @throws Error when the fetch itself fails.
+ */
+export declare function probeUrl(url: string, options: {
+    browser: boolean;
+} & CommonOptions): Promise<ProbeResult>;
+/**
+ * Fetches a page and reports where its data lives, to a `Terminal`.
+ *
+ * @param url - The page to probe.
+ * @param options - Browser path, TLS and user agent, plus whether to render.
+ * @param terminal - Where the report goes.
+ * @returns The exit code: 0, or 1 on a fetch failure.
+ */
+export declare function probePage(url: string, options: {
+    browser: boolean;
+} & CommonOptions, terminal: Terminal): Promise<number>;
+//# sourceMappingURL=probe-page.use-case.d.ts.map

package/dist/src/probe/probe-report.mapper.d.ts

@@ -0,0 +1,12 @@
+import type { ProbeFindings } from './find-data.algorithm.js';
+/**
+ * Renders a probe's findings as text.
+ *
+ * @param url - The URL probed.
+ * @param status - The HTTP status.
+ * @param findings - What `findData` found.
+ * @param observed - JSON responses a browser saw, when `--browser` was used.
+ * @returns The report.
+ */
+export declare function probeReport(url: string, status: number, findings: ProbeFindings, observed?: string[]): string;
+//# sourceMappingURL=probe-report.mapper.d.ts.map

package/dist/src/recipe-files/index.d.ts

@@ -0,0 +1,3 @@
+export { readRecipeFiles } from './read-recipe-files.repository.js';
+export type { RecipeFiles } from './read-recipe-files.repository.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/recipe-files/read-recipe-files.repository.d.ts

@@ -0,0 +1,26 @@
+import type { RecipeSource } from '@opencraw/core';
+/** Decoded recipes, split by their `kind`. */
+export interface RecipeFiles {
+    /** `path` is the file, `path:line` in a JSON Lines file, or a label (`recipes[0]`) for one from memory. */
+    outputs: {
+        path: string;
+        recipe: unknown;
+    }[];
+    inputs: {
+        path: string;
+        recipe: unknown;
+    }[];
+    /** Documents that are neither; listed so a typo in `kind` does not pass silently. */
+    others: string[];
+}
+/**
+ * Reads recipes, expanding directories to their `.json` and `.jsonl` files
+ * (sorted, not recursive), and sorts them by `kind`. The command line passes
+ * paths; the MCP server also passes recipes it was handed inline.
+ *
+ * @param sources - Files or directories, or any other recipe source core reads.
+ * @returns The decoded recipes.
+ * @throws Error naming a file that is not JSON or JSON Lines.
+ */
+export declare function readRecipeFiles(sources: RecipeSource): Promise<RecipeFiles>;
+//# sourceMappingURL=read-recipe-files.repository.d.ts.map

package/dist/src/run/index.d.ts

@@ -0,0 +1,3 @@
+export { runRecipes } from './run-recipes.use-case.js';
+export { loadForRun } from './load-for-run.use-case.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/run/load-for-run.use-case.d.ts

@@ -0,0 +1,13 @@
+import type { RecipeSet, RecipeSource } from '@opencraw/core';
+/**
+ * Loads and binds the recipes `run` needs, filtered to the input ids in `only`
+ * when given.
+ *
+ * @param sources - Recipe files or directories (or recipes the MCP server was handed inline).
+ * @param only - Input recipe ids to keep; empty keeps all.
+ * @returns The bound set.
+ * @throws Error when there is not exactly one output recipe.
+ * @throws RecipeValidationError, RecipeBindingError on a bad recipe.
+ */
+export declare function loadForRun(sources: RecipeSource, only: readonly string[]): Promise<RecipeSet>;
+//# sourceMappingURL=load-for-run.use-case.d.ts.map

package/dist/src/run/run-recipes.use-case.d.ts

@@ -0,0 +1,17 @@
+import type { Command } from '../arguments/index.js';
+import type { Terminal } from '../terminal/index.js';
+/**
+ * Crawls the recipes at `paths` and reports what happened.
+ *
+ * `--dry-run` caps every input at one record, attaches the scope each record
+ * was mapped from, and prints them side by side instead of writing them, so a
+ * recipe under construction can be checked without a full run.
+ *
+ * @param command - The parsed `run` command.
+ * @param terminal - Where records, the trace and errors go.
+ * @returns The exit code: 0, or 1 on a load failure or a recipe error.
+ */
+export declare function runRecipes(command: Extract<Command, {
+    name: 'run';
+}>, terminal: Terminal): Promise<number>;
+//# sourceMappingURL=run-recipes.use-case.d.ts.map

package/dist/src/terminal/index.d.ts

@@ -0,0 +1,3 @@
+export type { Terminal } from './terminal.contract.js';
+export { processTerminal } from './terminal.client.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/terminal/terminal.client.d.ts

@@ -0,0 +1,4 @@
+import type { Terminal } from './terminal.contract.js';
+/** The process's stdout and stderr. */
+export declare function processTerminal(): Terminal;
+//# sourceMappingURL=terminal.client.d.ts.map

package/dist/src/terminal/terminal.contract.d.ts

@@ -0,0 +1,6 @@
+/** Where a command writes: results to `out`, diagnostics and progress to `err`. Tests capture both. */
+export interface Terminal {
+    out: (line: string) => void;
+    err: (line: string) => void;
+}
+//# sourceMappingURL=terminal.contract.d.ts.map

package/dist/src/validate/index.d.ts

@@ -0,0 +1,2 @@
+export { validateRecipes } from './validate-recipes.use-case.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/validate/validate-recipes.use-case.d.ts

@@ -0,0 +1,10 @@
+import type { Terminal } from '../terminal/index.js';
+/**
+ * Loads and binds recipe files, printing every problem with its JSON path.
+ *
+ * @param paths - Recipe files or directories.
+ * @param terminal - Where results go.
+ * @returns The exit code: 0 when the recipes bind cleanly, 1 otherwise.
+ */
+export declare function validateRecipes(paths: readonly string[], terminal: Terminal): Promise<number>;
+//# sourceMappingURL=validate-recipes.use-case.d.ts.map

package/package.json

@@ -0,0 +1,85 @@
+{
+  "name": "@opencraw/cli",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "./dist/index.esm.js",
+  "module": "./dist/index.esm.js",
+  "types": "./dist/src/index.d.ts",
+  "bin": {
+    "opencraw": "./bin/opencraw.mjs"
+  },
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/index.esm.js",
+      "default": "./dist/index.esm.js"
+    }
+  },
+  "files": [
+    "dist",
+    "bin",
+    "!**/*.tsbuildinfo",
+    "!**/*.d.ts.map",
+    "!**/*.js.map"
+  ],
+  "dependencies": {
+    "@opencraw/core": "^0.1.0",
+    "playwright": "^1.63.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "Command-line tools for OpenCraw: probe a page for scrapable data, validate recipes, run a crawl.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/russoedu/open.craw.git",
+    "directory": "packages/cli"
+  },
+  "keywords": [
+    "crawler",
+    "scraper",
+    "cli",
+    "recipe"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "sideEffects": false,
+  "nx": {
+    "targets": {
+      "typecheck": {
+        "dependsOn": [
+          "^build"
+        ]
+      },
+      "lint": {
+        "dependsOn": [
+          "^build"
+        ]
+      },
+      "e2e": {
+        "executor": "nx:run-commands",
+        "dependsOn": [
+          "build",
+          {
+            "projects": [
+              "@opencraw/core"
+            ],
+            "target": "e2e"
+          }
+        ],
+        "inputs": [
+          "default",
+          "^production",
+          "{workspaceRoot}/packages/core/e2e/**"
+        ],
+        "options": {
+          "command": "tsc -p tsconfig.e2e.json && jest --config jest.e2e.config.cts",
+          "cwd": "packages/cli"
+        }
+      }
+    }
+  }
+}