@opencraw/cli 0.1.8 → 0.1.10

package/README.md

@@ -1,13 +1,17 @@
+<p align="center">
+  <img src="https://raw.githubusercontent.com/russoedu/open.craw/main/docs/assets/opencraw-logo.svg" alt="OpenCraw" width="360">
+</p>
+
 # @opencraw/cli
 
-Command-line tools for [`@opencraw/core`](../core): probe a page for scrapable data, validate recipes,
+Command-line tools for [`@opencraw/core`](https://github.com/russoedu/open.craw/blob/main/packages/core): probe a page for scrapable data, validate recipes,
 and run a crawl from the terminal.
 
 ## Install
 
 ```sh
 npm install -g @opencraw/cli
-# or, inside this workspace: node packages/cli/bin/opencraw.mjs <command>
+# or, inside this workspace, after `npm run cli:build`: node packages/cli/bin/opencraw.mjs <command>
 ```
 
 ## Commands
@@ -16,6 +20,7 @@ npm install -g @opencraw/cli
 opencraw validate <recipe files or directories...>
 opencraw run <recipe files or directories...> [options]
 opencraw probe <url> [options]
+opencraw diff <previous.jsonl> <current.jsonl> [--key a,b] [--ignore a,b] [--changes <file>]
 ```
 
 ### `validate`
@@ -43,11 +48,11 @@ input recipes) and reports what happened.
 | `--dry-run` | One record per input recipe, printed with the scope it was mapped from — for checking a recipe under construction without a full run. |
 | `--trace` | Print the crawl trace to stderr. |
 | `--headed` | Show the browser instead of running headless. |
-| `--parallel <n>` | Run this many input recipes at once (default 1). Iterations inside a recipe follow its `limits.concurrency`. See [§3.9 of the authoring guide](../../docs/recipes/authoring.md#39-concurrency). |
-| `--retries <n>` | Tries per request that fails in passing (a dropped connection, a timeout, a 429, a 5xx), for recipes whose `limits.retry` says nothing. Default 3; `1` turns retrying off. See [§7 of the authoring guide](../../docs/recipes/authoring.md#7-policies-what-happens-when-something-is-missing-or-fails). |
-| `--profiles <dir>` | Where the persistent browser profiles of `session.browserProfile` live (or `OPENCRAW_PROFILES`; default `.opencraw/profiles`). See [access.md](../../docs/recipes/access.md#persistent-browser-profiles). |
-| `--host-delay <ms>`, `--host-concurrency <n>` | Per-site politeness across every recipe: at least `ms` between two requests to one site, at most `n` in flight. They override the access config's `throttle` defaults. See [access.md](../../docs/recipes/access.md#throttling-per-site). |
-| `--plugins <file>` (or `--hooks`) | A plugins module: named exports `hooks` (the recipes' hook steps and transforms), `accessPlugins` (for `{ kind: "plugin" }` access profiles) and `captchaSolvers`. A module whose default export is `{ name: function }` is read as hooks alone. `OPENCRAW_PLUGINS` / `OPENCRAW_HOOKS` when not given; `probe` reads it too. It runs as your code; load only files you trust. See [§6 of the authoring guide](../../docs/recipes/authoring.md#6-hooks). |
+| `--parallel <n>` | Run this many input recipes at once (default 1). Iterations inside a recipe follow its `limits.concurrency`. See [§3.9 of the authoring guide](https://github.com/russoedu/open.craw/blob/main/docs/recipes/authoring.md#39-concurrency). |
+| `--retries <n>` | Tries per request that fails in passing (a dropped connection, a timeout, a 429, a 5xx), for recipes whose `limits.retry` says nothing. Default 3; `1` turns retrying off. See [§7 of the authoring guide](https://github.com/russoedu/open.craw/blob/main/docs/recipes/authoring.md#7-policies-what-happens-when-something-is-missing-or-fails). |
+| `--profiles <dir>` | Where the persistent browser profiles of `session.browserProfile` live (or `OPENCRAW_PROFILES`; default `.opencraw/profiles`). See [access.md](https://github.com/russoedu/open.craw/blob/main/docs/recipes/access.md#persistent-browser-profiles). |
+| `--host-delay <ms>`, `--host-concurrency <n>` | Per-site politeness across every recipe: at least `ms` between two requests to one site, at most `n` in flight. They override the access config's `throttle` defaults. See [access.md](https://github.com/russoedu/open.craw/blob/main/docs/recipes/access.md#throttling-per-site). |
+| `--plugins <file>` (or `--hooks`) | A plugins module: named exports `hooks` (the recipes' hook steps and transforms), `accessPlugins` (for `{ kind: "plugin" }` access profiles) and `captchaSolvers`. A module whose default export is `{ name: function }` is read as hooks alone. `OPENCRAW_PLUGINS` / `OPENCRAW_HOOKS` when not given; `probe` reads it too. It runs as your code; load only files you trust. See [§6 of the authoring guide](https://github.com/russoedu/open.craw/blob/main/docs/recipes/authoring.md#6-hooks). |
 
 ```sh
 opencraw run recipes/ --out out/products.jsonl --trace
@@ -113,7 +118,7 @@ opencraw probe ./decks/incentivi.pptx
 ```
 
 Use it before writing an input recipe, to find the shape a site's data actually takes (§9 of
-[the authoring guide](../../docs/recipes/authoring.md)).
+[the authoring guide](https://github.com/russoedu/open.craw/blob/main/docs/recipes/authoring.md)).
 
 ### Options common to `run` and `probe`
 
@@ -121,7 +126,7 @@ Use it before writing an input recipe, to find the shape a site's data actually
 |---|---|
 | `--browser-path <path>` | A browser binary other than the one Playwright installed (or `OPENCRAW_CHROMIUM`). |
 | `--insecure-tls` | Accept an intercepting proxy's certificate (or `OPENCRAW_INSECURE_TLS=1`). |
-| `--access <file>` | An access config: proxy profiles, with credentials as `{{env.NAME}}` (or `OPENCRAW_ACCESS`). See [access.md](../../docs/recipes/access.md). |
+| `--access <file>` | An access config: proxy profiles, with credentials as `{{env.NAME}}` (or `OPENCRAW_ACCESS`). See [access.md](https://github.com/russoedu/open.craw/blob/main/docs/recipes/access.md). |
 | `--access-profile <name>` | The profile used by recipes that name none, overriding the config's `default`. `probe` uses it for its fetch. |
 | `--user-agent <ua>` | The user agent to send. |
 

package/dist/index.esm.js

@@ -1,6 +1,6 @@
 import { parseArgs } from 'node:util';
 import { writeFile } from 'node:fs/promises';
-import { readableKey, readRecordsFile, diffRecords, loadAccessConfig, htmlTableSheets, AccessBroker, HttpClient, parseXml, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, variantLabel, diffOptionsFor, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
+import { readableKey, readRecordsFile, diffRecords, loadAccessConfig, htmlTableSheets, HostAllowlist, AccessBroker, HttpClient, parseXml, BrowserClient, readRecipeSource, parseOutputRecipe, parseInputRecipe, bindRecipeSet, memorySink, jsonLinesSink, createCrawler, RecipeSet, variantLabel, diffOptionsFor, RecipeValidationError, RecipeBindingError, traceLine } from '@opencraw/core';
 import { resolve } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { readFileSync } from 'node:fs';
@@ -1046,8 +1046,10 @@ const OBSERVE_MS = 4000;
  * @throws Error when the fetch itself fails.
  */
 async function probeUrl(url, options) {
-  const plugins = options.plugins === undefined ? undefined : await loadPlugins(options.plugins);
-  const lease = await new AccessBroker(await resolveAccess(options), plugins?.accessPlugins).lease({
+  const allowedHosts = HostAllowlist.of(options.allowedHosts);
+  const plugins = options.accessPlugins === undefined && options.plugins !== undefined ? await loadPlugins(options.plugins) : undefined;
+  const access = options.accessConfig === undefined ? await resolveAccess(options) : withDefault(options.accessConfig, options.accessProfile);
+  const lease = await new AccessBroker(access, options.accessPlugins ?? plugins?.accessPlugins).lease({
     recipeId: 'probe'
   });
   if (lease.cdp !== undefined) throw new Error(`access profile "${lease.profile}" is a remote browser; probe fetches over HTTP and needs a proxy profile`);
@@ -1055,7 +1057,8 @@ async function probeUrl(url, options) {
     userAgent: options.userAgent ?? BROWSER_USER_AGENT,
     ignoreHTTPSErrors: options.insecureTls || lease.ignoreHTTPSErrors === true,
     proxy: lease.proxy,
-    headers: lease.headers
+    headers: lease.headers,
+    allowedHosts
   });
   try {
     const target = /^[a-z][\w+.-]+:/i.test(url) ? url : pathToFileURL(resolve(url)).href;
@@ -1106,7 +1109,7 @@ async function probeUrl(url, options) {
       xml: describeXml(parseXml(body.xml, response.url))
     };
     const text = body.kind === 'html' ? body.html : body.text;
-    const observed = options.browser && !target.startsWith('file:') ? await observeBrowserJson(url, options, lease) : [];
+    const observed = options.browser && !target.startsWith('file:') ? await observeBrowserJson(url, options, lease, allowedHosts) : [];
     const html = body.kind === 'html' ? {
       html: describeHtml(body.html, response.format === 'markdown' || response.format === 'docx')
     } : {};
@@ -1139,7 +1142,7 @@ async function probePage(url, options, terminal) {
     return 1;
   }
 }
-async function observeBrowserJson(url, options, lease) {
+async function observeBrowserJson(url, options, lease, allowedHosts) {
   const browser = await BrowserClient.launch({
     executablePath: options.browserPath,
     ignoreHTTPSErrors: options.insecureTls
@@ -1149,7 +1152,8 @@ async function observeBrowserJson(url, options, lease) {
       userAgent: options.userAgent,
       proxy: lease.proxy,
       headers: lease.headers,
-      ignoreHTTPSErrors: lease.ignoreHTTPSErrors
+      ignoreHTTPSErrors: lease.ignoreHTTPSErrors,
+      allowedHosts
     });
     const seen = [];
     session.page.on('response', response => {
@@ -1169,6 +1173,15 @@ async function observeBrowserJson(url, options, lease) {
     await browser.close();
   }
 }
+/** An access config with its default profile overridden, when one is named. */
+function withDefault(config, profile) {
+  if (profile === undefined) return config;
+  if (!Object.hasOwn(config.profiles, profile)) throw new Error(`access profile "${profile}" is not in the access config (profiles: ${Object.keys(config.profiles).join(', ') || 'none'})`);
+  return {
+    ...config,
+    default: profile
+  };
+}
 function reportOf(result) {
   if (result.pdf !== undefined) return pdfReport(result.url, result.pdf);
   if (result.workbook !== undefined) return workbookReport(result.url, result.workbook);

package/dist/src/index.d.ts

@@ -4,7 +4,7 @@ export type { Command } from './arguments/index.js';
 export { processTerminal } from './terminal/index.js';
 export type { Terminal } from './terminal/index.js';
 export { findData, probeReport, probeUrl } from './probe/index.js';
-export type { ProbeFindings, ProbeResult } from './probe/index.js';
+export type { ProbeFindings, ProbeResult, ProbeOptions } from './probe/index.js';
 export { readRecipeFiles } from './recipe-files/index.js';
 export type { RecipeFiles } from './recipe-files/index.js';
 export { loadForRun } from './run/index.js';

package/dist/src/probe/index.d.ts

@@ -1,5 +1,5 @@
 export { probePage, probeUrl } from './probe-page.use-case.js';
-export type { ProbeResult } from './probe-page.use-case.js';
+export type { ProbeResult, ProbeOptions } from './probe-page.use-case.js';
 export { findData } from './find-data.algorithm.js';
 export type { ProbeFindings, JsonLdFinding, InlineJsonFinding } from './find-data.algorithm.js';
 export { probeReport, pdfReport, workbookReport, deckReport, jsonReport, xmlReport } from './probe-report.mapper.js';

package/dist/src/probe/probe-page.use-case.d.ts

@@ -1,3 +1,4 @@
+import type { AccessConfig, AccessPlugin } from '@opencraw/core';
 import type { CommonOptions } from '../arguments/index.js';
 import type { Terminal } from '../terminal/index.js';
 import { findData } from './find-data.algorithm.js';
@@ -27,6 +28,17 @@ export interface ProbeResult {
     /** Present for XML: its namespaces, repeated elements and structure. */
     xml?: XmlFindings;
 }
+/** How to probe: the cli's options, plus what a service probing for others sets. */
+export interface ProbeOptions extends CommonOptions {
+    /** Also render the page and list the JSON it fetches. */
+    browser: boolean;
+    /** The only hosts the probe may reach, as `CrawlOptions.allowedHosts`; local paths and `file:` are refused. */
+    allowedHosts?: string[];
+    /** An access config already loaded, instead of `access` (a file). `accessProfile` still picks the default. */
+    accessConfig?: AccessConfig;
+    /** Access plugins already loaded, instead of those in `plugins` (a module). */
+    accessPlugins?: AccessPlugin[];
+}
 /**
  * Fetches a page and finds where its data lives, through the access profile the
  * options name (direct without one). With `browser: true` it also
@@ -43,9 +55,7 @@ export interface ProbeResult {
  * @returns What was found.
  * @throws Error when the fetch itself fails.
  */
-export declare function probeUrl(url: string, options: {
-    browser: boolean;
-} & CommonOptions): Promise<ProbeResult>;
+export declare function probeUrl(url: string, options: ProbeOptions): Promise<ProbeResult>;
 /**
  * Fetches a page and reports where its data lives, to a `Terminal`.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/cli",
-  "version": "0.1.8",
+  "version": "0.1.10",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",
@@ -24,7 +24,7 @@
     "!**/*.js.map"
   ],
   "dependencies": {
-    "@opencraw/core": "^0.1.8",
+    "@opencraw/core": "^0.1.10",
     "playwright": "^1.63.0"
   },
   "publishConfig": {