@opencraw/cli 0.1.2 → 0.1.3

package/README.md

@@ -43,6 +43,10 @@ input recipes) and reports what happened.
 | `--dry-run` | One record per input recipe, printed with the scope it was mapped from — for checking a recipe under construction without a full run. |
 | `--trace` | Print the crawl trace to stderr. |
 | `--headed` | Show the browser instead of running headless. |
+| `--parallel <n>` | Run this many input recipes at once (default 1). Iterations inside a recipe follow its `limits.concurrency`. See [§3.9 of the authoring guide](../../docs/recipes/authoring.md#39-concurrency). |
+| `--retries <n>` | Tries per request that fails in passing (a dropped connection, a timeout, a 429, a 5xx), for recipes whose `limits.retry` says nothing. Default 3; `1` turns retrying off. See [§7 of the authoring guide](../../docs/recipes/authoring.md#7-policies-what-happens-when-something-is-missing-or-fails). |
+| `--profiles <dir>` | Where the persistent browser profiles of `session.browserProfile` live (or `OPENCRAW_PROFILES`; default `.opencraw/profiles`). See [access.md](../../docs/recipes/access.md#persistent-browser-profiles). |
+| `--host-delay <ms>`, `--host-concurrency <n>` | Per-site politeness across every recipe: at least `ms` between two requests to one site, at most `n` in flight. They override the access config's `throttle` defaults. See [access.md](../../docs/recipes/access.md#throttling-per-site). |
 | `--plugins <file>` (or `--hooks`) | A plugins module: named exports `hooks` (the recipes' hook steps and transforms), `accessPlugins` (for `{ kind: "plugin" }` access profiles) and `captchaSolvers`. A module whose default export is `{ name: function }` is read as hooks alone. `OPENCRAW_PLUGINS` / `OPENCRAW_HOOKS` when not given; `probe` reads it too. It runs as your code; load only files you trust. See [§6 of the authoring guide](../../docs/recipes/authoring.md#6-hooks). |
 
 ```sh

package/dist/index.esm.js

@@ -19,6 +19,16 @@ Options for run
   --dry-run           One record per input, printed with the scope it was mapped from.
   --trace             Print the crawl trace to stderr.
   --headed            Show the browser.
+  --parallel <n>      Run this many input recipes at once (default 1). Iterations inside a recipe
+                      follow its limits.concurrency.
+  --retries <n>       Tries per request that fails in passing (connection, timeout, 429, 5xx), for
+                      recipes whose limits.retry says nothing. Default 3; 1 turns retrying off.
+  --profiles <dir>    Where recipes' session.browserProfile profiles live (or OPENCRAW_PROFILES;
+                      default .opencraw/profiles).
+  --host-delay <ms>   At least this long between two requests to one site, across every recipe.
+  --host-concurrency <n>
+                      At most this many requests to one site in flight, across every recipe.
+                      Both override the access config's "throttle" defaults.
 
 Options for probe
   --browser           Also render the page in a browser and list the JSON it fetches.
@@ -58,6 +68,21 @@ const OPTIONS = {
   'headed': {
     type: 'boolean'
   },
+  'profiles': {
+    type: 'string'
+  },
+  'retries': {
+    type: 'string'
+  },
+  'parallel': {
+    type: 'string'
+  },
+  'host-delay': {
+    type: 'string'
+  },
+  'host-concurrency': {
+    type: 'string'
+  },
   'hooks': {
     type: 'string'
   },
@@ -154,6 +179,13 @@ function parseArguments(argv, env = {}) {
           dryRun: values['dry-run'] === true,
           only: values.only ?? [],
           headed: values.headed === true,
+          profiles: values.profiles ?? nonEmpty(env.OPENCRAW_PROFILES),
+          retries: integer(values.retries, '--retries', 1),
+          parallel: integer(values.parallel, '--parallel', 1),
+          throttle: {
+            delayMs: integer(values['host-delay'], '--host-delay', 0),
+            concurrency: integer(values['host-concurrency'], '--host-concurrency', 1)
+          },
           options
         };
       }
@@ -173,6 +205,12 @@ function parseArguments(argv, env = {}) {
       }
   }
 }
+function integer(value, name, minimum) {
+  if (value === undefined) return undefined;
+  const number = Number(value);
+  if (!Number.isSafeInteger(number) || number < minimum) throw new Error(`${name} needs a whole number of at least ${minimum}, not "${value}"`);
+  return number;
+}
 function nonEmpty(value) {
   return value === '' ? undefined : value;
 }
@@ -1018,6 +1056,12 @@ async function runRecipes(command, terminal) {
   const crawler = createCrawler({
     sink,
     access,
+    throttle: throttleFor(command, access?.throttle),
+    profilesDir: command.profiles,
+    parallel: command.parallel,
+    retry: command.retries === undefined ? undefined : {
+      attempts: command.retries
+    },
     hooks: plugins?.hooks,
     accessPlugins: plugins?.accessPlugins,
     captchaSolvers: plugins?.captchaSolvers,
@@ -1049,6 +1093,23 @@ async function runRecipes(command, terminal) {
     await crawler.close();
   }
 }
+/** The access config's throttle, with `--host-delay` / `--host-concurrency` over its defaults. */
+function throttleFor(command, fromConfig) {
+  const {
+    delayMs,
+    concurrency
+  } = command.throttle;
+  if (delayMs === undefined && concurrency === undefined) return fromConfig;
+  return {
+    ...fromConfig,
+    ...(delayMs !== undefined && {
+      delayMs
+    }),
+    ...(concurrency !== undefined && {
+      concurrency
+    })
+  };
+}
 function onEvent(event, command, terminal) {
   if (command.trace) {
     const line = traceLine(event);

package/dist/src/arguments/command.contract.d.ts

@@ -33,6 +33,17 @@ export type Command = {
     /** Run only the input recipes with these ids. */
     only: string[];
     headed: boolean;
+    /** How many input recipes run at once. */
+    parallel?: number;
+    /** Tries per request for recipes whose `limits.retry` says nothing (1 turns retrying off). */
+    retries?: number;
+    /** Where `session.browserProfile` profiles live; `OPENCRAW_PROFILES` when not given. */
+    profiles?: string;
+    /** Per-site politeness across every recipe: overrides the access config's `throttle` defaults. */
+    throttle: {
+        delayMs?: number;
+        concurrency?: number;
+    };
     options: CommonOptions;
 } | {
     name: 'probe';

package/dist/src/arguments/parse-arguments.algorithm.d.ts

@@ -1,5 +1,5 @@
 import type { Command } from './command.contract.js';
-export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.\n  probe <url>                                 Fetch a page and report where its data lives.\n\nOptions for run\n  --out <file>        Write records to this JSON Lines file (default: stdout).\n  --append            Keep what --out holds and add to it (each line carries _key).\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --plugins <file>    A JavaScript module exporting hooks (the recipes' hook steps and transforms),\n                      accessPlugins (for { kind: \"plugin\" } access profiles) and captchaSolvers\n                      (for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /\n                      OPENCRAW_HOOKS. It runs as your code: trust it.\n  --help, --version";
+export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out as JSON Lines, or to stdout.\n  probe <url>                                 Fetch a page and report where its data lives.\n\nOptions for run\n  --out <file>        Write records to this JSON Lines file (default: stdout).\n  --append            Keep what --out holds and add to it (each line carries _key).\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n  --parallel <n>      Run this many input recipes at once (default 1). Iterations inside a recipe\n                      follow its limits.concurrency.\n  --retries <n>       Tries per request that fails in passing (connection, timeout, 429, 5xx), for\n                      recipes whose limits.retry says nothing. Default 3; 1 turns retrying off.\n  --profiles <dir>    Where recipes' session.browserProfile profiles live (or OPENCRAW_PROFILES;\n                      default .opencraw/profiles).\n  --host-delay <ms>   At least this long between two requests to one site, across every recipe.\n  --host-concurrency <n>\n                      At most this many requests to one site in flight, across every recipe.\n                      Both override the access config's \"throttle\" defaults.\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --plugins <file>    A JavaScript module exporting hooks (the recipes' hook steps and transforms),\n                      accessPlugins (for { kind: \"plugin\" } access profiles) and captchaSolvers\n                      (for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /\n                      OPENCRAW_HOOKS. It runs as your code: trust it.\n  --help, --version";
 /**
  * Turns argv into a command.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/cli",
-  "version": "0.1.2",
+  "version": "0.1.3",
   "type": "module",
   "main": "./dist/index.esm.js",
   "module": "./dist/index.esm.js",
@@ -24,7 +24,7 @@
     "!**/*.js.map"
   ],
   "dependencies": {
-    "@opencraw/core": "^0.1.2",
+    "@opencraw/core": "^0.1.3",
     "playwright": "^1.63.0"
   },
   "publishConfig": {