@opencraw/cli 0.1.53 → 0.1.55

package/dist/access/index.d.ts

@@ -0,0 +1,2 @@
+export { resolveAccess } from './resolve-access.use-case';
+//# sourceMappingURL=index.d.ts.map

package/dist/access/resolve-access.use-case.d.ts

@@ -0,0 +1,12 @@
+import type { AccessConfig } from '@opencraw/core';
+import type { CommonOptions } from '../arguments';
+/**
+ * The access config a command runs with: the file `--access` (or
+ * `OPENCRAW_ACCESS`) names, with `--access-profile` as its default.
+ *
+ * @param options - The command's shared options.
+ * @returns The config, or `undefined` for direct access.
+ * @throws Error when a profile is asked for without a config, or is not in it.
+ */
+export declare function resolveAccess(options: CommonOptions): Promise<AccessConfig | undefined>;
+//# sourceMappingURL=resolve-access.use-case.d.ts.map

package/dist/arguments/command.contract.d.ts

@@ -0,0 +1,86 @@
+/** Options every command shares. */
+export interface CommonOptions {
+    /** A browser binary other than the one Playwright installed. */
+    browserPath?: string;
+    /** Accept an intercepting proxy's certificate. */
+    insecureTls: boolean;
+    /** A user agent for requests and the browser. */
+    userAgent?: string;
+    /** An access config file (proxy profiles); `OPENCRAW_ACCESS` when not given. */
+    access?: string;
+    /** The access profile to use by default, overriding the config's `default`. */
+    accessProfile?: string;
+    /** A plugins module: hooks, access plugins; `--hooks` / `OPENCRAW_PLUGINS` / `OPENCRAW_HOOKS` too. */
+    plugins?: string;
+}
+export type Command = {
+    name: 'help';
+} | {
+    name: 'version';
+} | {
+    name: 'validate';
+    paths: string[];
+} | {
+    name: 'export';
+    /** A finished run's JSON Lines file. */
+    records: string;
+    /** The output recipe the records were mapped to: it names the columns. */
+    output: string;
+    /** The file to write; its extension is the format. */
+    out: string;
+    options: CommonOptions;
+} | {
+    name: 'diff';
+    previous: string;
+    current: string;
+    /** The fields that identify a record; empty uses each line's `_key`. */
+    key: string[];
+    /** Fields left out of the comparison. */
+    ignore: string[];
+    /** Where to write the changes as JSON Lines. */
+    changes?: string;
+} | {
+    name: 'run';
+    paths: string[];
+    /** A JSON Lines file; records go to stdout without it. */
+    out?: string;
+    append: boolean;
+    resume: boolean;
+    trace: boolean;
+    /** One record per input, with the scope it was mapped from. */
+    dryRun: boolean;
+    /** Run only the input recipes with these ids. */
+    only: string[];
+    headed: boolean;
+    /** A previous run's JSON Lines file to compare this run's records with. */
+    diff?: string;
+    /** Where to write the changes `diff` finds, as JSON Lines. */
+    changes?: string;
+    /** How many input recipes run at once. */
+    parallel?: number;
+    /** Tries per request for recipes whose `limits.retry` says nothing (1 turns retrying off). */
+    retries?: number;
+    /** Where `session.browserProfile` profiles live; `OPENCRAW_PROFILES` when not given. */
+    profiles?: string;
+    /** Per-site politeness across every recipe: overrides the access config's `throttle` defaults. */
+    throttle: {
+        delayMs?: number;
+        concurrency?: number;
+    };
+    options: CommonOptions;
+} | {
+    name: 'probe';
+    url: string;
+    /** Also render the page in a browser and watch the JSON it fetches. */
+    browser: boolean;
+    options: CommonOptions;
+} | {
+    name: 'studio';
+    /** The recipe folder to open; the current directory when not given. */
+    folder?: string;
+    /** A hooks or plugins module to run recipes with (`--hooks`, `--plugins`, `OPENCRAW_HOOKS`, `OPENCRAW_PLUGINS`; issue #150). */
+    plugins?: string;
+    /** A folder, recipe file or document the printed link opens straight away (`--open`, issue #235). */
+    open?: string;
+};
+//# sourceMappingURL=command.contract.d.ts.map

package/dist/arguments/index.d.ts

@@ -0,0 +1,3 @@
+export type { Command, CommonOptions } from './command.contract';
+export { parseArguments, USAGE } from './parse-arguments.algorithm';
+//# sourceMappingURL=index.d.ts.map

package/dist/arguments/parse-arguments.algorithm.d.ts

@@ -0,0 +1,12 @@
+import type { Command } from './command.contract';
+export declare const USAGE = "opencraw <command> [options]\n\nCommands\n  validate <recipe files or directories...>   Parse and bind the recipes; list every problem with its path.\n  run <recipe files or directories...>        Crawl. Records go to --out (its extension picks the format), or to stdout as JSON Lines.\n  probe <url>                                 Fetch a page and report where its data lives.\n  diff <previous.jsonl> <current.jsonl>       Compare two runs' records by key: added, removed, changed.\n  export <records.jsonl> <output recipe> --out <file>\n                                              Write a finished run's records again in another format (the extension\n                                               of --out chooses it, as for run).\n  studio [folder] [--open <path>]             Open a recipe folder in the browser: run samples, read\n                                               the trace next to the page (needs @opencraw/studio).\n                                               --open prints a link that opens a folder, recipe or\n                                               document straight away.\n\nOptions for run\n  --out <file>        Write records to this file; the extension is the format: .jsonl, .json, .csv, .xlsx, .yml/.yaml,\n                      .html (a page that looks like Studio) or .pdf (that page, printed; needs Chromium). Any other\n                      extension is an error. Default: stdout as JSON Lines.\n  --append            Keep what --out holds and add to it (each line carries _key); .jsonl only.\n  --resume            Skip records --out already has (needs --append).\n  --only <id>         Run only this input recipe (repeatable).\n  --dry-run           One record per input, printed with the scope it was mapped from.\n  --trace             Print the crawl trace to stderr.\n  --headed            Show the browser.\n  --parallel <n>      Run this many input recipes at once (default 1). Iterations inside a recipe\n                      follow its limits.concurrency.\n  --retries <n>       Tries per request that fails in passing (connection, timeout, 429, 5xx), for\n                      recipes whose limits.retry says nothing. Default 3; 1 turns retrying off.\n  --profiles <dir>    Where recipes' session.browserProfile profiles live (or OPENCRAW_PROFILES;\n                      default .opencraw/profiles).\n  --diff <file>       Compare this run's records with a previous run's JSON Lines file (it may be\n                      --out itself: it is read before the crawl). Keys come from the output recipe.\n  --host-delay <ms>   At least this long between two requests to one site, across every recipe.\n  --host-concurrency <n>\n                      At most this many requests to one site in flight, across every recipe.\n                      Both override the access config's \"throttle\" defaults.\n\nOptions for diff\n  --key <a,b>         The fields that identify a record (default: each line's _key).\n  --ignore <a,b>      Fields left out of the comparison (a date the crawl stamps, say).\n  --changes <file>    Also write every change as a JSON line (run --diff takes it too).\n\nOptions for probe\n  --browser           Also render the page in a browser and list the JSON it fetches.\n\nOptions for both\n  --browser-path <p>  A browser binary other than the one Playwright installed (or OPENCRAW_CHROMIUM).\n  --insecure-tls      Accept an intercepting proxy's certificate (or OPENCRAW_INSECURE_TLS=1).\n  --user-agent <ua>   The user agent to send.\n  --access <file>     An access config: proxy profiles, credentials as {{env.NAME}} (or OPENCRAW_ACCESS).\n  --access-profile <name>\n                      The access profile to use when a recipe names none.\n  --plugins <file>    A JavaScript module exporting hooks (the recipes' hook steps and transforms),\n                      accessPlugins (for { kind: \"plugin\" } access profiles) and captchaSolvers\n                      (for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /\n                      OPENCRAW_HOOKS. It runs as your code: trust it.\n  --help, --version";
+/**
+ * Turns argv into a command.
+ *
+ * @param argv - The arguments after the program name.
+ * @param env - The environment, for the defaults some options have.
+ * @returns The command.
+ * @throws Error with a usage message on bad input.
+ */
+export declare function parseArguments(argv: readonly string[], env?: Record<string, string | undefined>): Command;
+//# sourceMappingURL=parse-arguments.algorithm.d.ts.map

package/dist/diff/diff-files.use-case.d.ts

@@ -0,0 +1,22 @@
+import type { RecordDiff } from '@opencraw/core';
+import type { Command } from '../arguments';
+import type { Terminal } from '../terminal';
+/**
+ * Compares two runs' JSON Lines files and prints what changed; with
+ * `--changes`, writes every change as a JSON line too.
+ *
+ * @param command - The parsed `diff` command.
+ * @param terminal - Where the report goes.
+ * @returns The exit code: 0, or 1 when a file cannot be read or has no key.
+ */
+export declare function diffFiles(command: Extract<Command, {
+    name: 'diff';
+}>, terminal: Terminal): Promise<number>;
+/**
+ * Writes a diff's changes as JSON Lines: `{ change, key, before?, after?, fields? }`.
+ *
+ * @param path - The file.
+ * @param diff - The diff.
+ */
+export declare function writeChanges(path: string, diff: RecordDiff): Promise<void>;
+//# sourceMappingURL=diff-files.use-case.d.ts.map

package/dist/diff/diff-report.mapper.d.ts

@@ -0,0 +1,12 @@
+import type { RecordDiff } from '@opencraw/core';
+/**
+ * A diff as people read it: the counts, a warning when the run lost most of
+ * its records, then one line per change, `-` removed, `~` changed (with each
+ * field's before → after), `+` added.
+ *
+ * @param diff - The diff.
+ * @param limit - Changes listed at most.
+ * @returns The lines.
+ */
+export declare function diffReport(diff: RecordDiff, limit?: number): string[];
+//# sourceMappingURL=diff-report.mapper.d.ts.map

package/dist/diff/index.d.ts

@@ -0,0 +1,3 @@
+export { diffFiles, writeChanges } from './diff-files.use-case';
+export { diffReport } from './diff-report.mapper';
+//# sourceMappingURL=index.d.ts.map

package/dist/export/export-records.use-case.d.ts

@@ -0,0 +1,14 @@
+import type { Command } from '../arguments';
+import type { Terminal } from '../terminal';
+/**
+ * Writes a finished run's JSON Lines file again in another format: the extension of `--out` chooses it, as it does for `run`. The output recipe
+ * supplies the columns (its fields, in order, with their types and descriptions).
+ *
+ * @param command - The parsed `export` command.
+ * @param terminal - Where the result and errors go.
+ * @returns The exit code: 0, or 1 when a file cannot be read, the recipe is not an output recipe, or the extension names no format.
+ */
+export declare function exportRecords(command: Extract<Command, {
+    name: 'export';
+}>, terminal: Terminal): Promise<number>;
+//# sourceMappingURL=export-records.use-case.d.ts.map

package/dist/export/index.d.ts

@@ -0,0 +1,2 @@
+export { exportRecords } from './export-records.use-case';
+//# sourceMappingURL=index.d.ts.map

package/dist/hooks-module/index.d.ts

@@ -0,0 +1,3 @@
+export { loadHooks, loadPlugins } from './load-hooks.repository';
+export type { ModuleImporter, PluginModule } from './load-hooks.repository';
+//# sourceMappingURL=index.d.ts.map

package/dist/hooks-module/load-hooks.repository.d.ts

@@ -0,0 +1,47 @@
+import type { AccessPlugin, CaptchaSolver, HookMap } from '@opencraw/core';
+/** What a plugins module provides: hooks for `hook` steps and transforms, and plugins the engine calls. */
+export interface PluginModule {
+    hooks: HookMap;
+    /** Access plugins: `{ kind: "plugin", name }` profiles in the access config use them. */
+    accessPlugins: AccessPlugin[];
+    /** Captcha solvers: recipes name them in `session.captcha.solver` and `captcha` steps. */
+    captchaSolvers: CaptchaSolver[];
+}
+/**
+ * Loads a plugins module: a JavaScript module whose named exports `hooks`
+ * (`{ name: function }`), `accessPlugins` (`[{ name, lease }]`) and
+ * `captchaSolvers` (`[{ name, solve }]`) give what recipes and access profiles
+ * refer to by name:
+ *
+ * ```js
+ * export const hooks = { positive: v => v > 0 }
+ * export const accessPlugins = [{ name: 'browserbase', lease: async () => ({ cdp: { endpoint } }) }]
+ * export const captchaSolvers = [{ name: 'capsolver', solve: async (challenge, { page }) => ({ status: 'solved' }) }]
+ * ```
+ *
+ * A module with none of those names is a bare hooks module, as before: its
+ * default export when that is an object of functions, else its named function
+ * exports.
+ *
+ * The module runs with the caller's privileges, like any code the caller
+ * imports: point this only at a file you trust.
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file, or a `.ts` / `.mts` file on a Node that strips types (22.18+).
+ * @param importModule - How to import it; `import()` unless a test says otherwise.
+ * @returns The hooks and plugins.
+ * @throws Error naming the file when it cannot be imported, provides nothing,
+ * or provides something that is not what its name promises.
+ */
+export declare function loadPlugins(path: string, importModule?: ModuleImporter): Promise<PluginModule>;
+/**
+ * Loads the hooks of a hooks or plugins module (see {@link loadPlugins}).
+ *
+ * @param path - A `.mjs` / `.js` / `.cjs` file.
+ * @param importModule - How to import it.
+ * @returns The hooks by name.
+ * @throws Error naming the file when it cannot be imported or exports no hooks.
+ */
+export declare function loadHooks(path: string, importModule?: ModuleImporter): Promise<HookMap>;
+/** Imports a module by URL, returning its namespace. */
+export type ModuleImporter = (url: string) => Promise<Record<string, unknown>>;
+//# sourceMappingURL=load-hooks.repository.d.ts.map

package/dist/index.d.ts

@@ -1 +1,18 @@
-export * from "./src/index.js";
+export { main } from './main';
+export { parseArguments, USAGE } from './arguments';
+export type { Command } from './arguments';
+export { processTerminal } from './terminal';
+export type { Terminal } from './terminal';
+export { findData, probeReport, probeUrl } from './probe';
+export type { ProbeFindings, ProbeResult, ProbeOptions } from './probe';
+export { readRecipeFiles } from './recipe-files';
+export type { RecipeFiles } from './recipe-files';
+export { loadForRun } from './run';
+export { diffReport, writeChanges } from './diff';
+export type { CommonOptions } from './arguments';
+export { resolveAccess } from './access';
+export { loadHooks, loadPlugins } from './hooks-module';
+export type { PluginModule } from './hooks-module';
+export { runStudio } from './studio';
+export type { StudioModule } from './studio';
+//# sourceMappingURL=index.d.ts.map