@opencraw/core 0.1.2 → 0.1.4

package/dist/record-diff/record-diff.algorithm.d.ts

@@ -0,0 +1,82 @@
+import type { OutputRecipe } from '../recipe-schema/index.js';
+/** A record as a JSON Lines file holds it: the data, plus the sink's `_source` (and `_key` in append mode). */
+export type StoredRecord = Record<string, unknown>;
+/** One field whose value changed; nested objects are compared field by field (`price.amount`). */
+export interface FieldChange {
+    field: string;
+    before: unknown;
+    after: unknown;
+}
+export type RecordChange = {
+    change: 'added';
+    key: string;
+    after: StoredRecord;
+} | {
+    change: 'removed';
+    key: string;
+    before: StoredRecord;
+} | {
+    change: 'changed';
+    key: string;
+    before: StoredRecord;
+    after: StoredRecord;
+    fields: FieldChange[];
+};
+/** How two runs differ, by record key. */
+export interface RecordDiff {
+    added: number;
+    removed: number;
+    changed: number;
+    unchanged: number;
+    /** Every change, removed first, then changed, then added, each in file order. */
+    changes: RecordChange[];
+    /** Records on either side the key cannot tell apart from an earlier one: only the first counts. */
+    repeated: number;
+    /** How many records each side holds. */
+    counts: {
+        previous: number;
+        current: number;
+    };
+    /** Set when the current run holds far fewer records: the usual sign a site changed and a recipe broke quietly. */
+    shrunk?: {
+        previous: number;
+        current: number;
+    };
+}
+export interface DiffOptions {
+    /** The fields that identify a record. Default: the `_key` each line carries (a sink in append mode writes it). */
+    key?: readonly string[];
+    /** Fields not compared (their dotted paths match nested ones too). `_source` and `_key` never are. */
+    ignore?: readonly string[];
+    /** The share of records a run may lose before `shrunk` is set; default 0.5 (half). */
+    shrink?: number;
+}
+/**
+ * Compares two runs' records by key: what was added, what was removed, and
+ * for the records in both, which fields changed, before and after. Like a
+ * `git diff` of a price list, but row by row instead of line by line, so a
+ * reordered file is not a change.
+ *
+ * @param previous - The earlier run's records.
+ * @param current - The later run's records.
+ * @param options - The key fields, the fields to ignore, the shrink threshold.
+ * @returns The differences.
+ * @throws Error when records carry no key: without `key`, every line needs `_key`.
+ */
+export declare function diffRecords(previous: readonly StoredRecord[], current: readonly StoredRecord[], options?: DiffOptions): RecordDiff;
+/**
+ * The diff options an output recipe implies: its key fields, and its fields
+ * the engine fills differently every run (`generated: now`, `uuid`).
+ *
+ * @param output - The output recipe.
+ * @returns The key and the fields to ignore.
+ */
+export declare function diffOptionsFor(output: OutputRecipe): Required<Pick<DiffOptions, 'key' | 'ignore'>>;
+/**
+ * A key as people read it: the key fields' values joined (`Pandina · 1.0 Hybrid`).
+ *
+ * @param key - A record key (a JSON array of the key values).
+ * @returns The readable form.
+ */
+export declare function readableKey(key: string): string;
+//# sourceMappingURL=record-diff.algorithm.d.ts.map

package/dist/record-diff/records-file.repository.d.ts

@@ -0,0 +1,10 @@
+import type { StoredRecord } from './record-diff.algorithm.js';
+/**
+ * The records of a JSON Lines file, as a sink wrote them.
+ *
+ * @param path - The file.
+ * @returns One record per non-empty line.
+ * @throws Error naming the file and the line that is not a JSON object.
+ */
+export declare function readRecordsFile(path: string): Promise<StoredRecord[]>;
+//# sourceMappingURL=records-file.repository.d.ts.map

package/dist/record-sink/dedupe.policy.d.ts

@@ -0,0 +1,29 @@
+import type { OutputRecord } from '../output-mapping/index.js';
+/** How far de-duplication reaches: the whole run, one input recipe, or not at all. */
+export type DedupeScope = 'run' | 'recipe' | 'off';
+/** One input recipe's view of de-duplication: whether a record repeats a key already seen. */
+export interface RecipeDedupe {
+    /**
+     * @param record - A validated record.
+     * @returns `true` when the record repeats an earlier key and must be dropped.
+     */
+    isDuplicate: (record: OutputRecord) => boolean;
+}
+/**
+ * Drops records whose key was already seen. First record wins; keyless
+ * records always pass. Recipes running in parallel each get their own view:
+ * under `recipe` scope they never see each other's keys, under `run` scope
+ * they share them (and whichever emits a key first keeps it).
+ */
+export declare class DedupePolicy {
+    readonly scope: DedupeScope;
+    private readonly shared;
+    constructor(scope?: DedupeScope);
+    /**
+     * The de-duplication one input recipe run uses.
+     *
+     * @returns Its view: keys shared with the run, its own, or none checked.
+     */
+    forRecipe(): RecipeDedupe;
+}
+//# sourceMappingURL=dedupe.policy.d.ts.map

package/dist/record-sink/index.d.ts

@@ -0,0 +1,9 @@
+export type { RecordSink, SinkSummary } from './record-sink.contract.js';
+export { memorySink } from './memory-sink.repository.js';
+export type { MemorySink } from './memory-sink.repository.js';
+export { jsonLinesSink } from './json-lines-sink.repository.js';
+export type { JsonLinesSinkOptions } from './json-lines-sink.repository.js';
+export { DedupePolicy } from './dedupe.policy.js';
+export type { RecipeDedupe } from './dedupe.policy.js';
+export type { DedupeScope } from './dedupe.policy.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/record-sink/json-lines-sink.repository.d.ts

@@ -0,0 +1,19 @@
+import type { RecordSink } from './record-sink.contract.js';
+export interface JsonLinesSinkOptions {
+    /**
+     * Keep what the file holds and add to it. Each line then carries the record
+     * key as `_key`, and `open` reads the keys already there so a resumed run
+     * (`CrawlOptions.resume`) can skip them.
+     */
+    append?: boolean;
+}
+/**
+ * A sink that writes one JSON object per line to a file (JSON Lines). Each line
+ * is the record's `data` plus a `_source` member (and `_key` in append mode).
+ *
+ * @param path - The file to write; created with its directories, truncated on open unless `append`.
+ * @param options - Append mode.
+ * @returns The sink.
+ */
+export declare function jsonLinesSink(path: string, options?: JsonLinesSinkOptions): RecordSink;
+//# sourceMappingURL=json-lines-sink.repository.d.ts.map

package/dist/record-sink/memory-sink.repository.d.ts

@@ -0,0 +1,9 @@
+import type { OutputRecord } from '../output-mapping/index.js';
+import type { RecordSink } from './record-sink.contract.js';
+/** A sink that keeps records in memory; `records` is readable at any time. */
+export interface MemorySink extends RecordSink {
+    readonly records: OutputRecord[];
+}
+/** @returns A new in-memory sink. */
+export declare function memorySink(): MemorySink;
+//# sourceMappingURL=memory-sink.repository.d.ts.map

package/dist/record-sink/record-sink.contract.d.ts

@@ -0,0 +1,17 @@
+import type { OutputRecord } from '../output-mapping/index.js';
+import type { OutputRecipe } from '../recipe-schema/index.js';
+/** What a sink reports when closed. */
+export interface SinkSummary {
+    written: number;
+    /** Where the records went, when the sink has a location. */
+    location?: string;
+}
+/** Where validated records go. Opened once per run, written per record, closed once. */
+export interface RecordSink {
+    open: (output: OutputRecipe) => Promise<void>;
+    write: (record: OutputRecord) => Promise<void>;
+    close: () => Promise<SinkSummary>;
+    /** Whether a record with this key is already there; what a resumed run asks before writing. */
+    has?: (key: string) => Promise<boolean>;
+}
+//# sourceMappingURL=record-sink.contract.d.ts.map

package/dist/selection/html-selector.algorithm.d.ts

@@ -0,0 +1,19 @@
+import type { Cheerio, CheerioAPI } from 'cheerio';
+import type { AnyNode } from 'domhandler';
+/** A matched HTML element together with the document it belongs to. */
+export interface HtmlMatch {
+    api: CheerioAPI;
+    element: Cheerio<AnyNode>;
+}
+/**
+ * Runs a CSS selector on static HTML. A whole document is parsed as one; anything
+ * else (a table row, a list item taken with `take: "html"`) is parsed as a
+ * fragment, so cells and rows outside a table survive instead of being dropped.
+ *
+ * @param html - The markup (a whole document or a fragment).
+ * @param selector - A CSS selector.
+ * @param xml - Read the markup as XML: names keep their case, `take: "html"` gives XML.
+ * @returns Every match, in document order.
+ */
+export declare function selectHtml(html: string, selector: string, xml?: boolean): HtmlMatch[];
+//# sourceMappingURL=html-selector.algorithm.d.ts.map

package/dist/selection/index.d.ts

@@ -0,0 +1,8 @@
+export { selectJson } from './json-path.algorithm.js';
+export { selectRegex } from './regex.algorithm.js';
+export { selectHtml } from './html-selector.algorithm.js';
+export type { HtmlMatch } from './html-selector.algorithm.js';
+export { takeFromHtml, takeFromJson, collapse } from './take-value.mapper.js';
+export type { Take } from './take-value.mapper.js';
+export { tryParseJson, parseJsonText, dataItemsOf, parseJsonLike, parseJsonLines } from './json-text.algorithm.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/selection/json-path.algorithm.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Evaluates a JSONPath expression on a decoded JSON document.
+ *
+ * @param document - The JSON value.
+ * @param path - A JSONPath such as `$.items[*].url`.
+ * @returns Every match, in document order.
+ */
+export declare function selectJson(document: unknown, path: string): unknown[];
+//# sourceMappingURL=json-path.algorithm.d.ts.map

package/dist/selection/json-text.algorithm.d.ts

@@ -0,0 +1,58 @@
+/**
+ * JSON that arrives as text: a `<script type="application/ld+json">` body, a
+ * `data-*` attribute, a fetched document read as text. Sites wrap JSON in
+ * things that are not JSON: comment guards around inline JSON-LD, prefixes
+ * that stop a page from loading an API as a script, a JSONP callback, an
+ * assignment in an inline script. Those wrappers are removed, but only after
+ * the text failed to parse as it is, and what is left must still be strict
+ * JSON: nothing is evaluated.
+ */
+/**
+ * Parses text as JSON, or as JSON inside one of the wrappers sites put around
+ * it: comment guards, an anti-hijacking prefix, a JSONP call, an assignment.
+ * Valid JSON is always read as it is; a wrapper is only removed when that
+ * fails.
+ *
+ * @param text - The text.
+ * @returns The value, or the error the text as it is gave.
+ */
+export declare function parseJsonLike(text: string): {
+    value: unknown;
+} | {
+    error: Error;
+};
+/**
+ * Parses JSON Lines (NDJSON): one JSON value per non-blank line.
+ *
+ * @param text - The text.
+ * @param source - Where it came from, for the error.
+ * @returns The values, in order.
+ * @throws Error naming the source and the line that does not parse.
+ */
+export declare function parseJsonLines(text: string, source: string): unknown[];
+/**
+ * Parses text as JSON, wrappers removed (see {@link parseJsonLike}).
+ *
+ * @param text - The text.
+ * @returns The value, or `undefined` when it is not JSON.
+ */
+export declare function tryParseJson(text: string): unknown;
+/**
+ * Parses text that must be JSON.
+ *
+ * @param text - The text.
+ * @param id - What the text is, for the error.
+ * @returns The value.
+ * @throws Error when it is not JSON.
+ */
+export declare function parseJsonText(text: string, id: string): unknown;
+/**
+ * The data a value holds, whatever shape it arrived in: JSON text is parsed, a
+ * list of texts becomes the list of its parsable entries, and an entry that
+ * parses to a list is spliced in. Data that is not text is kept as is.
+ *
+ * @param value - A bound value: data, text, or a list of either.
+ * @returns A list of items.
+ */
+export declare function dataItemsOf(value: unknown): unknown[];
+//# sourceMappingURL=json-text.algorithm.d.ts.map

package/dist/selection/regex.algorithm.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Evaluates a regular expression on text: the extract kind for values that
+ * live in inline scripts, attributes or prose rather than in elements or JSON.
+ *
+ * @param text - The document text.
+ * @param pattern - A regular expression source; group 1 is returned when the
+ * pattern has a capturing group, else the whole match.
+ * @returns Every match, in document order.
+ * @throws When the pattern is not a valid regular expression.
+ */
+export declare function selectRegex(text: string, pattern: string): string[];
+//# sourceMappingURL=regex.algorithm.d.ts.map

package/dist/selection/take-value.mapper.d.ts

@@ -0,0 +1,27 @@
+import type { HtmlMatch } from './html-selector.algorithm.js';
+/** What `extract` takes from a match: text, inner HTML, an attribute, an input's value, or the node as data. */
+export type Take = 'text' | 'html' | 'value' | 'json' | `attr:${string}`;
+/**
+ * The value of an HTML match.
+ *
+ * @param match - A selected element.
+ * @param take - What to take; `text` collapses whitespace.
+ * @returns The value; `undefined` for a missing attribute.
+ */
+export declare function takeFromHtml(match: HtmlMatch, take: Take): unknown;
+/**
+ * The value of a JSON match.
+ *
+ * @param node - A JSONPath result.
+ * @param take - `json` keeps the node; `text` stringifies scalars.
+ * @returns The value.
+ */
+export declare function takeFromJson(node: unknown, take: Take): unknown;
+/**
+ * Text as a human reads it: runs of whitespace collapsed, ends trimmed.
+ *
+ * @param text - Raw text content.
+ * @returns The collapsed text.
+ */
+export declare function collapse(text: string): string;
+//# sourceMappingURL=take-value.mapper.d.ts.map

package/dist/src/access/access-profile.contract.d.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import type { ThrottleConfig } from '../step-flow/index.js';
 /**
  * Where a crawl's traffic goes. Profiles live in the runner's access config,
  * never in a recipe: they hold the user's accounts. Every string may use
@@ -84,8 +85,11 @@ export interface AccessConfig {
     $schema?: string;
     profiles: Record<string, AccessProfile>;
     default?: string;
+    /** How gently each site is crawled, across every recipe: `{ delayMs?, concurrency?, domains? }`. */
+    throttle?: ThrottleConfig;
 }
 export declare const accessProfileSchema: z.ZodType<AccessProfile>;
+export declare const throttleConfigSchema: z.ZodType<ThrottleConfig>;
 export declare const accessConfigSchema: z.ZodType<AccessConfig>;
 export {};
 //# sourceMappingURL=access-profile.contract.d.ts.map

package/dist/src/access/index.d.ts

@@ -3,7 +3,7 @@ export { loadAccessConfig } from './access-config.repository.js';
 export { AccessConfigError } from './access-config.error.js';
 export { ACCESS_PRESETS } from './access-preset.store.js';
 export type { AccessPreset } from './access-preset.store.js';
-export { accessConfigSchema, accessProfileSchema, BLOCKABLE_RESOURCES } from './access-profile.contract.js';
+export { accessConfigSchema, accessProfileSchema, throttleConfigSchema, BLOCKABLE_RESOURCES } from './access-profile.contract.js';
 export type { AccessConfig, AccessProfile, ProxyProfile, PoolProfile, CdpProfile, PluginProfile, DirectProfile, ProxySettings, BlockableResource } from './access-profile.contract.js';
 export type { AccessPlugin, AccessLease, LeaseRequest, PluginLeaseRequest } from './access-plugin.contract.js';
 export { newSessionId } from './session-id.algorithm.js';

package/dist/src/api-steps/extract-from-document.use-case.d.ts

@@ -2,11 +2,11 @@ import type { ExtractionScope, ScopeDocument } from '../extraction-scope/index.j
 import type { ExtractStep } from '../recipe-schema/index.js';
 /**
  * Runs an `extract` step against a static document: the value bound under
- * `from`, else the scope's current document. `css` reads HTML, `jsonpath`
- * reads JSON (or a read PDF, workbook or deck as data), `table` reads the
- * tables of a PDF, a workbook (a spreadsheet, a CSV), a deck (a presentation)
- * or HTML (its `<table>`s), `regex` reads any document as text; `xpath` needs
- * a live page and is refused here.
+ * `from`, else the scope's current document. `css` reads HTML (or XML),
+ * `xpath` reads XML (a feed, a sitemap) or HTML parsed as a browser parses
+ * it, `jsonpath` reads JSON (or a read PDF, workbook or deck as data), `table`
+ * reads the tables of a PDF, a workbook (a spreadsheet, a CSV), a deck (a
+ * presentation) or HTML (its `<table>`s), `regex` reads any document as text.
  *
  * A `jsonpath` extract whose `from` is text parses that text as JSON, and a
  * list of texts (every `<script type="application/ld+json">` of a page) becomes

package/dist/src/api-steps/send-request.use-case.d.ts

@@ -5,8 +5,9 @@ import type { InputRecipe, RequestStep } from '../recipe-schema/index.js';
 import type { RunGate } from '../step-flow/index.js';
 /**
  * Sends a `request` step: renders its templates, waits for the gate's throttle,
- * sends, checks the response against the recipe's block rule, then binds it as
- * the scope's current document (and under the step id).
+ * sends (again, after a pause, while it fails in passing: `limits.retry`),
+ * checks the response against the recipe's block rule, then binds it as the
+ * scope's current document (and under the step id).
  *
  * @param step - The request step.
  * @param scope - The scope to render in and bind into.

package/dist/src/browser-session/browser-profile.store.d.ts

@@ -0,0 +1,52 @@
+import type { BrowserSessionConfig } from './browser-session.config.js';
+import { BrowserSession } from './browser.client.js';
+import type { SessionOptions } from './browser.client.js';
+/** A browser profile name: it becomes a directory, so no separators or dots. */
+export declare const BROWSER_PROFILE_NAME: RegExp;
+/**
+ * Browser profiles that persist between runs: each is a directory of a real
+ * browser's user data (cookies, local storage, IndexedDB, cache, service
+ * workers), so a login, a consent choice or a site's trust in a returning
+ * visitor carries over to the next run. The browser equivalent of a user who
+ * never clears their history.
+ *
+ * A profile directory can be open in one browser at a time. Within this
+ * crawler, a second use waits for the first to close; the same owner (one
+ * recipe run reopening after a rotation) takes it over instead. Another
+ * crawler holding it, in this process or another, is reported, not waited
+ * for: a lock file in the profile names the process, and one left by a
+ * process that died is taken over. (Chromium's own profile lock is not
+ * enough: headless builds do not take it.)
+ */
+export declare class BrowserProfiles {
+    readonly directory: string;
+    private readonly config;
+    private readonly held;
+    private readonly waiting;
+    /**
+     * @param directory - Where the profiles live, one subdirectory each.
+     * @param config - The crawler's browser settings (type, binary, headless, timeouts).
+     */
+    constructor(directory: string, config?: BrowserSessionConfig);
+    private take;
+    private free;
+    private launch;
+    /**
+     * The profile's directory.
+     *
+     * @param name - A profile name.
+     * @returns The absolute path.
+     */
+    pathOf(name: string): string;
+    /**
+     * Opens a profile in its own browser, waiting while another run of this
+     * crawler uses it.
+     *
+     * @param name - The profile.
+     * @param options - Proxy, headers, viewport, cookies to add. `storageState` is ignored: the profile has its own.
+     * @param owner - Who opens it; the same owner reopening closes its previous session first.
+     * @returns The session; closing it frees the profile.
+     */
+    open(name: string, options: SessionOptions, owner: object): Promise<BrowserSession>;
+}
+//# sourceMappingURL=browser-profile.store.d.ts.map

package/dist/src/browser-session/browser.client.d.ts

@@ -39,6 +39,14 @@ export declare class BrowserSession {
     storageState(): Promise<StorageState>;
     close(): Promise<void>;
 }
+/**
+ * What a context gets after it opened: the cookies to add and the resource
+ * types to skip.
+ *
+ * @param context - The context.
+ * @param options - The session options.
+ */
+export declare function applySessionExtras(context: BrowserContext, options: SessionOptions): Promise<void>;
 /** A launched browser; sessions are opened from it and closed independently. */
 export declare class BrowserClient {
     private readonly browser;

package/dist/src/browser-session/index.d.ts

@@ -1,4 +1,5 @@
 export { BrowserClient, BrowserSession } from './browser.client.js';
+export { BrowserProfiles, BROWSER_PROFILE_NAME } from './browser-profile.store.js';
 export type { StorageState, SessionOptions } from './browser.client.js';
 export { DEFAULT_BROWSER_CONFIG } from './browser-session.config.js';
 export type { BrowserSessionConfig } from './browser-session.config.js';

package/dist/src/crawl-events/crawl-event.contract.d.ts

@@ -49,6 +49,14 @@ export type CrawlEvent = (Base & {
     attempt: number;
     reason: string;
 }) | 
+/** A request failed in passing (a dropped connection, a 503, a 429) and is sent again after `delayMs`. */
+(Base & {
+    type: 'request:retry';
+    url: string;
+    attempt: number;
+    reason: string;
+    delayMs: number;
+}) | 
 /** A captcha challenge is on the page. */
 (Base & {
     type: 'captcha:detected';

package/dist/src/crawl-execution/bootstrap-session.use-case.d.ts

@@ -1,15 +1,20 @@
 import type { AccessLease } from '../access/index.js';
-import type { BrowserClient, BrowserSession, SessionOptions, StorageState } from '../browser-session/index.js';
+import type { BrowserClient, BrowserProfiles, BrowserSession, SessionOptions, StorageState } from '../browser-session/index.js';
 import type { CaptchaGuard } from '../captcha/index.js';
 import type { EventBus } from '../crawl-events/index.js';
 import type { HookRegistry } from '../hooks/index.js';
 import type { InputRecipe } from '../recipe-schema/index.js';
+import type { HostThrottle } from '../step-flow/index.js';
 export interface BootstrapDependencies {
     /** Launches (or returns) the shared browser; only called when a browser is needed. */
     browser: () => Promise<BrowserClient>;
     hooks: HookRegistry;
     events: EventBus;
     storageStateDir?: string;
+    /** The crawler's per-site throttle: the bootstrap's pages count too. */
+    hosts?: HostThrottle;
+    /** The runner's persistent browser profiles, for `session.browserProfile`. */
+    profiles?: BrowserProfiles;
 }
 /**
  * The session options an access lease contributes: its proxy, TLS leniency,
@@ -28,13 +33,29 @@ export declare function accessOptions(lease: AccessLease | undefined, headers: R
  * The bootstrap runs through the same access lease as the crawl that follows,
  * so a login and the requests that use its cookies come from one IP.
  *
+ * With `session.browserProfile`, the bootstrap runs in that profile, and
+ * without a bootstrap the profile's own cookies and storage are the state: an
+ * api recipe picks up a login a browser left in the profile.
+ *
  * @param recipe - The input recipe.
  * @param deps - Browser, hooks, events.
  * @param lease - The recipe run's access; direct when omitted.
  * @param captcha - Solves the bootstrap's captchas (a login form's).
+ * @param owner - The recipe run, which a browser profile is held by.
  * @returns The state, or `undefined` when the recipe declares none.
  */
-export declare function resolveStorageState(recipe: InputRecipe, deps: BootstrapDependencies, lease?: AccessLease, captcha?: CaptchaGuard): Promise<StorageState | undefined>;
+export declare function resolveStorageState(recipe: InputRecipe, deps: BootstrapDependencies, lease?: AccessLease, captcha?: CaptchaGuard, owner?: object): Promise<StorageState | undefined>;
+/**
+ * Opens the recipe's `session.browserProfile` with its session options and
+ * the lease's proxy.
+ *
+ * @param recipe - A recipe with `session.browserProfile`.
+ * @param deps - For `profiles`.
+ * @param lease - The access lease.
+ * @param owner - The recipe run.
+ * @returns The session in the profile.
+ */
+export declare function openBrowserProfile(recipe: InputRecipe, deps: Pick<BootstrapDependencies, 'profiles'>, lease: AccessLease | undefined, owner: object): Promise<BrowserSession>;
 /**
  * The storage state saved by an earlier bootstrap (`session.storageStatePath`), if the recipe names one.
  *

package/dist/src/crawl-execution/crawl-options.config.d.ts

@@ -3,6 +3,8 @@ import type { BrowserSessionConfig } from '../browser-session/index.js';
 import type { CaptchaSolver } from '../captcha/index.js';
 import type { CrawlListener } from '../crawl-events/index.js';
 import type { HookMap } from '../hooks/index.js';
+import type { ThrottleConfig } from '../step-flow/index.js';
+import type { RetryRule } from '../recipe-schema/index.js';
 import type { DedupeScope, RecordSink } from '../record-sink/index.js';
 /** How a crawler is created. Everything is optional. */
 export interface CrawlOptions {
@@ -15,10 +17,21 @@ export interface CrawlOptions {
     onEvent?: CrawlListener;
     /** Default `run`: a key seen once is dropped for the rest of the run. */
     dedupe?: DedupeScope;
+    /**
+     * How many input recipes of a set run at once; default 1, one after
+     * another. Each has its own browser context or HTTP session; the browser,
+     * the sink and the per-site `throttle` are shared.
+     */
+    parallel?: number;
     /** Whether a failed input recipe stops the run; default `continue`. */
     onRecipeError?: 'continue' | 'stop';
     /** Base directory for relative `storageStatePath` and `saveTo` values. */
     storageStateDir?: string;
+    /**
+     * Where `session.browserProfile` profiles live, one directory each. Default:
+     * `.opencraw/profiles` under `storageStateDir` (or the working directory).
+     */
+    profilesDir?: string;
     /**
      * Skip records whose key the sink already has (`sink.has`), reporting them as
      * `skipped`. Needs a sink that can answer, such as `jsonLinesSink(path, { append: true })`.
@@ -32,6 +45,20 @@ export interface CrawlOptions {
      * with `session.access.profile`. Without it every recipe goes direct.
      */
     access?: AccessConfig;
+    /**
+     * How gently each site is crawled, across every recipe this crawler runs:
+     * `delayMs` between request starts and `concurrency` requests in flight,
+     * per site, with `domains` for site-specific rules. Defaults to
+     * `access.throttle`; without either, only each recipe's `limits` apply.
+     */
+    throttle?: ThrottleConfig;
+    /**
+     * How a request that fails in passing (a dropped connection, a timeout, a
+     * 503, a 429) is sent again, for recipes whose `limits.retry` says
+     * nothing: `{ attempts?, backoffMs?, maxDelayMs?, statuses? }`. Default:
+     * three tries, one then two seconds apart, `Retry-After` honoured.
+     */
+    retry?: RetryRule;
     /** Plugins `{ kind: 'plugin', name }` profiles refer to. */
     accessPlugins?: AccessPlugin[];
     /** Solvers recipes name in `session.captcha.solver` and `captcha` steps. */

package/dist/src/crawl-execution/rotating-runner.use-case.d.ts

@@ -50,6 +50,15 @@ export declare class RotatingRunner implements StepRunner {
     private note;
     runLeaf(step: Step, scope: ExtractionScope): Promise<void>;
     nextPage(next: PaginateNext, scope: ExtractionScope): Promise<NextPageResult>;
+    /**
+     * A runner for one parallel iteration, forked from whichever runner is
+     * current when it runs a step: after a rotation it forks again from the new
+     * one, since the old context is gone (or going). Blocks are noted and
+     * rotated like the main runner's.
+     *
+     * @returns The iteration's runner.
+     */
+    fork(): Promise<StepRunner>;
     elements(selector: string, scope: ExtractionScope): Promise<LiveElement[]>;
     rotate(error: BlockedError): Promise<boolean>;
     dispose(): Promise<void>;

package/dist/src/crawl-execution/run-crawl.use-case.d.ts

@@ -2,12 +2,16 @@ import type { RecipeSet } from '../recipe-loading/index.js';
 import type { CrawlReport } from './crawl-report.model.js';
 import type { RecipeRunDependencies } from './run-input-recipe.use-case.js';
 /**
- * Runs every input recipe of a set, one after another, into one sink.
+ * Runs every input recipe of a set into one sink, `parallel` at a time
+ * (default one after another). Reports come back in the set's order whatever
+ * order the recipes finish in. Under `onRecipeError: 'stop'`, a failed recipe
+ * stops the ones not started yet; those already running finish.
  *
  * @param set - The bound recipes.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @param onRecipeError - Whether a failed recipe stops the run.
+ * @param parallel - How many input recipes run at once.
  * @returns The report.
  */
-export declare function runCrawl(set: RecipeSet, deps: RecipeRunDependencies, onRecipeError: 'continue' | 'stop'): Promise<CrawlReport>;
+export declare function runCrawl(set: RecipeSet, deps: RecipeRunDependencies, onRecipeError: 'continue' | 'stop', parallel?: number): Promise<CrawlReport>;
 //# sourceMappingURL=run-crawl.use-case.d.ts.map