@opencraw/core 0.1.1 → 0.1.3

package/dist/src/access/access-profile.contract.d.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import type { ThrottleConfig } from '../step-flow/index.js';
 /**
  * Where a crawl's traffic goes. Profiles live in the runner's access config,
  * never in a recipe: they hold the user's accounts. Every string may use
@@ -84,8 +85,11 @@ export interface AccessConfig {
     $schema?: string;
     profiles: Record<string, AccessProfile>;
     default?: string;
+    /** How gently each site is crawled, across every recipe: `{ delayMs?, concurrency?, domains? }`. */
+    throttle?: ThrottleConfig;
 }
 export declare const accessProfileSchema: z.ZodType<AccessProfile>;
+export declare const throttleConfigSchema: z.ZodType<ThrottleConfig>;
 export declare const accessConfigSchema: z.ZodType<AccessConfig>;
 export {};
 //# sourceMappingURL=access-profile.contract.d.ts.map

package/dist/src/access/index.d.ts

@@ -3,7 +3,7 @@ export { loadAccessConfig } from './access-config.repository.js';
 export { AccessConfigError } from './access-config.error.js';
 export { ACCESS_PRESETS } from './access-preset.store.js';
 export type { AccessPreset } from './access-preset.store.js';
-export { accessConfigSchema, accessProfileSchema, BLOCKABLE_RESOURCES } from './access-profile.contract.js';
+export { accessConfigSchema, accessProfileSchema, throttleConfigSchema, BLOCKABLE_RESOURCES } from './access-profile.contract.js';
 export type { AccessConfig, AccessProfile, ProxyProfile, PoolProfile, CdpProfile, PluginProfile, DirectProfile, ProxySettings, BlockableResource } from './access-profile.contract.js';
 export type { AccessPlugin, AccessLease, LeaseRequest, PluginLeaseRequest } from './access-plugin.contract.js';
 export { newSessionId } from './session-id.algorithm.js';

package/dist/src/api-steps/extract-from-document.use-case.d.ts

@@ -1,10 +1,12 @@
-import type { ExtractionScope } from '../extraction-scope/index.js';
+import type { ExtractionScope, ScopeDocument } from '../extraction-scope/index.js';
 import type { ExtractStep } from '../recipe-schema/index.js';
 /**
  * Runs an `extract` step against a static document: the value bound under
  * `from`, else the scope's current document. `css` reads HTML, `jsonpath`
- * reads JSON (or a read PDF's rows), `table` reads a PDF's tables, `regex`
- * reads any document as text; `xpath` needs a live page and is refused here.
+ * reads JSON (or a read PDF, workbook or deck as data), `table` reads the
+ * tables of a PDF, a workbook (a spreadsheet, a CSV), a deck (a presentation)
+ * or HTML (its `<table>`s), `regex` reads any document as text; `xpath` needs
+ * a live page and is refused here.
  *
  * A `jsonpath` extract whose `from` is text parses that text as JSON, and a
  * list of texts (every `<script type="application/ld+json">` of a page) becomes
@@ -25,4 +27,15 @@ export declare function extractFromDocument(step: ExtractStep, scope: Extraction
  * @returns The selector to run.
  */
 export declare function renderSelector(selector: string, scope: ExtractionScope): string;
+/**
+ * The tables a `table` extract finds in a document: a PDF's, a workbook's, a
+ * deck's, or an HTML document's `<table>`s (a fetched page, rendered Markdown,
+ * a live page's content).
+ *
+ * @param document - The document.
+ * @param step - The extract step.
+ * @param scope - Where its selector renders.
+ * @returns The tables.
+ */
+export declare function tablesIn(document: ScopeDocument, step: ExtractStep, scope: ExtractionScope): unknown[];
 //# sourceMappingURL=extract-from-document.use-case.d.ts.map

package/dist/src/api-steps/index.d.ts

@@ -1,4 +1,4 @@
 export { ApiStepRunner } from './run-api-step.use-case.js';
-export { extractFromDocument, renderSelector } from './extract-from-document.use-case.js';
+export { extractFromDocument, renderSelector, tablesIn } from './extract-from-document.use-case.js';
 export { sendRequest, documentValue } from './send-request.use-case.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/api-steps/send-request.use-case.d.ts

@@ -5,8 +5,9 @@ import type { InputRecipe, RequestStep } from '../recipe-schema/index.js';
 import type { RunGate } from '../step-flow/index.js';
 /**
  * Sends a `request` step: renders its templates, waits for the gate's throttle,
- * sends, checks the response against the recipe's block rule, then binds it as
- * the scope's current document (and under the step id).
+ * sends (again, after a pause, while it fails in passing: `limits.retry`),
+ * checks the response against the recipe's block rule, then binds it as the
+ * scope's current document (and under the step id).
  *
  * @param step - The request step.
  * @param scope - The scope to render in and bind into.
@@ -14,9 +15,9 @@ import type { RunGate } from '../step-flow/index.js';
  * @param recipe - The recipe: its limits, block rule and id.
  * @param gate - Spaces request starts by `delayMs`.
  * @param events - Where to report the visit.
- * @throws BlockedError when the response is a block; HttpError for any other 4xx/5xx.
+ * @throws BlockedError when the response is a block, or a captcha page under `session.captcha`; HttpError for any other 4xx/5xx.
  */
 export declare function sendRequest(step: RequestStep, scope: ExtractionScope, client: HttpSender, recipe: InputRecipe, gate: RunGate, events: EventBus): Promise<void>;
-/** What a step id holds for a document: parsed JSON, the read PDF, or the markup / text. */
+/** What a step id holds for a document: parsed JSON, the read PDF, workbook or deck, or the markup / text. */
 export declare function documentValue(body: HttpBody): unknown;
 //# sourceMappingURL=send-request.use-case.d.ts.map

package/dist/src/browser-session/browser-profile.store.d.ts

@@ -0,0 +1,52 @@
+import type { BrowserSessionConfig } from './browser-session.config.js';
+import { BrowserSession } from './browser.client.js';
+import type { SessionOptions } from './browser.client.js';
+/** A browser profile name: it becomes a directory, so no separators or dots. */
+export declare const BROWSER_PROFILE_NAME: RegExp;
+/**
+ * Browser profiles that persist between runs: each is a directory of a real
+ * browser's user data (cookies, local storage, IndexedDB, cache, service
+ * workers), so a login, a consent choice or a site's trust in a returning
+ * visitor carries over to the next run. The browser equivalent of a user who
+ * never clears their history.
+ *
+ * A profile directory can be open in one browser at a time. Within this
+ * crawler, a second use waits for the first to close; the same owner (one
+ * recipe run reopening after a rotation) takes it over instead. Another
+ * crawler holding it, in this process or another, is reported, not waited
+ * for: a lock file in the profile names the process, and one left by a
+ * process that died is taken over. (Chromium's own profile lock is not
+ * enough: headless builds do not take it.)
+ */
+export declare class BrowserProfiles {
+    readonly directory: string;
+    private readonly config;
+    private readonly held;
+    private readonly waiting;
+    /**
+     * @param directory - Where the profiles live, one subdirectory each.
+     * @param config - The crawler's browser settings (type, binary, headless, timeouts).
+     */
+    constructor(directory: string, config?: BrowserSessionConfig);
+    private take;
+    private free;
+    private launch;
+    /**
+     * The profile's directory.
+     *
+     * @param name - A profile name.
+     * @returns The absolute path.
+     */
+    pathOf(name: string): string;
+    /**
+     * Opens a profile in its own browser, waiting while another run of this
+     * crawler uses it.
+     *
+     * @param name - The profile.
+     * @param options - Proxy, headers, viewport, cookies to add. `storageState` is ignored: the profile has its own.
+     * @param owner - Who opens it; the same owner reopening closes its previous session first.
+     * @returns The session; closing it frees the profile.
+     */
+    open(name: string, options: SessionOptions, owner: object): Promise<BrowserSession>;
+}
+//# sourceMappingURL=browser-profile.store.d.ts.map

package/dist/src/browser-session/browser.client.d.ts

@@ -39,6 +39,14 @@ export declare class BrowserSession {
     storageState(): Promise<StorageState>;
     close(): Promise<void>;
 }
+/**
+ * What a context gets after it opened: the cookies to add and the resource
+ * types to skip.
+ *
+ * @param context - The context.
+ * @param options - The session options.
+ */
+export declare function applySessionExtras(context: BrowserContext, options: SessionOptions): Promise<void>;
 /** A launched browser; sessions are opened from it and closed independently. */
 export declare class BrowserClient {
     private readonly browser;

package/dist/src/browser-session/index.d.ts

@@ -1,4 +1,5 @@
 export { BrowserClient, BrowserSession } from './browser.client.js';
+export { BrowserProfiles, BROWSER_PROFILE_NAME } from './browser-profile.store.js';
 export type { StorageState, SessionOptions } from './browser.client.js';
 export { DEFAULT_BROWSER_CONFIG } from './browser-session.config.js';
 export type { BrowserSessionConfig } from './browser-session.config.js';

package/dist/src/captcha/captcha-budget.model.d.ts

@@ -0,0 +1,21 @@
+/** Default solves a recipe run may spend. */
+export declare const DEFAULT_MAX_SOLVES = 10;
+/**
+ * The solves a recipe run may still spend. Every solve costs money: a detector
+ * that matches the wrong element would drain a balance without it. Shared by
+ * every runner of the run, rotations included.
+ */
+export declare class CaptchaBudget {
+    readonly max: number;
+    private used;
+    constructor(max?: number);
+    /** Solves spent so far. */
+    get spent(): number;
+    /**
+     * Spends one solve.
+     *
+     * @returns Whether one was left.
+     */
+    take(): boolean;
+}
+//# sourceMappingURL=captcha-budget.model.d.ts.map

package/dist/src/captcha/captcha-detection.client.d.ts

@@ -0,0 +1,28 @@
+import type { Page } from 'playwright';
+import type { CaptchaChallenge } from './captcha-solver.contract.js';
+/** The widgets detection looks for when a recipe names none: reCAPTCHA v2, hCaptcha and Turnstile, as a container or as their iframe. */
+export declare const DEFAULT_CAPTCHA_SELECTOR: string;
+/**
+ * The first visible challenge on the page, if any. An element counts only
+ * when visible: sites keep hidden widgets around after a solve, and an
+ * invisible reCAPTCHA shows nothing until it challenges.
+ *
+ * @param page - The live page.
+ * @param selector - Where challenges are; `DEFAULT_CAPTCHA_SELECTOR` when omitted.
+ * @param options - `v3`: also report a reCAPTCHA v3 script (a `captcha` step asks for it; the automatic checks do not, since v3 never blocks a page by itself).
+ * @returns The challenge, or `undefined`.
+ */
+export declare function detectChallenge(page: Page, selector?: string, options?: {
+    v3?: boolean;
+}): Promise<CaptchaChallenge | undefined>;
+/**
+ * Whether the page is clear of challenges, tolerating a page that is
+ * navigating (a solve often submits a form): an evaluation cut short by the
+ * navigation counts as not clear yet.
+ *
+ * @param page - The live page.
+ * @param selector - Where challenges are.
+ * @returns Whether no challenge is visible.
+ */
+export declare function isClear(page: Page, selector: string): Promise<boolean>;
+//# sourceMappingURL=captcha-detection.client.d.ts.map

package/dist/src/captcha/captcha-guard.use-case.d.ts

@@ -0,0 +1,64 @@
+import type { Page } from 'playwright';
+import type { AccessLease } from '../access/index.js';
+import type { EventBus } from '../crawl-events/index.js';
+import type { CaptchaStep, InputRecipe } from '../recipe-schema/index.js';
+import type { BlockedError } from '../step-flow/index.js';
+import type { CaptchaBudget } from './captcha-budget.model.js';
+import type { CaptchaSolverRegistry } from './captcha-solver-registry.store.js';
+export interface CaptchaGuardOptions {
+    recipe: InputRecipe;
+    events: EventBus;
+    solvers: CaptchaSolverRegistry;
+    /** Shared by every runner of the recipe run, rotations and the bootstrap included. */
+    budget: CaptchaBudget;
+    /** The access the page goes through, handed to solvers. */
+    lease?: AccessLease;
+}
+/**
+ * Where a web runner meets captchas: after each navigation, click and key
+ * press (`session.captcha`), on a block page (`onBlock.solve`), and at a
+ * `captcha` step.
+ */
+export declare class CaptchaGuard {
+    private readonly options;
+    constructor(options: CaptchaGuardOptions);
+    private settings;
+    private solve;
+    /** Whether a block page is searched for a challenge before the block counts. */
+    get solvesBlocks(): boolean;
+    /**
+     * The automatic check: with `session.captcha`, solves the challenge the page
+     * shows, if any. Without it, nothing is looked for.
+     *
+     * @param page - The live page.
+     * @throws CaptchaError when the challenge could not be solved.
+     */
+    check(page: Page): Promise<void>;
+    /**
+     * A block page under `onBlock.solve`: solves the challenge it shows. A block
+     * without a challenge stays a block.
+     *
+     * @param page - The page showing the block.
+     * @param blocked - The block.
+     * @throws BlockedError (`blocked`) when the page shows no challenge; CaptchaError when it could not be solved.
+     */
+    solveBlock(page: Page, blocked: BlockedError): Promise<void>;
+    /**
+     * A `captcha` step: solves the challenge the page shows, reCAPTCHA v3
+     * included; a page without one is fine.
+     *
+     * @param page - The live page.
+     * @param step - The step.
+     * @throws CaptchaError when the challenge could not be solved.
+     */
+    step(page: Page, step: CaptchaStep): Promise<void>;
+}
+/**
+ * Every solver name a recipe uses (`session.captcha` and its `captcha`
+ * steps, the bootstrap's included), to check them before the run starts.
+ *
+ * @param recipe - The input recipe.
+ * @returns The names, without repeats.
+ */
+export declare function captchaSolverNames(recipe: InputRecipe): string[];
+//# sourceMappingURL=captcha-guard.use-case.d.ts.map

package/dist/src/captcha/captcha-solver-registry.store.d.ts

@@ -0,0 +1,19 @@
+import type { CaptchaSolver } from './captcha-solver.contract.js';
+/** The captcha solvers a crawler was given, by name. */
+export declare class CaptchaSolverRegistry {
+    private readonly solvers;
+    private readonly order;
+    constructor(solvers?: readonly CaptchaSolver[]);
+    /** The registered names. */
+    get names(): string[];
+    has(name: string): boolean;
+    /**
+     * The solver of that name.
+     *
+     * @param name - As a recipe names it.
+     * @returns The solver.
+     * @throws Error naming what is registered when it is not.
+     */
+    resolve(name: string): CaptchaSolver;
+}
+//# sourceMappingURL=captcha-solver-registry.store.d.ts.map

package/dist/src/captcha/captcha-solver.contract.d.ts

@@ -0,0 +1,47 @@
+import type { Page } from 'playwright';
+import type { AccessLease } from '../access/index.js';
+/** The kinds of challenge detection tells apart. */
+export type CaptchaKind = 'recaptcha-v2' | 'recaptcha-v3' | 'hcaptcha' | 'turnstile' | 'image' | 'unknown';
+/** What the engine found on the page: enough for a token service, or for a solver that works on the page itself. */
+export interface CaptchaChallenge {
+    kind: CaptchaKind;
+    /** The page the challenge is on. */
+    url: string;
+    /** The widget's site key (`data-sitekey`, or the `k` / `sitekey` of its iframe), when it has one. */
+    siteKey?: string;
+    /** The widget's action (`data-action`), when it declares one. */
+    action?: string;
+    /** A Playwright selector for the widget, or for the image of an image captcha. */
+    selector?: string;
+}
+export type CaptchaLog = (level: 'debug' | 'info' | 'warn' | 'error', message: string, meta?: Record<string, unknown>) => void;
+/** What a solver gets besides the challenge. */
+export interface CaptchaContext {
+    /** The live page: the solver may inject a token, fill a field, click. */
+    page: Page;
+    /** The access in use: token services often need the same IP (proxy) as the browser. */
+    lease?: AccessLease;
+    /** 1, 2, … within one solve loop. */
+    attempt: number;
+    /** Aborted when the engine's timeout for this attempt runs out. */
+    signal: AbortSignal;
+    log: CaptchaLog;
+}
+/** A solver's report. The engine checks it: `solved` is a claim until the page confirms it. */
+export type CaptchaOutcome = {
+    status: 'solved';
+} | {
+    status: 'failed';
+    reason: string;
+};
+/**
+ * A captcha solver: solves a challenge and applies the answer on the page
+ * (injects a token and calls the widget's callback, types an image's text,
+ * submits). Detection, verification, retries, rotation and the budget are the
+ * engine's.
+ */
+export interface CaptchaSolver {
+    name: string;
+    solve: (challenge: CaptchaChallenge, context: CaptchaContext) => Promise<CaptchaOutcome> | CaptchaOutcome;
+}
+//# sourceMappingURL=captcha-solver.contract.d.ts.map

package/dist/src/captcha/captcha.error.d.ts

@@ -0,0 +1,13 @@
+import { BlockedError } from '../step-flow/index.js';
+/**
+ * A challenge the engine could not get past: the solver failed, the page did
+ * not confirm it, or the budget ran out. It is a block, so a recipe with
+ * `onBlock.rotate` retries the step on a new access lease (a new IP often
+ * means an easier challenge, or none), then the step's error policy applies.
+ */
+export declare class CaptchaError extends BlockedError {
+    readonly kind: string;
+    readonly attempts: number;
+    constructor(url: string, kind: string, attempts: number, reason: string);
+}
+//# sourceMappingURL=captcha.error.d.ts.map

package/dist/src/captcha/index.d.ts

@@ -0,0 +1,10 @@
+export type { CaptchaSolver, CaptchaChallenge, CaptchaContext, CaptchaOutcome, CaptchaKind, CaptchaLog } from './captcha-solver.contract.js';
+export { CaptchaError } from './captcha.error.js';
+export { CaptchaSolverRegistry } from './captcha-solver-registry.store.js';
+export { CaptchaBudget, DEFAULT_MAX_SOLVES } from './captcha-budget.model.js';
+export { DEFAULT_CAPTCHA_SELECTOR, detectChallenge } from './captcha-detection.client.js';
+export { resolveCaptcha, DEFAULT_CAPTCHA_ATTEMPTS, DEFAULT_CAPTCHA_TIMEOUT_MS } from './resolve-captcha.use-case.js';
+export type { CaptchaAttemptPlan } from './resolve-captcha.use-case.js';
+export { CaptchaGuard, captchaSolverNames } from './captcha-guard.use-case.js';
+export type { CaptchaGuardOptions } from './captcha-guard.use-case.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/captcha/resolve-captcha.use-case.d.ts

@@ -0,0 +1,40 @@
+import type { Page } from 'playwright';
+import type { AccessLease } from '../access/index.js';
+import type { EventBus } from '../crawl-events/index.js';
+import type { CaptchaCheck } from '../recipe-schema/index.js';
+import type { CaptchaBudget } from './captcha-budget.model.js';
+import type { CaptchaChallenge, CaptchaSolver } from './captcha-solver.contract.js';
+/** Default solves tried per challenge. */
+export declare const DEFAULT_CAPTCHA_ATTEMPTS = 3;
+/** Default time one solve may take: token services take 10 to 60 seconds. */
+export declare const DEFAULT_CAPTCHA_TIMEOUT_MS = 120000;
+/** One challenge to get past, and everything the loop needs for it. */
+export interface CaptchaAttemptPlan {
+    recipeId: string;
+    page: Page;
+    challenge: CaptchaChallenge;
+    solver: CaptchaSolver;
+    /** Where challenges are, to re-detect and to confirm one is gone. */
+    selector: string;
+    verify?: CaptchaCheck;
+    attempts: number;
+    timeoutMs: number;
+    budget: CaptchaBudget;
+    events: EventBus;
+    lease?: AccessLease;
+}
+/**
+ * Gets past one challenge: asks the solver, then checks the page (a solver's
+ * `solved` is a claim; the challenge must be gone and/or the `verify` element
+ * must appear), and tries again with what the page shows next, up to
+ * `attempts`. Each try spends one solve of the run's budget.
+ *
+ * A failed try re-detects the challenge (a widget re-renders after a wrong
+ * answer). When it is gone without the page confirming, the page is reloaded
+ * for a fresh one; when a reload shows none, there is nothing left to solve.
+ *
+ * @param plan - The challenge, the solver and the limits.
+ * @throws CaptchaError when every try failed, or the budget is spent.
+ */
+export declare function resolveCaptcha(plan: CaptchaAttemptPlan): Promise<void>;
+//# sourceMappingURL=resolve-captcha.use-case.d.ts.map

package/dist/src/crawl-events/crawl-event.contract.d.ts

@@ -48,6 +48,54 @@ export type CrawlEvent = (Base & {
     type: 'access:rotate';
     attempt: number;
     reason: string;
+}) | 
+/** A request failed in passing (a dropped connection, a 503, a 429) and is sent again after `delayMs`. */
+(Base & {
+    type: 'request:retry';
+    url: string;
+    attempt: number;
+    reason: string;
+    delayMs: number;
+}) | 
+/** A captcha challenge is on the page. */
+(Base & {
+    type: 'captcha:detected';
+    url: string;
+    kind: string;
+    siteKey?: string;
+}) | 
+/** A solver is trying (`attempt` counts from 1 per challenge). */
+(Base & {
+    type: 'captcha:solve';
+    url: string;
+    kind: string;
+    solver: string;
+    attempt: number;
+}) | 
+/** The page confirmed the solve. */
+(Base & {
+    type: 'captcha:solved';
+    url: string;
+    kind: string;
+    solver: string;
+    attempt: number;
+    durationMs: number;
+}) | 
+/** The solver failed, timed out, or the page still shows the challenge. */
+(Base & {
+    type: 'captcha:failed';
+    url: string;
+    kind: string;
+    solver: string;
+    attempt: number;
+    reason: string;
+}) | 
+/** A challenge was left unsolved: the run spent its `maxSolves`. */
+(Base & {
+    type: 'captcha:budget';
+    url: string;
+    kind: string;
+    max: number;
 }) | (Base & {
     type: 'step:start';
     stepType: string;

package/dist/src/crawl-execution/bootstrap-session.use-case.d.ts

@@ -1,14 +1,20 @@
 import type { AccessLease } from '../access/index.js';
-import type { BrowserClient, BrowserSession, SessionOptions, StorageState } from '../browser-session/index.js';
+import type { BrowserClient, BrowserProfiles, BrowserSession, SessionOptions, StorageState } from '../browser-session/index.js';
+import type { CaptchaGuard } from '../captcha/index.js';
 import type { EventBus } from '../crawl-events/index.js';
 import type { HookRegistry } from '../hooks/index.js';
 import type { InputRecipe } from '../recipe-schema/index.js';
+import type { HostThrottle } from '../step-flow/index.js';
 export interface BootstrapDependencies {
     /** Launches (or returns) the shared browser; only called when a browser is needed. */
     browser: () => Promise<BrowserClient>;
     hooks: HookRegistry;
     events: EventBus;
     storageStateDir?: string;
+    /** The crawler's per-site throttle: the bootstrap's pages count too. */
+    hosts?: HostThrottle;
+    /** The runner's persistent browser profiles, for `session.browserProfile`. */
+    profiles?: BrowserProfiles;
 }
 /**
  * The session options an access lease contributes: its proxy, TLS leniency,
@@ -27,12 +33,29 @@ export declare function accessOptions(lease: AccessLease | undefined, headers: R
  * The bootstrap runs through the same access lease as the crawl that follows,
  * so a login and the requests that use its cookies come from one IP.
  *
+ * With `session.browserProfile`, the bootstrap runs in that profile, and
+ * without a bootstrap the profile's own cookies and storage are the state: an
+ * api recipe picks up a login a browser left in the profile.
+ *
  * @param recipe - The input recipe.
  * @param deps - Browser, hooks, events.
  * @param lease - The recipe run's access; direct when omitted.
+ * @param captcha - Solves the bootstrap's captchas (a login form's).
+ * @param owner - The recipe run, which a browser profile is held by.
  * @returns The state, or `undefined` when the recipe declares none.
  */
-export declare function resolveStorageState(recipe: InputRecipe, deps: BootstrapDependencies, lease?: AccessLease): Promise<StorageState | undefined>;
+export declare function resolveStorageState(recipe: InputRecipe, deps: BootstrapDependencies, lease?: AccessLease, captcha?: CaptchaGuard, owner?: object): Promise<StorageState | undefined>;
+/**
+ * Opens the recipe's `session.browserProfile` with its session options and
+ * the lease's proxy.
+ *
+ * @param recipe - A recipe with `session.browserProfile`.
+ * @param deps - For `profiles`.
+ * @param lease - The access lease.
+ * @param owner - The recipe run.
+ * @returns The session in the profile.
+ */
+export declare function openBrowserProfile(recipe: InputRecipe, deps: Pick<BootstrapDependencies, 'profiles'>, lease: AccessLease | undefined, owner: object): Promise<BrowserSession>;
 /**
  * The storage state saved by an earlier bootstrap (`session.storageStatePath`), if the recipe names one.
  *
@@ -49,7 +72,8 @@ export declare function readSavedState(recipe: InputRecipe, deps: Pick<Bootstrap
  * @param recipe - An input recipe with `session.bootstrap`.
  * @param browserSession - Where the steps run.
  * @param deps - Hooks, events, `storageStateDir`.
+ * @param captcha - Solves the bootstrap's captchas.
  * @returns The kept state.
  */
-export declare function runBootstrap(recipe: InputRecipe, browserSession: BrowserSession, deps: Omit<BootstrapDependencies, 'browser'>): Promise<StorageState>;
+export declare function runBootstrap(recipe: InputRecipe, browserSession: BrowserSession, deps: Omit<BootstrapDependencies, 'browser'>, captcha?: CaptchaGuard): Promise<StorageState>;
 //# sourceMappingURL=bootstrap-session.use-case.d.ts.map

package/dist/src/crawl-execution/crawl-options.config.d.ts

@@ -1,7 +1,10 @@
 import type { AccessConfig, AccessPlugin } from '../access/index.js';
 import type { BrowserSessionConfig } from '../browser-session/index.js';
+import type { CaptchaSolver } from '../captcha/index.js';
 import type { CrawlListener } from '../crawl-events/index.js';
 import type { HookMap } from '../hooks/index.js';
+import type { ThrottleConfig } from '../step-flow/index.js';
+import type { RetryRule } from '../recipe-schema/index.js';
 import type { DedupeScope, RecordSink } from '../record-sink/index.js';
 /** How a crawler is created. Everything is optional. */
 export interface CrawlOptions {
@@ -14,10 +17,21 @@ export interface CrawlOptions {
     onEvent?: CrawlListener;
     /** Default `run`: a key seen once is dropped for the rest of the run. */
     dedupe?: DedupeScope;
+    /**
+     * How many input recipes of a set run at once; default 1, one after
+     * another. Each has its own browser context or HTTP session; the browser,
+     * the sink and the per-site `throttle` are shared.
+     */
+    parallel?: number;
     /** Whether a failed input recipe stops the run; default `continue`. */
     onRecipeError?: 'continue' | 'stop';
     /** Base directory for relative `storageStatePath` and `saveTo` values. */
     storageStateDir?: string;
+    /**
+     * Where `session.browserProfile` profiles live, one directory each. Default:
+     * `.opencraw/profiles` under `storageStateDir` (or the working directory).
+     */
+    profilesDir?: string;
     /**
      * Skip records whose key the sink already has (`sink.has`), reporting them as
      * `skipped`. Needs a sink that can answer, such as `jsonLinesSink(path, { append: true })`.
@@ -31,7 +45,23 @@ export interface CrawlOptions {
      * with `session.access.profile`. Without it every recipe goes direct.
      */
     access?: AccessConfig;
+    /**
+     * How gently each site is crawled, across every recipe this crawler runs:
+     * `delayMs` between request starts and `concurrency` requests in flight,
+     * per site, with `domains` for site-specific rules. Defaults to
+     * `access.throttle`; without either, only each recipe's `limits` apply.
+     */
+    throttle?: ThrottleConfig;
+    /**
+     * How a request that fails in passing (a dropped connection, a timeout, a
+     * 503, a 429) is sent again, for recipes whose `limits.retry` says
+     * nothing: `{ attempts?, backoffMs?, maxDelayMs?, statuses? }`. Default:
+     * three tries, one then two seconds apart, `Retry-After` honoured.
+     */
+    retry?: RetryRule;
     /** Plugins `{ kind: 'plugin', name }` profiles refer to. */
     accessPlugins?: AccessPlugin[];
+    /** Solvers recipes name in `session.captcha.solver` and `captcha` steps. */
+    captchaSolvers?: CaptchaSolver[];
 }
 //# sourceMappingURL=crawl-options.config.d.ts.map

package/dist/src/crawl-execution/crawl-report.model.d.ts

@@ -12,6 +12,12 @@ export interface RecipeReport {
     stepsSkipped: number;
     pages: number;
     durationMs: number;
+    /** Captcha challenges met, solved, and solve attempts that failed; present when any was met. */
+    captchas?: {
+        detected: number;
+        solved: number;
+        failed: number;
+    };
     /** Set when the recipe stopped on a failure. */
     error?: string;
 }

package/dist/src/crawl-execution/create-crawler.use-case.d.ts

@@ -11,9 +11,10 @@ export interface Crawler {
  * Creates a crawler. The browser is launched lazily, on the first recipe or
  * bootstrap that needs it, and shared by every run until `close`.
  *
- * @param options - Hooks, sink, events, browser settings, access, policies.
+ * @param options - Hooks, sink, events, browser settings, access, captcha solvers, policies.
  * @returns The crawler.
  * @throws AccessConfigError when the access config cannot work.
+ * @throws Error when two captcha solvers share a name.
  */
 export declare function createCrawler(options?: CrawlOptions): Crawler;
 //# sourceMappingURL=create-crawler.use-case.d.ts.map

package/dist/src/crawl-execution/rotating-runner.use-case.d.ts

@@ -50,6 +50,15 @@ export declare class RotatingRunner implements StepRunner {
     private note;
     runLeaf(step: Step, scope: ExtractionScope): Promise<void>;
     nextPage(next: PaginateNext, scope: ExtractionScope): Promise<NextPageResult>;
+    /**
+     * A runner for one parallel iteration, forked from whichever runner is
+     * current when it runs a step: after a rotation it forks again from the new
+     * one, since the old context is gone (or going). Blocks are noted and
+     * rotated like the main runner's.
+     *
+     * @returns The iteration's runner.
+     */
+    fork(): Promise<StepRunner>;
     elements(selector: string, scope: ExtractionScope): Promise<LiveElement[]>;
     rotate(error: BlockedError): Promise<boolean>;
     dispose(): Promise<void>;