@opencraw/core 0.1.2 → 0.1.3

package/dist/src/access/access-profile.contract.d.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod';
+import type { ThrottleConfig } from '../step-flow/index.js';
 /**
  * Where a crawl's traffic goes. Profiles live in the runner's access config,
  * never in a recipe: they hold the user's accounts. Every string may use
@@ -84,8 +85,11 @@ export interface AccessConfig {
     $schema?: string;
     profiles: Record<string, AccessProfile>;
     default?: string;
+    /** How gently each site is crawled, across every recipe: `{ delayMs?, concurrency?, domains? }`. */
+    throttle?: ThrottleConfig;
 }
 export declare const accessProfileSchema: z.ZodType<AccessProfile>;
+export declare const throttleConfigSchema: z.ZodType<ThrottleConfig>;
 export declare const accessConfigSchema: z.ZodType<AccessConfig>;
 export {};
 //# sourceMappingURL=access-profile.contract.d.ts.map

package/dist/src/access/index.d.ts

@@ -3,7 +3,7 @@ export { loadAccessConfig } from './access-config.repository.js';
 export { AccessConfigError } from './access-config.error.js';
 export { ACCESS_PRESETS } from './access-preset.store.js';
 export type { AccessPreset } from './access-preset.store.js';
-export { accessConfigSchema, accessProfileSchema, BLOCKABLE_RESOURCES } from './access-profile.contract.js';
+export { accessConfigSchema, accessProfileSchema, throttleConfigSchema, BLOCKABLE_RESOURCES } from './access-profile.contract.js';
 export type { AccessConfig, AccessProfile, ProxyProfile, PoolProfile, CdpProfile, PluginProfile, DirectProfile, ProxySettings, BlockableResource } from './access-profile.contract.js';
 export type { AccessPlugin, AccessLease, LeaseRequest, PluginLeaseRequest } from './access-plugin.contract.js';
 export { newSessionId } from './session-id.algorithm.js';

package/dist/src/api-steps/send-request.use-case.d.ts

@@ -5,8 +5,9 @@ import type { InputRecipe, RequestStep } from '../recipe-schema/index.js';
 import type { RunGate } from '../step-flow/index.js';
 /**
  * Sends a `request` step: renders its templates, waits for the gate's throttle,
- * sends, checks the response against the recipe's block rule, then binds it as
- * the scope's current document (and under the step id).
+ * sends (again, after a pause, while it fails in passing: `limits.retry`),
+ * checks the response against the recipe's block rule, then binds it as the
+ * scope's current document (and under the step id).
  *
  * @param step - The request step.
  * @param scope - The scope to render in and bind into.

package/dist/src/browser-session/browser-profile.store.d.ts

@@ -0,0 +1,52 @@
+import type { BrowserSessionConfig } from './browser-session.config.js';
+import { BrowserSession } from './browser.client.js';
+import type { SessionOptions } from './browser.client.js';
+/** A browser profile name: it becomes a directory, so no separators or dots. */
+export declare const BROWSER_PROFILE_NAME: RegExp;
+/**
+ * Browser profiles that persist between runs: each is a directory of a real
+ * browser's user data (cookies, local storage, IndexedDB, cache, service
+ * workers), so a login, a consent choice or a site's trust in a returning
+ * visitor carries over to the next run. The browser equivalent of a user who
+ * never clears their history.
+ *
+ * A profile directory can be open in one browser at a time. Within this
+ * crawler, a second use waits for the first to close; the same owner (one
+ * recipe run reopening after a rotation) takes it over instead. Another
+ * crawler holding it, in this process or another, is reported, not waited
+ * for: a lock file in the profile names the process, and one left by a
+ * process that died is taken over. (Chromium's own profile lock is not
+ * enough: headless builds do not take it.)
+ */
+export declare class BrowserProfiles {
+    readonly directory: string;
+    private readonly config;
+    private readonly held;
+    private readonly waiting;
+    /**
+     * @param directory - Where the profiles live, one subdirectory each.
+     * @param config - The crawler's browser settings (type, binary, headless, timeouts).
+     */
+    constructor(directory: string, config?: BrowserSessionConfig);
+    private take;
+    private free;
+    private launch;
+    /**
+     * The profile's directory.
+     *
+     * @param name - A profile name.
+     * @returns The absolute path.
+     */
+    pathOf(name: string): string;
+    /**
+     * Opens a profile in its own browser, waiting while another run of this
+     * crawler uses it.
+     *
+     * @param name - The profile.
+     * @param options - Proxy, headers, viewport, cookies to add. `storageState` is ignored: the profile has its own.
+     * @param owner - Who opens it; the same owner reopening closes its previous session first.
+     * @returns The session; closing it frees the profile.
+     */
+    open(name: string, options: SessionOptions, owner: object): Promise<BrowserSession>;
+}
+//# sourceMappingURL=browser-profile.store.d.ts.map

package/dist/src/browser-session/browser.client.d.ts

@@ -39,6 +39,14 @@ export declare class BrowserSession {
     storageState(): Promise<StorageState>;
     close(): Promise<void>;
 }
+/**
+ * What a context gets after it opened: the cookies to add and the resource
+ * types to skip.
+ *
+ * @param context - The context.
+ * @param options - The session options.
+ */
+export declare function applySessionExtras(context: BrowserContext, options: SessionOptions): Promise<void>;
 /** A launched browser; sessions are opened from it and closed independently. */
 export declare class BrowserClient {
     private readonly browser;

package/dist/src/browser-session/index.d.ts

@@ -1,4 +1,5 @@
 export { BrowserClient, BrowserSession } from './browser.client.js';
+export { BrowserProfiles, BROWSER_PROFILE_NAME } from './browser-profile.store.js';
 export type { StorageState, SessionOptions } from './browser.client.js';
 export { DEFAULT_BROWSER_CONFIG } from './browser-session.config.js';
 export type { BrowserSessionConfig } from './browser-session.config.js';

package/dist/src/crawl-events/crawl-event.contract.d.ts

@@ -49,6 +49,14 @@ export type CrawlEvent = (Base & {
     attempt: number;
     reason: string;
 }) | 
+/** A request failed in passing (a dropped connection, a 503, a 429) and is sent again after `delayMs`. */
+(Base & {
+    type: 'request:retry';
+    url: string;
+    attempt: number;
+    reason: string;
+    delayMs: number;
+}) | 
 /** A captcha challenge is on the page. */
 (Base & {
     type: 'captcha:detected';

package/dist/src/crawl-execution/bootstrap-session.use-case.d.ts

@@ -1,15 +1,20 @@
 import type { AccessLease } from '../access/index.js';
-import type { BrowserClient, BrowserSession, SessionOptions, StorageState } from '../browser-session/index.js';
+import type { BrowserClient, BrowserProfiles, BrowserSession, SessionOptions, StorageState } from '../browser-session/index.js';
 import type { CaptchaGuard } from '../captcha/index.js';
 import type { EventBus } from '../crawl-events/index.js';
 import type { HookRegistry } from '../hooks/index.js';
 import type { InputRecipe } from '../recipe-schema/index.js';
+import type { HostThrottle } from '../step-flow/index.js';
 export interface BootstrapDependencies {
     /** Launches (or returns) the shared browser; only called when a browser is needed. */
     browser: () => Promise<BrowserClient>;
     hooks: HookRegistry;
     events: EventBus;
     storageStateDir?: string;
+    /** The crawler's per-site throttle: the bootstrap's pages count too. */
+    hosts?: HostThrottle;
+    /** The runner's persistent browser profiles, for `session.browserProfile`. */
+    profiles?: BrowserProfiles;
 }
 /**
  * The session options an access lease contributes: its proxy, TLS leniency,
@@ -28,13 +33,29 @@ export declare function accessOptions(lease: AccessLease | undefined, headers: R
  * The bootstrap runs through the same access lease as the crawl that follows,
  * so a login and the requests that use its cookies come from one IP.
  *
+ * With `session.browserProfile`, the bootstrap runs in that profile, and
+ * without a bootstrap the profile's own cookies and storage are the state: an
+ * api recipe picks up a login a browser left in the profile.
+ *
  * @param recipe - The input recipe.
  * @param deps - Browser, hooks, events.
  * @param lease - The recipe run's access; direct when omitted.
  * @param captcha - Solves the bootstrap's captchas (a login form's).
+ * @param owner - The recipe run, which a browser profile is held by.
  * @returns The state, or `undefined` when the recipe declares none.
  */
-export declare function resolveStorageState(recipe: InputRecipe, deps: BootstrapDependencies, lease?: AccessLease, captcha?: CaptchaGuard): Promise<StorageState | undefined>;
+export declare function resolveStorageState(recipe: InputRecipe, deps: BootstrapDependencies, lease?: AccessLease, captcha?: CaptchaGuard, owner?: object): Promise<StorageState | undefined>;
+/**
+ * Opens the recipe's `session.browserProfile` with its session options and
+ * the lease's proxy.
+ *
+ * @param recipe - A recipe with `session.browserProfile`.
+ * @param deps - For `profiles`.
+ * @param lease - The access lease.
+ * @param owner - The recipe run.
+ * @returns The session in the profile.
+ */
+export declare function openBrowserProfile(recipe: InputRecipe, deps: Pick<BootstrapDependencies, 'profiles'>, lease: AccessLease | undefined, owner: object): Promise<BrowserSession>;
 /**
  * The storage state saved by an earlier bootstrap (`session.storageStatePath`), if the recipe names one.
  *

package/dist/src/crawl-execution/crawl-options.config.d.ts

@@ -3,6 +3,8 @@ import type { BrowserSessionConfig } from '../browser-session/index.js';
 import type { CaptchaSolver } from '../captcha/index.js';
 import type { CrawlListener } from '../crawl-events/index.js';
 import type { HookMap } from '../hooks/index.js';
+import type { ThrottleConfig } from '../step-flow/index.js';
+import type { RetryRule } from '../recipe-schema/index.js';
 import type { DedupeScope, RecordSink } from '../record-sink/index.js';
 /** How a crawler is created. Everything is optional. */
 export interface CrawlOptions {
@@ -15,10 +17,21 @@ export interface CrawlOptions {
     onEvent?: CrawlListener;
     /** Default `run`: a key seen once is dropped for the rest of the run. */
     dedupe?: DedupeScope;
+    /**
+     * How many input recipes of a set run at once; default 1, one after
+     * another. Each has its own browser context or HTTP session; the browser,
+     * the sink and the per-site `throttle` are shared.
+     */
+    parallel?: number;
     /** Whether a failed input recipe stops the run; default `continue`. */
     onRecipeError?: 'continue' | 'stop';
     /** Base directory for relative `storageStatePath` and `saveTo` values. */
     storageStateDir?: string;
+    /**
+     * Where `session.browserProfile` profiles live, one directory each. Default:
+     * `.opencraw/profiles` under `storageStateDir` (or the working directory).
+     */
+    profilesDir?: string;
     /**
      * Skip records whose key the sink already has (`sink.has`), reporting them as
      * `skipped`. Needs a sink that can answer, such as `jsonLinesSink(path, { append: true })`.
@@ -32,6 +45,20 @@ export interface CrawlOptions {
      * with `session.access.profile`. Without it every recipe goes direct.
      */
     access?: AccessConfig;
+    /**
+     * How gently each site is crawled, across every recipe this crawler runs:
+     * `delayMs` between request starts and `concurrency` requests in flight,
+     * per site, with `domains` for site-specific rules. Defaults to
+     * `access.throttle`; without either, only each recipe's `limits` apply.
+     */
+    throttle?: ThrottleConfig;
+    /**
+     * How a request that fails in passing (a dropped connection, a timeout, a
+     * 503, a 429) is sent again, for recipes whose `limits.retry` says
+     * nothing: `{ attempts?, backoffMs?, maxDelayMs?, statuses? }`. Default:
+     * three tries, one then two seconds apart, `Retry-After` honoured.
+     */
+    retry?: RetryRule;
     /** Plugins `{ kind: 'plugin', name }` profiles refer to. */
     accessPlugins?: AccessPlugin[];
     /** Solvers recipes name in `session.captcha.solver` and `captcha` steps. */

package/dist/src/crawl-execution/rotating-runner.use-case.d.ts

@@ -50,6 +50,15 @@ export declare class RotatingRunner implements StepRunner {
     private note;
     runLeaf(step: Step, scope: ExtractionScope): Promise<void>;
     nextPage(next: PaginateNext, scope: ExtractionScope): Promise<NextPageResult>;
+    /**
+     * A runner for one parallel iteration, forked from whichever runner is
+     * current when it runs a step: after a rotation it forks again from the new
+     * one, since the old context is gone (or going). Blocks are noted and
+     * rotated like the main runner's.
+     *
+     * @returns The iteration's runner.
+     */
+    fork(): Promise<StepRunner>;
     elements(selector: string, scope: ExtractionScope): Promise<LiveElement[]>;
     rotate(error: BlockedError): Promise<boolean>;
     dispose(): Promise<void>;

package/dist/src/crawl-execution/run-crawl.use-case.d.ts

@@ -2,12 +2,16 @@ import type { RecipeSet } from '../recipe-loading/index.js';
 import type { CrawlReport } from './crawl-report.model.js';
 import type { RecipeRunDependencies } from './run-input-recipe.use-case.js';
 /**
- * Runs every input recipe of a set, one after another, into one sink.
+ * Runs every input recipe of a set into one sink, `parallel` at a time
+ * (default one after another). Reports come back in the set's order whatever
+ * order the recipes finish in. Under `onRecipeError: 'stop'`, a failed recipe
+ * stops the ones not started yet; those already running finish.
  *
  * @param set - The bound recipes.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @param onRecipeError - Whether a failed recipe stops the run.
+ * @param parallel - How many input recipes run at once.
  * @returns The report.
  */
-export declare function runCrawl(set: RecipeSet, deps: RecipeRunDependencies, onRecipeError: 'continue' | 'stop'): Promise<CrawlReport>;
+export declare function runCrawl(set: RecipeSet, deps: RecipeRunDependencies, onRecipeError: 'continue' | 'stop', parallel?: number): Promise<CrawlReport>;
 //# sourceMappingURL=run-crawl.use-case.d.ts.map

package/dist/src/crawl-execution/run-input-recipe.use-case.d.ts

@@ -1,10 +1,12 @@
 import type { AccessBroker } from '../access/index.js';
 import { BrowserClient } from '../browser-session/index.js';
+import type { BrowserProfiles } from '../browser-session/index.js';
 import { CaptchaSolverRegistry } from '../captcha/index.js';
 import type { EventBus } from '../crawl-events/index.js';
 import type { HookRegistry } from '../hooks/index.js';
-import type { InputRecipe, OutputRecipe } from '../recipe-schema/index.js';
+import type { InputRecipe, OutputRecipe, RetryRule } from '../recipe-schema/index.js';
 import type { DedupePolicy, RecordSink } from '../record-sink/index.js';
+import type { HostThrottle } from '../step-flow/index.js';
 import type { RecipeReport } from './crawl-report.model.js';
 export interface RecipeRunDependencies {
     browser: () => Promise<BrowserClient>;
@@ -23,6 +25,12 @@ export interface RecipeRunDependencies {
     access: AccessBroker;
     /** The solvers recipes name; none when omitted. */
     captchaSolvers?: CaptchaSolverRegistry;
+    /** The crawler's per-site throttle, shared by every recipe. */
+    hosts?: HostThrottle;
+    /** The runner's persistent browser profiles, for `session.browserProfile`. */
+    profiles?: BrowserProfiles;
+    /** The crawler's retry rule, under each recipe's `limits.retry`. */
+    retry?: RetryRule;
 }
 /**
  * Runs one input recipe end to end: session, runner, the step walk, and for
@@ -34,10 +42,10 @@ export interface RecipeRunDependencies {
  * the sink sees one record at a time and `maxRecords` is exact: once reached,
  * every later emit returns `stop` before mapping.
  *
- * @param input - The input recipe.
+ * @param recipe - The input recipe.
  * @param output - The output recipe it feeds.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @returns What happened.
  */
-export declare function runInputRecipe(input: InputRecipe, output: OutputRecipe, deps: RecipeRunDependencies): Promise<RecipeReport>;
+export declare function runInputRecipe(recipe: InputRecipe, output: OutputRecipe, deps: RecipeRunDependencies): Promise<RecipeReport>;
 //# sourceMappingURL=run-input-recipe.use-case.d.ts.map

package/dist/src/index.d.ts

@@ -1,11 +1,11 @@
 export { createCrawler } from './crawl-execution/index.js';
-export { AccessBroker, AccessConfigError, ACCESS_PRESETS, loadAccessConfig, accessConfigSchema, accessConfigJsonSchema } from './access/index.js';
+export { AccessBroker, AccessConfigError, ACCESS_PRESETS, loadAccessConfig, accessConfigSchema, accessConfigJsonSchema, throttleConfigSchema } from './access/index.js';
 export type { AccessConfig, AccessProfile, AccessPlugin, AccessLease, LeaseRequest, PluginLeaseRequest, ProxySettings, AccessPreset } from './access/index.js';
 export type { Crawler, CrawlOptions, CrawlReport, RecipeReport } from './crawl-execution/index.js';
 export { loadRecipeSet, loadRecipes, readRecipeSource, bindRecipeSet, RecipeSet, RecipeBindingError, validateBinding } from './recipe-loading/index.js';
 export type { RecipeSetSource, RecipeSource, RecipeBytes, RecipeDocument, BindingIssue } from './recipe-loading/index.js';
-export { parseInputRecipe, parseOutputRecipe, RecipeValidationError, inputRecipeJsonSchema, outputRecipeJsonSchema, inputRecipeSchema, outputRecipeSchema } from './recipe-schema/index.js';
-export type { InputRecipe, OutputRecipe, FieldSpec, Step, StepType, MappingRule, TransformRule, ErrorPolicy, PaginateNext, SessionSpec, SessionAccess, CaptchaSettings, CaptchaStep, RecipeIssue } from './recipe-schema/index.js';
+export { parseInputRecipe, parseOutputRecipe, RecipeValidationError, inputRecipeJsonSchema, outputRecipeJsonSchema, inputRecipeSchema, outputRecipeSchema, retryRuleSchema } from './recipe-schema/index.js';
+export type { InputRecipe, OutputRecipe, FieldSpec, Step, StepType, MappingRule, TransformRule, ErrorPolicy, PaginateNext, SessionSpec, SessionAccess, CaptchaSettings, CaptchaStep, RetryRule, RecipeIssue } from './recipe-schema/index.js';
 export type { Hook, HookMap, HookContext } from './hooks/index.js';
 export { UnknownHookError } from './hooks/index.js';
 export type { OutputRecord } from './output-mapping/index.js';
@@ -27,7 +27,8 @@ export { readYaml } from './yaml-document/index.js';
 export { findDeckTables, deckText, isDeckDocument } from './deck-document/index.js';
 export type { DeckDocument, DeckSlide, DeckShape, DeckChart, DeckTable, DeckTableQuery } from './deck-document/index.js';
 export type { WorkbookDocument, WorkbookCell, Sheet, CsvFormat, GridTable, GridTableQuery } from './workbook-document/index.js';
-export { StepFailure } from './step-flow/index.js';
+export { StepFailure, HostThrottle, DEFAULT_RETRY_RULE } from './step-flow/index.js';
+export type { ThrottleConfig, HostRule } from './step-flow/index.js';
 export { TransformError } from './transformation/index.js';
 export { CaptchaError, DEFAULT_CAPTCHA_SELECTOR, detectChallenge } from './captcha/index.js';
 export type { CaptchaSolver, CaptchaChallenge, CaptchaContext, CaptchaOutcome, CaptchaKind, CaptchaLog } from './captcha/index.js';

package/dist/src/recipe-schema/index.d.ts

@@ -5,8 +5,8 @@ export { stepSchema, errorPolicySchema, paginateNextSchema } from './step.contra
 export type { Step, StepType, StepBaseFields, TargetFields, ErrorPolicy, PaginateNext, TakeKind, GotoStep, ClickStep, FillStep, PressStep, SelectStep, ScrollStep, WaitStep, EvaluateStep, ScreenshotStep, RequestStep, ExtractStep, SetStep, CollectStep, ForEachStep, IfStep, PaginateStep, EmitStep, HookStep, CaptchaStep, CaptchaCheck, } from './step.contract.js';
 export { transformRuleSchema, mappingRuleSchema } from './transform-rule.contract.js';
 export type { TransformRule, TransformOp, MappingRule, FromRule, EachRule } from './transform-rule.contract.js';
-export { inputRecipeSchema, sessionSpecSchema, startPointSchema } from './input-recipe.contract.js';
-export type { InputRecipe, SessionSpec, SessionBootstrap, SessionAccess, BlockRule, BlockRotation, CaptchaSettings, StartPoint, CrawlLimits, RecipeCookie } from './input-recipe.contract.js';
+export { inputRecipeSchema, sessionSpecSchema, startPointSchema, retryRuleSchema } from './input-recipe.contract.js';
+export type { InputRecipe, SessionSpec, SessionBootstrap, SessionAccess, BlockRule, BlockRotation, CaptchaSettings, RetryRule, StartPoint, CrawlLimits, RecipeCookie } from './input-recipe.contract.js';
 export { parseInputRecipe, parseOutputRecipe, recipeKindOf } from './recipe.validator.js';
 export { RecipeValidationError } from './recipe-validation.error.js';
 export type { RecipeIssue } from './recipe-validation.error.js';

package/dist/src/recipe-schema/input-recipe.contract.d.ts

@@ -99,14 +99,36 @@ export interface SessionSpec {
     blockedWhen?: BlockRule;
     onBlock?: BlockRotation;
     captcha?: CaptchaSettings;
+    /**
+     * A browser profile of the runner that persists between runs (cookies,
+     * storage, cache): web recipes and bootstraps run in it. A name; the runner
+     * decides where profiles live.
+     */
+    browserProfile?: string;
+}
+/**
+ * How a request that fails in passing (a dropped connection, a timeout, a
+ * 503, a 429) is sent again. On by default: three tries in all.
+ */
+export interface RetryRule {
+    /** Tries per request, the first included; `1` turns retrying off. Default 3. */
+    attempts?: number;
+    /** The first pause; it doubles on every retry. Default 1000. */
+    backoffMs?: number;
+    /** The longest pause, `Retry-After` included; a server asking for longer is not retried. Default 30000. */
+    maxDelayMs?: number;
+    /** The statuses retried. Default `[408, 425, 429, 500, 502, 503, 504]`. */
+    statuses?: number[];
 }
 export interface CrawlLimits {
     maxRecords?: number;
     /** Minimum interval between two request starts across the recipe, whatever runs in parallel. */
     delayMs?: number;
     timeoutMs?: number;
-    /** How many `forEach` iterations may run at once (api mode; a web recipe drives one page). Default 1. */
+    /** How many `forEach` iterations over a list may run at once: requests in api mode, tabs in web mode. Default 1. */
     concurrency?: number;
+    /** How requests that fail in passing are sent again; the crawler's `retry`, else three tries, when omitted. */
+    retry?: RetryRule;
 }
 /** Where to start, how to navigate, what to extract, and how it maps to one output recipe. */
 export interface InputRecipe {
@@ -129,5 +151,6 @@ export interface InputRecipe {
 }
 export declare const startPointSchema: z.ZodType<StartPoint>;
 export declare const sessionSpecSchema: z.ZodType<SessionSpec>;
+export declare const retryRuleSchema: z.ZodType<RetryRule>;
 export declare const inputRecipeSchema: z.ZodType<InputRecipe>;
 //# sourceMappingURL=input-recipe.contract.d.ts.map

package/dist/src/record-sink/dedupe.policy.d.ts

@@ -1,17 +1,29 @@
 import type { OutputRecord } from '../output-mapping/index.js';
 /** How far de-duplication reaches: the whole run, one input recipe, or not at all. */
 export type DedupeScope = 'run' | 'recipe' | 'off';
-/** Drops records whose key was already seen. First record wins; keyless records always pass. */
+/** One input recipe's view of de-duplication: whether a record repeats a key already seen. */
+export interface RecipeDedupe {
+    /**
+     * @param record - A validated record.
+     * @returns `true` when the record repeats an earlier key and must be dropped.
+     */
+    isDuplicate: (record: OutputRecord) => boolean;
+}
+/**
+ * Drops records whose key was already seen. First record wins; keyless
+ * records always pass. Recipes running in parallel each get their own view:
+ * under `recipe` scope they never see each other's keys, under `run` scope
+ * they share them (and whichever emits a key first keeps it).
+ */
 export declare class DedupePolicy {
     readonly scope: DedupeScope;
-    private seen;
+    private readonly shared;
     constructor(scope?: DedupeScope);
-    /** Called when an input recipe starts; forgets keys under `recipe` scope. */
-    startRecipe(): void;
     /**
-     * @param record - A validated record.
-     * @returns `true` when the record repeats an earlier key and must be dropped.
+     * The de-duplication one input recipe run uses.
+     *
+     * @returns Its view: keys shared with the run, its own, or none checked.
      */
-    isDuplicate(record: OutputRecord): boolean;
+    forRecipe(): RecipeDedupe;
 }
 //# sourceMappingURL=dedupe.policy.d.ts.map

package/dist/src/record-sink/index.d.ts

@@ -4,5 +4,6 @@ export type { MemorySink } from './memory-sink.repository.js';
 export { jsonLinesSink } from './json-lines-sink.repository.js';
 export type { JsonLinesSinkOptions } from './json-lines-sink.repository.js';
 export { DedupePolicy } from './dedupe.policy.js';
+export type { RecipeDedupe } from './dedupe.policy.js';
 export type { DedupeScope } from './dedupe.policy.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/step-flow/for-each.use-case.d.ts

@@ -6,8 +6,10 @@ import type { EmitOutcome, StepWalk } from './run-steps.use-case.js';
  * matching `selector`, each in a fresh child scope with the item bound under
  * `as`; emits a record per iteration when asked.
  *
- * With a concurrent gate, iterations run as permits allow and records come
- * out in completion order; without one, in list order.
+ * With a concurrent gate, iterations of a list run as permits allow and
+ * records come out in completion order; without one, in list order. In web
+ * mode each parallel iteration runs in a tab of its own (`runner.fork`); a
+ * loop over live elements stays sequential, since its elements live on one page.
  *
  * @param step - The forEach step.
  * @param scope - The scope the list lives in.

package/dist/src/step-flow/host-throttle.policy.d.ts

@@ -0,0 +1,49 @@
+/** How gently one site is crawled. */
+export interface HostRule {
+    /** Minimum time between two request starts to the site, whatever recipe sends them. */
+    delayMs?: number;
+    /** Requests to the site in flight at once. */
+    concurrency?: number;
+}
+/**
+ * The crawler's politeness towards each site, across every recipe and run it
+ * executes: a default rule for any host, and rules by domain (`example.com`
+ * also covers `www.example.com`; the longest match wins).
+ */
+export interface ThrottleConfig extends HostRule {
+    domains?: Record<string, HostRule>;
+}
+/**
+ * Spaces and bounds requests per site, shared by every recipe of a crawler,
+ * so two recipes (or two parallel iterations) that hit one site add up to one
+ * polite client rather than two. A recipe's own `limits.delayMs` still applies
+ * on top, per recipe.
+ *
+ * Like a single-lane bridge with a traffic light: whoever arrives waits for
+ * the car ahead to be far enough, and for a free lane.
+ */
+export declare class HostThrottle {
+    private readonly config;
+    private readonly buckets;
+    private readonly domains;
+    constructor(config?: ThrottleConfig);
+    private bucketFor;
+    /** Whether any rule can hold a request back. */
+    get active(): boolean;
+    /**
+     * Waits until a request to `url` may start, then holds one of its site's
+     * lanes until the returned release is called.
+     *
+     * @param url - Where the request goes; anything but `http(s):` passes at once.
+     * @returns The release: call it once, when the response arrived or the request failed.
+     */
+    slot(url: string): Promise<() => void>;
+    /**
+     * Holds every request to the site of `url` back until `untilMs` (a `Retry-After`).
+     *
+     * @param url - A URL of the site.
+     * @param untilMs - An epoch time.
+     */
+    pause(url: string, untilMs: number): void;
+}
+//# sourceMappingURL=host-throttle.policy.d.ts.map

package/dist/src/step-flow/index.d.ts

@@ -1,9 +1,14 @@
 export { runSteps } from './run-steps.use-case.js';
 export type { StepWalkOptions, EmitOutcome } from './run-steps.use-case.js';
 export type { StepRunner, NextPageResult } from './step-runner.contract.js';
+export { disposeQuietly } from './step-runner.contract.js';
 export { StepFailure, NoMatchError } from './step-failure.error.js';
 export { resolveErrorPolicy, backoffFor, sleep } from './retry.policy.js';
 export { RunGate } from './run-gate.policy.js';
+export { HostThrottle } from './host-throttle.policy.js';
+export { withTransportRetry, resolveRetryRule, transientError, retryDelay, DEFAULT_RETRY_RULE, RETRY_STATUSES } from './transport-retry.policy.js';
+export type { ResolvedRetryRule, Transient, TransportAttempt, RetryContext } from './transport-retry.policy.js';
+export type { HostRule, ThrottleConfig } from './host-throttle.policy.js';
 export { BlockedError } from './blocked.error.js';
 export { detectBlock, DEFAULT_BLOCK_RULE } from './block-rule.policy.js';
 export type { ObservedResponse } from './block-rule.policy.js';

package/dist/src/step-flow/run-gate.policy.d.ts

@@ -1,3 +1,4 @@
+import type { HostThrottle } from './host-throttle.policy.js';
 /**
  * What bounds a recipe run: how many `forEach` iterations may be in flight and
  * how close together requests may start. One gate per recipe run, shared by
@@ -10,6 +11,7 @@
 export declare class RunGate {
     readonly permits: number;
     readonly minIntervalMs: number;
+    readonly hosts?: HostThrottle | undefined;
     private readonly shared?;
     private inFlight;
     private readonly waiting;
@@ -17,9 +19,10 @@ export declare class RunGate {
     /**
      * @param permits - Iterations allowed in flight; 1 is sequential.
      * @param minIntervalMs - Minimum time between two request starts across the run.
+     * @param hosts - The crawler's per-site throttle, shared with every other recipe.
      * @param shared - The throttle state to share (internal: `nested` gates keep their parent's).
      */
-    constructor(permits: number, minIntervalMs: number, shared?: RunGate | undefined);
+    constructor(permits: number, minIntervalMs: number, hosts?: HostThrottle | undefined, shared?: RunGate | undefined);
     /** Whether this gate lets more than one iteration run at once. */
     get concurrent(): boolean;
     /**
@@ -33,6 +36,14 @@ export declare class RunGate {
      * whichever loop started it. Returns at once when the interval has passed.
      */
     throttle(): Promise<void>;
+    /**
+     * Waits until a request to `url` may start: the recipe's interval, then its
+     * site's turn in the crawler's per-site throttle.
+     *
+     * @param url - Where the request goes.
+     * @returns The release of the site's lane: call it once the response arrived or the request failed.
+     */
+    request(url: string): Promise<() => void>;
     /** The gate for a body running inside an iteration that holds a permit: sequential, same throttle. */
     nested(): RunGate;
 }

package/dist/src/step-flow/step-runner.contract.d.ts

@@ -1,6 +1,13 @@
 import type { ExtractionScope, LiveElement } from '../extraction-scope/index.js';
 import type { PaginateNext, Step } from '../recipe-schema/index.js';
 import type { BlockedError } from './blocked.error.js';
+/**
+ * Disposes a runner, ignoring a failure: a tab whose browser already went
+ * away (a rotation, a crash) has nothing left to close.
+ *
+ * @param runner - The runner.
+ */
+export declare function disposeQuietly(runner: StepRunner): Promise<void>;
 /** What `paginate` learns from the runner after a page body ran. */
 export type NextPageResult = 
 /** The next page is at this URL (the runner already navigated in web mode). */
@@ -34,6 +41,12 @@ export interface StepRunner {
      * `false` means it cannot, and the block fails the step like any error.
      */
     rotate?: (error: BlockedError) => Promise<boolean>;
+    /**
+     * A runner of its own for one parallel `forEach` iteration: a new tab in the
+     * same browser context (same cookies, its own page), disposed when the
+     * iteration ends. Web mode; an api runner is shared as it is.
+     */
+    fork?: () => Promise<StepRunner>;
     dispose: () => Promise<void>;
 }
 //# sourceMappingURL=step-runner.contract.d.ts.map