@opencraw/core 0.1.2 → 0.1.4

package/dist/src/crawl-execution/run-input-recipe.use-case.d.ts

@@ -1,10 +1,12 @@
 import type { AccessBroker } from '../access/index.js';
 import { BrowserClient } from '../browser-session/index.js';
+import type { BrowserProfiles } from '../browser-session/index.js';
 import { CaptchaSolverRegistry } from '../captcha/index.js';
 import type { EventBus } from '../crawl-events/index.js';
 import type { HookRegistry } from '../hooks/index.js';
-import type { InputRecipe, OutputRecipe } from '../recipe-schema/index.js';
+import type { InputRecipe, OutputRecipe, RetryRule } from '../recipe-schema/index.js';
 import type { DedupePolicy, RecordSink } from '../record-sink/index.js';
+import type { HostThrottle } from '../step-flow/index.js';
 import type { RecipeReport } from './crawl-report.model.js';
 export interface RecipeRunDependencies {
     browser: () => Promise<BrowserClient>;
@@ -23,6 +25,12 @@ export interface RecipeRunDependencies {
     access: AccessBroker;
     /** The solvers recipes name; none when omitted. */
     captchaSolvers?: CaptchaSolverRegistry;
+    /** The crawler's per-site throttle, shared by every recipe. */
+    hosts?: HostThrottle;
+    /** The runner's persistent browser profiles, for `session.browserProfile`. */
+    profiles?: BrowserProfiles;
+    /** The crawler's retry rule, under each recipe's `limits.retry`. */
+    retry?: RetryRule;
 }
 /**
  * Runs one input recipe end to end: session, runner, the step walk, and for
@@ -34,10 +42,10 @@ export interface RecipeRunDependencies {
  * the sink sees one record at a time and `maxRecords` is exact: once reached,
  * every later emit returns `stop` before mapping.
  *
- * @param input - The input recipe.
+ * @param recipe - The input recipe.
  * @param output - The output recipe it feeds.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @returns What happened.
  */
-export declare function runInputRecipe(input: InputRecipe, output: OutputRecipe, deps: RecipeRunDependencies): Promise<RecipeReport>;
+export declare function runInputRecipe(recipe: InputRecipe, output: OutputRecipe, deps: RecipeRunDependencies): Promise<RecipeReport>;
 //# sourceMappingURL=run-input-recipe.use-case.d.ts.map

package/dist/src/docx-document/index.d.ts

@@ -0,0 +1,2 @@
+export { readDocxHtml } from './read-docx-html.client.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/docx-document/read-docx-html.client.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Reads a `.docx` Word document into HTML, through `@opencraw/office-reader`,
+ * so every selector that reads a page reads a Word document too:
+ *
+ * - headings become `<h1>`…`<h6>` inside `<section data-heading="…">`, as in
+ *   rendered Markdown;
+ * - list items become nested `<ul>` / `<ol>`;
+ * - tables become `<table>`s with their merged cells as `colspan` / `rowspan`,
+ *   so a `table` extract reads them like any HTML table;
+ * - a paragraph keeps its style as `data-style` and its links as `<a href>`;
+ * - headers, footers and notes follow the body, in `<header>`, `<footer>`
+ *   and `<aside data-part="notes">`; the title goes to `<title>`.
+ *
+ * The reader is imported on first use.
+ *
+ * @param bytes - The file.
+ * @param source - Where it came from, for messages.
+ * @returns The HTML document.
+ * @throws Error naming the source, and saying what to do, for a file that is
+ * not a readable Word document (a legacy `.doc`, a password-protected file, an `.odt`…).
+ */
+export declare function readDocxHtml(bytes: Uint8Array, source: string): Promise<string>;
+//# sourceMappingURL=read-docx-html.client.d.ts.map

package/dist/src/extraction-scope/extraction-scope.model.d.ts

@@ -7,11 +7,12 @@
  * Page state (the current URL, page number and the document `extract` reads by
  * default) is scope state too, bound in the innermost scope that navigated.
  */
+import type { XmlDocument } from '../xml-document/index.js';
 import type { DeckDocument } from '../deck-document/index.js';
 import type { PdfDocument } from '../pdf-document/index.js';
 import type { WorkbookDocument } from '../workbook-document/index.js';
 /** A fetched or rendered document a later `extract` can read. */
-export type ScopeDocument = PdfDocument | WorkbookDocument | DeckDocument | {
+export type ScopeDocument = PdfDocument | WorkbookDocument | DeckDocument | XmlDocument | {
     kind: 'json';
     data: unknown;
 } | {

package/dist/src/http-session/http-response.contract.d.ts

@@ -1,6 +1,7 @@
 import type { DeckDocument } from '../deck-document/index.js';
 import type { PdfDocument } from '../pdf-document/index.js';
 import type { WorkbookDocument } from '../workbook-document/index.js';
+import type { XmlDocument } from '../xml-document/index.js';
 import type { BodyKind, HttpMethod } from '../recipe-schema/index.js';
 /** One HTTP request as the api runner sends it, templates already rendered. */
 export interface HttpRequest {
@@ -20,7 +21,7 @@ export interface HttpRequest {
     timeoutMs?: number;
 }
 /** A parsed response body. Structurally the same as a scope document, on purpose. */
-export type HttpBody = PdfDocument | WorkbookDocument | DeckDocument | {
+export type HttpBody = PdfDocument | WorkbookDocument | DeckDocument | XmlDocument | {
     kind: 'json';
     data: unknown;
 } | {

package/dist/src/index.d.ts

@@ -1,11 +1,11 @@
 export { createCrawler } from './crawl-execution/index.js';
-export { AccessBroker, AccessConfigError, ACCESS_PRESETS, loadAccessConfig, accessConfigSchema, accessConfigJsonSchema } from './access/index.js';
+export { AccessBroker, AccessConfigError, ACCESS_PRESETS, loadAccessConfig, accessConfigSchema, accessConfigJsonSchema, throttleConfigSchema } from './access/index.js';
 export type { AccessConfig, AccessProfile, AccessPlugin, AccessLease, LeaseRequest, PluginLeaseRequest, ProxySettings, AccessPreset } from './access/index.js';
 export type { Crawler, CrawlOptions, CrawlReport, RecipeReport } from './crawl-execution/index.js';
 export { loadRecipeSet, loadRecipes, readRecipeSource, bindRecipeSet, RecipeSet, RecipeBindingError, validateBinding } from './recipe-loading/index.js';
 export type { RecipeSetSource, RecipeSource, RecipeBytes, RecipeDocument, BindingIssue } from './recipe-loading/index.js';
-export { parseInputRecipe, parseOutputRecipe, RecipeValidationError, inputRecipeJsonSchema, outputRecipeJsonSchema, inputRecipeSchema, outputRecipeSchema } from './recipe-schema/index.js';
-export type { InputRecipe, OutputRecipe, FieldSpec, Step, StepType, MappingRule, TransformRule, ErrorPolicy, PaginateNext, SessionSpec, SessionAccess, CaptchaSettings, CaptchaStep, RecipeIssue } from './recipe-schema/index.js';
+export { parseInputRecipe, parseOutputRecipe, RecipeValidationError, inputRecipeJsonSchema, outputRecipeJsonSchema, inputRecipeSchema, outputRecipeSchema, retryRuleSchema } from './recipe-schema/index.js';
+export type { InputRecipe, OutputRecipe, FieldSpec, Step, StepType, MappingRule, TransformRule, ErrorPolicy, PaginateNext, SessionSpec, SessionAccess, CaptchaSettings, CaptchaStep, RetryRule, RecipeIssue } from './recipe-schema/index.js';
 export type { Hook, HookMap, HookContext } from './hooks/index.js';
 export { UnknownHookError } from './hooks/index.js';
 export type { OutputRecord } from './output-mapping/index.js';
@@ -24,11 +24,16 @@ export type { PdfDocument, PdfPage, PdfRow, PdfCell, PdfTable, TableQuery, Table
 export { parseCsv, detectDelimiter, csvWorkbook, findGridTables, fillDown, workbookText, isWorkbookDocument, htmlTableSheets } from './workbook-document/index.js';
 export { readMarkdown } from './markdown-document/index.js';
 export { readYaml } from './yaml-document/index.js';
+export { parseXml, selectXpath, takeFromXml, isXmlDocument } from './xml-document/index.js';
+export type { XmlDocument, XpathOptions, XpathValue } from './xml-document/index.js';
 export { findDeckTables, deckText, isDeckDocument } from './deck-document/index.js';
 export type { DeckDocument, DeckSlide, DeckShape, DeckChart, DeckTable, DeckTableQuery } from './deck-document/index.js';
 export type { WorkbookDocument, WorkbookCell, Sheet, CsvFormat, GridTable, GridTableQuery } from './workbook-document/index.js';
-export { StepFailure } from './step-flow/index.js';
+export { StepFailure, HostThrottle, DEFAULT_RETRY_RULE } from './step-flow/index.js';
+export type { ThrottleConfig, HostRule } from './step-flow/index.js';
 export { TransformError } from './transformation/index.js';
+export { diffRecords, diffOptionsFor, readableKey, readRecordsFile } from './record-diff/index.js';
+export type { RecordDiff, RecordChange, FieldChange, DiffOptions, StoredRecord } from './record-diff/index.js';
 export { CaptchaError, DEFAULT_CAPTCHA_SELECTOR, detectChallenge } from './captcha/index.js';
 export type { CaptchaSolver, CaptchaChallenge, CaptchaContext, CaptchaOutcome, CaptchaKind, CaptchaLog } from './captcha/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/markdown-document/index.d.ts

@@ -1,3 +1,3 @@
-export { readMarkdown } from './read-markdown.client.js';
+export { readMarkdown, sectioned } from './read-markdown.client.js';
 export type { MarkdownRead } from './read-markdown.client.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/markdown-document/read-markdown.client.d.ts

@@ -26,4 +26,14 @@ export interface MarkdownRead {
  * @throws Error naming the source when the front matter is not YAML.
  */
 export declare function readMarkdown(text: string, source: string): Promise<MarkdownRead>;
+/**
+ * Wraps each heading and what follows it, up to the next heading of the same
+ * or a higher level, in `<section data-heading="…" data-level="…">`, sections
+ * nesting; headings get slug ids. Rendered Markdown and Word documents both go
+ * through it, so one selector finds "the table under *Prezzi*" in either.
+ *
+ * @param html - Top-level HTML: headings among paragraphs, lists, tables.
+ * @returns The same content, sectioned.
+ */
+export declare function sectioned(html: string): string;
 //# sourceMappingURL=read-markdown.client.d.ts.map

package/dist/src/recipe-schema/index.d.ts

@@ -5,8 +5,8 @@ export { stepSchema, errorPolicySchema, paginateNextSchema } from './step.contra
 export type { Step, StepType, StepBaseFields, TargetFields, ErrorPolicy, PaginateNext, TakeKind, GotoStep, ClickStep, FillStep, PressStep, SelectStep, ScrollStep, WaitStep, EvaluateStep, ScreenshotStep, RequestStep, ExtractStep, SetStep, CollectStep, ForEachStep, IfStep, PaginateStep, EmitStep, HookStep, CaptchaStep, CaptchaCheck, } from './step.contract.js';
 export { transformRuleSchema, mappingRuleSchema } from './transform-rule.contract.js';
 export type { TransformRule, TransformOp, MappingRule, FromRule, EachRule } from './transform-rule.contract.js';
-export { inputRecipeSchema, sessionSpecSchema, startPointSchema } from './input-recipe.contract.js';
-export type { InputRecipe, SessionSpec, SessionBootstrap, SessionAccess, BlockRule, BlockRotation, CaptchaSettings, StartPoint, CrawlLimits, RecipeCookie } from './input-recipe.contract.js';
+export { inputRecipeSchema, sessionSpecSchema, startPointSchema, retryRuleSchema } from './input-recipe.contract.js';
+export type { InputRecipe, SessionSpec, SessionBootstrap, SessionAccess, BlockRule, BlockRotation, CaptchaSettings, RetryRule, StartPoint, CrawlLimits, RecipeCookie } from './input-recipe.contract.js';
 export { parseInputRecipe, parseOutputRecipe, recipeKindOf } from './recipe.validator.js';
 export { RecipeValidationError } from './recipe-validation.error.js';
 export type { RecipeIssue } from './recipe-validation.error.js';

package/dist/src/recipe-schema/input-recipe.contract.d.ts

@@ -99,14 +99,36 @@ export interface SessionSpec {
     blockedWhen?: BlockRule;
     onBlock?: BlockRotation;
     captcha?: CaptchaSettings;
+    /**
+     * A browser profile of the runner that persists between runs (cookies,
+     * storage, cache): web recipes and bootstraps run in it. A name; the runner
+     * decides where profiles live.
+     */
+    browserProfile?: string;
+}
+/**
+ * How a request that fails in passing (a dropped connection, a timeout, a
+ * 503, a 429) is sent again. On by default: three tries in all.
+ */
+export interface RetryRule {
+    /** Tries per request, the first included; `1` turns retrying off. Default 3. */
+    attempts?: number;
+    /** The first pause; it doubles on every retry. Default 1000. */
+    backoffMs?: number;
+    /** The longest pause, `Retry-After` included; a server asking for longer is not retried. Default 30000. */
+    maxDelayMs?: number;
+    /** The statuses retried. Default `[408, 425, 429, 500, 502, 503, 504]`. */
+    statuses?: number[];
 }
 export interface CrawlLimits {
     maxRecords?: number;
     /** Minimum interval between two request starts across the recipe, whatever runs in parallel. */
     delayMs?: number;
     timeoutMs?: number;
-    /** How many `forEach` iterations may run at once (api mode; a web recipe drives one page). Default 1. */
+    /** How many `forEach` iterations over a list may run at once: requests in api mode, tabs in web mode. Default 1. */
     concurrency?: number;
+    /** How requests that fail in passing are sent again; the crawler's `retry`, else three tries, when omitted. */
+    retry?: RetryRule;
 }
 /** Where to start, how to navigate, what to extract, and how it maps to one output recipe. */
 export interface InputRecipe {
@@ -129,5 +151,6 @@ export interface InputRecipe {
 }
 export declare const startPointSchema: z.ZodType<StartPoint>;
 export declare const sessionSpecSchema: z.ZodType<SessionSpec>;
+export declare const retryRuleSchema: z.ZodType<RetryRule>;
 export declare const inputRecipeSchema: z.ZodType<InputRecipe>;
 //# sourceMappingURL=input-recipe.contract.d.ts.map

package/dist/src/recipe-schema/recipe-kind.enum.d.ts

@@ -4,7 +4,7 @@ export declare const CRAWL_MODES: readonly ["web", "api"];
 export declare const SELECTOR_KINDS: readonly ["css", "xpath", "jsonpath", "regex", "table"];
 /** `take` also accepts `attr:<name>`, which is validated by pattern rather than listed. */
 export declare const TAKE_KINDS: readonly ["text", "html", "value", "json"];
-export declare const BODY_KINDS: readonly ["json", "jsonl", "html", "text", "pdf", "csv", "xlsx", "pptx", "yaml", "markdown"];
+export declare const BODY_KINDS: readonly ["json", "jsonl", "html", "text", "pdf", "csv", "xlsx", "pptx", "yaml", "markdown", "xml", "docx"];
 export declare const YAML_SCALARS: readonly ["typed", "text"];
 /** How a PDF table aligns a row's values against a cell wrapped over several lines. */
 export declare const TABLE_ALIGNS: readonly ["auto", "top", "center", "bottom"];

package/dist/src/recipe-schema/step.contract.d.ts

@@ -119,6 +119,10 @@ export interface ExtractStep extends StepBaseFields {
     slide?: string;
     /** `table` only: read text boxes laid out as a table instead of native tables (deck). */
     shapes?: boolean;
+    /** `xpath` on XML only: prefix to namespace URI (`{ "atom": "http://www.w3.org/2005/Atom" }`). Prefixes the root declares are known already. */
+    namespaces?: Record<string, string>;
+    /** `xpath` on XML only: drop the document's namespaces, so `//entry/title` matches `<entry xmlns="…">`. */
+    ignoreNamespaces?: boolean;
 }
 export interface SetStep extends StepBaseFields {
     type: 'set';

package/dist/src/record-diff/index.d.ts

@@ -0,0 +1,4 @@
+export { diffRecords, diffOptionsFor, readableKey } from './record-diff.algorithm.js';
+export type { RecordDiff, RecordChange, FieldChange, DiffOptions, StoredRecord } from './record-diff.algorithm.js';
+export { readRecordsFile } from './records-file.repository.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/record-diff/record-diff.algorithm.d.ts

@@ -0,0 +1,82 @@
+import type { OutputRecipe } from '../recipe-schema/index.js';
+/** A record as a JSON Lines file holds it: the data, plus the sink's `_source` (and `_key` in append mode). */
+export type StoredRecord = Record<string, unknown>;
+/** One field whose value changed; nested objects are compared field by field (`price.amount`). */
+export interface FieldChange {
+    field: string;
+    before: unknown;
+    after: unknown;
+}
+export type RecordChange = {
+    change: 'added';
+    key: string;
+    after: StoredRecord;
+} | {
+    change: 'removed';
+    key: string;
+    before: StoredRecord;
+} | {
+    change: 'changed';
+    key: string;
+    before: StoredRecord;
+    after: StoredRecord;
+    fields: FieldChange[];
+};
+/** How two runs differ, by record key. */
+export interface RecordDiff {
+    added: number;
+    removed: number;
+    changed: number;
+    unchanged: number;
+    /** Every change, removed first, then changed, then added, each in file order. */
+    changes: RecordChange[];
+    /** Records on either side the key cannot tell apart from an earlier one: only the first counts. */
+    repeated: number;
+    /** How many records each side holds. */
+    counts: {
+        previous: number;
+        current: number;
+    };
+    /** Set when the current run holds far fewer records: the usual sign a site changed and a recipe broke quietly. */
+    shrunk?: {
+        previous: number;
+        current: number;
+    };
+}
+export interface DiffOptions {
+    /** The fields that identify a record. Default: the `_key` each line carries (a sink in append mode writes it). */
+    key?: readonly string[];
+    /** Fields not compared (their dotted paths match nested ones too). `_source` and `_key` never are. */
+    ignore?: readonly string[];
+    /** The share of records a run may lose before `shrunk` is set; default 0.5 (half). */
+    shrink?: number;
+}
+/**
+ * Compares two runs' records by key: what was added, what was removed, and
+ * for the records in both, which fields changed, before and after. Like a
+ * `git diff` of a price list, but row by row instead of line by line, so a
+ * reordered file is not a change.
+ *
+ * @param previous - The earlier run's records.
+ * @param current - The later run's records.
+ * @param options - The key fields, the fields to ignore, the shrink threshold.
+ * @returns The differences.
+ * @throws Error when records carry no key: without `key`, every line needs `_key`.
+ */
+export declare function diffRecords(previous: readonly StoredRecord[], current: readonly StoredRecord[], options?: DiffOptions): RecordDiff;
+/**
+ * The diff options an output recipe implies: its key fields, and its fields
+ * the engine fills differently every run (`generated: now`, `uuid`).
+ *
+ * @param output - The output recipe.
+ * @returns The key and the fields to ignore.
+ */
+export declare function diffOptionsFor(output: OutputRecipe): Required<Pick<DiffOptions, 'key' | 'ignore'>>;
+/**
+ * A key as people read it: the key fields' values joined (`Pandina · 1.0 Hybrid`).
+ *
+ * @param key - A record key (a JSON array of the key values).
+ * @returns The readable form.
+ */
+export declare function readableKey(key: string): string;
+//# sourceMappingURL=record-diff.algorithm.d.ts.map

package/dist/src/record-diff/records-file.repository.d.ts

@@ -0,0 +1,10 @@
+import type { StoredRecord } from './record-diff.algorithm.js';
+/**
+ * The records of a JSON Lines file, as a sink wrote them.
+ *
+ * @param path - The file.
+ * @returns One record per non-empty line.
+ * @throws Error naming the file and the line that is not a JSON object.
+ */
+export declare function readRecordsFile(path: string): Promise<StoredRecord[]>;
+//# sourceMappingURL=records-file.repository.d.ts.map

package/dist/src/record-sink/dedupe.policy.d.ts

@@ -1,17 +1,29 @@
 import type { OutputRecord } from '../output-mapping/index.js';
 /** How far de-duplication reaches: the whole run, one input recipe, or not at all. */
 export type DedupeScope = 'run' | 'recipe' | 'off';
-/** Drops records whose key was already seen. First record wins; keyless records always pass. */
+/** One input recipe's view of de-duplication: whether a record repeats a key already seen. */
+export interface RecipeDedupe {
+    /**
+     * @param record - A validated record.
+     * @returns `true` when the record repeats an earlier key and must be dropped.
+     */
+    isDuplicate: (record: OutputRecord) => boolean;
+}
+/**
+ * Drops records whose key was already seen. First record wins; keyless
+ * records always pass. Recipes running in parallel each get their own view:
+ * under `recipe` scope they never see each other's keys, under `run` scope
+ * they share them (and whichever emits a key first keeps it).
+ */
 export declare class DedupePolicy {
     readonly scope: DedupeScope;
-    private seen;
+    private readonly shared;
     constructor(scope?: DedupeScope);
-    /** Called when an input recipe starts; forgets keys under `recipe` scope. */
-    startRecipe(): void;
     /**
-     * @param record - A validated record.
-     * @returns `true` when the record repeats an earlier key and must be dropped.
+     * The de-duplication one input recipe run uses.
+     *
+     * @returns Its view: keys shared with the run, its own, or none checked.
      */
-    isDuplicate(record: OutputRecord): boolean;
+    forRecipe(): RecipeDedupe;
 }
 //# sourceMappingURL=dedupe.policy.d.ts.map

package/dist/src/record-sink/index.d.ts

@@ -4,5 +4,6 @@ export type { MemorySink } from './memory-sink.repository.js';
 export { jsonLinesSink } from './json-lines-sink.repository.js';
 export type { JsonLinesSinkOptions } from './json-lines-sink.repository.js';
 export { DedupePolicy } from './dedupe.policy.js';
+export type { RecipeDedupe } from './dedupe.policy.js';
 export type { DedupeScope } from './dedupe.policy.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/selection/html-selector.algorithm.d.ts

@@ -12,7 +12,8 @@ export interface HtmlMatch {
  *
  * @param html - The markup (a whole document or a fragment).
  * @param selector - A CSS selector.
+ * @param xml - Read the markup as XML: names keep their case, `take: "html"` gives XML.
  * @returns Every match, in document order.
  */
-export declare function selectHtml(html: string, selector: string): HtmlMatch[];
+export declare function selectHtml(html: string, selector: string, xml?: boolean): HtmlMatch[];
 //# sourceMappingURL=html-selector.algorithm.d.ts.map

package/dist/src/step-flow/for-each.use-case.d.ts

@@ -6,8 +6,10 @@ import type { EmitOutcome, StepWalk } from './run-steps.use-case.js';
  * matching `selector`, each in a fresh child scope with the item bound under
  * `as`; emits a record per iteration when asked.
  *
- * With a concurrent gate, iterations run as permits allow and records come
- * out in completion order; without one, in list order.
+ * With a concurrent gate, iterations of a list run as permits allow and
+ * records come out in completion order; without one, in list order. In web
+ * mode each parallel iteration runs in a tab of its own (`runner.fork`); a
+ * loop over live elements stays sequential, since its elements live on one page.
  *
  * @param step - The forEach step.
  * @param scope - The scope the list lives in.

package/dist/src/step-flow/host-throttle.policy.d.ts

@@ -0,0 +1,49 @@
+/** How gently one site is crawled. */
+export interface HostRule {
+    /** Minimum time between two request starts to the site, whatever recipe sends them. */
+    delayMs?: number;
+    /** Requests to the site in flight at once. */
+    concurrency?: number;
+}
+/**
+ * The crawler's politeness towards each site, across every recipe and run it
+ * executes: a default rule for any host, and rules by domain (`example.com`
+ * also covers `www.example.com`; the longest match wins).
+ */
+export interface ThrottleConfig extends HostRule {
+    domains?: Record<string, HostRule>;
+}
+/**
+ * Spaces and bounds requests per site, shared by every recipe of a crawler,
+ * so two recipes (or two parallel iterations) that hit one site add up to one
+ * polite client rather than two. A recipe's own `limits.delayMs` still applies
+ * on top, per recipe.
+ *
+ * Like a single-lane bridge with a traffic light: whoever arrives waits for
+ * the car ahead to be far enough, and for a free lane.
+ */
+export declare class HostThrottle {
+    private readonly config;
+    private readonly buckets;
+    private readonly domains;
+    constructor(config?: ThrottleConfig);
+    private bucketFor;
+    /** Whether any rule can hold a request back. */
+    get active(): boolean;
+    /**
+     * Waits until a request to `url` may start, then holds one of its site's
+     * lanes until the returned release is called.
+     *
+     * @param url - Where the request goes; anything but `http(s):` passes at once.
+     * @returns The release: call it once, when the response arrived or the request failed.
+     */
+    slot(url: string): Promise<() => void>;
+    /**
+     * Holds every request to the site of `url` back until `untilMs` (a `Retry-After`).
+     *
+     * @param url - A URL of the site.
+     * @param untilMs - An epoch time.
+     */
+    pause(url: string, untilMs: number): void;
+}
+//# sourceMappingURL=host-throttle.policy.d.ts.map

package/dist/src/step-flow/index.d.ts

@@ -1,9 +1,14 @@
 export { runSteps } from './run-steps.use-case.js';
 export type { StepWalkOptions, EmitOutcome } from './run-steps.use-case.js';
 export type { StepRunner, NextPageResult } from './step-runner.contract.js';
+export { disposeQuietly } from './step-runner.contract.js';
 export { StepFailure, NoMatchError } from './step-failure.error.js';
 export { resolveErrorPolicy, backoffFor, sleep } from './retry.policy.js';
 export { RunGate } from './run-gate.policy.js';
+export { HostThrottle } from './host-throttle.policy.js';
+export { withTransportRetry, resolveRetryRule, transientError, retryDelay, DEFAULT_RETRY_RULE, RETRY_STATUSES } from './transport-retry.policy.js';
+export type { ResolvedRetryRule, Transient, TransportAttempt, RetryContext } from './transport-retry.policy.js';
+export type { HostRule, ThrottleConfig } from './host-throttle.policy.js';
 export { BlockedError } from './blocked.error.js';
 export { detectBlock, DEFAULT_BLOCK_RULE } from './block-rule.policy.js';
 export type { ObservedResponse } from './block-rule.policy.js';

package/dist/src/step-flow/run-gate.policy.d.ts

@@ -1,3 +1,4 @@
+import type { HostThrottle } from './host-throttle.policy.js';
 /**
  * What bounds a recipe run: how many `forEach` iterations may be in flight and
  * how close together requests may start. One gate per recipe run, shared by
@@ -10,6 +11,7 @@
 export declare class RunGate {
     readonly permits: number;
     readonly minIntervalMs: number;
+    readonly hosts?: HostThrottle | undefined;
     private readonly shared?;
     private inFlight;
     private readonly waiting;
@@ -17,9 +19,10 @@ export declare class RunGate {
     /**
      * @param permits - Iterations allowed in flight; 1 is sequential.
      * @param minIntervalMs - Minimum time between two request starts across the run.
+     * @param hosts - The crawler's per-site throttle, shared with every other recipe.
      * @param shared - The throttle state to share (internal: `nested` gates keep their parent's).
      */
-    constructor(permits: number, minIntervalMs: number, shared?: RunGate | undefined);
+    constructor(permits: number, minIntervalMs: number, hosts?: HostThrottle | undefined, shared?: RunGate | undefined);
     /** Whether this gate lets more than one iteration run at once. */
     get concurrent(): boolean;
     /**
@@ -33,6 +36,14 @@ export declare class RunGate {
      * whichever loop started it. Returns at once when the interval has passed.
      */
     throttle(): Promise<void>;
+    /**
+     * Waits until a request to `url` may start: the recipe's interval, then its
+     * site's turn in the crawler's per-site throttle.
+     *
+     * @param url - Where the request goes.
+     * @returns The release of the site's lane: call it once the response arrived or the request failed.
+     */
+    request(url: string): Promise<() => void>;
     /** The gate for a body running inside an iteration that holds a permit: sequential, same throttle. */
     nested(): RunGate;
 }

package/dist/src/step-flow/step-runner.contract.d.ts

@@ -1,6 +1,13 @@
 import type { ExtractionScope, LiveElement } from '../extraction-scope/index.js';
 import type { PaginateNext, Step } from '../recipe-schema/index.js';
 import type { BlockedError } from './blocked.error.js';
+/**
+ * Disposes a runner, ignoring a failure: a tab whose browser already went
+ * away (a rotation, a crash) has nothing left to close.
+ *
+ * @param runner - The runner.
+ */
+export declare function disposeQuietly(runner: StepRunner): Promise<void>;
 /** What `paginate` learns from the runner after a page body ran. */
 export type NextPageResult = 
 /** The next page is at this URL (the runner already navigated in web mode). */
@@ -34,6 +41,12 @@ export interface StepRunner {
      * `false` means it cannot, and the block fails the step like any error.
      */
     rotate?: (error: BlockedError) => Promise<boolean>;
+    /**
+     * A runner of its own for one parallel `forEach` iteration: a new tab in the
+     * same browser context (same cookies, its own page), disposed when the
+     * iteration ends. Web mode; an api runner is shared as it is.
+     */
+    fork?: () => Promise<StepRunner>;
     dispose: () => Promise<void>;
 }
 //# sourceMappingURL=step-runner.contract.d.ts.map

package/dist/src/step-flow/transport-retry.policy.d.ts

@@ -0,0 +1,76 @@
+import type { EventBus } from '../crawl-events/index.js';
+import type { RetryRule } from '../recipe-schema/index.js';
+import type { RunGate } from './run-gate.policy.js';
+/** A retry rule with every default filled in. */
+export type ResolvedRetryRule = Required<RetryRule>;
+/** Statuses a server uses for "not now": timeout, too early, too many requests, and the 5xx that pass. */
+export declare const RETRY_STATUSES: readonly number[];
+/** Three tries, one second then two apart, never a wait over 30 seconds. */
+export declare const DEFAULT_RETRY_RULE: ResolvedRetryRule;
+/** Why an attempt may be worth repeating, and the server's own `Retry-After`, if it sent one. */
+export interface Transient {
+    reason: string;
+    retryAfter?: string;
+}
+/** One try of a request, and how to tell a passing failure in what it gave. */
+export interface TransportAttempt<T> {
+    run: () => Promise<T>;
+    /** A transient problem in the outcome (a retry status, a connection error), or `undefined` when the outcome stands. */
+    problem: (outcome: {
+        value: T;
+    } | {
+        error: unknown;
+    }) => Transient | undefined;
+}
+export interface RetryContext {
+    recipeId: string;
+    gate: RunGate;
+    events: EventBus;
+    rule: ResolvedRetryRule;
+}
+/**
+ * The retry rule a recipe runs with: its `limits.retry` over the crawler's
+ * default over `DEFAULT_RETRY_RULE`.
+ *
+ * @param own - The recipe's `limits.retry`.
+ * @param crawler - The crawler's `retry` option.
+ * @returns The rule.
+ */
+export declare function resolveRetryRule(own?: RetryRule, crawler?: RetryRule): ResolvedRetryRule;
+/**
+ * Whether an error is a connection failure worth another try.
+ *
+ * @param error - What the request threw.
+ * @returns The reason, or `undefined`.
+ */
+export declare function transientError(error: unknown): Transient | undefined;
+/**
+ * How long to wait before attempt `attempt + 1`: the server's `Retry-After`
+ * when it gave one, else `backoffMs` doubling per attempt with a little
+ * jitter; `undefined` when the server asks for longer than `maxDelayMs` (it
+ * means "come back much later", which a crawl cannot wait for).
+ *
+ * @param rule - The retry rule.
+ * @param attempt - The attempt that just failed, from 1.
+ * @param retryAfter - The `Retry-After` header: seconds, or an HTTP date.
+ * @param now - The current time, for dates.
+ * @returns Milliseconds, or `undefined` for no retry.
+ */
+export declare function retryDelay(rule: ResolvedRetryRule, attempt: number, retryAfter?: string, now?: number): number | undefined;
+/**
+ * Sends a request through the gate (the recipe's rate, the site's lane), and
+ * sends it again after a pause while it fails in a passing way, up to
+ * `rule.attempts` tries in all. A `Retry-After` holds back every request to
+ * that site, not only this one. Each retry is reported as `request:retry`.
+ *
+ * Like redialling a busy number: wait a moment, dial again, give up after a
+ * few tries; and if the other end said "call back in a minute", wait that minute.
+ *
+ * @param url - Where the request goes.
+ * @param attempt - How to send it and how to judge the outcome.
+ * @param context - The recipe, gate, events and rule.
+ * @returns What the last try gave.
+ * @throws What the last try threw.
+ */
+export declare function withTransportRetry<T>(url: string, attempt: TransportAttempt<T>, context: RetryContext): Promise<T>;
+//# sourceMappingURL=transport-retry.policy.d.ts.map

package/dist/src/web-steps/navigate.use-case.d.ts

@@ -5,8 +5,9 @@ import type { GotoStep, InputRecipe } from '../recipe-schema/index.js';
 import type { RunGate } from '../step-flow/index.js';
 /**
  * Runs a `goto` step: renders the URL (relative to the current page), waits for
- * the gate's throttle (`delayMs`), navigates, records the page's real URL in the
- * scope, and checks the response against the recipe's block rule.
+ * the gate's throttle (`delayMs`), navigates (again, after a pause, while it
+ * fails in passing: `limits.retry`), records the page's real URL in the scope,
+ * and checks the response against the recipe's block rule.
  *
  * @throws BlockedError when the response is a block.
  */