@opencraw/core 0.1.1 → 0.1.3

package/dist/src/crawl-execution/run-crawl.use-case.d.ts

@@ -2,12 +2,16 @@ import type { RecipeSet } from '../recipe-loading/index.js';
 import type { CrawlReport } from './crawl-report.model.js';
 import type { RecipeRunDependencies } from './run-input-recipe.use-case.js';
 /**
- * Runs every input recipe of a set, one after another, into one sink.
+ * Runs every input recipe of a set into one sink, `parallel` at a time
+ * (default one after another). Reports come back in the set's order whatever
+ * order the recipes finish in. Under `onRecipeError: 'stop'`, a failed recipe
+ * stops the ones not started yet; those already running finish.
  *
  * @param set - The bound recipes.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @param onRecipeError - Whether a failed recipe stops the run.
+ * @param parallel - How many input recipes run at once.
  * @returns The report.
  */
-export declare function runCrawl(set: RecipeSet, deps: RecipeRunDependencies, onRecipeError: 'continue' | 'stop'): Promise<CrawlReport>;
+export declare function runCrawl(set: RecipeSet, deps: RecipeRunDependencies, onRecipeError: 'continue' | 'stop', parallel?: number): Promise<CrawlReport>;
 //# sourceMappingURL=run-crawl.use-case.d.ts.map

package/dist/src/crawl-execution/run-input-recipe.use-case.d.ts

@@ -1,9 +1,12 @@
 import type { AccessBroker } from '../access/index.js';
 import { BrowserClient } from '../browser-session/index.js';
+import type { BrowserProfiles } from '../browser-session/index.js';
+import { CaptchaSolverRegistry } from '../captcha/index.js';
 import type { EventBus } from '../crawl-events/index.js';
 import type { HookRegistry } from '../hooks/index.js';
-import type { InputRecipe, OutputRecipe } from '../recipe-schema/index.js';
+import type { InputRecipe, OutputRecipe, RetryRule } from '../recipe-schema/index.js';
 import type { DedupePolicy, RecordSink } from '../record-sink/index.js';
+import type { HostThrottle } from '../step-flow/index.js';
 import type { RecipeReport } from './crawl-report.model.js';
 export interface RecipeRunDependencies {
     browser: () => Promise<BrowserClient>;
@@ -20,6 +23,14 @@ export interface RecipeRunDependencies {
     debug?: boolean;
     /** Leases each recipe run its network access. */
     access: AccessBroker;
+    /** The solvers recipes name; none when omitted. */
+    captchaSolvers?: CaptchaSolverRegistry;
+    /** The crawler's per-site throttle, shared by every recipe. */
+    hosts?: HostThrottle;
+    /** The runner's persistent browser profiles, for `session.browserProfile`. */
+    profiles?: BrowserProfiles;
+    /** The crawler's retry rule, under each recipe's `limits.retry`. */
+    retry?: RetryRule;
 }
 /**
  * Runs one input recipe end to end: session, runner, the step walk, and for
@@ -31,10 +42,10 @@ export interface RecipeRunDependencies {
  * the sink sees one record at a time and `maxRecords` is exact: once reached,
  * every later emit returns `stop` before mapping.
  *
- * @param input - The input recipe.
+ * @param recipe - The input recipe.
  * @param output - The output recipe it feeds.
  * @param deps - Shared browser, hooks, events, sink and de-duplication.
  * @returns What happened.
  */
-export declare function runInputRecipe(input: InputRecipe, output: OutputRecipe, deps: RecipeRunDependencies): Promise<RecipeReport>;
+export declare function runInputRecipe(recipe: InputRecipe, output: OutputRecipe, deps: RecipeRunDependencies): Promise<RecipeReport>;
 //# sourceMappingURL=run-input-recipe.use-case.d.ts.map

package/dist/src/deck-document/deck-document.model.d.ts

@@ -0,0 +1,58 @@
+import type { Sheet } from '../workbook-document/index.js';
+/** A text box on a slide, in points from the slide's top-left corner. */
+export interface DeckShape {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    text: string;
+    /** The placeholder it fills (`title`, `body`…). */
+    placeholder?: string;
+}
+/** A chart's data, from the values the chart caches. */
+export interface DeckChart {
+    type: string;
+    title?: string;
+    series: {
+        name: string;
+        categories: string[];
+        values: (number | null)[];
+    }[];
+}
+/** One slide. */
+export interface DeckSlide {
+    number: number;
+    title?: string;
+    hidden: boolean;
+    /** Text boxes in reading order. */
+    shapes: DeckShape[];
+    /** Native tables, as sheets (`table 1`…) with their merged cells. */
+    tables: Sheet[];
+    charts: DeckChart[];
+    notes: string;
+}
+/** A presentation read into slides: what `extract` works on. */
+export interface DeckDocument {
+    kind: 'deck';
+    /** The slide size, in points. */
+    width: number;
+    height: number;
+    slides: DeckSlide[];
+}
+/**
+ * The text a `regex` extract reads: per visible slide, its title, its text
+ * boxes in reading order, its tables' rows (cells separated by a tab) and its
+ * notes after `Notes:`; slides separated by a blank line.
+ *
+ * @param document - The deck.
+ * @returns The text.
+ */
+export declare function deckText(document: DeckDocument): string;
+/**
+ * Whether a value bound in scope is a read deck (so `extract … from` can take it).
+ *
+ * @param value - Anything.
+ * @returns Whether it is a {@link DeckDocument}.
+ */
+export declare function isDeckDocument(value: unknown): value is DeckDocument;
+//# sourceMappingURL=deck-document.model.d.ts.map

package/dist/src/deck-document/deck-table.algorithm.d.ts

@@ -0,0 +1,35 @@
+import type { TableAlign } from '../pdf-document/index.js';
+import type { GridTableQuery, WorkbookCell } from '../workbook-document/index.js';
+import type { DeckDocument } from './deck-document.model.js';
+/** What a table extract looks for in a deck. */
+export interface DeckTableQuery extends Omit<GridTableQuery, 'sheet'> {
+    /** Matches the titles of the slides to read; default every slide. */
+    slide?: RegExp;
+    /** Read text boxes laid out as a table instead of native tables. */
+    shapes?: boolean;
+    /** With `shapes`: how a row's values sit against a box wrapped over several lines. */
+    align?: TableAlign;
+}
+/** One table found in a deck. */
+export interface DeckTable {
+    /** The slide's number, from 1. */
+    slide: number;
+    slideTitle: string;
+    /** The first header cell. */
+    title: string;
+    header: string[];
+    rows: Record<string, WorkbookCell>[];
+}
+/**
+ * Finds tables in a deck: native tables through the workbook table reader
+ * (merged cells filled, a header over several rows joined), or, with
+ * `shapes`, text boxes laid out as a table through the PDF table reader (a box
+ * is a cell, boxes whose heights overlap a row, columns from where the body's
+ * boxes start). Hidden slides are skipped unless `includeHidden`.
+ *
+ * @param document - The deck.
+ * @param query - Which tables, on which slides, and how to name their columns.
+ * @returns The tables, slide by slide.
+ */
+export declare function findDeckTables(document: DeckDocument, query: DeckTableQuery): DeckTable[];
+//# sourceMappingURL=deck-table.algorithm.d.ts.map

package/dist/src/deck-document/index.d.ts

@@ -0,0 +1,6 @@
+export { deckText, isDeckDocument } from './deck-document.model.js';
+export type { DeckDocument, DeckSlide, DeckShape, DeckChart } from './deck-document.model.js';
+export { readPptxDeck } from './read-pptx.client.js';
+export { findDeckTables } from './deck-table.algorithm.js';
+export type { DeckTable, DeckTableQuery } from './deck-table.algorithm.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/deck-document/read-pptx.client.d.ts

@@ -0,0 +1,16 @@
+import type { DeckDocument } from './deck-document.model.js';
+/**
+ * Reads a `.pptx` presentation into a deck document, through
+ * `@opencraw/office-reader`: every slide's text boxes with their positions,
+ * its tables with their merged cells, its charts' cached data and its notes.
+ * The reader is imported on first use, so recipes that never read a
+ * presentation never load it.
+ *
+ * @param bytes - The file.
+ * @param source - Where it came from, for messages.
+ * @returns The deck.
+ * @throws Error naming the source, and saying what to do, for a file that is
+ * not a readable presentation (a legacy `.ppt`, a password-protected file, an `.odp`…).
+ */
+export declare function readPptxDeck(bytes: Uint8Array, source: string): Promise<DeckDocument>;
+//# sourceMappingURL=read-pptx.client.d.ts.map

package/dist/src/extraction-scope/extraction-scope.model.d.ts

@@ -7,9 +7,11 @@
  * Page state (the current URL, page number and the document `extract` reads by
  * default) is scope state too, bound in the innermost scope that navigated.
  */
+import type { DeckDocument } from '../deck-document/index.js';
 import type { PdfDocument } from '../pdf-document/index.js';
+import type { WorkbookDocument } from '../workbook-document/index.js';
 /** A fetched or rendered document a later `extract` can read. */
-export type ScopeDocument = PdfDocument | {
+export type ScopeDocument = PdfDocument | WorkbookDocument | DeckDocument | {
     kind: 'json';
     data: unknown;
 } | {

package/dist/src/http-session/http-response.contract.d.ts

@@ -1,4 +1,6 @@
+import type { DeckDocument } from '../deck-document/index.js';
 import type { PdfDocument } from '../pdf-document/index.js';
+import type { WorkbookDocument } from '../workbook-document/index.js';
 import type { BodyKind, HttpMethod } from '../recipe-schema/index.js';
 /** One HTTP request as the api runner sends it, templates already rendered. */
 export interface HttpRequest {
@@ -9,10 +11,16 @@ export interface HttpRequest {
     body?: unknown;
     /** How to read the body; default: from the response content type. */
     as?: BodyKind;
+    /** The encoding of a text body (a WHATWG label); default: the BOM, the declared charset, UTF-8, else Windows-1252. */
+    encoding?: string;
+    /** A CSV body's delimiter (one character); default: detected. */
+    delimiter?: string;
+    /** A YAML body's scalars: `typed` (default) or `text`. */
+    scalars?: 'typed' | 'text';
     timeoutMs?: number;
 }
 /** A parsed response body. Structurally the same as a scope document, on purpose. */
-export type HttpBody = PdfDocument | {
+export type HttpBody = PdfDocument | WorkbookDocument | DeckDocument | {
     kind: 'json';
     data: unknown;
 } | {
@@ -28,6 +36,10 @@ export interface HttpResponse {
     url: string;
     headers: Record<string, string>;
     body: HttpBody;
+    /** What reading the body noticed but read anyway (an unknown YAML tag…). */
+    warnings?: string[];
+    /** The format the body was read as (`yaml` and `jsonl` both give JSON data). */
+    format?: BodyKind;
 }
 /** The part of the client the api runner needs; tests fake it. */
 export interface HttpSender {

package/dist/src/http-session/text-decoding.algorithm.d.ts

@@ -0,0 +1,35 @@
+/** Decoded text and the encoding it was read as. */
+export interface DecodedText {
+    text: string;
+    encoding: string;
+}
+/**
+ * Decodes a body, in this order: a byte-order mark (UTF-8, UTF-16 LE/BE; Excel's
+ * "Unicode text" export is UTF-16 LE), the encoding a recipe asks for, the
+ * charset the server declares, strict UTF-8, and Windows-1252 (a superset of
+ * Latin-1) for text that is not UTF-8.
+ *
+ * The Windows-1252 fallback is only taken when the text holds no valid UTF-8
+ * beyond ASCII: a UTF-8 page with one stray byte keeps its accents, with a
+ * replacement character for the stray byte, instead of turning every accent
+ * into mojibake.
+ *
+ * @param bytes - The body.
+ * @param options - `encoding`: the recipe's choice, a WHATWG label (wins over
+ * the charset, not over a BOM); `charset`: from the content type (ignored when
+ * not a known label).
+ * @returns The text, without its BOM, and the encoding used.
+ * @throws Error when `encoding` is not a known label.
+ */
+export declare function decodeText(bytes: Uint8Array, options?: {
+    encoding?: string;
+    charset?: string;
+}): DecodedText;
+/**
+ * The charset a content type declares (`text/csv; charset=ISO-8859-1`).
+ *
+ * @param contentType - The header value.
+ * @returns The charset, or `undefined`.
+ */
+export declare function charsetOf(contentType: string): string | undefined;
+//# sourceMappingURL=text-decoding.algorithm.d.ts.map

package/dist/src/index.d.ts

@@ -1,11 +1,11 @@
 export { createCrawler } from './crawl-execution/index.js';
-export { AccessBroker, AccessConfigError, ACCESS_PRESETS, loadAccessConfig, accessConfigSchema, accessConfigJsonSchema } from './access/index.js';
+export { AccessBroker, AccessConfigError, ACCESS_PRESETS, loadAccessConfig, accessConfigSchema, accessConfigJsonSchema, throttleConfigSchema } from './access/index.js';
 export type { AccessConfig, AccessProfile, AccessPlugin, AccessLease, LeaseRequest, PluginLeaseRequest, ProxySettings, AccessPreset } from './access/index.js';
 export type { Crawler, CrawlOptions, CrawlReport, RecipeReport } from './crawl-execution/index.js';
 export { loadRecipeSet, loadRecipes, readRecipeSource, bindRecipeSet, RecipeSet, RecipeBindingError, validateBinding } from './recipe-loading/index.js';
 export type { RecipeSetSource, RecipeSource, RecipeBytes, RecipeDocument, BindingIssue } from './recipe-loading/index.js';
-export { parseInputRecipe, parseOutputRecipe, RecipeValidationError, inputRecipeJsonSchema, outputRecipeJsonSchema, inputRecipeSchema, outputRecipeSchema } from './recipe-schema/index.js';
-export type { InputRecipe, OutputRecipe, FieldSpec, Step, StepType, MappingRule, TransformRule, ErrorPolicy, PaginateNext, SessionSpec, SessionAccess, RecipeIssue } from './recipe-schema/index.js';
+export { parseInputRecipe, parseOutputRecipe, RecipeValidationError, inputRecipeJsonSchema, outputRecipeJsonSchema, inputRecipeSchema, outputRecipeSchema, retryRuleSchema } from './recipe-schema/index.js';
+export type { InputRecipe, OutputRecipe, FieldSpec, Step, StepType, MappingRule, TransformRule, ErrorPolicy, PaginateNext, SessionSpec, SessionAccess, CaptchaSettings, CaptchaStep, RetryRule, RecipeIssue } from './recipe-schema/index.js';
 export type { Hook, HookMap, HookContext } from './hooks/index.js';
 export { UnknownHookError } from './hooks/index.js';
 export type { OutputRecord } from './output-mapping/index.js';
@@ -21,6 +21,15 @@ export type { HttpClientOptions, HttpRequest, HttpResponse, HttpBody } from './h
 export { tryParseJson } from './selection/index.js';
 export { readPdf, PdfReadError, findTables, pdfText } from './pdf-document/index.js';
 export type { PdfDocument, PdfPage, PdfRow, PdfCell, PdfTable, TableQuery, TableAlign } from './pdf-document/index.js';
-export { StepFailure } from './step-flow/index.js';
+export { parseCsv, detectDelimiter, csvWorkbook, findGridTables, fillDown, workbookText, isWorkbookDocument, htmlTableSheets } from './workbook-document/index.js';
+export { readMarkdown } from './markdown-document/index.js';
+export { readYaml } from './yaml-document/index.js';
+export { findDeckTables, deckText, isDeckDocument } from './deck-document/index.js';
+export type { DeckDocument, DeckSlide, DeckShape, DeckChart, DeckTable, DeckTableQuery } from './deck-document/index.js';
+export type { WorkbookDocument, WorkbookCell, Sheet, CsvFormat, GridTable, GridTableQuery } from './workbook-document/index.js';
+export { StepFailure, HostThrottle, DEFAULT_RETRY_RULE } from './step-flow/index.js';
+export type { ThrottleConfig, HostRule } from './step-flow/index.js';
 export { TransformError } from './transformation/index.js';
+export { CaptchaError, DEFAULT_CAPTCHA_SELECTOR, detectChallenge } from './captcha/index.js';
+export type { CaptchaSolver, CaptchaChallenge, CaptchaContext, CaptchaOutcome, CaptchaKind, CaptchaLog } from './captcha/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/markdown-document/index.d.ts

@@ -0,0 +1,3 @@
+export { readMarkdown } from './read-markdown.client.js';
+export type { MarkdownRead } from './read-markdown.client.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/markdown-document/read-markdown.client.d.ts

@@ -0,0 +1,29 @@
+/** Markdown, rendered. */
+export interface MarkdownRead {
+    /** A full HTML document: front matter in the head, the rendered body wrapped in sections. */
+    html: string;
+    /** The front matter's data, when there is some. */
+    frontMatter: unknown;
+    warnings: string[];
+}
+/**
+ * Renders Markdown (GitHub-flavoured: tables, task lists, strikethrough,
+ * autolinks) to HTML with `marked`, imported on first use, so every `css`
+ * selector works on it:
+ *
+ * - each heading and everything up to the next heading of the same or a higher
+ *   level is wrapped in `<section data-heading="…" data-level="…">`, sections
+ *   nesting, so "the table under *Prezzi*" is one selector;
+ * - headings get slug ids (`<h2 id="prezzi">`);
+ * - a leading `---` YAML block is parsed (YAML 1.2, as `as: "yaml"` reads it)
+ *   and put in the head as `<script type="application/json" data-front-matter>`.
+ *
+ * Raw HTML in the Markdown is kept: it is data, parsed by cheerio, never run.
+ *
+ * @param text - The Markdown.
+ * @param source - Where it came from, for messages.
+ * @returns The HTML, the front matter's data and the YAML parser's warnings.
+ * @throws Error naming the source when the front matter is not YAML.
+ */
+export declare function readMarkdown(text: string, source: string): Promise<MarkdownRead>;
+//# sourceMappingURL=read-markdown.client.d.ts.map

package/dist/src/pdf-document/index.d.ts

@@ -1,7 +1,7 @@
 export { readPdf, PdfReadError } from './read-pdf.client.js';
 export { pdfText, isPdfDocument } from './pdf-document.model.js';
 export type { PdfDocument, PdfPage, PdfRow, PdfCell, PositionedText } from './pdf-document.model.js';
-export { assembleRows } from './row-assembly.algorithm.js';
+export { assembleRows, rowsOfCells } from './row-assembly.algorithm.js';
 export { findTables } from './pdf-table.algorithm.js';
 export type { PdfTable, TableQuery, TableAlign } from './pdf-table.algorithm.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/src/pdf-document/row-assembly.algorithm.d.ts

@@ -1,4 +1,4 @@
-import type { PdfRow, PositionedText } from './pdf-document.model.js';
+import type { PdfCell, PdfRow, PositionedText } from './pdf-document.model.js';
 /**
  * Turns a page's text runs into rows of cells, top to bottom.
  *
@@ -12,6 +12,15 @@ import type { PdfRow, PositionedText } from './pdf-document.model.js';
  * @returns The rows.
  */
 export declare function assembleRows(runs: readonly PositionedText[]): PdfRow[];
+/**
+ * Groups finished cells into rows, top to bottom: cells whose vertical extents
+ * overlap share a row. For cells that need no joining, such as a slide's text
+ * boxes, each already a cell.
+ *
+ * @param cells - The cells, in any order.
+ * @returns The rows.
+ */
+export declare function rowsOfCells(cells: readonly PdfCell[]): PdfRow[];
 /** `Array#findLast`, which the es2022 library does not declare. */
 export declare function lastOf<T>(items: readonly T[], test: (item: T) => boolean): T | undefined;
 //# sourceMappingURL=row-assembly.algorithm.d.ts.map

package/dist/src/recipe-loading/recipe-binding.validator.d.ts

@@ -14,7 +14,9 @@ import type { BindingIssue } from './recipe-binding.error.js';
  * - web-only steps appear only in web recipes or inside a bootstrap, api-only
  *   steps only in api recipes, and `next.selector` only in web mode;
  * - exactly one emitting construct exists on any path (the two branches of an
- *   `if` are separate paths).
+ *   `if` are separate paths);
+ * - a `captcha` step, and `onBlock.solve`, have a solver: their own or
+ *   `session.captcha.solver`.
  *
  * @param input - A parsed input recipe.
  * @param output - The parsed output recipe it names.

package/dist/src/recipe-schema/index.d.ts

@@ -2,11 +2,11 @@ export * from './recipe-kind.enum.js';
 export { fieldSpecSchema, outputRecipeSchema } from './output-recipe.contract.js';
 export type { FieldSpec, OutputRecipe } from './output-recipe.contract.js';
 export { stepSchema, errorPolicySchema, paginateNextSchema } from './step.contract.js';
-export type { Step, StepType, StepBaseFields, TargetFields, ErrorPolicy, PaginateNext, TakeKind, GotoStep, ClickStep, FillStep, PressStep, SelectStep, ScrollStep, WaitStep, EvaluateStep, ScreenshotStep, RequestStep, ExtractStep, SetStep, CollectStep, ForEachStep, IfStep, PaginateStep, EmitStep, HookStep, } from './step.contract.js';
+export type { Step, StepType, StepBaseFields, TargetFields, ErrorPolicy, PaginateNext, TakeKind, GotoStep, ClickStep, FillStep, PressStep, SelectStep, ScrollStep, WaitStep, EvaluateStep, ScreenshotStep, RequestStep, ExtractStep, SetStep, CollectStep, ForEachStep, IfStep, PaginateStep, EmitStep, HookStep, CaptchaStep, CaptchaCheck, } from './step.contract.js';
 export { transformRuleSchema, mappingRuleSchema } from './transform-rule.contract.js';
 export type { TransformRule, TransformOp, MappingRule, FromRule, EachRule } from './transform-rule.contract.js';
-export { inputRecipeSchema, sessionSpecSchema, startPointSchema } from './input-recipe.contract.js';
-export type { InputRecipe, SessionSpec, SessionBootstrap, SessionAccess, BlockRule, BlockRotation, StartPoint, CrawlLimits, RecipeCookie } from './input-recipe.contract.js';
+export { inputRecipeSchema, sessionSpecSchema, startPointSchema, retryRuleSchema } from './input-recipe.contract.js';
+export type { InputRecipe, SessionSpec, SessionBootstrap, SessionAccess, BlockRule, BlockRotation, CaptchaSettings, RetryRule, StartPoint, CrawlLimits, RecipeCookie } from './input-recipe.contract.js';
 export { parseInputRecipe, parseOutputRecipe, recipeKindOf } from './recipe.validator.js';
 export { RecipeValidationError } from './recipe-validation.error.js';
 export type { RecipeIssue } from './recipe-validation.error.js';

package/dist/src/recipe-schema/input-recipe.contract.d.ts

@@ -1,6 +1,6 @@
 import { z } from 'zod';
 import type { CrawlMode, KeepKind } from './recipe-kind.enum.js';
-import type { ErrorPolicy, Step } from './step.contract.js';
+import type { CaptchaCheck, ErrorPolicy, Step } from './step.contract.js';
 import type { MappingRule } from './transform-rule.contract.js';
 /** A URL the crawl starts from, with variables visible to its templates as `vars.*`. */
 export interface StartPoint {
@@ -49,12 +49,41 @@ export interface BlockRule {
     /** A regular expression the response body must match (case-insensitive). */
     text?: string;
 }
-/** What to do when blocked: take a new access lease (a new IP), reopen the session, and retry the step. */
+/**
+ * What to do when blocked. `solve`: when the block page shows a captcha, the
+ * `session.captcha` solver solves it on the spot. `rotate`: take a new access
+ * lease (a new IP), reopen the session, and retry the step; with `solve`, only
+ * once solving failed.
+ */
 export interface BlockRotation {
-    rotate: boolean;
+    rotate?: boolean;
+    solve?: boolean;
     /** How many rotations a recipe run may use. Default 2. */
     attempts?: number;
 }
+/**
+ * How a web recipe gets past captchas: a solver the runner registered (a
+ * plugin's `captchaSolvers`), where to look for challenges, how to confirm
+ * one is solved, and what the run may spend. With it, the engine checks for a
+ * challenge after each navigation, click and key press, and solves it before
+ * the next step runs.
+ */
+export interface CaptchaSettings {
+    /** The name of a registered captcha solver. */
+    solver: string;
+    /** Where challenges are (a Playwright selector); the common widgets when omitted. */
+    detect?: {
+        selector: string;
+    };
+    /** How a solve is confirmed. Default: the challenge is gone. */
+    verify?: CaptchaCheck;
+    /** Solves tried per challenge before it counts as a block. Default 3. */
+    attempts?: number;
+    /** How long one solve may take. Default 120000. */
+    timeoutMs?: number;
+    /** Solves the whole run may spend (a paid solver bills each). Default 10; 0 detects without solving. */
+    maxSolves?: number;
+}
 export interface SessionSpec {
     headers?: Record<string, string>;
     cookies?: RecipeCookie[];
@@ -69,14 +98,37 @@ export interface SessionSpec {
     access?: SessionAccess;
     blockedWhen?: BlockRule;
     onBlock?: BlockRotation;
+    captcha?: CaptchaSettings;
+    /**
+     * A browser profile of the runner that persists between runs (cookies,
+     * storage, cache): web recipes and bootstraps run in it. A name; the runner
+     * decides where profiles live.
+     */
+    browserProfile?: string;
+}
+/**
+ * How a request that fails in passing (a dropped connection, a timeout, a
+ * 503, a 429) is sent again. On by default: three tries in all.
+ */
+export interface RetryRule {
+    /** Tries per request, the first included; `1` turns retrying off. Default 3. */
+    attempts?: number;
+    /** The first pause; it doubles on every retry. Default 1000. */
+    backoffMs?: number;
+    /** The longest pause, `Retry-After` included; a server asking for longer is not retried. Default 30000. */
+    maxDelayMs?: number;
+    /** The statuses retried. Default `[408, 425, 429, 500, 502, 503, 504]`. */
+    statuses?: number[];
 }
 export interface CrawlLimits {
     maxRecords?: number;
     /** Minimum interval between two request starts across the recipe, whatever runs in parallel. */
     delayMs?: number;
     timeoutMs?: number;
-    /** How many `forEach` iterations may run at once (api mode; a web recipe drives one page). Default 1. */
+    /** How many `forEach` iterations over a list may run at once: requests in api mode, tabs in web mode. Default 1. */
     concurrency?: number;
+    /** How requests that fail in passing are sent again; the crawler's `retry`, else three tries, when omitted. */
+    retry?: RetryRule;
 }
 /** Where to start, how to navigate, what to extract, and how it maps to one output recipe. */
 export interface InputRecipe {
@@ -99,5 +151,6 @@ export interface InputRecipe {
 }
 export declare const startPointSchema: z.ZodType<StartPoint>;
 export declare const sessionSpecSchema: z.ZodType<SessionSpec>;
+export declare const retryRuleSchema: z.ZodType<RetryRule>;
 export declare const inputRecipeSchema: z.ZodType<InputRecipe>;
 //# sourceMappingURL=input-recipe.contract.d.ts.map

package/dist/src/recipe-schema/recipe-kind.enum.d.ts

@@ -4,7 +4,8 @@ export declare const CRAWL_MODES: readonly ["web", "api"];
 export declare const SELECTOR_KINDS: readonly ["css", "xpath", "jsonpath", "regex", "table"];
 /** `take` also accepts `attr:<name>`, which is validated by pattern rather than listed. */
 export declare const TAKE_KINDS: readonly ["text", "html", "value", "json"];
-export declare const BODY_KINDS: readonly ["json", "html", "text", "pdf"];
+export declare const BODY_KINDS: readonly ["json", "jsonl", "html", "text", "pdf", "csv", "xlsx", "pptx", "yaml", "markdown"];
+export declare const YAML_SCALARS: readonly ["typed", "text"];
 /** How a PDF table aligns a row's values against a cell wrapped over several lines. */
 export declare const TABLE_ALIGNS: readonly ["auto", "top", "center", "bottom"];
 export declare const FIELD_TYPES: readonly ["string", "number", "integer", "boolean", "date", "datetime", "currency", "url", "enum", "array", "object", "json"];
@@ -16,7 +17,7 @@ export declare const KEEP_KINDS: readonly ["cookies", "localStorage"];
 export declare const WAIT_UNTIL: readonly ["load", "domcontentloaded", "networkidle", "commit"];
 export declare const HTTP_METHODS: readonly ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"];
 /** Steps that only make sense with a live browser page. */
-export declare const WEB_ONLY_STEPS: readonly ["goto", "click", "fill", "press", "select", "scroll", "wait", "evaluate", "screenshot"];
+export declare const WEB_ONLY_STEPS: readonly ["goto", "click", "fill", "press", "select", "scroll", "wait", "evaluate", "screenshot", "captcha"];
 /** Steps that only make sense against an HTTP request context. */
 export declare const API_ONLY_STEPS: readonly ["request"];
 export type RecipeKind = typeof RECIPE_KINDS[number];

package/dist/src/recipe-schema/step.contract.d.ts

@@ -86,6 +86,12 @@ export interface RequestStep extends StepBaseFields {
     headers?: Record<string, string>;
     body?: unknown;
     as?: BodyKind;
+    /** The body's text encoding (a WHATWG label, `windows-1252`); default: the BOM, the declared charset, UTF-8, else Windows-1252. */
+    encoding?: string;
+    /** A CSV body's delimiter (one character); default: detected among `,` `;` tab `|`. */
+    delimiter?: string;
+    /** A YAML body's scalars: `typed` (default, YAML 1.2) or `text`, every scalar as written (`0123` stays `"0123"`). */
+    scalars?: 'typed' | 'text';
 }
 export interface ExtractStep extends StepBaseFields {
     type: 'extract';
@@ -99,8 +105,20 @@ export interface ExtractStep extends StepBaseFields {
     columns?: Record<string, string>;
     /** `table` only: a pattern (case-insensitive) for the row that ends a table. */
     until?: string;
-    /** `table` only: how a row's values sit against a cell wrapped over several lines; default `auto`. */
+    /** `table` only: how a row's values sit against a cell wrapped over several lines (PDF); default `auto`. */
     align?: TableAlign;
+    /** `table` only: a pattern (case-insensitive) for the names of the sheets to read (workbook); default every sheet. */
+    sheet?: string;
+    /** `table` only: how many rows the header spans (workbook); a column's key joins its header texts. Default 1. */
+    headerRows?: number;
+    /** `table` only: output keys whose empty cells take the value of the row above. */
+    fillDown?: string[];
+    /** `table` only: read hidden sheets and rows (workbook) or hidden slides (deck) too. */
+    includeHidden?: boolean;
+    /** `table` only: a pattern (case-insensitive) for the titles of the slides to read (deck); default every slide. */
+    slide?: string;
+    /** `table` only: read text boxes laid out as a table instead of native tables (deck). */
+    shapes?: boolean;
 }
 export interface SetStep extends StepBaseFields {
     type: 'set';
@@ -150,9 +168,35 @@ export interface HookStep extends StepBaseFields {
     name: string;
     args?: Record<string, unknown>;
 }
-export type Step = GotoStep | ClickStep | FillStep | PressStep | SelectStep | ScrollStep | WaitStep | EvaluateStep | ScreenshotStep | RequestStep | ExtractStep | SetStep | CollectStep | ForEachStep | IfStep | PaginateStep | EmitStep | HookStep;
+/**
+ * How the engine confirms a captcha was solved (it never takes the solver's
+ * word): the challenge is gone, and/or an element appears.
+ */
+export interface CaptchaCheck {
+    /** The detected challenge must be off the page. Default `true`. */
+    gone?: boolean;
+    /** An element that must appear once solved. */
+    selector?: string;
+}
+/**
+ * Solves the captcha on the live page, if there is one (none is not an error):
+ * a challenge known to sit at one point of the crawl, a login form. The solver
+ * and the defaults come from `session.captcha`, unless the step names its own.
+ */
+export interface CaptchaStep extends StepBaseFields {
+    type: 'captcha';
+    /** A captcha solver the runner registered; `session.captcha.solver` when omitted. */
+    solver?: string;
+    /** Where the challenge is (a Playwright selector); the common widgets when omitted. */
+    selector?: string;
+    verify?: CaptchaCheck;
+    attempts?: number;
+    timeoutMs?: number;
+}
+export type Step = GotoStep | ClickStep | FillStep | PressStep | SelectStep | ScrollStep | WaitStep | EvaluateStep | ScreenshotStep | RequestStep | ExtractStep | SetStep | CollectStep | ForEachStep | IfStep | PaginateStep | EmitStep | HookStep | CaptchaStep;
 export type StepType = Step['type'];
 export declare const errorPolicySchema: z.ZodType<ErrorPolicy>;
 export declare const paginateNextSchema: z.ZodType<PaginateNext>;
+export declare const captchaCheckSchema: z.ZodType<CaptchaCheck>;
 export declare const stepSchema: z.ZodType<Step>;
 //# sourceMappingURL=step.contract.d.ts.map

package/dist/src/record-sink/dedupe.policy.d.ts

@@ -1,17 +1,29 @@
 import type { OutputRecord } from '../output-mapping/index.js';
 /** How far de-duplication reaches: the whole run, one input recipe, or not at all. */
 export type DedupeScope = 'run' | 'recipe' | 'off';
-/** Drops records whose key was already seen. First record wins; keyless records always pass. */
+/** One input recipe's view of de-duplication: whether a record repeats a key already seen. */
+export interface RecipeDedupe {
+    /**
+     * @param record - A validated record.
+     * @returns `true` when the record repeats an earlier key and must be dropped.
+     */
+    isDuplicate: (record: OutputRecord) => boolean;
+}
+/**
+ * Drops records whose key was already seen. First record wins; keyless
+ * records always pass. Recipes running in parallel each get their own view:
+ * under `recipe` scope they never see each other's keys, under `run` scope
+ * they share them (and whichever emits a key first keeps it).
+ */
 export declare class DedupePolicy {
     readonly scope: DedupeScope;
-    private seen;
+    private readonly shared;
     constructor(scope?: DedupeScope);
-    /** Called when an input recipe starts; forgets keys under `recipe` scope. */
-    startRecipe(): void;
     /**
-     * @param record - A validated record.
-     * @returns `true` when the record repeats an earlier key and must be dropped.
+     * The de-duplication one input recipe run uses.
+     *
+     * @returns Its view: keys shared with the run, its own, or none checked.
      */
-    isDuplicate(record: OutputRecord): boolean;
+    forRecipe(): RecipeDedupe;
 }
 //# sourceMappingURL=dedupe.policy.d.ts.map

package/dist/src/record-sink/index.d.ts

@@ -4,5 +4,6 @@ export type { MemorySink } from './memory-sink.repository.js';
 export { jsonLinesSink } from './json-lines-sink.repository.js';
 export type { JsonLinesSinkOptions } from './json-lines-sink.repository.js';
 export { DedupePolicy } from './dedupe.policy.js';
+export type { RecipeDedupe } from './dedupe.policy.js';
 export type { DedupeScope } from './dedupe.policy.js';
 //# sourceMappingURL=index.d.ts.map