@octocrawl/sdk 0.3.0

package/dist/types/esm/contracts/api.d.ts

@@ -0,0 +1,891 @@
+/**
+ * Native REST contract for scrape + crawl.
+ *
+ * Request fields match the product CLI. Responses are FetchResult /
+ * CrawlReport — no second result enum. Types only.
+ */
+import { type CrawlMode, type RobotsOverride } from './compliance.js';
+import type { CrawlError, CrawlPage, CrawlPageList, CrawlReport, SitemapMode } from './crawl.js';
+import { type FetchOptions } from './execution.js';
+import type { FetchResult, FetchWarning, LadderRunAudit, TraceEvent } from './result.js';
+import type { DocumentExtraction, PageMetadata } from './extractor.js';
+import type { EvidenceRecord } from './evidenceRecord.js';
+import type { ScrapeFormat, StructuredExtractionResult } from './structured.js';
+import type { WebhookPayloadFormat } from './delivery.js';
+export declare const CRAWL_MODES: readonly ["research", "standard", "authed"];
+export type ApiCrawlMode = (typeof CRAWL_MODES)[number];
+/** A scrape's deadline when the request sets no `timeout`, and the largest one it may set. */
+export declare const DEFAULT_SCRAPE_TIMEOUT_MS = 300000;
+export declare const MIN_SCRAPE_TIMEOUT_MS = 1000;
+export declare const MAX_WAIT_FOR_MS = 60000;
+/**
+ * Per-page capture options shared by scrape, batch and crawl (for batch and
+ * crawl they apply to every page). A robots override is never one of them:
+ * it names one URL (`ScrapeRequest.robotsOverride`, `BatchStartRequest.robotsOverrides`).
+ * Nor are `includeHtml`, `includeRawHtml`, `includeImages`, `attributes` and
+ * `screenshot`: the `html`, `rawHtml`, `images`, `attributes` and `screenshot`
+ * formats ask for those.
+ */
+export interface PageOptions extends Omit<FetchOptions, 'robotsOverride' | 'includeHtml' | 'includeRawHtml' | 'includeImages' | 'attributes' | 'screenshot' | 'list'> {
+    /**
+     * The whole scrape's deadline in milliseconds, 1 000 to 300 000; default
+     * 300 000. When it fires the result is `partial` with the best content a
+     * rung produced so far, or `failed` with `timeout`, never an error.
+     */
+    timeout?: number;
+    /**
+     * The http lane alone, no browser escalation: a page that needs script
+     * execution returns the http lane's own verdict (a shell is
+     * `failed`/`empty_unverified`, never rendered), the ladder audit records
+     * the rungs it dropped (`ladder_channels_filtered`), and `agentHints` says
+     * when the http lane asked for the browser lane. Not a lane option: the
+     * engine selects channels by it. Default false. A URL the server binds to
+     * the browser lane refuses it with HTTP 400.
+     */
+    fastMode?: boolean;
+    /**
+     * Reuse a stored result of this page fetched at most this many
+     * milliseconds ago, under the same options, instead of fetching it; 0 to
+     * MAX_CACHE_AGE_MS. Default 0: nothing is looked up and the page is
+     * fetched. A reused result says so (`cacheState: "hit"`, `cachedAt`) and
+     * carries the original fetch's evidence; a looked-up page that had none
+     * says `miss`. Not available in mode `authed`.
+     */
+    maxAge?: number;
+    /**
+     * Reuse only a stored result at least this many milliseconds old; 0 to
+     * MAX_CACHE_AGE_MS, at most `maxAge`. Without `maxAge` it looks up a
+     * result of any age from this one on.
+     */
+    minAge?: number;
+    /**
+     * Store this page's result for later reuse when it succeeds. Default true,
+     * except for a request with custom `headers`, which stores only with
+     * `true` (the stored trace keeps their values); mode `authed` never stores.
+     */
+    storeInCache?: boolean;
+    /**
+     * Cache only: answer from a stored result and never fetch; a page with
+     * none is `failed` with `cache_miss`. `maxAge` and `minAge` still bound
+     * the age when given; `maxAge: 0` contradicts it. A crawl in this mode
+     * reads no sitemap, so it needs `sitemap: "skip"`.
+     */
+    lockdown?: boolean;
+}
+/** The largest `maxAge` or `minAge` a request may set: ten years in milliseconds. */
+export declare const MAX_CACHE_AGE_MS = 315360000000;
+/** The cache options of a request, as PageOptions names them. */
+export type CacheOptions = Pick<PageOptions, 'maxAge' | 'minAge' | 'storeInCache' | 'lockdown'>;
+/**
+ * Whether a request looks a page up in the cache: a `maxAge` above 0, a
+ * `minAge` or `lockdown`, unless `maxAge` is 0. Otherwise nothing is looked
+ * up, and the result carries no `cacheState`.
+ */
+export declare function cacheLookupRequested(options: CacheOptions): boolean;
+/**
+ * What the cache did for a result, read from its trace: `hit` with the
+ * reused fetch's time after a `cache_hit` event, `miss` after a
+ * `cache_miss` event, nothing when the cache was not asked. The last such
+ * event decides.
+ */
+export declare function cacheStateOf(trace: readonly TraceEvent[]): Pick<ScrapeMetadata, 'cacheState' | 'cachedAt'>;
+/** The caveats a scrape response may carry for an agent: what to change about the request, in one sentence each. */
+export type AgentHints = readonly string[];
+/**
+ * Who a request is from, for W2L's own records only: the scrape record and
+ * the task checkpoint carry both, the crawl and batch status reports them as
+ * `attribution`, and nothing sent to the target changes. Each is 1 to 100
+ * printable ASCII characters without spaces (`^[\x21-\x7e]+$`).
+ */
+export interface RequestAttribution {
+    /** The client that made the request: the SDK sends `js-sdk@<version>`, the MCP server `mcp-<client name>@<client version>`. */
+    origin?: string;
+    /** The caller's own label for the integration or workflow the request belongs to. */
+    integration?: string;
+}
+/**
+ * The facts of one scrape call, under Firecrawl's names, on the full and
+ * compact scrape responses (`metadata`, beside the page's own declarations)
+ * and on `/fc`. Nothing here is guessed: `proxyUsed` is the route the
+ * answering lane recorded, `timezone` the time zone the browser lane
+ * declares (null on the HTTP lane, where none goes on the wire), and the
+ * concurrency pair says whether the per-origin ceiling held an attempt back.
+ */
+export interface ScrapeMetadata {
+    /** A UUID minted for this call; `GET /v1/scrapes/:id` returns its record. */
+    scrapeId: string;
+    /** The URL as requested (`requestedUrl`). */
+    sourceURL: string;
+    /** The final URL, after redirects (`evidence.finalUrl`). */
+    url: string;
+    /** The status of the response that answered `url` (`evidence.httpStatus`); null when none did. */
+    statusCode: number | null;
+    /** That response's `content-type` header (`evidence.contentType`); null when there was none. */
+    contentType: string | null;
+    /** `operator` when the request went through the server's environment proxy, `user` when the compliance record names the caller's own egress, else null. */
+    proxyUsed: 'operator' | 'user' | null;
+    /** The IANA time zone the browser lane declares; null for an HTTP-lane result. */
+    timezone: string | null;
+    /** True when the per-origin concurrency ceiling (`W2L_PER_HOST_CONCURRENCY`) held an attempt of this scrape back. */
+    concurrencyLimited: boolean;
+    /** Milliseconds the attempts of this scrape waited on that ceiling, cooldown and pacing excluded; 0 when none did. */
+    concurrencyQueueDurationMs: number;
+    /**
+     * Whether the cache answered: `hit` when a stored result was reused (the
+     * rest of the response is that fetch's), `miss` when one was looked up
+     * and none fit. Absent when nothing was looked up (no `maxAge` above 0,
+     * no `minAge`, no `lockdown`): never a guessed `miss`.
+     */
+    cacheState?: 'hit' | 'miss';
+    /** On a hit, when the reused result was fetched (its `evidenceRecord.fetchedAt`). Absent otherwise. */
+    cachedAt?: string;
+}
+/** A scrape response's `metadata`: the page's declarations (all null on a page that was not read as content) and the call's facts. */
+export type ScrapeResponseMetadata = PageMetadata & ScrapeMetadata;
+export interface ScrapeRequest extends PageOptions, RequestAttribution {
+    url: string;
+    mode?: ApiCrawlMode;
+    allowlistedDomains?: readonly string[];
+    formats?: readonly ScrapeFormat[];
+    /** Include outbound links. Kept separate from content formats. */
+    includeLinks?: boolean;
+    /** Omitted preserves the legacy full REST/SDK response. MCP sends false by default. */
+    debug?: boolean;
+    /**
+     * A recorded decision to fetch this URL although its host's robots.txt
+     * disallows it. The reason is required; robots.txt is still read, and the
+     * override is reported in the trace, the warnings and, in the browser lane,
+     * the compliance record. A hosted server refuses the field
+     * (`unsupported_parameter`).
+     */
+    robotsOverride?: RobotsOverride;
+    /**
+     * When W2L is stopped at a check it does not pass (a captcha, a challenge,
+     * a login wall), hand the page to the person in their own Chrome and answer
+     * with the page they get through to (`true`, or `{ waitMs }`: how long to
+     * wait for them, 10 s to 30 min, default 10 min). Offered only by a server
+     * on the person's own machine; refused elsewhere, and with `actions` or a
+     * screenshot (`unsupported_parameter`).
+     */
+    handoff?: {
+        waitMs?: number;
+    };
+}
+/** A recorded robots override for one URL of a batch. */
+export interface RobotsUrlOverride extends RobotsOverride {
+    url: string;
+}
+/** One scrape as the ladder ran it, before the API shapes the response: the result, its routing audit, the request's hints and, once shaped, the `warning` string. */
+export type ScrapeRun = FetchResult & LadderRunAudit & {
+    agentHints?: AgentHints;
+    warning?: string;
+};
+/** The `warnings` as one string, their messages joined with a space (Firecrawl's `warning`); undefined when there are none. */
+export declare function warningOf(warnings: readonly FetchWarning[] | undefined): string | undefined;
+/**
+ * The full scrape response: the run, its `scrapeId`, `metadata` carrying the
+ * call's facts beside the page's declarations, and the snapshot and Evidence
+ * Record set on every response the API sends (see evidenceRecord.ts).
+ */
+export type ScrapeResponse = ScrapeRun & {
+    scrapeId: string;
+    metadata: ScrapeResponseMetadata;
+    snapshot?: CompactScrapeResponse['snapshot'];
+    evidenceRecord?: EvidenceRecord;
+};
+/**
+ * What `GET /v1/scrapes/:id` returns: the record of one scrape call, written
+ * beside the task root (`scrapes/<scrapeId>.json`) before the response was
+ * sent. It holds no page body: the request (header values replaced by their
+ * names), who made it, the verdict and the facts of the fetch.
+ */
+export interface ScrapeRecord extends RequestAttribution {
+    scrapeId: string;
+    /** UTC ISO time the API took the request. */
+    requestedAt: string;
+    /** The parsed request; each custom header's value is replaced by its name. */
+    request: ScrapeRequest;
+    status: FetchResult['status'];
+    failureReason: FetchResult['failureReason'];
+    blockReason: FetchResult['blockReason'];
+    budgetExceeded: FetchResult['budgetExceeded'];
+    lane: FetchResult['lane'];
+    channelsTried: readonly string[];
+    metadata: ScrapeResponseMetadata;
+    snapshot: CompactScrapeResponse['snapshot'];
+    usage: {
+        wallMs: number;
+        totalMs: number;
+        requestCount: number;
+        attemptCount: number;
+        browserMs: number;
+    };
+    warnings?: readonly FetchWarning[];
+    agentHints?: AgentHints;
+}
+export interface CompactScrapeResponse {
+    requestedUrl: string;
+    finalUrl: string;
+    /**
+     * Small capture identity for field audits; the HTML body remains local.
+     * `httpStatus` and `contentType` are the status and `content-type` header
+     * of the response that answered `finalUrl` (`evidence.httpStatus`,
+     * `evidence.contentType`): in the browser lane, of the document the page
+     * shows after a script or a meta refresh moved it on.
+     */
+    snapshot: {
+        rawBodySha256: string | null;
+        artifacts: readonly string[];
+        httpStatus: number | null;
+        contentType: string | null;
+    };
+    /** The result's Evidence Record v1, the same as on the full response. */
+    evidenceRecord: EvidenceRecord;
+    status: FetchResult['status'];
+    failureReason: FetchResult['failureReason'];
+    blockReason: FetchResult['blockReason'];
+    budgetExceeded: FetchResult['budgetExceeded'];
+    retryAt?: number;
+    lane: FetchResult['lane'];
+    formats: readonly ('markdown' | 'html' | 'rawHtml' | 'links' | 'json' | 'images' | 'tables' | 'attributes' | 'screenshot' | 'list')[];
+    markdown?: string | null;
+    /** Present when `html` was asked for, as on the full response; null when the result carries none (a file, a page that was not read as content). */
+    html?: string | null;
+    /** Present when `rawHtml` was asked for, as on the full response; null when the result carries none. */
+    rawHtml?: string | null;
+    links?: readonly string[];
+    /** Present when `images` was asked for and the page was read as content: every image URL of the whole document, as on the full response. */
+    images?: readonly string[];
+    /** Present when `tables` was asked for and the page was read as content: its data tables, as on the full response. */
+    tables?: FetchResult['tables'];
+    /** Present when a `pdf` parser entry asked for `pages` and a PDF's text was read, as on the full response. */
+    pages?: FetchResult['pages'];
+    /** Present when an `attributes` entry was asked for and the page was read as content, as on the full response. */
+    attributes?: FetchResult['attributes'];
+    /** Present when a `list` entry was asked for and the page was read: its records. */
+    list?: FetchResult['list'];
+    /** Present when a `screenshot` entry was asked for, as on the full response: the capture, or null when the browser lane rendered no page or could not capture it. */
+    screenshot?: FetchResult['screenshot'];
+    /** Present when the request ran `actions`: what the steps produced, and the step that failed if one did. */
+    actions?: FetchResult['actions'];
+    document?: Pick<DocumentExtraction, 'title' | 'pageType' | 'strategy' | 'confidence' | 'adapter' | 'adapterValidation'> | null;
+    /** The call's facts (`scrapeId`, `proxyUsed`, the concurrency pair, ...) and the page's own declarations, as on the full response. */
+    metadata: ScrapeResponseMetadata;
+    json?: StructuredExtractionResult | null;
+    /** The file the response was, as on the full response; absent for a web page. */
+    file?: FetchResult['file'];
+    /** The fetch's caveats (a recorded robots override, a suspected client-rendered shell, a thin http answer kept), as on the full response; absent when it had none. */
+    warnings?: FetchResult['warnings'];
+    /** The warnings' messages joined with a space, present exactly when `warnings` is (Firecrawl's `warning`). */
+    warning?: string;
+    /** Present when the request itself left something on the table (`fastMode` declined a browser hop the http lane asked for), as on the full response. */
+    agentHints?: AgentHints;
+    /** A page stopped at a check a person can get through, on a server that hands pages to them: why, and how (`handoff: true`), as on the full response. */
+    handoff?: FetchResult['handoff'];
+    truncated: boolean;
+    truncatedAt: number | null;
+    usage: FetchResult['usage'] & {
+        totalMs: number;
+    };
+    channelsTried: readonly string[];
+}
+/** The events a job webhook can be sent: Firecrawl's four, plus `cancelled`, which W2L tells apart from `failed`. */
+export declare const WEBHOOK_EVENTS: readonly ["started", "page", "completed", "failed", "cancelled"];
+export type WebhookEvent = (typeof WEBHOOK_EVENTS)[number];
+/** The bounds of a job webhook's configuration. */
+export declare const MAX_WEBHOOK_URL_LENGTH = 2048;
+export declare const MAX_WEBHOOK_HEADERS = 32;
+export declare const MAX_WEBHOOK_HEADERS_BYTES = 8192;
+export declare const MAX_WEBHOOK_METADATA_ENTRIES = 32;
+export declare const MAX_WEBHOOK_METADATA_VALUE_LENGTH = 1000;
+export declare const MAX_WEBHOOK_METADATA_BYTES = 8192;
+/**
+ * Where a crawl or batch posts its events (`webhook` on `POST /v1/crawl` and
+ * `POST /v1/batches`; a plain string is `{ url }`). Each event is one durable
+ * delivery with retries, signed when `secretEnv` names an operator secret;
+ * `GET /v1/deliveries?jobId=<taskId>` lists them. The receiver must be https;
+ * a local server also takes plain http to a loopback receiver.
+ */
+export interface WebhookConfig {
+    /** The receiver: an http(s) URL of at most 2048 characters, without credentials or a fragment. */
+    url: string;
+    /**
+     * Headers sent with every delivery, retries included: at most 32, 8 KiB in
+     * all, RFC 7230 token names (lower-cased), values without line breaks.
+     * `content-type`, `content-length`, `host`, `connection`,
+     * `transfer-encoding` and every `x-w2l-*` name are W2L's and refused by
+     * name. Stored in the control database alone, never on the task or in any
+     * response, which show their names only; `secretEnv` is the signing path.
+     */
+    headers?: Readonly<Record<string, string>>;
+    /** Strings echoed as `metadata` in every payload: at most 32, each of at most 1000 characters, 8 KiB in all. */
+    metadata?: Readonly<Record<string, string>>;
+    /** The events to deliver; default all five. A filtered event is never enqueued. */
+    events?: readonly WebhookEvent[];
+    /** An operator `W2L_WEBHOOK_SECRET_*` variable whose value signs each delivery (`x-w2l-timestamp`, `x-w2l-signature`); never a literal. */
+    secretEnv?: string;
+}
+/** The `webhook` field as a request may write it: a URL string, a configuration object, or null for none. */
+export type WebhookOption = string | WebhookConfig | null;
+/**
+ * What a crawl or batch status says about its webhook: the destination
+ * (`GET /v1/deliveries?jobId=`), the receiver as origin and path (no query),
+ * the events taken, and how its deliveries stand. `pending` counts the
+ * deliveries not yet acknowledged, those in flight included.
+ */
+export interface JobWebhookStatus {
+    destinationId: string;
+    url: string;
+    events: readonly WebhookEvent[];
+    pending: number;
+    delivered: number;
+    deadLetter: number;
+}
+export interface CrawlStartRequest extends PageOptions, RequestAttribution {
+    url: string;
+    mode?: ApiCrawlMode;
+    maxPages?: number | null;
+    maxDepth?: number | null;
+    /**
+     * A resume (`POST /v1/crawl/:id/resume`, or a restart) reuses the pages
+     * this crawl already fetched instead of fetching them again. It reaches no
+     * other request's pages: `maxAge` reuses a stored result of any request.
+     */
+    useCached?: boolean;
+    allowlistedDomains?: readonly string[];
+    /** Formats for every page, validated as for scrape. Omitted selects Markdown. */
+    formats?: readonly ScrapeFormat[];
+    /** Include each page's outbound links, like a `links` format. */
+    includeLinks?: boolean;
+    /** Pathname regexes a discovered link must match. The seed URL is always fetched. */
+    includePaths?: readonly string[];
+    /** Pathname regexes that skip a discovered link; they win over includePaths. */
+    excludePaths?: readonly string[];
+    /**
+     * Match includePaths / excludePaths against a discovered link's canonical
+     * URL (scheme, host, path and query) instead of its pathname. Default false.
+     */
+    regexOnFullURL?: boolean;
+    /**
+     * URLs that differ only in their query string are one page: the first
+     * variant seen is fetched, later ones are reported as collapsed. Default false.
+     */
+    ignoreQueryParameters?: boolean;
+    /**
+     * `/a` and `/a/`, `/` and `/index.html`, `www.` and the apex, http and https
+     * name one page: the first variant seen is fetched, later ones are reported
+     * as collapsed. Default true.
+     */
+    deduplicateSimilarURLs?: boolean;
+    /**
+     * Follow links anywhere on the start URL's host. Default false: links on
+     * that host are followed only inside the start URL's path subtree.
+     */
+    crawlEntireDomain?: boolean;
+    /** Follow links to subdomains of the start URL's host (`*.apex`, with one leading `www.` removed). Default false. */
+    allowSubdomains?: boolean;
+    /** Follow links to any host; cannot be combined with allowlistedDomains. Default false. */
+    allowExternalLinks?: boolean;
+    /**
+     * How the crawl uses the site's sitemap: `include` (default) reads the
+     * sitemaps the start URL's robots.txt names, or `/sitemap.xml`, and queues
+     * their URLs ahead of the start page's links; `skip` reads none; `only`
+     * follows no page link, so the pages are the start URL and the sitemap's
+     * entries. Entries pass the same host, subtree, path and depth rules as
+     * links. The files read are listed in the report's `discovery.sitemap`.
+     */
+    sitemap?: SitemapMode;
+    /**
+     * Pages this crawl fetches at once, at most: an integer >= 1, refused above
+     * the service's worker count. It can only lower the crawl's parallelism; the
+     * per-host ceiling and minimum interval still apply. Null or omitted takes
+     * the worker count.
+     */
+    maxConcurrency?: number | null;
+    /**
+     * A client-chosen key, 1 to 200 characters without control characters,
+     * that makes a retried start return the first start's answer instead of a
+     * second job (also the `x-idempotency-key` header on REST). The same key
+     * with a different request is HTTP 409 `conflict`. Keys live 24 hours.
+     */
+    idempotencyKey?: string;
+    /** A receiver for the crawl's events (`started`, one `page` per page recorded, then `completed`, `failed` or `cancelled`); see WebhookConfig. */
+    webhook?: WebhookOption;
+}
+/** What the parser hands the engine: the request plus, from the `/fc` shim, the payload shape its receiver expects. */
+export type ParsedCrawlStartRequest = CrawlStartRequest & {
+    webhookPayloadFormat?: WebhookPayloadFormat;
+};
+/** A map's `limit` when the request names none, and the largest it may name: Firecrawl's documented default and maximum (read 2026-10-03). */
+export declare const DEFAULT_MAP_LIMIT = 5000;
+export declare const MAX_MAP_LIMIT = 100000;
+/** One deadline for the whole map when the request names no `timeout`; a map answers synchronously. */
+export declare const DEFAULT_MAP_TIMEOUT_MS = 60000;
+export declare const MAX_MAP_TIMEOUT_MS = 300000;
+/** A hosted server's map caps: Firecrawl's default limit, and the default deadline, since a map answers synchronously. */
+export declare const HOSTED_MAP_MAX_LIMIT = 5000;
+export declare const HOSTED_MAP_MAX_TIMEOUT_MS = 60000;
+/** POST /v1/map: the URLs of a site from its sitemaps and its start page's links, without fetching each page. */
+export interface MapRequest extends RequestAttribution {
+    url: string;
+    /** standard (default) or research; authed is refused: a map reads public sitemaps and one public page. */
+    mode?: 'standard' | 'research';
+    /** Links returned at most, 1 to MAX_MAP_LIMIT; default DEFAULT_MAP_LIMIT. */
+    limit?: number;
+    /** Milliseconds for the whole map, 1000 to MAX_MAP_TIMEOUT_MS; default DEFAULT_MAP_TIMEOUT_MS. At the deadline the map answers with what it found. */
+    timeout?: number;
+    /**
+     * Keep only the URLs in which every word (1 to MAP_SEARCH_MAX_WORDS words,
+     * 1 to MAP_SEARCH_MAX_CHARS characters, trimmed) appears, case-insensitively,
+     * in the percent-decoded URL or the link's title. A filter, not a ranking:
+     * discovery order is kept, and it runs before `limit` is counted.
+     */
+    search?: string;
+    /** How the map uses the site's sitemap: include (default), skip (the start page alone), only (no page body read). */
+    sitemap?: SitemapMode;
+    /** Admit every host under the start URL's apex (the crawl's allowSubdomains). Default false. */
+    includeSubdomains?: boolean;
+    /** Fold URLs that differ only in their query string into the first one seen; the returned URL has no query. Default false. */
+    ignoreQueryParameters?: boolean;
+    /** The crawl's scope options, under their crawl names and rules. */
+    includePaths?: readonly string[];
+    excludePaths?: readonly string[];
+    regexOnFullURL?: boolean;
+    crawlEntireDomain?: boolean;
+    /** Default true, as on a crawl; a returned http link gives way to its https variant when that comes too, on an origin whose robots.txt the map read anyway and which allows it. */
+    deduplicateSimilarURLs?: boolean;
+}
+/** A map's `search`: at most this many characters after trimming, and this many whitespace-separated words. */
+export declare const MAP_SEARCH_MAX_CHARS = 200;
+export declare const MAP_SEARCH_MAX_WORDS = 10;
+export interface CrawlAccepted {
+    taskId: string;
+    /** Present and true when `idempotencyKey` matched an earlier start and this is its stored answer; nothing was started. */
+    replayed?: boolean;
+}
+/** The request headers REST reads an idempotency key from, merged into the body as `idempotencyKey` before parsing. */
+export declare const IDEMPOTENCY_KEY_HEADERS: readonly ["x-idempotency-key", "idempotency-key"];
+export declare const MAX_IDEMPOTENCY_KEY_LENGTH = 200;
+/** The options a running crawl was started with, as `GET /v1/crawl/active` reports them: its task's stored options plus its page budget. */
+export interface ActiveCrawlOptions {
+    maxPages: number | null;
+    maxDepth: number | null;
+    allowlistedDomains: readonly string[];
+    includePaths: readonly string[];
+    excludePaths: readonly string[];
+    useCached: boolean;
+    sitemap: SitemapMode;
+    ignoreQueryParameters: boolean;
+    deduplicateSimilarURLs: boolean;
+    crawlEntireDomain: boolean;
+    allowSubdomains: boolean;
+    allowExternalLinks: boolean;
+    regexOnFullURL: boolean;
+    maxConcurrency: number | null;
+    /** The per-page options every page of the crawl gets: its formats, `includeLinks` and the page options. */
+    scrapeOptions: PageOptions & {
+        formats: readonly ScrapeFormat[];
+        includeLinks: boolean;
+    };
+}
+/** One crawl this API process is running (a crawl it resumed at startup included); batches are not listed. */
+export interface ActiveCrawl {
+    id: string;
+    /** The start URL. */
+    url: string;
+    status: 'pending' | 'running' | 'paused';
+    /** When the latest attempt started; the task's creation time while no attempt has opened yet. */
+    startedAt: string;
+    /** The latest attempt's pages so far. */
+    pagesFetched: number;
+    options: ActiveCrawlOptions;
+}
+/** `GET /v1/crawl/active`: always 200, with an empty list when nothing runs. */
+export interface ActiveCrawlList {
+    crawls: readonly ActiveCrawl[];
+}
+export interface BatchStartRequest extends PageOptions, RequestAttribution {
+    urls: readonly string[];
+    mode?: ApiCrawlMode;
+    formats?: readonly ScrapeFormat[];
+    includeLinks?: boolean;
+    /** Recorded robots overrides, each for one URL of `urls`. A hosted server refuses the field (`unsupported_parameter`). */
+    robotsOverrides?: readonly RobotsUrlOverride[];
+    /**
+     * Pages of this batch in flight at once, at most: an integer from 1 to 4.
+     * It only lowers the service's worker count (4 locally, 2 on the hosted
+     * MCP host); the per-host ceiling and minimum interval still apply.
+     * Omitted takes the worker count. Stored with the task, so a resumed batch
+     * runs under the same cap.
+     */
+    maxConcurrency?: number;
+    /**
+     * Start the batch with the entries of `urls` that are http(s) URLs and
+     * report the rest as `invalidURLs` instead of refusing the request. A
+     * non-string entry is still refused (`urls[i] must be a string`), and so
+     * is a duplicate: neither is an invalid URL. Default false.
+     */
+    ignoreInvalidURLs?: boolean;
+    /**
+     * Firecrawl's extract scope flags, accepted in their no-op form only: a
+     * batch fetches exactly the URLs given and follows no link, which is what
+     * `false` says. `true` is refused with HTTP 400 naming the alternative (a
+     * crawl's `allowExternalLinks` / `allowSubdomains`; extraction across
+     * links is the M5 multi-URL extract).
+     */
+    allowExternalLinks?: false;
+    includeSubdomains?: false;
+    /**
+     * A client-chosen key, 1 to 200 characters without control characters,
+     * that makes a retried submission return the first one's answer (with
+     * `replayed: true`) instead of a second job; also the `x-idempotency-key`
+     * header on REST. The same key with a different request is HTTP 409
+     * `conflict`. Keys live 24 hours, per task root.
+     */
+    idempotencyKey?: string;
+    /**
+     * The id of an existing batch to add `urls` to instead of starting a new
+     * job. The body may then carry only `urls`, `ignoreInvalidURLs`,
+     * `idempotencyKey`, `robotsOverrides` and the attribution labels: the job's
+     * `mode`, `formats`, `includeLinks`, `maxConcurrency` and page options
+     * stay as they were (`appendToId keeps the job's options; <key> cannot be
+     * changed`). The appended URLs go to the end of the job's list, in order.
+     */
+    appendToId?: string;
+    /** A receiver for the batch's events (`started`, one `page` per item recorded, then `completed`, `failed` or `cancelled`); see WebhookConfig. */
+    webhook?: WebhookOption;
+}
+/** What the parser hands the engine: the request plus, when `ignoreInvalidURLs` was on, the entries it skipped (possibly none), and from a shim the payload shape its receiver expects. */
+export type ParsedBatchStartRequest = BatchStartRequest & {
+    invalidURLs?: readonly string[];
+    webhookPayloadFormat?: WebhookPayloadFormat;
+};
+/**
+ * `POST /v1/batches` 202: the task id and, when `ignoreInvalidURLs` was on,
+ * the entries skipped, possibly none. An append answers with the job's id,
+ * `requested` (the job's URLs after the append) and `appended` (the URLs this
+ * request added); a replayed submission carries `replayed: true`.
+ */
+export interface BatchAccepted extends CrawlAccepted {
+    invalidURLs?: string[];
+    requested?: number;
+    appended?: number;
+}
+export interface BatchStatusResponse extends CrawlReport {
+    requested: number;
+    completed: number;
+    remaining: number;
+    /** Items recorded `success`, `partial` or `empty_verified` (a page read, with or without content), every attempt counted. */
+    succeeded: number;
+    /** Items the errors report lists: `failed`, `blocked`, `cancelled` or `budget_exceeded`, every attempt counted. With one step per URL, `completed` is `succeeded + failed`. */
+    failed: number;
+    /** The cap in force: the request's `maxConcurrency` or the service's worker count, whichever is lower. */
+    maxConcurrency: number;
+    /** The entries `ignoreInvalidURLs` skipped at submission; present exactly when the option was on. */
+    invalidURLs?: readonly string[];
+    /** Items stopped at a check a person can get through in their own Chrome (`POST /v1/batches/:id/handoff`); present on a server that offers the handoff. */
+    waitingForPerson?: number;
+}
+/**
+ * The checks a batch item can be handed to a person for, and the routing
+ * reason each is handed over as: a captcha, a bot check or challenge, a
+ * login wall. A rate limit or a region block is not something a person gets
+ * through in a browser.
+ */
+export declare const HANDOFF_REASONS: Readonly<Record<string, 'captcha_required' | 'bot_gate' | 'login_required'>>;
+/**
+ * `POST /v1/logins/import`: save the person's login to `site` (a domain or a
+ * page URL) from the Chrome they use, as `octocrawl login import` does, on a server
+ * on their machine. `approveTimeoutMs`: how long to wait for them to click
+ * Allow in Chrome, 10 s to 10 min; default 2 min.
+ */
+export interface LoginImportRequest {
+    site: string;
+    approveTimeoutMs?: number;
+}
+export declare function parseLoginImportRequest(body: unknown): LoginImportRequest;
+/** The localStorage a saved login holds: the origins, and how many items in all; never a value. */
+export interface LoginStorage {
+    origins: string[];
+    itemCount: number;
+}
+/** A login saved for a domain: never its cookies or storage values, only how many and the hash a record names it by. */
+export interface SavedLogin {
+    domain: string;
+    savedAt: string;
+    cookieCount: number;
+    /** The localStorage saved with it, read from the site's tabs open in Chrome when it was imported; null for none. */
+    localStorage: LoginStorage | null;
+    sessionSha256: string;
+}
+/** What an import saved, and whether the site's localStorage was read: false when no tab of the site was open in Chrome. */
+export interface LoginImportResponse extends SavedLogin {
+    localStorageRead: boolean;
+    /** The origins of the site's open tabs whose localStorage Chrome did not give (a tab that crashed or was discarded): saved without it. */
+    localStorageUnread: string[];
+    /** Why each of localStorageUnread was not read, a tab at a time: the request to Chrome that failed (`Target.attachToTarget`, `Page.getFrameTree` or `DOMStorage.getDOMStorageItems`) and Chrome's answer, or the wait that ran out. */
+    localStorageUnreadReasons: {
+        origin: string;
+        step: string;
+        error: string;
+    }[];
+}
+/** `POST /v1/batches/:id/handoff`: how long to wait for the person on each page, 10 s to 30 min; default 10 min. */
+export interface BatchHandoffRequest {
+    waitMs?: number;
+}
+export declare const MAX_HANDOFF_WAIT_MS = 1800000;
+export declare function parseBatchHandoffRequest(body: unknown): BatchHandoffRequest;
+/**
+ * What a handoff did: each item it handed over, in order, and whether the
+ * person got it through. An item that was through is read in their browser
+ * and its result replaces the stopped one (`status`, the item's new one); an
+ * item that was not (they did not get through in time, closed its tab, or it
+ * ended off its site) keeps its stopped result, with `reason` saying why.
+ */
+export interface BatchHandoffResponse {
+    id: string;
+    handedOff: number;
+    through: number;
+    notThrough: number;
+    items: Array<{
+        id: string;
+        url: string;
+        through: boolean;
+        status: string;
+        reason?: string;
+    }>;
+}
+/** The step statuses `GET /v1/batches/:id/errors` lists: the same ones `/v1/crawl/:id/errors` does. */
+export declare const BATCH_ERROR_STATUSES: readonly ["failed", "blocked", "cancelled", "budget_exceeded"];
+export type BatchErrorStatus = (typeof BATCH_ERROR_STATUSES)[number];
+/**
+ * One item of a batch that did not succeed, under Firecrawl's names (`id`,
+ * `timestamp`, `url`, `code`, `error`) with W2L's status vocabulary beside
+ * them: `status` and `code` (the `failureReason`, `blockReason` or
+ * `budgetExceeded` the status carries, else the status itself) are the same
+ * values the item on `/items` carries.
+ */
+export interface BatchErrorItem {
+    /** The item's step id, as on `GET /v1/batches/:id/items`. */
+    id: string;
+    /** When the item was recorded (its `createdAt`). */
+    timestamp: string;
+    url: string;
+    status: BatchErrorStatus;
+    code: string;
+    /**
+     * A sentence for a reader: the result's first warning when it has one,
+     * else `<status>: <code>`, with ` (HTTP <n>)` when the status is known and,
+     * for a robots.txt refusal, the rule that applied.
+     */
+    error: string;
+    httpStatus: number | null;
+}
+/** `GET /v1/batches/:id/errors`: the batch's errors across every attempt, one page at a time, and the URLs robots.txt refused (every attempt, not paginated). */
+export interface BatchErrorsResponse {
+    errors: BatchErrorItem[];
+    /**
+     * URLs of the batch whose result is `policy_denied` by a `robots_disallowed`
+     * trace event that no recorded override set aside. A governance or SSRF
+     * refusal is `policy_denied` too but is not robots.txt, and stays out of
+     * this list (it is still in `errors`).
+     */
+    robotsBlocked: string[];
+    nextCursor: string | null;
+    hasMore: boolean;
+}
+/** The page of errors asked for: `limit` 1 to 1000 (default 1000: a batch has at most 1000 URLs and errors carry no bodies). */
+export interface BatchErrorsQuery {
+    cursor?: string;
+    limit?: number;
+}
+export declare const BATCH_ERRORS_MAX_LIMIT = 1000;
+export type CrawlStatusResponse = CrawlReport;
+export interface CrawlPageQuery {
+    attemptId?: string;
+    cursor?: string;
+    limit?: number;
+    debug?: boolean;
+    /** List the pages whose content repeated an earlier page's (status `duplicate`) too; left out by default. */
+    includeDuplicates?: boolean;
+}
+export type CrawlPagesResponse = CrawlPageList<CrawlPage>;
+export type CrawlErrorsResponse = CrawlPageList<CrawlError>;
+/**
+ * The events a job stream sends (`GET /v1/crawl/:id/events`,
+ * `GET /v1/batches/:id/events` as server-sent events, and the same routes'
+ * `/ws` as WebSocket frames): `catchup` with the job's report as the stream
+ * opens, one `document` per page recorded (the compact page the items routes
+ * list, with the step cursor the listing routes take, so `after=<cursor>` or
+ * `Last-Event-ID` resumes a stream), `snapshot` with the report after each
+ * page, `done` with the terminal report, `error` with `{ code, message }`.
+ * A document is sent once per step id on one stream; a client that resumes
+ * or switches transports deduplicates by it.
+ */
+export declare const JOB_STREAM_EVENTS: readonly ["catchup", "document", "snapshot", "done", "error"];
+export type JobStreamEventType = (typeof JOB_STREAM_EVENTS)[number];
+export type JobStreamReport = CrawlReport | BatchStatusResponse;
+export type JobStreamFrame = {
+    type: 'catchup';
+    data: JobStreamReport;
+} | {
+    type: 'document';
+    data: CrawlPage;
+    cursor: string;
+} | {
+    type: 'snapshot';
+    data: JobStreamReport;
+} | {
+    type: 'done';
+    data: JobStreamReport;
+} | {
+    type: 'error';
+    error: {
+        code: string;
+        message: string;
+    };
+};
+/**
+ * The WebSocket subprotocol a client presents its bearer token in, since the
+ * WebSocket API sets no headers: `w2l.token.<token>`, echoed back as the
+ * selected protocol. A token must then be made of the characters a
+ * subprotocol name allows (RFC 6455 token characters); the SDK falls back to
+ * the SSE route, which carries the Authorization header, for any other token.
+ */
+export declare const WS_TOKEN_PROTOCOL_PREFIX = "w2l.token.";
+export declare function isApiCrawlMode(value: string): value is ApiCrawlMode;
+export declare function defaultApiMode(mode: CrawlMode | undefined): ApiCrawlMode;
+/**
+ * The one set of request-error codes, shared by the REST API, /fc, the SDK and
+ * MCP. A request error means W2L refused or failed the request itself; what
+ * happened to a fetched page is its result status (blocked, failed, ...), not
+ * one of these.
+ */
+export declare const API_ERROR_CODES: readonly ["invalid_json", "invalid_request", "unsupported_parameter", "unsupported_format", "unauthorized", "not_found", "conflict", "internal_error"];
+export type ApiErrorCode = (typeof API_ERROR_CODES)[number];
+/** The HTTP status each code is returned with. */
+export declare const API_ERROR_STATUS: Readonly<Record<ApiErrorCode, 400 | 401 | 404 | 409 | 500>>;
+export declare function isApiErrorCode(value: unknown): value is ApiErrorCode;
+/** What a request named that W2L cannot honour, as sent (for example `scrapeOptions.actions`). */
+export interface ApiErrorDetails {
+    parameters?: readonly string[];
+    formats?: readonly string[];
+}
+/** Native error body. /fc sends the same fields after `success: false`, with the hints as `agent_hints`. */
+export interface ApiErrorBody {
+    error: string;
+    code: ApiErrorCode;
+    details?: ApiErrorDetails;
+    /** For a refused option W2L does not offer: the supported route, one sentence each. */
+    agentHints?: AgentHints;
+}
+/**
+ * The code of the one answer that is not a request error: the request was
+ * well formed, and the caller is over the server's per-minute budget
+ * (HTTP 429, `Retry-After`). It is not one of API_ERROR_CODES, which name
+ * what was wrong with a request.
+ */
+export declare const RATE_LIMITED_CODE: "rate_limited";
+export declare const RATE_LIMITED_STATUS = 429;
+/** The native 429 body; /fc sends `{ success: false, error, code, agent_hints }` with the same header. */
+export interface RateLimitedBody {
+    error: string;
+    code: typeof RATE_LIMITED_CODE;
+    /** Seconds until the window admits a request again, at least 1; also the `Retry-After` header. */
+    retryAfterSeconds: number;
+    agentHints: AgentHints;
+}
+export declare function rateLimitedBody(perMinute: number, retryAfterSeconds: number): RateLimitedBody;
+export declare class RequestError extends Error {
+    readonly code: 'invalid_request' | 'unsupported_parameter' | 'unsupported_format';
+    readonly details?: ApiErrorDetails | undefined;
+    /** The supported route, when the refused option is one W2L does not offer. */
+    readonly agentHints?: AgentHints | undefined;
+    readonly status = 400;
+    constructor(message: string, code?: 'invalid_request' | 'unsupported_parameter' | 'unsupported_format', details?: ApiErrorDetails | undefined, 
+    /** The supported route, when the refused option is one W2L does not offer. */
+    agentHints?: AgentHints | undefined);
+}
+/** The hints a refusal carries for the options W2L does not offer: the next honest step, never a way around the refusal. */
+export declare const REFUSAL_HINTS: {
+    readonly stealth: "W2L does not offer a stealth mode or stealth proxies; a proxy or session you own (mode authed) is the supported route";
+    readonly ignoreRobotsTxt: "robots.txt is always read; a robotsOverride with a recorded reason fetches one URL past its rule, on the record";
+    readonly hostedSkipTlsVerification: "a hosted server verifies every certificate; run W2L locally to use skipTlsVerification, which is recorded in the trace and a tls_unverified warning";
+    readonly useIndex: "W2L keeps no URL index: a map reads the sitemaps the site declares and its start page, on the record; crawl reads further pages";
+    readonly actions: "actions run on scrape and batch, where each page named gets the same steps; a crawl or a map does not take them";
+};
+/** The hint for a refused request key, or null when the key has none (an option W2L simply does not know). */
+export declare function refusalHint(key: string, value: unknown): string | null;
+export declare const PAGE_KEYS: readonly ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown"];
+export declare const ATTRIBUTION_KEYS: readonly ["origin", "integration"];
+export declare const SCRAPE_KEYS: readonly ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const CRAWL_SCOPE_KEYS: readonly ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
+export declare const CRAWL_KEYS: readonly ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", "regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks", "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const BATCH_KEYS: readonly ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+/** What a batch body may carry beside `appendToId`: the job's own options are not among them (the scope no-ops change nothing, so they may come along). */
+export declare const BATCH_APPEND_KEYS: readonly ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", "origin", "integration"];
+/** The scope options a map takes under their crawl names; allowSubdomains is includeSubdomains on a map, and allowExternalLinks is not offered. */
+export declare const MAP_SCOPE_KEYS: readonly ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
+/** What a map takes. No page option (headers, mobile, skipTlsVerification, formats, ...): a map has nothing to loosen. */
+export declare const MAP_KEYS: readonly ["url", "mode", "limit", "timeout", "search", "sitemap", "includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs", "includePaths", "excludePaths", "origin", "integration"];
+/** Why a webhook header (lower-cased name) cannot be sent, or null when it can: W2L's own and the transport's names are reserved. */
+export declare function webhookHeaderRefusal(name: string): string | null;
+/**
+ * The `webhook` of a crawl or batch request: a URL string is `{ url }`, null
+ * or undefined is none, an object takes url, headers, metadata, events and
+ * secretEnv and nothing else (`unknown webhook option: <key>`). The URL must
+ * be http(s) of at most 2048 characters without credentials or a fragment;
+ * whether http is admitted (a loopback receiver of a local service) and
+ * whether a hosted server takes the address is the engine's, mode-aware check.
+ */
+export declare function readWebhook(value: unknown): WebhookConfig | undefined;
+/** The smallest window a screenshot may ask for; the largest is the declared screen (`BROWSER_FINGERPRINT.screen`, 1920x1080). */
+export declare const MIN_SCREENSHOT_VIEWPORT: {
+    readonly width: 320;
+    readonly height: 240;
+};
+/** Largest number of custom request headers, and the longest value, a request may carry. */
+export declare const MAX_REQUEST_HEADERS = 32;
+export declare const MAX_REQUEST_HEADER_VALUE_LENGTH = 4096;
+/**
+ * Why a request header (lower-cased name) cannot be sent on a caller's
+ * behalf, or null when it can. The User-Agent and the client hints are
+ * W2L's declared identity, which no option overrides; credentials belong to
+ * the authed session path, which the record names; transport headers are
+ * the lane's. The lanes apply the same rule to what reaches them.
+ */
+export declare function headerRefusal(name: string): string | null;
+/**
+ * Custom request headers: an object of at most 32 string values, each name
+ * an RFC 7230 token given once (lower-cased here), each value at most 4096
+ * characters without CR, LF or NUL. A name the lane cannot send on the
+ * caller's behalf (headerRefusal) is refused by name.
+ */
+export declare function readHeaders(value: unknown, name?: string): Readonly<Record<string, string>> | undefined;
+export declare function parseScrapeRequest(body: unknown): ScrapeRequest;
+export declare function parseCrawlStartRequest(body: unknown): CrawlStartRequest;
+/**
+ * A map request: url, mode (standard or research), limit, timeout, search,
+ * sitemap, includeSubdomains, the crawl's scope options under their crawl
+ * names (ignoreQueryParameters, includePaths, excludePaths, regexOnFullURL,
+ * crawlEntireDomain, deduplicateSimilarURLs), origin and integration.
+ * Anything else is refused by name, `useIndex` with the supported route;
+ * nothing is silently ignored.
+ */
+export declare function parseMapRequest(body: unknown): MapRequest;
+/**
+ * A batch entry that is not an http(s) URL is refused by its index
+ * (`urls[2] must be http(s)`), or with `ignoreInvalidURLs` collected into
+ * `invalidURLs` instead; an entry that is not a string is refused either
+ * way. The 1..1000 cap counts the submitted entries; `requested` later
+ * counts the valid ones. With `appendToId` the body may carry only the
+ * entries to add and what binds to them: an option of the job itself is
+ * refused by name (`appendToId keeps the job's options; formats cannot be
+ * changed`).
+ */
+export declare function parseBatchStartRequest(body: unknown): ParsedBatchStartRequest;
+/** `GET /v1/batches/:id/errors`: `cursor` and `limit` (1 to 1000) from the query string. */
+export declare function parseBatchErrorsQuery(query: Record<string, string | undefined>): BatchErrorsQuery;
+export declare function parseCrawlPageQuery(query: Record<string, string | undefined>): CrawlPageQuery;
+//# sourceMappingURL=api.d.ts.map