@octocrawl/sdk 0.3.0

package/dist/types/cjs/contracts/file.d.ts

@@ -0,0 +1,117 @@
+/**
+ * A response W2L took for a file rather than a web page: a PDF, CSV, JSON,
+ * plain-text, XLSX, XLS or ZIP file. The bytes are saved as received, with
+ * their SHA-256 and size, and never sent to the browser as a page would be.
+ * Types and the size cap only; detection lives in @w2l/extract-tf, saving in
+ * @w2l/bench.
+ */
+import type { NetworkPolicy } from './policy.js';
+export declare const FILE_KINDS: readonly ["pdf", "csv", "json", "text", "xlsx", "xls", "zip"];
+export type FileKind = (typeof FILE_KINDS)[number];
+/** The operator cap on a file's size when `W2L_MAX_FILE_BYTES` is not set: 50 MiB. */
+export declare const DEFAULT_MAX_FILE_BYTES: number;
+/** The largest cap `W2L_MAX_FILE_BYTES` may set: 500 MiB. A file is held in memory while it is hashed and read. */
+export declare const MAX_FILE_BYTES_CEILING: number;
+/** What the PDF declares about itself; null where it declares nothing (pdfToMarkdown's `info`). */
+export interface PdfDocumentInfo {
+    title: string | null;
+    author: string | null;
+    subject: string | null;
+    keywords: string | null;
+    creator: string | null;
+    producer: string | null;
+    /** PDF date strings as declared, e.g. `D:20240315120000+01'00'`. */
+    creationDate: string | null;
+    modificationDate: string | null;
+    language: string | null;
+    pdfVersion: string | null;
+    encrypted: boolean;
+}
+/** The text of a PDF as W2L read it into `markdown`. */
+export interface FilePdfText {
+    /** Pages in the file; null when it could not be opened. */
+    pageCount: number | null;
+    /** Pages read into `markdown`, from the first. */
+    pagesRead: number;
+    /**
+     * Every page read: its 1-based number, the page label the PDF declares
+     * (a printed number such as "xii"), and the offsets of its text in
+     * `markdown` (`markdown.slice(start, end)`), which follows the line
+     * `<!-- page N -->`.
+     */
+    pages: readonly {
+        number: number;
+        label: string | null;
+        start: number;
+        end: number;
+    }[];
+    info: PdfDocumentInfo | null;
+    /**
+     * `tables_unverified` on every PDF with text; `no_text_layer` per page
+     * without text (no OCR is run); `page_error`, `page_cap`, `time_budget`.
+     */
+    warnings: readonly {
+        code: string;
+        message: string;
+        page?: number;
+    }[];
+    /** Why no text could be read at all (`not_pdf`, `encrypted`, `malformed`, `time_budget`); null otherwise. */
+    error: {
+        code: string;
+        message: string;
+    } | null;
+}
+export interface FileWarning {
+    /**
+     * `text_not_decoded`: the bytes are not valid text in their declared or
+     * default (UTF-8) encoding, so no text is returned. `pdf_not_parsed`: the
+     * request's `parsers` named no `pdf` entry, so a PDF's text was not read.
+     */
+    code: 'text_not_decoded' | 'pdf_not_parsed';
+    message: string;
+}
+export interface FileDescription {
+    kind: FileKind;
+    /**
+     * `content_type`: the response's Content-Type named the kind. `content`:
+     * the bytes did, when the Content-Type did not (a `%PDF-` or ZIP header
+     * under `application/octet-stream`, or no Content-Type at all).
+     */
+    detectedBy: 'content_type' | 'content';
+    /** The Content-Type header as received; null when the response had none. */
+    contentType: string | null;
+    /** The Content-Length the server declared; null when it declared none. */
+    declaredBytes: number | null;
+    /** The size cap that applied: the operator's `W2L_MAX_FILE_BYTES`, or a request's lower `maxFileBytes`. */
+    maxBytes: number;
+    /** Bytes received, all of them; null when the file was not read (over the cap). */
+    bytes: number | null;
+    /** SHA-256 (hex) of the bytes received; null when not read. */
+    sha256: string | null;
+    /**
+     * Where the bytes were saved, as received: `<task root>/files/<sha256>.<ext>`.
+     * Null when they were not saved (over the cap, or no file store is configured).
+     */
+    path: string | null;
+    /**
+     * What `markdown` carries: `pdf_text` (the PDF's text layer, one
+     * `<!-- page N -->` marker per page), `text` (a CSV, JSON or plain-text
+     * file's text as received), or null (no text: binary kinds, or text that
+     * could not be decoded).
+     */
+    markdownFrom: 'pdf_text' | 'text' | null;
+    /** The encoding text was decoded with (the declared charset, else UTF-8); null without text. */
+    encoding: string | null;
+    warnings: readonly FileWarning[];
+    /** PDF files that were read; null otherwise. */
+    pdf: FilePdfText | null;
+}
+/**
+ * The operator's file cap from `W2L_MAX_FILE_BYTES` (bytes, an integer from 1
+ * to MAX_FILE_BYTES_CEILING), else `fallback`. Anything else is an error, so
+ * a mistyped cap stops the service at start instead of being ignored.
+ */
+export declare function maxFileBytesFromEnv(env: Readonly<Record<string, string | undefined>>, fallback?: number): number;
+/** The cap for one fetch: the policy's (operator) cap, lowered by the request's `maxFileBytes` when it sets one. */
+export declare function fileByteCap(policy: Pick<NetworkPolicy, 'maxFileBytes'>, requested?: number): number;
+//# sourceMappingURL=file.d.ts.map

package/dist/types/cjs/contracts/firecrawl.d.ts

@@ -0,0 +1,258 @@
+/**
+ * Firecrawl v1 scrape/crawl snapshot, frozen 2026-09-18, and its map path (added 2026-10-03).
+ *
+ * A one-shot migration shim: map the main paths onto the native
+ * contract. Not a compatibility layer. A parameter or format the shim cannot
+ * honour is rejected by name (HTTP 400, success: false), never ignored.
+ */
+import type { AgentHints, CrawlAccepted, MapRequest, ParsedCrawlStartRequest, ScrapeMetadata, ScrapeRequest, ScrapeResponse } from './api.js';
+import type { MapResponse } from './map.js';
+import type { CrawlReport } from './crawl.js';
+import type { JobWebhookEnvelope } from './delivery.js';
+import type { FetchResult } from './result.js';
+import type { StepRecord, StepStatus, TaskStatus } from './checkpoint.js';
+export declare const FIRECRAWL_SHIM_SNAPSHOT: {
+    capturedAt: string;
+    apiVersion: "v1";
+    docs: {
+        scrape: string;
+        crawl: string;
+        crawlStatus: string;
+        map: string;
+    };
+    paths: readonly ["/scrape", "/crawl", "/crawl/:id", "/map"];
+    notCovered: readonly ["search", "interact", "agent", "monitor", "extract"];
+};
+export declare const FIRECRAWL_SHIM_DIFFS: readonly ["Challenge / block pages are success: false (Firecrawl often returns them as success markdown).", "A page with no main content is success: false (failed: empty_unverified) with the whole page in data.markdown as evidence; with onlyMainContent: false it is success: true.", "A page whose server HTML is a shell for data its scripts fill in is fetched again on the browser rung, and the rendered page is the answer when it holds more; otherwise the HTTP page is returned with client_rendered_suspected and low_content_yield warnings on the native response, whose messages /fc passes through as data.warning (one string, joined with a space), as it does every native warning. Firecrawl renders every page in a browser.", "No fire-engine, proxy pools or JSON extract.", "actions (scrape only; a crawl's scrapeOptions.actions is refused) run on the local browser rung alone, which such a request selects, after load, stability and waitFor and before the formats are read: wait (milliseconds up to 60000, or a selector, waited for up to 60 s within the scrape's timeout), click (all: true clicks every match), write (into the focused element), press, scroll (one screen up or down, of the page or the element a selector names), screenshot, scrape, executeJavascript (a function body; return gives the value) and pdf; at most 50 steps. data.actions holds screenshots and pdfs as data: URIs (Firecrawl returns URLs), scrapes as { url, html } and javascriptReturns as { type, value }. A step that fails stops the steps after it: success is false with data.actions.failed naming the step, its code and message, and data.markdown is the page as it stood. A step that leads the page to a URL robots.txt or the egress policy refuses fails with navigation_refused and that page is not read. A hosted server refuses actions, and the cache is not used with them.", "An omitted maxAge reuses nothing: every page is fetched live unless the request sets maxAge above 0, minAge or lockdown (Firecrawl reuses its own index by default; its Python SDK sends maxAge 4 hours). A reused page is one W2L itself stored, under the same options, on this server (its task root), never a shared index; only a success is stored, and data.metadata says cacheState hit with cachedAt (its fetch time) or miss when one was looked up. lockdown with no stored result is HTTP 404 SCRAPE_LOCKDOWN_CACHE_MISS on scrape, and nothing is fetched; a crawl in lockdown needs sitemap skip, and each page with no stored result is failed with cache_miss. Mode authed neither stores nor reuses. A Firecrawl body never sets useCached, W2L's reuse of a crawl's own pages on resume.", "Omitted limit / maxDepth stay unbounded on a local server; a hosted server takes its crawl limit for an omitted or null limit and refuses a larger one. Firecrawl defaults are 10000 / 10.", "maxDepth counts link hops from the start URL (Firecrawl calls that maxDiscoveryDepth); Firecrawl maxDepth counts URL path depth.", "Crawl start is mapped onto native POST /v1/crawl; the shim itself returns 200 {success,id,url}.", "creditsUsed and expiresAt are null: W2L counts no credits and keeps crawl results until their task directory is deleted.", "Crawl status describes the latest attempt: completed counts its successful pages, total adds its failed, blocked and duplicate pages and, while this API process runs the crawl, the pages in flight and queued (null for a paused crawl); data lists the failed and blocked pages too (with metadata.error) but not the duplicates, whose content is an earlier entry's, up to 100 per response (limit 1 to 1000) with next carrying a W2L cursor; skip is rejected.", "Scrape maps url, formats, onlyMainContent, includeTags, excludeTags, waitFor, timeout, headers, mobile, skipTlsVerification, fastMode, blockAds, removeBase64Images, maxAge, minAge, storeInCache, lockdown, actions, origin and integration; crawl maps url, limit (as maxPages), maxDepth, includePaths, excludePaths, regexOnFullURL, ignoreQueryParameters, deduplicateSimilarURLs, crawlEntireDomain (and its v1 name allowBackwardLinks), allowSubdomains, allowExternalLinks, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only), maxConcurrency, origin, integration and the same scrapeOptions (applied to every page). The formats are markdown, links, html, rawHtml, images, screenshot (also screenshot@fullPage, and { type: \"screenshot\", fullPage, quality, viewport }) and an { type: \"attributes\", selectors } entry; other formats and parameters the shim does not map (proxy, location, json, ...) are rejected by name with HTTP 400 and success: false; a refusal of stealth, proxy: stealth or enhanced, or ignoreRobotsTxt names the supported route in agent_hints.", "A crawl follows links inside the start URL's path subtree on its host and www twin by default (crawlEntireDomain false), folds /a and /a/, / and /index.html, www and apex, http and https into one page (deduplicateSimilarURLs true) and reports every collapsed or refused link in the native crawl status (discovery) and each page's trace (links_offered); allowSubdomains takes every host under the start URL's apex (no public-suffix list), allowExternalLinks every host, each page with its own robots.txt read.", "sitemap (default include, as in Firecrawl) reads the sitemaps the start URL's robots.txt names, or /sitemap.xml, with the crawl's own http identity, robots.txt verdict, SSRF checks and proxy, and queues their URLs ahead of the start page's links under the same host, subtree, path and depth rules; only follows no page link; skip reads none. The native crawl status lists every sitemap file read, refused or unreadable in discovery.sitemap; the shim's status carries nothing of it, and sitemap fetches have no signed compliance record. maxConcurrency caps the pages one crawl fetches at once, at most the service's worker count (HTTP 400 above it), and never raises the per-host ceiling.", "screenshot (data.screenshot, a data:image/png;base64 string, or image/jpeg with quality 1 to 100) is captured on the local browser rung alone, which such a request selects (no http attempt; a server without a browser rung refuses the format with HTTP 400): after load, stability and waitFor, before the DOM is read, CSS-pixel sized at the declared 1280x800 viewport (device scale factor 2 is declared, not baked into the image) or at the viewport asked for (integers 320..1920 by 240..1080, within the declared screen; a window size, not a change of identity); fullPage captures the document's whole height at that width without scrolling first, so sections a page loads on scroll may show unloaded. A capture the browser could not make leaves data.screenshot null with a screenshot_unavailable warning while the page stands; a file or a page that was not rendered has null too. Firecrawl captures at its own viewport and may return a URL instead of the image.", "images (data.images) lists every image URL of the whole document as received: img src and srcset candidates, picture sources, lazy data-src/data-srcset/data-lazy-src/data-original, video posters, image_src links, og:image and twitter:image, absolute http(s) with the fragment stripped, each once, in document order, data: URIs left out; includeTags, excludeTags and onlyMainContent do not narrow it. attributes (data.attributes) gives, per selector, the named attribute's values as written, elements without it skipped; a selector W2L does not match is HTTP 400 by name, as for includeTags. Both are absent for a file and for a page that is success: false. removeBase64Images (default true) keeps an image's alt text where Firecrawl writes a (<Base64-Image-Removed>) placeholder; false keeps the data: URI in the Markdown.", "origin (the Firecrawl SDKs' client label) and integration are stored, not echoed: the scrape record (GET /v1/scrapes/:id) and the crawl task carry them, and nothing sent to the target changes.", "data.metadata carries scrapeId (a UUID per call, which GET /v1/scrapes/:id looks up), proxyUsed (operator for the server's environment proxy, user for the caller's own egress, else null), timezone (the browser rung's declared zone, null on the HTTP rung), creditsUsed: null (W2L counts no credits), concurrencyLimited and concurrencyQueueDurationMs (whether and how long the per-origin ceiling held the fetch back), and cacheState and cachedAt when the cache was asked (never a guessed miss).", "A page whose result W2L has advice about (a login wall, a robots.txt rule, a gate, a cut, a script-filled shell) carries data.agent_hints, one sentence each; the native response calls them agentHints. A request refused for an option W2L does not offer carries agent_hints in the error envelope, and a caller over the server's per-minute rate limit gets HTTP 429 { success: false, error, code: rate_limited, agent_hints } with Retry-After.", "headers never override the User-Agent, the client hints, a credential (authorization, cookie) or a transport header: such a header is HTTP 400 naming it, where Firecrawl sends it. The headers go to the requested origin after the declared identity and are on the record (the trace, the browser lane's signed sentHeaders); both rungs withhold them from a redirect hop to another origin and say so (custom_headers_withheld).", "mobile selects a declared Android Chrome identity (User-Agent, client hints, 412x915 viewport, touch) that robots.txt is evaluated against and the record carries; the page is whatever the site serves to it, with no DOM rewriting. It is refused with mode research.", "skipTlsVerification relaxes certificate verification for one local request and its robots.txt lookup, recorded in the trace (tls_verification_skipped) and a tls_unverified warning the native response carries; a hosted W2L refuses it with HTTP 400. Without it a bad certificate is success: false with failed: tls_error. Firecrawl's Python SDK sends true by default; W2L verifies by default.", "fastMode keeps the http rung alone: a page that needs scripts is success: false with failed: empty_unverified, never rendered; waitFor has no effect under it. Firecrawl's fast mode still renders.", "blockAds (default true) aborts requests to a bundled list of about 50 ad-serving hosts on the local browser rung and removes ad and cookie-banner elements before extraction; false keeps them. The list is curated, not EasyList: ads from hosts outside it are not blocked.", "html is the cleaned HTML the markdown is written from: the main content, the whole page without scripts, styles, form controls and embedded media when onlyMainContent is false, or a <body> holding the includeTags elements. rawHtml is the page as the answering rung received it: the response body on the HTTP rung, the rendered DOM on a browser rung. Both are null for a file and for a page that is success: false.", "includeTags keeps only the named elements, in document order, whatever onlyMainContent says; excludeTags removes elements from the main content, the whole page and an includeTags selection. A selector that does not parse, or that uses a sibling combinator, a positional pseudo-class, :has() or another pseudo-class W2L does not match, is rejected with HTTP 400.", "An omitted timeout stays 300000 ms (Firecrawl: 30000). A timeout is answered with HTTP 200: success: true with the content fetched so far (native status partial), or success: false with failed: timeout; Firecrawl answers it with an error.", "waitFor skips the HTTP rung, which cannot run scripts, and starts at the browser rung; the wait counts toward timeout.", "metadata has title, description, language, keywords, robots and favicon only when the page declares them, and the Open Graph (ogTitle, ogDescription, ogUrl, ogImage, ogAudio, ogVideo, ogDeterminer, ogLocale, ogLocaleAlternate, ogSiteName), Dublin Core (dcTermsCreated, dcDateCreated, dcDate, dcTermsType, dcType, dcTermsAudience, dcTermsSubject, dcSubject, dcDescription, dcTermsKeywords) and article (publishedTime, modifiedTime, articleTag, articleSection) tags under Firecrawl's names, each only when the page states it, as written (no date normalisation, no fallback from another tag); twitter:* and other meta tags are not passed through, and a failed or blocked page has none.", "A PDF answers success: true with its text layer as markdown and metadata.numPages (the document's page count). parsers maps Firecrawl's pdf entry (the string or { type: \"pdf\", mode, maxPages, pages, pageMarkers }); mode fast and auto both read the text layer, and mode ocr and the image parser are refused by name. pageMarkers is false unless asked, as on Firecrawl; with true W2L writes a <!-- page N --> line before each page (Firecrawl writes --- and the marker between pages). pages: true adds data.pages, [{ pageNumber, markdown }]. maxPages (1 to 10000) reads the first pages, and a cut it asked for stays success: true. parsers [] or v1 parsePDF false reads no PDF: success: true with markdown null and the file saved as received. A PDF without a text layer is success: false with failed: empty_unverified (no OCR). CSV, JSON and text files give their text as received; XLSX, XLS and ZIP files are success: true with markdown null. A file over W2L_MAX_FILE_BYTES is success: false with failed: body_too_large.", "Map (POST /fc/v1/map) maps url, search, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only; both v1 flags true is HTTP 400), includeSubdomains, ignoreQueryParameters, limit (1 to 100000, default 5000), timeout (1000 to 300000 ms for the whole map, default 60000; Firecrawl documents no default), origin and integration onto native POST /v1/map, and answers 200 { success: true, id, links: [url strings], warning?, agent_hints? }, or 200 { success: false, id, error, links: [] } when the map found nothing because a source failed or its deadline passed; useIndex, location, ignoreCache, threatProtection and auditMetadata are refused by name (useIndex with the hint that W2L keeps no URL index). Omitted options take W2L's defaults: includeSubdomains and ignoreQueryParameters are false, where Firecrawl v2 documents true for both. search keeps the URLs in which every word appears in the decoded URL or the title in hand, in discovery order; Firecrawl orders by relevance. A map reads the sitemaps the site declares and one page body (the start URL, http rung only), so a site without a sitemap maps only its start page's links; a title is the start page's own, an anchor's text or a sitemap's <news:title>, never fetched from the target; robots-disallowed URLs are left out and counted on the native response (GET /v1/maps/:id), which also records every sitemap file read.", "A crawl's webhook (a URL string or { url, headers, metadata, events }) is mapped onto the native webhook and its receiver gets Firecrawl's payload shape: { success, type: crawl.started | crawl.page | crawl.completed | crawl.failed, id, data: [page], metadata, error? }, one durable delivery per event with retries, every request carrying x-w2l-event-id, x-w2l-event-version and x-w2l-delivery-id (and the signature pair with secretEnv, a native option). A cancelled crawl is crawl.failed with error \"cancelled\". The native rules apply: https (plain http for a loopback receiver of a local server only), no content-type, host or x-w2l-* header, at most 32 headers and 32 metadata strings; a hosted server takes public https receivers only. GET /v1/deliveries?jobId=<id> on the native API lists the deliveries."];
+export interface FirecrawlPage {
+    markdown: string | null;
+    /** Present when the `html` format was asked for; null when the page has none (a file, a page that did not succeed). */
+    html?: string | null;
+    /** Present when the `rawHtml` format was asked for; null when the page has none. */
+    rawHtml?: string | null;
+    links?: string[];
+    /** A PDF's pages, when the request's pdf parser asked for them (`pages: true`). */
+    pages?: {
+        pageNumber: number;
+        markdown: string;
+    }[];
+    /** Present when the `images` format was asked for and the page was read as content: every image URL of the whole document. */
+    images?: string[];
+    /** Present when an `attributes` entry was asked for and the page was read as content: per selector, the attribute's values as written. */
+    attributes?: Array<{
+        selector: string;
+        attribute: string;
+        values: string[];
+    }>;
+    /** Present when a `screenshot` entry was asked for: the capture as a `data:image/png;base64,…` (or `image/jpeg`) string, as Firecrawl v1 clients read it; null when the browser rung rendered no page or could not capture it. */
+    screenshot?: string | null;
+    /**
+     * Present when the request ran `actions`: screenshots and PDFs as data: URIs, the HTML of each scrape step, each
+     * script's return, and the step that failed if one did.
+     */
+    actions?: {
+        screenshots: string[];
+        scrapes: Array<{
+            url: string;
+            html: string;
+        }>;
+        javascriptReturns: Array<{
+            type: string;
+            value: unknown;
+        }>;
+        pdfs: string[];
+        /** W2L's own list steps (scrollToEnd, loadMore, paginate): what each did and why it stopped; present when the request had one. */
+        lists?: Array<{
+            index: number;
+            type: string;
+            stoppedBy: string;
+            rounds: number;
+            items: number | null;
+            itemsRead?: number | null;
+        }>;
+        failed?: {
+            index: number;
+            type: string;
+            code: string;
+            message: string;
+        };
+    };
+    /** The native `warnings` as one string, their messages joined with a space; present when the result has any. */
+    warning?: string;
+    /** What to change about the request next time, one sentence each (the native `agentHints`); present when W2L has any. */
+    agent_hints?: string[];
+    /** Page fields appear only when the page declares them (W2L's `metadata`, null values left out). */
+    metadata: {
+        title?: string;
+        description?: string;
+        language?: string;
+        keywords?: string;
+        robots?: string;
+        favicon?: string;
+        /** Open Graph, Dublin Core and article tags under Firecrawl's names, each only when the page states it (see PageMetadata). */
+        ogTitle?: string;
+        ogDescription?: string;
+        ogUrl?: string;
+        ogImage?: string;
+        ogAudio?: string;
+        ogVideo?: string;
+        ogDeterminer?: string;
+        ogLocale?: string;
+        ogLocaleAlternate?: string[];
+        ogSiteName?: string;
+        dcTermsCreated?: string;
+        dcDateCreated?: string;
+        dcDate?: string;
+        dcTermsType?: string;
+        dcType?: string;
+        dcTermsAudience?: string;
+        dcTermsSubject?: string;
+        dcSubject?: string;
+        dcDescription?: string;
+        dcTermsKeywords?: string;
+        publishedTime?: string;
+        modifiedTime?: string;
+        /** Every `article:tag`, joined with `, ` as Firecrawl writes it. */
+        articleTag?: string;
+        articleSection?: string;
+        sourceURL: string;
+        /** The final URL, after redirects (`evidence.finalUrl`). */
+        url: string;
+        /** The status of the response that answered `url` (`evidence.httpStatus`); null when none did. */
+        statusCode: number | null;
+        /** That response's `content-type` header; left out when there was none. */
+        contentType?: string;
+        /** A PDF's page count, read or not; left out for anything else. */
+        numPages?: number;
+        /** On a page that did not succeed: W2L's failure, block or budget reason code (`http_error`, `cloudflare_challenge`, ...), or its status when it has none (`empty_verified`). */
+        error?: string;
+        /** The facts of the scrape call (native `metadata`), on a scrape response; a crawl status page has no call of its own and leaves them out. */
+        scrapeId?: string;
+        proxyUsed?: ScrapeMetadata['proxyUsed'];
+        timezone?: string | null;
+        /** Null: W2L counts no credits. */
+        creditsUsed?: null;
+        concurrencyLimited?: boolean;
+        concurrencyQueueDurationMs?: number;
+        /** `hit` when a stored result answered (`maxAge`, `minAge`, `lockdown`), `miss` when one was looked up and none fit; absent when the cache was not asked. */
+        cacheState?: 'hit' | 'miss';
+        /** On a hit, when the reused result was fetched. */
+        cachedAt?: string;
+    };
+}
+export interface FirecrawlScrapeResponse {
+    success: boolean;
+    data: FirecrawlPage;
+    error?: string;
+}
+/** POST /fc/v1/map: the URLs as strings; a map that found nothing because a source failed or its deadline passed is `success: false`. */
+export type FirecrawlMapResponse = {
+    success: true;
+    id: string;
+    links: string[];
+    warning?: string;
+    agent_hints?: AgentHints;
+} | {
+    success: false;
+    id: string;
+    error: string;
+    links: [];
+    agent_hints?: AgentHints;
+};
+export interface FirecrawlCrawlStarted {
+    success: true;
+    id: string;
+    url: string;
+}
+/** The type of one Firecrawl-shaped webhook payload: the job kind and the event (`cancelled` is sent as `failed` with `error: 'cancelled'`). */
+export type FirecrawlWebhookType = `${'crawl' | 'batch_scrape'}.${'started' | 'page' | 'completed' | 'failed'}`;
+/**
+ * A job event as Firecrawl's webhook receivers read it: `data` holds the page
+ * of a `page` event (as `/fc` crawl status lists it) and is empty otherwise;
+ * `metadata` is the request's `webhook.metadata`. The W2L envelope's fields
+ * are on the delivery's headers (`x-w2l-event-id`, `x-w2l-event-version`).
+ */
+export interface FirecrawlWebhookPayload {
+    success: true;
+    type: FirecrawlWebhookType;
+    id: string;
+    data: FirecrawlPage[];
+    metadata: Readonly<Record<string, string>>;
+    error?: string;
+}
+export type FirecrawlCrawlJobStatus = 'scraping' | 'completed' | 'failed' | 'cancelled';
+export interface FirecrawlCrawlStatus {
+    status: FirecrawlCrawlJobStatus;
+    /**
+     * The latest attempt's pages and errors, plus, while this API process runs
+     * the crawl, the pages in flight and those queued within its page limit.
+     * Null while the crawl is unfinished and no process here runs it (paused).
+     */
+    total: number | null;
+    /** The latest attempt's pages that succeeded (status success or partial): the data entries without metadata.error. */
+    completed: number;
+    /** Null: W2L counts no credits, and an unknown count is not zero. */
+    creditsUsed: number | null;
+    /** Null: crawl results stay until their task directory is deleted. */
+    expiresAt: string | null;
+    /** The URL of the next page of `data`: there while more pages are stored or the crawl is running, left out after the last. */
+    next?: string;
+    /** One page of the latest attempt's steps in the order they were recorded, errors included (with `metadata.error`). */
+    data: FirecrawlPage[];
+}
+/** What the API knows about a crawl beyond its steps, for its Firecrawl status. */
+export interface FirecrawlCrawlCounts {
+    completed: number;
+    total: number | null;
+    next?: string;
+}
+/**
+ * completed and total from how many of the latest attempt's steps have each
+ * status: completed is its success and partial pages, total every step it
+ * recorded plus, while the crawl is unfinished, `ahead`, the pages it will
+ * still record, null when no process here runs it.
+ */
+export declare function firecrawlCrawlCounts(status: TaskStatus, steps: Partial<Record<StepStatus, number>>, ahead: number | null): {
+    completed: number;
+    total: number | null;
+};
+/** Default and largest number of steps one `GET /fc/v1/crawl/:id` returns in `data`. */
+export declare const FIRECRAWL_STATUS_PAGE_SIZE: {
+    readonly default: 100;
+    readonly max: 1000;
+};
+export declare function parseFirecrawlScrapeRequest(body: unknown): ScrapeRequest;
+export declare function parseFirecrawlCrawlRequest(body: unknown): ParsedCrawlStartRequest;
+/**
+ * A Firecrawl map request as the native one. v1 ignoreSitemap true is
+ * sitemap skip and false include, sitemapOnly true is only, both true is
+ * refused; the v2 `sitemap` wins over the v1 flags, as on the crawl shim.
+ * The native parser validates the rest; an omitted option takes W2L's
+ * default (includeSubdomains and ignoreQueryParameters false).
+ */
+export declare function parseFirecrawlMapRequest(body: unknown): MapRequest;
+/** The native map response as Firecrawl's: the links as URL strings, the warnings' messages joined, the hints as `agent_hints`. */
+export declare function wrapMap(response: MapResponse): FirecrawlMapResponse;
+/**
+ * A job event in Firecrawl's webhook shape: the type from the job kind and
+ * the event (`cancelled` becomes `failed` with `error: 'cancelled'`), the
+ * page of a `page` event as `/fc` crawl status would list it, `metadata` as
+ * the request gave it.
+ */
+export declare function wrapJobWebhook(envelope: JobWebhookEnvelope, result: FetchResult | null): FirecrawlWebhookPayload;
+/** The native scrape response as Firecrawl's envelope: the page with the call's facts in `data.metadata` and its hints as `data.agent_hints`. */
+export declare function wrapScrape(response: ScrapeResponse): FirecrawlScrapeResponse;
+export declare function wrapCrawlAccepted(native: CrawlAccepted, seedUrl: string): FirecrawlCrawlStarted;
+/**
+ * The query of `GET /fc/v1/crawl/:id`: `cursor` (from a `next` URL) and
+ * `limit` (1 to 1000, default 100). Anything else, Firecrawl's `skip`
+ * included, is rejected by name: pages follow `next`.
+ */
+export declare function parseFirecrawlCrawlStatusQuery(query: Record<string, string | undefined>): {
+    cursor?: string;
+    limit: number;
+};
+export declare function wrapCrawlStatus(report: Pick<CrawlReport, 'status'>, steps: readonly StepRecord[], counts: FirecrawlCrawlCounts): FirecrawlCrawlStatus;
+/** A scrape a cache-only request (`lockdown`) could not answer: nothing was fetched, and Firecrawl answers it with HTTP 404 `SCRAPE_LOCKDOWN_CACHE_MISS`. */
+export declare function isLockdownCacheMiss(response: Pick<FetchResult, 'status' | 'failureReason'>): boolean;
+//# sourceMappingURL=firecrawl.d.ts.map

package/dist/types/cjs/contracts/groundTruth.d.ts

@@ -0,0 +1,77 @@
+import type { BlockReason, Lane, ResultStatus } from './status.js';
+import type { ExpectedTable } from './tableMarkdown.js';
+/** The five false-success checks. Fixtures evaluate all five; canaries evaluate the evidence-bearing subset. */
+export declare const FALSE_SUCCESS_CHECK: readonly ["missing_required_content", "challenge_text_returned", "content_yield_below_floor", "wrong_page_content", "silent_truncation"];
+export type FalseSuccessCheck = (typeof FALSE_SUCCESS_CHECK)[number];
+/**
+ * Checks computable without per-page annotation. Canary runs evaluate exactly these;
+ * the annotation-dependent ones are reported as unknown, never as pass.
+ */
+export declare const EVIDENCE_ONLY_CHECKS: readonly FalseSuccessCheck[];
+export declare const ANNOTATION_REQUIRED_CHECKS: readonly FalseSuccessCheck[];
+export type CheckOutcome = 'pass' | 'fail' | 'unknown';
+export interface CheckResult {
+    check: FalseSuccessCheck;
+    outcome: CheckOutcome;
+    /** Why it failed, or why it could not be evaluated. */
+    detail: string | null;
+}
+export interface CaseBudget {
+    maxTokens: number;
+    maxWallMs: number;
+    maxAttempts: number;
+}
+/**
+ * Ground truth for one benchmark case.
+ * Fixtures carry the full annotation; canaries may omit the annotation-only fields.
+ */
+export interface GroundTruth {
+    /** Stable case id, unique across the suite. */
+    id: string;
+    /** Path relative to the fixture server root, or an absolute https URL for canaries. */
+    target: string;
+    kind: 'fixture' | 'canary';
+    /** What this case is designed to probe. */
+    category: string;
+    evaluationSet?: 'development' | 'holdout';
+    /** Substrings that MUST appear in the extracted markdown. */
+    mustContain: readonly string[];
+    /** Substrings that MUST NOT appear in delivered content (nav, footer, cookie banner, ads). */
+    mustNotContain: readonly string[];
+    /**
+     * Structural table assertion, evaluated by check `missing_required_content`
+     * against the extracted markdown. Omitted for cases without table annotation.
+     * Use unique cell strings so cells/anchors locate unambiguously.
+     */
+    expectedTable?: ExpectedTable | null;
+    /** The lane the runtime is expected to settle on. */
+    expectedLane: Lane;
+    /** True when an empty extraction is the correct answer. */
+    emptyIsLegit: boolean;
+    /** Inclusive token range for the main content. Null when unannotated (canary). */
+    expectedMainTokens: {
+        min: number;
+        max: number;
+    } | null;
+    budget: CaseBudget;
+    /** Expected terminal status. Lets non-success cases (blocked, failed) be asserted too. */
+    expectedStatus: ResultStatus;
+    /**
+     * Which gate the case is expected to be classified as. Required whenever
+     * `expectedStatus` is 'blocked' — an unscored reason field is a reason field
+     * that silently drifts. Omitted (undefined) for every other case, which the
+     * runner reports as "not graded" rather than as a pass.
+     */
+    expectedBlockReason?: BlockReason | null;
+    notes?: string;
+}
+export interface SuiteMeta {
+    name: string;
+    version: string;
+    /** ISO date the suite content was last curated. */
+    curatedAt: string;
+}
+export interface Suite extends SuiteMeta {
+    cases: readonly GroundTruth[];
+}
+//# sourceMappingURL=groundTruth.d.ts.map

package/dist/types/cjs/contracts/identityBundle.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Identity bundle: the anti-bot layer we can actually own.
+ *
+ * FP-Inconsistent (IMC 2025) showed that bots which "escape" DataDome/BotD
+ * still leak because they rewrite fingerprint fields that no longer agree
+ * with each other. 2606.30119 showed JS stealth often *increases*
+ * detectability. So the product rule is: one coherent identity, fail closed
+ * on contradiction. This is not disguise — it is refusing to ship a lie.
+ */
+import { type BrowserFingerprint, type CrawlMode, type IdentityDevice, type ModeIdentity } from './compliance.js';
+export interface IdentityBundle {
+    userAgent: string;
+    clientHints: Readonly<Record<string, string>>;
+    locale: string;
+    timezoneId: string;
+    viewport: {
+        width: number;
+        height: number;
+    };
+    screen: {
+        width: number;
+        height: number;
+    };
+    /** Device pixels per CSS pixel; absent on a bundle built before the mobile identity existed. */
+    deviceScaleFactor?: number;
+    /** Whether the context reports a mobile device; must agree with `sec-ch-ua-mobile` and the UA. */
+    isMobile?: boolean;
+    hasTouch?: boolean;
+}
+/** Problems in a bundle. Empty means the identity is internally coherent. */
+export declare function identityBundleIssues(bundle: IdentityBundle): string[];
+/**
+ * L4 / vendor wire identity. We never inject our Chrome bundle into a vendor
+ * browser — we measure theirs. This check is: does what they actually send
+ * contradict the *mode* we claimed, or contradict itself?
+ *
+ * Missing client hints are allowed (most vendors do not surface them).
+ * HeadlessChrome, research-mode Chromium, and UA/hint disagreement are not.
+ */
+export declare function vendorIdentityIssues(mode: CrawlMode, userAgent: string, clientHints?: Readonly<Record<string, string>>): string[];
+export declare function assertIdentityBundle(bundle: IdentityBundle): void;
+/**
+ * Wire headers implied by a bundle. Asserts first: a contradictory bundle
+ * never becomes bytes. HTTP subjects send this object verbatim.
+ */
+export declare function headersFromIdentity(bundle: IdentityBundle): Record<string, string>;
+/**
+ * Route extras (proxy, cookies, vendor resume) that MUST NOT retune the face.
+ * Changing IP ≠ changing identity.
+ */
+export interface RouteAccess {
+    proxy?: unknown;
+    session?: unknown;
+    resume?: unknown;
+}
+export type IdentityOverride = Partial<Pick<IdentityBundle, 'userAgent' | 'clientHints' | 'locale' | 'timezoneId' | 'viewport' | 'screen'>>;
+/**
+ * The identity a fetch presents, given a mode and optional user route.
+ * `access` is accepted so callers cannot "forget" it — it is ignored.
+ * An override that would retune timezone/locale/viewport to a proxy geo is
+ * refused: that is fingerprint spoofing wearing a routing costume. `device`
+ * picks between the two declared browser identities (desktop, mobile): a
+ * declared second identity is not an override, and both pass the same checks.
+ */
+export declare function identityForRoute(mode: CrawlMode, access?: RouteAccess | null | undefined, chromeMajor?: number, override?: IdentityOverride | null, device?: IdentityDevice): IdentityBundle;
+/** The bundle of an identity with its fingerprint: the device's own (browserFingerprintFor) unless one is given. */
+export declare function identityBundleFrom(identity: ModeIdentity, fingerprint?: Readonly<BrowserFingerprint>): IdentityBundle;
+/** One-line identity for CLI stdout. Never empty for a coherent bundle. */
+export declare function formatIdentitySummary(bundle: IdentityBundle): string;
+//# sourceMappingURL=identityBundle.d.ts.map

package/dist/types/cjs/contracts/index.d.ts

@@ -0,0 +1,30 @@
+export * from './status.js';
+export * from './tokens.js';
+export * from './result.js';
+export * from './groundTruth.js';
+export * from './tableMarkdown.js';
+export * from './extractor.js';
+export * from './policy.js';
+export * from './ssrf.js';
+export * from './proxy.js';
+export * from './benchmark.js';
+export * from './compliance.js';
+export * from './identityBundle.js';
+export * from './access.js';
+export * from './checkpoint.js';
+export * from './crawl.js';
+export * from './regexSafety.js';
+export * from './api.js';
+export * from './firecrawl.js';
+export * from './monitor.js';
+export * from './monitorConfig.js';
+export * from './session.js';
+export * from './recipe.js';
+export * from './execution.js';
+export * from './delivery.js';
+export * from './structured.js';
+export * from './evidenceRecord.js';
+export * from './file.js';
+export * from './map.js';
+export * from './actions.js';
+//# sourceMappingURL=index.d.ts.map