@octocrawl/sdk 0.3.1 → 0.3.2

package/README.md

@@ -14,4 +14,4 @@ const { report, items } = await client.batchAndWait(['https://example.com/a', 'h
 
 Start a local API with `npx octocrawl serve`. Requires a runtime with `fetch` (Node.js 18 or later, browsers, Deno, Bun).
 
-Licence: MIT. Source and the API reference: https://github.com/77777R7/Octocrawl
+Licence: MIT. Website and docs: https://octocrawl.dev. Source and the API reference: https://github.com/77777R7/Octocrawl

package/dist/index.cjs

@@ -216,10 +216,10 @@ function isApiErrorCode(value) {
 var RATE_LIMITED_CODE = "rate_limited";
 var PAGE_KEYS = ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown"];
 var ATTRIBUTION_KEYS = ["origin", "integration"];
-var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "lane", "access", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var CRAWL_SCOPE_KEYS = ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-var CRAWL_KEYS = ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
-var BATCH_KEYS = ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var CRAWL_KEYS = ["url", "mode", "access", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var BATCH_KEYS = ["urls", "mode", "lane", "access", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var BATCH_APPEND_KEYS = ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", ...ATTRIBUTION_KEYS];
 var MAP_SCOPE_KEYS = ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
 var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", "ignoreRobotsTxt", ...ATTRIBUTION_KEYS];
@@ -254,11 +254,16 @@ var EVIDENCE_RECORD_KEYS = {
   pageActions: keysOf()(["steps", "scriptRan"]),
   pageActionStep: keysOf()(["type", "outcome"]),
   requestHeader: keysOf()(["name", "valueSha256"]),
-  access: keysOf()(["route", "executor", "executorVersion", "profile", "externalCostUsd"])
+  access: keysOf()(["route", "executor", "executorVersion", "profile", "externalCostUsd", "completion", "egress", "session", "paidCalls", "grant"]),
+  accessEgress: keysOf()(["proxy", "source", "switchedFrom", "exit"]),
+  accessEgressExit: keysOf()(["ip", "country", "observedAt"]),
+  accessSession: keysOf()(["id"]),
+  accessPaidCall: keysOf()(["provider", "rung", "capabilities", "ceilingUsd", "chargedUsd", "reportedCostUsd", "outcome", "reason", "answer"]),
+  accessGrant: keysOf()(["sha256", "tier", "attestedAt"])
 };
 
 // packages/sdk/src/version.ts
-var SDK_VERSION = "0.3.1";
+var SDK_VERSION = "0.3.2";
 
 // packages/sdk/src/watcher.ts
 var DEFAULT_WATCH_POLL_INTERVAL_MS = 2e3;
@@ -791,7 +796,7 @@ var W2L = class {
   }
   async scrape(url, opts = {}, request = {}) {
     const deadlineMs = Number.isInteger(opts.timeout) ? Math.min(Math.max(opts.timeout, 0), DEFAULT_SCRAPE_TIMEOUT_MS) : DEFAULT_SCRAPE_TIMEOUT_MS;
-    const handedOver = opts.handoff !== void 0 && opts.handoff !== false;
+    const handedOver = opts.handoff !== void 0 && opts.handoff !== false || opts.lane === "my-browser" || opts.access === "my-browser";
     return this.post("/v1/scrape", { ...opts, url, origin: originOf(opts, request) }, 200, request, handedOver ? 0 : deadlineMs + SCRAPE_ANSWER_MARGIN_MS);
   }
   /** The record of one scrape call, by the `scrapeId` its response carried (`metadata.scrapeId`); a W2LError with code `not_found` for an id the server has no record of. */

package/dist/index.js

@@ -179,10 +179,10 @@ function isApiErrorCode(value) {
 var RATE_LIMITED_CODE = "rate_limited";
 var PAGE_KEYS = ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown"];
 var ATTRIBUTION_KEYS = ["origin", "integration"];
-var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "lane", "access", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var CRAWL_SCOPE_KEYS = ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-var CRAWL_KEYS = ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
-var BATCH_KEYS = ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var CRAWL_KEYS = ["url", "mode", "access", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var BATCH_KEYS = ["urls", "mode", "lane", "access", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var BATCH_APPEND_KEYS = ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", ...ATTRIBUTION_KEYS];
 var MAP_SCOPE_KEYS = ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
 var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", "ignoreRobotsTxt", ...ATTRIBUTION_KEYS];
@@ -217,11 +217,16 @@ var EVIDENCE_RECORD_KEYS = {
   pageActions: keysOf()(["steps", "scriptRan"]),
   pageActionStep: keysOf()(["type", "outcome"]),
   requestHeader: keysOf()(["name", "valueSha256"]),
-  access: keysOf()(["route", "executor", "executorVersion", "profile", "externalCostUsd"])
+  access: keysOf()(["route", "executor", "executorVersion", "profile", "externalCostUsd", "completion", "egress", "session", "paidCalls", "grant"]),
+  accessEgress: keysOf()(["proxy", "source", "switchedFrom", "exit"]),
+  accessEgressExit: keysOf()(["ip", "country", "observedAt"]),
+  accessSession: keysOf()(["id"]),
+  accessPaidCall: keysOf()(["provider", "rung", "capabilities", "ceilingUsd", "chargedUsd", "reportedCostUsd", "outcome", "reason", "answer"]),
+  accessGrant: keysOf()(["sha256", "tier", "attestedAt"])
 };
 
 // packages/sdk/src/version.ts
-var SDK_VERSION = "0.3.1";
+var SDK_VERSION = "0.3.2";
 
 // packages/sdk/src/watcher.ts
 var DEFAULT_WATCH_POLL_INTERVAL_MS = 2e3;
@@ -754,7 +759,7 @@ var W2L = class {
   }
   async scrape(url, opts = {}, request = {}) {
     const deadlineMs = Number.isInteger(opts.timeout) ? Math.min(Math.max(opts.timeout, 0), DEFAULT_SCRAPE_TIMEOUT_MS) : DEFAULT_SCRAPE_TIMEOUT_MS;
-    const handedOver = opts.handoff !== void 0 && opts.handoff !== false;
+    const handedOver = opts.handoff !== void 0 && opts.handoff !== false || opts.lane === "my-browser" || opts.access === "my-browser";
     return this.post("/v1/scrape", { ...opts, url, origin: originOf(opts, request) }, 200, request, handedOver ? 0 : deadlineMs + SCRAPE_ANSWER_MARGIN_MS);
   }
   /** The record of one scrape call, by the `scrapeId` its response carried (`metadata.scrapeId`); a W2LError with code `not_found` for an id the server has no record of. */

package/dist/types/cjs/client.d.ts

@@ -33,7 +33,7 @@ export interface RequestOptions {
     origin?: string;
 }
 /** What the SDK records as `origin` unless the caller or the host says otherwise. */
-export declare const SDK_ORIGIN = "js-sdk@0.3.1";
+export declare const SDK_ORIGIN = "js-sdk@0.3.2";
 /**
  * Polling for waitBatch and waitCrawl. A status request that fails with a
  * network error, HTTP 408, 429 or 5xx is retried: after 1, 2, 4, 8, then

package/dist/types/cjs/contracts/actions.d.ts

@@ -4,6 +4,7 @@
  * Each step is recorded in the trace with its outcome and timing; a step
  * that fails ends the pipeline, and the result keeps the page as it stood.
  */
+import type { BlockReason } from './status.js';
 import type { ScreenshotEvidence } from './result.js';
 import type { ScreenshotViewport } from './structured.js';
 /** The most steps one request may run. */
@@ -149,8 +150,20 @@ export interface ActionPdf {
     path: string | null;
     base64: string;
 }
-/** Why a scrollToEnd, loadMore or paginate step stopped. `max` and `deadline` stop short of the list's end. */
-export type ListStop = 'end' | 'no_growth' | 'repeat' | 'max' | 'deadline';
+/**
+ * Why a scrollToEnd, loadMore or paginate step stopped. `max` and `deadline` stop short of the list's end; so does
+ * `challenge`: a check the site put up on the next page (a Cloudflare interstitial, a page of nothing but a CAPTCHA),
+ * which paginate stops at without reading it, the result then `blocked` with the check's reason.
+ */
+export type ListStop = 'end' | 'no_growth' | 'repeat' | 'max' | 'deadline' | 'challenge';
+/** The check a paginate step stopped at (ListStop `challenge`): the page it would have been, its URL, and what the gate saw. */
+export interface ListChallenge {
+    /** 1-based position the page would have had among the pages read. */
+    page: number;
+    url: string;
+    reason: BlockReason;
+    signals: readonly string[];
+}
 /** What a scrollToEnd, loadMore or paginate step did. */
 export interface ListRun {
     index: number;
@@ -160,8 +173,27 @@ export interface ListRun {
     rounds: number;
     /** Elements matching `itemSelector` at the end (on the last page for paginate; summed over its pages in `itemsRead`); null without one. */
     items: number | null;
-    /** paginate: elements matching `itemSelector` over every page read; null without one. */
+    /**
+     * paginate: elements matching `itemSelector` over every page read; null without one. After a continuation in the person's
+     * Chrome (`continued`), the kept pages' count plus each page they showed, as their tab counted it, only when the addresses show
+     * no page can be counted twice (the kept pages each at its own, the check's page and the pages shown at none of them, the check
+     * not at the list's own address), no page shows again what another shows (its items' whole text, as the list merge tells it:
+     * a result set tied to the session that made it comes back at new addresses), and every count is known; null (unknown) otherwise.
+     */
     itemsRead?: number | null;
+    /** paginate: pages taken from the task's checkpoint after a run cut at page N (ExecutionContext.listResume), counted in `rounds`; absent when none. */
+    resumed?: number;
+    /** paginate: the check the step stopped at, with `stoppedBy` `challenge`; absent otherwise. */
+    challenge?: ListChallenge;
+    /**
+     * paginate: the pages read after the check at `from`, by the person paging on in their own browser once they got through
+     * it (the batch handoff), counted in `rounds`; `stoppedBy` then says how that reading ended. Absent otherwise.
+     */
+    continued?: {
+        from: number;
+        pages: number;
+        by: 'user_browser';
+    };
 }
 /** What the steps produced, each list in the order of its steps. */
 export interface ActionsResult {
@@ -170,7 +202,8 @@ export interface ActionsResult {
     scrapes: {
         url: string;
         html: string;
-        step?: number;
+        step?: number; /** Set when the person read the page in their own browser (a list's continuation after a check); absent for W2L's own browser. */
+        by?: 'user_browser';
     }[];
     /** `type` is the JavaScript `typeof` of the value (`null` for null). */
     javascriptReturns: {

package/dist/types/cjs/contracts/api.d.ts

@@ -170,7 +170,28 @@ export interface ScrapeRequest extends PageOptions, RequestAttribution {
     handoff?: {
         waitMs?: number;
     };
+    /**
+     * `my-browser`: read the page in the person's own Chrome, over remote
+     * debugging, without W2L fetching it first. The person allows the
+     * connection in Chrome, then the site in a page W2L opens there; the page
+     * is read without a click of theirs only on a site they allowed, and a
+     * check it shows waits for them (`handoff.waitMs`, default 10 min). Lane
+     * `my_browser`; never cached. Offered only by a server on the person's
+     * own machine; refused elsewhere, with `actions` or a screenshot, and with
+     * a mode other than standard (`unsupported_parameter`).
+     */
+    lane?: 'my-browser';
+    /** One of three plain choices of how the page is reached (ACCESS_CHOICES); omitted, the server's own configuration. */
+    access?: AccessChoice;
 }
+/**
+ * How pages are reached, as three plain choices (ROADMAP PA item 7) beside the per-route options:
+ * `standard` (Octocrawl's own lanes and none that costs a third party), `enhanced` (also what the server's
+ * access grant of tier enhanced approves, within its budget; refused on a server without one), `my-browser`
+ * (the person's own Chrome, as `lane: "my-browser"`; a scrape or a batch only).
+ */
+export declare const ACCESS_CHOICES: readonly ["standard", "enhanced", "my-browser"];
+export type AccessChoice = (typeof ACCESS_CHOICES)[number];
 /** A recorded robots override for one URL of a batch. */
 export interface RobotsUrlOverride extends RobotsOverride {
     url: string;
@@ -344,6 +365,8 @@ export interface JobWebhookStatus {
 export interface CrawlStartRequest extends PageOptions, RequestAttribution {
     url: string;
     mode?: ApiCrawlMode;
+    /** `standard` or `enhanced` (ACCESS_CHOICES); a crawl does not take `my-browser`. */
+    access?: Exclude<AccessChoice, 'my-browser'>;
     maxPages?: number | null;
     maxDepth?: number | null;
     /**
@@ -526,6 +549,16 @@ export interface ActiveCrawlList {
 export interface BatchStartRequest extends PageOptions, RequestAttribution {
     urls: readonly string[];
     mode?: ApiCrawlMode;
+    /** One of three plain choices of how pages are reached (ACCESS_CHOICES); `my-browser` is `lane: "my-browser"`. */
+    access?: AccessChoice;
+    /**
+     * `my-browser`: read every page in the person's own Chrome, one at a time, as a scrape's `lane` does. The person
+     * allows the connection in Chrome, then all the batch's sites (host and port) in the page W2L opens there, once for
+     * the run; a site not among them is not read. A resumed run asks again. Offered only by a server on the person's own
+     * machine; refused elsewhere, with `actions`, a screenshot, lockdown, a mode other than standard, `maxConcurrency`
+     * above 1 or a webhook (`unsupported_parameter`).
+     */
+    lane?: 'my-browser';
     formats?: readonly ScrapeFormat[];
     includeLinks?: boolean;
     /** Recorded robots overrides, each for one URL of `urls`. A hosted server refuses the field (`unsupported_parameter`). */
@@ -604,6 +637,8 @@ export interface BatchStatusResponse extends CrawlReport {
     invalidURLs?: readonly string[];
     /** Items stopped at a check a person can get through in their own Chrome (`POST /v1/batches/:id/handoff`); present on a server that offers the handoff. */
     waitingForPerson?: number;
+    /** A batch on the my-browser lane waiting for the person to allow its sites in the page Octocrawl opened in their Chrome; present only while it waits. */
+    waitingForApproval?: true;
 }
 /**
  * The checks a batch item can be handed to a person for, and the routing
@@ -838,10 +873,12 @@ export declare const REFUSAL_HINTS: {
 export declare function refusalHint(key: string, value: unknown): string | null;
 export declare const PAGE_KEYS: readonly ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown"];
 export declare const ATTRIBUTION_KEYS: readonly ["origin", "integration"];
-export declare const SCRAPE_KEYS: readonly ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const SCRAPE_KEYS: readonly ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "lane", "access", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+/** The lanes a request may ask for by name. */
+export declare const REQUEST_LANES: readonly ["my-browser"];
 export declare const CRAWL_SCOPE_KEYS: readonly ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-export declare const CRAWL_KEYS: readonly ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", "regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks", "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
-export declare const BATCH_KEYS: readonly ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const CRAWL_KEYS: readonly ["url", "mode", "access", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", "regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks", "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const BATCH_KEYS: readonly ["urls", "mode", "lane", "access", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
 /** What a batch body may carry beside `appendToId`: the job's own options are not among them (the scope no-ops change nothing, so they may come along). */
 export declare const BATCH_APPEND_KEYS: readonly ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", "origin", "integration"];
 /** The scope options a map takes under their crawl names; allowSubdomains is includeSubdomains on a map, and allowExternalLinks is not offered. */

package/dist/types/cjs/contracts/checkpoint.d.ts

@@ -7,7 +7,7 @@
  * Granularity is the page. A partial parse inside a page is not a step; the
  * whole URL is retried. Block-level checkpoint is out of Phase 1.
  */
-import type { PageOptions, RequestAttribution, RobotsUrlOverride, WebhookEvent } from './api.js';
+import type { AccessChoice, PageOptions, RequestAttribution, RobotsUrlOverride, WebhookEvent } from './api.js';
 import type { CrawlMode } from './compliance.js';
 import type { WebhookPayloadFormat } from './delivery.js';
 import type { CrawlDiscovery, SitemapMode } from './crawl.js';
@@ -29,6 +29,11 @@ export interface CrawlBudget {
     maxWallMs: number | null;
     maxCostUsd: number | null;
     maxTokens: number | null;
+    /**
+     * What one page may spend on third parties (an access grant's `perRequestUsd`, ROADMAP PA item 4): its own cap
+     * within the run's, held by the run's spend ledger. Absent or null: none of its own.
+     */
+    maxCostPerPageUsd?: number | null;
 }
 export declare const DEFAULT_CRAWL_BUDGET: CrawlBudget;
 /**
@@ -72,12 +77,16 @@ export interface Task {
         maxConcurrency?: number;
         invalidURLs?: readonly string[];
         webhook?: StoredJobWebhook;
+        lane?: 'my-browser';
+        access?: AccessChoice;
     } & PageOptions;
     /**
      * Every crawl option but the page budget (`budget`), stored when the crawl
      * starts so a resumed crawl runs with the options it was started with.
      */
     crawl?: {
+        /** The plain access choice the crawl was started with (`standard` or `enhanced`); absent: the server's configuration. */
+        access?: 'standard' | 'enhanced';
         formats?: readonly ScrapeFormat[];
         includeLinks?: boolean;
         includePaths?: readonly string[];
@@ -136,6 +145,11 @@ export interface Attempt {
     recoveredFromAttemptId?: string | null;
     /** What this attempt's pages offered the frontier and what became of it, written after every page of a crawl; absent for a batch and for an attempt stored before it was kept. */
     discovery?: CrawlDiscovery | null;
+    /**
+     * What the spend ledger charged this attempt's paid calls (ROADMAP PA item 4): a resumed or appended run of the task
+     * opens its ledger with every earlier attempt's charge, so its cap is the task's, not each run's. Absent: none.
+     */
+    chargedUsd?: number | null;
 }
 /**
  * One URL inside one attempt. The atomic checkpoint unit.

package/dist/types/cjs/contracts/evidenceRecord.d.ts

@@ -56,7 +56,101 @@ export interface EvidenceAccess {
     profile: string | null;
     /** Third-party spend of the run that produced the result: 0 when no paid service was called, null when a called service stated no price. */
     externalCostUsd: number | null;
+    /**
+     * How the page was read, counted apart (ROADMAP PA items 7 and 8): `unattended` (W2L's own lanes, no session of the
+     * person's), `authorized_session` (with the person's saved login), `user_browser` (in the person's own Chrome, on a
+     * site they allowed, without a step of theirs), `handed_to_person` (in their Chrome, after they got through a check).
+     * Null when no page was read: the result is not success, partial or empty_verified, or no lane produced it. Added with
+     * the my-browser lane: optional, so that records written before it stay valid.
+     */
+    completion?: AccessCompletion | null;
+    /**
+     * The egress the page left through (ROADMAP PA item 3): the proxy's `host:port` and whether it came from the
+     * operator's pool (`W2L_EGRESS_PROXIES`) or the environment variables, or `direct` with no proxy when W2L's own lane
+     * recorded none and a page response shows a request was sent. `switchedFrom` names the pool egress the task last
+     * moved off before this page was read here (`egress_switched`); null otherwise. Null when nothing says where the
+     * requests left from: a vendor's service, the person's own browser, no lane at all, or a lane that stopped before
+     * a page request (robots.txt, an address check, a deadline). Added with the egress pool: optional, so that earlier
+     * records stay valid.
+     */
+    egress?: EvidenceAccessEgress | null;
+    /**
+     * The task cookie session the page was read with (`egress_sessions`): its id alone, never its cookies. Null when
+     * the page was read with none. Added with the egress pool: optional, so that earlier records stay valid.
+     */
+    session?: EvidenceAccessSession | null;
+    /**
+     * Every paid provider call the page was read with (ROADMAP PA item 4), in order: the provider, what the spend ledger
+     * reserved and charged, the price the provider stated, and what Octocrawl made of what came back. Null when no
+     * provider was called. Added with the spend ledger: optional, so that earlier records stay valid.
+     */
+    paidCalls?: readonly EvidencePaidCall[] | null;
+    /** The access grant the paid calls were made under; null when no provider was called. Added with `paidCalls`. */
+    grant?: EvidenceAccessGrant | null;
+}
+/** One paid provider call (ROADMAP PA item 4). A provider's own word on the page is never its outcome. */
+export interface EvidencePaidCall {
+    /** The provider's id, as the access grant's tariffs name it (`browserbase`, `steel`). */
+    provider: string;
+    /** The rung that made the call; a retry after the person's handoff is `provider(retry)`. */
+    rung: string;
+    /** The ADR 0005 capabilities the provider's session was created with: `vendor_remote_browser`, and solving or stealth when the grant named them. */
+    capabilities: readonly string[];
+    /** What the ledger reserved before the call: its price ceiling, from the grant's tariff. */
+    ceilingUsd: number;
+    /** What the ledger charged: the price the provider stated, else the ceiling (a call that threw or was cut included). */
+    chargedUsd: number;
+    /** The price the provider stated for the call; null when it stated none (Browserbase and Steel state none per call). */
+    reportedCostUsd: number | null;
+    /**
+     * What Octocrawl made of the page the call returned, by its own checks (a block page, an empty or unverified read, an
+     * identity it did not send): never the provider's word that it succeeded. Null when the call returned no page (it
+     * threw, or the deadline cut it).
+     */
+    outcome: ResultStatus | null;
+    /** Why the outcome is not a read page, as the record's own `reason`; null otherwise. */
+    reason: FailureReason | BlockReason | BudgetKind | null;
+    /** The record's own page is this call's. */
+    answer: boolean;
+}
+/** The access grant paid calls were made under: enough to name it, not a copy. */
+export interface EvidenceAccessGrant {
+    /** SHA-256 of the grant's text as Octocrawl read it (`shasum -a 256 grant.json`); null for a grant not read from text. */
+    sha256: string | null;
+    /** The grant's tier. */
+    tier: string;
+    /** When its attestation says the operator accepted the providers' terms and costs; null when it has none. */
+    attestedAt: string | null;
+}
+export declare const ACCESS_EGRESS_SOURCES: readonly ["pool", "environment", "direct"];
+export type AccessEgressSource = (typeof ACCESS_EGRESS_SOURCES)[number];
+export interface EvidenceAccessEgress {
+    /** The proxy's `host:port`, never its credentials; null when the request went direct. */
+    proxy: string | null;
+    source: AccessEgressSource;
+    /** The pool egress the task last left before this page was read here; null when it did not move. */
+    switchedFrom: string | null;
+    /**
+     * Where the pool egress leaves from, as the operator's echo URL (`W2L_EGRESS_ECHO_URL`) saw it through that proxy.
+     * Null when no echo URL is set, the echo did not answer, or the egress is not the pool's. Added after `egress`:
+     * optional, so that earlier records stay valid.
+     */
+    exit?: EvidenceAccessEgressExit | null;
+}
+export interface EvidenceAccessEgressExit {
+    /** The address the echo service saw the request come from. */
+    ip: string;
+    /** Its two-letter country code, when the echo service gives one; null otherwise. */
+    country: string | null;
+    /** When the echo was asked (UTC ISO); an exit is asked again after ten minutes. */
+    observedAt: string;
+}
+export interface EvidenceAccessSession {
+    /** The session's id, as `session_cookies` traces it. */
+    id: string;
 }
+export declare const ACCESS_COMPLETIONS: readonly ["unattended", "authorized_session", "user_browser", "handed_to_person"];
+export type AccessCompletion = (typeof ACCESS_COMPLETIONS)[number];
 export interface EvidenceRedirectChain {
     /**
      * Every URL W2L requested for the page, in order: the requested URL first,
@@ -211,7 +305,12 @@ export declare const EVIDENCE_RECORD_KEYS: {
     readonly pageActions: readonly ["steps", "scriptRan"];
     readonly pageActionStep: readonly ["type", "outcome"];
     readonly requestHeader: readonly ["name", "valueSha256"];
-    readonly access: readonly ["route", "executor", "executorVersion", "profile", "externalCostUsd"];
+    readonly access: readonly ["route", "executor", "executorVersion", "profile", "externalCostUsd", "completion", "egress", "session", "paidCalls", "grant"];
+    readonly accessEgress: readonly ["proxy", "source", "switchedFrom", "exit"];
+    readonly accessEgressExit: readonly ["ip", "country", "observedAt"];
+    readonly accessSession: readonly ["id"];
+    readonly accessPaidCall: readonly ["provider", "rung", "capabilities", "ceilingUsd", "chargedUsd", "reportedCostUsd", "outcome", "reason", "answer"];
+    readonly accessGrant: readonly ["sha256", "tier", "attestedAt"];
 };
 /**
  * Keys added to v1 after it was first published: optional in the schema, so

package/dist/types/cjs/contracts/execution.d.ts

@@ -20,6 +20,74 @@ export interface ExecutionContext {
      * kept or sent, as before.
      */
     cookieSession?: CookieSession;
+    /**
+     * Hear of each trace event the moment the HTTP lane records it (robots.txt checked, the response arrived, the
+     * page extracted), for a caller that shows progress while the fetch runs. The result's `trace` stays the record;
+     * a listener's error never changes the fetch. Absent: nothing is told early.
+     */
+    onTrace?: (event: TraceEvent) => void;
+    /**
+     * Hear of each page a paginate step reads the moment it is read (ROADMAP PA item 3), for a caller that keeps
+     * them in the task's checkpoint: a run cut at page N then resumes from them through `listResume`. A listener's
+     * error never changes the fetch. Absent: nothing is told.
+     */
+    onListPage?: (page: ListPageRead) => void;
+    /**
+     * The pages a paginate step of this URL read before an earlier run was cut, from the task's checkpoint. The
+     * browser lane counts them as read: it passes over them on its way along the site's own Next links (the only way
+     * to page N+1 when pages have no address of their own), tells and reads the pages after them, and merges every
+     * page once. Absent: the list starts from its first page.
+     */
+    listResume?: {
+        pages: readonly ListPageRead[];
+    };
+    /**
+     * The run's spend ledger (ROADMAP PA item 4): a rung that costs a third party reserves its price ceiling here before
+     * its call and settles after, so a run's concurrent pages, retries and providers share one cap and none overruns it.
+     * Absent: nothing is reserved, as before (a run with no third-party rung).
+     */
+    spend?: SpendLedger;
+}
+/**
+ * A spend ledger: the cap a run (or one page of it) may spend on third parties, what is settled and what is reserved.
+ * A paid call reserves its price ceiling first and is not made when the ceiling does not fit.
+ */
+export interface SpendLedger {
+    /** Reserve `ceilingUsd` against this ledger's cap and every enclosing one; null when it does not fit. */
+    reserve(ceilingUsd: number): SpendReservation | null;
+    /** A ledger over the same totals whose own reservations are also capped at `capUsd` (one page's `perRequestUsd`); null: no cap of its own. */
+    child(capUsd: number | null): SpendLedger;
+    /** Spend settled so far, in US dollars: each call at its reported price, or at its ceiling when none was reported. */
+    readonly settledUsd: number;
+    /** Ceilings reserved by calls not yet settled. */
+    readonly reservedUsd: number;
+    /** The cap; null: none. */
+    readonly capUsd: number | null;
+}
+/** One reserved call. */
+export interface SpendReservation {
+    /** The call was made: settle at the price the provider reported, or at the ceiling when it reported none. Returns what was charged. */
+    settle(reportedUsd: number | null): number;
+    /** The call was not made: give the reservation back. */
+    release(): void;
+}
+/** One page a paginate step read: what `onListPage` tells and `listResume` gives back. */
+export interface ListPageRead {
+    /** Index of the paginate step among the request's actions. */
+    step: number;
+    /** 1-based position among the pages the step read, the resumed ones included. */
+    page: number;
+    /** The URL the browser showed when the page was read. */
+    url: string;
+    html: string;
+    /** The page's state key as the lane computed it (its URL and its items or words), to know the page again on a resume. */
+    state: string;
+    /** The hash of the elements `itemSelector` matched, or null without one. */
+    items: string | null;
+    /** The hash of the links and sources inside those elements alone, which a changed price or date leaves as it was; null without `itemSelector` or when no item has one. */
+    itemRefs: string | null;
+    /** How many elements `itemSelector` matched on the page; null without one. */
+    count: number | null;
 }
 /** A cookie as a browser context takes and gives it (Playwright's shape). */
 export interface ContextCookie {

package/dist/types/cjs/contracts/firecrawl.d.ts

@@ -23,7 +23,7 @@ export declare const FIRECRAWL_SHIM_SNAPSHOT: {
     paths: readonly ["/scrape", "/crawl", "/crawl/:id", "/map"];
     notCovered: readonly ["search", "interact", "agent", "monitor", "extract"];
 };
-export declare const FIRECRAWL_SHIM_DIFFS: readonly ["Challenge / block pages are success: false (Firecrawl often returns them as success markdown).", "A page with no main content is success: false (failed: empty_unverified) with the whole page in data.markdown as evidence; with onlyMainContent: false it is success: true.", "A page whose server HTML is a shell for data its scripts fill in is fetched again on the browser rung, and the rendered page is the answer when it holds more; otherwise the HTTP page is returned with client_rendered_suspected and low_content_yield warnings on the native response, whose messages /fc passes through as data.warning (one string, joined with a space), as it does every native warning. Firecrawl renders every page in a browser.", "No fire-engine, proxy pools or JSON extract.", "actions (scrape only; a crawl's scrapeOptions.actions is refused) run on the local browser rung alone, which such a request selects, after load, stability and waitFor and before the formats are read: wait (milliseconds up to 60000, or a selector, waited for up to 60 s within the scrape's timeout), click (all: true clicks every match), write (into the focused element), press, scroll (one screen up or down, of the page or the element a selector names), screenshot, scrape, executeJavascript (a function body; return gives the value) and pdf; at most 50 steps. data.actions holds screenshots and pdfs as data: URIs (Firecrawl returns URLs), scrapes as { url, html } and javascriptReturns as { type, value }. A step that fails stops the steps after it: success is false with data.actions.failed naming the step, its code and message, and data.markdown is the page as it stood. A step that leads the page to a URL robots.txt or the egress policy refuses fails with navigation_refused and that page is not read. A hosted server refuses actions, and the cache is not used with them.", "An omitted maxAge reuses nothing: every page is fetched live unless the request sets maxAge above 0, minAge or lockdown (Firecrawl reuses its own index by default; its Python SDK sends maxAge 4 hours). A reused page is one Octocrawl itself stored, under the same options, on this server (its task root), never a shared index; only a success is stored, and data.metadata says cacheState hit with cachedAt (its fetch time) or miss when one was looked up. lockdown with no stored result is HTTP 404 SCRAPE_LOCKDOWN_CACHE_MISS on scrape, and nothing is fetched; a crawl in lockdown needs sitemap skip, and each page with no stored result is failed with cache_miss. Mode authed neither stores nor reuses. A Firecrawl body never sets useCached, Octocrawl's reuse of a crawl's own pages on resume.", "Omitted limit / maxDepth stay unbounded on a local server; a hosted server takes its crawl limit for an omitted or null limit and refuses a larger one. Firecrawl defaults are 10000 / 10.", "maxDepth counts link hops from the start URL (Firecrawl calls that maxDiscoveryDepth); Firecrawl maxDepth counts URL path depth.", "Crawl start is mapped onto native POST /v1/crawl; the shim itself returns 200 {success,id,url}.", "creditsUsed and expiresAt are null: Octocrawl counts no credits and keeps crawl results until their task directory is deleted.", "Crawl status describes the latest attempt: completed counts its successful pages, total adds its failed, blocked and duplicate pages and, while this API process runs the crawl, the pages in flight and queued (null for a paused crawl); data lists the failed and blocked pages too (with metadata.error) but not the duplicates, whose content is an earlier entry's, up to 100 per response (limit 1 to 1000) with next carrying an Octocrawl cursor; skip is rejected.", "Scrape maps url, formats, onlyMainContent, includeTags, excludeTags, waitFor, timeout, headers, mobile, skipTlsVerification, fastMode, blockAds, removeBase64Images, maxAge, minAge, storeInCache, lockdown, actions, origin and integration; crawl maps url, limit (as maxPages), maxDepth, includePaths, excludePaths, regexOnFullURL, ignoreQueryParameters, deduplicateSimilarURLs, crawlEntireDomain (and its v1 name allowBackwardLinks), allowSubdomains, allowExternalLinks, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only), maxConcurrency, ignoreRobotsTxt (v2; a local server only), origin, integration and the same scrapeOptions (applied to every page). The formats are markdown, links, html, rawHtml, images, screenshot (also screenshot@fullPage, and { type: \"screenshot\", fullPage, quality, viewport }) and an { type: \"attributes\", selectors } entry; other formats and parameters the shim does not map (proxy, location, json, ...) are rejected by name with HTTP 400 and success: false; a refusal of stealth, proxy: stealth or enhanced, or ignoreRobotsTxt on a scrape or map names the supported route in agent_hints.", "A crawl follows links inside the start URL's path subtree on its host and www twin by default (crawlEntireDomain false), folds /a and /a/, / and /index.html, www and apex, http and https into one page (deduplicateSimilarURLs true) and reports every collapsed or refused link in the native crawl status (discovery) and each page's trace (links_offered); allowSubdomains takes every host under the start URL's apex (no public-suffix list), allowExternalLinks every host, each page with its own robots.txt read.", "sitemap (default include, as in Firecrawl) reads the sitemaps the start URL's robots.txt names, or /sitemap.xml, with the crawl's own http identity, robots.txt verdict, SSRF checks and proxy, and queues their URLs ahead of the start page's links under the same host, subtree, path and depth rules; only follows no page link; skip reads none. The native crawl status lists every sitemap file read, refused or unreadable in discovery.sitemap; the shim's status carries nothing of it, and sitemap fetches have no signed compliance record. maxConcurrency caps the pages one crawl fetches at once, at most the service's worker count (HTTP 400 above it), and never raises the per-host ceiling.", "screenshot (data.screenshot, a data:image/png;base64 string, or image/jpeg with quality 1 to 100) is captured on the local browser rung alone, which such a request selects (no http attempt; a server without a browser rung refuses the format with HTTP 400): after load, stability and waitFor, before the DOM is read, CSS-pixel sized at the declared 1280x800 viewport (device scale factor 2 is declared, not baked into the image) or at the viewport asked for (integers 320..1920 by 240..1080, within the declared screen; a window size, not a change of identity); fullPage captures the document's whole height at that width without scrolling first, so sections a page loads on scroll may show unloaded. A capture the browser could not make leaves data.screenshot null with a screenshot_unavailable warning while the page stands; a file or a page that was not rendered has null too. Firecrawl captures at its own viewport and may return a URL instead of the image.", "images (data.images) lists every image URL of the whole document as received: img src and srcset candidates, picture sources, lazy data-src/data-srcset/data-lazy-src/data-original, video posters, image_src links, og:image and twitter:image, absolute http(s) with the fragment stripped, each once, in document order, data: URIs left out; includeTags, excludeTags and onlyMainContent do not narrow it. attributes (data.attributes) gives, per selector, the named attribute's values as written, elements without it skipped; a selector Octocrawl does not match is HTTP 400 by name, as for includeTags. Both are absent for a file and for a page that is success: false. removeBase64Images (default true) keeps an image's alt text where Firecrawl writes a (<Base64-Image-Removed>) placeholder; false keeps the data: URI in the Markdown.", "origin (the Firecrawl SDKs' client label) and integration are stored, not echoed: the scrape record (GET /v1/scrapes/:id) and the crawl task carry them, and nothing sent to the target changes.", "data.metadata carries scrapeId (a UUID per call, which GET /v1/scrapes/:id looks up), proxyUsed (operator for the server's environment proxy, user for the caller's own egress, else null), timezone (the browser rung's declared zone, null on the HTTP rung), creditsUsed: null (Octocrawl counts no credits), concurrencyLimited and concurrencyQueueDurationMs (whether and how long the per-origin ceiling held the fetch back), and cacheState and cachedAt when the cache was asked (never a guessed miss).", "A page whose result Octocrawl has advice about (a login wall, a robots.txt rule, a gate, a cut, a script-filled shell) carries data.agent_hints, one sentence each; the native response calls them agentHints. A request refused for an option Octocrawl does not offer carries agent_hints in the error envelope, and a caller over the server's per-minute rate limit gets HTTP 429 { success: false, error, code: rate_limited, agent_hints } with Retry-After.", "headers never override the User-Agent, the client hints, a credential (authorization, cookie) or a transport header: such a header is HTTP 400 naming it, where Firecrawl sends it. The headers go to the requested origin after the declared identity and are on the record (the trace, the browser lane's signed sentHeaders); both rungs withhold them from a redirect hop to another origin and say so (custom_headers_withheld).", "mobile selects a declared Android Chrome identity (User-Agent, client hints, 412x915 viewport, touch) that robots.txt is evaluated against and the record carries; the page is whatever the site serves to it, with no DOM rewriting. It is refused with mode research.", "skipTlsVerification relaxes certificate verification for one local request and its robots.txt lookup, recorded in the trace (tls_verification_skipped) and a tls_unverified warning the native response carries; a hosted Octocrawl refuses it with HTTP 400. Without it a bad certificate is success: false with failed: tls_error. Firecrawl's Python SDK sends true by default; Octocrawl verifies by default.", "fastMode keeps the http rung alone: a page that needs scripts is success: false with failed: empty_unverified, never rendered; waitFor has no effect under it. Firecrawl's fast mode still renders.", "blockAds (default true) aborts requests to a bundled list of about 50 ad-serving hosts on the local browser rung and removes ad and cookie-banner elements before extraction; false keeps them. The list is curated, not EasyList: ads from hosts outside it are not blocked.", "html is the cleaned HTML the markdown is written from: the main content, the whole page without scripts, styles, form controls and embedded media when onlyMainContent is false, or a <body> holding the includeTags elements. rawHtml is the page as the answering rung received it: the response body on the HTTP rung, the rendered DOM on a browser rung. Both are null for a file and for a page that is success: false.", "includeTags keeps only the named elements, in document order, whatever onlyMainContent says; excludeTags removes elements from the main content, the whole page and an includeTags selection. A selector that does not parse, or that uses a sibling combinator, a positional pseudo-class, :has() or another pseudo-class Octocrawl does not match, is rejected with HTTP 400.", "An omitted timeout stays 300000 ms (Firecrawl: 30000). A timeout is answered with HTTP 200: success: true with the content fetched so far (native status partial), or success: false with failed: timeout; Firecrawl answers it with an error.", "waitFor skips the HTTP rung, which cannot run scripts, and starts at the browser rung; the wait counts toward timeout.", "metadata has title, description, language, keywords, robots and favicon only when the page declares them, and the Open Graph (ogTitle, ogDescription, ogUrl, ogImage, ogAudio, ogVideo, ogDeterminer, ogLocale, ogLocaleAlternate, ogSiteName), Dublin Core (dcTermsCreated, dcDateCreated, dcDate, dcTermsType, dcType, dcTermsAudience, dcTermsSubject, dcSubject, dcDescription, dcTermsKeywords) and article (publishedTime, modifiedTime, articleTag, articleSection) tags under Firecrawl's names, each only when the page states it, as written (no date normalisation, no fallback from another tag); twitter:* and other meta tags are not passed through, and a failed or blocked page has none.", "A PDF answers success: true with its text layer as markdown and metadata.numPages (the document's page count). parsers maps Firecrawl's pdf entry (the string or { type: \"pdf\", mode, maxPages, pages, pageMarkers }); mode fast and auto both read the text layer, and mode ocr and the image parser are refused by name. pageMarkers is false unless asked, as on Firecrawl; with true Octocrawl writes a <!-- page N --> line before each page (Firecrawl writes --- and the marker between pages). pages: true adds data.pages, [{ pageNumber, markdown }]. maxPages (1 to 10000) reads the first pages, and a cut it asked for stays success: true. parsers [] or v1 parsePDF false reads no PDF: success: true with markdown null and the file saved as received. A PDF without a text layer is success: false with failed: empty_unverified (no OCR). CSV, JSON and text files give their text as received; XLSX, XLS and ZIP files are success: true with markdown null. A file over W2L_MAX_FILE_BYTES is success: false with failed: body_too_large.", "Map (POST /fc/v1/map) maps url, search, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only; both v1 flags true is HTTP 400), includeSubdomains, ignoreQueryParameters, limit (1 to 100000, default 5000), timeout (1000 to 300000 ms for the whole map, default 60000; Firecrawl documents no default), origin and integration onto native POST /v1/map, and answers 200 { success: true, id, links: [url strings], warning?, agent_hints? }, or 200 { success: false, id, error, links: [] } when the map found nothing because a source failed or its deadline passed; useIndex, location, ignoreCache, threatProtection and auditMetadata are refused by name (useIndex with the hint that Octocrawl keeps no URL index). Omitted options take Octocrawl's defaults: includeSubdomains and ignoreQueryParameters are false, where Firecrawl v2 documents true for both. search keeps the URLs in which every word appears in the decoded URL or the title in hand, in discovery order; Firecrawl orders by relevance. A map reads the sitemaps the site declares and one page body (the start URL, http rung only), so a site without a sitemap maps only its start page's links; a title is the start page's own, an anchor's text or a sitemap's <news:title>, never fetched from the target; robots-disallowed URLs are left out and counted on the native response (GET /v1/maps/:id), which also records every sitemap file read.", "A crawl's webhook (a URL string or { url, headers, metadata, events }) is mapped onto the native webhook and its receiver gets Firecrawl's payload shape: { success, type: crawl.started | crawl.page | crawl.completed | crawl.failed, id, data: [page], metadata, error? }, one durable delivery per event with retries, every request carrying x-w2l-event-id, x-w2l-event-version and x-w2l-delivery-id (and the signature pair with secretEnv, a native option). A cancelled crawl is crawl.failed with error \"cancelled\". The native rules apply: https (plain http for a loopback receiver of a local server only), no content-type, host or x-w2l-* header, at most 32 headers and 32 metadata strings; a hosted server takes public https receivers only. GET /v1/deliveries?jobId=<id> on the native API lists the deliveries."];
+export declare const FIRECRAWL_SHIM_DIFFS: readonly ["Challenge / block pages are success: false (Firecrawl often returns them as success markdown).", "A page with no main content is success: false (failed: empty_unverified) with the whole page in data.markdown as evidence; with onlyMainContent: false it is success: true.", "A page whose server HTML is a shell for data its scripts fill in is fetched again on the browser rung, and the rendered page is the answer when it holds more; otherwise the HTTP page is returned with client_rendered_suspected and low_content_yield warnings on the native response, whose messages /fc passes through as data.warning (one string, joined with a space), as it does every native warning. Firecrawl renders every page in a browser.", "No fire-engine, proxy pools or JSON extract.", "actions (scrape only; a crawl's scrapeOptions.actions is refused) run on the local browser rung alone, which such a request selects, after load, stability and waitFor and before the formats are read: wait (milliseconds up to 60000, or a selector, waited for up to 60 s within the scrape's timeout), click (all: true clicks every match), write (into the focused element), press, scroll (one screen up or down, of the page or the element a selector names), screenshot, scrape, executeJavascript (a function body; return gives the value) and pdf; at most 50 steps. data.actions holds screenshots and pdfs as data: URIs (Firecrawl returns URLs), scrapes as { url, html } and javascriptReturns as { type, value }. A step that fails stops the steps after it: success is false with data.actions.failed naming the step, its code and message, and data.markdown is the page as it stood. A step that leads the page to a URL robots.txt or the egress policy refuses fails with navigation_refused and that page is not read. A hosted server refuses actions, and the cache is not used with them.", "An omitted maxAge reuses nothing: every page is fetched live unless the request sets maxAge above 0, minAge or lockdown (Firecrawl reuses its own index by default; its Python SDK sends maxAge 4 hours). A reused page is one Octocrawl itself stored, under the same options, on this server (its task root), never a shared index; only a success is stored, and data.metadata says cacheState hit with cachedAt (its fetch time) or miss when one was looked up. lockdown with no stored result is HTTP 404 SCRAPE_LOCKDOWN_CACHE_MISS on scrape, and nothing is fetched; a crawl in lockdown needs sitemap skip, and each page with no stored result is failed with cache_miss. Mode authed neither stores nor reuses. A Firecrawl body never sets useCached, Octocrawl's reuse of a crawl's own pages on resume.", "Omitted limit / maxDepth stay unbounded on a local server; a hosted server takes its crawl limit for an omitted or null limit and refuses a larger one. Firecrawl defaults are 10000 / 10.", "maxDepth counts link hops from the start URL (Firecrawl calls that maxDiscoveryDepth); Firecrawl maxDepth counts URL path depth.", "Crawl start is mapped onto native POST /v1/crawl; the shim itself returns 200 {success,id,url}.", "creditsUsed and expiresAt are null: Octocrawl counts no credits and keeps crawl results until their task directory is deleted.", "Crawl status describes the latest attempt: completed counts its successful pages, total adds its failed, blocked and duplicate pages and, while this API process runs the crawl, the pages in flight and queued (null for a paused crawl); data lists the failed and blocked pages too (with metadata.error) but not the duplicates, whose content is an earlier entry's, up to 100 per response (limit 1 to 1000) with next carrying an Octocrawl cursor; skip is rejected.", "Scrape maps url, formats, onlyMainContent, includeTags, excludeTags, waitFor, timeout, headers, mobile, skipTlsVerification, fastMode, blockAds, removeBase64Images, maxAge, minAge, storeInCache, lockdown, actions, proxy (basic as access standard; stealth and auto as access enhanced, which a server without an access grant of tier enhanced refuses by name), origin and integration; crawl maps url, limit (as maxPages), maxDepth, includePaths, excludePaths, regexOnFullURL, ignoreQueryParameters, deduplicateSimilarURLs, crawlEntireDomain (and its v1 name allowBackwardLinks), allowSubdomains, allowExternalLinks, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only), maxConcurrency, ignoreRobotsTxt (v2; a local server only), origin, integration and the same scrapeOptions (applied to every page). The formats are markdown, links, html, rawHtml, images, screenshot (also screenshot@fullPage, and { type: \"screenshot\", fullPage, quality, viewport }) and an { type: \"attributes\", selectors } entry; other formats and parameters the shim does not map (location, json, ...) are rejected by name with HTTP 400 and success: false; a refusal of stealth, of access enhanced (proxy: stealth or auto) on a server without the grant, or of ignoreRobotsTxt on a scrape or map names the supported route in agent_hints.", "A crawl follows links inside the start URL's path subtree on its host and www twin by default (crawlEntireDomain false), folds /a and /a/, / and /index.html, www and apex, http and https into one page (deduplicateSimilarURLs true) and reports every collapsed or refused link in the native crawl status (discovery) and each page's trace (links_offered); allowSubdomains takes every host under the start URL's apex (no public-suffix list), allowExternalLinks every host, each page with its own robots.txt read.", "sitemap (default include, as in Firecrawl) reads the sitemaps the start URL's robots.txt names, or /sitemap.xml, with the crawl's own http identity, robots.txt verdict, SSRF checks and proxy, and queues their URLs ahead of the start page's links under the same host, subtree, path and depth rules; only follows no page link; skip reads none. The native crawl status lists every sitemap file read, refused or unreadable in discovery.sitemap; the shim's status carries nothing of it, and sitemap fetches have no signed compliance record. maxConcurrency caps the pages one crawl fetches at once, at most the service's worker count (HTTP 400 above it), and never raises the per-host ceiling.", "screenshot (data.screenshot, a data:image/png;base64 string, or image/jpeg with quality 1 to 100) is captured on the local browser rung alone, which such a request selects (no http attempt; a server without a browser rung refuses the format with HTTP 400): after load, stability and waitFor, before the DOM is read, CSS-pixel sized at the declared 1280x800 viewport (device scale factor 2 is declared, not baked into the image) or at the viewport asked for (integers 320..1920 by 240..1080, within the declared screen; a window size, not a change of identity); fullPage captures the document's whole height at that width without scrolling first, so sections a page loads on scroll may show unloaded. A capture the browser could not make leaves data.screenshot null with a screenshot_unavailable warning while the page stands; a file or a page that was not rendered has null too. Firecrawl captures at its own viewport and may return a URL instead of the image.", "images (data.images) lists every image URL of the whole document as received: img src and srcset candidates, picture sources, lazy data-src/data-srcset/data-lazy-src/data-original, video posters, image_src links, og:image and twitter:image, absolute http(s) with the fragment stripped, each once, in document order, data: URIs left out; includeTags, excludeTags and onlyMainContent do not narrow it. attributes (data.attributes) gives, per selector, the named attribute's values as written, elements without it skipped; a selector Octocrawl does not match is HTTP 400 by name, as for includeTags. Both are absent for a file and for a page that is success: false. removeBase64Images (default true) keeps an image's alt text where Firecrawl writes a (<Base64-Image-Removed>) placeholder; false keeps the data: URI in the Markdown.", "origin (the Firecrawl SDKs' client label) and integration are stored, not echoed: the scrape record (GET /v1/scrapes/:id) and the crawl task carry them, and nothing sent to the target changes.", "data.metadata carries scrapeId (a UUID per call, which GET /v1/scrapes/:id looks up), proxyUsed (operator for the server's environment proxy, user for the caller's own egress, else null), timezone (the browser rung's declared zone, null on the HTTP rung), creditsUsed: null (Octocrawl counts no credits), concurrencyLimited and concurrencyQueueDurationMs (whether and how long the per-origin ceiling held the fetch back), and cacheState and cachedAt when the cache was asked (never a guessed miss).", "A page whose result Octocrawl has advice about (a login wall, a robots.txt rule, a gate, a cut, a script-filled shell) carries data.agent_hints, one sentence each; the native response calls them agentHints. A request refused for an option Octocrawl does not offer carries agent_hints in the error envelope, and a caller over the server's per-minute rate limit gets HTTP 429 { success: false, error, code: rate_limited, agent_hints } with Retry-After.", "headers never override the User-Agent, the client hints, a credential (authorization, cookie) or a transport header: such a header is HTTP 400 naming it, where Firecrawl sends it. The headers go to the requested origin after the declared identity and are on the record (the trace, the browser lane's signed sentHeaders); both rungs withhold them from a redirect hop to another origin and say so (custom_headers_withheld).", "mobile selects a declared Android Chrome identity (User-Agent, client hints, 412x915 viewport, touch) that robots.txt is evaluated against and the record carries; the page is whatever the site serves to it, with no DOM rewriting. It is refused with mode research.", "skipTlsVerification relaxes certificate verification for one local request and its robots.txt lookup, recorded in the trace (tls_verification_skipped) and a tls_unverified warning the native response carries; a hosted Octocrawl refuses it with HTTP 400. Without it a bad certificate is success: false with failed: tls_error. Firecrawl's Python SDK sends true by default; Octocrawl verifies by default.", "fastMode keeps the http rung alone: a page that needs scripts is success: false with failed: empty_unverified, never rendered; waitFor has no effect under it. Firecrawl's fast mode still renders.", "blockAds (default true) aborts requests to a bundled list of about 50 ad-serving hosts on the local browser rung and removes ad and cookie-banner elements before extraction; false keeps them. The list is curated, not EasyList: ads from hosts outside it are not blocked.", "html is the cleaned HTML the markdown is written from: the main content, the whole page without scripts, styles, form controls and embedded media when onlyMainContent is false, or a <body> holding the includeTags elements. rawHtml is the page as the answering rung received it: the response body on the HTTP rung, the rendered DOM on a browser rung. Both are null for a file and for a page that is success: false.", "includeTags keeps only the named elements, in document order, whatever onlyMainContent says; excludeTags removes elements from the main content, the whole page and an includeTags selection. A selector that does not parse, or that uses a sibling combinator, a positional pseudo-class, :has() or another pseudo-class Octocrawl does not match, is rejected with HTTP 400.", "An omitted timeout stays 300000 ms (Firecrawl: 30000). A timeout is answered with HTTP 200: success: true with the content fetched so far (native status partial), or success: false with failed: timeout; Firecrawl answers it with an error.", "waitFor skips the HTTP rung, which cannot run scripts, and starts at the browser rung; the wait counts toward timeout.", "metadata has title, description, language, keywords, robots and favicon only when the page declares them, and the Open Graph (ogTitle, ogDescription, ogUrl, ogImage, ogAudio, ogVideo, ogDeterminer, ogLocale, ogLocaleAlternate, ogSiteName), Dublin Core (dcTermsCreated, dcDateCreated, dcDate, dcTermsType, dcType, dcTermsAudience, dcTermsSubject, dcSubject, dcDescription, dcTermsKeywords) and article (publishedTime, modifiedTime, articleTag, articleSection) tags under Firecrawl's names, each only when the page states it, as written (no date normalisation, no fallback from another tag); twitter:* and other meta tags are not passed through, and a failed or blocked page has none.", "A PDF answers success: true with its text layer as markdown and metadata.numPages (the document's page count). parsers maps Firecrawl's pdf entry (the string or { type: \"pdf\", mode, maxPages, pages, pageMarkers }); mode fast and auto both read the text layer, and mode ocr and the image parser are refused by name. pageMarkers is false unless asked, as on Firecrawl; with true Octocrawl writes a <!-- page N --> line before each page (Firecrawl writes --- and the marker between pages). pages: true adds data.pages, [{ pageNumber, markdown }]. maxPages (1 to 10000) reads the first pages, and a cut it asked for stays success: true. parsers [] or v1 parsePDF false reads no PDF: success: true with markdown null and the file saved as received. A PDF without a text layer is success: false with failed: empty_unverified (no OCR). CSV, JSON and text files give their text as received; XLSX, XLS and ZIP files are success: true with markdown null. A file over W2L_MAX_FILE_BYTES is success: false with failed: body_too_large.", "Map (POST /fc/v1/map) maps url, search, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only; both v1 flags true is HTTP 400), includeSubdomains, ignoreQueryParameters, limit (1 to 100000, default 5000), timeout (1000 to 300000 ms for the whole map, default 60000; Firecrawl documents no default), origin and integration onto native POST /v1/map, and answers 200 { success: true, id, links: [url strings], warning?, agent_hints? }, or 200 { success: false, id, error, links: [] } when the map found nothing because a source failed or its deadline passed; useIndex, location, ignoreCache, threatProtection and auditMetadata are refused by name (useIndex with the hint that Octocrawl keeps no URL index). Omitted options take Octocrawl's defaults: includeSubdomains and ignoreQueryParameters are false, where Firecrawl v2 documents true for both. search keeps the URLs in which every word appears in the decoded URL or the title in hand, in discovery order; Firecrawl orders by relevance. A map reads the sitemaps the site declares and one page body (the start URL, http rung only), so a site without a sitemap maps only its start page's links; a title is the start page's own, an anchor's text or a sitemap's <news:title>, never fetched from the target; robots-disallowed URLs are left out and counted on the native response (GET /v1/maps/:id), which also records every sitemap file read.", "A crawl's webhook (a URL string or { url, headers, metadata, events }) is mapped onto the native webhook and its receiver gets Firecrawl's payload shape: { success, type: crawl.started | crawl.page | crawl.completed | crawl.failed, id, data: [page], metadata, error? }, one durable delivery per event with retries, every request carrying x-w2l-event-id, x-w2l-event-version and x-w2l-delivery-id (and the signature pair with secretEnv, a native option). A cancelled crawl is crawl.failed with error \"cancelled\". The native rules apply: https (plain http for a loopback receiver of a local server only), no content-type, host or x-w2l-* header, at most 32 headers and 32 metadata strings; a hosted server takes public https receivers only. GET /v1/deliveries?jobId=<id> on the native API lists the deliveries."];
 export interface FirecrawlPage {
     markdown: string | null;
     /** Present when the `html` format was asked for; null when the page has none (a file, a page that did not succeed). */