@octocrawl/sdk 0.3.0 → 0.3.1

package/dist/index.cjs

@@ -218,11 +218,11 @@ var PAGE_KEYS = ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "inclu
 var ATTRIBUTION_KEYS = ["origin", "integration"];
 var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var CRAWL_SCOPE_KEYS = ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-var CRAWL_KEYS = ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var CRAWL_KEYS = ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var BATCH_KEYS = ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var BATCH_APPEND_KEYS = ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", ...ATTRIBUTION_KEYS];
 var MAP_SCOPE_KEYS = ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
-var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", ...ATTRIBUTION_KEYS];
+var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", "ignoreRobotsTxt", ...ATTRIBUTION_KEYS];
 var WEBHOOK_HEADERS_MESSAGE = `webhook.headers must be an object of at most ${MAX_WEBHOOK_HEADERS} string values`;
 var WEBHOOK_METADATA_MESSAGE = `webhook.metadata must be an object of at most ${MAX_WEBHOOK_METADATA_ENTRIES} string values of at most ${MAX_WEBHOOK_METADATA_VALUE_LENGTH} characters`;
 var WEBHOOK_EVENTS_MESSAGE = `webhook.events must be a non-empty array of ${WEBHOOK_EVENTS.join(", ")} without duplicates`;
@@ -243,9 +243,9 @@ var SHIM_MAP_KEYS = ["url", "search", "sitemap", "ignoreSitemap", "sitemapOnly",
 // packages/contracts/dist/evidenceRecord.js
 var keysOf = () => (keys) => keys;
 var EVIDENCE_RECORD_KEYS = {
-  record: keysOf()(["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions"]),
+  record: keysOf()(["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions", "access"]),
   redirectChain: keysOf()(["urls", "complete"]),
-  robotsDecision: keysOf()(["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride"]),
+  robotsDecision: keysOf()(["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride", "overrideBasis"]),
   outputSha256: keysOf()(["markdown", "json"]),
   extractor: keysOf()(["name", "version", "commit"]),
   fieldEvidence: keysOf()(["source", "locator"]),
@@ -253,11 +253,12 @@ var EVIDENCE_RECORD_KEYS = {
   identity: keysOf()(["userAgent", "mode", "contact", "device", "requestHeaders"]),
   pageActions: keysOf()(["steps", "scriptRan"]),
   pageActionStep: keysOf()(["type", "outcome"]),
-  requestHeader: keysOf()(["name", "valueSha256"])
+  requestHeader: keysOf()(["name", "valueSha256"]),
+  access: keysOf()(["route", "executor", "executorVersion", "profile", "externalCostUsd"])
 };
 
 // packages/sdk/src/version.ts
-var SDK_VERSION = "0.3.0";
+var SDK_VERSION = "0.3.1";
 
 // packages/sdk/src/watcher.ts
 var DEFAULT_WATCH_POLL_INTERVAL_MS = 2e3;

package/dist/index.js

@@ -181,11 +181,11 @@ var PAGE_KEYS = ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "inclu
 var ATTRIBUTION_KEYS = ["origin", "integration"];
 var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var CRAWL_SCOPE_KEYS = ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-var CRAWL_KEYS = ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var CRAWL_KEYS = ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var BATCH_KEYS = ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var BATCH_APPEND_KEYS = ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", ...ATTRIBUTION_KEYS];
 var MAP_SCOPE_KEYS = ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
-var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", ...ATTRIBUTION_KEYS];
+var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", "ignoreRobotsTxt", ...ATTRIBUTION_KEYS];
 var WEBHOOK_HEADERS_MESSAGE = `webhook.headers must be an object of at most ${MAX_WEBHOOK_HEADERS} string values`;
 var WEBHOOK_METADATA_MESSAGE = `webhook.metadata must be an object of at most ${MAX_WEBHOOK_METADATA_ENTRIES} string values of at most ${MAX_WEBHOOK_METADATA_VALUE_LENGTH} characters`;
 var WEBHOOK_EVENTS_MESSAGE = `webhook.events must be a non-empty array of ${WEBHOOK_EVENTS.join(", ")} without duplicates`;
@@ -206,9 +206,9 @@ var SHIM_MAP_KEYS = ["url", "search", "sitemap", "ignoreSitemap", "sitemapOnly",
 // packages/contracts/dist/evidenceRecord.js
 var keysOf = () => (keys) => keys;
 var EVIDENCE_RECORD_KEYS = {
-  record: keysOf()(["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions"]),
+  record: keysOf()(["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions", "access"]),
   redirectChain: keysOf()(["urls", "complete"]),
-  robotsDecision: keysOf()(["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride"]),
+  robotsDecision: keysOf()(["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride", "overrideBasis"]),
   outputSha256: keysOf()(["markdown", "json"]),
   extractor: keysOf()(["name", "version", "commit"]),
   fieldEvidence: keysOf()(["source", "locator"]),
@@ -216,11 +216,12 @@ var EVIDENCE_RECORD_KEYS = {
   identity: keysOf()(["userAgent", "mode", "contact", "device", "requestHeaders"]),
   pageActions: keysOf()(["steps", "scriptRan"]),
   pageActionStep: keysOf()(["type", "outcome"]),
-  requestHeader: keysOf()(["name", "valueSha256"])
+  requestHeader: keysOf()(["name", "valueSha256"]),
+  access: keysOf()(["route", "executor", "executorVersion", "profile", "externalCostUsd"])
 };
 
 // packages/sdk/src/version.ts
-var SDK_VERSION = "0.3.0";
+var SDK_VERSION = "0.3.1";
 
 // packages/sdk/src/watcher.ts
 var DEFAULT_WATCH_POLL_INTERVAL_MS = 2e3;

package/dist/types/cjs/client.d.ts

@@ -33,7 +33,7 @@ export interface RequestOptions {
     origin?: string;
 }
 /** What the SDK records as `origin` unless the caller or the host says otherwise. */
-export declare const SDK_ORIGIN = "js-sdk@0.3.0";
+export declare const SDK_ORIGIN = "js-sdk@0.3.1";
 /**
  * Polling for waitBatch and waitCrawl. A status request that fails with a
  * network error, HTTP 408, 429 or 5xx is retried: after 1, 2, 4, 8, then

package/dist/types/cjs/contracts/api.d.ts

@@ -411,6 +411,16 @@ export interface CrawlStartRequest extends PageOptions, RequestAttribution {
     idempotencyKey?: string;
     /** A receiver for the crawl's events (`started`, one `page` per page recorded, then `completed`, `failed` or `cancelled`); see WebhookConfig. */
     webhook?: WebhookOption;
+    /**
+     * Fetch the pages and sitemap files robots.txt disallows, or whose
+     * robots.txt could not be read (Firecrawl v2's name). robots.txt is still
+     * read for every host and its verdict recorded on each page, Crawl-delay
+     * applied, with a `robots_overridden` warning and `overrideBasis:
+     * "ignore_robots_txt"` where a rule was set aside. A local server only: a
+     * hosted one refuses it by name. Default false: a crawl's links obey
+     * robots.txt.
+     */
+    ignoreRobotsTxt?: boolean;
 }
 /** What the parser hands the engine: the request plus, from the `/fc` shim, the payload shape its receiver expects. */
 export type ParsedCrawlStartRequest = CrawlStartRequest & {
@@ -454,6 +464,14 @@ export interface MapRequest extends RequestAttribution {
     crawlEntireDomain?: boolean;
     /** Default true, as on a crawl; a returned http link gives way to its https variant when that comes too, on an origin whose robots.txt the map read anyway and which allows it. */
     deduplicateSimilarURLs?: boolean;
+    /**
+     * Return the URLs robots.txt disallows, or whose robots.txt could not be
+     * read, with that verdict on each link (`robots: "disallowed"` or
+     * `"unreachable"`), and read the start page and sitemap files past it.
+     * robots.txt is still read, within MAP_MAX_ROBOTS_HOSTS. A local server
+     * only: a hosted one refuses it by name. Default false.
+     */
+    ignoreRobotsTxt?: boolean;
 }
 /** A map's `search`: at most this many characters after trimming, and this many whitespace-separated words. */
 export declare const MAP_SEARCH_MAX_CHARS = 200;
@@ -482,6 +500,7 @@ export interface ActiveCrawlOptions {
     allowExternalLinks: boolean;
     regexOnFullURL: boolean;
     maxConcurrency: number | null;
+    ignoreRobotsTxt: boolean;
     /** The per-page options every page of the crawl gets: its formats, `includeLinks` and the page options. */
     scrapeOptions: PageOptions & {
         formats: readonly ScrapeFormat[];
@@ -809,10 +828,10 @@ export declare class RequestError extends Error {
 }
 /** The hints a refusal carries for the options W2L does not offer: the next honest step, never a way around the refusal. */
 export declare const REFUSAL_HINTS: {
-    readonly stealth: "W2L does not offer a stealth mode or stealth proxies; a proxy or session you own (mode authed) is the supported route";
-    readonly ignoreRobotsTxt: "robots.txt is always read; a robotsOverride with a recorded reason fetches one URL past its rule, on the record";
-    readonly hostedSkipTlsVerification: "a hosted server verifies every certificate; run W2L locally to use skipTlsVerification, which is recorded in the trace and a tls_unverified warning";
-    readonly useIndex: "W2L keeps no URL index: a map reads the sitemaps the site declares and its start page, on the record; crawl reads further pages";
+    readonly stealth: "Octocrawl has no stealth option on a request: a provider's stealth or challenge solving runs only on a server started with an access grant that names it (--access-grant, ADR 0005); a proxy or session you own (mode authed) is the other route";
+    readonly ignoreRobotsTxt: "robots.txt is always read and recorded; on a local server a URL a scrape or batch names is fetched whatever it says, and ignoreRobotsTxt on a crawl or map fetches the links it disallows, on the record";
+    readonly hostedSkipTlsVerification: "a hosted server verifies every certificate; run Octocrawl locally to use skipTlsVerification, which is recorded in the trace and a tls_unverified warning";
+    readonly useIndex: "Octocrawl keeps no URL index: a map reads the sitemaps the site declares and its start page, on the record; crawl reads further pages";
     readonly actions: "actions run on scrape and batch, where each page named gets the same steps; a crawl or a map does not take them";
 };
 /** The hint for a refused request key, or null when the key has none (an option W2L simply does not know). */
@@ -821,14 +840,14 @@ export declare const PAGE_KEYS: readonly ["onlyMainContent", "waitFor", "timeout
 export declare const ATTRIBUTION_KEYS: readonly ["origin", "integration"];
 export declare const SCRAPE_KEYS: readonly ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
 export declare const CRAWL_SCOPE_KEYS: readonly ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-export declare const CRAWL_KEYS: readonly ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", "regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks", "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const CRAWL_KEYS: readonly ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", "regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks", "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
 export declare const BATCH_KEYS: readonly ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
 /** What a batch body may carry beside `appendToId`: the job's own options are not among them (the scope no-ops change nothing, so they may come along). */
 export declare const BATCH_APPEND_KEYS: readonly ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", "origin", "integration"];
 /** The scope options a map takes under their crawl names; allowSubdomains is includeSubdomains on a map, and allowExternalLinks is not offered. */
 export declare const MAP_SCOPE_KEYS: readonly ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
 /** What a map takes. No page option (headers, mobile, skipTlsVerification, formats, ...): a map has nothing to loosen. */
-export declare const MAP_KEYS: readonly ["url", "mode", "limit", "timeout", "search", "sitemap", "includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs", "includePaths", "excludePaths", "origin", "integration"];
+export declare const MAP_KEYS: readonly ["url", "mode", "limit", "timeout", "search", "sitemap", "includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs", "includePaths", "excludePaths", "ignoreRobotsTxt", "origin", "integration"];
 /** Why a webhook header (lower-cased name) cannot be sent, or null when it can: W2L's own and the transport's names are reserved. */
 export declare function webhookHeaderRefusal(name: string): string | null;
 /**
@@ -869,8 +888,8 @@ export declare function parseCrawlStartRequest(body: unknown): CrawlStartRequest
  * A map request: url, mode (standard or research), limit, timeout, search,
  * sitemap, includeSubdomains, the crawl's scope options under their crawl
  * names (ignoreQueryParameters, includePaths, excludePaths, regexOnFullURL,
- * crawlEntireDomain, deduplicateSimilarURLs), origin and integration.
- * Anything else is refused by name, `useIndex` with the supported route;
+ * crawlEntireDomain, deduplicateSimilarURLs), ignoreRobotsTxt, origin and
+ * integration. Anything else is refused by name, `useIndex` with the supported route;
  * nothing is silently ignored.
  */
 export declare function parseMapRequest(body: unknown): MapRequest;

package/dist/types/cjs/contracts/checkpoint.d.ts

@@ -106,6 +106,8 @@ export interface Task {
         maxConcurrency?: number | null;
         /** The crawl's webhook, when the request set one. */
         webhook?: StoredJobWebhook;
+        /** The crawl fetches what robots.txt disallows, on the record (CrawlStartRequest.ignoreRobotsTxt); absent: it obeys. A server that takes no override resumes it obeying. */
+        ignoreRobotsTxt?: boolean;
     } & PageOptions;
     /** Who started the task (`origin`, `integration`), stored with it and reported as `attribution` on its status; absent when the request named neither. */
     attribution?: RequestAttribution;

package/dist/types/cjs/contracts/compliance.d.ts

@@ -109,13 +109,24 @@ export declare function researchUserAgent(contact?: string | null, host?: string
 export declare function declaredContact(userAgent: string): string | null;
 /** Whether a User-Agent is one research mode declares, in either format. */
 export declare function isResearchUserAgent(userAgent: string): boolean;
+/**
+ * The product token a robots.txt names to address Octocrawl itself
+ * (`User-agent: Octocrawl`), whatever User-Agent header a request sends. It
+ * is matched in every mode. A rule written for it (or for research mode's own
+ * token) is not set aside for a URL the request names or for a crawl or map
+ * started with ignoreRobotsTxt; only a recorded robotsOverride sets it aside.
+ */
+export declare const PRODUCT_ROBOTS_TOKEN = "octocrawl";
 /**
  * The text robots.txt `User-agent` lines are matched against for a
- * User-Agent W2L sends. SEC's format names no product token, so the research
- * token is added: a group for w2l-research governs research requests to
- * SEC.gov as it does on every other host.
+ * User-Agent Octocrawl sends: the header, with the product token added. SEC's
+ * format names no product token, so the research token is added as well: a
+ * group for w2l-research governs research requests to SEC.gov as it does on
+ * every other host.
  */
 export declare function robotsAgent(userAgent: string): string;
+/** Whether the robots.txt group that decided names Octocrawl itself (its product token, or research mode's) rather than every crawler (`*`). */
+export declare function isOctocrawlRobotsGroup(matchedAgent: string | null | undefined): boolean;
 /** The operator's contact from `W2L_CONTACT`, trimmed; null when unset or blank. The error never repeats the value. */
 export declare function operatorContact(env: Readonly<Record<string, string | undefined>>): string | null;
 /** An operator policy whose research-mode requests declare `W2L_CONTACT`, when it is set. */
@@ -254,6 +265,23 @@ export interface RobotsOverride {
     /** Who recorded the decision, when the caller wants that on the record. */
     recordedBy?: string;
 }
+/**
+ * On whose word a fetch went past robots.txt. `robots_override`: the
+ * caller's recorded decision for this URL (`RobotsOverride`).
+ * `user_named_url`: a local server fetching a URL the request named (a
+ * scrape, a batch entry), since robots.txt addresses crawlers that discover
+ * links, not the pages a person names. `ignore_robots_txt`: a crawl or map
+ * the caller started with `ignoreRobotsTxt` on a local server.
+ */
+export type RobotsOverrideBasis = 'robots_override' | 'user_named_url' | 'ignore_robots_txt';
+/**
+ * A robots override as a lane applies it: the caller's recorded one, or one
+ * W2L applies by rule, which says so in `basis` (absent: the caller's,
+ * `robots_override`). Set by W2L, never read from a request.
+ */
+export interface AppliedRobotsOverride extends RobotsOverride {
+    basis?: Exclude<RobotsOverrideBasis, 'robots_override'>;
+}
 /**
  * The outcome of consulting robots.txt for a single target URL. One record per
  * fetch. `consulted` distinguishes "we checked and it said X" from "there was
@@ -288,11 +316,11 @@ export interface RobotsDecision {
      */
     unreachable?: RobotsUnreachable;
     /**
-     * Present when a disallow the publisher wrote was set aside by a recorded
-     * decision: the fetch went ahead (`skippedFetch: false`) and this says on
-     * whose word. Never set for an unreachable robots.txt.
+     * Present when a disallow was set aside, the publisher's or the one an
+     * unreachable robots.txt implies: the fetch went ahead (`skippedFetch:
+     * false`) and this says on whose word.
      */
-    override?: RobotsOverride;
+    override?: AppliedRobotsOverride;
 }
 /**
  * What actually went on the wire. Sorted by header name, lowercased names.

package/dist/types/cjs/contracts/crawl.d.ts

@@ -86,7 +86,7 @@ export interface SitemapFileRecord {
     kind: SitemapFileKind;
     /** `<loc>` entries the file holds (child sitemaps for an index), http(s) ones only; null when the file was not parsed. */
     entries: number | null;
-    /** The robots.txt verdict for the file's own URL under the crawl's identity (an unreachable robots.txt is `disallowed`, as for a page); null when the URL failed its egress check before robots.txt was consulted. */
+    /** The robots.txt verdict for the file's own URL under the crawl's identity (an unreachable robots.txt is `disallowed`, as for a page); null when the URL failed its egress check before robots.txt was consulted. A `disallowed` file is `refused`, unless the crawl or map was started with ignoreRobotsTxt and read it. */
     robots: 'allowed' | 'disallowed' | 'no_robots' | null;
     /** Whether the request left through the operator's environment proxy (local mode); a hosted server never has one. */
     proxyUsed: boolean;

package/dist/types/cjs/contracts/evidenceRecord.d.ts

@@ -15,7 +15,7 @@ import type { PageActionType } from './actions.js';
  * changes its meaning. A change that cannot follow this rule gets a new
  * schemaVersion (`w2l.evidence/2`) and a new schema file.
  */
-import type { CrawlMode, RobotsUnreachable } from './compliance.js';
+import type { CrawlMode, RobotsOverrideBasis, RobotsUnreachable } from './compliance.js';
 import type { BlockReason, BudgetKind, FailureReason, Lane, ResultStatus } from './status.js';
 export declare const EVIDENCE_SCHEMA_VERSION = "w2l.evidence/1";
 /**
@@ -33,6 +33,30 @@ export type FieldEvidenceSource = (typeof FIELD_EVIDENCE_SOURCES)[number];
  */
 export declare const EVIDENCE_ARTIFACT_KINDS: readonly ["snapshot", "screenshot", "file"];
 export type EvidenceArtifactKind = (typeof EVIDENCE_ARTIFACT_KINDS)[number];
+/**
+ * The route that produced a result (ADR 0005): `http` (W2L's HTTP client, undici), `http_compat` (the
+ * browser-compatible HTTP transport, impit), `browser` (the local headless browser), `enhanced_browser`
+ * (the local browser on Patchright), `authed_browser` (the local browser with the user's saved login),
+ * `user_browser` (the person's own browser after a handoff), `vendor` (a third-party browser service).
+ */
+export declare const ACCESS_ROUTES: readonly ["http", "http_compat", "browser", "enhanced_browser", "authed_browser", "user_browser", "vendor"];
+export type AccessRoute = (typeof ACCESS_ROUTES)[number];
+/**
+ * How a result was reached, read from the result's own trace. Added to v1 with enhanced access
+ * (EVIDENCE_RECORD_ADDED_KEYS).
+ */
+export interface EvidenceAccess {
+    /** Null when no lane produced the result (a run cut before a rung answered, a rung that threw, a lockdown miss). */
+    route: AccessRoute | null;
+    /** The client that sent the requests: undici, impit, playwright, patchright, the person's browser, or the vendor's id; null when the result does not say. */
+    executor: string | null;
+    /** The executor's version as the lane reported it; null when it reported none. */
+    executorVersion: string | null;
+    /** The browser profile the HTTP transport sent (impit's); null on every other route. Its TLS fingerprint was not observed. */
+    profile: string | null;
+    /** Third-party spend of the run that produced the result: 0 when no paid service was called, null when a called service stated no price. */
+    externalCostUsd: number | null;
+}
 export interface EvidenceRedirectChain {
     /**
      * Every URL W2L requested for the page, in order: the requested URL first,
@@ -61,8 +85,10 @@ export interface EvidenceRobotsDecision {
     /** Why robots.txt could not be fetched (then `decision` is `disallowed`, RFC 9309 §2.3.1.4); null when it was. */
     unreachable: RobotsUnreachable | null;
     crawlDelayMs: number | null;
-    /** Whether the fetch went ahead under a recorded robots override although `decision` is `disallowed`; the reason is in the trace, the warnings and, in the browser lane, the compliance record. */
+    /** Whether the fetch went ahead although `decision` is `disallowed`; on whose word is `overrideBasis`, and the reason is in the trace, the warnings and, in the browser lane, the compliance record. */
     userOverride: boolean;
+    /** On whose word the disallow was set aside (`RobotsOverrideBasis`); null when it was not. Optional in the v1 schema file, written on every record. */
+    overrideBasis: RobotsOverrideBasis | null;
 }
 export interface EvidenceOutputSha256 {
     /** SHA-256 of the UTF-8 bytes of the delivered `markdown`; null when none was delivered. */
@@ -169,12 +195,14 @@ export interface EvidenceRecord {
      * the caller's ran in it, so its content may be the script's.
      */
     pageActions: EvidencePageActions | null;
+    /** How the result was reached (EvidenceAccess). Added to v1 later (EVIDENCE_RECORD_ADDED_KEYS). */
+    access: EvidenceAccess;
 }
 /** Field order of the record and of each nested object, as in the schema file. */
 export declare const EVIDENCE_RECORD_KEYS: {
-    readonly record: readonly ["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions"];
+    readonly record: readonly ["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions", "access"];
     readonly redirectChain: readonly ["urls", "complete"];
-    readonly robotsDecision: readonly ["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride"];
+    readonly robotsDecision: readonly ["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride", "overrideBasis"];
     readonly outputSha256: readonly ["markdown", "json"];
     readonly extractor: readonly ["name", "version", "commit"];
     readonly fieldEvidence: readonly ["source", "locator"];
@@ -183,6 +211,7 @@ export declare const EVIDENCE_RECORD_KEYS: {
     readonly pageActions: readonly ["steps", "scriptRan"];
     readonly pageActionStep: readonly ["type", "outcome"];
     readonly requestHeader: readonly ["name", "valueSha256"];
+    readonly access: readonly ["route", "executor", "executorVersion", "profile", "externalCostUsd"];
 };
 /**
  * Keys added to v1 after it was first published: optional in the schema, so

package/dist/types/cjs/contracts/execution.d.ts

@@ -1,4 +1,4 @@
-import type { RobotsOverride } from './compliance.js';
+import type { AppliedRobotsOverride } from './compliance.js';
 import type { FetchWarning, TraceEvent } from './result.js';
 import type { AttributeSelector, ListFormatRequest, ScreenshotOptions } from './structured.js';
 import type { PageAction } from './actions.js';
@@ -14,6 +14,47 @@ export interface ExecutionContext {
      * when that lane never returns (a deadline) or another rung's result answers.
      */
     onRobotsOverride?: (applied: RobotsOverrideApplied) => void;
+    /**
+     * The task's cookie session (ADR 0005 `egress_sessions`): the cookies a page's responses set,
+     * sent again to their site on the task's later pages, by every local rung. Absent: no cookie is
+     * kept or sent, as before.
+     */
+    cookieSession?: CookieSession;
+}
+/** A cookie as a browser context takes and gives it (Playwright's shape). */
+export interface ContextCookie {
+    name: string;
+    value: string;
+    domain: string;
+    path: string;
+    /** Seconds since the epoch; -1 for a session cookie. */
+    expires: number;
+    httpOnly: boolean;
+    secure: boolean;
+    sameSite: 'Strict' | 'Lax' | 'None';
+}
+/**
+ * One task's cookies, matched to a URL by RFC 6265 (domain, path, secure, expiry). Values never
+ * leave it into a record or a trace: a lane reports the session's `id` and counts.
+ */
+export interface CookieSession {
+    /** An opaque id for the record, unrelated to any cookie value. */
+    readonly id: string;
+    /** The `Cookie` header for a request to this URL; empty when none applies. */
+    cookieHeader(url: string): Promise<string>;
+    /** Keep the `Set-Cookie` lines a response to this URL carried; returns how many were kept. */
+    store(url: string, setCookies: readonly string[]): Promise<number>;
+    /** The cookies a browser context loading this URL should start with. */
+    browserCookies(url: string): Promise<ContextCookie[]>;
+    /**
+     * Keep what a browser context changed: given the cookies it started with and the ones it holds
+     * after the page, store the new and changed ones and delete the ones it dropped, unless another
+     * page of the task changed that cookie meanwhile. Unchanged cookies are left as the session has them.
+     */
+    storeBrowserChanges(startedWith: readonly ContextCookie[], held: readonly ContextCookie[]): Promise<{
+        kept: number;
+        removed: number;
+    }>;
 }
 /** What a lane reports when it sets a robots.txt rule aside under a recorded override. */
 export interface RobotsOverrideApplied {
@@ -52,15 +93,18 @@ export interface FetchOptions {
      */
     parsers?: readonly PdfParser[];
     /**
-     * A recorded decision to fetch this one URL although its host's robots.txt
-     * disallows it. robots.txt is still read and its verdict recorded; the
-     * override goes into the trace, the warnings and, in the browser lane, the
-     * compliance record. An unreachable robots.txt is not set aside. The HTTP
-     * and local browser lanes apply it; the provider lane takes none.
-     * Set per URL by the caller (a scrape's `robotsOverride`, a batch's
-     * `robotsOverrides` entry), never by a batch or crawl for every page.
-     */
-    robotsOverride?: RobotsOverride;
+     * A decision to fetch this one URL although its host's robots.txt
+     * disallows it or could not be read. robots.txt is still read and its
+     * verdict recorded, Crawl-delay included; the override goes into the
+     * trace, the warnings and, in the browser lane, the compliance record. The
+     * HTTP and local browser lanes apply it; the provider lane takes none.
+     * Set by the engine: a scrape's `robotsOverride` or a batch's
+     * `robotsOverrides` entry, else, on a local server, `user_named_url` for
+     * every URL a scrape or batch names and `ignore_robots_txt` for a crawl's
+     * pages when the crawl asked. A navigation a page's steps make to another
+     * URL is checked against robots.txt whatever this says.
+     */
+    robotsOverride?: AppliedRobotsOverride;
     /**
      * CSS selectors naming the only elements to keep. The content is those
      * elements, in document order, copied from the page before anything is

package/dist/types/cjs/contracts/extractor.d.ts

@@ -237,7 +237,7 @@ export interface DocumentExtraction {
     labelledValues?: readonly LabelledValue[];
 }
 /** The rule that decided a page's data is most likely rendered client-side (see RenderSignals). */
-export type RenderReason = 'empty_table_with_scripts' | 'empty_app_root' | 'script_shell' | 'js_fallback' | 'hydration_shell' | 'aria_busy';
+export type RenderReason = 'empty_table_with_scripts' | 'empty_app_root' | 'script_shell' | 'js_fallback' | 'hydration_shell' | 'aria_busy' | 'hydration_list_partial';
 /** A client-side rendering marker found in the page as received (see RenderSignals). */
 export type RenderMarker = 'hydration_state' | 'app_root_empty' | 'noscript_notice' | 'js_fallback_marker' | 'aria_busy';
 /**
@@ -255,6 +255,16 @@ export interface RenderSignals {
     emptyTables: number;
     /** Markers found in the page as received. */
     markers: readonly RenderMarker[];
+    /**
+     * On a listing page, the hydration data's list of named records that most
+     * outnumbers the ones its markup shows: how many it lists, and how many
+     * of their names are in the visible text. Present only when such a list
+     * decided `hydration_list_partial`.
+     */
+    listRecords?: {
+        declared: number;
+        shown: number;
+    };
     /** True when the signals say the data is most likely rendered client-side. */
     clientRendered: boolean;
     /**

package/dist/types/cjs/contracts/firecrawl.d.ts

@@ -23,7 +23,7 @@ export declare const FIRECRAWL_SHIM_SNAPSHOT: {
     paths: readonly ["/scrape", "/crawl", "/crawl/:id", "/map"];
     notCovered: readonly ["search", "interact", "agent", "monitor", "extract"];
 };
-export declare const FIRECRAWL_SHIM_DIFFS: readonly ["Challenge / block pages are success: false (Firecrawl often returns them as success markdown).", "A page with no main content is success: false (failed: empty_unverified) with the whole page in data.markdown as evidence; with onlyMainContent: false it is success: true.", "A page whose server HTML is a shell for data its scripts fill in is fetched again on the browser rung, and the rendered page is the answer when it holds more; otherwise the HTTP page is returned with client_rendered_suspected and low_content_yield warnings on the native response, whose messages /fc passes through as data.warning (one string, joined with a space), as it does every native warning. Firecrawl renders every page in a browser.", "No fire-engine, proxy pools or JSON extract.", "actions (scrape only; a crawl's scrapeOptions.actions is refused) run on the local browser rung alone, which such a request selects, after load, stability and waitFor and before the formats are read: wait (milliseconds up to 60000, or a selector, waited for up to 60 s within the scrape's timeout), click (all: true clicks every match), write (into the focused element), press, scroll (one screen up or down, of the page or the element a selector names), screenshot, scrape, executeJavascript (a function body; return gives the value) and pdf; at most 50 steps. data.actions holds screenshots and pdfs as data: URIs (Firecrawl returns URLs), scrapes as { url, html } and javascriptReturns as { type, value }. A step that fails stops the steps after it: success is false with data.actions.failed naming the step, its code and message, and data.markdown is the page as it stood. A step that leads the page to a URL robots.txt or the egress policy refuses fails with navigation_refused and that page is not read. A hosted server refuses actions, and the cache is not used with them.", "An omitted maxAge reuses nothing: every page is fetched live unless the request sets maxAge above 0, minAge or lockdown (Firecrawl reuses its own index by default; its Python SDK sends maxAge 4 hours). A reused page is one W2L itself stored, under the same options, on this server (its task root), never a shared index; only a success is stored, and data.metadata says cacheState hit with cachedAt (its fetch time) or miss when one was looked up. lockdown with no stored result is HTTP 404 SCRAPE_LOCKDOWN_CACHE_MISS on scrape, and nothing is fetched; a crawl in lockdown needs sitemap skip, and each page with no stored result is failed with cache_miss. Mode authed neither stores nor reuses. A Firecrawl body never sets useCached, W2L's reuse of a crawl's own pages on resume.", "Omitted limit / maxDepth stay unbounded on a local server; a hosted server takes its crawl limit for an omitted or null limit and refuses a larger one. Firecrawl defaults are 10000 / 10.", "maxDepth counts link hops from the start URL (Firecrawl calls that maxDiscoveryDepth); Firecrawl maxDepth counts URL path depth.", "Crawl start is mapped onto native POST /v1/crawl; the shim itself returns 200 {success,id,url}.", "creditsUsed and expiresAt are null: W2L counts no credits and keeps crawl results until their task directory is deleted.", "Crawl status describes the latest attempt: completed counts its successful pages, total adds its failed, blocked and duplicate pages and, while this API process runs the crawl, the pages in flight and queued (null for a paused crawl); data lists the failed and blocked pages too (with metadata.error) but not the duplicates, whose content is an earlier entry's, up to 100 per response (limit 1 to 1000) with next carrying a W2L cursor; skip is rejected.", "Scrape maps url, formats, onlyMainContent, includeTags, excludeTags, waitFor, timeout, headers, mobile, skipTlsVerification, fastMode, blockAds, removeBase64Images, maxAge, minAge, storeInCache, lockdown, actions, origin and integration; crawl maps url, limit (as maxPages), maxDepth, includePaths, excludePaths, regexOnFullURL, ignoreQueryParameters, deduplicateSimilarURLs, crawlEntireDomain (and its v1 name allowBackwardLinks), allowSubdomains, allowExternalLinks, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only), maxConcurrency, origin, integration and the same scrapeOptions (applied to every page). The formats are markdown, links, html, rawHtml, images, screenshot (also screenshot@fullPage, and { type: \"screenshot\", fullPage, quality, viewport }) and an { type: \"attributes\", selectors } entry; other formats and parameters the shim does not map (proxy, location, json, ...) are rejected by name with HTTP 400 and success: false; a refusal of stealth, proxy: stealth or enhanced, or ignoreRobotsTxt names the supported route in agent_hints.", "A crawl follows links inside the start URL's path subtree on its host and www twin by default (crawlEntireDomain false), folds /a and /a/, / and /index.html, www and apex, http and https into one page (deduplicateSimilarURLs true) and reports every collapsed or refused link in the native crawl status (discovery) and each page's trace (links_offered); allowSubdomains takes every host under the start URL's apex (no public-suffix list), allowExternalLinks every host, each page with its own robots.txt read.", "sitemap (default include, as in Firecrawl) reads the sitemaps the start URL's robots.txt names, or /sitemap.xml, with the crawl's own http identity, robots.txt verdict, SSRF checks and proxy, and queues their URLs ahead of the start page's links under the same host, subtree, path and depth rules; only follows no page link; skip reads none. The native crawl status lists every sitemap file read, refused or unreadable in discovery.sitemap; the shim's status carries nothing of it, and sitemap fetches have no signed compliance record. maxConcurrency caps the pages one crawl fetches at once, at most the service's worker count (HTTP 400 above it), and never raises the per-host ceiling.", "screenshot (data.screenshot, a data:image/png;base64 string, or image/jpeg with quality 1 to 100) is captured on the local browser rung alone, which such a request selects (no http attempt; a server without a browser rung refuses the format with HTTP 400): after load, stability and waitFor, before the DOM is read, CSS-pixel sized at the declared 1280x800 viewport (device scale factor 2 is declared, not baked into the image) or at the viewport asked for (integers 320..1920 by 240..1080, within the declared screen; a window size, not a change of identity); fullPage captures the document's whole height at that width without scrolling first, so sections a page loads on scroll may show unloaded. A capture the browser could not make leaves data.screenshot null with a screenshot_unavailable warning while the page stands; a file or a page that was not rendered has null too. Firecrawl captures at its own viewport and may return a URL instead of the image.", "images (data.images) lists every image URL of the whole document as received: img src and srcset candidates, picture sources, lazy data-src/data-srcset/data-lazy-src/data-original, video posters, image_src links, og:image and twitter:image, absolute http(s) with the fragment stripped, each once, in document order, data: URIs left out; includeTags, excludeTags and onlyMainContent do not narrow it. attributes (data.attributes) gives, per selector, the named attribute's values as written, elements without it skipped; a selector W2L does not match is HTTP 400 by name, as for includeTags. Both are absent for a file and for a page that is success: false. removeBase64Images (default true) keeps an image's alt text where Firecrawl writes a (<Base64-Image-Removed>) placeholder; false keeps the data: URI in the Markdown.", "origin (the Firecrawl SDKs' client label) and integration are stored, not echoed: the scrape record (GET /v1/scrapes/:id) and the crawl task carry them, and nothing sent to the target changes.", "data.metadata carries scrapeId (a UUID per call, which GET /v1/scrapes/:id looks up), proxyUsed (operator for the server's environment proxy, user for the caller's own egress, else null), timezone (the browser rung's declared zone, null on the HTTP rung), creditsUsed: null (W2L counts no credits), concurrencyLimited and concurrencyQueueDurationMs (whether and how long the per-origin ceiling held the fetch back), and cacheState and cachedAt when the cache was asked (never a guessed miss).", "A page whose result W2L has advice about (a login wall, a robots.txt rule, a gate, a cut, a script-filled shell) carries data.agent_hints, one sentence each; the native response calls them agentHints. A request refused for an option W2L does not offer carries agent_hints in the error envelope, and a caller over the server's per-minute rate limit gets HTTP 429 { success: false, error, code: rate_limited, agent_hints } with Retry-After.", "headers never override the User-Agent, the client hints, a credential (authorization, cookie) or a transport header: such a header is HTTP 400 naming it, where Firecrawl sends it. The headers go to the requested origin after the declared identity and are on the record (the trace, the browser lane's signed sentHeaders); both rungs withhold them from a redirect hop to another origin and say so (custom_headers_withheld).", "mobile selects a declared Android Chrome identity (User-Agent, client hints, 412x915 viewport, touch) that robots.txt is evaluated against and the record carries; the page is whatever the site serves to it, with no DOM rewriting. It is refused with mode research.", "skipTlsVerification relaxes certificate verification for one local request and its robots.txt lookup, recorded in the trace (tls_verification_skipped) and a tls_unverified warning the native response carries; a hosted W2L refuses it with HTTP 400. Without it a bad certificate is success: false with failed: tls_error. Firecrawl's Python SDK sends true by default; W2L verifies by default.", "fastMode keeps the http rung alone: a page that needs scripts is success: false with failed: empty_unverified, never rendered; waitFor has no effect under it. Firecrawl's fast mode still renders.", "blockAds (default true) aborts requests to a bundled list of about 50 ad-serving hosts on the local browser rung and removes ad and cookie-banner elements before extraction; false keeps them. The list is curated, not EasyList: ads from hosts outside it are not blocked.", "html is the cleaned HTML the markdown is written from: the main content, the whole page without scripts, styles, form controls and embedded media when onlyMainContent is false, or a <body> holding the includeTags elements. rawHtml is the page as the answering rung received it: the response body on the HTTP rung, the rendered DOM on a browser rung. Both are null for a file and for a page that is success: false.", "includeTags keeps only the named elements, in document order, whatever onlyMainContent says; excludeTags removes elements from the main content, the whole page and an includeTags selection. A selector that does not parse, or that uses a sibling combinator, a positional pseudo-class, :has() or another pseudo-class W2L does not match, is rejected with HTTP 400.", "An omitted timeout stays 300000 ms (Firecrawl: 30000). A timeout is answered with HTTP 200: success: true with the content fetched so far (native status partial), or success: false with failed: timeout; Firecrawl answers it with an error.", "waitFor skips the HTTP rung, which cannot run scripts, and starts at the browser rung; the wait counts toward timeout.", "metadata has title, description, language, keywords, robots and favicon only when the page declares them, and the Open Graph (ogTitle, ogDescription, ogUrl, ogImage, ogAudio, ogVideo, ogDeterminer, ogLocale, ogLocaleAlternate, ogSiteName), Dublin Core (dcTermsCreated, dcDateCreated, dcDate, dcTermsType, dcType, dcTermsAudience, dcTermsSubject, dcSubject, dcDescription, dcTermsKeywords) and article (publishedTime, modifiedTime, articleTag, articleSection) tags under Firecrawl's names, each only when the page states it, as written (no date normalisation, no fallback from another tag); twitter:* and other meta tags are not passed through, and a failed or blocked page has none.", "A PDF answers success: true with its text layer as markdown and metadata.numPages (the document's page count). parsers maps Firecrawl's pdf entry (the string or { type: \"pdf\", mode, maxPages, pages, pageMarkers }); mode fast and auto both read the text layer, and mode ocr and the image parser are refused by name. pageMarkers is false unless asked, as on Firecrawl; with true W2L writes a <!-- page N --> line before each page (Firecrawl writes --- and the marker between pages). pages: true adds data.pages, [{ pageNumber, markdown }]. maxPages (1 to 10000) reads the first pages, and a cut it asked for stays success: true. parsers [] or v1 parsePDF false reads no PDF: success: true with markdown null and the file saved as received. A PDF without a text layer is success: false with failed: empty_unverified (no OCR). CSV, JSON and text files give their text as received; XLSX, XLS and ZIP files are success: true with markdown null. A file over W2L_MAX_FILE_BYTES is success: false with failed: body_too_large.", "Map (POST /fc/v1/map) maps url, search, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only; both v1 flags true is HTTP 400), includeSubdomains, ignoreQueryParameters, limit (1 to 100000, default 5000), timeout (1000 to 300000 ms for the whole map, default 60000; Firecrawl documents no default), origin and integration onto native POST /v1/map, and answers 200 { success: true, id, links: [url strings], warning?, agent_hints? }, or 200 { success: false, id, error, links: [] } when the map found nothing because a source failed or its deadline passed; useIndex, location, ignoreCache, threatProtection and auditMetadata are refused by name (useIndex with the hint that W2L keeps no URL index). Omitted options take W2L's defaults: includeSubdomains and ignoreQueryParameters are false, where Firecrawl v2 documents true for both. search keeps the URLs in which every word appears in the decoded URL or the title in hand, in discovery order; Firecrawl orders by relevance. A map reads the sitemaps the site declares and one page body (the start URL, http rung only), so a site without a sitemap maps only its start page's links; a title is the start page's own, an anchor's text or a sitemap's <news:title>, never fetched from the target; robots-disallowed URLs are left out and counted on the native response (GET /v1/maps/:id), which also records every sitemap file read.", "A crawl's webhook (a URL string or { url, headers, metadata, events }) is mapped onto the native webhook and its receiver gets Firecrawl's payload shape: { success, type: crawl.started | crawl.page | crawl.completed | crawl.failed, id, data: [page], metadata, error? }, one durable delivery per event with retries, every request carrying x-w2l-event-id, x-w2l-event-version and x-w2l-delivery-id (and the signature pair with secretEnv, a native option). A cancelled crawl is crawl.failed with error \"cancelled\". The native rules apply: https (plain http for a loopback receiver of a local server only), no content-type, host or x-w2l-* header, at most 32 headers and 32 metadata strings; a hosted server takes public https receivers only. GET /v1/deliveries?jobId=<id> on the native API lists the deliveries."];
+export declare const FIRECRAWL_SHIM_DIFFS: readonly ["Challenge / block pages are success: false (Firecrawl often returns them as success markdown).", "A page with no main content is success: false (failed: empty_unverified) with the whole page in data.markdown as evidence; with onlyMainContent: false it is success: true.", "A page whose server HTML is a shell for data its scripts fill in is fetched again on the browser rung, and the rendered page is the answer when it holds more; otherwise the HTTP page is returned with client_rendered_suspected and low_content_yield warnings on the native response, whose messages /fc passes through as data.warning (one string, joined with a space), as it does every native warning. Firecrawl renders every page in a browser.", "No fire-engine, proxy pools or JSON extract.", "actions (scrape only; a crawl's scrapeOptions.actions is refused) run on the local browser rung alone, which such a request selects, after load, stability and waitFor and before the formats are read: wait (milliseconds up to 60000, or a selector, waited for up to 60 s within the scrape's timeout), click (all: true clicks every match), write (into the focused element), press, scroll (one screen up or down, of the page or the element a selector names), screenshot, scrape, executeJavascript (a function body; return gives the value) and pdf; at most 50 steps. data.actions holds screenshots and pdfs as data: URIs (Firecrawl returns URLs), scrapes as { url, html } and javascriptReturns as { type, value }. A step that fails stops the steps after it: success is false with data.actions.failed naming the step, its code and message, and data.markdown is the page as it stood. A step that leads the page to a URL robots.txt or the egress policy refuses fails with navigation_refused and that page is not read. A hosted server refuses actions, and the cache is not used with them.", "An omitted maxAge reuses nothing: every page is fetched live unless the request sets maxAge above 0, minAge or lockdown (Firecrawl reuses its own index by default; its Python SDK sends maxAge 4 hours). A reused page is one Octocrawl itself stored, under the same options, on this server (its task root), never a shared index; only a success is stored, and data.metadata says cacheState hit with cachedAt (its fetch time) or miss when one was looked up. lockdown with no stored result is HTTP 404 SCRAPE_LOCKDOWN_CACHE_MISS on scrape, and nothing is fetched; a crawl in lockdown needs sitemap skip, and each page with no stored result is failed with cache_miss. Mode authed neither stores nor reuses. A Firecrawl body never sets useCached, Octocrawl's reuse of a crawl's own pages on resume.", "Omitted limit / maxDepth stay unbounded on a local server; a hosted server takes its crawl limit for an omitted or null limit and refuses a larger one. Firecrawl defaults are 10000 / 10.", "maxDepth counts link hops from the start URL (Firecrawl calls that maxDiscoveryDepth); Firecrawl maxDepth counts URL path depth.", "Crawl start is mapped onto native POST /v1/crawl; the shim itself returns 200 {success,id,url}.", "creditsUsed and expiresAt are null: Octocrawl counts no credits and keeps crawl results until their task directory is deleted.", "Crawl status describes the latest attempt: completed counts its successful pages, total adds its failed, blocked and duplicate pages and, while this API process runs the crawl, the pages in flight and queued (null for a paused crawl); data lists the failed and blocked pages too (with metadata.error) but not the duplicates, whose content is an earlier entry's, up to 100 per response (limit 1 to 1000) with next carrying an Octocrawl cursor; skip is rejected.", "Scrape maps url, formats, onlyMainContent, includeTags, excludeTags, waitFor, timeout, headers, mobile, skipTlsVerification, fastMode, blockAds, removeBase64Images, maxAge, minAge, storeInCache, lockdown, actions, origin and integration; crawl maps url, limit (as maxPages), maxDepth, includePaths, excludePaths, regexOnFullURL, ignoreQueryParameters, deduplicateSimilarURLs, crawlEntireDomain (and its v1 name allowBackwardLinks), allowSubdomains, allowExternalLinks, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only), maxConcurrency, ignoreRobotsTxt (v2; a local server only), origin, integration and the same scrapeOptions (applied to every page). The formats are markdown, links, html, rawHtml, images, screenshot (also screenshot@fullPage, and { type: \"screenshot\", fullPage, quality, viewport }) and an { type: \"attributes\", selectors } entry; other formats and parameters the shim does not map (proxy, location, json, ...) are rejected by name with HTTP 400 and success: false; a refusal of stealth, proxy: stealth or enhanced, or ignoreRobotsTxt on a scrape or map names the supported route in agent_hints.", "A crawl follows links inside the start URL's path subtree on its host and www twin by default (crawlEntireDomain false), folds /a and /a/, / and /index.html, www and apex, http and https into one page (deduplicateSimilarURLs true) and reports every collapsed or refused link in the native crawl status (discovery) and each page's trace (links_offered); allowSubdomains takes every host under the start URL's apex (no public-suffix list), allowExternalLinks every host, each page with its own robots.txt read.", "sitemap (default include, as in Firecrawl) reads the sitemaps the start URL's robots.txt names, or /sitemap.xml, with the crawl's own http identity, robots.txt verdict, SSRF checks and proxy, and queues their URLs ahead of the start page's links under the same host, subtree, path and depth rules; only follows no page link; skip reads none. The native crawl status lists every sitemap file read, refused or unreadable in discovery.sitemap; the shim's status carries nothing of it, and sitemap fetches have no signed compliance record. maxConcurrency caps the pages one crawl fetches at once, at most the service's worker count (HTTP 400 above it), and never raises the per-host ceiling.", "screenshot (data.screenshot, a data:image/png;base64 string, or image/jpeg with quality 1 to 100) is captured on the local browser rung alone, which such a request selects (no http attempt; a server without a browser rung refuses the format with HTTP 400): after load, stability and waitFor, before the DOM is read, CSS-pixel sized at the declared 1280x800 viewport (device scale factor 2 is declared, not baked into the image) or at the viewport asked for (integers 320..1920 by 240..1080, within the declared screen; a window size, not a change of identity); fullPage captures the document's whole height at that width without scrolling first, so sections a page loads on scroll may show unloaded. A capture the browser could not make leaves data.screenshot null with a screenshot_unavailable warning while the page stands; a file or a page that was not rendered has null too. Firecrawl captures at its own viewport and may return a URL instead of the image.", "images (data.images) lists every image URL of the whole document as received: img src and srcset candidates, picture sources, lazy data-src/data-srcset/data-lazy-src/data-original, video posters, image_src links, og:image and twitter:image, absolute http(s) with the fragment stripped, each once, in document order, data: URIs left out; includeTags, excludeTags and onlyMainContent do not narrow it. attributes (data.attributes) gives, per selector, the named attribute's values as written, elements without it skipped; a selector Octocrawl does not match is HTTP 400 by name, as for includeTags. Both are absent for a file and for a page that is success: false. removeBase64Images (default true) keeps an image's alt text where Firecrawl writes a (<Base64-Image-Removed>) placeholder; false keeps the data: URI in the Markdown.", "origin (the Firecrawl SDKs' client label) and integration are stored, not echoed: the scrape record (GET /v1/scrapes/:id) and the crawl task carry them, and nothing sent to the target changes.", "data.metadata carries scrapeId (a UUID per call, which GET /v1/scrapes/:id looks up), proxyUsed (operator for the server's environment proxy, user for the caller's own egress, else null), timezone (the browser rung's declared zone, null on the HTTP rung), creditsUsed: null (Octocrawl counts no credits), concurrencyLimited and concurrencyQueueDurationMs (whether and how long the per-origin ceiling held the fetch back), and cacheState and cachedAt when the cache was asked (never a guessed miss).", "A page whose result Octocrawl has advice about (a login wall, a robots.txt rule, a gate, a cut, a script-filled shell) carries data.agent_hints, one sentence each; the native response calls them agentHints. A request refused for an option Octocrawl does not offer carries agent_hints in the error envelope, and a caller over the server's per-minute rate limit gets HTTP 429 { success: false, error, code: rate_limited, agent_hints } with Retry-After.", "headers never override the User-Agent, the client hints, a credential (authorization, cookie) or a transport header: such a header is HTTP 400 naming it, where Firecrawl sends it. The headers go to the requested origin after the declared identity and are on the record (the trace, the browser lane's signed sentHeaders); both rungs withhold them from a redirect hop to another origin and say so (custom_headers_withheld).", "mobile selects a declared Android Chrome identity (User-Agent, client hints, 412x915 viewport, touch) that robots.txt is evaluated against and the record carries; the page is whatever the site serves to it, with no DOM rewriting. It is refused with mode research.", "skipTlsVerification relaxes certificate verification for one local request and its robots.txt lookup, recorded in the trace (tls_verification_skipped) and a tls_unverified warning the native response carries; a hosted Octocrawl refuses it with HTTP 400. Without it a bad certificate is success: false with failed: tls_error. Firecrawl's Python SDK sends true by default; Octocrawl verifies by default.", "fastMode keeps the http rung alone: a page that needs scripts is success: false with failed: empty_unverified, never rendered; waitFor has no effect under it. Firecrawl's fast mode still renders.", "blockAds (default true) aborts requests to a bundled list of about 50 ad-serving hosts on the local browser rung and removes ad and cookie-banner elements before extraction; false keeps them. The list is curated, not EasyList: ads from hosts outside it are not blocked.", "html is the cleaned HTML the markdown is written from: the main content, the whole page without scripts, styles, form controls and embedded media when onlyMainContent is false, or a <body> holding the includeTags elements. rawHtml is the page as the answering rung received it: the response body on the HTTP rung, the rendered DOM on a browser rung. Both are null for a file and for a page that is success: false.", "includeTags keeps only the named elements, in document order, whatever onlyMainContent says; excludeTags removes elements from the main content, the whole page and an includeTags selection. A selector that does not parse, or that uses a sibling combinator, a positional pseudo-class, :has() or another pseudo-class Octocrawl does not match, is rejected with HTTP 400.", "An omitted timeout stays 300000 ms (Firecrawl: 30000). A timeout is answered with HTTP 200: success: true with the content fetched so far (native status partial), or success: false with failed: timeout; Firecrawl answers it with an error.", "waitFor skips the HTTP rung, which cannot run scripts, and starts at the browser rung; the wait counts toward timeout.", "metadata has title, description, language, keywords, robots and favicon only when the page declares them, and the Open Graph (ogTitle, ogDescription, ogUrl, ogImage, ogAudio, ogVideo, ogDeterminer, ogLocale, ogLocaleAlternate, ogSiteName), Dublin Core (dcTermsCreated, dcDateCreated, dcDate, dcTermsType, dcType, dcTermsAudience, dcTermsSubject, dcSubject, dcDescription, dcTermsKeywords) and article (publishedTime, modifiedTime, articleTag, articleSection) tags under Firecrawl's names, each only when the page states it, as written (no date normalisation, no fallback from another tag); twitter:* and other meta tags are not passed through, and a failed or blocked page has none.", "A PDF answers success: true with its text layer as markdown and metadata.numPages (the document's page count). parsers maps Firecrawl's pdf entry (the string or { type: \"pdf\", mode, maxPages, pages, pageMarkers }); mode fast and auto both read the text layer, and mode ocr and the image parser are refused by name. pageMarkers is false unless asked, as on Firecrawl; with true Octocrawl writes a <!-- page N --> line before each page (Firecrawl writes --- and the marker between pages). pages: true adds data.pages, [{ pageNumber, markdown }]. maxPages (1 to 10000) reads the first pages, and a cut it asked for stays success: true. parsers [] or v1 parsePDF false reads no PDF: success: true with markdown null and the file saved as received. A PDF without a text layer is success: false with failed: empty_unverified (no OCR). CSV, JSON and text files give their text as received; XLSX, XLS and ZIP files are success: true with markdown null. A file over W2L_MAX_FILE_BYTES is success: false with failed: body_too_large.", "Map (POST /fc/v1/map) maps url, search, sitemap (v2; v1 ignoreSitemap true is skip and false include, sitemapOnly true is only; both v1 flags true is HTTP 400), includeSubdomains, ignoreQueryParameters, limit (1 to 100000, default 5000), timeout (1000 to 300000 ms for the whole map, default 60000; Firecrawl documents no default), origin and integration onto native POST /v1/map, and answers 200 { success: true, id, links: [url strings], warning?, agent_hints? }, or 200 { success: false, id, error, links: [] } when the map found nothing because a source failed or its deadline passed; useIndex, location, ignoreCache, threatProtection and auditMetadata are refused by name (useIndex with the hint that Octocrawl keeps no URL index). Omitted options take Octocrawl's defaults: includeSubdomains and ignoreQueryParameters are false, where Firecrawl v2 documents true for both. search keeps the URLs in which every word appears in the decoded URL or the title in hand, in discovery order; Firecrawl orders by relevance. A map reads the sitemaps the site declares and one page body (the start URL, http rung only), so a site without a sitemap maps only its start page's links; a title is the start page's own, an anchor's text or a sitemap's <news:title>, never fetched from the target; robots-disallowed URLs are left out and counted on the native response (GET /v1/maps/:id), which also records every sitemap file read.", "A crawl's webhook (a URL string or { url, headers, metadata, events }) is mapped onto the native webhook and its receiver gets Firecrawl's payload shape: { success, type: crawl.started | crawl.page | crawl.completed | crawl.failed, id, data: [page], metadata, error? }, one durable delivery per event with retries, every request carrying x-w2l-event-id, x-w2l-event-version and x-w2l-delivery-id (and the signature pair with secretEnv, a native option). A cancelled crawl is crawl.failed with error \"cancelled\". The native rules apply: https (plain http for a loopback receiver of a local server only), no content-type, host or x-w2l-* header, at most 32 headers and 32 metadata strings; a hosted server takes public https receivers only. GET /v1/deliveries?jobId=<id> on the native API lists the deliveries."];
 export interface FirecrawlPage {
     markdown: string | null;
     /** Present when the `html` format was asked for; null when the page has none (a file, a page that did not succeed). */

package/dist/types/cjs/contracts/map.d.ts

@@ -40,11 +40,16 @@ export interface MapLink {
     sitemapFile?: string;
     /** The `<lastmod>` the sitemap gave, as written. */
     lastmod?: string;
-    /** The robots.txt verdict for the URL under the map's declared identity; a disallowed URL is never a link. */
-    robots: 'allowed' | 'no_robots';
+    /**
+     * The robots.txt verdict for the URL under the map's declared identity. A
+     * URL robots.txt disallows (`disallowed`), or on a host whose robots.txt
+     * could not be read (`unreachable`), is a link only in a map started with
+     * ignoreRobotsTxt; otherwise it is refused.
+     */
+    robots: 'allowed' | 'no_robots' | 'disallowed' | 'unreachable';
 }
 export type MapStatus = 'completed' | 'partial' | 'failed';
-/** What the start page read gave, or why it was not read (`failed/policy_denied` when robots.txt disallows the start URL or could not be read). */
+/** What the start page read gave, or why it was not read (`failed/policy_denied` when robots.txt disallows the start URL or could not be read, unless the map was started with ignoreRobotsTxt). */
 export interface MapStartPage {
     url: string;
     finalUrl: string | null;
@@ -164,7 +169,8 @@ export interface MapStartPageRead {
 /** A robots.txt verdict for one URL under the map's declared identity. */
 export type MapRobotsVerdict = 'allowed' | 'no_robots' | {
     disallowed: true;
-    unreachable?: string;
+    unreachable?: string; /** A rule robots.txt wrote for Octocrawl itself, which ignoreRobotsTxt does not set aside. */
+    octocrawl?: true;
 };
 /** The fetch paths a map uses; the API engine wires the real ones, tests inject fakes. */
 export interface MapSources {

package/dist/types/cjs/contracts/policy.d.ts

@@ -63,7 +63,8 @@ export interface NetworkPolicy {
  * because the browser lane can route through only one proxy.
  */
 export interface EgressProxy {
-    source: 'environment';
+    /** `environment`: HTTPS_PROXY / HTTP_PROXY. `pool`: one of the operator's W2L_EGRESS_PROXIES (ADR 0005 egress_sessions). */
+    source: 'environment' | 'pool';
     /** From HTTPS_PROXY / https_proxy. Null sends https: URLs direct. */
     https: ProxyServer | null;
     /** From HTTP_PROXY / http_proxy. Null sends http: URLs direct. */

package/dist/types/cjs/contracts/proxy.d.ts

@@ -25,6 +25,14 @@ export declare class ProxyConfigError extends Error {
 export declare function environmentProxy(env: Env): EgressProxy | null;
 /** A local-mode policy that routes through the environment's proxy, when one is set. */
 export declare function withEnvironmentProxy(policy: NetworkPolicy, env: Env): NetworkPolicy;
+/**
+ * The operator's egress proxies (`W2L_EGRESS_PROXIES`, ADR 0005 `egress_sessions`): http:// or https://
+ * URLs, credentials in their userinfo, separated by commas or white space. Each one is a whole egress:
+ * every URL's scheme goes through it. Duplicates (the same endpoint) are kept once.
+ */
+export declare function egressProxies(raw: string | undefined): ProxyServer[];
+/** A policy whose every request leaves through this egress proxy; NO_PROXY entries and loopback still go direct. */
+export declare function withPoolProxy(policy: NetworkPolicy, server: ProxyServer): NetworkPolicy;
 /** The operator proxy a URL leaves through under this policy; null means a direct connection. */
 export declare function proxyFor(url: string | URL, policy: Pick<NetworkPolicy, 'origin' | 'egressProxy'>): ProxyServer | null;
 export type NoProxyEntry = {