@octocrawl/sdk 0.3.0 → 0.3.2

package/dist/types/esm/contracts/actions.d.ts

@@ -4,6 +4,7 @@
  * Each step is recorded in the trace with its outcome and timing; a step
  * that fails ends the pipeline, and the result keeps the page as it stood.
  */
+import type { BlockReason } from './status.js';
 import type { ScreenshotEvidence } from './result.js';
 import type { ScreenshotViewport } from './structured.js';
 /** The most steps one request may run. */
@@ -149,8 +150,20 @@ export interface ActionPdf {
     path: string | null;
     base64: string;
 }
-/** Why a scrollToEnd, loadMore or paginate step stopped. `max` and `deadline` stop short of the list's end. */
-export type ListStop = 'end' | 'no_growth' | 'repeat' | 'max' | 'deadline';
+/**
+ * Why a scrollToEnd, loadMore or paginate step stopped. `max` and `deadline` stop short of the list's end; so does
+ * `challenge`: a check the site put up on the next page (a Cloudflare interstitial, a page of nothing but a CAPTCHA),
+ * which paginate stops at without reading it, the result then `blocked` with the check's reason.
+ */
+export type ListStop = 'end' | 'no_growth' | 'repeat' | 'max' | 'deadline' | 'challenge';
+/** The check a paginate step stopped at (ListStop `challenge`): the page it would have been, its URL, and what the gate saw. */
+export interface ListChallenge {
+    /** 1-based position the page would have had among the pages read. */
+    page: number;
+    url: string;
+    reason: BlockReason;
+    signals: readonly string[];
+}
 /** What a scrollToEnd, loadMore or paginate step did. */
 export interface ListRun {
     index: number;
@@ -160,8 +173,27 @@ export interface ListRun {
     rounds: number;
     /** Elements matching `itemSelector` at the end (on the last page for paginate; summed over its pages in `itemsRead`); null without one. */
     items: number | null;
-    /** paginate: elements matching `itemSelector` over every page read; null without one. */
+    /**
+     * paginate: elements matching `itemSelector` over every page read; null without one. After a continuation in the person's
+     * Chrome (`continued`), the kept pages' count plus each page they showed, as their tab counted it, only when the addresses show
+     * no page can be counted twice (the kept pages each at its own, the check's page and the pages shown at none of them, the check
+     * not at the list's own address), no page shows again what another shows (its items' whole text, as the list merge tells it:
+     * a result set tied to the session that made it comes back at new addresses), and every count is known; null (unknown) otherwise.
+     */
     itemsRead?: number | null;
+    /** paginate: pages taken from the task's checkpoint after a run cut at page N (ExecutionContext.listResume), counted in `rounds`; absent when none. */
+    resumed?: number;
+    /** paginate: the check the step stopped at, with `stoppedBy` `challenge`; absent otherwise. */
+    challenge?: ListChallenge;
+    /**
+     * paginate: the pages read after the check at `from`, by the person paging on in their own browser once they got through
+     * it (the batch handoff), counted in `rounds`; `stoppedBy` then says how that reading ended. Absent otherwise.
+     */
+    continued?: {
+        from: number;
+        pages: number;
+        by: 'user_browser';
+    };
 }
 /** What the steps produced, each list in the order of its steps. */
 export interface ActionsResult {
@@ -170,7 +202,8 @@ export interface ActionsResult {
     scrapes: {
         url: string;
         html: string;
-        step?: number;
+        step?: number; /** Set when the person read the page in their own browser (a list's continuation after a check); absent for W2L's own browser. */
+        by?: 'user_browser';
     }[];
     /** `type` is the JavaScript `typeof` of the value (`null` for null). */
     javascriptReturns: {

package/dist/types/esm/contracts/api.d.ts

@@ -170,7 +170,28 @@ export interface ScrapeRequest extends PageOptions, RequestAttribution {
     handoff?: {
         waitMs?: number;
     };
+    /**
+     * `my-browser`: read the page in the person's own Chrome, over remote
+     * debugging, without W2L fetching it first. The person allows the
+     * connection in Chrome, then the site in a page W2L opens there; the page
+     * is read without a click of theirs only on a site they allowed, and a
+     * check it shows waits for them (`handoff.waitMs`, default 10 min). Lane
+     * `my_browser`; never cached. Offered only by a server on the person's
+     * own machine; refused elsewhere, with `actions` or a screenshot, and with
+     * a mode other than standard (`unsupported_parameter`).
+     */
+    lane?: 'my-browser';
+    /** One of three plain choices of how the page is reached (ACCESS_CHOICES); omitted, the server's own configuration. */
+    access?: AccessChoice;
 }
+/**
+ * How pages are reached, as three plain choices (ROADMAP PA item 7) beside the per-route options:
+ * `standard` (Octocrawl's own lanes and none that costs a third party), `enhanced` (also what the server's
+ * access grant of tier enhanced approves, within its budget; refused on a server without one), `my-browser`
+ * (the person's own Chrome, as `lane: "my-browser"`; a scrape or a batch only).
+ */
+export declare const ACCESS_CHOICES: readonly ["standard", "enhanced", "my-browser"];
+export type AccessChoice = (typeof ACCESS_CHOICES)[number];
 /** A recorded robots override for one URL of a batch. */
 export interface RobotsUrlOverride extends RobotsOverride {
     url: string;
@@ -344,6 +365,8 @@ export interface JobWebhookStatus {
 export interface CrawlStartRequest extends PageOptions, RequestAttribution {
     url: string;
     mode?: ApiCrawlMode;
+    /** `standard` or `enhanced` (ACCESS_CHOICES); a crawl does not take `my-browser`. */
+    access?: Exclude<AccessChoice, 'my-browser'>;
     maxPages?: number | null;
     maxDepth?: number | null;
     /**
@@ -411,6 +434,16 @@ export interface CrawlStartRequest extends PageOptions, RequestAttribution {
     idempotencyKey?: string;
     /** A receiver for the crawl's events (`started`, one `page` per page recorded, then `completed`, `failed` or `cancelled`); see WebhookConfig. */
     webhook?: WebhookOption;
+    /**
+     * Fetch the pages and sitemap files robots.txt disallows, or whose
+     * robots.txt could not be read (Firecrawl v2's name). robots.txt is still
+     * read for every host and its verdict recorded on each page, Crawl-delay
+     * applied, with a `robots_overridden` warning and `overrideBasis:
+     * "ignore_robots_txt"` where a rule was set aside. A local server only: a
+     * hosted one refuses it by name. Default false: a crawl's links obey
+     * robots.txt.
+     */
+    ignoreRobotsTxt?: boolean;
 }
 /** What the parser hands the engine: the request plus, from the `/fc` shim, the payload shape its receiver expects. */
 export type ParsedCrawlStartRequest = CrawlStartRequest & {
@@ -454,6 +487,14 @@ export interface MapRequest extends RequestAttribution {
     crawlEntireDomain?: boolean;
     /** Default true, as on a crawl; a returned http link gives way to its https variant when that comes too, on an origin whose robots.txt the map read anyway and which allows it. */
     deduplicateSimilarURLs?: boolean;
+    /**
+     * Return the URLs robots.txt disallows, or whose robots.txt could not be
+     * read, with that verdict on each link (`robots: "disallowed"` or
+     * `"unreachable"`), and read the start page and sitemap files past it.
+     * robots.txt is still read, within MAP_MAX_ROBOTS_HOSTS. A local server
+     * only: a hosted one refuses it by name. Default false.
+     */
+    ignoreRobotsTxt?: boolean;
 }
 /** A map's `search`: at most this many characters after trimming, and this many whitespace-separated words. */
 export declare const MAP_SEARCH_MAX_CHARS = 200;
@@ -482,6 +523,7 @@ export interface ActiveCrawlOptions {
     allowExternalLinks: boolean;
     regexOnFullURL: boolean;
     maxConcurrency: number | null;
+    ignoreRobotsTxt: boolean;
     /** The per-page options every page of the crawl gets: its formats, `includeLinks` and the page options. */
     scrapeOptions: PageOptions & {
         formats: readonly ScrapeFormat[];
@@ -507,6 +549,16 @@ export interface ActiveCrawlList {
 export interface BatchStartRequest extends PageOptions, RequestAttribution {
     urls: readonly string[];
     mode?: ApiCrawlMode;
+    /** One of three plain choices of how pages are reached (ACCESS_CHOICES); `my-browser` is `lane: "my-browser"`. */
+    access?: AccessChoice;
+    /**
+     * `my-browser`: read every page in the person's own Chrome, one at a time, as a scrape's `lane` does. The person
+     * allows the connection in Chrome, then all the batch's sites (host and port) in the page W2L opens there, once for
+     * the run; a site not among them is not read. A resumed run asks again. Offered only by a server on the person's own
+     * machine; refused elsewhere, with `actions`, a screenshot, lockdown, a mode other than standard, `maxConcurrency`
+     * above 1 or a webhook (`unsupported_parameter`).
+     */
+    lane?: 'my-browser';
     formats?: readonly ScrapeFormat[];
     includeLinks?: boolean;
     /** Recorded robots overrides, each for one URL of `urls`. A hosted server refuses the field (`unsupported_parameter`). */
@@ -585,6 +637,8 @@ export interface BatchStatusResponse extends CrawlReport {
     invalidURLs?: readonly string[];
     /** Items stopped at a check a person can get through in their own Chrome (`POST /v1/batches/:id/handoff`); present on a server that offers the handoff. */
     waitingForPerson?: number;
+    /** A batch on the my-browser lane waiting for the person to allow its sites in the page Octocrawl opened in their Chrome; present only while it waits. */
+    waitingForApproval?: true;
 }
 /**
  * The checks a batch item can be handed to a person for, and the routing
@@ -809,26 +863,28 @@ export declare class RequestError extends Error {
 }
 /** The hints a refusal carries for the options W2L does not offer: the next honest step, never a way around the refusal. */
 export declare const REFUSAL_HINTS: {
-    readonly stealth: "W2L does not offer a stealth mode or stealth proxies; a proxy or session you own (mode authed) is the supported route";
-    readonly ignoreRobotsTxt: "robots.txt is always read; a robotsOverride with a recorded reason fetches one URL past its rule, on the record";
-    readonly hostedSkipTlsVerification: "a hosted server verifies every certificate; run W2L locally to use skipTlsVerification, which is recorded in the trace and a tls_unverified warning";
-    readonly useIndex: "W2L keeps no URL index: a map reads the sitemaps the site declares and its start page, on the record; crawl reads further pages";
+    readonly stealth: "Octocrawl has no stealth option on a request: a provider's stealth or challenge solving runs only on a server started with an access grant that names it (--access-grant, ADR 0005); a proxy or session you own (mode authed) is the other route";
+    readonly ignoreRobotsTxt: "robots.txt is always read and recorded; on a local server a URL a scrape or batch names is fetched whatever it says, and ignoreRobotsTxt on a crawl or map fetches the links it disallows, on the record";
+    readonly hostedSkipTlsVerification: "a hosted server verifies every certificate; run Octocrawl locally to use skipTlsVerification, which is recorded in the trace and a tls_unverified warning";
+    readonly useIndex: "Octocrawl keeps no URL index: a map reads the sitemaps the site declares and its start page, on the record; crawl reads further pages";
     readonly actions: "actions run on scrape and batch, where each page named gets the same steps; a crawl or a map does not take them";
 };
 /** The hint for a refused request key, or null when the key has none (an option W2L simply does not know). */
 export declare function refusalHint(key: string, value: unknown): string | null;
 export declare const PAGE_KEYS: readonly ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown"];
 export declare const ATTRIBUTION_KEYS: readonly ["origin", "integration"];
-export declare const SCRAPE_KEYS: readonly ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const SCRAPE_KEYS: readonly ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "lane", "access", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+/** The lanes a request may ask for by name. */
+export declare const REQUEST_LANES: readonly ["my-browser"];
 export declare const CRAWL_SCOPE_KEYS: readonly ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-export declare const CRAWL_KEYS: readonly ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", "regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks", "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
-export declare const BATCH_KEYS: readonly ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const CRAWL_KEYS: readonly ["url", "mode", "access", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", "regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks", "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const BATCH_KEYS: readonly ["urls", "mode", "lane", "access", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
 /** What a batch body may carry beside `appendToId`: the job's own options are not among them (the scope no-ops change nothing, so they may come along). */
 export declare const BATCH_APPEND_KEYS: readonly ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", "origin", "integration"];
 /** The scope options a map takes under their crawl names; allowSubdomains is includeSubdomains on a map, and allowExternalLinks is not offered. */
 export declare const MAP_SCOPE_KEYS: readonly ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
 /** What a map takes. No page option (headers, mobile, skipTlsVerification, formats, ...): a map has nothing to loosen. */
-export declare const MAP_KEYS: readonly ["url", "mode", "limit", "timeout", "search", "sitemap", "includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs", "includePaths", "excludePaths", "origin", "integration"];
+export declare const MAP_KEYS: readonly ["url", "mode", "limit", "timeout", "search", "sitemap", "includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs", "includePaths", "excludePaths", "ignoreRobotsTxt", "origin", "integration"];
 /** Why a webhook header (lower-cased name) cannot be sent, or null when it can: W2L's own and the transport's names are reserved. */
 export declare function webhookHeaderRefusal(name: string): string | null;
 /**
@@ -869,8 +925,8 @@ export declare function parseCrawlStartRequest(body: unknown): CrawlStartRequest
  * A map request: url, mode (standard or research), limit, timeout, search,
  * sitemap, includeSubdomains, the crawl's scope options under their crawl
  * names (ignoreQueryParameters, includePaths, excludePaths, regexOnFullURL,
- * crawlEntireDomain, deduplicateSimilarURLs), origin and integration.
- * Anything else is refused by name, `useIndex` with the supported route;
+ * crawlEntireDomain, deduplicateSimilarURLs), ignoreRobotsTxt, origin and
+ * integration. Anything else is refused by name, `useIndex` with the supported route;
  * nothing is silently ignored.
  */
 export declare function parseMapRequest(body: unknown): MapRequest;

package/dist/types/esm/contracts/checkpoint.d.ts

@@ -7,7 +7,7 @@
  * Granularity is the page. A partial parse inside a page is not a step; the
  * whole URL is retried. Block-level checkpoint is out of Phase 1.
  */
-import type { PageOptions, RequestAttribution, RobotsUrlOverride, WebhookEvent } from './api.js';
+import type { AccessChoice, PageOptions, RequestAttribution, RobotsUrlOverride, WebhookEvent } from './api.js';
 import type { CrawlMode } from './compliance.js';
 import type { WebhookPayloadFormat } from './delivery.js';
 import type { CrawlDiscovery, SitemapMode } from './crawl.js';
@@ -29,6 +29,11 @@ export interface CrawlBudget {
     maxWallMs: number | null;
     maxCostUsd: number | null;
     maxTokens: number | null;
+    /**
+     * What one page may spend on third parties (an access grant's `perRequestUsd`, ROADMAP PA item 4): its own cap
+     * within the run's, held by the run's spend ledger. Absent or null: none of its own.
+     */
+    maxCostPerPageUsd?: number | null;
 }
 export declare const DEFAULT_CRAWL_BUDGET: CrawlBudget;
 /**
@@ -72,12 +77,16 @@ export interface Task {
         maxConcurrency?: number;
         invalidURLs?: readonly string[];
         webhook?: StoredJobWebhook;
+        lane?: 'my-browser';
+        access?: AccessChoice;
     } & PageOptions;
     /**
      * Every crawl option but the page budget (`budget`), stored when the crawl
      * starts so a resumed crawl runs with the options it was started with.
      */
     crawl?: {
+        /** The plain access choice the crawl was started with (`standard` or `enhanced`); absent: the server's configuration. */
+        access?: 'standard' | 'enhanced';
         formats?: readonly ScrapeFormat[];
         includeLinks?: boolean;
         includePaths?: readonly string[];
@@ -106,6 +115,8 @@ export interface Task {
         maxConcurrency?: number | null;
         /** The crawl's webhook, when the request set one. */
         webhook?: StoredJobWebhook;
+        /** The crawl fetches what robots.txt disallows, on the record (CrawlStartRequest.ignoreRobotsTxt); absent: it obeys. A server that takes no override resumes it obeying. */
+        ignoreRobotsTxt?: boolean;
     } & PageOptions;
     /** Who started the task (`origin`, `integration`), stored with it and reported as `attribution` on its status; absent when the request named neither. */
     attribution?: RequestAttribution;
@@ -134,6 +145,11 @@ export interface Attempt {
     recoveredFromAttemptId?: string | null;
     /** What this attempt's pages offered the frontier and what became of it, written after every page of a crawl; absent for a batch and for an attempt stored before it was kept. */
     discovery?: CrawlDiscovery | null;
+    /**
+     * What the spend ledger charged this attempt's paid calls (ROADMAP PA item 4): a resumed or appended run of the task
+     * opens its ledger with every earlier attempt's charge, so its cap is the task's, not each run's. Absent: none.
+     */
+    chargedUsd?: number | null;
 }
 /**
  * One URL inside one attempt. The atomic checkpoint unit.

package/dist/types/esm/contracts/compliance.d.ts

@@ -109,13 +109,24 @@ export declare function researchUserAgent(contact?: string | null, host?: string
 export declare function declaredContact(userAgent: string): string | null;
 /** Whether a User-Agent is one research mode declares, in either format. */
 export declare function isResearchUserAgent(userAgent: string): boolean;
+/**
+ * The product token a robots.txt names to address Octocrawl itself
+ * (`User-agent: Octocrawl`), whatever User-Agent header a request sends. It
+ * is matched in every mode. A rule written for it (or for research mode's own
+ * token) is not set aside for a URL the request names or for a crawl or map
+ * started with ignoreRobotsTxt; only a recorded robotsOverride sets it aside.
+ */
+export declare const PRODUCT_ROBOTS_TOKEN = "octocrawl";
 /**
  * The text robots.txt `User-agent` lines are matched against for a
- * User-Agent W2L sends. SEC's format names no product token, so the research
- * token is added: a group for w2l-research governs research requests to
- * SEC.gov as it does on every other host.
+ * User-Agent Octocrawl sends: the header, with the product token added. SEC's
+ * format names no product token, so the research token is added as well: a
+ * group for w2l-research governs research requests to SEC.gov as it does on
+ * every other host.
  */
 export declare function robotsAgent(userAgent: string): string;
+/** Whether the robots.txt group that decided names Octocrawl itself (its product token, or research mode's) rather than every crawler (`*`). */
+export declare function isOctocrawlRobotsGroup(matchedAgent: string | null | undefined): boolean;
 /** The operator's contact from `W2L_CONTACT`, trimmed; null when unset or blank. The error never repeats the value. */
 export declare function operatorContact(env: Readonly<Record<string, string | undefined>>): string | null;
 /** An operator policy whose research-mode requests declare `W2L_CONTACT`, when it is set. */
@@ -254,6 +265,23 @@ export interface RobotsOverride {
     /** Who recorded the decision, when the caller wants that on the record. */
     recordedBy?: string;
 }
+/**
+ * On whose word a fetch went past robots.txt. `robots_override`: the
+ * caller's recorded decision for this URL (`RobotsOverride`).
+ * `user_named_url`: a local server fetching a URL the request named (a
+ * scrape, a batch entry), since robots.txt addresses crawlers that discover
+ * links, not the pages a person names. `ignore_robots_txt`: a crawl or map
+ * the caller started with `ignoreRobotsTxt` on a local server.
+ */
+export type RobotsOverrideBasis = 'robots_override' | 'user_named_url' | 'ignore_robots_txt';
+/**
+ * A robots override as a lane applies it: the caller's recorded one, or one
+ * W2L applies by rule, which says so in `basis` (absent: the caller's,
+ * `robots_override`). Set by W2L, never read from a request.
+ */
+export interface AppliedRobotsOverride extends RobotsOverride {
+    basis?: Exclude<RobotsOverrideBasis, 'robots_override'>;
+}
 /**
  * The outcome of consulting robots.txt for a single target URL. One record per
  * fetch. `consulted` distinguishes "we checked and it said X" from "there was
@@ -288,11 +316,11 @@ export interface RobotsDecision {
      */
     unreachable?: RobotsUnreachable;
     /**
-     * Present when a disallow the publisher wrote was set aside by a recorded
-     * decision: the fetch went ahead (`skippedFetch: false`) and this says on
-     * whose word. Never set for an unreachable robots.txt.
+     * Present when a disallow was set aside, the publisher's or the one an
+     * unreachable robots.txt implies: the fetch went ahead (`skippedFetch:
+     * false`) and this says on whose word.
      */
-    override?: RobotsOverride;
+    override?: AppliedRobotsOverride;
 }
 /**
  * What actually went on the wire. Sorted by header name, lowercased names.

package/dist/types/esm/contracts/crawl.d.ts

@@ -86,7 +86,7 @@ export interface SitemapFileRecord {
     kind: SitemapFileKind;
     /** `<loc>` entries the file holds (child sitemaps for an index), http(s) ones only; null when the file was not parsed. */
     entries: number | null;
-    /** The robots.txt verdict for the file's own URL under the crawl's identity (an unreachable robots.txt is `disallowed`, as for a page); null when the URL failed its egress check before robots.txt was consulted. */
+    /** The robots.txt verdict for the file's own URL under the crawl's identity (an unreachable robots.txt is `disallowed`, as for a page); null when the URL failed its egress check before robots.txt was consulted. A `disallowed` file is `refused`, unless the crawl or map was started with ignoreRobotsTxt and read it. */
     robots: 'allowed' | 'disallowed' | 'no_robots' | null;
     /** Whether the request left through the operator's environment proxy (local mode); a hosted server never has one. */
     proxyUsed: boolean;

package/dist/types/esm/contracts/evidenceRecord.d.ts

@@ -15,7 +15,7 @@ import type { PageActionType } from './actions.js';
  * changes its meaning. A change that cannot follow this rule gets a new
  * schemaVersion (`w2l.evidence/2`) and a new schema file.
  */
-import type { CrawlMode, RobotsUnreachable } from './compliance.js';
+import type { CrawlMode, RobotsOverrideBasis, RobotsUnreachable } from './compliance.js';
 import type { BlockReason, BudgetKind, FailureReason, Lane, ResultStatus } from './status.js';
 export declare const EVIDENCE_SCHEMA_VERSION = "w2l.evidence/1";
 /**
@@ -33,6 +33,124 @@ export type FieldEvidenceSource = (typeof FIELD_EVIDENCE_SOURCES)[number];
  */
 export declare const EVIDENCE_ARTIFACT_KINDS: readonly ["snapshot", "screenshot", "file"];
 export type EvidenceArtifactKind = (typeof EVIDENCE_ARTIFACT_KINDS)[number];
+/**
+ * The route that produced a result (ADR 0005): `http` (W2L's HTTP client, undici), `http_compat` (the
+ * browser-compatible HTTP transport, impit), `browser` (the local headless browser), `enhanced_browser`
+ * (the local browser on Patchright), `authed_browser` (the local browser with the user's saved login),
+ * `user_browser` (the person's own browser after a handoff), `vendor` (a third-party browser service).
+ */
+export declare const ACCESS_ROUTES: readonly ["http", "http_compat", "browser", "enhanced_browser", "authed_browser", "user_browser", "vendor"];
+export type AccessRoute = (typeof ACCESS_ROUTES)[number];
+/**
+ * How a result was reached, read from the result's own trace. Added to v1 with enhanced access
+ * (EVIDENCE_RECORD_ADDED_KEYS).
+ */
+export interface EvidenceAccess {
+    /** Null when no lane produced the result (a run cut before a rung answered, a rung that threw, a lockdown miss). */
+    route: AccessRoute | null;
+    /** The client that sent the requests: undici, impit, playwright, patchright, the person's browser, or the vendor's id; null when the result does not say. */
+    executor: string | null;
+    /** The executor's version as the lane reported it; null when it reported none. */
+    executorVersion: string | null;
+    /** The browser profile the HTTP transport sent (impit's); null on every other route. Its TLS fingerprint was not observed. */
+    profile: string | null;
+    /** Third-party spend of the run that produced the result: 0 when no paid service was called, null when a called service stated no price. */
+    externalCostUsd: number | null;
+    /**
+     * How the page was read, counted apart (ROADMAP PA items 7 and 8): `unattended` (W2L's own lanes, no session of the
+     * person's), `authorized_session` (with the person's saved login), `user_browser` (in the person's own Chrome, on a
+     * site they allowed, without a step of theirs), `handed_to_person` (in their Chrome, after they got through a check).
+     * Null when no page was read: the result is not success, partial or empty_verified, or no lane produced it. Added with
+     * the my-browser lane: optional, so that records written before it stay valid.
+     */
+    completion?: AccessCompletion | null;
+    /**
+     * The egress the page left through (ROADMAP PA item 3): the proxy's `host:port` and whether it came from the
+     * operator's pool (`W2L_EGRESS_PROXIES`) or the environment variables, or `direct` with no proxy when W2L's own lane
+     * recorded none and a page response shows a request was sent. `switchedFrom` names the pool egress the task last
+     * moved off before this page was read here (`egress_switched`); null otherwise. Null when nothing says where the
+     * requests left from: a vendor's service, the person's own browser, no lane at all, or a lane that stopped before
+     * a page request (robots.txt, an address check, a deadline). Added with the egress pool: optional, so that earlier
+     * records stay valid.
+     */
+    egress?: EvidenceAccessEgress | null;
+    /**
+     * The task cookie session the page was read with (`egress_sessions`): its id alone, never its cookies. Null when
+     * the page was read with none. Added with the egress pool: optional, so that earlier records stay valid.
+     */
+    session?: EvidenceAccessSession | null;
+    /**
+     * Every paid provider call the page was read with (ROADMAP PA item 4), in order: the provider, what the spend ledger
+     * reserved and charged, the price the provider stated, and what Octocrawl made of what came back. Null when no
+     * provider was called. Added with the spend ledger: optional, so that earlier records stay valid.
+     */
+    paidCalls?: readonly EvidencePaidCall[] | null;
+    /** The access grant the paid calls were made under; null when no provider was called. Added with `paidCalls`. */
+    grant?: EvidenceAccessGrant | null;
+}
+/** One paid provider call (ROADMAP PA item 4). A provider's own word on the page is never its outcome. */
+export interface EvidencePaidCall {
+    /** The provider's id, as the access grant's tariffs name it (`browserbase`, `steel`). */
+    provider: string;
+    /** The rung that made the call; a retry after the person's handoff is `provider(retry)`. */
+    rung: string;
+    /** The ADR 0005 capabilities the provider's session was created with: `vendor_remote_browser`, and solving or stealth when the grant named them. */
+    capabilities: readonly string[];
+    /** What the ledger reserved before the call: its price ceiling, from the grant's tariff. */
+    ceilingUsd: number;
+    /** What the ledger charged: the price the provider stated, else the ceiling (a call that threw or was cut included). */
+    chargedUsd: number;
+    /** The price the provider stated for the call; null when it stated none (Browserbase and Steel state none per call). */
+    reportedCostUsd: number | null;
+    /**
+     * What Octocrawl made of the page the call returned, by its own checks (a block page, an empty or unverified read, an
+     * identity it did not send): never the provider's word that it succeeded. Null when the call returned no page (it
+     * threw, or the deadline cut it).
+     */
+    outcome: ResultStatus | null;
+    /** Why the outcome is not a read page, as the record's own `reason`; null otherwise. */
+    reason: FailureReason | BlockReason | BudgetKind | null;
+    /** The record's own page is this call's. */
+    answer: boolean;
+}
+/** The access grant paid calls were made under: enough to name it, not a copy. */
+export interface EvidenceAccessGrant {
+    /** SHA-256 of the grant's text as Octocrawl read it (`shasum -a 256 grant.json`); null for a grant not read from text. */
+    sha256: string | null;
+    /** The grant's tier. */
+    tier: string;
+    /** When its attestation says the operator accepted the providers' terms and costs; null when it has none. */
+    attestedAt: string | null;
+}
+export declare const ACCESS_EGRESS_SOURCES: readonly ["pool", "environment", "direct"];
+export type AccessEgressSource = (typeof ACCESS_EGRESS_SOURCES)[number];
+export interface EvidenceAccessEgress {
+    /** The proxy's `host:port`, never its credentials; null when the request went direct. */
+    proxy: string | null;
+    source: AccessEgressSource;
+    /** The pool egress the task last left before this page was read here; null when it did not move. */
+    switchedFrom: string | null;
+    /**
+     * Where the pool egress leaves from, as the operator's echo URL (`W2L_EGRESS_ECHO_URL`) saw it through that proxy.
+     * Null when no echo URL is set, the echo did not answer, or the egress is not the pool's. Added after `egress`:
+     * optional, so that earlier records stay valid.
+     */
+    exit?: EvidenceAccessEgressExit | null;
+}
+export interface EvidenceAccessEgressExit {
+    /** The address the echo service saw the request come from. */
+    ip: string;
+    /** Its two-letter country code, when the echo service gives one; null otherwise. */
+    country: string | null;
+    /** When the echo was asked (UTC ISO); an exit is asked again after ten minutes. */
+    observedAt: string;
+}
+export interface EvidenceAccessSession {
+    /** The session's id, as `session_cookies` traces it. */
+    id: string;
+}
+export declare const ACCESS_COMPLETIONS: readonly ["unattended", "authorized_session", "user_browser", "handed_to_person"];
+export type AccessCompletion = (typeof ACCESS_COMPLETIONS)[number];
 export interface EvidenceRedirectChain {
     /**
      * Every URL W2L requested for the page, in order: the requested URL first,
@@ -61,8 +179,10 @@ export interface EvidenceRobotsDecision {
     /** Why robots.txt could not be fetched (then `decision` is `disallowed`, RFC 9309 §2.3.1.4); null when it was. */
     unreachable: RobotsUnreachable | null;
     crawlDelayMs: number | null;
-    /** Whether the fetch went ahead under a recorded robots override although `decision` is `disallowed`; the reason is in the trace, the warnings and, in the browser lane, the compliance record. */
+    /** Whether the fetch went ahead although `decision` is `disallowed`; on whose word is `overrideBasis`, and the reason is in the trace, the warnings and, in the browser lane, the compliance record. */
     userOverride: boolean;
+    /** On whose word the disallow was set aside (`RobotsOverrideBasis`); null when it was not. Optional in the v1 schema file, written on every record. */
+    overrideBasis: RobotsOverrideBasis | null;
 }
 export interface EvidenceOutputSha256 {
     /** SHA-256 of the UTF-8 bytes of the delivered `markdown`; null when none was delivered. */
@@ -169,12 +289,14 @@ export interface EvidenceRecord {
      * the caller's ran in it, so its content may be the script's.
      */
     pageActions: EvidencePageActions | null;
+    /** How the result was reached (EvidenceAccess). Added to v1 later (EVIDENCE_RECORD_ADDED_KEYS). */
+    access: EvidenceAccess;
 }
 /** Field order of the record and of each nested object, as in the schema file. */
 export declare const EVIDENCE_RECORD_KEYS: {
-    readonly record: readonly ["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions"];
+    readonly record: readonly ["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions", "access"];
     readonly redirectChain: readonly ["urls", "complete"];
-    readonly robotsDecision: readonly ["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride"];
+    readonly robotsDecision: readonly ["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride", "overrideBasis"];
     readonly outputSha256: readonly ["markdown", "json"];
     readonly extractor: readonly ["name", "version", "commit"];
     readonly fieldEvidence: readonly ["source", "locator"];
@@ -183,6 +305,12 @@ export declare const EVIDENCE_RECORD_KEYS: {
     readonly pageActions: readonly ["steps", "scriptRan"];
     readonly pageActionStep: readonly ["type", "outcome"];
     readonly requestHeader: readonly ["name", "valueSha256"];
+    readonly access: readonly ["route", "executor", "executorVersion", "profile", "externalCostUsd", "completion", "egress", "session", "paidCalls", "grant"];
+    readonly accessEgress: readonly ["proxy", "source", "switchedFrom", "exit"];
+    readonly accessEgressExit: readonly ["ip", "country", "observedAt"];
+    readonly accessSession: readonly ["id"];
+    readonly accessPaidCall: readonly ["provider", "rung", "capabilities", "ceilingUsd", "chargedUsd", "reportedCostUsd", "outcome", "reason", "answer"];
+    readonly accessGrant: readonly ["sha256", "tier", "attestedAt"];
 };
 /**
  * Keys added to v1 after it was first published: optional in the schema, so