@octocrawl/sdk 0.3.0 → 0.3.2

package/README.md

@@ -14,4 +14,4 @@ const { report, items } = await client.batchAndWait(['https://example.com/a', 'h
 
 Start a local API with `npx octocrawl serve`. Requires a runtime with `fetch` (Node.js 18 or later, browsers, Deno, Bun).
 
-Licence: MIT. Source and the API reference: https://github.com/77777R7/Octocrawl
+Licence: MIT. Website and docs: https://octocrawl.dev. Source and the API reference: https://github.com/77777R7/Octocrawl

package/dist/index.cjs

@@ -216,13 +216,13 @@ function isApiErrorCode(value) {
 var RATE_LIMITED_CODE = "rate_limited";
 var PAGE_KEYS = ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown"];
 var ATTRIBUTION_KEYS = ["origin", "integration"];
-var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "lane", "access", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var CRAWL_SCOPE_KEYS = ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-var CRAWL_KEYS = ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
-var BATCH_KEYS = ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var CRAWL_KEYS = ["url", "mode", "access", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var BATCH_KEYS = ["urls", "mode", "lane", "access", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var BATCH_APPEND_KEYS = ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", ...ATTRIBUTION_KEYS];
 var MAP_SCOPE_KEYS = ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
-var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", ...ATTRIBUTION_KEYS];
+var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", "ignoreRobotsTxt", ...ATTRIBUTION_KEYS];
 var WEBHOOK_HEADERS_MESSAGE = `webhook.headers must be an object of at most ${MAX_WEBHOOK_HEADERS} string values`;
 var WEBHOOK_METADATA_MESSAGE = `webhook.metadata must be an object of at most ${MAX_WEBHOOK_METADATA_ENTRIES} string values of at most ${MAX_WEBHOOK_METADATA_VALUE_LENGTH} characters`;
 var WEBHOOK_EVENTS_MESSAGE = `webhook.events must be a non-empty array of ${WEBHOOK_EVENTS.join(", ")} without duplicates`;
@@ -243,9 +243,9 @@ var SHIM_MAP_KEYS = ["url", "search", "sitemap", "ignoreSitemap", "sitemapOnly",
 // packages/contracts/dist/evidenceRecord.js
 var keysOf = () => (keys) => keys;
 var EVIDENCE_RECORD_KEYS = {
-  record: keysOf()(["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions"]),
+  record: keysOf()(["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions", "access"]),
   redirectChain: keysOf()(["urls", "complete"]),
-  robotsDecision: keysOf()(["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride"]),
+  robotsDecision: keysOf()(["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride", "overrideBasis"]),
   outputSha256: keysOf()(["markdown", "json"]),
   extractor: keysOf()(["name", "version", "commit"]),
   fieldEvidence: keysOf()(["source", "locator"]),
@@ -253,11 +253,17 @@ var EVIDENCE_RECORD_KEYS = {
   identity: keysOf()(["userAgent", "mode", "contact", "device", "requestHeaders"]),
   pageActions: keysOf()(["steps", "scriptRan"]),
   pageActionStep: keysOf()(["type", "outcome"]),
-  requestHeader: keysOf()(["name", "valueSha256"])
+  requestHeader: keysOf()(["name", "valueSha256"]),
+  access: keysOf()(["route", "executor", "executorVersion", "profile", "externalCostUsd", "completion", "egress", "session", "paidCalls", "grant"]),
+  accessEgress: keysOf()(["proxy", "source", "switchedFrom", "exit"]),
+  accessEgressExit: keysOf()(["ip", "country", "observedAt"]),
+  accessSession: keysOf()(["id"]),
+  accessPaidCall: keysOf()(["provider", "rung", "capabilities", "ceilingUsd", "chargedUsd", "reportedCostUsd", "outcome", "reason", "answer"]),
+  accessGrant: keysOf()(["sha256", "tier", "attestedAt"])
 };
 
 // packages/sdk/src/version.ts
-var SDK_VERSION = "0.3.0";
+var SDK_VERSION = "0.3.2";
 
 // packages/sdk/src/watcher.ts
 var DEFAULT_WATCH_POLL_INTERVAL_MS = 2e3;
@@ -790,7 +796,7 @@ var W2L = class {
   }
   async scrape(url, opts = {}, request = {}) {
     const deadlineMs = Number.isInteger(opts.timeout) ? Math.min(Math.max(opts.timeout, 0), DEFAULT_SCRAPE_TIMEOUT_MS) : DEFAULT_SCRAPE_TIMEOUT_MS;
-    const handedOver = opts.handoff !== void 0 && opts.handoff !== false;
+    const handedOver = opts.handoff !== void 0 && opts.handoff !== false || opts.lane === "my-browser" || opts.access === "my-browser";
     return this.post("/v1/scrape", { ...opts, url, origin: originOf(opts, request) }, 200, request, handedOver ? 0 : deadlineMs + SCRAPE_ANSWER_MARGIN_MS);
   }
   /** The record of one scrape call, by the `scrapeId` its response carried (`metadata.scrapeId`); a W2LError with code `not_found` for an id the server has no record of. */

package/dist/index.js

@@ -179,13 +179,13 @@ function isApiErrorCode(value) {
 var RATE_LIMITED_CODE = "rate_limited";
 var PAGE_KEYS = ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown"];
 var ATTRIBUTION_KEYS = ["origin", "integration"];
-var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var SCRAPE_KEYS = ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "lane", "access", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var CRAWL_SCOPE_KEYS = ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-var CRAWL_KEYS = ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
-var BATCH_KEYS = ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var CRAWL_KEYS = ["url", "mode", "access", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", ...CRAWL_SCOPE_KEYS, "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
+var BATCH_KEYS = ["urls", "mode", "lane", "access", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", ...PAGE_KEYS, ...ATTRIBUTION_KEYS];
 var BATCH_APPEND_KEYS = ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", ...ATTRIBUTION_KEYS];
 var MAP_SCOPE_KEYS = ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
-var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", ...ATTRIBUTION_KEYS];
+var MAP_KEYS = ["url", "mode", "limit", "timeout", "search", "sitemap", ...MAP_SCOPE_KEYS, "includePaths", "excludePaths", "ignoreRobotsTxt", ...ATTRIBUTION_KEYS];
 var WEBHOOK_HEADERS_MESSAGE = `webhook.headers must be an object of at most ${MAX_WEBHOOK_HEADERS} string values`;
 var WEBHOOK_METADATA_MESSAGE = `webhook.metadata must be an object of at most ${MAX_WEBHOOK_METADATA_ENTRIES} string values of at most ${MAX_WEBHOOK_METADATA_VALUE_LENGTH} characters`;
 var WEBHOOK_EVENTS_MESSAGE = `webhook.events must be a non-empty array of ${WEBHOOK_EVENTS.join(", ")} without duplicates`;
@@ -206,9 +206,9 @@ var SHIM_MAP_KEYS = ["url", "search", "sitemap", "ignoreSitemap", "sitemapOnly",
 // packages/contracts/dist/evidenceRecord.js
 var keysOf = () => (keys) => keys;
 var EVIDENCE_RECORD_KEYS = {
-  record: keysOf()(["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions"]),
+  record: keysOf()(["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions", "access"]),
   redirectChain: keysOf()(["urls", "complete"]),
-  robotsDecision: keysOf()(["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride"]),
+  robotsDecision: keysOf()(["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride", "overrideBasis"]),
   outputSha256: keysOf()(["markdown", "json"]),
   extractor: keysOf()(["name", "version", "commit"]),
   fieldEvidence: keysOf()(["source", "locator"]),
@@ -216,11 +216,17 @@ var EVIDENCE_RECORD_KEYS = {
   identity: keysOf()(["userAgent", "mode", "contact", "device", "requestHeaders"]),
   pageActions: keysOf()(["steps", "scriptRan"]),
   pageActionStep: keysOf()(["type", "outcome"]),
-  requestHeader: keysOf()(["name", "valueSha256"])
+  requestHeader: keysOf()(["name", "valueSha256"]),
+  access: keysOf()(["route", "executor", "executorVersion", "profile", "externalCostUsd", "completion", "egress", "session", "paidCalls", "grant"]),
+  accessEgress: keysOf()(["proxy", "source", "switchedFrom", "exit"]),
+  accessEgressExit: keysOf()(["ip", "country", "observedAt"]),
+  accessSession: keysOf()(["id"]),
+  accessPaidCall: keysOf()(["provider", "rung", "capabilities", "ceilingUsd", "chargedUsd", "reportedCostUsd", "outcome", "reason", "answer"]),
+  accessGrant: keysOf()(["sha256", "tier", "attestedAt"])
 };
 
 // packages/sdk/src/version.ts
-var SDK_VERSION = "0.3.0";
+var SDK_VERSION = "0.3.2";
 
 // packages/sdk/src/watcher.ts
 var DEFAULT_WATCH_POLL_INTERVAL_MS = 2e3;
@@ -753,7 +759,7 @@ var W2L = class {
   }
   async scrape(url, opts = {}, request = {}) {
     const deadlineMs = Number.isInteger(opts.timeout) ? Math.min(Math.max(opts.timeout, 0), DEFAULT_SCRAPE_TIMEOUT_MS) : DEFAULT_SCRAPE_TIMEOUT_MS;
-    const handedOver = opts.handoff !== void 0 && opts.handoff !== false;
+    const handedOver = opts.handoff !== void 0 && opts.handoff !== false || opts.lane === "my-browser" || opts.access === "my-browser";
     return this.post("/v1/scrape", { ...opts, url, origin: originOf(opts, request) }, 200, request, handedOver ? 0 : deadlineMs + SCRAPE_ANSWER_MARGIN_MS);
   }
   /** The record of one scrape call, by the `scrapeId` its response carried (`metadata.scrapeId`); a W2LError with code `not_found` for an id the server has no record of. */

package/dist/types/cjs/client.d.ts

@@ -33,7 +33,7 @@ export interface RequestOptions {
     origin?: string;
 }
 /** What the SDK records as `origin` unless the caller or the host says otherwise. */
-export declare const SDK_ORIGIN = "js-sdk@0.3.0";
+export declare const SDK_ORIGIN = "js-sdk@0.3.2";
 /**
  * Polling for waitBatch and waitCrawl. A status request that fails with a
  * network error, HTTP 408, 429 or 5xx is retried: after 1, 2, 4, 8, then

package/dist/types/cjs/contracts/actions.d.ts

@@ -4,6 +4,7 @@
  * Each step is recorded in the trace with its outcome and timing; a step
  * that fails ends the pipeline, and the result keeps the page as it stood.
  */
+import type { BlockReason } from './status.js';
 import type { ScreenshotEvidence } from './result.js';
 import type { ScreenshotViewport } from './structured.js';
 /** The most steps one request may run. */
@@ -149,8 +150,20 @@ export interface ActionPdf {
     path: string | null;
     base64: string;
 }
-/** Why a scrollToEnd, loadMore or paginate step stopped. `max` and `deadline` stop short of the list's end. */
-export type ListStop = 'end' | 'no_growth' | 'repeat' | 'max' | 'deadline';
+/**
+ * Why a scrollToEnd, loadMore or paginate step stopped. `max` and `deadline` stop short of the list's end; so does
+ * `challenge`: a check the site put up on the next page (a Cloudflare interstitial, a page of nothing but a CAPTCHA),
+ * which paginate stops at without reading it, the result then `blocked` with the check's reason.
+ */
+export type ListStop = 'end' | 'no_growth' | 'repeat' | 'max' | 'deadline' | 'challenge';
+/** The check a paginate step stopped at (ListStop `challenge`): the page it would have been, its URL, and what the gate saw. */
+export interface ListChallenge {
+    /** 1-based position the page would have had among the pages read. */
+    page: number;
+    url: string;
+    reason: BlockReason;
+    signals: readonly string[];
+}
 /** What a scrollToEnd, loadMore or paginate step did. */
 export interface ListRun {
     index: number;
@@ -160,8 +173,27 @@ export interface ListRun {
     rounds: number;
     /** Elements matching `itemSelector` at the end (on the last page for paginate; summed over its pages in `itemsRead`); null without one. */
     items: number | null;
-    /** paginate: elements matching `itemSelector` over every page read; null without one. */
+    /**
+     * paginate: elements matching `itemSelector` over every page read; null without one. After a continuation in the person's
+     * Chrome (`continued`), the kept pages' count plus each page they showed, as their tab counted it, only when the addresses show
+     * no page can be counted twice (the kept pages each at its own, the check's page and the pages shown at none of them, the check
+     * not at the list's own address), no page shows again what another shows (its items' whole text, as the list merge tells it:
+     * a result set tied to the session that made it comes back at new addresses), and every count is known; null (unknown) otherwise.
+     */
     itemsRead?: number | null;
+    /** paginate: pages taken from the task's checkpoint after a run cut at page N (ExecutionContext.listResume), counted in `rounds`; absent when none. */
+    resumed?: number;
+    /** paginate: the check the step stopped at, with `stoppedBy` `challenge`; absent otherwise. */
+    challenge?: ListChallenge;
+    /**
+     * paginate: the pages read after the check at `from`, by the person paging on in their own browser once they got through
+     * it (the batch handoff), counted in `rounds`; `stoppedBy` then says how that reading ended. Absent otherwise.
+     */
+    continued?: {
+        from: number;
+        pages: number;
+        by: 'user_browser';
+    };
 }
 /** What the steps produced, each list in the order of its steps. */
 export interface ActionsResult {
@@ -170,7 +202,8 @@ export interface ActionsResult {
     scrapes: {
         url: string;
         html: string;
-        step?: number;
+        step?: number; /** Set when the person read the page in their own browser (a list's continuation after a check); absent for W2L's own browser. */
+        by?: 'user_browser';
     }[];
     /** `type` is the JavaScript `typeof` of the value (`null` for null). */
     javascriptReturns: {

package/dist/types/cjs/contracts/api.d.ts

@@ -170,7 +170,28 @@ export interface ScrapeRequest extends PageOptions, RequestAttribution {
     handoff?: {
         waitMs?: number;
     };
+    /**
+     * `my-browser`: read the page in the person's own Chrome, over remote
+     * debugging, without W2L fetching it first. The person allows the
+     * connection in Chrome, then the site in a page W2L opens there; the page
+     * is read without a click of theirs only on a site they allowed, and a
+     * check it shows waits for them (`handoff.waitMs`, default 10 min). Lane
+     * `my_browser`; never cached. Offered only by a server on the person's
+     * own machine; refused elsewhere, with `actions` or a screenshot, and with
+     * a mode other than standard (`unsupported_parameter`).
+     */
+    lane?: 'my-browser';
+    /** One of three plain choices of how the page is reached (ACCESS_CHOICES); omitted, the server's own configuration. */
+    access?: AccessChoice;
 }
+/**
+ * How pages are reached, as three plain choices (ROADMAP PA item 7) beside the per-route options:
+ * `standard` (Octocrawl's own lanes and none that costs a third party), `enhanced` (also what the server's
+ * access grant of tier enhanced approves, within its budget; refused on a server without one), `my-browser`
+ * (the person's own Chrome, as `lane: "my-browser"`; a scrape or a batch only).
+ */
+export declare const ACCESS_CHOICES: readonly ["standard", "enhanced", "my-browser"];
+export type AccessChoice = (typeof ACCESS_CHOICES)[number];
 /** A recorded robots override for one URL of a batch. */
 export interface RobotsUrlOverride extends RobotsOverride {
     url: string;
@@ -344,6 +365,8 @@ export interface JobWebhookStatus {
 export interface CrawlStartRequest extends PageOptions, RequestAttribution {
     url: string;
     mode?: ApiCrawlMode;
+    /** `standard` or `enhanced` (ACCESS_CHOICES); a crawl does not take `my-browser`. */
+    access?: Exclude<AccessChoice, 'my-browser'>;
     maxPages?: number | null;
     maxDepth?: number | null;
     /**
@@ -411,6 +434,16 @@ export interface CrawlStartRequest extends PageOptions, RequestAttribution {
     idempotencyKey?: string;
     /** A receiver for the crawl's events (`started`, one `page` per page recorded, then `completed`, `failed` or `cancelled`); see WebhookConfig. */
     webhook?: WebhookOption;
+    /**
+     * Fetch the pages and sitemap files robots.txt disallows, or whose
+     * robots.txt could not be read (Firecrawl v2's name). robots.txt is still
+     * read for every host and its verdict recorded on each page, Crawl-delay
+     * applied, with a `robots_overridden` warning and `overrideBasis:
+     * "ignore_robots_txt"` where a rule was set aside. A local server only: a
+     * hosted one refuses it by name. Default false: a crawl's links obey
+     * robots.txt.
+     */
+    ignoreRobotsTxt?: boolean;
 }
 /** What the parser hands the engine: the request plus, from the `/fc` shim, the payload shape its receiver expects. */
 export type ParsedCrawlStartRequest = CrawlStartRequest & {
@@ -454,6 +487,14 @@ export interface MapRequest extends RequestAttribution {
     crawlEntireDomain?: boolean;
     /** Default true, as on a crawl; a returned http link gives way to its https variant when that comes too, on an origin whose robots.txt the map read anyway and which allows it. */
     deduplicateSimilarURLs?: boolean;
+    /**
+     * Return the URLs robots.txt disallows, or whose robots.txt could not be
+     * read, with that verdict on each link (`robots: "disallowed"` or
+     * `"unreachable"`), and read the start page and sitemap files past it.
+     * robots.txt is still read, within MAP_MAX_ROBOTS_HOSTS. A local server
+     * only: a hosted one refuses it by name. Default false.
+     */
+    ignoreRobotsTxt?: boolean;
 }
 /** A map's `search`: at most this many characters after trimming, and this many whitespace-separated words. */
 export declare const MAP_SEARCH_MAX_CHARS = 200;
@@ -482,6 +523,7 @@ export interface ActiveCrawlOptions {
     allowExternalLinks: boolean;
     regexOnFullURL: boolean;
     maxConcurrency: number | null;
+    ignoreRobotsTxt: boolean;
     /** The per-page options every page of the crawl gets: its formats, `includeLinks` and the page options. */
     scrapeOptions: PageOptions & {
         formats: readonly ScrapeFormat[];
@@ -507,6 +549,16 @@ export interface ActiveCrawlList {
 export interface BatchStartRequest extends PageOptions, RequestAttribution {
     urls: readonly string[];
     mode?: ApiCrawlMode;
+    /** One of three plain choices of how pages are reached (ACCESS_CHOICES); `my-browser` is `lane: "my-browser"`. */
+    access?: AccessChoice;
+    /**
+     * `my-browser`: read every page in the person's own Chrome, one at a time, as a scrape's `lane` does. The person
+     * allows the connection in Chrome, then all the batch's sites (host and port) in the page W2L opens there, once for
+     * the run; a site not among them is not read. A resumed run asks again. Offered only by a server on the person's own
+     * machine; refused elsewhere, with `actions`, a screenshot, lockdown, a mode other than standard, `maxConcurrency`
+     * above 1 or a webhook (`unsupported_parameter`).
+     */
+    lane?: 'my-browser';
     formats?: readonly ScrapeFormat[];
     includeLinks?: boolean;
     /** Recorded robots overrides, each for one URL of `urls`. A hosted server refuses the field (`unsupported_parameter`). */
@@ -585,6 +637,8 @@ export interface BatchStatusResponse extends CrawlReport {
     invalidURLs?: readonly string[];
     /** Items stopped at a check a person can get through in their own Chrome (`POST /v1/batches/:id/handoff`); present on a server that offers the handoff. */
     waitingForPerson?: number;
+    /** A batch on the my-browser lane waiting for the person to allow its sites in the page Octocrawl opened in their Chrome; present only while it waits. */
+    waitingForApproval?: true;
 }
 /**
  * The checks a batch item can be handed to a person for, and the routing
@@ -809,26 +863,28 @@ export declare class RequestError extends Error {
 }
 /** The hints a refusal carries for the options W2L does not offer: the next honest step, never a way around the refusal. */
 export declare const REFUSAL_HINTS: {
-    readonly stealth: "W2L does not offer a stealth mode or stealth proxies; a proxy or session you own (mode authed) is the supported route";
-    readonly ignoreRobotsTxt: "robots.txt is always read; a robotsOverride with a recorded reason fetches one URL past its rule, on the record";
-    readonly hostedSkipTlsVerification: "a hosted server verifies every certificate; run W2L locally to use skipTlsVerification, which is recorded in the trace and a tls_unverified warning";
-    readonly useIndex: "W2L keeps no URL index: a map reads the sitemaps the site declares and its start page, on the record; crawl reads further pages";
+    readonly stealth: "Octocrawl has no stealth option on a request: a provider's stealth or challenge solving runs only on a server started with an access grant that names it (--access-grant, ADR 0005); a proxy or session you own (mode authed) is the other route";
+    readonly ignoreRobotsTxt: "robots.txt is always read and recorded; on a local server a URL a scrape or batch names is fetched whatever it says, and ignoreRobotsTxt on a crawl or map fetches the links it disallows, on the record";
+    readonly hostedSkipTlsVerification: "a hosted server verifies every certificate; run Octocrawl locally to use skipTlsVerification, which is recorded in the trace and a tls_unverified warning";
+    readonly useIndex: "Octocrawl keeps no URL index: a map reads the sitemaps the site declares and its start page, on the record; crawl reads further pages";
     readonly actions: "actions run on scrape and batch, where each page named gets the same steps; a crawl or a map does not take them";
 };
 /** The hint for a refused request key, or null when the key has none (an option W2L simply does not know). */
 export declare function refusalHint(key: string, value: unknown): string | null;
 export declare const PAGE_KEYS: readonly ["onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown"];
 export declare const ATTRIBUTION_KEYS: readonly ["origin", "integration"];
-export declare const SCRAPE_KEYS: readonly ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const SCRAPE_KEYS: readonly ["url", "mode", "allowlistedDomains", "formats", "includeLinks", "debug", "robotsOverride", "actions", "handoff", "lane", "access", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+/** The lanes a request may ask for by name. */
+export declare const REQUEST_LANES: readonly ["my-browser"];
 export declare const CRAWL_SCOPE_KEYS: readonly ["regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks"];
-export declare const CRAWL_KEYS: readonly ["url", "mode", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", "regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks", "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
-export declare const BATCH_KEYS: readonly ["urls", "mode", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const CRAWL_KEYS: readonly ["url", "mode", "access", "maxPages", "maxDepth", "useCached", "allowlistedDomains", "formats", "includeLinks", "includePaths", "excludePaths", "regexOnFullURL", "ignoreQueryParameters", "deduplicateSimilarURLs", "crawlEntireDomain", "allowSubdomains", "allowExternalLinks", "sitemap", "maxConcurrency", "idempotencyKey", "webhook", "ignoreRobotsTxt", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
+export declare const BATCH_KEYS: readonly ["urls", "mode", "lane", "access", "formats", "includeLinks", "robotsOverrides", "maxConcurrency", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "appendToId", "webhook", "actions", "onlyMainContent", "waitFor", "timeout", "maxFileBytes", "includeTags", "excludeTags", "headers", "mobile", "skipTlsVerification", "fastMode", "blockAds", "removeBase64Images", "parsers", "maxAge", "minAge", "storeInCache", "lockdown", "origin", "integration"];
 /** What a batch body may carry beside `appendToId`: the job's own options are not among them (the scope no-ops change nothing, so they may come along). */
 export declare const BATCH_APPEND_KEYS: readonly ["urls", "appendToId", "ignoreInvalidURLs", "allowExternalLinks", "includeSubdomains", "idempotencyKey", "robotsOverrides", "origin", "integration"];
 /** The scope options a map takes under their crawl names; allowSubdomains is includeSubdomains on a map, and allowExternalLinks is not offered. */
 export declare const MAP_SCOPE_KEYS: readonly ["includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs"];
 /** What a map takes. No page option (headers, mobile, skipTlsVerification, formats, ...): a map has nothing to loosen. */
-export declare const MAP_KEYS: readonly ["url", "mode", "limit", "timeout", "search", "sitemap", "includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs", "includePaths", "excludePaths", "origin", "integration"];
+export declare const MAP_KEYS: readonly ["url", "mode", "limit", "timeout", "search", "sitemap", "includeSubdomains", "ignoreQueryParameters", "regexOnFullURL", "crawlEntireDomain", "deduplicateSimilarURLs", "includePaths", "excludePaths", "ignoreRobotsTxt", "origin", "integration"];
 /** Why a webhook header (lower-cased name) cannot be sent, or null when it can: W2L's own and the transport's names are reserved. */
 export declare function webhookHeaderRefusal(name: string): string | null;
 /**
@@ -869,8 +925,8 @@ export declare function parseCrawlStartRequest(body: unknown): CrawlStartRequest
  * A map request: url, mode (standard or research), limit, timeout, search,
  * sitemap, includeSubdomains, the crawl's scope options under their crawl
  * names (ignoreQueryParameters, includePaths, excludePaths, regexOnFullURL,
- * crawlEntireDomain, deduplicateSimilarURLs), origin and integration.
- * Anything else is refused by name, `useIndex` with the supported route;
+ * crawlEntireDomain, deduplicateSimilarURLs), ignoreRobotsTxt, origin and
+ * integration. Anything else is refused by name, `useIndex` with the supported route;
  * nothing is silently ignored.
  */
 export declare function parseMapRequest(body: unknown): MapRequest;

package/dist/types/cjs/contracts/checkpoint.d.ts

@@ -7,7 +7,7 @@
  * Granularity is the page. A partial parse inside a page is not a step; the
  * whole URL is retried. Block-level checkpoint is out of Phase 1.
  */
-import type { PageOptions, RequestAttribution, RobotsUrlOverride, WebhookEvent } from './api.js';
+import type { AccessChoice, PageOptions, RequestAttribution, RobotsUrlOverride, WebhookEvent } from './api.js';
 import type { CrawlMode } from './compliance.js';
 import type { WebhookPayloadFormat } from './delivery.js';
 import type { CrawlDiscovery, SitemapMode } from './crawl.js';
@@ -29,6 +29,11 @@ export interface CrawlBudget {
     maxWallMs: number | null;
     maxCostUsd: number | null;
     maxTokens: number | null;
+    /**
+     * What one page may spend on third parties (an access grant's `perRequestUsd`, ROADMAP PA item 4): its own cap
+     * within the run's, held by the run's spend ledger. Absent or null: none of its own.
+     */
+    maxCostPerPageUsd?: number | null;
 }
 export declare const DEFAULT_CRAWL_BUDGET: CrawlBudget;
 /**
@@ -72,12 +77,16 @@ export interface Task {
         maxConcurrency?: number;
         invalidURLs?: readonly string[];
         webhook?: StoredJobWebhook;
+        lane?: 'my-browser';
+        access?: AccessChoice;
     } & PageOptions;
     /**
      * Every crawl option but the page budget (`budget`), stored when the crawl
      * starts so a resumed crawl runs with the options it was started with.
      */
     crawl?: {
+        /** The plain access choice the crawl was started with (`standard` or `enhanced`); absent: the server's configuration. */
+        access?: 'standard' | 'enhanced';
         formats?: readonly ScrapeFormat[];
         includeLinks?: boolean;
         includePaths?: readonly string[];
@@ -106,6 +115,8 @@ export interface Task {
         maxConcurrency?: number | null;
         /** The crawl's webhook, when the request set one. */
         webhook?: StoredJobWebhook;
+        /** The crawl fetches what robots.txt disallows, on the record (CrawlStartRequest.ignoreRobotsTxt); absent: it obeys. A server that takes no override resumes it obeying. */
+        ignoreRobotsTxt?: boolean;
     } & PageOptions;
     /** Who started the task (`origin`, `integration`), stored with it and reported as `attribution` on its status; absent when the request named neither. */
     attribution?: RequestAttribution;
@@ -134,6 +145,11 @@ export interface Attempt {
     recoveredFromAttemptId?: string | null;
     /** What this attempt's pages offered the frontier and what became of it, written after every page of a crawl; absent for a batch and for an attempt stored before it was kept. */
     discovery?: CrawlDiscovery | null;
+    /**
+     * What the spend ledger charged this attempt's paid calls (ROADMAP PA item 4): a resumed or appended run of the task
+     * opens its ledger with every earlier attempt's charge, so its cap is the task's, not each run's. Absent: none.
+     */
+    chargedUsd?: number | null;
 }
 /**
  * One URL inside one attempt. The atomic checkpoint unit.

package/dist/types/cjs/contracts/compliance.d.ts

@@ -109,13 +109,24 @@ export declare function researchUserAgent(contact?: string | null, host?: string
 export declare function declaredContact(userAgent: string): string | null;
 /** Whether a User-Agent is one research mode declares, in either format. */
 export declare function isResearchUserAgent(userAgent: string): boolean;
+/**
+ * The product token a robots.txt names to address Octocrawl itself
+ * (`User-agent: Octocrawl`), whatever User-Agent header a request sends. It
+ * is matched in every mode. A rule written for it (or for research mode's own
+ * token) is not set aside for a URL the request names or for a crawl or map
+ * started with ignoreRobotsTxt; only a recorded robotsOverride sets it aside.
+ */
+export declare const PRODUCT_ROBOTS_TOKEN = "octocrawl";
 /**
  * The text robots.txt `User-agent` lines are matched against for a
- * User-Agent W2L sends. SEC's format names no product token, so the research
- * token is added: a group for w2l-research governs research requests to
- * SEC.gov as it does on every other host.
+ * User-Agent Octocrawl sends: the header, with the product token added. SEC's
+ * format names no product token, so the research token is added as well: a
+ * group for w2l-research governs research requests to SEC.gov as it does on
+ * every other host.
  */
 export declare function robotsAgent(userAgent: string): string;
+/** Whether the robots.txt group that decided names Octocrawl itself (its product token, or research mode's) rather than every crawler (`*`). */
+export declare function isOctocrawlRobotsGroup(matchedAgent: string | null | undefined): boolean;
 /** The operator's contact from `W2L_CONTACT`, trimmed; null when unset or blank. The error never repeats the value. */
 export declare function operatorContact(env: Readonly<Record<string, string | undefined>>): string | null;
 /** An operator policy whose research-mode requests declare `W2L_CONTACT`, when it is set. */
@@ -254,6 +265,23 @@ export interface RobotsOverride {
     /** Who recorded the decision, when the caller wants that on the record. */
     recordedBy?: string;
 }
+/**
+ * On whose word a fetch went past robots.txt. `robots_override`: the
+ * caller's recorded decision for this URL (`RobotsOverride`).
+ * `user_named_url`: a local server fetching a URL the request named (a
+ * scrape, a batch entry), since robots.txt addresses crawlers that discover
+ * links, not the pages a person names. `ignore_robots_txt`: a crawl or map
+ * the caller started with `ignoreRobotsTxt` on a local server.
+ */
+export type RobotsOverrideBasis = 'robots_override' | 'user_named_url' | 'ignore_robots_txt';
+/**
+ * A robots override as a lane applies it: the caller's recorded one, or one
+ * W2L applies by rule, which says so in `basis` (absent: the caller's,
+ * `robots_override`). Set by W2L, never read from a request.
+ */
+export interface AppliedRobotsOverride extends RobotsOverride {
+    basis?: Exclude<RobotsOverrideBasis, 'robots_override'>;
+}
 /**
  * The outcome of consulting robots.txt for a single target URL. One record per
  * fetch. `consulted` distinguishes "we checked and it said X" from "there was
@@ -288,11 +316,11 @@ export interface RobotsDecision {
      */
     unreachable?: RobotsUnreachable;
     /**
-     * Present when a disallow the publisher wrote was set aside by a recorded
-     * decision: the fetch went ahead (`skippedFetch: false`) and this says on
-     * whose word. Never set for an unreachable robots.txt.
+     * Present when a disallow was set aside, the publisher's or the one an
+     * unreachable robots.txt implies: the fetch went ahead (`skippedFetch:
+     * false`) and this says on whose word.
      */
-    override?: RobotsOverride;
+    override?: AppliedRobotsOverride;
 }
 /**
  * What actually went on the wire. Sorted by header name, lowercased names.

package/dist/types/cjs/contracts/crawl.d.ts

@@ -86,7 +86,7 @@ export interface SitemapFileRecord {
     kind: SitemapFileKind;
     /** `<loc>` entries the file holds (child sitemaps for an index), http(s) ones only; null when the file was not parsed. */
     entries: number | null;
-    /** The robots.txt verdict for the file's own URL under the crawl's identity (an unreachable robots.txt is `disallowed`, as for a page); null when the URL failed its egress check before robots.txt was consulted. */
+    /** The robots.txt verdict for the file's own URL under the crawl's identity (an unreachable robots.txt is `disallowed`, as for a page); null when the URL failed its egress check before robots.txt was consulted. A `disallowed` file is `refused`, unless the crawl or map was started with ignoreRobotsTxt and read it. */
     robots: 'allowed' | 'disallowed' | 'no_robots' | null;
     /** Whether the request left through the operator's environment proxy (local mode); a hosted server never has one. */
     proxyUsed: boolean;