@octocrawl/sdk 0.3.0

package/dist/types/esm/contracts/crawl.d.ts

@@ -0,0 +1,302 @@
+/**
+ * Crawl composition contract: one scrape atom plus a crawl-level report.
+ *
+ * The orchestrator never opens Playwright. A page is one `scrape(url)`.
+ * Types only — no I/O.
+ */
+import type { AgentHints, JobWebhookStatus, RequestAttribution } from './api.js';
+import type { CrawlBudget, StepStatus, TaskStatus } from './checkpoint.js';
+import type { CrawlMode } from './compliance.js';
+import type { Evidence, FetchResult, FetchWarning, HandoffRequest, LadderRunAudit, TraceEvent } from './result.js';
+import type { EvidenceRecord } from './evidenceRecord.js';
+import type { Lane } from './status.js';
+import type { BudgetKind } from './status.js';
+import type { ExecutionContext } from './execution.js';
+/**
+ * How a crawl uses the site's sitemap: beside the links it finds (`include`,
+ * the default), not at all (`skip`), or as its only source of URLs beside the
+ * start URL (`only`: page links are returned when asked for, never followed).
+ */
+export declare const SITEMAP_MODES: readonly ["include", "skip", "only"];
+export type SitemapMode = (typeof SITEMAP_MODES)[number];
+export interface ScrapeOutcome {
+    result: FetchResult;
+    links: readonly string[];
+    audit?: LadderRunAudit;
+    crawlDelayMs?: number | null;
+    /**
+     * True when the result is a stored one the cache answered with: nothing
+     * was fetched, so the page costs no fetch and says nothing new about its
+     * host's robots.txt (its Crawl-delay is left as it was).
+     */
+    cached?: boolean;
+}
+/**
+ * One URL in, one page outcome out. The ladder is the production
+ * implementation; Phase 4 tests inject a fake.
+ */
+export interface ScrapeAtom {
+    scrape(url: string, context?: ExecutionContext): Promise<ScrapeOutcome>;
+    close(): Promise<void>;
+}
+export interface SitemapLoadRequest {
+    seedUrl: string;
+    /** Stop collecting entries once this many are in hand: the crawl's maxPages, or 50 000 when it is unbounded. */
+    maxUrls: number;
+    /** At most this many sitemap files are fetched for one load, an index and its children each counting as one. */
+    maxFiles: number;
+    /**
+     * Opt-in (a map passes it, a crawl does not): judges each entry before it is
+     * collected. An entry it refuses is not collected and does not count toward
+     * `maxUrls`. Once `maxUrls` entries are in hand, the rest of the file in
+     * flight is still offered, and those it accepts are counted as left over
+     * (`truncated: 'urls'`), never collected.
+     */
+    accept?: (entry: SitemapEntry) => boolean | Promise<boolean>;
+    /**
+     * Opt-in (a map passes it, a crawl does not): an absolute UTC time after
+     * which the load stops and returns what it has, `truncated: 'time'`. The
+     * file in flight is aborted and recorded as `unreadable`, error `timeout`.
+     * The caller's own cancellation still throws.
+     */
+    softDeadlineAt?: number;
+}
+/** Where a load looked for sitemaps: the `Sitemap:` lines of the start URL's robots.txt, or the conventional `/sitemap.xml` when it lists none. */
+export type SitemapSourceKind = 'robots' | 'guess';
+/**
+ * What one sitemap file turned out to be: a `<sitemapindex>`, a `<urlset>`, a
+ * 4xx (`absent`), a 2xx body that is neither (`not_sitemap`), a file that
+ * could not be read (`unreadable`: too large, over the decompression cap, a
+ * Content-Encoding W2L does not decode or bytes that do not decode as theirs,
+ * a 5xx, a transport failure; `error` says which) or one its host's robots.txt
+ * disallows for the crawl's identity (`refused`, never requested).
+ */
+export type SitemapFileKind = 'index' | 'urlset' | 'absent' | 'not_sitemap' | 'unreadable' | 'refused';
+/** One sitemap file a load fetched or refused, on the record of the crawl that used it. These fetches carry no signed compliance record. */
+export interface SitemapFileRecord {
+    url: string;
+    /** Where the file was read from after redirects; null when no response answered. */
+    finalUrl: string | null;
+    status: number | null;
+    contentType: string | null;
+    /** Bytes on the wire, before any gzip inflation; null when no body was read. */
+    bytes: number | null;
+    /** SHA-256 of the bytes as received; null when no body was read. */
+    sha256: string | null;
+    kind: SitemapFileKind;
+    /** `<loc>` entries the file holds (child sitemaps for an index), http(s) ones only; null when the file was not parsed. */
+    entries: number | null;
+    /** The robots.txt verdict for the file's own URL under the crawl's identity (an unreachable robots.txt is `disallowed`, as for a page); null when the URL failed its egress check before robots.txt was consulted. */
+    robots: 'allowed' | 'disallowed' | 'no_robots' | null;
+    /** Whether the request left through the operator's environment proxy (local mode); a hosted server never has one. */
+    proxyUsed: boolean;
+    error: string | null;
+}
+/** A URL a sitemap listed and the file that listed it. */
+export interface SitemapEntry {
+    url: string;
+    file: string;
+    /** The entry's `<lastmod>`, as written (trimmed, entities decoded, not normalised); absent when it has none. A crawl ignores it. */
+    lastmod?: string;
+    /** The entry's `<news:title>`, entity-decoded and trimmed; absent when it has none. A crawl ignores it. */
+    title?: string;
+}
+export interface SitemapLoadResult {
+    /** The declared identity the files were requested with: the crawl mode's http identity. */
+    identity: {
+        mode: CrawlMode;
+        userAgent: string;
+    };
+    sources: SitemapSourceKind[];
+    files: SitemapFileRecord[];
+    /** The entries collected, in listed order, each once, up to `maxUrls`. */
+    urls: SitemapEntry[];
+    /** Set when the load stopped before reading everything: at `maxFiles` with files unread, at `maxUrls` with entries uncollected, or at `softDeadlineAt` (`time`, never on a crawl). */
+    truncated: 'files' | 'urls' | 'time' | null;
+    /** Only with `truncated: 'time'`: the files located (declared, or children of an index read) and not requested, the aborted one excluded; absent when the deadline came before any was located. */
+    unreadFiles?: number;
+}
+/**
+ * Reads a site's sitemaps for a crawl: an auxiliary fetch path beside
+ * robots.txt, never a page fetcher. The bench's HttpSitemapSource is the
+ * production implementation; tests inject a fake. One source serves one crawl
+ * and is closed with it.
+ */
+export interface SitemapSource {
+    load(request: SitemapLoadRequest, context?: ExecutionContext): Promise<SitemapLoadResult>;
+    close(): Promise<void>;
+}
+/** What a crawl attempt's sitemap load found and what the frontier made of it. Null when the crawl read no sitemap. */
+export interface SitemapDiscovery {
+    mode: SitemapMode;
+    sources: SitemapSourceKind[];
+    files: SitemapFileRecord[];
+    /** Entries the load returned and offered to the frontier. */
+    listed: number;
+    /** Entries the frontier accepted as pages to fetch. */
+    enqueued: number;
+    /** As SitemapLoadResult says; a crawl never produces `time`. */
+    truncated: 'files' | 'urls' | 'time' | null;
+    /** Why the load itself failed, when it threw before returning; `files` then holds what it had read. Null otherwise. */
+    error: string | null;
+}
+/**
+ * What one run of the orchestrator crawls. A new task stores its budget,
+ * maxDepth, allowlistedDomains, includePaths, excludePaths, URL-scope options,
+ * sitemap mode and concurrency cap; a resumed (`resumeFrom`) or existing
+ * (`taskId`) task runs with the ones it stored, so a resume never widens the
+ * crawl it continues.
+ */
+export interface CrawlSpec {
+    seedUrl: string;
+    seedUrls?: readonly string[];
+    taskDir: string;
+    mode: CrawlMode;
+    budget: CrawlBudget;
+    maxDepth: number | null;
+    /** Hosts links may lead to beside the seed's host, its www twin and where the seed redirected. */
+    allowlistedDomains: readonly string[];
+    resumeFrom: string | null;
+    useCached: boolean;
+    /** When set, openRun updates this existing task instead of inserting a new id. */
+    taskId?: string;
+    /**
+     * Pathname regexes for discovered links (the seed is always fetched); a
+     * match in excludePaths wins.
+     */
+    includePaths?: readonly string[];
+    excludePaths?: readonly string[];
+    /** The URL-scope options, as CrawlStartRequest names them; DEFAULT_CRAWL_SPEC has their defaults. */
+    regexOnFullURL?: boolean;
+    ignoreQueryParameters?: boolean;
+    deduplicateSimilarURLs?: boolean;
+    crawlEntireDomain?: boolean;
+    allowSubdomains?: boolean;
+    allowExternalLinks?: boolean;
+    /** How the crawl uses the site's sitemap; `skip` when the run has no SitemapSource. */
+    sitemap?: SitemapMode;
+    /** Pages this crawl fetches at once, at most; null takes the service's worker count. Never raises the per-host ceiling. */
+    maxConcurrency?: number | null;
+}
+/**
+ * What a crawl attempt's pages offered the frontier and what became of each
+ * link: enqueued, an exact repeat (`duplicate`), a variant folded into a
+ * first-seen page (`collapsed`: a query the crawl ignores, `/a/` after `/a`,
+ * `/index.html` after `/`, the www twin), or refused by the host scope, the
+ * start URL's path subtree, includePaths / excludePaths or maxDepth. The
+ * `offered` total also counts links no counter names (assets, non-http
+ * schemes, a path a filter could not decide). Sitemap entries the crawl
+ * offered count here too, and `sitemap` says what the load read. `duplicateContent`
+ * counts pages fetched and then found to repeat an earlier page's body. Null
+ * on a batch, which discovers nothing.
+ */
+export interface CrawlDiscovery {
+    offered: number;
+    enqueued: number;
+    duplicate: number;
+    collapsed: number;
+    hostDenied: number;
+    subtreeDenied: number;
+    pathDenied: number;
+    depthDenied: number;
+    duplicateContent: number;
+    /** The attempt's sitemap load; null when the crawl read no sitemap (`sitemap: skip`, or a task stored before the option). */
+    sitemap: SitemapDiscovery | null;
+}
+export declare const EMPTY_CRAWL_DISCOVERY: CrawlDiscovery;
+export interface CrawlReport {
+    taskId: string;
+    attemptId: string;
+    status: TaskStatus;
+    pagesFetched: number;
+    cachedPages: number;
+    budgetExceeded: BudgetKind | null;
+    loopDetected: boolean;
+    wallMs: number;
+    costUsd: number | null;
+    costUnknown?: boolean;
+    contentTokens: number | null;
+    contentTokensUnknown?: boolean;
+    /** The latest attempt's link discovery counters; null for a batch and for a crawl stored before they were kept. */
+    discovery: CrawlDiscovery | null;
+    /** Who started the task, as the request said (`origin`, `integration`); absent when it named neither. */
+    attribution?: RequestAttribution;
+    /** The task's webhook and how its deliveries stand; absent when the request set none. */
+    webhook?: JobWebhookStatus;
+}
+export interface CrawlPage {
+    id: string;
+    url: string;
+    canonicalUrl: string;
+    depth: number;
+    status: StepStatus;
+    lane: Lane | null;
+    markdown: string | null;
+    /** The fetch's caveats (a recorded robots override), as on a scrape result; absent when it had none. */
+    warnings?: readonly FetchWarning[];
+    /** The warnings' messages joined with a space, present exactly when `warnings` is, as on a scrape response. */
+    warning?: string;
+    /** What to change about the request next time (a login wall, a robots.txt rule, a cut), as on a scrape response; absent when nothing applies. */
+    agentHints?: AgentHints;
+    /**
+     * A batch item stopped at a check W2L does not pass (a captcha, a
+     * challenge, a login wall), on a server that can hand it to a person in
+     * their own Chrome (`POST /v1/batches/:id/handoff`): why, and how. Absent
+     * otherwise.
+     */
+    handoff?: HandoffRequest;
+    /** Present when the task asked for the `html` format, as on a scrape result; null when the page has none. */
+    html?: string | null;
+    /** Present when the task asked for the `rawHtml` format, as on a scrape result; null when the page has none. */
+    rawHtml?: string | null;
+    /** Present when the task asked for the `images` format and the page was read as content, as on a scrape result. */
+    images?: readonly string[];
+    /** Present when the task asked for the `tables` format and the page was read as content, as on a scrape result. */
+    tables?: FetchResult['tables'];
+    /** Present when the task's `pdf` parser asked for `pages` and the page is a PDF whose text was read, as on a scrape result. */
+    pages?: FetchResult['pages'];
+    /** Present when the task asked for an `attributes` format and the page was read as content, as on a scrape result. */
+    attributes?: FetchResult['attributes'];
+    /** Present when the task asked for the `screenshot` format and the page rendered, as on a scrape result: the capture, or null when the browser lane could not capture it. */
+    screenshot?: FetchResult['screenshot'];
+    /** Present when the batch ran `actions` on the page: what the steps produced, and the step that failed if one did. */
+    actions?: FetchResult['actions'];
+    /** Present when the task asked for a `list` entry and the page was read: its records. */
+    list?: FetchResult['list'];
+    /** Absolute outbound links; present when the task requested links. */
+    links?: readonly string[];
+    /** The page's own title, description, language, ... as on a scrape result; absent when no page was extracted. */
+    metadata?: FetchResult['metadata'];
+    json?: import('./structured.js').StructuredExtractionResult | null;
+    /** The file the page was (PDF, CSV, ...), as on a scrape result; absent for a web page. */
+    file?: FetchResult['file'];
+    failureReason: string | null;
+    blockReason: string | null;
+    budgetExceeded: BudgetKind | null;
+    evidence: Evidence | null;
+    /** The page's Evidence Record v1; null while the page has no result yet. */
+    evidenceRecord: EvidenceRecord | null;
+    /** Per-URL latency, attempts and metering without loading the full audit. */
+    usage?: import('./result.js').ResourceUsage | null;
+    trace: readonly TraceEvent[];
+    audit?: LadderRunAudit;
+    /** True when the page was not fetched in this attempt: a stored result was reused (the cache, or a resume's own pages). */
+    cached: boolean;
+    /** Whether the cache answered, as on a scrape response's `metadata`: present when the task looked pages up (`maxAge`, `minAge`, `lockdown`). */
+    cacheState?: 'hit' | 'miss';
+    /** On a hit, when the reused result was fetched (its `evidenceRecord.fetchedAt`). */
+    cachedAt?: string;
+    contentHash: string | null;
+    createdAt: string;
+    updatedAt: string;
+}
+export interface CrawlError extends CrawlPage {
+    trace: readonly TraceEvent[];
+}
+export interface CrawlPageList<T> {
+    items: readonly T[];
+    nextCursor: string | null;
+    hasMore: boolean;
+}
+export declare const DEFAULT_CRAWL_SPEC: Omit<CrawlSpec, 'seedUrl' | 'taskDir'>;
+//# sourceMappingURL=crawl.d.ts.map

package/dist/types/esm/contracts/delivery.d.ts

@@ -0,0 +1,136 @@
+import type { BatchStatusResponse, WebhookEvent } from './api.js';
+import type { CrawlPage, CrawlReport } from './crawl.js';
+import type { FirecrawlWebhookPayload } from './firecrawl.js';
+import type { MonitorEvent, MonitorSnapshot } from './monitor.js';
+/** Whose events a destination receives: a Monitor's, or a job's (a crawl or batch, destination `job:<taskId>`). */
+export type DeliveryDestinationKind = 'monitor' | 'job';
+/** The shape of a job destination's payloads: W2L's JobWebhookEnvelope, or Firecrawl's for a job started through the `/fc` shim. */
+export type WebhookPayloadFormat = 'w2l' | 'firecrawl';
+export interface DeliveryDestinationInput {
+    id: string;
+    /** The Monitor the destination belongs to; for a job destination, `job:<taskId>`. */
+    monitorId: string;
+    url: string;
+    maxAttempts?: number;
+    /** Operator environment reference; literal secrets are never stored or returned. */
+    secretEnv?: string;
+    enabled?: boolean;
+    /** Default `monitor`. A `job` destination takes the four options below; a Monitor destination takes none of them yet. */
+    kind?: DeliveryDestinationKind;
+    /** The job events to deliver; default all. */
+    events?: readonly WebhookEvent[];
+    /** Headers sent with every delivery (lower-cased names); stored in the control database alone and never returned. */
+    headers?: Readonly<Record<string, string>>;
+    /** Strings copied into every payload's `metadata`. */
+    metadata?: Readonly<Record<string, string>>;
+    payloadFormat?: WebhookPayloadFormat;
+}
+export interface DeliveryDestination {
+    id: string;
+    monitorId: string;
+    url: string;
+    maxAttempts: number;
+    secretEnv?: string;
+    enabled: boolean;
+    createdAt: number;
+    kind: DeliveryDestinationKind;
+    /** The job the destination belongs to (`kind: 'job'`). */
+    jobId?: string;
+    events?: readonly WebhookEvent[];
+    /** The names of the custom headers sent with each delivery; their values are never returned. */
+    headerNames: string[];
+    metadata?: Readonly<Record<string, string>>;
+    payloadFormat?: WebhookPayloadFormat;
+}
+/** eventVersion is the committed snapshot version, scoped to this monitor/entity/view. */
+export interface WebhookEventEnvelope {
+    schemaVersion: 'w2l.monitor-event/v1';
+    eventId: string;
+    eventVersion: number;
+    monitorId: string;
+    workspaceId: string;
+    entityKey: string;
+    viewKey: string;
+    event: MonitorEvent;
+    snapshot: MonitorSnapshot;
+}
+/**
+ * One event of a crawl or batch, as its webhook receiver gets it. `eventId`
+ * is `<taskId>:started`, `<taskId>:page:<stepId>` or `<taskId>:<status>`
+ * (a job run again, a batch appended after completion, suffixes its later
+ * terminal ids with the attempt id); `sequence` is 0 for `started`, then
+ * one more for every page and terminal event in the order they were
+ * enqueued, so a job of n pages ends at n+1. Also the `x-w2l-event-id` and
+ * `x-w2l-event-version` headers of the delivery.
+ */
+export interface JobWebhookEnvelope {
+    schemaVersion: 'w2l.job-event/v1';
+    eventId: string;
+    sequence: number;
+    jobId: string;
+    jobKind: 'crawl' | 'batch';
+    event: WebhookEvent;
+    /** When the event was recorded, ISO 8601. */
+    at: string;
+    /** The request's `webhook.metadata`, `{}` when none. */
+    metadata: Readonly<Record<string, string>>;
+    /** On a `page` event: the page as `GET /v1/crawl/:id/pages` or `/v1/batches/:id/items` lists it (no audit, empty trace). */
+    page?: CrawlPage;
+    /** On a terminal event: the job's status as `GET /v1/crawl/:id` or `GET /v1/batches/:id` reports it then. */
+    report?: CrawlReport | BatchStatusResponse;
+    /** On `failed`: why. */
+    error?: string;
+}
+/** What a delivery carries: a Monitor event, a job event, or a job event in Firecrawl's shape. */
+export type WebhookPayload = WebhookEventEnvelope | JobWebhookEnvelope | FirecrawlWebhookPayload;
+export type DeliveryState = 'pending' | 'delivering' | 'delivered' | 'dead_letter';
+export interface WebhookDelivery {
+    id: string;
+    destinationId: string;
+    monitorId: string;
+    eventId: string;
+    eventVersion: number;
+    state: DeliveryState;
+    attemptCount: number;
+    maxAttempts: number;
+    nextAttemptAt: number;
+    leaseUntil: number | null;
+    fencingToken: number;
+    createdAt: number;
+    deliveredAt: number | null;
+    lastStatus: number | null;
+    lastError: string | null;
+    payload: WebhookPayload;
+}
+export interface DeliveryAttempt {
+    id: string;
+    deliveryId: string;
+    fencingToken: number;
+    startedAt: number;
+    endedAt: number | null;
+    outcome: 'sending' | 'delivered' | 'retry' | 'dead_letter' | 'lease_expired';
+    status: number | null;
+    error: string | null;
+    retryAfterAt: number | null;
+}
+export interface DeliveryQuery {
+    monitorId?: string;
+    /** The crawl or batch whose deliveries to list: `monitorId` `job:<taskId>` under its own name; not beside `monitorId`. */
+    jobId?: string;
+    destinationId?: string;
+    state?: DeliveryState;
+}
+export interface DeliveryPageQuery extends DeliveryQuery {
+    cursor?: string;
+    limit?: number;
+}
+export interface DeliveryPage {
+    items: WebhookDelivery[];
+    nextCursor: string | null;
+    hasMore: boolean;
+}
+export interface DeliveryDetail {
+    delivery: WebhookDelivery;
+    attempts: DeliveryAttempt[];
+}
+//# sourceMappingURL=delivery.d.ts.map

package/dist/types/esm/contracts/evidenceRecord.d.ts

@@ -0,0 +1,192 @@
+import type { PageActionType } from './actions.js';
+/**
+ * Evidence Record v1: the evidence every result carries, stated the same way
+ * whichever lane produced it. The published JSON Schema is
+ * `schemas/evidence-record.v1.json` in this package; the key lists below and
+ * `test/evidenceRecord.test.ts` keep the file and this type in step. Types
+ * only: the builder (`toEvidenceRecord`) lives in @w2l/runtime.
+ *
+ * Unknown is null, never zero or a guess. Every field is always present.
+ *
+ * Versioning: `w2l.evidence/1` changes only additively. A later revision of
+ * the schema file accepts every record an earlier one accepted: a field added
+ * later is optional in the schema (EVIDENCE_RECORD_ADDED_KEYS), though W2L
+ * writes it on every record from then on; enums only gain values; no field
+ * changes its meaning. A change that cannot follow this rule gets a new
+ * schemaVersion (`w2l.evidence/2`) and a new schema file.
+ */
+import type { CrawlMode, RobotsUnreachable } from './compliance.js';
+import type { BlockReason, BudgetKind, FailureReason, Lane, ResultStatus } from './status.js';
+export declare const EVIDENCE_SCHEMA_VERSION = "w2l.evidence/1";
+/**
+ * Where a JSON field's value was read. `fetch` is the fetch's own value (its
+ * final or requested URL), not read from the page. `pdf`: a `Label: value`
+ * line of a PDF's text, whose locator names the page and the label.
+ */
+export declare const FIELD_EVIDENCE_SOURCES: readonly ["jsonld", "microdata", "meta", "hydration", "dom", "text", "inferred", "model", "fetch", "pdf"];
+export type FieldEvidenceSource = (typeof FIELD_EVIDENCE_SOURCES)[number];
+/**
+ * `snapshot` is the page's HTML as W2L read it (`W2L_CAPTURE_RAW_DIR`);
+ * `file` is a file (PDF, CSV, ...) saved as received; `screenshot` is the
+ * browser lane's capture for the `screenshot` format, saved under
+ * `W2L_CAPTURE_RAW_DIR` as `<sha256>.png` or `.jpg`, with its size and type.
+ */
+export declare const EVIDENCE_ARTIFACT_KINDS: readonly ["snapshot", "screenshot", "file"];
+export type EvidenceArtifactKind = (typeof EVIDENCE_ARTIFACT_KINDS)[number];
+export interface EvidenceRedirectChain {
+    /**
+     * Every URL W2L requested for the page, in order: the requested URL first,
+     * the final URL last, with each redirect between (in the browser lane, a
+     * document a script or a meta refresh loaded is one). `[requestedUrl]` when
+     * there was no redirect; empty when W2L sent no request for the page.
+     */
+    urls: readonly string[];
+    /**
+     * True when every hop is listed. The HTTP lane follows redirects itself and
+     * lists each; the browser lane lists each redirect Chromium followed and
+     * each document a script or a meta refresh loaded, and is false only when a
+     * follow-up navigation did not start at the requested URL, a document came
+     * without a request, or the chain of a page that kept moving on was cut to
+     * its first URL and last 20. The provider lane sees only where its vendor
+     * started and ended, so its chain is `[requested, final]` and this is false.
+     */
+    complete: boolean;
+}
+export interface EvidenceRobotsDecision {
+    /** `no_robots`: robots.txt was requested and the site has none (a 4xx, or a body that is not text/plain). */
+    decision: 'allowed' | 'disallowed' | 'no_robots';
+    robotsUrl: string | null;
+    /** SHA-256 of the robots.txt bytes parsed; null when none was parsed. */
+    robotsSha256: string | null;
+    /** Why robots.txt could not be fetched (then `decision` is `disallowed`, RFC 9309 §2.3.1.4); null when it was. */
+    unreachable: RobotsUnreachable | null;
+    crawlDelayMs: number | null;
+    /** Whether the fetch went ahead under a recorded robots override although `decision` is `disallowed`; the reason is in the trace, the warnings and, in the browser lane, the compliance record. */
+    userOverride: boolean;
+}
+export interface EvidenceOutputSha256 {
+    /** SHA-256 of the UTF-8 bytes of the delivered `markdown`; null when none was delivered. */
+    markdown: string | null;
+    /** SHA-256 of the canonical JSON of the delivered `json.data`; null when JSON was not requested or has no data. */
+    json: string | null;
+}
+export interface EvidenceExtractor {
+    name: string;
+    /** EXTRACTOR_VERSION of @w2l/extract-tf for a web page; PDF_TEXT_VERSION for a PDF; FILE_TEXT_VERSION for another file. */
+    version: string;
+    /** The source commit of the running W2L (`W2L_SOURCE_COMMIT`); null when not declared. */
+    commit: string | null;
+}
+export interface EvidenceFieldLocation {
+    source: FieldEvidenceSource;
+    /** JSON-LD path, DOM selector, `table[i] tr[j] "label"`, `h1[0]`, a result field such as `finalUrl`, or `page N "label"` for a PDF; null when the source gives none. */
+    locator: string | null;
+}
+export interface EvidenceArtifact {
+    /** Null when W2L cannot tell what the file is. */
+    kind: EvidenceArtifactKind | null;
+    path: string;
+    sha256: string | null;
+    /** Size of the saved file in bytes; null when unknown. Added to v1 for files (EVIDENCE_RECORD_ADDED_KEYS). */
+    bytes: number | null;
+    /** The Content-Type the file was received with; null when unknown or none was sent. Added to v1 for files. */
+    contentType: string | null;
+}
+export interface EvidenceIdentity {
+    /** The User-Agent observed on the wire; null when the lane did not observe it. */
+    userAgent: string | null;
+    mode: CrawlMode;
+    /** The contact the User-Agent declares (research mode with `W2L_CONTACT`); null when none. */
+    contact: string | null;
+    /**
+     * The device the answering lane's identity declared (`mobile` for a
+     * request with `mobile: true`); null when W2L sent no request for the page,
+     * in mode `research` (a bot declares no device) or when the lane recorded
+     * none (the provider lane). Added to v1 (EVIDENCE_RECORD_ADDED_KEYS).
+     */
+    device: 'desktop' | 'mobile' | null;
+    /**
+     * The caller's custom headers (`headers`) the answering lane sent, names
+     * lower-cased and sorted, each with the SHA-256 of its value, never the
+     * value: a value may be a key the caller would not publish with the data,
+     * as the scrape record keeps names only. Empty when it sent none; null when
+     * W2L sent no request for the page. Added to v1 (EVIDENCE_RECORD_ADDED_KEYS).
+     */
+    requestHeaders: readonly EvidenceRequestHeader[] | null;
+}
+/** One custom request header as sent: its name and the SHA-256 (hex) of its value's UTF-8 bytes. */
+export interface EvidenceRequestHeader {
+    name: string;
+    valueSha256: string;
+}
+/** The steps a request ran on the page before it was read. */
+export interface EvidencePageActions {
+    steps: readonly EvidencePageActionStep[];
+    /** Whether an `executeJavascript` step ran in the page. */
+    scriptRan: boolean;
+}
+export interface EvidencePageActionStep {
+    type: PageActionType;
+    outcome: 'ok' | 'failed';
+}
+export interface EvidenceRecord {
+    schemaVersion: typeof EVIDENCE_SCHEMA_VERSION;
+    requestedUrl: string;
+    /** The last URL W2L requested for the page; null when it sent none (robots.txt disallow, DNS failure, policy). */
+    finalUrl: string | null;
+    redirectChain: EvidenceRedirectChain;
+    /** UTC ISO 8601 time the response W2L reports was received; null when there was none. */
+    fetchedAt: string | null;
+    httpStatus: number | null;
+    status: ResultStatus;
+    /** The failure, block or budget reason; null for other statuses. */
+    reason: FailureReason | BlockReason | BudgetKind | null;
+    lane: Lane;
+    /** Null when no robots.txt decision was made for this result. */
+    robotsDecision: EvidenceRobotsDecision | null;
+    /** `evidence.rawBodySha256`: the body W2L read (see the schema for what each lane reads). */
+    rawSha256: string | null;
+    /**
+     * The Content-Encoding the body was read from, as the HTTP lane received it
+     * (lower-cased codings in applied order, `x-gzip` read as `gzip`), or
+     * `identity` when it had none: what was decoded before `rawSha256` was
+     * taken. Null when no body was read, and in the browser and provider lanes.
+     * Added to v1 later (EVIDENCE_RECORD_ADDED_KEYS).
+     */
+    contentEncoding: string | null;
+    outputSha256: EvidenceOutputSha256;
+    extractor: EvidenceExtractor;
+    /** JSON Pointer into `json.data` → where that value was read; null when JSON was not requested. */
+    fieldEvidence: Readonly<Record<string, EvidenceFieldLocation>> | null;
+    artifacts: readonly EvidenceArtifact[];
+    /** `host:port` of the operator's environment proxy the request went through; null when it went direct or the lane does not report its route. */
+    proxy: string | null;
+    identity: EvidenceIdentity;
+    /**
+     * The request's `actions` that ran on the page before it was read, in order,
+     * with their outcome; null when the request had none. The hashes above are
+     * of the page as the steps left it, and `scriptRan` says when a script of
+     * the caller's ran in it, so its content may be the script's.
+     */
+    pageActions: EvidencePageActions | null;
+}
+/** Field order of the record and of each nested object, as in the schema file. */
+export declare const EVIDENCE_RECORD_KEYS: {
+    readonly record: readonly ["schemaVersion", "requestedUrl", "finalUrl", "redirectChain", "fetchedAt", "httpStatus", "status", "reason", "lane", "robotsDecision", "rawSha256", "contentEncoding", "outputSha256", "extractor", "fieldEvidence", "artifacts", "proxy", "identity", "pageActions"];
+    readonly redirectChain: readonly ["urls", "complete"];
+    readonly robotsDecision: readonly ["decision", "robotsUrl", "robotsSha256", "unreachable", "crawlDelayMs", "userOverride"];
+    readonly outputSha256: readonly ["markdown", "json"];
+    readonly extractor: readonly ["name", "version", "commit"];
+    readonly fieldEvidence: readonly ["source", "locator"];
+    readonly artifact: readonly ["kind", "path", "sha256", "bytes", "contentType"];
+    readonly identity: readonly ["userAgent", "mode", "contact", "device", "requestHeaders"];
+    readonly pageActions: readonly ["steps", "scriptRan"];
+    readonly pageActionStep: readonly ["type", "outcome"];
+    readonly requestHeader: readonly ["name", "valueSha256"];
+};
+/**
+ * Keys added to v1 after it was first published: optional in the schema, so
+ * that records written before them stay valid, though W2L always writes them.
+ */
+export declare const EVIDENCE_RECORD_ADDED_KEYS: Partial<Record<keyof typeof EVIDENCE_RECORD_KEYS, readonly string[]>>;
+//# sourceMappingURL=evidenceRecord.d.ts.map