@bluefields/fetcher 0.2.0

package/dist/fetcher-robots-gate.js

@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Robots-gated fetcher — the central robots.txt choke point (see ROBOTS_POLICY.md).
+ *
+ * Wraps any FetcherAdapter so every route that fetches through it gets uniform
+ * robots handling (instead of the per-path drift the audit found). When
+ * robots-respect is enabled it:
+ *   - refuses URLs the target's robots.txt disallows for our product token →
+ *     throws `RobotsDisallowedError` (routes map it to 403 `robots_disallowed`);
+ *   - paces same-host requests by `max(Crawl-delay, politeness floor)`.
+ *
+ * Authorized opt-out: when the request carries the internal
+ * `__robotsBypassAuthorized` flag (resolved UPSTREAM at the API layer from the
+ * customer's `ignoreRobots` + `robotsAttestation` AND an account capability), the
+ * Disallow check is skipped and the bypass is logged for audit. The gate never
+ * decides authorization itself — that's an API/control concern (four-layer rule).
+ *
+ * Phase-in: respect is OFF by default; enable per-environment via
+ * `ROBOTS_RESPECT=1` or the explicit `respectRobots` option. When off, the gate
+ * is a transparent pass-through (zero behavior change). robots.txt evaluation is
+ * fail-open (a target's robots outage never blocks the fetch) — see
+ * `evaluateRobots`. robots.txt URLs are SSRF-validated inside `evaluateRobots`;
+ * this gate adds no new trust-boundary crossing.
+ */
+import { RobotsDisallowedError, createLayerLogger, evaluateRobots } from '@bluefields/primitives';
+const log = createLayerLogger('fetcher');
+/** The product token we match robots.txt against — NOT the (possibly spoofed) fetch UA. */
+const ROBOTS_USER_AGENT = 'BlueFields-Fetcher';
+/** Minimum spacing between requests to the same host, even absent a Crawl-delay. */
+const DEFAULT_POLITENESS_FLOOR_MS = 1_000;
+/**
+ * Build a robots-gated fetcher wrapping `opts.inner`. Drop-in `FetcherAdapter`,
+ * so adopting it is just wrapping the fetcher a route already constructs.
+ */
+export function createRobotsGatedFetcher(opts) {
+    const env = opts.env ?? process.env;
+    const respectRobots = opts.respectRobots ?? env.ROBOTS_RESPECT === '1';
+    const floorMs = opts.politenessFloorMs ?? DEFAULT_POLITENESS_FLOOR_MS;
+    const evaluate = opts.evaluate ?? evaluateRobots;
+    const now = opts.nowMsFn ?? Date.now;
+    const sleep = opts.sleepFn ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    /** Per-instance last-request time per host, for politeness pacing. */
+    const lastFetchByHost = new Map();
+    return {
+        async fetch(url, options) {
+            // Phase-in off → transparent pass-through (current behavior, no robots fetch).
+            if (!respectRobots)
+                return opts.inner.fetch(url, options);
+            const host = hostOf(url);
+            // Authorized opt-out: skip the Disallow check, log for audit. Resolved
+            // upstream — we only honor a resolved boolean here.
+            if (options?.__robotsBypassAuthorized === true) {
+                const audit = { url, host: host ?? '', atMs: now() };
+                opts.onBypass?.(audit);
+                log.info({ outcome: 'ok', url, host, robots_bypass: true }, 'robots opt-out honored (authorized)');
+                return opts.inner.fetch(url, options);
+            }
+            const { allowed, crawlDelaySeconds } = await evaluate(url, ROBOTS_USER_AGENT, {
+                ...(opts.robotsFetchImpl ? { fetchImpl: opts.robotsFetchImpl } : {}),
+            });
+            if (!allowed) {
+                log.info({ outcome: 'blocked', url, host, reason: 'robots_disallowed' }, 'robots.txt disallows fetch');
+                throw new RobotsDisallowedError(url);
+            }
+            // Politeness: space same-host requests by max(Crawl-delay, floor). For a
+            // one-shot scrape the host is unseen → no wait; binds on crawl/batch.
+            if (host) {
+                const delayMs = Math.max(floorMs, (crawlDelaySeconds ?? 0) * 1_000);
+                const last = lastFetchByHost.get(host);
+                const wait = last === undefined ? 0 : delayMs - (now() - last);
+                if (wait > 0)
+                    await sleep(wait);
+                lastFetchByHost.set(host, now());
+            }
+            return opts.inner.fetch(url, options);
+        },
+    };
+}
+function hostOf(url) {
+    try {
+        return new URL(url).host.toLowerCase();
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=fetcher-robots-gate.js.map

package/dist/fetcher-robots-gate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher-robots-gate.js","sourceRoot":"","sources":["../src/fetcher-robots-gate.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAKlG,MAAM,GAAG,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEzC,2FAA2F;AAC3F,MAAM,iBAAiB,GAAG,oBAAoB,CAAC;AAC/C,oFAAoF;AACpF,MAAM,2BAA2B,GAAG,KAAK,CAAC;AAmC1C;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAuB;IAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACpC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,GAAG,CAAC,cAAc,KAAK,GAAG,CAAC;IACvE,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,IAAI,2BAA2B,CAAC;IACtE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,cAAc,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC;IACrC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5F,sEAAsE;IACtE,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;IAElD,OAAO;QACL,KAAK,CAAC,KAAK,CAAC,GAAW,EAAE,OAAsB;YAC7C,+EAA+E;YAC/E,IAAI,CAAC,aAAa;gBAAE,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAE1D,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAEzB,uEAAuE;YACvE,oDAAoD;YACpD,IAAK,OAA4C,EAAE,wBAAwB,KAAK,IAAI,EAAE,CAAC;gBACrF,MAAM,KAAK,GAAsB,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;gBACxE,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC;gBACvB,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,EACjD,qCAAqC,CACtC,CAAC;gBACF,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACxC,CAAC;YAED,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,iBAAiB,EAAE;gBAC5E,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrE,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,EAAE,EAC9D,4BAA4B,CAC7B,CAAC;gBACF,MAAM,IAAI,qBAAqB,CAAC,GAAG,CAAC,CAAC;YACvC,CAAC;YAED,yEAAyE;YACzE,sEAAsE;YACtE,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,iBAAiB,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;gBACpE,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvC,MAAM,IAAI,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;gBAC/D,IAAI,IAAI,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;gBAChC,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;YACnC,CAAC;YAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,GAAW;IACzB,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,93 @@
+/**
+ * @bluefields/fetcher — public surface (chunks 8 + 9).
+ *
+ * Per the four-layer rule:
+ *   - Fetcher returns raw HTML + metadata.
+ *   - It does NOT extract, diff, or notify.
+ *
+ * Production wiring:
+ *   - Fetcher: HTTP (default) → Patchright (when IPROYAL_* env set + dep
+ *     installed).
+ *   - Storage: in-memory (default) → R2 (when R2_* env set).
+ *
+ * `fetchAndStore` orchestrates one end-to-end fetch: robots-check →
+ * fetch via adapter → store content-addressed → sign the attestation
+ * manifest (when a signer is configured — A3, Review #3) → insert
+ * snapshot row → write cost_records.
+ *
+ * The return is a discriminated outcome — chunk 21's poll handler
+ * switches on `outcome.ok` to decide whether to feed the extractor or
+ * record a blocked-poll event.
+ */
+import { type Signer } from '@bluefields/attest';
+import { type DbClient } from '@bluefields/db';
+import type { FetchOptions, FetchResult, FetcherAdapter, StorageAdapter } from './types.js';
+export { type FetchOptions, type FetchResult, type FetcherAdapter, type PageAction, type RequestCookie, type StorageAdapter, type StoredSnapshot, type WaitCondition, buildR2Key, } from './types.js';
+export { createHttpFetcherAdapter } from './fetcher-http.js';
+export { PATCHRIGHT_REQUIRED_ENV, tryCreatePatchrightAdapter } from './fetcher-patchright.js';
+export { type EscalatingFetcherOptions, type FetchOutcome, classifyFetchResult, createEscalatingFetcherAdapter, } from './fetcher-escalating.js';
+export { type RobotsBypassAudit, type RobotsGateOptions, createRobotsGatedFetcher, } from './fetcher-robots-gate.js';
+export { createInMemoryStorageAdapter } from './storage-memory.js';
+export { R2_REQUIRED_ENV, type R2Config, createR2StorageAdapter, tryCreateR2StorageAdapter, } from './storage-r2.js';
+export { COMPUTE_BLOCKED_COST_MICRO_USD, type FetchCostType, type RecordFetchCostArgs, TIER1_FIXED_COST_MICRO_USD, recordBlockedFetch, recordFetchCost, } from './cost.js';
+/**
+ * Default fetcher factory — Patchright when env-configured, otherwise
+ * the plain HTTP fetcher.
+ */
+export declare function getDefaultFetcher(env?: NodeJS.ProcessEnv): FetcherAdapter;
+/**
+ * Default storage factory — R2 when env-configured, otherwise in-memory.
+ */
+export declare function getDefaultStorage(env?: NodeJS.ProcessEnv): StorageAdapter;
+export interface FetchAndStoreArgs {
+    db: DbClient;
+    fetcher: FetcherAdapter;
+    storage: StorageAdapter;
+    url: string;
+    customerId: string;
+    subscriptionId: string | null;
+    fetchOptions?: FetchOptions;
+    /**
+     * When true (default), check robots.txt before fetching. Set false in
+     * tests that don't want to mock robots fetches; production callers
+     * should leave it on.
+     */
+    respectRobotsTxt?: boolean;
+    /** Test seam: override the robots.txt fetch. */
+    robotsFetchImpl?: typeof fetch;
+    /**
+     * Attestation signer (Review #3) — KMS-backed in production via
+     * `tryCreateKmsSigner`. When set, every snapshot row carries a signature
+     * over the JCS-canonical v1 manifest, and a signing failure fails the
+     * whole fetch: a configured signer is a provenance promise, and silently
+     * persisting unsigned rows would fake it. When null/absent, rows keep the
+     * explicit no-attestation placeholders (pre-KMS deployments, tests).
+     */
+    signer?: Signer | null;
+}
+export type FetchAndStoreOutcome = {
+    ok: true;
+    snapshotId: string;
+    contentHash: string;
+    r2Key: string;
+    fetchResult: FetchResult;
+} | {
+    ok: false;
+    reason: 'robots_disallowed';
+    urlCanonical: string;
+};
+/**
+ * Orchestrate one full fetch with robots-check + cost recording.
+ *
+ *   1. Canonicalize URL
+ *   2. Check robots.txt (skipped when respectRobotsTxt=false)
+ *      - Disallowed: update host_records.robots_status='disallowed',
+ *        write a compute cost row, return ok=false
+ *   3. Fetch via the adapter (SSRF validation lives in the HTTP adapter)
+ *   4. Store content-addressed in R2 / memory
+ *   5. Insert a snapshot row
+ *   6. Write a cost_records row (fetch_tier1, http_only)
+ *
+ * Returns a discriminated outcome — callers switch on `outcome.ok`.
+ */
+export declare function fetchAndStore(args: FetchAndStoreArgs): Promise<FetchAndStoreOutcome>;

package/dist/index.js

@@ -0,0 +1,219 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * @bluefields/fetcher — public surface (chunks 8 + 9).
+ *
+ * Per the four-layer rule:
+ *   - Fetcher returns raw HTML + metadata.
+ *   - It does NOT extract, diff, or notify.
+ *
+ * Production wiring:
+ *   - Fetcher: HTTP (default) → Patchright (when IPROYAL_* env set + dep
+ *     installed).
+ *   - Storage: in-memory (default) → R2 (when R2_* env set).
+ *
+ * `fetchAndStore` orchestrates one end-to-end fetch: robots-check →
+ * fetch via adapter → store content-addressed → sign the attestation
+ * manifest (when a signer is configured — A3, Review #3) → insert
+ * snapshot row → write cost_records.
+ *
+ * The return is a discriminated outcome — chunk 21's poll handler
+ * switches on `outcome.ok` to decide whether to feed the extractor or
+ * record a blocked-poll event.
+ */
+import { createHash } from 'node:crypto';
+import { MANIFEST_VERSION, buildManifest, canonicalize, signManifest, } from '@bluefields/attest';
+import { hostRecords, snapshots } from '@bluefields/db';
+import { canonicalizeUrl, createLayerLogger, isAllowedByRobotsTxt } from '@bluefields/primitives';
+import { eq } from 'drizzle-orm';
+const log = createLayerLogger('fetcher');
+import { TIER1_FIXED_COST_MICRO_USD, recordBlockedFetch, recordFetchCost } from './cost.js';
+import { createHttpFetcherAdapter } from './fetcher-http.js';
+import { tryCreatePatchrightAdapter } from './fetcher-patchright.js';
+import { createInMemoryStorageAdapter } from './storage-memory.js';
+import { tryCreateR2StorageAdapter } from './storage-r2.js';
+export { buildR2Key, } from './types.js';
+export { createHttpFetcherAdapter } from './fetcher-http.js';
+export { PATCHRIGHT_REQUIRED_ENV, tryCreatePatchrightAdapter } from './fetcher-patchright.js';
+export { classifyFetchResult, createEscalatingFetcherAdapter, } from './fetcher-escalating.js';
+export { createRobotsGatedFetcher, } from './fetcher-robots-gate.js';
+export { createInMemoryStorageAdapter } from './storage-memory.js';
+export { R2_REQUIRED_ENV, createR2StorageAdapter, tryCreateR2StorageAdapter, } from './storage-r2.js';
+export { COMPUTE_BLOCKED_COST_MICRO_USD, TIER1_FIXED_COST_MICRO_USD, recordBlockedFetch, recordFetchCost, } from './cost.js';
+/**
+ * Default fetcher factory — Patchright when env-configured, otherwise
+ * the plain HTTP fetcher.
+ */
+export function getDefaultFetcher(env = process.env) {
+    return tryCreatePatchrightAdapter(env) ?? createHttpFetcherAdapter();
+}
+/**
+ * Default storage factory — R2 when env-configured, otherwise in-memory.
+ */
+export function getDefaultStorage(env = process.env) {
+    return tryCreateR2StorageAdapter(env) ?? createInMemoryStorageAdapter();
+}
+/**
+ * Orchestrate one full fetch with robots-check + cost recording.
+ *
+ *   1. Canonicalize URL
+ *   2. Check robots.txt (skipped when respectRobotsTxt=false)
+ *      - Disallowed: update host_records.robots_status='disallowed',
+ *        write a compute cost row, return ok=false
+ *   3. Fetch via the adapter (SSRF validation lives in the HTTP adapter)
+ *   4. Store content-addressed in R2 / memory
+ *   5. Insert a snapshot row
+ *   6. Write a cost_records row (fetch_tier1, http_only)
+ *
+ * Returns a discriminated outcome — callers switch on `outcome.ok`.
+ */
+export async function fetchAndStore(args) {
+    const canonical = canonicalizeUrl(args.url);
+    const respectRobots = args.respectRobotsTxt ?? true;
+    const userAgent = args.fetchOptions?.userAgent ?? 'BlueFields-Fetcher/1.0';
+    // ── Robots-check ──
+    if (respectRobots) {
+        const allowed = await isAllowedByRobotsTxt(args.url, userAgent, {
+            ...(args.robotsFetchImpl ? { fetchImpl: args.robotsFetchImpl } : {}),
+        });
+        if (!allowed) {
+            const host = hostnameOf(args.url);
+            if (host) {
+                // Mark the host as disallowed so dashboards / future polls can
+                // short-circuit. Don't fail the request if this UPDATE doesn't
+                // affect a row — host_records may not exist yet (first time we
+                // see this host); the chunk-7 host-records repo will populate
+                // on the next call.
+                const updated = await args.db
+                    .update(hostRecords)
+                    .set({ robotsStatus: 'disallowed' })
+                    .where(eq(hostRecords.host, host))
+                    .returning({ host: hostRecords.host });
+                if (updated.length === 0) {
+                    log.debug({
+                        outcome: 'partial',
+                        subscription_id: args.subscriptionId,
+                        customer_id: args.customerId,
+                        host,
+                    }, 'robots_disallowed flag not set: host_records row missing (first-seen host)');
+                }
+            }
+            await recordBlockedFetch({
+                db: args.db,
+                customerId: args.customerId,
+                subscriptionId: args.subscriptionId,
+                urlCanonical: canonical.url_canonical,
+                reason: 'robots_disallowed',
+            });
+            log.info({
+                outcome: 'blocked',
+                subscription_id: args.subscriptionId,
+                customer_id: args.customerId,
+                url_canonical: canonical.url_canonical,
+            }, 'fetch blocked by robots.txt');
+            return { ok: false, reason: 'robots_disallowed', urlCanonical: canonical.url_canonical };
+        }
+    }
+    // ── Fetch + store + sign + snapshot ──
+    const fetchResult = await args.fetcher.fetch(args.url, args.fetchOptions);
+    const stored = await args.storage.storeSnapshot(fetchResult.html);
+    // Sign BEFORE persisting (spec §8 order) so a row never exists in a
+    // configured-but-unsigned state. The manifest carries a HASH of the
+    // JCS-canonicalized response headers, never the headers themselves —
+    // Set-Cookie and friends must not enter the permanent manifest (Review #3).
+    const attested = args.signer
+        ? await signManifest(buildManifest({
+            urlCanonical: canonical.url_canonical,
+            canonVersion: canonical.canon_version,
+            contentHash: stored.contentHash,
+            rawHtmlSizeBytes: fetchResult.rawHtmlSizeBytes,
+            fetchedAt: fetchResult.fetchedAt.toISOString(),
+            fetchDurationMs: fetchResult.fetchDurationMs,
+            fetcherId: fetchResult.fetcherId,
+            proxyEgressIp: fetchResult.proxyEgressIp,
+            userAgent: fetchResult.userAgent,
+            responseStatus: fetchResult.responseStatus,
+            responseHeadersHash: createHash('sha256')
+                .update(canonicalize(fetchResult.responseHeaders))
+                .digest('hex'),
+        }), args.signer)
+        : null;
+    const inserted = await args.db
+        .insert(snapshots)
+        .values({
+        urlCanonical: canonical.url_canonical,
+        urlSurt: canonical.url_surt,
+        canonVersion: canonical.canon_version,
+        contentHash: stored.contentHash,
+        fetchedAt: fetchResult.fetchedAt,
+        fetcherId: fetchResult.fetcherId,
+        proxyEgressIp: fetchResult.proxyEgressIp,
+        userAgent: fetchResult.userAgent,
+        responseStatus: fetchResult.responseStatus,
+        responseHeaders: fetchResult.responseHeaders,
+        rawHtmlR2Key: stored.r2Key,
+        rawHtmlSizeBytes: fetchResult.rawHtmlSizeBytes,
+        fetchDurationMs: fetchResult.fetchDurationMs,
+        // Signed when a signer is configured; explicit placeholders otherwise.
+        attestationSignature: attested?.signature.signature ?? '',
+        attestationKeyId: attested?.signature.keyId ?? 'v0-no-attestation',
+        attestationAlgorithm: attested?.signature.algorithm ?? 'EdDSA',
+        manifestVersion: MANIFEST_VERSION,
+        // Per-snapshot cost = the flat Tier 1 cost so the snapshot row is
+        // self-describing; cost_records is the authoritative billing row.
+        costMicroUsd: TIER1_FIXED_COST_MICRO_USD,
+    })
+        .returning({ id: snapshots.id });
+    const row = inserted[0];
+    if (!row)
+        throw new Error('snapshot insert returned no row');
+    // ── cost_records (chunk 9) ──
+    // Failure here means the snapshot is in DB but no billing row was
+    // written — partial success. Log audibly so a runaway billing gap
+    // is visible; don't fail the request (caller already has a valid
+    // snapshot to extract from).
+    try {
+        await recordFetchCost({
+            db: args.db,
+            customerId: args.customerId,
+            subscriptionId: args.subscriptionId,
+            urlCanonical: canonical.url_canonical,
+            units: fetchResult.rawHtmlSizeBytes,
+        });
+    }
+    catch (err) {
+        log.error({
+            outcome: 'partial',
+            subscription_id: args.subscriptionId,
+            customer_id: args.customerId,
+            snapshot_id: row.id,
+            error: err.message,
+        }, 'snapshot inserted but cost_records write failed — billing gap');
+    }
+    log.info({
+        outcome: 'ok',
+        subscription_id: args.subscriptionId,
+        customer_id: args.customerId,
+        snapshot_id: row.id,
+        content_hash: stored.contentHash,
+        response_status: fetchResult.responseStatus,
+        raw_html_size_bytes: fetchResult.rawHtmlSizeBytes,
+        duration_ms: fetchResult.fetchDurationMs,
+        attestation_key_id: attested?.signature.keyId ?? null,
+    }, 'fetch ok');
+    return {
+        ok: true,
+        snapshotId: row.id,
+        contentHash: stored.contentHash,
+        r2Key: stored.r2Key,
+        fetchResult,
+    };
+}
+function hostnameOf(url) {
+    try {
+        return new URL(url).hostname.toLowerCase();
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EACL,gBAAgB,EAGhB,aAAa,EACb,YAAY,EACZ,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAiB,WAAW,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AACvE,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAClG,OAAO,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAEjC,MAAM,GAAG,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEzC,OAAO,EAAE,0BAA0B,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5F,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AACrE,OAAO,EAAE,4BAA4B,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAG5D,OAAO,EASL,UAAU,GACX,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAC9F,OAAO,EAGL,mBAAmB,EACnB,8BAA8B,GAC/B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAGL,wBAAwB,GACzB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,4BAA4B,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EACL,eAAe,EAEf,sBAAsB,EACtB,yBAAyB,GAC1B,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,8BAA8B,EAG9B,0BAA0B,EAC1B,kBAAkB,EAClB,eAAe,GAChB,MAAM,WAAW,CAAC;AAEnB;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAyB,OAAO,CAAC,GAAG;IACpE,OAAO,0BAA0B,CAAC,GAAG,CAAC,IAAI,wBAAwB,EAAE,CAAC;AACvE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAyB,OAAO,CAAC,GAAG;IACpE,OAAO,yBAAyB,CAAC,GAAG,CAAC,IAAI,4BAA4B,EAAE,CAAC;AAC1E,CAAC;AA2CD;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAuB;IACzD,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC;IACpD,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,SAAS,IAAI,wBAAwB,CAAC;IAE3E,qBAAqB;IACrB,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE;YAC9D,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,IAAI,EAAE,CAAC;gBACT,+DAA+D;gBAC/D,+DAA+D;gBAC/D,+DAA+D;gBAC/D,8DAA8D;gBAC9D,oBAAoB;gBACpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE;qBAC1B,MAAM,CAAC,WAAW,CAAC;qBACnB,GAAG,CAAC,EAAE,YAAY,EAAE,YAAY,EAAE,CAAC;qBACnC,KAAK,CAAC,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;qBACjC,SAAS,CAAC,EAAE,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACzB,GAAG,CAAC,KAAK,CACP;wBACE,OAAO,EAAE,SAAS;wBAClB,eAAe,EAAE,IAAI,CAAC,cAAc;wBACpC,WAAW,EAAE,IAAI,CAAC,UAAU;wBAC5B,IAAI;qBACL,EACD,4EAA4E,CAC7E,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,MAAM,kBAAkB,CAAC;gBACvB,EAAE,EAAE,IAAI,CAAC,EAAE;gBACX,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,YAAY,EAAE,SAAS,CAAC,aAAa;gBACrC,MAAM,EAAE,mBAAmB;aAC5B,CAAC,CAAC;YACH,GAAG,CAAC,IAAI,CACN;gBACE,OAAO,EAAE,SAAS;gBAClB,eAAe,EAAE,IAAI,CAAC,cAAc;gBACpC,WAAW,EAAE,IAAI,CAAC,UAAU;gBAC5B,aAAa,EAAE,SAAS,CAAC,aAAa;aACvC,EACD,6BAA6B,CAC9B,CAAC;YACF,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,YAAY,EAAE,SAAS,CAAC,aAAa,EAAE,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAElE,oEAAoE;IACpE,oEAAoE;IACpE,qEAAqE;IACrE,4EAA4E;IAC5E,MAAM,QAAQ,GAA0B,IAAI,CAAC,MAAM;QACjD,CAAC,CAAC,MAAM,YAAY,CAChB,aAAa,CAAC;YACZ,YAAY,EAAE,SAAS,CAAC,aAAa;YACrC,YAAY,EAAE,SAAS,CAAC,aAAa;YACrC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,gBAAgB,EAAE,WAAW,CAAC,gBAAgB;YAC9C,SAAS,EAAE,WAAW,CAAC,SAAS,CAAC,WAAW,EAAE;YAC9C,eAAe,EAAE,WAAW,CAAC,eAAe;YAC5C,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,aAAa,EAAE,WAAW,CAAC,aAAa;YACxC,SAAS,EAAE,WAAW,CAAC,SAAS;YAChC,cAAc,EAAE,WAAW,CAAC,cAAc;YAC1C,mBAAmB,EAAE,UAAU,CAAC,QAAQ,CAAC;iBACtC,MAAM,CAAC,YAAY,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;iBACjD,MAAM,CAAC,KAAK,CAAC;SACjB,CAAC,EACF,IAAI,CAAC,MAAM,CACZ;QACH,CAAC,CAAC,IAAI,CAAC;IAET,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,EAAE;SAC3B,MAAM,CAAC,SAAS,CAAC;SACjB,MAAM,CAAC;QACN,YAAY,EAAE,SAAS,CAAC,aAAa;QACrC,OAAO,EAAE,SAAS,CAAC,QAAQ;QAC3B,YAAY,EAAE,SAAS,CAAC,aAAa;QACrC,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,SAAS,EAAE,WAAW,CAAC,SAAS;QAChC,SAAS,EAAE,WAAW,CAAC,SAAS;QAChC,aAAa,EAAE,WAAW,CAAC,aAAa;QACxC,SAAS,EAAE,WAAW,CAAC,SAAS;QAChC,cAAc,EAAE,WAAW,CAAC,cAAc;QAC1C,eAAe,EAAE,WAAW,CAAC,eAAe;QAC5C,YAAY,EAAE,MAAM,CAAC,KAAK;QAC1B,gBAAgB,EAAE,WAAW,CAAC,gBAAgB;QAC9C,eAAe,EAAE,WAAW,CAAC,eAAe;QAC5C,uEAAuE;QACvE,oBAAoB,EAAE,QAAQ,EAAE,SAAS,CAAC,SAAS,IAAI,EAAE;QACzD,gBAAgB,EAAE,QAAQ,EAAE,SAAS,CAAC,KAAK,IAAI,mBAAmB;QAClE,oBAAoB,EAAE,QAAQ,EAAE,SAAS,CAAC,SAAS,IAAI,OAAO;QAC9D,eAAe,EAAE,gBAAgB;QACjC,kEAAkE;QAClE,kEAAkE;QAClE,YAAY,EAAE,0BAA0B;KACzC,CAAC;SACD,SAAS,CAAC,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;IAEnC,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IAE7D,+BAA+B;IAC/B,kEAAkE;IAClE,kEAAkE;IAClE,iEAAiE;IACjE,6BAA6B;IAC7B,IAAI,CAAC;QACH,MAAM,eAAe,CAAC;YACpB,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,YAAY,EAAE,SAAS,CAAC,aAAa;YACrC,KAAK,EAAE,WAAW,CAAC,gBAAgB;SACpC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CACP;YACE,OAAO,EAAE,SAAS;YAClB,eAAe,EAAE,IAAI,CAAC,cAAc;YACpC,WAAW,EAAE,IAAI,CAAC,UAAU;YAC5B,WAAW,EAAE,GAAG,CAAC,EAAE;YACnB,KAAK,EAAG,GAAa,CAAC,OAAO;SAC9B,EACD,+DAA+D,CAChE,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,IAAI,CACN;QACE,OAAO,EAAE,IAAI;QACb,eAAe,EAAE,IAAI,CAAC,cAAc;QACpC,WAAW,EAAE,IAAI,CAAC,UAAU;QAC5B,WAAW,EAAE,GAAG,CAAC,EAAE;QACnB,YAAY,EAAE,MAAM,CAAC,WAAW;QAChC,eAAe,EAAE,WAAW,CAAC,cAAc;QAC3C,mBAAmB,EAAE,WAAW,CAAC,gBAAgB;QACjD,WAAW,EAAE,WAAW,CAAC,eAAe;QACxC,kBAAkB,EAAE,QAAQ,EAAE,SAAS,CAAC,KAAK,IAAI,IAAI;KACtD,EACD,UAAU,CACX,CAAC;IAEF,OAAO;QACL,EAAE,EAAE,IAAI;QACR,UAAU,EAAE,GAAG,CAAC,EAAE;QAClB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/internal-options.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Fetcher-internal escalation hints.
+ *
+ * These are NOT customer-facing. The API route and the SDKs build a public
+ * `FetchOptions` (see `types.ts`); they never set or expose anything here.
+ * `InternalFetchOptions` exists only inside `packages/fetcher` so the
+ * escalating fetcher (`fetcher-escalating.ts`) can tune how a higher tier
+ * behaves on a re-fetch, without changing the public `FetcherAdapter` contract
+ * (`fetch(url, options?)`) or the customer-visible option shape.
+ *
+ * Adapters read these defensively (a missing field means "behave as before"),
+ * so passing them to an adapter that doesn't understand them is a no-op.
+ */
+import type { FetchOptions } from './types.js';
+export interface InternalFetchOptions extends FetchOptions {
+    /**
+     * Fully render a JS app before reading content: best-effort `networkidle`,
+     * then a bounded autoscroll + content-settle loop (see `fullyRenderPage` in
+     * `fetcher-patchright.ts`). Honored only by browser tiers; the HTTP and
+     * unlocker tiers ignore it.
+     *
+     * Set by `createEscalatingFetcherAdapter` when it escalates a JS shell /
+     * anti-bot block to a higher tier AND the customer gave no explicit
+     * `waitFor` (an explicit `waitFor` always wins). The whole settle sequence
+     * is clamped to `timeoutMs`, so it never extends a request past its deadline.
+     */
+    __fullRender?: boolean;
+    /**
+     * Authorized robots.txt opt-out. When true, the robots-gated fetcher skips the
+     * Disallow check (and logs an audit entry). Resolved UPSTREAM at the API layer
+     * from the customer's `ignoreRobots` + `robotsAttestation` AND an account-level
+     * capability — the fetcher never decides authorization itself (four-layer rule).
+     * Absent/false → robots is enforced normally. See `ROBOTS_POLICY.md`.
+     */
+    __robotsBypassAuthorized?: boolean;
+    /**
+     * Abandonment signal. `createEscalatingFetcherAdapter` aborts this when a tier
+     * exceeds its wall-clock budget (`raceTierBudget`) so the abandoned tier can
+     * release resources instead of running on. The browser tier (Patchright)
+     * honors it by closing its context — without this, an abandoned browser fetch
+     * leaks headless Chromium. The HTTP / unlocker tiers ignore it (they leak at
+     * most a socket, bounded by undici's own timeouts).
+     */
+    __abortSignal?: AbortSignal;
+    /**
+     * Tier-0 early-block escalation (perf round, L1). Set ONLY by the escalating
+     * fetcher on its tier-0 stage. When true, the HTTP adapter — on a DECISIVE block
+     * status (403/429/503, the statuses classifyFetchResult always maps to
+     * challenge|blocked, both `needsEscalation`) — skips draining the doomed response
+     * body and returns an empty-body result immediately, so the escalating layer
+     * escalates at ~time-to-headers instead of after the full (often slow-rolled) block
+     * body or the tier-0 fast-fail wall. Coverage-SAFE by construction: it never emits
+     * an early `success` and fires only on statuses that already escalate regardless of
+     * body. Absent/false → today's full-drain behavior, so a standalone HTTP-adapter
+     * caller (which never sets this) is byte-for-byte unchanged. Browser/unlocker tiers
+     * ignore it.
+     */
+    __earlyClassify?: boolean;
+}

package/dist/internal-options.js

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Fetcher-internal escalation hints.
+ *
+ * These are NOT customer-facing. The API route and the SDKs build a public
+ * `FetchOptions` (see `types.ts`); they never set or expose anything here.
+ * `InternalFetchOptions` exists only inside `packages/fetcher` so the
+ * escalating fetcher (`fetcher-escalating.ts`) can tune how a higher tier
+ * behaves on a re-fetch, without changing the public `FetcherAdapter` contract
+ * (`fetch(url, options?)`) or the customer-visible option shape.
+ *
+ * Adapters read these defensively (a missing field means "behave as before"),
+ * so passing them to an adapter that doesn't understand them is a no-op.
+ */
+export {};
+//# sourceMappingURL=internal-options.js.map

package/dist/internal-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"internal-options.js","sourceRoot":"","sources":["../src/internal-options.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;GAYG"}

package/dist/perf-harness.d.ts

@@ -0,0 +1,105 @@
+/**
+ * Deterministic fixture perf harness for the escalating fetcher (perf round).
+ *
+ * The mission requires a before/after p50/p95/cost/coverage number PER LEVER on a
+ * FIXTURE corpus (never live sites). This library scripts each tier's latency +
+ * outcome per fixture URL, drives the real `createEscalatingFetcherAdapter` against
+ * those mocks, and reports the distribution. It is deliberately dependency-free and
+ * vitest-free: the CALLER injects the clock (a fake-timer test drives `now` +
+ * `advanceAndFlush`), so the whole run is deterministic and fast — no real waiting,
+ * no network. The runnable entrypoint is `fetcher-escalating.perf.test.ts`
+ * (FETCH_PERF=1-gated, inert in the normal suite); `perf-harness.test.ts` guards the
+ * metrics math in CI.
+ *
+ * Fetcher-layer only: it exercises tier selection + timing, nothing else.
+ */
+import type { EscalatingFetcherOptions } from './fetcher-escalating.js';
+import type { FetcherAdapter } from './types.js';
+/** Billed credits per tier (mirrors billing: tier-0 HTTP=1, tier-1 browser=3, tier-2 unlocker=8). */
+export declare const TIER_CREDITS: Readonly<Record<number, number>>;
+/** fetcherId each tier's mock reports — matches the real adapters so we can map result→tier. */
+export declare const TIER_FETCHER_ID: Readonly<Record<number, string>>;
+/** Scripted behavior of one tier for one fixture. */
+export type TierScript = 
+/** Resolves with `status`+`html` after `latencyMs` of (fake) time. */
+{
+    latencyMs: number;
+    status: number;
+    html: string;
+}
+/** Never resolves on its own — models a slow-roll/hung tier bounded only by the budget/deadline. */
+ | {
+    latencyMs?: number;
+    kind: 'hang';
+}
+/** Tier not configured (adapter is null). */
+ | {
+    kind: 'absent';
+};
+export interface FixtureSpec {
+    url: string;
+    /** Short human label for reports (e.g. 'fast-success', 'slow-block', 'shell→browser'). */
+    label: string;
+    tier0: TierScript;
+    tier1?: TierScript;
+    tier2?: TierScript;
+}
+/**
+ * A mock tier adapter honoring the abort seam the escalating fetcher uses: a hung
+ * tier stays pending until aborted (then rejects, like a real abandoned tier); a
+ * scripted tier resolves after `latencyMs` via setTimeout (fake-clock driven). Returns
+ * null for an absent tier so the ladder omits it.
+ */
+export declare function mockTierAdapter(tier: number, script: TierScript): FetcherAdapter | null;
+export interface PerRequest {
+    url: string;
+    label: string;
+    elapsedMs: number;
+    /** Winning tier (0/1/2) or -1 if the fetch threw. */
+    tier: number;
+    credits: number;
+    outcome: string;
+    covered: boolean;
+}
+export interface CorpusReport {
+    n: number;
+    /** Latency percentiles (ms) over end-to-end fake-clock elapsed. */
+    p50: number;
+    p95: number;
+    max: number;
+    /** Coverage = fraction whose final outcome is `success`. This is the HARD FLOOR metric. */
+    successRate: number;
+    /** Cost per successful scrape = total credits billed / number covered. */
+    creditsPerSuccess: number;
+    /** Mean credits per request (blended tier mix). */
+    meanCredits: number;
+    /** Count of requests won by each tier. */
+    perTier: Record<number, number>;
+    requests: PerRequest[];
+}
+/** Nearest-rank percentile (no interpolation) — same convention as the SLA harness. */
+export declare function percentile(sorted: number[], q: number): number;
+export interface RunOptions {
+    /** Escalating-fetcher config under test (tier0TimeoutMs, hedgeDelayMs, …). nowMsFn is injected below. */
+    escalatingOpts?: Partial<Omit<EscalatingFetcherOptions, 'tier0' | 'tier1' | 'tier2' | 'nowMsFn'>>;
+    /** Repeat the corpus this many times (default 1). */
+    iterations?: number;
+    /** Read the current (fake) clock — pass the test's faked Date.now. */
+    now: () => number;
+    /** Advance the (fake) clock by `ms` AND flush microtasks/timers. Pass `vi.advanceTimersByTimeAsync`. */
+    advanceAndFlush: (ms: number) => Promise<void>;
+    /** Advance step (ms). Smaller = finer latency resolution, more iterations. Default 25. */
+    stepMs?: number;
+    /** Safety cap on total advance per request (ms). Default 120000. */
+    capMs?: number;
+}
+/**
+ * Run the fixture corpus through the real escalating fetcher under the caller's fake
+ * clock, returning the latency/cost/coverage distribution. For each fixture it builds
+ * `createEscalatingFetcherAdapter` with the scripted mock tiers + the injected
+ * `nowMsFn`, starts the fetch, and drives `advanceAndFlush(stepMs)` until it settles —
+ * measuring end-to-end elapsed on the fake clock (quantized to stepMs).
+ */
+export declare function runCorpus(fixtures: FixtureSpec[], o: RunOptions): Promise<CorpusReport>;
+/** Pretty one-line summary for a config's report (for the perf entrypoint's console output). */
+export declare function formatReport(name: string, r: CorpusReport): string;