@bluefields/fetcher 0.2.0

package/dist/asset-block-measure.js

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Manual bandwidth measurement (gated, NOT a test). Fetches a few image-heavy
+ * pages through the browser tier with asset-blocking ON (the normal path) vs
+ * OFF (a screenshot request disables blocking) and prints bytesTransferred for
+ * each, to confirm the metered-proxy saving is real.
+ *
+ *   ASSET_BLOCK_MEASURE=1 packages/fetcher/node_modules/.bin/tsx \
+ *     packages/fetcher/src/asset-block-measure.ts
+ *   (source .env.local first for IPROYAL_*). Spends a little IPRoyal egress.
+ *
+ * Robots: a crude `Disallow: /` check for the `*` group skips a host that
+ * forbids it; we only target bot-permissive homepages anyway.
+ */
+import { tryCreatePatchrightAdapter } from './fetcher-patchright.js';
+const URLS = [
+    'https://www.theverge.com/',
+    'https://www.bbc.com/',
+    'https://www.nationalgeographic.com/',
+    'https://www.cnn.com/',
+];
+const UA = 'BlueFields-Fetcher/1.0 (+https://bluefields.dev)';
+async function rootAllowed(url) {
+    try {
+        const res = await fetch(`${new URL(url).origin}/robots.txt`, {
+            headers: { 'user-agent': UA },
+            signal: AbortSignal.timeout(8_000),
+        });
+        if (!res.ok)
+            return true;
+        const txt = (await res.text()).toLowerCase();
+        const star = txt.split(/user-agent:/).find((b) => b.trimStart().startsWith('*'));
+        return !(star && /\n\s*disallow:\s*\/\s*(\n|$)/.test(`\n${star}`));
+    }
+    catch {
+        return true;
+    }
+}
+const kb = (n) => n == null ? '     n/a' : `${Math.round(n / 1024)}KB`.padStart(8);
+async function main() {
+    if (process.env.ASSET_BLOCK_MEASURE !== '1') {
+        console.log('gated: set ASSET_BLOCK_MEASURE=1 (live; spends a little IPRoyal egress).');
+        return;
+    }
+    const adapter = tryCreatePatchrightAdapter(process.env);
+    if (!adapter) {
+        console.log('no Patchright/IPRoyal env (IPROYAL_GATEWAY + creds) — cannot measure.');
+        return;
+    }
+    console.log(`${'url'.padEnd(38)} OFF(before)  ON(after)     saved`);
+    let totBefore = 0;
+    let totAfter = 0;
+    for (const url of URLS) {
+        if (!(await rootAllowed(url))) {
+            console.log(`${url.padEnd(38)} skip (robots)`);
+            continue;
+        }
+        try {
+            // before: a screenshot request disables blocking → full asset download
+            const before = await adapter.fetch(url, { timeoutMs: 45_000, captureScreenshot: true });
+            // after: normal browser tier → image/media/font/stylesheet aborted
+            const after = await adapter.fetch(url, { timeoutMs: 45_000 });
+            const b = before.bytesTransferred ?? 0;
+            const a = after.bytesTransferred ?? 0;
+            totBefore += b;
+            totAfter += a;
+            const pct = b ? Math.round((100 * (b - a)) / b) : 0;
+            console.log(`${url.padEnd(38)} ${kb(b)} ${kb(a)}  ${kb(b - a)} (${pct}%)`);
+        }
+        catch (err) {
+            console.log(`${url.padEnd(38)} ERROR ${err.message.slice(0, 50)}`);
+        }
+        await new Promise((r) => setTimeout(r, 1_000));
+    }
+    const pct = totBefore ? Math.round((100 * (totBefore - totAfter)) / totBefore) : 0;
+    console.log(`${'TOTAL'.padEnd(38)} ${kb(totBefore)} ${kb(totAfter)}  ${kb(totBefore - totAfter)} (${pct}%)`);
+}
+main().catch((e) => {
+    console.error('measure failed:', e);
+    process.exit(1);
+});
+//# sourceMappingURL=asset-block-measure.js.map

package/dist/asset-block-measure.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"asset-block-measure.js","sourceRoot":"","sources":["../src/asset-block-measure.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAErE,MAAM,IAAI,GAAG;IACX,2BAA2B;IAC3B,sBAAsB;IACtB,qCAAqC;IACrC,sBAAsB;CACvB,CAAC;AACF,MAAM,EAAE,GAAG,kDAAkD,CAAC;AAE9D,KAAK,UAAU,WAAW,CAAC,GAAW;IACpC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,aAAa,EAAE;YAC3D,OAAO,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE;YAC7B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;SACnC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACjF,OAAO,CAAC,CAAC,IAAI,IAAI,8BAA8B,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;IACrE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,EAAE,GAAG,CAAC,CAAU,EAAU,EAAE,CAChC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAEnE,KAAK,UAAU,IAAI;IACjB,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,GAAG,EAAE,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,0EAA0E,CAAC,CAAC;QACxF,OAAO;IACT,CAAC;IACD,MAAM,OAAO,GAAG,0BAA0B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,uEAAuE,CAAC,CAAC;QACrF,OAAO;IACT,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,mCAAmC,CAAC,CAAC;IACpE,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,MAAM,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC;YAC/C,SAAS;QACX,CAAC;QACD,IAAI,CAAC;YACH,uEAAuE;YACvE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;YACxF,mEAAmE;YACnE,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9D,MAAM,CAAC,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;YACvC,MAAM,CAAC,GAAG,KAAK,CAAC,gBAAgB,IAAI,CAAC,CAAC;YACtC,SAAS,IAAI,CAAC,CAAC;YACf,QAAQ,IAAI,CAAC,CAAC;YACd,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;QAC7E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,UAAW,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,OAAO,CAAC,GAAG,CACT,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,SAAS,GAAG,QAAQ,CAAC,KAAK,GAAG,IAAI,CAChG,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IACjB,OAAO,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;IACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/browser-pool.d.ts

@@ -0,0 +1,65 @@
+/**
+ * Bounded Chromium browser pool for the Patchright adapter.
+ *
+ * Why: launching Chromium costs ~500ms-1.5s per fetch (Patchright + IPRoyal
+ * proxy + headless boot). At sub-1-rps load that's tolerable; at 5+ rps it
+ * doubles end-to-end latency. With a pool of N browsers, only the FIRST
+ * fetch eats the launch cost; subsequent fetches just newContext (~30ms).
+ *
+ * Isolation: every fetch still gets a fresh `browser.newContext()` (own
+ * cookies, storage, fingerprint slot) so reused browsers don't leak state.
+ * The cookie jar / localStorage live on the context, not the browser.
+ *
+ * Crash resilience: if any context-level op fails, the slot is *disposed*
+ * (browser killed) and replaced on next acquire. Browsers also age out
+ * after MAX_USES contexts to avoid memory creep from a long-lived headless
+ * Chromium.
+ *
+ * Fail-open: when a launch fails, acquire throws and the caller falls
+ * back to per-fetch launch (the adapter wraps this with try/catch).
+ */
+import type { ChromiumLauncher } from './fetcher-patchright.js';
+export interface BrowserPoolOptions {
+    launcher: ChromiumLauncher;
+    proxy: {
+        server: string;
+        username: string;
+        password: string;
+    };
+    /** Number of browsers held alive concurrently. Default 2. */
+    size?: number;
+    /** Max time to wait for a free slot before rejecting. Default 30s. */
+    acquireTimeoutMs?: number;
+}
+type LaunchedBrowser = Awaited<ReturnType<ChromiumLauncher['launch']>>;
+interface Slot {
+    browser: LaunchedBrowser;
+    usesRemaining: number;
+    inUse: boolean;
+}
+export declare class BrowserPool {
+    private readonly opts;
+    private readonly slots;
+    private readonly waiters;
+    private stopped;
+    constructor(opts: BrowserPoolOptions);
+    /** Pre-launch all browsers. Safe to call multiple times. */
+    warm(): Promise<void>;
+    /**
+     * Acquire a browser. Caller MUST call release(slot) or dispose(slot)
+     * exactly once when done. Throws on acquire timeout or launch failure.
+     */
+    acquire(): Promise<Slot>;
+    /** Return a healthy slot to the pool. Decrements its remaining lifespan. */
+    release(slot: Slot): void;
+    /** Mark a slot as crashed; kill the browser + replace. */
+    dispose(slot: Slot): void;
+    /** Close all browsers. */
+    stop(): Promise<void>;
+    /** Visible for testing. */
+    get size(): number;
+    private handoff;
+    private launchOne;
+    private replaceSlot;
+}
+export {};

package/dist/browser-pool.js

@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Bounded Chromium browser pool for the Patchright adapter.
+ *
+ * Why: launching Chromium costs ~500ms-1.5s per fetch (Patchright + IPRoyal
+ * proxy + headless boot). At sub-1-rps load that's tolerable; at 5+ rps it
+ * doubles end-to-end latency. With a pool of N browsers, only the FIRST
+ * fetch eats the launch cost; subsequent fetches just newContext (~30ms).
+ *
+ * Isolation: every fetch still gets a fresh `browser.newContext()` (own
+ * cookies, storage, fingerprint slot) so reused browsers don't leak state.
+ * The cookie jar / localStorage live on the context, not the browser.
+ *
+ * Crash resilience: if any context-level op fails, the slot is *disposed*
+ * (browser killed) and replaced on next acquire. Browsers also age out
+ * after MAX_USES contexts to avoid memory creep from a long-lived headless
+ * Chromium.
+ *
+ * Fail-open: when a launch fails, acquire throws and the caller falls
+ * back to per-fetch launch (the adapter wraps this with try/catch).
+ */
+import { createLayerLogger } from '@bluefields/primitives';
+const log = createLayerLogger('fetcher');
+/** Replace a browser after this many contexts to bound memory. */
+const MAX_USES_PER_BROWSER = 200;
+/** Hard ceiling on acquire wait time. */
+const DEFAULT_ACQUIRE_TIMEOUT_MS = 30_000;
+export class BrowserPool {
+    opts;
+    slots = [];
+    waiters = [];
+    stopped = false;
+    constructor(opts) {
+        this.opts = {
+            launcher: opts.launcher,
+            proxy: opts.proxy,
+            size: opts.size ?? 2,
+            acquireTimeoutMs: opts.acquireTimeoutMs ?? DEFAULT_ACQUIRE_TIMEOUT_MS,
+        };
+    }
+    /** Pre-launch all browsers. Safe to call multiple times. */
+    async warm() {
+        if (this.stopped)
+            return;
+        while (this.slots.length < this.opts.size) {
+            const browser = await this.launchOne();
+            this.slots.push({ browser, usesRemaining: MAX_USES_PER_BROWSER, inUse: false });
+            log.info({ outcome: 'ok', pool_size: this.slots.length }, 'browser-pool: warmed slot');
+        }
+    }
+    /**
+     * Acquire a browser. Caller MUST call release(slot) or dispose(slot)
+     * exactly once when done. Throws on acquire timeout or launch failure.
+     */
+    async acquire() {
+        if (this.stopped)
+            throw new Error('browser-pool: stopped');
+        const free = this.slots.find((s) => !s.inUse);
+        if (free) {
+            free.inUse = true;
+            return free;
+        }
+        if (this.slots.length < this.opts.size) {
+            const browser = await this.launchOne();
+            const slot = { browser, usesRemaining: MAX_USES_PER_BROWSER, inUse: true };
+            this.slots.push(slot);
+            return slot;
+        }
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                const idx = this.waiters.findIndex((w) => w.timer === timer);
+                if (idx >= 0)
+                    this.waiters.splice(idx, 1);
+                reject(new Error(`browser-pool: acquire timeout after ${this.opts.acquireTimeoutMs}ms`));
+            }, this.opts.acquireTimeoutMs);
+            this.waiters.push({ resolve, reject, timer });
+        });
+    }
+    /** Return a healthy slot to the pool. Decrements its remaining lifespan. */
+    release(slot) {
+        slot.usesRemaining -= 1;
+        if (slot.usesRemaining <= 0) {
+            void this.replaceSlot(slot);
+            return;
+        }
+        this.handoff(slot);
+    }
+    /** Mark a slot as crashed; kill the browser + replace. */
+    dispose(slot) {
+        void this.replaceSlot(slot);
+    }
+    /** Close all browsers. */
+    async stop() {
+        this.stopped = true;
+        for (const w of this.waiters) {
+            clearTimeout(w.timer);
+            w.reject(new Error('browser-pool: stopping'));
+        }
+        this.waiters.length = 0;
+        const toClose = [...this.slots];
+        this.slots.length = 0;
+        await Promise.all(toClose.map((s) => s.browser.close().catch((err) => {
+            log.warn({ error: err instanceof Error ? err.message : String(err) }, 'browser-pool: close failed');
+        })));
+    }
+    /** Visible for testing. */
+    get size() {
+        return this.slots.length;
+    }
+    // ── private ────────────────────────────────────────────────────────────
+    handoff(slot) {
+        if (this.waiters.length === 0) {
+            slot.inUse = false;
+            return;
+        }
+        const next = this.waiters.shift();
+        clearTimeout(next.timer);
+        // Slot stays inUse; just transferred to the next caller.
+        next.resolve(slot);
+    }
+    async launchOne() {
+        const browser = await this.opts.launcher.launch({
+            proxy: this.opts.proxy,
+            headless: true,
+            args: ['--no-sandbox', '--disable-dev-shm-usage', '--disable-gpu'],
+            timeout: 20_000,
+        });
+        return browser;
+    }
+    async replaceSlot(slot) {
+        const idx = this.slots.indexOf(slot);
+        if (idx < 0)
+            return;
+        this.slots.splice(idx, 1);
+        await slot.browser.close().catch((err) => {
+            log.warn({ error: err instanceof Error ? err.message : String(err) }, 'browser-pool: close-on-replace failed');
+        });
+        if (this.stopped)
+            return;
+        // Replace only if we have waiters or we're below target — otherwise
+        // let the next acquire lazily refill.
+        if (this.waiters.length > 0 || this.slots.length < this.opts.size) {
+            try {
+                const browser = await this.launchOne();
+                const fresh = { browser, usesRemaining: MAX_USES_PER_BROWSER, inUse: false };
+                this.slots.push(fresh);
+                this.handoff(fresh);
+            }
+            catch (err) {
+                log.warn({ error: err instanceof Error ? err.message : String(err) }, 'browser-pool: replacement launch failed');
+                // Reject the head waiter so the caller can fall back to per-fetch launch.
+                const w = this.waiters.shift();
+                if (w) {
+                    clearTimeout(w.timer);
+                    w.reject(err instanceof Error ? err : new Error(String(err)));
+                }
+            }
+        }
+    }
+}
+//# sourceMappingURL=browser-pool.js.map

package/dist/browser-pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-pool.js","sourceRoot":"","sources":["../src/browser-pool.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAI3D,MAAM,GAAG,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEzC,kEAAkE;AAClE,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,yCAAyC;AACzC,MAAM,0BAA0B,GAAG,MAAM,CAAC;AAyB1C,MAAM,OAAO,WAAW;IACL,IAAI,CAGnB;IACe,KAAK,GAAW,EAAE,CAAC;IACnB,OAAO,GAAa,EAAE,CAAC;IAChC,OAAO,GAAG,KAAK,CAAC;IAExB,YAAY,IAAwB;QAClC,IAAI,CAAC,IAAI,GAAG;YACV,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC;YACpB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,IAAI,0BAA0B;SACtE,CAAC;IACJ,CAAC;IAED,4DAA4D;IAC5D,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAChF,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,2BAA2B,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAE3D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACvC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YACvC,MAAM,IAAI,GAAS,EAAE,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YACjF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC;gBAC7D,IAAI,GAAG,IAAI,CAAC;oBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC1C,MAAM,CAAC,IAAI,KAAK,CAAC,uCAAuC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC,CAAC;YAC3F,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC/B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4EAA4E;IAC5E,OAAO,CAAC,IAAU;QAChB,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;QACxB,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC;YAC5B,KAAK,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC5B,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IAED,0DAA0D;IAC1D,OAAO,CAAC,IAAU;QAChB,KAAK,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,0BAA0B;IAC1B,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACtB,MAAM,OAAO,CAAC,GAAG,CACf,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAChB,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC9B,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC3D,4BAA4B,CAC7B,CAAC;QACJ,CAAC,CAAC,CACH,CACF,CAAC;IACJ,CAAC;IAED,2BAA2B;IAC3B,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,0EAA0E;IAElE,OAAO,CAAC,IAAU;QACxB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAG,CAAC;QACnC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,yDAAyD;QACzD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC9C,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YACtB,QAAQ,EAAE,IAAI;YACd,IAAI,EAAE,CAAC,cAAc,EAAE,yBAAyB,EAAE,eAAe,CAAC;YAClE,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAU;QAClC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,GAAG,GAAG,CAAC;YAAE,OAAO;QACpB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC1B,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvC,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC3D,uCAAuC,CACxC,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,oEAAoE;QACpE,sCAAsC;QACtC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAClE,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAS,EAAE,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;gBACnF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACvB,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACtB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CACN,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC3D,yCAAyC,CAC1C,CAAC;gBACF,0EAA0E;gBAC1E,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBAC/B,IAAI,CAAC,EAAE,CAAC;oBACN,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBACtB,CAAC,CAAC,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/cost.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Fetcher cost recording (chunk 9, Review #11).
+ *
+ * Per Review #11, every fetch writes a cost_records row. The cost_type
+ * discriminates the source ('fetch_tier1' for Patchright; 'fetch_tier0'
+ * later for HTTP-only Tier 0); the cost_subtype refines it (proxy
+ * provider name, or 'http_only' for no-proxy fetches).
+ *
+ * v1 uses a flat $0.0014 per Tier 1 fetch (Review #4 weighted average).
+ * Later chunks will refine this from actual proxy bytes + Patchright
+ * runtime once the production fetcher ships.
+ */
+import { type DbClient } from '@bluefields/db';
+/** Per-fetch cost in micro-USD (Review #4: ~$0.0014 weighted average). */
+export declare const TIER1_FIXED_COST_MICRO_USD = 1400;
+/** Compute-only row when a fetch is blocked (robots disallow). */
+export declare const COMPUTE_BLOCKED_COST_MICRO_USD = 50;
+export type FetchCostType = 'fetch_tier0' | 'fetch_tier1' | 'fetch_tier2' | 'fetch_tier3' | 'fetch_304' | 'proxy_bandwidth' | 'compute';
+export interface RecordFetchCostArgs {
+    db: DbClient;
+    customerId: string;
+    subscriptionId: string | null;
+    urlCanonical: string;
+    costType?: FetchCostType;
+    /** Subtype refinement — e.g. proxy provider name, or 'http_only'. */
+    costSubtype?: string | null;
+    costMicroUsd?: number;
+    /** Units recorded for analytics (bytes, ms, etc.). */
+    units?: string | number | null;
+}
+/**
+ * Write a cost_records row for one fetch attempt.
+ *
+ * Defaults to Tier 1's flat cost with subtype='http_only' to reflect the
+ * v1 default fetcher. Override `costType` / `costSubtype` when you know
+ * (e.g. proxy_bandwidth row after a Patchright fetch, or 'fetch_304'
+ * when conditional GET returned 304 once chunk M8-10 wires that).
+ */
+export declare function recordFetchCost(args: RecordFetchCostArgs): Promise<void>;
+/**
+ * Write a near-zero `compute` cost_records row for a fetch that was
+ * blocked before reaching the wire (e.g. robots.txt disallow). We still
+ * record the row so the customer can see the request in usage history.
+ */
+export declare function recordBlockedFetch(args: {
+    db: DbClient;
+    customerId: string;
+    subscriptionId: string | null;
+    urlCanonical: string;
+    reason: string;
+}): Promise<void>;

package/dist/cost.js

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Fetcher cost recording (chunk 9, Review #11).
+ *
+ * Per Review #11, every fetch writes a cost_records row. The cost_type
+ * discriminates the source ('fetch_tier1' for Patchright; 'fetch_tier0'
+ * later for HTTP-only Tier 0); the cost_subtype refines it (proxy
+ * provider name, or 'http_only' for no-proxy fetches).
+ *
+ * v1 uses a flat $0.0014 per Tier 1 fetch (Review #4 weighted average).
+ * Later chunks will refine this from actual proxy bytes + Patchright
+ * runtime once the production fetcher ships.
+ */
+import { costRecords } from '@bluefields/db';
+/** Per-fetch cost in micro-USD (Review #4: ~$0.0014 weighted average). */
+export const TIER1_FIXED_COST_MICRO_USD = 1_400;
+/** Compute-only row when a fetch is blocked (robots disallow). */
+export const COMPUTE_BLOCKED_COST_MICRO_USD = 50;
+/**
+ * Write a cost_records row for one fetch attempt.
+ *
+ * Defaults to Tier 1's flat cost with subtype='http_only' to reflect the
+ * v1 default fetcher. Override `costType` / `costSubtype` when you know
+ * (e.g. proxy_bandwidth row after a Patchright fetch, or 'fetch_304'
+ * when conditional GET returned 304 once chunk M8-10 wires that).
+ */
+export async function recordFetchCost(args) {
+    await args.db.insert(costRecords).values({
+        customerId: args.customerId,
+        subscriptionId: args.subscriptionId,
+        urlCanonical: args.urlCanonical,
+        costType: args.costType ?? 'fetch_tier1',
+        costSubtype: args.costSubtype ?? 'http_only',
+        costMicroUsd: args.costMicroUsd ?? TIER1_FIXED_COST_MICRO_USD,
+        units: args.units !== undefined ? String(args.units) : null,
+    });
+}
+/**
+ * Write a near-zero `compute` cost_records row for a fetch that was
+ * blocked before reaching the wire (e.g. robots.txt disallow). We still
+ * record the row so the customer can see the request in usage history.
+ */
+export async function recordBlockedFetch(args) {
+    await args.db.insert(costRecords).values({
+        customerId: args.customerId,
+        subscriptionId: args.subscriptionId,
+        urlCanonical: args.urlCanonical,
+        costType: 'compute',
+        costSubtype: args.reason,
+        costMicroUsd: COMPUTE_BLOCKED_COST_MICRO_USD,
+        units: null,
+    });
+}
+//# sourceMappingURL=cost.js.map

package/dist/cost.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.js","sourceRoot":"","sources":["../src/cost.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;GAWG;AAEH,OAAO,EAAiB,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE5D,0EAA0E;AAC1E,MAAM,CAAC,MAAM,0BAA0B,GAAG,KAAK,CAAC;AAEhD,kEAAkE;AAClE,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,CAAC;AAwBjD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAyB;IAC7D,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;QACvC,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,aAAa;QACxC,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,WAAW;QAC5C,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,0BAA0B;QAC7D,KAAK,EAAE,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;KAC5D,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAMxC;IACC,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;QACvC,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,QAAQ,EAAE,SAAS;QACnB,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,YAAY,EAAE,8BAA8B;QAC5C,KAAK,EAAE,IAAI;KACZ,CAAC,CAAC;AACL,CAAC"}

package/dist/fetcher-brightdata.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Bright Data Web Unlocker fetcher adapter (Tier 2).
+ *
+ * Tier 2 of the coverage ladder: a managed anti-bot "unlocker" endpoint
+ * that solves challenges (Cloudflare, PerimeterX, DataDome, …) server-side
+ * and returns the final HTML. Roughly 10-20x the per-request cost of the
+ * Tier-1 browser path, so it is escalation-only — Tier 0 (HTTP) and Tier 1
+ * (Patchright + residential proxy) run first; we only reach for the
+ * unlocker when a page is STILL shell/challenge/blocked.
+ *
+ * Env-gated exactly like the Patchright adapter: without BRIGHT_DATA_TOKEN
+ * + BRIGHT_DATA_ZONE, `tryCreateBrightDataAdapter` returns null and callers
+ * stay on the cheaper tiers.
+ *
+ * Fetch-only by design (v1): no page actions, no screenshots. The
+ * unlocker's /request endpoint returns the final HTML body; it does not
+ * expose the interactive page surface that actions/screenshots need.
+ * Callers passing `actions` get an explicit rejection (mirrors the HTTP
+ * adapter's stance on browser-only features). The unlocker also manages
+ * its own UA / fingerprint, so a caller-supplied `userAgent` does not reach
+ * the target — we report a managed-UA sentinel rather than pretend.
+ *
+ * Security: the TARGET url is SSRF-validated (HTTPS-only, no internal /
+ * metadata hosts) before we hand it to the unlocker — the managed egress
+ * must not become an open SSRF relay. The POST to api.brightdata.com itself
+ * goes through ssrfSafeFetch like every other external call.
+ */
+import type { FetcherAdapter } from './types.js';
+export declare const BRIGHT_DATA_REQUIRED_ENV: readonly ["BRIGHT_DATA_TOKEN", "BRIGHT_DATA_ZONE"];
+export interface BrightDataAdapterConfig {
+    /** Bright Data API token, sent as `Authorization: Bearer <token>`. */
+    token: string;
+    /** Web Unlocker zone name. */
+    zone: string;
+    /** Override the API endpoint (default https://api.brightdata.com/request). */
+    endpoint?: string;
+    /** Test seam: override the HTTP fetch. */
+    fetchImpl?: typeof fetch;
+    /** Hard cap (bytes) on the response body. Default `DEFAULT_MAX_BODY_BYTES` (5 MiB). */
+    maxBodyBytes?: number;
+}
+/**
+ * Returns the env-gated Bright Data adapter when BRIGHT_DATA_TOKEN +
+ * BRIGHT_DATA_ZONE are both set; null otherwise. Callers (the coverage
+ * runner) fall back to the lower tiers when this returns null.
+ */
+export declare function tryCreateBrightDataAdapter(env?: NodeJS.ProcessEnv): FetcherAdapter | null;
+/**
+ * Direct constructor — bypasses the env gate. Used by the factory above and
+ * by tests (which inject `fetchImpl`).
+ */
+export declare function createBrightDataAdapter(config: BrightDataAdapterConfig): FetcherAdapter;

package/dist/fetcher-brightdata.js

@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Bright Data Web Unlocker fetcher adapter (Tier 2).
+ *
+ * Tier 2 of the coverage ladder: a managed anti-bot "unlocker" endpoint
+ * that solves challenges (Cloudflare, PerimeterX, DataDome, …) server-side
+ * and returns the final HTML. Roughly 10-20x the per-request cost of the
+ * Tier-1 browser path, so it is escalation-only — Tier 0 (HTTP) and Tier 1
+ * (Patchright + residential proxy) run first; we only reach for the
+ * unlocker when a page is STILL shell/challenge/blocked.
+ *
+ * Env-gated exactly like the Patchright adapter: without BRIGHT_DATA_TOKEN
+ * + BRIGHT_DATA_ZONE, `tryCreateBrightDataAdapter` returns null and callers
+ * stay on the cheaper tiers.
+ *
+ * Fetch-only by design (v1): no page actions, no screenshots. The
+ * unlocker's /request endpoint returns the final HTML body; it does not
+ * expose the interactive page surface that actions/screenshots need.
+ * Callers passing `actions` get an explicit rejection (mirrors the HTTP
+ * adapter's stance on browser-only features). The unlocker also manages
+ * its own UA / fingerprint, so a caller-supplied `userAgent` does not reach
+ * the target — we report a managed-UA sentinel rather than pretend.
+ *
+ * Security: the TARGET url is SSRF-validated (HTTPS-only, no internal /
+ * metadata hosts) before we hand it to the unlocker — the managed egress
+ * must not become an open SSRF relay. The POST to api.brightdata.com itself
+ * goes through ssrfSafeFetch like every other external call.
+ */
+import { envInt, ssrfSafeFetch, validateUrlForExternalRequest } from '@bluefields/primitives';
+import { fetchWithRetry, readBodyCapped } from './fetcher-http.js';
+// The unlocker may sit on a challenge for tens of seconds while it solves
+// it server-side; 30s (the HTTP default) is too tight. Callers can still
+// override via options.timeoutMs (the coverage runner passes a longer one).
+const DEFAULT_TIMEOUT_MS = envInt('BLUEFIELDS_UNLOCKER_TIMEOUT_MS', 60_000);
+const DEFAULT_ENDPOINT = 'https://api.brightdata.com/request';
+const FETCHER_ID = 'brightdata-v1';
+/** The unlocker fetches from its own managed egress; we never see that IP. */
+const MANAGED_EGRESS_PLACEHOLDER_IP = '0.0.0.0';
+/** The unlocker picks + rotates its own UA; the caller's never reaches the target. */
+const MANAGED_UA = 'brightdata-web-unlocker (managed fingerprint)';
+/** Hard cap on the unlocker response body buffered into memory (mirrors tier-0). */
+const DEFAULT_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MiB
+export const BRIGHT_DATA_REQUIRED_ENV = ['BRIGHT_DATA_TOKEN', 'BRIGHT_DATA_ZONE'];
+/**
+ * Returns the env-gated Bright Data adapter when BRIGHT_DATA_TOKEN +
+ * BRIGHT_DATA_ZONE are both set; null otherwise. Callers (the coverage
+ * runner) fall back to the lower tiers when this returns null.
+ */
+export function tryCreateBrightDataAdapter(env = process.env) {
+    for (const key of BRIGHT_DATA_REQUIRED_ENV) {
+        if (!env[key] || env[key]?.length === 0)
+            return null;
+    }
+    return createBrightDataAdapter({
+        token: env.BRIGHT_DATA_TOKEN ?? '',
+        zone: env.BRIGHT_DATA_ZONE ?? '',
+        ...(env.BRIGHT_DATA_ENDPOINT ? { endpoint: env.BRIGHT_DATA_ENDPOINT } : {}),
+    });
+}
+/**
+ * Direct constructor — bypasses the env gate. Used by the factory above and
+ * by tests (which inject `fetchImpl`).
+ */
+export function createBrightDataAdapter(config) {
+    const endpoint = config.endpoint ?? DEFAULT_ENDPOINT;
+    // Tests inject config.fetchImpl; production uses ssrfSafeFetch (undici's
+    // fetch with the SSRF-safe dispatcher), which also validates that
+    // api.brightdata.com resolves to a public IP.
+    const fetchImpl = (config.fetchImpl ?? ssrfSafeFetch);
+    const maxBodyBytes = config.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES;
+    return {
+        async fetch(url, options = {}) {
+            if (options.actions && options.actions.length > 0) {
+                throw new Error('Bright Data unlocker is fetch-only; page actions require the Patchright fetcher');
+            }
+            // SSRF-validate the TARGET (not just our vendor endpoint) so the
+            // managed egress can't be steered at internal / metadata hosts.
+            await validateUrlForExternalRequest(url);
+            const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+            const now = options.nowFn ?? (() => new Date());
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(), timeoutMs);
+            const startedAt = Date.now();
+            const fetchedAt = now();
+            // `format: 'raw'` → the response body IS the target's HTML. Bright Data
+            // proxies the upstream status through res.status (a still-blocked or
+            // unsolvable page surfaces as a non-2xx, which the block-detector maps
+            // honestly rather than counting as success).
+            const requestBody = JSON.stringify({ zone: config.zone, url, format: 'raw' });
+            let res;
+            try {
+                res = await fetchWithRetry(fetchImpl, endpoint, {
+                    method: 'POST',
+                    headers: {
+                        authorization: `Bearer ${config.token}`,
+                        'content-type': 'application/json',
+                    },
+                    body: requestBody,
+                    signal: controller.signal,
+                    redirect: 'follow',
+                });
+            }
+            finally {
+                clearTimeout(timeout);
+            }
+            // Bound the body read by the request deadline (the unlocker can stream a
+            // large solved page slowly; res.text() after clearTimeout was unbounded).
+            const html = await readBodyCapped(res, maxBodyBytes, { deadlineMs: startedAt + timeoutMs });
+            const fetchDurationMs = Math.max(1, Date.now() - startedAt);
+            const responseHeaders = {};
+            res.headers.forEach((value, key) => {
+                responseHeaders[key.toLowerCase()] = value;
+            });
+            return {
+                html,
+                responseStatus: res.status,
+                responseHeaders,
+                fetchedAt,
+                fetchDurationMs,
+                fetcherId: FETCHER_ID,
+                // Managed egress: the real exit IP belongs to Bright Data and isn't
+                // exposed to us, so we report the same placeholder as the no-proxy path.
+                proxyEgressIp: MANAGED_EGRESS_PLACEHOLDER_IP,
+                userAgent: MANAGED_UA,
+                rawHtmlSizeBytes: Buffer.byteLength(html, 'utf8'),
+                // Fetch-only: the unlocker can't render-and-screenshot in this mode.
+                screenshot: null,
+                // Raw mode doesn't return a trustworthy post-redirect URL header;
+                // report the requested URL.
+                finalUrl: url,
+            };
+        },
+    };
+}
+//# sourceMappingURL=fetcher-brightdata.js.map

package/dist/fetcher-brightdata.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetcher-brightdata.js","sourceRoot":"","sources":["../src/fetcher-brightdata.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,6BAA6B,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAGnE,0EAA0E;AAC1E,yEAAyE;AACzE,4EAA4E;AAC5E,MAAM,kBAAkB,GAAG,MAAM,CAAC,gCAAgC,EAAE,MAAM,CAAC,CAAC;AAC5E,MAAM,gBAAgB,GAAG,oCAAoC,CAAC;AAC9D,MAAM,UAAU,GAAG,eAAe,CAAC;AACnC,8EAA8E;AAC9E,MAAM,6BAA6B,GAAG,SAAS,CAAC;AAChD,sFAAsF;AACtF,MAAM,UAAU,GAAG,+CAA+C,CAAC;AACnE,oFAAoF;AACpF,MAAM,sBAAsB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,QAAQ;AAExD,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,mBAAmB,EAAE,kBAAkB,CAAU,CAAC;AAe3F;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CACxC,MAAyB,OAAO,CAAC,GAAG;IAEpC,KAAK,MAAM,GAAG,IAAI,wBAAwB,EAAE,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;IACvD,CAAC;IACD,OAAO,uBAAuB,CAAC;QAC7B,KAAK,EAAE,GAAG,CAAC,iBAAiB,IAAI,EAAE;QAClC,IAAI,EAAE,GAAG,CAAC,gBAAgB,IAAI,EAAE;QAChC,GAAG,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5E,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAA+B;IACrE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,gBAAgB,CAAC;IACrD,yEAAyE;IACzE,kEAAkE;IAClE,8CAA8C;IAC9C,MAAM,SAAS,GAAiB,CAAC,MAAM,CAAC,SAAS,IAAI,aAAa,CAAiB,CAAC;IACpF,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,sBAAsB,CAAC;IAEnE,OAAO;QACL,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,UAAwB,EAAE;YACzC,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAC;YACJ,CAAC;YACD,iEAAiE;YACjE,gEAAgE;YAChE,MAAM,6BAA6B,CAAC,GAAG,CAAC,CAAC;YAEzC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;YAC1D,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAEhD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;YAChE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC;YAExB,wEAAwE;YACxE,qEAAqE;YACrE,uEAAuE;YACvE,6CAA6C;YAC7C,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAE9E,IAAI,GAAa,CAAC;YAClB,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE;oBAC9C,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,aAAa,EAAE,UAAU,MAAM,CAAC,KAAK,EAAE;wBACvC,cAAc,EAAE,kBAAkB;qBACnC;oBACD,IAAI,EAAE,WAAW;oBACjB,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,QAAQ,EAAE,QAAQ;iBACnB,CAAC,CAAC;YACL,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;YAED,yEAAyE;YACzE,0EAA0E;YAC1E,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,YAAY,EAAE,EAAE,UAAU,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC;YAC5F,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;YAE5D,MAAM,eAAe,GAA2B,EAAE,CAAC;YACnD,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBACjC,eAAe,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,GAAG,KAAK,CAAC;YAC7C,CAAC,CAAC,CAAC;YAEH,OAAO;gBACL,IAAI;gBACJ,cAAc,EAAE,GAAG,CAAC,MAAM;gBAC1B,eAAe;gBACf,SAAS;gBACT,eAAe;gBACf,SAAS,EAAE,UAAU;gBACrB,oEAAoE;gBACpE,yEAAyE;gBACzE,aAAa,EAAE,6BAA6B;gBAC5C,SAAS,EAAE,UAAU;gBACrB,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC;gBACjD,qEAAqE;gBACrE,UAAU,EAAE,IAAI;gBAChB,kEAAkE;gBAClE,4BAA4B;gBAC5B,QAAQ,EAAE,GAAG;aACQ,CAAC;QAC1B,CAAC;KACF,CAAC;AACJ,CAAC"}