jumpy-lion 0.1.6-beta.7 → 1.0.0-beta.0

package/dist/navigation/page-navigation.js

@@ -0,0 +1,122 @@
+/**
+ * PageNavigation - the deep module beside NetworkWatch/ElementTarget that owns
+ * the navigation + load-wait protocol: enable Page, fire navigate/reload, race
+ * `domContentEventFired` / `loadEventFired` against a timeout, then reconcile
+ * the landed URL via `getNavigationHistory` (NOT `Runtime.evaluate`, which is a
+ * CDP detection vector). It also caches the last reconciled URL so `url()` can
+ * answer without re-hitting CDP.
+ *
+ * Talks to the page only through the narrow `NavigationPort`, so the whole
+ * protocol is unit-testable with a scripted port - no `new CdpPage(...)` and no
+ * fake CDP client. `CdpPage.goto` / `reload` / `url` are thin delegates and the
+ * `loadedUrl` state lives here, not on the god object.
+ *
+ * The URL reconcile prefers `getNavigationHistory` (Page domain only) and falls
+ * back to `evaluateUrl` (Runtime) only when history is unavailable.
+ */
+import { log } from 'crawlee';
+const DEFAULT_TIMEOUT_MS = 30000;
+export class PageNavigation {
+    constructor(port) {
+        Object.defineProperty(this, "port", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "loadedUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        this.port = port;
+    }
+    /** Last reconciled URL, or - before any navigation - the live URL from CDP. */
+    async url() {
+        if (!this.loadedUrl) {
+            return this.currentUrl();
+        }
+        return this.loadedUrl;
+    }
+    /** Navigate to `url`, wait for the requested load milestone, reconcile the landed URL. */
+    async goto(url, options) {
+        await this.port.enablePage();
+        // Arm the load/domcontentloaded listener BEFORE navigating. The
+        // milestone events (esp. domcontentloaded) can fire in the microtask
+        // gap right after `navigate()` resolves; attaching the listener after
+        // navigate would miss them and hang until the timeout. Creating the
+        // wait promise first registers the CDP listener synchronously.
+        const waitForMilestone = this.waitForLoad(options, 'Navigation timeout');
+        waitForMilestone.catch(() => { }); // pre-handle so a navigate() throw can't orphan it
+        await this.port.navigate(url);
+        await waitForMilestone;
+        this.loadedUrl = await this.currentUrl();
+    }
+    /** Reload the current page, wait for the requested load milestone, reconcile the landed URL. */
+    async reload(options) {
+        await this.port.enablePage();
+        const waitForMilestone = this.waitForLoad(options, 'Reload timeout');
+        waitForMilestone.catch(() => { }); // pre-handle so a reload() throw can't orphan it
+        await this.port.reload();
+        await waitForMilestone;
+        this.loadedUrl = await this.currentUrl();
+    }
+    /**
+     * Race the requested load event against the timeout. Resolves when the
+     * event fires; rejects with `timeoutMessage` once the timeout elapses.
+     */
+    async waitForLoad(options, timeoutMessage) {
+        const timeout = options?.timeout ?? DEFAULT_TIMEOUT_MS;
+        const waitUntil = options?.waitUntil ?? 'load';
+        if (waitUntil !== 'load' && waitUntil !== 'domcontentloaded') {
+            throw new Error(`Unsupported waitUntil event: ${waitUntil}`);
+        }
+        // The CDP event subscription returns an unsubscribe fn. We MUST call it
+        // on every outcome - especially on timeout, where the milestone never
+        // fires. Leaving it attached leaks one Page.loadEventFired /
+        // domContentEventFired listener on the CDP client per timed-out
+        // navigation, which on a stuck crawl piles up into a
+        // MaxListenersExceededWarning and unbounded memory growth.
+        let unsubscribe;
+        const waitEvent = new Promise((resolve) => {
+            unsubscribe = waitUntil === 'load'
+                ? this.port.onLoad(() => resolve())
+                : this.port.onDomContent(() => resolve());
+        });
+        let timer;
+        const timeoutPromise = new Promise((_, reject) => {
+            timer = setTimeout(() => reject(new Error(timeoutMessage)), timeout);
+        });
+        try {
+            await Promise.race([waitEvent, timeoutPromise]);
+        }
+        finally {
+            // Clear the timer on whichever outcome wins so the timeout closure
+            // does not dangle for ~timeout-length after the event resolves.
+            if (timer)
+                clearTimeout(timer);
+            // Detach the milestone listener so it cannot accumulate.
+            unsubscribe?.();
+        }
+    }
+    /**
+     * Resolve the current URL without activating the Runtime domain when
+     * possible: `getNavigationHistory` needs only the Page domain. Falls back
+     * to a Runtime evaluation only when history is empty or unavailable.
+     */
+    async currentUrl() {
+        try {
+            await this.port.enablePage();
+            const { currentIndex, entries } = await this.port.navigationHistory();
+            if (entries && entries.length > 0 && currentIndex >= 0) {
+                return entries[currentIndex].url;
+            }
+        }
+        catch {
+            log.debug('getNavigationHistory failed, falling back to evaluate');
+        }
+        return this.port.evaluateUrl();
+    }
+}
+//# sourceMappingURL=page-navigation.js.map

package/dist/navigation/page-navigation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-navigation.js","sourceRoot":"","sources":["../../src/navigation/page-navigation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,GAAG,EAAE,MAAM,SAAS,CAAC;AA+B9B,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAElC,MAAM,OAAO,cAAc;IAIvB,YAAY,IAAoB;QAHf;;;;;WAAqB;QAC9B;;;;mBAA2B,IAAI;WAAC;QAGpC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,+EAA+E;IAC/E,KAAK,CAAC,GAAG;QACL,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;QAC7B,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAED,0FAA0F;IAC1F,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,OAAqB;QACzC,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;QAC7B,gEAAgE;QAChE,qEAAqE;QACrE,sEAAsE;QACtE,oEAAoE;QACpE,+DAA+D;QAC/D,MAAM,gBAAgB,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;QACzE,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,mDAAmD;QACrF,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,gBAAgB,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;IAC7C,CAAC;IAED,gGAAgG;IAChG,KAAK,CAAC,MAAM,CAAC,OAAqB;QAC9B,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;QAC7B,MAAM,gBAAgB,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QACrE,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,iDAAiD;QACnF,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,gBAAgB,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,WAAW,CAAC,OAAgC,EAAE,cAAsB;QAC9E,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,kBAAkB,CAAC;QACvD,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,MAAM,CAAC;QAE/C,IAAI,SAAS,KAAK,MAAM,IAAI,SAAS,KAAK,kBAAkB,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,wEAAwE;QACxE,sEAAsE;QACtE,6DAA6D;QAC7D,gEAAgE;QAChE,qDAAqD;QACrD,2DAA2D;QAC3D,IAAI,WAAqC,CAAC;QAC1C,MAAM,SAAS,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5C,WAAW,GAAG,SAAS,KAAK,MAAM;gBAC9B,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;gBACnC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,IAAI,KAAgD,CAAC;QACrD,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACpD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACzE,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACD,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC;QACpD,CAAC;gBAAS,CAAC;YACP,mEAAmE;YACnE,gEAAgE;YAChE,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,yDAAyD;YACzD,WAAW,EAAE,EAAE,CAAC;QACpB,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,UAAU;QACpB,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAC7B,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACtE,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;gBACrD,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC,GAAG,CAAC;YACrC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACL,GAAG,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACnC,CAAC;CACJ"}

package/dist/network/network-watch.d.ts

@@ -0,0 +1,110 @@
+/**
+ * NetworkWatch - the deep module beside ElementTarget/DropdownSelector that
+ * owns the network interception protocol: response waiting (match, timeout,
+ * body fetch with backoff, listener cleanup), request blocking, and
+ * fetch-metadata header repair.
+ *
+ * Talks to the page only through the narrow `NetworkWatchPort`, so the whole
+ * protocol is unit-testable with a scripted port. `CdpPage.waitForResponse`,
+ * `CdpPage.blockRequests`, and `CdpPage.setupFetchMetadataHeaders` are thin
+ * delegates.
+ */
+export interface WatchedResponse {
+    url: string;
+    status: number;
+    [key: string]: unknown;
+}
+/** A matched response with its (JSON-parsed, or raw-string) body attached. */
+export type WatchedResponseWithBody = WatchedResponse & {
+    body: unknown;
+};
+export interface InterceptedRequest {
+    interceptionId: string;
+    request: {
+        url: string;
+        method: string;
+        headers: Record<string, string>;
+    };
+}
+export interface NetworkWatchPort {
+    /** Lazily enable the Network domain (a CDP detection-surface concern owned by the page). */
+    ensureNetworkEnabled(): Promise<void>;
+    /** Subscribe to responseReceived; returns the unsubscribe function. */
+    onResponseReceived(handler: (params: {
+        requestId: string;
+        response: WatchedResponse;
+    }) => void): () => void;
+    getResponseBody(requestId: string): Promise<{
+        body: string;
+    }>;
+    setRequestInterception(patterns: {
+        urlPattern: string;
+    }[]): Promise<void>;
+    onRequestIntercepted(handler: (params: InterceptedRequest) => Promise<void>): void;
+    continueInterceptedRequest(params: {
+        interceptionId: string;
+        headers?: Record<string, string>;
+        errorReason?: string;
+    }): Promise<void>;
+}
+export interface NetworkWatchOptions {
+    /** Backoff before body-fetch retry `attempt` (1-based). Injectable so tests run instantly. */
+    backoffMs?: (attempt: number) => number;
+}
+/**
+ * Compile glob-ish patterns (`*` = wildcard) to RegExp once, so the hot matcher
+ * never rebuilds them.
+ *
+ * Every character except `*` is escaped first. URLs are full of regex
+ * metacharacters, so treating the pattern as a raw regex was both wrong and
+ * crash-prone: a pattern containing `ads(1).js` threw
+ * `SyntaxError: Unterminated group`, and a plain `?`/`+`/`.` silently changed
+ * what matched - an unescaped `.` matches any character, so a pattern for
+ * `.google-analytics.com` also matched `xgoogle-analytics.com`.
+ * Same bug class as the injected-script template double-escaping fix.
+ */
+export declare function compilePatterns(urlPatterns: string[]): RegExp[];
+/** True when the URL matches any precompiled pattern. */
+export declare function matchesAnyCompiled(url: string, regexes: RegExp[]): boolean;
+/** True when the URL matches any glob-ish pattern (`*` = wildcard). */
+export declare function matchesAnyPattern(url: string, urlPatterns: string[]): boolean;
+/**
+ * Repair missing sec-fetch-* headers based on request type and destination.
+ * Pure: returns the headers to send.
+ */
+export declare function withFetchMetadataHeaders(request: InterceptedRequest['request']): Record<string, string>;
+export declare class NetworkWatch {
+    private readonly port;
+    private readonly backoffMs;
+    /** Compiled block patterns, or null when `blockRequests` was never called. */
+    private blockedPatterns;
+    /** Whether `setupFetchMetadataHeaders` asked for sec-fetch-* repair. */
+    private repairFetchMetadata;
+    /** Guards against installing a second interception handler. */
+    private interceptionInstalled;
+    constructor(port: NetworkWatchPort, options?: NetworkWatchOptions);
+    /**
+     * Resolve with the first response whose URL contains `urlPart` and whose
+     * status matches, with its JSON body attached. The listener is removed on
+     * match AND on timeout - a long-lived page must not accumulate handlers.
+     */
+    waitForResponse(urlPart: string, statusCode?: number, timeoutMs?: number): Promise<WatchedResponseWithBody>;
+    /** Abort every request whose URL matches one of the glob-ish patterns. */
+    blockRequests(urlPatterns: string[]): Promise<void>;
+    /** Add proper Fetch Metadata (sec-fetch-*) headers to avoid CDP detection. */
+    setupFetchMetadataHeaders(): Promise<void>;
+    /**
+     * Install the SINGLE request-interception handler, at most once per page.
+     *
+     * Both interception features (blocking, fetch-metadata repair) route
+     * through here because CDP allows exactly one continue per
+     * `interceptionId`. Registering a handler per feature meant that calling
+     * both `blockRequests` and `setupFetchMetadataHeaders` on one page made two
+     * handlers race to continue the same request - the second call fails with
+     * `Invalid InterceptionId`, and the second `setRequestInterception` silently
+     * replaced the first's patterns. One handler consults both pieces of state
+     * instead, so the features compose in either call order.
+     */
+    private ensureInterception;
+}
+//# sourceMappingURL=network-watch.d.ts.map

package/dist/network/network-watch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"network-watch.d.ts","sourceRoot":"","sources":["../../src/network/network-watch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,eAAe;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,8EAA8E;AAC9E,MAAM,MAAM,uBAAuB,GAAG,eAAe,GAAG;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,CAAC;AAE1E,MAAM,WAAW,kBAAkB;IAC/B,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE;QACL,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACnC,CAAC;CACL;AAED,MAAM,WAAW,gBAAgB;IAC7B,4FAA4F;IAC5F,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,uEAAuE;IACvE,kBAAkB,CACd,OAAO,EAAE,CAAC,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,eAAe,CAAA;KAAE,KAAK,IAAI,GAC5E,MAAM,IAAI,CAAC;IACd,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9D,sBAAsB,CAAC,QAAQ,EAAE;QAAE,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1E,oBAAoB,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,kBAAkB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACnF,0BAA0B,CAAC,MAAM,EAAE;QAC/B,cAAc,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACjC,WAAW,CAAC,EAAE,MAAM,CAAC;KACxB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrB;AAED,MAAM,WAAW,mBAAmB;IAChC,8FAA8F;IAC9F,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAC;CAC3C;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAS/D;AAED,yDAAyD;AACzD,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAE1E;AAED,uEAAuE;AACvE,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAE7E;AAWD;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,kBAAkB,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CA0BvG;AAED,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAmB;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA8B;IAGxD,8EAA8E;IAC9E,OAAO,CAAC,eAAe,CAAyB;IAChD,wEAAwE;IACxE,OAAO,CAAC,mBAAmB,CAAS;IACpC,+DAA+D;IAC/D,OAAO,CAAC,qBAAqB,CAAS;gBAE1B,IAAI,EAAE,gBAAgB,EAAE,OAAO,GAAE,mBAAwB;IAKrE;;;;OAIG;IACG,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,SAAM,EAAE,SAAS,SAAU,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAmD/G,0EAA0E;IACpE,aAAa,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAOzD,8EAA8E;IACxE,yBAAyB,IAAI,OAAO,CAAC,IAAI,CAAC;IAKhD;;;;;;;;;;;OAWG;YACW,kBAAkB;CAwBnC"}

package/dist/network/network-watch.js

@@ -0,0 +1,223 @@
+/**
+ * NetworkWatch - the deep module beside ElementTarget/DropdownSelector that
+ * owns the network interception protocol: response waiting (match, timeout,
+ * body fetch with backoff, listener cleanup), request blocking, and
+ * fetch-metadata header repair.
+ *
+ * Talks to the page only through the narrow `NetworkWatchPort`, so the whole
+ * protocol is unit-testable with a scripted port. `CdpPage.waitForResponse`,
+ * `CdpPage.blockRequests`, and `CdpPage.setupFetchMetadataHeaders` are thin
+ * delegates.
+ */
+/**
+ * Compile glob-ish patterns (`*` = wildcard) to RegExp once, so the hot matcher
+ * never rebuilds them.
+ *
+ * Every character except `*` is escaped first. URLs are full of regex
+ * metacharacters, so treating the pattern as a raw regex was both wrong and
+ * crash-prone: a pattern containing `ads(1).js` threw
+ * `SyntaxError: Unterminated group`, and a plain `?`/`+`/`.` silently changed
+ * what matched - an unescaped `.` matches any character, so a pattern for
+ * `.google-analytics.com` also matched `xgoogle-analytics.com`.
+ * Same bug class as the injected-script template double-escaping fix.
+ */
+export function compilePatterns(urlPatterns) {
+    return urlPatterns.map((pattern) => {
+        const source = pattern
+            // Escape every regex metacharacter, `*` included...
+            .replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+            // ...then re-expand the escaped `\*` back into the wildcard.
+            .replace(/\\\*/g, '.*');
+        return new RegExp(source);
+    });
+}
+/** True when the URL matches any precompiled pattern. */
+export function matchesAnyCompiled(url, regexes) {
+    return regexes.some((regex) => regex.test(url));
+}
+/** True when the URL matches any glob-ish pattern (`*` = wildcard). */
+export function matchesAnyPattern(url, urlPatterns) {
+    return matchesAnyCompiled(url, compilePatterns(urlPatterns));
+}
+/** Parse a response body as JSON, falling back to the raw string when it is not JSON. */
+function parseResponseBody(body) {
+    try {
+        return JSON.parse(body);
+    }
+    catch {
+        return body;
+    }
+}
+/**
+ * Repair missing sec-fetch-* headers based on request type and destination.
+ * Pure: returns the headers to send.
+ */
+export function withFetchMetadataHeaders(request) {
+    const url = new URL(request.url);
+    const headers = { ...request.headers };
+    if (request.method === 'GET' && !headers['sec-fetch-dest']) {
+        if (url.pathname.endsWith('.js')) {
+            headers['sec-fetch-dest'] = 'script';
+            headers['sec-fetch-mode'] = 'no-cors';
+            headers['sec-fetch-site'] = url.origin === headers.origin ? 'same-origin' : 'cross-site';
+        }
+        else if (url.pathname.endsWith('.css')) {
+            headers['sec-fetch-dest'] = 'style';
+            headers['sec-fetch-mode'] = 'no-cors';
+            headers['sec-fetch-site'] = url.origin === headers.origin ? 'same-origin' : 'cross-site';
+        }
+        else if (url.pathname.match(/\.(png|jpg|jpeg|gif|webp|svg)$/i)) {
+            headers['sec-fetch-dest'] = 'image';
+            headers['sec-fetch-mode'] = 'no-cors';
+            headers['sec-fetch-site'] = url.origin === headers.origin ? 'same-origin' : 'cross-site';
+        }
+        else if (url.pathname === '/' || !url.pathname.includes('.')) {
+            headers['sec-fetch-dest'] = 'document';
+            headers['sec-fetch-mode'] = 'navigate';
+            headers['sec-fetch-site'] = 'none';
+            headers['sec-fetch-user'] = '?1';
+        }
+    }
+    return headers;
+}
+export class NetworkWatch {
+    constructor(port, options = {}) {
+        Object.defineProperty(this, "port", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "backoffMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // ── request-interception state, read by the single shared handler ──
+        /** Compiled block patterns, or null when `blockRequests` was never called. */
+        Object.defineProperty(this, "blockedPatterns", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        /** Whether `setupFetchMetadataHeaders` asked for sec-fetch-* repair. */
+        Object.defineProperty(this, "repairFetchMetadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        /** Guards against installing a second interception handler. */
+        Object.defineProperty(this, "interceptionInstalled", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        this.port = port;
+        this.backoffMs = options.backoffMs ?? ((attempt) => 1000 * 2 ** attempt);
+    }
+    /**
+     * Resolve with the first response whose URL contains `urlPart` and whose
+     * status matches, with its JSON body attached. The listener is removed on
+     * match AND on timeout - a long-lived page must not accumulate handlers.
+     */
+    async waitForResponse(urlPart, statusCode = 200, timeoutMs = 180000) {
+        await this.port.ensureNetworkEnabled(); // Lazy enable Network only when needed
+        return new Promise((resolve, reject) => {
+            let unsubscribe = () => { };
+            const timer = setTimeout(() => {
+                unsubscribe();
+                reject(new Error(`Timeout after ${timeoutMs / 1000} seconds`));
+            }, timeoutMs);
+            const onResponseReceived = ({ requestId, response }) => {
+                if (!(response.url.includes(urlPart) && response.status === statusCode))
+                    return;
+                clearTimeout(timer);
+                unsubscribe();
+                void (async () => {
+                    let body;
+                    try {
+                        ({ body } = await this.port.getResponseBody(requestId));
+                    }
+                    catch {
+                        // Retry with exponential backoff only for genuine
+                        // transport failures: the body may not be ready yet, or
+                        // the Network domain dropped mid-flight. A non-JSON body
+                        // is NOT a transient error and never reaches this loop.
+                        for (let attempt = 1; attempt <= 3; attempt++) {
+                            try {
+                                await this.port.ensureNetworkEnabled();
+                                await new Promise((resolveTimeout) => {
+                                    setTimeout(resolveTimeout, this.backoffMs(attempt));
+                                });
+                                const retried = await this.port.getResponseBody(requestId);
+                                resolve({ ...response, body: parseResponseBody(retried.body) });
+                                return;
+                            }
+                            catch {
+                                if (attempt === 3)
+                                    reject(new Error(`Failed to get response body after ${attempt} attempts`));
+                            }
+                        }
+                        return;
+                    }
+                    // Transport succeeded: parse JSON, or resolve the raw string
+                    // immediately when the body is not JSON.
+                    resolve({ ...response, body: parseResponseBody(body) });
+                })();
+            };
+            unsubscribe = this.port.onResponseReceived(onResponseReceived);
+        });
+    }
+    /** Abort every request whose URL matches one of the glob-ish patterns. */
+    async blockRequests(urlPatterns) {
+        // Compile once; the interception handler reads the compiled array so no
+        // RegExp is rebuilt on the hot path.
+        this.blockedPatterns = compilePatterns(urlPatterns);
+        await this.ensureInterception();
+    }
+    /** Add proper Fetch Metadata (sec-fetch-*) headers to avoid CDP detection. */
+    async setupFetchMetadataHeaders() {
+        this.repairFetchMetadata = true;
+        await this.ensureInterception();
+    }
+    /**
+     * Install the SINGLE request-interception handler, at most once per page.
+     *
+     * Both interception features (blocking, fetch-metadata repair) route
+     * through here because CDP allows exactly one continue per
+     * `interceptionId`. Registering a handler per feature meant that calling
+     * both `blockRequests` and `setupFetchMetadataHeaders` on one page made two
+     * handlers race to continue the same request - the second call fails with
+     * `Invalid InterceptionId`, and the second `setRequestInterception` silently
+     * replaced the first's patterns. One handler consults both pieces of state
+     * instead, so the features compose in either call order.
+     */
+    async ensureInterception() {
+        await this.port.ensureNetworkEnabled(); // Lazy enable Network only when needed
+        if (this.interceptionInstalled)
+            return;
+        this.interceptionInstalled = true;
+        await this.port.setRequestInterception([{ urlPattern: '*' }]);
+        this.port.onRequestIntercepted(async (params) => {
+            // Blocking wins: an aborted request must not also be continued.
+            if (this.blockedPatterns && matchesAnyCompiled(params.request.url, this.blockedPatterns)) {
+                await this.port.continueInterceptedRequest({
+                    interceptionId: params.interceptionId,
+                    errorReason: 'Aborted',
+                });
+                return;
+            }
+            await this.port.continueInterceptedRequest({
+                interceptionId: params.interceptionId,
+                ...(this.repairFetchMetadata
+                    ? { headers: withFetchMetadataHeaders(params.request) }
+                    : {}),
+            });
+        });
+    }
+}
+//# sourceMappingURL=network-watch.js.map

package/dist/network/network-watch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"network-watch.js","sourceRoot":"","sources":["../../src/network/network-watch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AA0CH;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,eAAe,CAAC,WAAqB;IACjD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC/B,MAAM,MAAM,GAAG,OAAO;YAClB,oDAAoD;aACnD,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC;YACvC,6DAA6D;aAC5D,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC5B,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;AACP,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAAE,OAAiB;IAC7D,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,iBAAiB,CAAC,GAAW,EAAE,WAAqB;IAChE,OAAO,kBAAkB,CAAC,GAAG,EAAE,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,yFAAyF;AACzF,SAAS,iBAAiB,CAAC,IAAY;IACnC,IAAI,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC;IAChB,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAAsC;IAC3E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAEvC,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACzD,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,gBAAgB,CAAC,GAAG,QAAQ,CAAC;YACrC,OAAO,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;YACtC,OAAO,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC;QAC7F,CAAC;aAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,gBAAgB,CAAC,GAAG,OAAO,CAAC;YACpC,OAAO,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;YACtC,OAAO,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC;QAC7F,CAAC;aAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,iCAAiC,CAAC,EAAE,CAAC;YAC/D,OAAO,CAAC,gBAAgB,CAAC,GAAG,OAAO,CAAC;YACpC,OAAO,CAAC,gBAAgB,CAAC,GAAG,SAAS,CAAC;YACtC,OAAO,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC;QAC7F,CAAC;aAAM,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,gBAAgB,CAAC,GAAG,UAAU,CAAC;YACvC,OAAO,CAAC,gBAAgB,CAAC,GAAG,UAAU,CAAC;YACvC,OAAO,CAAC,gBAAgB,CAAC,GAAG,MAAM,CAAC;YACnC,OAAO,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QACrC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,MAAM,OAAO,YAAY;IAYrB,YAAY,IAAsB,EAAE,UAA+B,EAAE;QAXpD;;;;;WAAuB;QACvB;;;;;WAAuC;QAExD,sEAAsE;QACtE,8EAA8E;QACtE;;;;mBAAmC,IAAI;WAAC;QAChD,wEAAwE;QAChE;;;;mBAAsB,KAAK;WAAC;QACpC,+DAA+D;QACvD;;;;mBAAwB,KAAK;WAAC;QAGlC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC;IAC7E,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,eAAe,CAAC,OAAe,EAAE,UAAU,GAAG,GAAG,EAAE,SAAS,GAAG,MAAO;QACxE,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC,uCAAuC;QAE/E,OAAO,IAAI,OAAO,CAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5D,IAAI,WAAW,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;YAEvC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC1B,WAAW,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,KAAK,CAAC,iBAAiB,SAAS,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC;YACnE,CAAC,EAAE,SAAS,CAAC,CAAC;YAEd,MAAM,kBAAkB,GAAG,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAoD,EAAQ,EAAE;gBAC3G,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,CAAC;oBAAE,OAAO;gBAEhF,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,WAAW,EAAE,CAAC;gBAEd,KAAK,CAAC,KAAK,IAAI,EAAE;oBACb,IAAI,IAAY,CAAC;oBACjB,IAAI,CAAC;wBACD,CAAC,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC;oBAC5D,CAAC;oBAAC,MAAM,CAAC;wBACL,kDAAkD;wBAClD,wDAAwD;wBACxD,yDAAyD;wBACzD,wDAAwD;wBACxD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC;4BAC5C,IAAI,CAAC;gCACD,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;gCACvC,MAAM,IAAI,OAAO,CAAC,CAAC,cAAc,EAAE,EAAE;oCACjC,UAAU,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;gCACxD,CAAC,CAAC,CAAC;gCACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;gCAC3D,OAAO,CAAC,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gCAChE,OAAO;4BACX,CAAC;4BAAC,MAAM,CAAC;gCACL,IAAI,OAAO,KAAK,CAAC;oCAAE,MAAM,CAAC,IAAI,KAAK,CAAC,qCAAqC,OAAO,WAAW,CAAC,CAAC,CAAC;4BAClG,CAAC;wBACL,CAAC;wBACD,OAAO;oBACX,CAAC;oBACD,6DAA6D;oBAC7D,yCAAyC;oBACzC,OAAO,CAAC,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC5D,CAAC,CAAC,EAAE,CAAC;YACT,CAAC,CAAC;YAEF,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;IACP,CAAC;IAED,0EAA0E;IAC1E,KAAK,CAAC,aAAa,CAAC,WAAqB;QACrC,wEAAwE;QACxE,qCAAqC;QACrC,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;QACpD,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;IACpC,CAAC;IAED,8EAA8E;IAC9E,KAAK,CAAC,yBAAyB;QAC3B,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;OAWG;IACK,KAAK,CAAC,kBAAkB;QAC5B,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC,CAAC,uCAAuC;QAC/E,IAAI,IAAI,CAAC,qBAAqB;YAAE,OAAO;QACvC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAElC,MAAM,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAE9D,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAC5C,gEAAgE;YAChE,IAAI,IAAI,CAAC,eAAe,IAAI,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvF,MAAM,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC;oBACvC,cAAc,EAAE,MAAM,CAAC,cAAc;oBACrC,WAAW,EAAE,SAAS;iBACzB,CAAC,CAAC;gBACH,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC;gBACvC,cAAc,EAAE,MAAM,CAAC,cAAc;gBACrC,GAAG,CAAC,IAAI,CAAC,mBAAmB;oBACxB,CAAC,CAAC,EAAE,OAAO,EAAE,wBAAwB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;oBACvD,CAAC,CAAC,EAAE,CAAC;aACZ,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;CACJ"}