@crawlee/browser 4.0.0-beta.14 → 4.0.0-beta.141

package/internals/browser-crawler.js

@@ -1,8 +1,35 @@
-import { BasicCrawler, BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES, Configuration, ContextPipeline, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, handleRequestTimeout, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, tryAbsoluteURL, validators, } from '@crawlee/basic';
-import { BrowserPool } from '@crawlee/browser-pool';
-import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
-import ow from 'ow';
-import { tryCancel } from '@apify/timeout';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
+import { CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
+import { sleep } from '@crawlee/utils';
+import { z } from 'zod';
+import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
+/**
+ * Rejects options that exist only to configure the browser pool the crawler would have built for itself.
+ * Accepting them alongside a pre-built `browserPool` and quietly ignoring them is how `browserPoolOptions` grew
+ * into a second, half-working way of configuring the same pool.
+ */
+export function assertBrowserPoolNotConfigured(crawlerName, ignoredOptions) {
+    const names = Object.keys(ignoredOptions).filter((name) => ignoredOptions[name] !== undefined);
+    if (names.length === 0) {
+        return;
+    }
+    throw new Error(`${crawlerName}: ${names.map((name) => `\`${name}\``).join(', ')} cannot be combined with \`browserPool\`, ` +
+        `${names.length > 1 ? 'they configure' : 'it configures'} the browser pool the crawler would build for ` +
+        'itself. Configure the pool you pass in instead.');
+}
+const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
+const readContextField = (ctx, key) => ctx[key];
+/**
+ * Whether an error thrown by `page.goto()` is a navigation timeout - either our own {@link TimeoutError}
+ * or the driver's, which Playwright/Puppeteer report with their own class and a `Timeout ... exceeded` message
+ * naming the raw millisecond value rather than the configured window.
+ */
+function isNavigationTimeoutError(error) {
+    return (error instanceof TimeoutError ||
+        error?.name === 'TimeoutError' ||
+        error?.constructor?.name === 'TimeoutError' ||
+        /timeout.*exceeded/i.test(error?.message ?? ''));
+}
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -15,115 +42,138 @@ import { tryCancel } from '@apify/timeout';
  * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
- * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
+ * The source URLs are represented by the {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
  * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
  * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
  *
- * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
+ * combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
+ * and pass the result as `requestManager`.
+ *
+ * > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
  * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
  *
- * New pages are only opened when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
- * parameter of the `BrowserCrawler` constructor.
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
- * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
  *
  * @category Crawlers
  */
 export class BrowserCrawler extends BasicCrawler {
-    config;
+    /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
+    #browserPoolDep;
     /**
-     * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
+     * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
+     * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
-    browserPool;
+    get browserPool() {
+        return this.#browserPoolDep.value;
+    }
     launchContext;
     ignoreShadowRoots;
     ignoreIframes;
-    navigationTimeoutMillis;
-    preNavigationHooks;
-    postNavigationHooks;
-    persistCookiesPerSession;
+    #navigationTimeoutMillis;
+    #preNavigationHooks;
+    #postNavigationHooks;
+    #saveResponseCookies;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BasicCrawler.optionsShape,
-        navigationTimeoutSecs: ow.optional.number.greaterThan(0),
-        preNavigationHooks: ow.optional.array,
-        postNavigationHooks: ow.optional.array,
-        launchContext: ow.optional.object,
-        headless: ow.optional.any(ow.boolean, ow.string),
-        browserPoolOptions: ow.object,
-        sessionPoolOptions: ow.optional.object,
-        persistCookiesPerSession: ow.optional.boolean,
-        useSessionPool: ow.optional.boolean,
-        proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
+        navigationTimeoutSecs: schemas.anyNumber
+            .refine((value) => value > 0, 'Expected a number greater than 0')
+            .default(60),
+        preNavigationHooks: schemas.anyArray.default(() => []),
+        postNavigationHooks: schemas.anyArray.default(() => []),
+        launchContext: schemas.anyObject.default(() => ({})),
+        browserPool: validators.browserPool.optional(),
+        browserPoolBuilder: schemas.anyFunction.optional(),
+        remoteBrowser: schemas.anyObject.optional(),
+        saveResponseCookies: z.boolean().default(true),
+        proxyConfiguration: validators.proxyConfiguration.optional(),
+        ignoreIframes: z.boolean().default(false),
+        ignoreShadowRoots: z.boolean().default(false),
     };
+    /** @internal */
+    static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
      */
-    constructor(options, config = Configuration.getGlobalConfig()) {
-        ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
-        const { navigationTimeoutSecs = 60, persistCookiesPerSession, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, proxyConfiguration, ...basicCrawlerOptions } = options;
+    constructor(options) {
+        const { navigationTimeoutSecs, saveResponseCookies, launchContext, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
+        if (browserPool) {
+            assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
+        }
+        const skipGuard = (action) => ({
+            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
+        });
         super({
             ...basicCrawlerOptions,
-            contextPipelineBuilder: () => contextPipelineBuilder()
-                .compose({ action: this.performNavigation.bind(this) })
-                .compose({ action: this.handleBlockedRequestByContent.bind(this) })
-                .compose({ action: this.restoreRequestState.bind(this) }),
-            extendContext: extendContext,
-        }, config);
-        this.config = config;
-        // Cookies should be persisted per session only if session pool is used
-        if (!this.useSessionPool && persistCookiesPerSession) {
-            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
-        }
+            contextPipelineBuilder: () => {
+                // A single navigation window covers the pre-navigation hooks, the navigation, and the
+                // post-navigation hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow
+                // hook eats into the same window the navigation uses. The navigation itself is bounded by
+                // capping its `gotoOptions.timeout` to the remaining budget.
+                const windowGuard = (step) => skipGuard(async (ctx) => {
+                    const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
+                    if (remaining <= 0) {
+                        throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+                    }
+                    return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+                });
+                let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
+                for (const hook of this.#preNavigationHooks) {
+                    pipeline = pipeline.compose(windowGuard(hook));
+                }
+                pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
+                for (const hook of this.#postNavigationHooks) {
+                    pipeline = pipeline.compose(windowGuard(hook));
+                }
+                return pipeline
+                    .compose(skipGuard(this.finalizeNavigation.bind(this)))
+                    .compose({ action: this.handleBlockedRequestByContent.bind(this) })
+                    .compose({ action: this.restoreRequestState.bind(this) });
+            },
+            extendContext,
+        });
         this.launchContext = launchContext;
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.proxyConfiguration = proxyConfiguration;
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = postNavigationHooks;
+        this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        // The public option hooks are extension-aware; internal storage uses the base context type
+        // (the pipeline composes hooks against the concrete context, which does not statically carry
+        // `ContextExtension`). The extension members are present at runtime regardless.
+        this.#preNavigationHooks = preNavigationHooks;
+        this.#postNavigationHooks = postNavigationHooks;
         this.ignoreIframes = ignoreIframes;
         this.ignoreShadowRoots = ignoreShadowRoots;
-        if (headless != null) {
-            this.launchContext.launchOptions ??= {};
-            this.launchContext.launchOptions.headless = headless;
-        }
-        if (this.useSessionPool) {
-            this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
-        }
-        else {
-            this.persistCookiesPerSession = false;
-        }
-        if (launchContext?.userAgent) {
-            if (browserPoolOptions.useFingerprints)
-                this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
-            browserPoolOptions.useFingerprints = false;
-        }
-        const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
-        this.browserPool = new BrowserPool({
-            ...rest,
-            preLaunchHooks: [this._extendLaunchContext.bind(this), ...preLaunchHooks],
-            postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
-        });
+        this.#saveResponseCookies = saveResponseCookies;
+        this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));
+    }
+    getNavigationTimeoutMillis() {
+        return this.#navigationTimeoutMillis;
     }
     buildContextPipeline() {
         return ContextPipeline.create().compose({
             action: this.preparePage.bind(this),
             cleanup: async (context) => {
                 context.registerDeferredCleanup(async () => {
-                    await context.page
-                        .close()
-                        .catch((error) => this.log.debug('Error while closing page', { error }));
+                    const error = !context.session.isUsable()
+                        ? new SessionError('Session is no longer usable')
+                        : undefined;
+                    await this.browserPool
+                        .closePage(context.page, { error })
+                        .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
                 });
             },
         });
@@ -137,12 +187,6 @@ export class BrowserCrawler extends BasicCrawler {
     }
     async isRequestBlocked(crawlingContext) {
         const { page, response } = crawlingContext;
-        const blockedStatusCodes = 
-        // eslint-disable-next-line dot-notation
-        (this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
-            ? // eslint-disable-next-line dot-notation
-                this.sessionPool['blockedStatusCodes']
-            : DEFAULT_BLOCKED_STATUS_CODES;
         // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
         if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
             await sleep(5000);
@@ -153,117 +197,161 @@ export class BrowserCrawler extends BasicCrawler {
             return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
         }
         const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
-        const blockedStatusCode = blockedStatusCodes.find((x) => x === (response?.status() ?? 0));
+        const statusCode = response?.status() ?? 0;
         if (foundSelectors)
             return `Found selectors: ${foundSelectors.join(', ')}`;
-        if (blockedStatusCode)
-            return `Received blocked status code: ${blockedStatusCode}`;
+        if (this.blockedStatusCodes.has(statusCode))
+            return `Received blocked status code: ${statusCode}`;
         return false;
     }
     async preparePage(crawlingContext) {
-        const newPageOptions = {
+        const page = await this.browserPool.newPage({
             id: crawlingContext.id,
-        };
-        const useIncognitoPages = this.launchContext?.useIncognitoPages;
-        if (crawlingContext.session?.proxyInfo) {
-            const proxyInfo = crawlingContext.session.proxyInfo;
-            crawlingContext.proxyInfo = proxyInfo;
-            newPageOptions.proxyUrl = proxyInfo?.url;
-            newPageOptions.proxyTier = proxyInfo?.proxyTier;
-            if (proxyInfo?.ignoreTlsErrors) {
-                /**
-                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
-                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
-                 */
-                newPageOptions.pageOptions = {
-                    ignoreHTTPSErrors: true,
-                    acceptInsecureCerts: true,
-                };
-            }
-        }
-        const page = (await this.browserPool.newPage(newPageOptions));
+            session: crawlingContext.session,
+        });
         tryCancel();
-        const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
-        const contextEnqueueLinks = crawlingContext.enqueueLinks;
-        const session = useIncognitoPages
-            ? crawlingContext.session
-            : browserControllerInstance.launchContext.session;
+        const addRequests = crawlingContext.addRequests;
+        const extractLinks = async (options) => {
+            return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
+        };
         return {
             page,
             get response() {
                 throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
             },
-            browserController: browserControllerInstance,
-            session,
-            proxyInfo: session?.proxyInfo,
-            enqueueLinks: async (enqueueOptions = {}) => {
-                return (await browserCrawlerEnqueueLinks({
-                    options: { ...enqueueOptions, limit: this.calculateEnqueuedRequestLimit(enqueueOptions?.limit) },
-                    page,
-                    requestQueue: await this.getRequestQueue(),
-                    robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                    onSkippedRequest: this.handleSkippedRequest,
-                    originalRequestUrl: crawlingContext.request.url,
+            get gotoOptions() {
+                throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
+            },
+            extractLinks,
+            enqueueLinks: async (options = {}) => {
+                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
+                    enqueueStrategy: options.strategy,
                     finalRequestUrl: crawlingContext.request.loadedUrl,
-                    enqueueLinks: contextEnqueueLinks,
-                })); // TODO make this type safe
+                    originalRequestUrl: crawlingContext.request.url,
+                    userProvidedBaseUrl: options.baseUrl,
+                });
+                const urls = await extractLinks(options);
+                return addRequests(urls, {
+                    ...options,
+                    baseUrl,
+                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+                });
             },
         };
     }
-    async performNavigation(crawlingContext) {
+    async prepareNavigation(crawlingContext) {
         if (crawlingContext.request.skipNavigation) {
             return {
                 request: new Proxy(crawlingContext.request, {
                     get(target, propertyName, receiver) {
                         if (propertyName === 'loadedUrl') {
-                            throw new Error('The `request.loadedUrl` property is not available - `skipNavigation` was used');
+                            throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
                         }
                         return Reflect.get(target, propertyName, receiver);
                     },
                 }),
                 get response() {
-                    throw new Error('The `response` property is not available - `skipNavigation` was used');
+                    throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
                 },
             };
         }
-        const gotoOptions = { timeout: this.navigationTimeoutMillis };
-        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
         crawlingContext.request.state = RequestState.BEFORE_NAV;
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
+        return {
+            // Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it
+            // to the remaining shared window unless a hook overrode it (see there).
+            gotoOptions: { timeout: this.#navigationTimeoutMillis },
+            [COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
+        };
+    }
+    async navigate(crawlingContext) {
         tryCancel();
-        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
-        await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
+        const gotoOptions = crawlingContext.gotoOptions;
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
+        if (remaining <= 0) {
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
+        }
+        // If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the
+        // shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour
+        // that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is
+        // aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.
+        const gotoTimeout = gotoOptions;
+        if (gotoTimeout.timeout === this.#navigationTimeoutMillis) {
+            gotoTimeout.timeout = remaining;
+        }
+        const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
+        const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
+        await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
         let response;
         try {
-            response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
+            response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
         }
         catch (error) {
-            await this._handleNavigationTimeout(crawlingContext, error);
+            await this.handleNavigationTimeout(crawlingContext, error);
             crawlingContext.request.state = RequestState.ERROR;
-            this._throwIfProxyError(error);
+            this.throwIfProxyError(error);
             throw error;
         }
         tryCancel();
         crawlingContext.request.state = RequestState.AFTER_NAV;
-        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
+        return { response };
+    }
+    async finalizeNavigation(crawlingContext) {
+        tryCancel();
+        let response;
+        try {
+            response = crawlingContext.response;
+        }
+        catch {
+            // `preparePage` installs a throwing getter for `response`; reaching this branch means
+            // navigation produced no response and no hook overrode it. Treat as undefined.
+        }
         await this.processResponse(response, crawlingContext);
         tryCancel();
-        // save cookies
-        // TODO: Should we save the cookies also after/only the handle page?
-        if (this.persistCookiesPerSession) {
-            const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
-            tryCancel();
-            crawlingContext.session?.setCookies(cookies, crawlingContext.request.loadedUrl);
+        // Persist cookies from the navigation response before the user handler runs.
+        // Cookies set during `requestHandler` are saved again afterwards.
+        await this.persistCookiesFromPage(crawlingContext);
+        return { request: crawlingContext.request };
+    }
+    /**
+     * Copies cookies from the live browser page into the session cookie jar.
+     */
+    async persistCookiesFromPage(crawlingContext) {
+        if (!this.#saveResponseCookies || !crawlingContext.session) {
+            return;
         }
-        if (response !== undefined) {
-            return {
-                request: crawlingContext.request,
-                response,
-            };
+        const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
+        tryCancel();
+        // Prefer the live page URL — the handler may have navigated after the initial load.
+        const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
+        for (const cookie of cookies) {
+            try {
+                await crawlingContext.session.cookieJar.setCookie(browserPoolCookieToToughCookie(cookie), url, {
+                    ignoreError: false,
+                });
+            }
+            catch (e) {
+                this.log.debug(`Could not set cookie: ${e.message}`);
+            }
+        }
+    }
+    /**
+     * Runs the user request handler, then re-reads browser cookies so login flows /
+     * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+     */
+    async runRequestHandler(crawlingContext) {
+        try {
+            await super.runRequestHandler(crawlingContext);
+        }
+        finally {
+            if (!crawlingContext.request.skipNavigation) {
+                try {
+                    await this.persistCookiesFromPage(crawlingContext);
+                }
+                catch {
+                    // Page may already be closed on some failure paths; ignore.
+                }
+            }
         }
-        return {
-            request: crawlingContext.request,
-        };
     }
     async handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {
@@ -277,41 +365,64 @@ export class BrowserCrawler extends BasicCrawler {
         crawlingContext.request.state = RequestState.REQUEST_HANDLER;
         return {};
     }
-    async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
-        const sessionCookie = session?.getCookies(request.url) ?? [];
+    async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
+        const sessionCookie = session
+            ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
+            : [];
         const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
-        await browserController.setCookies(page, [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
+        const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
             .filter((c) => typeof c !== 'undefined' && c !== null)
-            .map((c) => ({ ...c, url: c.domain ? undefined : request.url })));
+            .map((c) => ({ ...c, url: c.domain ? undefined : request.url }));
+        await this.browserPool.injectPageState(page, { cookies });
     }
     /**
-     * Marks session bad in case of navigation timeout.
+     * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
      */
-    async _handleNavigationTimeout(crawlingContext, error) {
-        const { session } = crawlingContext;
-        if (error && error.constructor.name === 'TimeoutError') {
-            handleRequestTimeout({ session, errorMessage: error.message });
+    async handleNavigationTimeout(crawlingContext, error) {
+        const { session, page } = crawlingContext;
+        // Fire-and-forget: no user code will run on this page after a failed navigation.
+        // Swallow rejections: the page may already be detached.
+        void page.evaluate(() => window.stop()).catch(() => { });
+        if (isNavigationTimeoutError(error)) {
+            session?.markBad();
+            // The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the
+            // hooks have run), so it names that value in its own error; report the configured window instead.
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
-        await crawlingContext.page.close();
     }
     /**
      * Transforms proxy-related errors to `SessionError`.
      */
-    _throwIfProxyError(error) {
+    throwIfProxyError(error) {
         if (this.isProxyError(error)) {
-            throw new SessionError(this._getMessageFromError(error));
+            throw new SessionError(this.getMessageFromError(error));
         }
     }
     async processResponse(response, crawlingContext) {
         const { session, request, page } = crawlingContext;
         if (typeof response === 'object' && typeof response.status === 'function') {
             const status = response.status();
-            this.stats.registerStatusCode(status);
+            this.statistics.registerStatusCode(status);
+            // Ahead of the error-status throw below: a 429 the user opted into treating as an error is still a
+            // rate limit the domain should back off from.
+            if (status === 429) {
+                // Both drivers lower-case header names and join duplicates, so a plain lookup is enough.
+                const retryAfter = response.headers?.()['retry-after'];
+                if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
+                    throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
+                }
+            }
+            if (this.isErrorStatusCode(status)) {
+                if (this.additionalHttpErrorStatusCodes.has(status)) {
+                    throw new Error(`${status} - Error status code was set by user.`);
+                }
+                throw new Error(`${status} - Internal Server Error`);
+            }
         }
         if (this.sessionPool && response && session) {
             if (typeof response === 'object' && typeof response.status === 'function') {
-                this._throwOnBlockedRequest(session, response.status());
+                this.throwOnBlockedRequest(response.status());
             }
             else {
                 this.log.debug('Got a malformed Browser response.', { request, response });
@@ -319,84 +430,15 @@ export class BrowserCrawler extends BasicCrawler {
         }
         request.loadedUrl = await page.url();
     }
-    async _extendLaunchContext(_pageId, launchContext) {
-        const launchContextExtends = {};
-        if (this.sessionPool) {
-            launchContextExtends.session = await this.sessionPool.newSession({
-                proxyInfo: await this.proxyConfiguration?.newProxyInfo({
-                // cannot pass a request here, since session is created on browser launch
-                }),
-            });
-        }
-        if (!launchContext.proxyUrl && launchContextExtends.session?.proxyInfo) {
-            const proxyInfo = launchContextExtends.session.proxyInfo;
-            launchContext.proxyUrl = proxyInfo?.url;
-            launchContextExtends.proxyInfo = proxyInfo;
-            // Disable SSL verification for MITM proxies
-            if (proxyInfo?.ignoreTlsErrors) {
-                /**
-                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
-                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
-                 */
-                launchContext.launchOptions.ignoreHTTPSErrors = true;
-                launchContext.launchOptions.acceptInsecureCerts = true;
-            }
-        }
-        launchContext.extend(launchContextExtends);
-    }
-    _maybeAddSessionRetiredListener(_pageId, browserController) {
-        if (this.sessionPool) {
-            const listener = (session) => {
-                const { launchContext } = browserController;
-                if (session.id === launchContext.session.id) {
-                    this.browserPool.retireBrowserController(browserController);
-                }
-            };
-            this.sessionPool.on(EVENT_SESSION_RETIRED, listener);
-            browserController.on("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
-                return this.sessionPool.removeListener(EVENT_SESSION_RETIRED, listener);
-            });
-        }
-    }
     /**
      * Function for cleaning up after all requests are processed.
      * @ignore
      */
     async teardown() {
-        await this.browserPool.destroy();
+        await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
         await super.teardown();
     }
 }
-/** @internal */
-function containsEnqueueLinks(options) {
-    return !!options.enqueueLinks;
-}
-/** @internal */
-export async function browserCrawlerEnqueueLinks(options) {
-    const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
-    const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-        enqueueStrategy: enqueueLinksOptions?.strategy,
-        finalRequestUrl,
-        originalRequestUrl,
-        userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
-    });
-    const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
-    if (containsEnqueueLinks(options)) {
-        return options.enqueueLinks({
-            urls,
-            baseUrl,
-            ...enqueueLinksOptions,
-        });
-    }
-    return enqueueLinks({
-        requestQueue: options.requestQueue,
-        robotsTxtFile: options.robotsTxtFile,
-        onSkippedRequest: options.onSkippedRequest,
-        urls,
-        baseUrl,
-        ...enqueueLinksOptions,
-    });
-}
 /**
  * Extracts URLs from a given page.
  * @ignore
@@ -422,4 +464,3 @@ page, selector, baseUrl) {
     })
         .filter((href) => !!href);
 }
-//# sourceMappingURL=browser-crawler.js.map