@crawlee/browser 4.0.0-beta.1 → 4.0.0-beta.100

package/internals/browser-crawler.js

@@ -1,8 +1,21 @@
-import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler, BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES, Configuration, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, handleRequestTimeout, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, tryAbsoluteURL, validators, } from '@crawlee/basic';
-import { BrowserPool } from '@crawlee/browser-pool';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
+import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
 import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
 import ow from 'ow';
-import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
+import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
+const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
+const readContextField = (ctx, key) => ctx[key];
+/**
+ * Whether an error thrown by `page.goto()` is a navigation timeout - either our own {@link TimeoutError}
+ * or the driver's, which Playwright/Puppeteer report with their own class and a `Timeout ... exceeded` message
+ * naming the raw millisecond value rather than the configured window.
+ */
+function isNavigationTimeoutError(error) {
+    return (error instanceof TimeoutError ||
+        error?.name === 'TimeoutError' ||
+        error?.constructor?.name === 'TimeoutError' ||
+        /timeout.*exceeded/i.test(error?.message ?? ''));
+}
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -15,51 +28,51 @@ import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
  * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
- * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
+ * The source URLs are represented by the {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
  * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
  * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
  *
- * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
+ * combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
+ * and pass the result as `requestManager`.
+ *
+ * > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
  * `BrowserCrawler` opens a new browser page (i.e. tab or window) for each {@link Request} object to crawl
  * and then calls the function provided by user as the {@link BrowserCrawlerOptions.requestHandler|`requestHandler`} option.
  *
- * New pages are only opened when there is enough free CPU and memory available,
- * using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BrowserCrawlerOptions.autoscaledPoolOptions|`autoscaledPoolOptions`}
- * parameter of the `BrowserCrawler` constructor.
- * For user convenience, the {@link AutoscaledPoolOptions.minConcurrency|`minConcurrency`} and
- * {@link AutoscaledPoolOptions.maxConcurrency|`maxConcurrency`} options of the
- * underlying {@link AutoscaledPool} constructor are available directly in the `BrowserCrawler` constructor.
+ * New pages are only opened when there is enough free CPU and memory available, as judged by the crawler's
+ * {@link ConcurrencySystem}.
+ * Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the
+ * `BrowserCrawler` constructor, or, for finer control, by injecting a pre-configured
+ * {@link ConcurrencySystem|`concurrencySystem`}.
  *
  * > *NOTE:* the pool of browser instances is internally managed by the {@link BrowserPool} class.
  *
  * @category Crawlers
  */
 export class BrowserCrawler extends BasicCrawler {
-    config;
-    /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-     * Only available if used by the crawler.
-     */
-    proxyConfiguration;
+    /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
+    browserPoolDep;
     /**
-     * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
+     * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
+     * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
-    browserPool;
+    get browserPool() {
+        return this.browserPoolDep.value;
+    }
     launchContext;
-    userProvidedRequestHandler;
+    ignoreShadowRoots;
+    ignoreIframes;
     navigationTimeoutMillis;
-    requestHandlerTimeoutInnerMillis;
     preNavigationHooks;
     postNavigationHooks;
-    persistCookiesPerSession;
+    saveResponseCookies;
     static optionsShape = {
         ...BasicCrawler.optionsShape,
         navigationTimeoutSecs: ow.optional.number.greaterThan(0),
@@ -67,67 +80,110 @@ export class BrowserCrawler extends BasicCrawler {
         postNavigationHooks: ow.optional.array,
         launchContext: ow.optional.object,
         headless: ow.optional.any(ow.boolean, ow.string),
-        browserPoolOptions: ow.object,
-        sessionPoolOptions: ow.optional.object,
-        persistCookiesPerSession: ow.optional.boolean,
-        useSessionPool: ow.optional.boolean,
+        browserPool: ow.optional.object.validate(validators.browserPool),
+        remoteBrowser: ow.optional.object,
+        browserPoolOptions: ow.optional.object,
+        saveResponseCookies: ow.optional.boolean,
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
-        ignoreShadowRoots: ow.optional.boolean,
-        ignoreIframes: ow.optional.boolean,
     };
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+    constructor(options) {
         ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
-        const { navigationTimeoutSecs = 60, requestHandlerTimeoutSecs = 60, persistCookiesPerSession, proxyConfiguration, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], requestHandler, failedRequestHandler, headless, ignoreShadowRoots, ignoreIframes, ...basicCrawlerOptions } = options;
+        const { navigationTimeoutSecs = 60, saveResponseCookies = true, launchContext = {}, browserPool, remoteBrowser, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, ...basicCrawlerOptions } = options;
+        const skipGuard = (action) => ({
+            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
+        });
         super({
             ...basicCrawlerOptions,
-            requestHandler: async (...args) => this._runRequestHandler(...args),
-            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
-        }, config);
-        this.config = config;
-        // FIXME any
-        this.userProvidedRequestHandler = requestHandler ?? this.router;
-        this.failedRequestHandler = failedRequestHandler; // FIXME is this even needed?
-        // Cookies should be persisted per session only if session pool is used
-        if (!this.useSessionPool && persistCookiesPerSession) {
-            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
-        }
+            contextPipelineBuilder: () => {
+                // A single navigation window covers the pre-navigation hooks, the navigation, and the
+                // post-navigation hooks: the whole phase shares one `navigationTimeoutSecs` budget, so a slow
+                // hook eats into the same window the navigation uses. The navigation itself is bounded by
+                // capping its `gotoOptions.timeout` to the remaining budget.
+                const windowGuard = (step) => skipGuard(async (ctx) => {
+                    const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);
+                    if (remaining <= 0) {
+                        throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+                    }
+                    return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+                });
+                let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
+                for (const hook of this.preNavigationHooks) {
+                    pipeline = pipeline.compose(windowGuard(hook));
+                }
+                pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
+                for (const hook of this.postNavigationHooks) {
+                    pipeline = pipeline.compose(windowGuard(hook));
+                }
+                return pipeline
+                    .compose(skipGuard(this.finalizeNavigation.bind(this)))
+                    .compose({ action: this.handleBlockedRequestByContent.bind(this) })
+                    .compose({ action: this.restoreRequestState.bind(this) });
+            },
+            extendContext,
+        });
         this.launchContext = launchContext;
         this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.requestHandlerTimeoutInnerMillis = requestHandlerTimeoutSecs * 1000;
-        this.proxyConfiguration = proxyConfiguration;
+        // The public option hooks are extension-aware; internal storage uses the base context type
+        // (the pipeline composes hooks against the concrete context, which does not statically carry
+        // `ContextExtension`). The extension members are present at runtime regardless.
         this.preNavigationHooks = preNavigationHooks;
         this.postNavigationHooks = postNavigationHooks;
+        this.ignoreIframes = ignoreIframes;
+        this.ignoreShadowRoots = ignoreShadowRoots;
         if (headless != null) {
             this.launchContext.launchOptions ??= {};
             this.launchContext.launchOptions.headless = headless;
         }
-        if (this.useSessionPool) {
-            this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
-        }
-        else {
-            this.persistCookiesPerSession = false;
-        }
-        if (launchContext?.userAgent) {
-            if (browserPoolOptions.useFingerprints)
-                this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
-            browserPoolOptions.useFingerprints = false;
-        }
-        const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
-        this.browserPool = new BrowserPool({
-            ...rest,
-            preLaunchHooks: [this._extendLaunchContext.bind(this), ...preLaunchHooks],
-            postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
+        this.saveResponseCookies = saveResponseCookies;
+        // `browserPool` wins over `remoteBrowser` — a passed-in pool is used as-is (borrowed), the sugar is ignored.
+        // The default is only built when no pool was injected, so all the option/launchContext fiddling below stays
+        // inside the factory.
+        this.browserPoolDep = OwnedOrInjected.resolve(browserPool, () => {
+            const resolvedBrowserPoolOptions = browserPoolOptions ?? {};
+            if (launchContext?.userAgent) {
+                if (resolvedBrowserPoolOptions.useFingerprints)
+                    this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
+                resolvedBrowserPoolOptions.useFingerprints = false;
+            }
+            if (remoteBrowser) {
+                // The crawler already built the right plugin for its browser — hand it to a RemoteBrowserPool so the
+                // remote connection is always for the matching browser (no plugin to construct, no way to mismatch).
+                const { browserPlugins, ...remoteBrowserPoolOptions } = resolvedBrowserPoolOptions;
+                return new RemoteBrowserPool({
+                    browserPlugins: browserPlugins,
+                    ...remoteBrowser,
+                    browserPoolOptions: remoteBrowserPoolOptions,
+                });
+            }
+            // Double cast: `BrowserPool` implements `IBrowserPool<PageReturn>`, where `PageReturn` is derived from the
+            // plugin/controller generics and doesn't overlap with the crawler's free `Page` type param, so TS won't
+            // narrow it directly. The concrete pool does satisfy the `Page`/`destroy` contract at runtime — this is the
+            // long-standing `Page` variance gap, not a `destroy`-related hole.
+            return new BrowserPool({
+                ...resolvedBrowserPoolOptions,
+            });
         });
     }
-    async _cleanupContext(crawlingContext) {
-        const { page } = crawlingContext;
-        // Page creation may be aborted
-        if (page) {
-            await page.close().catch((error) => this.log.debug('Error while closing page', { error }));
-        }
+    getNavigationTimeoutMillis() {
+        return this.navigationTimeoutMillis;
+    }
+    buildContextPipeline() {
+        return ContextPipeline.create().compose({
+            action: this.preparePage.bind(this),
+            cleanup: async (context) => {
+                context.registerDeferredCleanup(async () => {
+                    const error = !context.session.isUsable()
+                        ? new SessionError('Session is no longer usable')
+                        : undefined;
+                    await this.browserPool
+                        .closePage(context.page, { error })
+                        .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
+                });
+            },
+        });
     }
     async containsSelectors(page, selectors) {
         const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
@@ -138,12 +194,6 @@ export class BrowserCrawler extends BasicCrawler {
     }
     async isRequestBlocked(crawlingContext) {
         const { page, response } = crawlingContext;
-        const blockedStatusCodes = 
-        // eslint-disable-next-line dot-notation
-        (this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
-            ? // eslint-disable-next-line dot-notation
-                this.sessionPool['blockedStatusCodes']
-            : DEFAULT_BLOCKED_STATUS_CODES;
         // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
         if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
             await sleep(5000);
@@ -154,170 +204,218 @@ export class BrowserCrawler extends BasicCrawler {
             return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
         }
         const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
-        const blockedStatusCode = blockedStatusCodes.find((x) => x === (response?.status() ?? 0));
+        const statusCode = response?.status() ?? 0;
         if (foundSelectors)
             return `Found selectors: ${foundSelectors.join(', ')}`;
-        if (blockedStatusCode)
-            return `Received blocked status code: ${blockedStatusCode}`;
+        if (this.blockedStatusCodes.has(statusCode))
+            return `Received blocked status code: ${statusCode}`;
         return false;
     }
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    async _runRequestHandler(crawlingContext) {
-        const newPageOptions = {
+    async preparePage(crawlingContext) {
+        const page = await this.browserPool.newPage({
             id: crawlingContext.id,
+            session: crawlingContext.session,
+        });
+        tryCancel();
+        const contextEnqueueLinks = crawlingContext.enqueueLinks;
+        return {
+            page,
+            get response() {
+                throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
+            },
+            get gotoOptions() {
+                throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
+            },
+            enqueueLinks: async (enqueueOptions = {}) => {
+                return (await browserCrawlerEnqueueLinks({
+                    options: {
+                        ...enqueueOptions,
+                        limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
+                    },
+                    page,
+                    requestManager: await this.getRequestManager(),
+                    robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
+                    onSkippedRequest: this.handleSkippedRequest,
+                    originalRequestUrl: crawlingContext.request.url,
+                    finalRequestUrl: crawlingContext.request.loadedUrl,
+                    enqueueLinks: contextEnqueueLinks,
+                })); // TODO make this type safe
+            },
         };
-        const useIncognitoPages = this.launchContext?.useIncognitoPages;
-        if (this.proxyConfiguration) {
-            const { session } = crawlingContext;
-            const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, {
-                request: crawlingContext.request,
-            });
-            crawlingContext.proxyInfo = proxyInfo;
-            newPageOptions.proxyUrl = proxyInfo?.url;
-            newPageOptions.proxyTier = proxyInfo?.proxyTier;
-            if (this.proxyConfiguration.isManInTheMiddle) {
-                /**
-                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
-                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
-                 */
-                newPageOptions.pageOptions = {
-                    ignoreHTTPSErrors: true,
-                    acceptInsecureCerts: true,
-                };
-            }
+    }
+    async prepareNavigation(crawlingContext) {
+        if (crawlingContext.request.skipNavigation) {
+            return {
+                request: new Proxy(crawlingContext.request, {
+                    get(target, propertyName, receiver) {
+                        if (propertyName === 'loadedUrl') {
+                            throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
+                        }
+                        return Reflect.get(target, propertyName, receiver);
+                    },
+                }),
+                get response() {
+                    throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
+                },
+            };
         }
-        const page = (await this.browserPool.newPage(newPageOptions));
+        crawlingContext.request.state = RequestState.BEFORE_NAV;
+        return {
+            // Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it
+            // to the remaining shared window unless a hook overrode it (see there).
+            gotoOptions: { timeout: this.navigationTimeoutMillis },
+            [COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),
+        };
+    }
+    async navigate(crawlingContext) {
         tryCancel();
-        this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
-        // DO NOT MOVE THIS LINE ABOVE!
-        // `enhanceCrawlingContextWithPageInfo` gives us a valid session.
-        // For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`.
-        // So we must not save the session prior to making sure it was used only once, otherwise we would use it twice.
-        const { request, session } = crawlingContext;
-        if (!request.skipNavigation) {
-            await this._handleNavigation(crawlingContext);
-            tryCancel();
-            await this._responseHandler(crawlingContext);
-            tryCancel();
-            // save cookies
-            // TODO: Should we save the cookies also after/only the handle page?
-            if (this.persistCookiesPerSession) {
-                const cookies = await crawlingContext.browserController.getCookies(page);
-                tryCancel();
-                session?.setCookies(cookies, request.loadedUrl);
-            }
+        const gotoOptions = crawlingContext.gotoOptions;
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);
+        if (remaining <= 0) {
+            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
         }
-        if (!this.requestMatchesEnqueueStrategy(request)) {
-            this.log.debug(
-            // eslint-disable-next-line dot-notation
-            `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`);
-            request.noRetry = true;
-            request.state = RequestState.SKIPPED;
-            return;
+        // If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the
+        // shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour
+        // that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is
+        // aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.
+        const gotoTimeout = gotoOptions;
+        if (gotoTimeout.timeout === this.navigationTimeoutMillis) {
+            gotoTimeout.timeout = remaining;
         }
-        if (this.retryOnBlocked) {
-            const error = await this.isRequestBlocked(crawlingContext);
-            if (error)
-                throw new SessionError(error);
-        }
-        request.state = RequestState.REQUEST_HANDLER;
+        const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
+        const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);
+        await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
+        let response;
         try {
-            await addTimeoutToPromise(async () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)), this.requestHandlerTimeoutInnerMillis, `requestHandler timed out after ${this.requestHandlerTimeoutInnerMillis / 1000} seconds.`);
-            request.state = RequestState.DONE;
+            response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
         }
-        catch (e) {
-            request.state = RequestState.ERROR;
-            throw e;
+        catch (error) {
+            await this.handleNavigationTimeout(crawlingContext, error);
+            crawlingContext.request.state = RequestState.ERROR;
+            this.throwIfProxyError(error);
+            throw error;
         }
         tryCancel();
+        crawlingContext.request.state = RequestState.AFTER_NAV;
+        return { response };
     }
-    _enhanceCrawlingContextWithPageInfo(crawlingContext, page, createNewSession) {
-        crawlingContext.page = page;
-        // This switch is because the crawlingContexts are created on per request basis.
-        // However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
-        // We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,
-        // which configures proxy and session for every new request
-        const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
-        crawlingContext.browserController = browserControllerInstance;
-        if (!createNewSession) {
-            crawlingContext.session = browserControllerInstance.launchContext.session;
+    async finalizeNavigation(crawlingContext) {
+        tryCancel();
+        let response;
+        try {
+            response = crawlingContext.response;
         }
-        if (!crawlingContext.proxyInfo) {
-            crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo;
+        catch {
+            // `preparePage` installs a throwing getter for `response`; reaching this branch means
+            // navigation produced no response and no hook overrode it. Treat as undefined.
         }
-        crawlingContext.enqueueLinks = async (enqueueOptions) => {
-            return browserCrawlerEnqueueLinks({
-                options: enqueueOptions,
-                page,
-                requestQueue: await this.getRequestQueue(),
-                robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                onSkippedRequest: this.onSkippedRequest,
-                originalRequestUrl: crawlingContext.request.url,
-                finalRequestUrl: crawlingContext.request.loadedUrl,
-            });
-        };
+        await this.processResponse(response, crawlingContext);
+        tryCancel();
+        // Persist cookies from the navigation response before the user handler runs.
+        // Cookies set during `requestHandler` are saved again afterwards.
+        await this.persistCookiesFromPage(crawlingContext);
+        return { request: crawlingContext.request };
     }
-    async _handleNavigation(crawlingContext) {
-        const gotoOptions = { timeout: this.navigationTimeoutMillis };
-        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
-        crawlingContext.request.state = RequestState.BEFORE_NAV;
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
+    /**
+     * Copies cookies from the live browser page into the session cookie jar.
+     */
+    async persistCookiesFromPage(crawlingContext) {
+        if (!this.saveResponseCookies || !crawlingContext.session) {
+            return;
+        }
+        const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
         tryCancel();
-        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
-        await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
+        // Prefer the live page URL — the handler may have navigated after the initial load.
+        const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
+        for (const cookie of cookies) {
+            try {
+                crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
+                    ignoreError: false,
+                });
+            }
+            catch (e) {
+                this.log.debug(`Could not set cookie: ${e.message}`);
+            }
+        }
+    }
+    /**
+     * Runs the user request handler, then re-reads browser cookies so login flows /
+     * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+     */
+    async runRequestHandler(crawlingContext) {
         try {
-            crawlingContext.response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
+            await super.runRequestHandler(crawlingContext);
         }
-        catch (error) {
-            await this._handleNavigationTimeout(crawlingContext, error);
-            crawlingContext.request.state = RequestState.ERROR;
-            this._throwIfProxyError(error);
-            throw error;
+        finally {
+            if (!crawlingContext.request.skipNavigation) {
+                try {
+                    await this.persistCookiesFromPage(crawlingContext);
+                }
+                catch {
+                    // Page may already be closed on some failure paths; ignore.
+                }
+            }
         }
-        tryCancel();
-        crawlingContext.request.state = RequestState.AFTER_NAV;
-        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
     }
-    async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
-        const sessionCookie = session?.getCookies(request.url) ?? [];
+    async handleBlockedRequestByContent(crawlingContext) {
+        if (this.retryOnBlocked) {
+            const error = await this.isRequestBlocked(crawlingContext);
+            if (error)
+                throw new SessionError(error);
+        }
+        return {};
+    }
+    async restoreRequestState(crawlingContext) {
+        crawlingContext.request.state = RequestState.REQUEST_HANDLER;
+        return {};
+    }
+    async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
+        const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];
         const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
-        await browserController.setCookies(page, [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
+        const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
             .filter((c) => typeof c !== 'undefined' && c !== null)
-            .map((c) => ({ ...c, url: c.domain ? undefined : request.url })));
+            .map((c) => ({ ...c, url: c.domain ? undefined : request.url }));
+        await this.browserPool.injectPageState(page, { cookies });
     }
     /**
-     * Marks session bad in case of navigation timeout.
+     * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
      */
-    async _handleNavigationTimeout(crawlingContext, error) {
-        const { session } = crawlingContext;
-        if (error && error.constructor.name === 'TimeoutError') {
-            handleRequestTimeout({ session, errorMessage: error.message });
+    async handleNavigationTimeout(crawlingContext, error) {
+        const { session, page } = crawlingContext;
+        // Fire-and-forget: no user code will run on this page after a failed navigation.
+        // Swallow rejections: the page may already be detached.
+        void page.evaluate(() => window.stop()).catch(() => { });
+        if (isNavigationTimeoutError(error)) {
+            session?.markBad();
+            // The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the
+            // hooks have run), so it names that value in its own error; report the configured window instead.
+            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
         }
-        await crawlingContext.page.close();
     }
     /**
      * Transforms proxy-related errors to `SessionError`.
      */
-    _throwIfProxyError(error) {
+    throwIfProxyError(error) {
         if (this.isProxyError(error)) {
             throw new SessionError(this._getMessageFromError(error));
         }
     }
-    /**
-     * Should be overridden in case of different automation library that does not support this response API.
-     */
-    async _responseHandler(crawlingContext) {
-        const { response, session, request, page } = crawlingContext;
+    async processResponse(response, crawlingContext) {
+        const { session, request, page } = crawlingContext;
         if (typeof response === 'object' && typeof response.status === 'function') {
             const status = response.status();
             this.stats.registerStatusCode(status);
+            if (this.isErrorStatusCode(status)) {
+                if (this.additionalHttpErrorStatusCodes.has(status)) {
+                    throw new Error(`${status} - Error status code was set by user.`);
+                }
+                throw new Error(`${status} - Internal Server Error`);
+            }
         }
         if (this.sessionPool && response && session) {
             if (typeof response === 'object' && typeof response.status === 'function') {
-                this._throwOnBlockedRequest(session, response.status());
+                this._throwOnBlockedRequest(response.status());
             }
             else {
                 this.log.debug('Got a malformed Browser response.', { request, response });
@@ -325,68 +423,43 @@ export class BrowserCrawler extends BasicCrawler {
         }
         request.loadedUrl = await page.url();
     }
-    async _extendLaunchContext(_pageId, launchContext) {
-        const launchContextExtends = {};
-        if (this.sessionPool) {
-            launchContextExtends.session = await this.sessionPool.getSession();
-        }
-        if (this.proxyConfiguration && !launchContext.proxyUrl) {
-            const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id, {
-                proxyTier: launchContext.proxyTier ?? undefined,
-            });
-            launchContext.proxyUrl = proxyInfo?.url;
-            launchContextExtends.proxyInfo = proxyInfo;
-            // Disable SSL verification for MITM proxies
-            if (this.proxyConfiguration.isManInTheMiddle) {
-                /**
-                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
-                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
-                 */
-                launchContext.launchOptions.ignoreHTTPSErrors = true;
-                launchContext.launchOptions.acceptInsecureCerts = true;
-            }
-        }
-        launchContext.extend(launchContextExtends);
-    }
-    _maybeAddSessionRetiredListener(_pageId, browserController) {
-        if (this.sessionPool) {
-            const listener = (session) => {
-                const { launchContext } = browserController;
-                if (session.id === launchContext.session.id) {
-                    this.browserPool.retireBrowserController(browserController);
-                }
-            };
-            this.sessionPool.on(EVENT_SESSION_RETIRED, listener);
-            browserController.on("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
-                return this.sessionPool.removeListener(EVENT_SESSION_RETIRED, listener);
-            });
-        }
-    }
     /**
      * Function for cleaning up after all requests are processed.
      * @ignore
      */
     async teardown() {
-        await this.browserPool.destroy();
+        await this.browserPoolDep.ifOwned((pool) => pool.destroy());
         await super.teardown();
     }
 }
 /** @internal */
-export async function browserCrawlerEnqueueLinks({ options, page, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }) {
+function containsEnqueueLinks(options) {
+    return !!options.enqueueLinks;
+}
+/** @internal */
+export async function browserCrawlerEnqueueLinks(options) {
+    const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
     const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-        enqueueStrategy: options?.strategy,
+        enqueueStrategy: enqueueLinksOptions?.strategy,
         finalRequestUrl,
         originalRequestUrl,
-        userProvidedBaseUrl: options?.baseUrl,
+        userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
     });
-    const urls = await extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
+    const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
+    if (containsEnqueueLinks(options)) {
+        return options.enqueueLinks({
+            urls,
+            baseUrl,
+            ...enqueueLinksOptions,
+        });
+    }
     return enqueueLinks({
-        requestQueue,
-        robotsTxtFile,
-        onSkippedRequest,
+        requestManager: options.requestManager,
+        robotsTxtFile: options.robotsTxtFile,
+        onSkippedRequest: options.onSkippedRequest,
         urls,
         baseUrl,
-        ...options,
+        ...enqueueLinksOptions,
     });
 }
 /**
@@ -414,4 +487,3 @@ page, selector, baseUrl) {
     })
         .filter((href) => !!href);
 }
-//# sourceMappingURL=browser-crawler.js.map