@crawlee/browser 4.0.0-beta.8 → 4.0.0-beta.81

package/internals/browser-crawler.js

@@ -1,8 +1,10 @@
-import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS, BasicCrawler, BLOCKED_STATUS_CODES as DEFAULT_BLOCKED_STATUS_CODES, Configuration, cookieStringToToughCookie, enqueueLinks, EVENT_SESSION_RETIRED, handleRequestTimeout, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, tryAbsoluteURL, validators, } from '@crawlee/basic';
-import { BrowserPool } from '@crawlee/browser-pool';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, handleRequestTimeout, NavigationSkippedError, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
+import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
 import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
 import ow from 'ow';
-import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
+import { tryCancel } from '@apify/timeout';
+const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
+const readContextField = (ctx, key) => ctx[key];
 /**
  * Provides a simple framework for parallel crawling of web pages
  * using headless browsers with [Puppeteer](https://github.com/puppeteer/puppeteer)
@@ -15,15 +17,18 @@ import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
  * If the target website doesn't need JavaScript, we should consider using the {@link CheerioCrawler},
  * which downloads the pages using raw HTTP requests and is about 10x faster.
  *
- * The source URLs are represented by the {@link Request} objects that are fed from the {@link RequestList} or {@link RequestQueue} instances
- * provided by the {@link BrowserCrawlerOptions.requestList|`requestList`} or {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
- * constructor options, respectively. If neither `requestList` nor `requestQueue` options are provided,
+ * The source URLs are represented by the {@link Request} objects that are fed from the
+ * {@link IRequestManager|request manager} provided via the {@link BrowserCrawlerOptions.requestManager|`requestManager`}
+ * constructor option (a {@link RequestQueue} is itself a request manager). If no `requestManager` is provided,
  * the crawler will open the default request queue either when the {@link BrowserCrawler.addRequests|`crawler.addRequests()`} function is called,
  * or if `requests` parameter (representing the initial requests) of the {@link BrowserCrawler.run|`crawler.run()`} function is provided.
  *
- * If both {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`} options are used,
- * the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
- * to the {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
+ * To read from a read-only source such as a {@link RequestList} while still being able to enqueue new requests,
+ * combine it with a queue into a {@link RequestManagerTandem} via {@link IRequestLoader.toTandem|`requestLoader.toTandem()`}
+ * and pass the result as `requestManager`.
+ *
+ * > The {@link BrowserCrawlerOptions.requestList|`requestList`} and {@link BrowserCrawlerOptions.requestQueue|`requestQueue`}
+ * > options are deprecated; they are still accepted and folded into a single `requestManager` for back-compat.
  *
  * The crawler finishes when there are no more {@link Request} objects to crawl.
  *
@@ -43,23 +48,24 @@ import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
  * @category Crawlers
  */
 export class BrowserCrawler extends BasicCrawler {
-    config;
     /**
-     * A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
-     * Only available if used by the crawler.
+     * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
+     * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
-    proxyConfiguration;
+    browserPool;
     /**
-     * A reference to the underlying {@link BrowserPool} class that manages the crawler's browsers.
+     * Set when the crawler constructed its own pool (a {@link BrowserPool}, or a {@link RemoteBrowserPool}
+     * built from the `remoteBrowser` option). Holds the same instance as `browserPool` but is the only reference
+     * the crawler tears down — a user-supplied `browserPool` is never owned and never destroyed by the crawler.
      */
-    browserPool;
+    ownedBrowserPool;
     launchContext;
-    userProvidedRequestHandler;
+    ignoreShadowRoots;
+    ignoreIframes;
     navigationTimeoutMillis;
-    requestHandlerTimeoutInnerMillis;
     preNavigationHooks;
     postNavigationHooks;
-    persistCookiesPerSession;
+    saveResponseCookies;
     static optionsShape = {
         ...BasicCrawler.optionsShape,
         navigationTimeoutSecs: ow.optional.number.greaterThan(0),
@@ -67,65 +73,94 @@ export class BrowserCrawler extends BasicCrawler {
         postNavigationHooks: ow.optional.array,
         launchContext: ow.optional.object,
         headless: ow.optional.any(ow.boolean, ow.string),
-        browserPoolOptions: ow.object,
-        sessionPoolOptions: ow.optional.object,
-        persistCookiesPerSession: ow.optional.boolean,
-        useSessionPool: ow.optional.boolean,
+        browserPool: ow.optional.object.validate(validators.browserPool),
+        remoteBrowser: ow.optional.object,
+        browserPoolOptions: ow.optional.object,
+        saveResponseCookies: ow.optional.boolean,
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
-        ignoreShadowRoots: ow.optional.boolean,
-        ignoreIframes: ow.optional.boolean,
     };
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
      */
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
+    constructor(options) {
         ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
-        const { navigationTimeoutSecs = 60, requestHandlerTimeoutSecs = 60, persistCookiesPerSession, proxyConfiguration, launchContext = {}, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], requestHandler, headless, ignoreShadowRoots, ignoreIframes, ...basicCrawlerOptions } = options;
+        const { navigationTimeoutSecs = 60, saveResponseCookies = true, launchContext = {}, browserPool, remoteBrowser, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, ...basicCrawlerOptions } = options;
+        const skipGuard = (action) => ({
+            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
+        });
         super({
             ...basicCrawlerOptions,
-            requestHandler: async (...args) => this._runRequestHandler(...args),
-            requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
-        }, config);
-        this.config = config;
-        this.userProvidedRequestHandler = requestHandler ?? this.router;
-        // Cookies should be persisted per session only if session pool is used
-        if (!this.useSessionPool && persistCookiesPerSession) {
-            throw new Error('You cannot use "persistCookiesPerSession" without "useSessionPool" set to true.');
-        }
+            contextPipelineBuilder: () => {
+                let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
+                for (const hook of this.preNavigationHooks) {
+                    pipeline = pipeline.compose(skipGuard(hook));
+                }
+                pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
+                for (const hook of this.postNavigationHooks) {
+                    pipeline = pipeline.compose(skipGuard(hook));
+                }
+                return pipeline
+                    .compose(skipGuard(this.finalizeNavigation.bind(this)))
+                    .compose({ action: this.handleBlockedRequestByContent.bind(this) })
+                    .compose({ action: this.restoreRequestState.bind(this) });
+            },
+            extendContext: extendContext,
+        });
         this.launchContext = launchContext;
         this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
-        this.requestHandlerTimeoutInnerMillis = requestHandlerTimeoutSecs * 1000;
-        this.proxyConfiguration = proxyConfiguration;
         this.preNavigationHooks = preNavigationHooks;
         this.postNavigationHooks = postNavigationHooks;
+        this.ignoreIframes = ignoreIframes;
+        this.ignoreShadowRoots = ignoreShadowRoots;
         if (headless != null) {
             this.launchContext.launchOptions ??= {};
             this.launchContext.launchOptions.headless = headless;
         }
-        if (this.useSessionPool) {
-            this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
-        }
-        else {
-            this.persistCookiesPerSession = false;
+        this.saveResponseCookies = saveResponseCookies;
+        // `browserPool` wins over `remoteBrowser` — a passed-in pool is used as-is, the sugar is ignored.
+        if (browserPool) {
+            this.browserPool = browserPool;
+            return;
         }
+        const resolvedBrowserPoolOptions = browserPoolOptions ?? {};
         if (launchContext?.userAgent) {
-            if (browserPoolOptions.useFingerprints)
+            if (resolvedBrowserPoolOptions.useFingerprints)
                 this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
-            browserPoolOptions.useFingerprints = false;
+            resolvedBrowserPoolOptions.useFingerprints = false;
         }
-        const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
-        this.browserPool = new BrowserPool({
-            ...rest,
-            preLaunchHooks: [this._extendLaunchContext.bind(this), ...preLaunchHooks],
-            postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
+        if (remoteBrowser) {
+            // The crawler already built the right plugin for its browser — hand it to a RemoteBrowserPool so the
+            // remote connection is always for the matching browser (no plugin to construct, no way to mismatch).
+            const { browserPlugins, ...remoteBrowserPoolOptions } = resolvedBrowserPoolOptions;
+            const remotePool = new RemoteBrowserPool({
+                browserPlugins: browserPlugins,
+                ...remoteBrowser,
+                browserPoolOptions: remoteBrowserPoolOptions,
+            });
+            this.ownedBrowserPool = remotePool;
+            this.browserPool = remotePool;
+            return;
+        }
+        const ownedBrowserPool = new BrowserPool({
+            ...resolvedBrowserPoolOptions,
         });
+        this.ownedBrowserPool = ownedBrowserPool;
+        this.browserPool = ownedBrowserPool;
     }
-    async _cleanupContext(crawlingContext) {
-        const { page } = crawlingContext;
-        // Page creation may be aborted
-        if (page) {
-            await page.close().catch((error) => this.log.debug('Error while closing page', { error }));
-        }
+    buildContextPipeline() {
+        return ContextPipeline.create().compose({
+            action: this.preparePage.bind(this),
+            cleanup: async (context) => {
+                context.registerDeferredCleanup(async () => {
+                    const error = !context.session.isUsable()
+                        ? new SessionError('Session is no longer usable')
+                        : undefined;
+                    await this.browserPool
+                        .closePage(context.page, { error })
+                        .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
+                });
+            },
+        });
     }
     async containsSelectors(page, selectors) {
         const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
@@ -136,12 +171,6 @@ export class BrowserCrawler extends BasicCrawler {
     }
     async isRequestBlocked(crawlingContext) {
         const { page, response } = crawlingContext;
-        const blockedStatusCodes = 
-        // eslint-disable-next-line dot-notation
-        (this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
-            ? // eslint-disable-next-line dot-notation
-                this.sessionPool['blockedStatusCodes']
-            : DEFAULT_BLOCKED_STATUS_CODES;
         // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
         if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
             await sleep(5000);
@@ -152,170 +181,173 @@ export class BrowserCrawler extends BasicCrawler {
             return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
         }
         const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
-        const blockedStatusCode = blockedStatusCodes.find((x) => x === (response?.status() ?? 0));
+        const statusCode = response?.status() ?? 0;
         if (foundSelectors)
             return `Found selectors: ${foundSelectors.join(', ')}`;
-        if (blockedStatusCode)
-            return `Received blocked status code: ${blockedStatusCode}`;
+        if (this.blockedStatusCodes.has(statusCode))
+            return `Received blocked status code: ${statusCode}`;
         return false;
     }
-    /**
-     * Wrapper around requestHandler that opens and closes pages etc.
-     */
-    async _runRequestHandler(crawlingContext) {
-        const newPageOptions = {
+    async preparePage(crawlingContext) {
+        const page = await this.browserPool.newPage({
             id: crawlingContext.id,
-        };
-        const useIncognitoPages = this.launchContext?.useIncognitoPages;
-        if (this.proxyConfiguration) {
-            const { session } = crawlingContext;
-            const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, {
-                request: crawlingContext.request,
-            });
-            crawlingContext.proxyInfo = proxyInfo;
-            newPageOptions.proxyUrl = proxyInfo?.url;
-            newPageOptions.proxyTier = proxyInfo?.proxyTier;
-            if (this.proxyConfiguration.isManInTheMiddle) {
-                /**
-                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
-                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
-                 */
-                newPageOptions.pageOptions = {
-                    ignoreHTTPSErrors: true,
-                    acceptInsecureCerts: true,
-                };
-            }
-        }
-        const page = (await this.browserPool.newPage(newPageOptions));
-        tryCancel();
-        this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);
-        // DO NOT MOVE THIS LINE ABOVE!
-        // `enhanceCrawlingContextWithPageInfo` gives us a valid session.
-        // For example, `sessionPoolOptions.sessionOptions.maxUsageCount` can be `1`.
-        // So we must not save the session prior to making sure it was used only once, otherwise we would use it twice.
-        const { request, session } = crawlingContext;
-        if (!request.skipNavigation) {
-            await this._handleNavigation(crawlingContext);
-            tryCancel();
-            await this._responseHandler(crawlingContext);
-            tryCancel();
-            // save cookies
-            // TODO: Should we save the cookies also after/only the handle page?
-            if (this.persistCookiesPerSession) {
-                const cookies = await crawlingContext.browserController.getCookies(page);
-                tryCancel();
-                session?.setCookies(cookies, request.loadedUrl);
-            }
-        }
-        if (!this.requestMatchesEnqueueStrategy(request)) {
-            this.log.debug(
-            // eslint-disable-next-line dot-notation
-            `Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`);
-            request.noRetry = true;
-            request.state = RequestState.SKIPPED;
-            return;
-        }
-        if (this.retryOnBlocked) {
-            const error = await this.isRequestBlocked(crawlingContext);
-            if (error)
-                throw new SessionError(error);
-        }
-        request.state = RequestState.REQUEST_HANDLER;
-        try {
-            await addTimeoutToPromise(async () => Promise.resolve(this.userProvidedRequestHandler(crawlingContext)), this.requestHandlerTimeoutInnerMillis, `requestHandler timed out after ${this.requestHandlerTimeoutInnerMillis / 1000} seconds.`);
-            request.state = RequestState.DONE;
-        }
-        catch (e) {
-            request.state = RequestState.ERROR;
-            throw e;
-        }
+            session: crawlingContext.session,
+        });
         tryCancel();
+        const contextEnqueueLinks = crawlingContext.enqueueLinks;
+        return {
+            page,
+            get response() {
+                throw new Error("The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)");
+            },
+            get gotoOptions() {
+                throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
+            },
+            enqueueLinks: async (enqueueOptions = {}) => {
+                return (await browserCrawlerEnqueueLinks({
+                    options: {
+                        ...enqueueOptions,
+                        limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
+                    },
+                    page,
+                    requestManager: await this.getRequestManager(),
+                    robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
+                    onSkippedRequest: this.handleSkippedRequest,
+                    originalRequestUrl: crawlingContext.request.url,
+                    finalRequestUrl: crawlingContext.request.loadedUrl,
+                    enqueueLinks: contextEnqueueLinks,
+                })); // TODO make this type safe
+            },
+        };
     }
-    _enhanceCrawlingContextWithPageInfo(crawlingContext, page, createNewSession) {
-        crawlingContext.page = page;
-        // This switch is because the crawlingContexts are created on per request basis.
-        // However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
-        // We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,
-        // which configures proxy and session for every new request
-        const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
-        crawlingContext.browserController = browserControllerInstance;
-        if (!createNewSession) {
-            crawlingContext.session = browserControllerInstance.launchContext.session;
-        }
-        if (!crawlingContext.proxyInfo) {
-            crawlingContext.proxyInfo = browserControllerInstance.launchContext.proxyInfo;
+    async prepareNavigation(crawlingContext) {
+        if (crawlingContext.request.skipNavigation) {
+            return {
+                request: new Proxy(crawlingContext.request, {
+                    get(target, propertyName, receiver) {
+                        if (propertyName === 'loadedUrl') {
+                            throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
+                        }
+                        return Reflect.get(target, propertyName, receiver);
+                    },
+                }),
+                get response() {
+                    throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
+                },
+            };
         }
-        crawlingContext.enqueueLinks = async (enqueueOptions) => {
-            return browserCrawlerEnqueueLinks({
-                options: enqueueOptions,
-                page,
-                requestQueue: await this.getRequestQueue(),
-                robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                onSkippedRequest: this.onSkippedRequest,
-                originalRequestUrl: crawlingContext.request.url,
-                finalRequestUrl: crawlingContext.request.loadedUrl,
-            });
+        crawlingContext.request.state = RequestState.BEFORE_NAV;
+        return {
+            gotoOptions: { timeout: this.navigationTimeoutMillis },
+            [COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),
         };
     }
-    async _handleNavigation(crawlingContext) {
-        const gotoOptions = { timeout: this.navigationTimeoutMillis };
-        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
-        crawlingContext.request.state = RequestState.BEFORE_NAV;
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
+    async navigate(crawlingContext) {
         tryCancel();
-        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
-        await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
+        const gotoOptions = crawlingContext.gotoOptions;
+        const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
+        const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);
+        await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
+        let response;
         try {
-            crawlingContext.response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
+            response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
         }
         catch (error) {
-            await this._handleNavigationTimeout(crawlingContext, error);
+            await this.handleNavigationTimeout(crawlingContext, error);
             crawlingContext.request.state = RequestState.ERROR;
-            this._throwIfProxyError(error);
+            this.throwIfProxyError(error);
             throw error;
         }
         tryCancel();
         crawlingContext.request.state = RequestState.AFTER_NAV;
-        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
+        return { response };
+    }
+    async finalizeNavigation(crawlingContext) {
+        tryCancel();
+        let response;
+        try {
+            response = crawlingContext.response;
+        }
+        catch {
+            // `preparePage` installs a throwing getter for `response`; reaching this branch means
+            // navigation produced no response and no hook overrode it. Treat as undefined.
+        }
+        await this.processResponse(response, crawlingContext);
+        tryCancel();
+        // TODO: Should we save the cookies also after/only the handle page?
+        if (this.saveResponseCookies && crawlingContext.session) {
+            const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
+            tryCancel();
+            const url = crawlingContext.request.loadedUrl;
+            for (const cookie of cookies) {
+                try {
+                    crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
+                        ignoreError: false,
+                    });
+                }
+                catch (e) {
+                    this.log.debug(`Could not set cookie: ${e.message}`);
+                }
+            }
+        }
+        return { request: crawlingContext.request };
+    }
+    async handleBlockedRequestByContent(crawlingContext) {
+        if (this.retryOnBlocked) {
+            const error = await this.isRequestBlocked(crawlingContext);
+            if (error)
+                throw new SessionError(error);
+        }
+        return {};
     }
-    async _applyCookies({ session, request, page, browserController }, preHooksCookies, postHooksCookies) {
-        const sessionCookie = session?.getCookies(request.url) ?? [];
+    async restoreRequestState(crawlingContext) {
+        crawlingContext.request.state = RequestState.REQUEST_HANDLER;
+        return {};
+    }
+    async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
+        const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];
         const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
-        await browserController.setCookies(page, [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
+        const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
             .filter((c) => typeof c !== 'undefined' && c !== null)
-            .map((c) => ({ ...c, url: c.domain ? undefined : request.url })));
+            .map((c) => ({ ...c, url: c.domain ? undefined : request.url }));
+        await this.browserPool.injectPageState(page, { cookies });
     }
     /**
-     * Marks session bad in case of navigation timeout.
+     * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
      */
-    async _handleNavigationTimeout(crawlingContext, error) {
-        const { session } = crawlingContext;
-        if (error && error.constructor.name === 'TimeoutError') {
+    async handleNavigationTimeout(crawlingContext, error) {
+        const { session, page } = crawlingContext;
+        if (error?.constructor.name === 'TimeoutError') {
             handleRequestTimeout({ session, errorMessage: error.message });
         }
-        await crawlingContext.page.close();
+        // Fire-and-forget: no user code will run on this page after a failed navigation.
+        // Swallow rejections: the page may already be detached.
+        void page.evaluate(() => window.stop()).catch(() => { });
     }
     /**
      * Transforms proxy-related errors to `SessionError`.
      */
-    _throwIfProxyError(error) {
+    throwIfProxyError(error) {
         if (this.isProxyError(error)) {
             throw new SessionError(this._getMessageFromError(error));
         }
     }
-    /**
-     * Should be overridden in case of different automation library that does not support this response API.
-     */
-    async _responseHandler(crawlingContext) {
-        const { response, session, request, page } = crawlingContext;
+    async processResponse(response, crawlingContext) {
+        const { session, request, page } = crawlingContext;
         if (typeof response === 'object' && typeof response.status === 'function') {
             const status = response.status();
             this.stats.registerStatusCode(status);
+            if (this.isErrorStatusCode(status)) {
+                if (this.additionalHttpErrorStatusCodes.has(status)) {
+                    throw new Error(`${status} - Error status code was set by user.`);
+                }
+                throw new Error(`${status} - Internal Server Error`);
+            }
         }
         if (this.sessionPool && response && session) {
             if (typeof response === 'object' && typeof response.status === 'function') {
-                this._throwOnBlockedRequest(session, response.status());
+                this._throwOnBlockedRequest(response.status());
             }
             else {
                 this.log.debug('Got a malformed Browser response.', { request, response });
@@ -323,68 +355,43 @@ export class BrowserCrawler extends BasicCrawler {
         }
         request.loadedUrl = await page.url();
     }
-    async _extendLaunchContext(_pageId, launchContext) {
-        const launchContextExtends = {};
-        if (this.sessionPool) {
-            launchContextExtends.session = await this.sessionPool.getSession();
-        }
-        if (this.proxyConfiguration && !launchContext.proxyUrl) {
-            const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id, {
-                proxyTier: launchContext.proxyTier ?? undefined,
-            });
-            launchContext.proxyUrl = proxyInfo?.url;
-            launchContextExtends.proxyInfo = proxyInfo;
-            // Disable SSL verification for MITM proxies
-            if (this.proxyConfiguration.isManInTheMiddle) {
-                /**
-                 * @see https://playwright.dev/docs/api/class-browser/#browser-new-context
-                 * @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
-                 */
-                launchContext.launchOptions.ignoreHTTPSErrors = true;
-                launchContext.launchOptions.acceptInsecureCerts = true;
-            }
-        }
-        launchContext.extend(launchContextExtends);
-    }
-    _maybeAddSessionRetiredListener(_pageId, browserController) {
-        if (this.sessionPool) {
-            const listener = (session) => {
-                const { launchContext } = browserController;
-                if (session.id === launchContext.session.id) {
-                    this.browserPool.retireBrowserController(browserController);
-                }
-            };
-            this.sessionPool.on(EVENT_SESSION_RETIRED, listener);
-            browserController.on("browserClosed" /* BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED */, () => {
-                return this.sessionPool.removeListener(EVENT_SESSION_RETIRED, listener);
-            });
-        }
-    }
     /**
      * Function for cleaning up after all requests are processed.
      * @ignore
      */
     async teardown() {
-        await this.browserPool.destroy();
+        await this.ownedBrowserPool?.destroy();
         await super.teardown();
     }
 }
 /** @internal */
-export async function browserCrawlerEnqueueLinks({ options, page, requestQueue, robotsTxtFile, onSkippedRequest, originalRequestUrl, finalRequestUrl, }) {
+function containsEnqueueLinks(options) {
+    return !!options.enqueueLinks;
+}
+/** @internal */
+export async function browserCrawlerEnqueueLinks(options) {
+    const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
     const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-        enqueueStrategy: options?.strategy,
+        enqueueStrategy: enqueueLinksOptions?.strategy,
         finalRequestUrl,
         originalRequestUrl,
-        userProvidedBaseUrl: options?.baseUrl,
+        userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
     });
-    const urls = await extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
+    const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
+    if (containsEnqueueLinks(options)) {
+        return options.enqueueLinks({
+            urls,
+            baseUrl,
+            ...enqueueLinksOptions,
+        });
+    }
     return enqueueLinks({
-        requestQueue,
-        robotsTxtFile,
-        onSkippedRequest,
+        requestManager: options.requestManager,
+        robotsTxtFile: options.robotsTxtFile,
+        onSkippedRequest: options.onSkippedRequest,
         urls,
         baseUrl,
-        ...options,
+        ...enqueueLinksOptions,
     });
 }
 /**
@@ -412,4 +419,3 @@ page, selector, baseUrl) {
     })
         .filter((href) => !!href);
 }
-//# sourceMappingURL=browser-crawler.js.map

package/internals/browser-launcher.d.ts

@@ -44,6 +44,11 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
      * to reduce the chance of detection of the crawler.
      */
     userAgent?: string;
+    /**
+     * If set to `true`, TLS certificate errors from the upstream proxy will be ignored.
+     * This is useful when using HTTPS proxies with self-signed certificates.
+     */
+    ignoreProxyCertificate?: boolean;
     /**
      * The type of browser to be launched.
      * By default, `chromium` is used. Other browsers like `webkit` or `firefox` can be used.
@@ -81,6 +86,8 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
         useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
+// @ts-ignore optional peer dependency or compatibility with es2022
+        ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
         userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -103,12 +110,11 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
      */
     launch(): LaunchResult;
     createLaunchOptions(): Dictionary;
-    protected _getDefaultHeadlessOption(): boolean;
-    protected _getChromeExecutablePath(): string;
+    protected getDefaultHeadlessOption(): boolean;
+    private getChromeExecutablePath;
     /**
      * Gets a typical path to Chrome executable, depending on the current operating system.
      */
-    protected _getTypicalChromeExecutablePath(): string;
-    protected _validateProxyUrlProtocol(proxyUrl?: string): void;
+    private getTypicalChromeExecutablePath;
+    private validateProxyUrlProtocol;
 }
-//# sourceMappingURL=browser-launcher.d.ts.map