@crawlee/browser 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/browser-crawler.js

@@ -1,7 +1,7 @@
-import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, enqueueLinks, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, resolveBaseUrlForEnqueueLinksFiltering, SessionError, toughCookieToBrowserPoolCookie, tryAbsoluteURL, validators, } from '@crawlee/basic';
-import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
-import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
-import ow from 'ow';
+import { BasicCrawler, browserPoolCookieToToughCookie, ContextPipeline, cookieStringToToughCookie, EnqueueStrategy, NavigationSkippedError, OwnedOrInjected, remainingNavigationWindowMillis, RequestState, ContextPipelineInitializationError, RequestHandlerError, RequestThrottledError, resolveBaseUrlForEnqueueLinksFiltering, SessionError, SessionRetiredError, toughCookieToBrowserPoolCookie, validators, } from '@crawlee/basic';
+import { assertBrowserPoolNotConfigured, CLOUDFLARE_RETRY_CSS_SELECTORS, parseArgument, RETRY_CSS_SELECTORS, schemas, tryAbsoluteURL, } from '@crawlee/utils/internal';
+import { sleep } from '@crawlee/utils';
+import { z } from 'zod';
 import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
 const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
 const readContextField = (ctx, key) => ctx[key];
@@ -58,43 +58,50 @@ function isNavigationTimeoutError(error) {
  */
 export class BrowserCrawler extends BasicCrawler {
     /** Backs the {@link BrowserCrawler.browserPool|`browserPool`} getter. */
-    browserPoolDep;
+    #browserPoolDep;
     /**
      * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
      * {@link IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
     get browserPool() {
-        return this.browserPoolDep.value;
+        return this.#browserPoolDep.value;
     }
-    launchContext;
     ignoreShadowRoots;
     ignoreIframes;
-    navigationTimeoutMillis;
-    preNavigationHooks;
-    postNavigationHooks;
-    saveResponseCookies;
+    #navigationTimeoutMillis;
+    #preNavigationHooks;
+    #postNavigationHooks;
+    #saveResponseCookies;
+    /**
+     * @internal
+     */
     static optionsShape = {
         ...BasicCrawler.optionsShape,
-        navigationTimeoutSecs: ow.optional.number.greaterThan(0),
-        preNavigationHooks: ow.optional.array,
-        postNavigationHooks: ow.optional.array,
-        launchContext: ow.optional.object,
-        headless: ow.optional.any(ow.boolean, ow.string),
-        browserPool: ow.optional.object.validate(validators.browserPool),
-        remoteBrowser: ow.optional.object,
-        browserPoolOptions: ow.optional.object,
-        saveResponseCookies: ow.optional.boolean,
-        proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
+        navigationTimeoutSecs: schemas.anyNumber
+            .refine((value) => value > 0, 'Expected a number greater than 0')
+            .default(60),
+        preNavigationHooks: schemas.anyArray.default(() => []),
+        postNavigationHooks: schemas.anyArray.default(() => []),
+        browserPool: validators.browserPool.optional(),
+        browserPoolBuilder: schemas.anyFunction.optional(),
+        remoteBrowser: schemas.anyObject.optional(),
+        saveResponseCookies: z.boolean().default(true),
+        proxyConfiguration: validators.proxyConfiguration.optional(),
+        ignoreIframes: z.boolean().default(false),
+        ignoreShadowRoots: z.boolean().default(false),
     };
+    /** @internal */
+    static optionsSchema = z.strictObject(BrowserCrawler.optionsShape);
     /**
      * All `BrowserCrawler` parameters are passed via an options object.
+     * @internal
      */
     constructor(options) {
-        ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
-        const { navigationTimeoutSecs = 60, saveResponseCookies = true, launchContext = {}, browserPool, remoteBrowser, browserPoolOptions, preNavigationHooks = [], postNavigationHooks = [], headless, ignoreIframes = false, ignoreShadowRoots = false, contextPipelineBuilder, extendContext, ...basicCrawlerOptions } = options;
-        const skipGuard = (action) => ({
-            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
-        });
+        const { navigationTimeoutSecs, saveResponseCookies, browserPool, remoteBrowser, preNavigationHooks, postNavigationHooks, ignoreIframes, ignoreShadowRoots, contextPipelineBuilder, browserPoolBuilder, extendContext, ...basicCrawlerOptions } = parseArgument(options, BrowserCrawler.optionsSchema, 'BrowserCrawlerOptions');
+        if (browserPool) {
+            assertBrowserPoolNotConfigured(new.target.name, { remoteBrowser });
+        }
+        const skipGuard = (action) => async (ctx) => ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {});
         super({
             ...basicCrawlerOptions,
             contextPipelineBuilder: () => {
@@ -103,107 +110,64 @@ export class BrowserCrawler extends BasicCrawler {
                 // hook eats into the same window the navigation uses. The navigation itself is bounded by
                 // capping its `gotoOptions.timeout` to the remaining budget.
                 const windowGuard = (step) => skipGuard(async (ctx) => {
-                    const remaining = remainingNavigationWindowMillis(ctx, this.navigationTimeoutMillis);
+                    const remaining = remainingNavigationWindowMillis(ctx, this.#navigationTimeoutMillis);
                     if (remaining <= 0) {
-                        throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+                        throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
                     }
-                    return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+                    return addTimeoutToPromise(async () => step(ctx), remaining, `Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
                 });
-                let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
-                for (const hook of this.preNavigationHooks) {
+                let pipeline = contextPipelineBuilder().compose(this.#prepareNavigation.bind(this));
+                for (const hook of this.#preNavigationHooks) {
                     pipeline = pipeline.compose(windowGuard(hook));
                 }
                 pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
-                for (const hook of this.postNavigationHooks) {
+                for (const hook of this.#postNavigationHooks) {
                     pipeline = pipeline.compose(windowGuard(hook));
                 }
                 return pipeline
-                    .compose(skipGuard(this.finalizeNavigation.bind(this)))
-                    .compose({ action: this.handleBlockedRequestByContent.bind(this) })
-                    .compose({ action: this.restoreRequestState.bind(this) });
+                    .compose(skipGuard(this.#finalizeNavigation.bind(this)))
+                    .compose(this.#handleBlockedRequestByContent.bind(this))
+                    .compose(this.#restoreRequestState.bind(this));
             },
             extendContext,
         });
-        this.launchContext = launchContext;
-        this.navigationTimeoutMillis = navigationTimeoutSecs * 1000;
+        this.#navigationTimeoutMillis = navigationTimeoutSecs * 1000;
         // The public option hooks are extension-aware; internal storage uses the base context type
         // (the pipeline composes hooks against the concrete context, which does not statically carry
         // `ContextExtension`). The extension members are present at runtime regardless.
-        this.preNavigationHooks = preNavigationHooks;
-        this.postNavigationHooks = postNavigationHooks;
+        this.#preNavigationHooks = preNavigationHooks;
+        this.#postNavigationHooks = postNavigationHooks;
         this.ignoreIframes = ignoreIframes;
         this.ignoreShadowRoots = ignoreShadowRoots;
-        if (headless != null) {
-            this.launchContext.launchOptions ??= {};
-            this.launchContext.launchOptions.headless = headless;
-        }
-        this.saveResponseCookies = saveResponseCookies;
-        // `browserPool` wins over `remoteBrowser` — a passed-in pool is used as-is (borrowed), the sugar is ignored.
-        // The default is only built when no pool was injected, so all the option/launchContext fiddling below stays
-        // inside the factory.
-        this.browserPoolDep = OwnedOrInjected.resolve(browserPool, () => {
-            const resolvedBrowserPoolOptions = browserPoolOptions ?? {};
-            if (launchContext?.userAgent) {
-                if (resolvedBrowserPoolOptions.useFingerprints)
-                    this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
-                resolvedBrowserPoolOptions.useFingerprints = false;
-            }
-            if (remoteBrowser) {
-                // The crawler already built the right plugin for its browser — hand it to a RemoteBrowserPool so the
-                // remote connection is always for the matching browser (no plugin to construct, no way to mismatch).
-                const { browserPlugins, ...remoteBrowserPoolOptions } = resolvedBrowserPoolOptions;
-                return new RemoteBrowserPool({
-                    browserPlugins: browserPlugins,
-                    ...remoteBrowser,
-                    browserPoolOptions: remoteBrowserPoolOptions,
-                });
-            }
-            // Double cast: `BrowserPool` implements `IBrowserPool<PageReturn>`, where `PageReturn` is derived from the
-            // plugin/controller generics and doesn't overlap with the crawler's free `Page` type param, so TS won't
-            // narrow it directly. The concrete pool does satisfy the `Page`/`destroy` contract at runtime — this is the
-            // long-standing `Page` variance gap, not a `destroy`-related hole.
-            return new BrowserPool({
-                ...resolvedBrowserPoolOptions,
-            });
-        });
+        this.#saveResponseCookies = saveResponseCookies;
+        this.#browserPoolDep = OwnedOrInjected.resolve(browserPool, () => browserPoolBuilder(remoteBrowser));
     }
+    /** @internal */
     getNavigationTimeoutMillis() {
-        return this.navigationTimeoutMillis;
+        return this.#navigationTimeoutMillis;
     }
     buildContextPipeline() {
-        return ContextPipeline.create().compose({
-            action: this.preparePage.bind(this),
-            cleanup: async (context) => {
-                context.registerDeferredCleanup(async () => {
-                    const error = !context.session.isUsable()
-                        ? new SessionError('Session is no longer usable')
-                        : undefined;
-                    await this.browserPool
-                        .closePage(context.page, { error })
-                        .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
-                });
-            },
-        });
+        return ContextPipeline.create().compose(this.#preparePage.bind(this));
     }
-    async containsSelectors(page, selectors) {
+    async #containsSelectors(page, selectors) {
         const foundSelectors = (await Promise.all(selectors.map((selector) => page.$(selector))))
             .map((x, i) => [x, selectors[i]])
             .filter(([x]) => x !== null)
             .map(([, selector]) => selector);
         return foundSelectors.length > 0 ? foundSelectors : null;
     }
-    async isRequestBlocked(crawlingContext) {
+    async #isRequestBlocked(crawlingContext) {
         const { page, response } = crawlingContext;
         // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
-        if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
+        if ((await this.#containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
             await sleep(5000);
             // here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
-            const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
+            const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
             if (!foundSelectors)
                 return false;
             return `Cloudflare challenge failed, found selectors: ${foundSelectors.join(', ')}`;
         }
-        const foundSelectors = await this.containsSelectors(page, RETRY_CSS_SELECTORS);
+        const foundSelectors = await this.#containsSelectors(page, RETRY_CSS_SELECTORS);
         const statusCode = response?.status() ?? 0;
         if (foundSelectors)
             return `Found selectors: ${foundSelectors.join(', ')}`;
@@ -211,13 +175,38 @@ export class BrowserCrawler extends BasicCrawler {
             return `Received blocked status code: ${statusCode}`;
         return false;
     }
-    async preparePage(crawlingContext) {
+    async #preparePage(crawlingContext, onCleanup) {
         const page = await this.browserPool.newPage({
             id: crawlingContext.id,
             session: crawlingContext.session,
         });
         tryCancel();
-        const contextEnqueueLinks = crawlingContext.enqueueLinks;
+        onCleanup((failure) => {
+            crawlingContext.registerDeferredCleanup(async () => {
+                const cause = failure instanceof RequestHandlerError || failure instanceof ContextPipelineInitializationError
+                    ? failure.cause
+                    : failure;
+                // Only tells the pool what to do with the page's browser state - the request's failure, if any, has
+                // already been handled. A thrown `SessionError` or a blocked session counts as a block; a session
+                // unusable for any other reason is reported as retired. `retire()`/`markBad()` also bump the usage
+                // count, so the block check must come first.
+                const { session } = crawlingContext;
+                const closeReason = cause instanceof SessionError
+                    ? cause
+                    : session.isBlocked()
+                        ? new SessionError()
+                        : session.isUsable()
+                            ? undefined
+                            : new SessionRetiredError();
+                await this.browserPool
+                    .closePage(page, { error: closeReason })
+                    .catch((closeError) => this.log.debug('Error while closing page', { error: closeError }));
+            });
+        });
+        const addRequests = crawlingContext.addRequests;
+        const extractLinks = async (options) => {
+            return extractUrlsFromPage(page, options?.selector ?? 'a', options?.baseUrl ?? crawlingContext.request.loadedUrl ?? crawlingContext.request.url);
+        };
         return {
             page,
             get response() {
@@ -226,33 +215,36 @@ export class BrowserCrawler extends BasicCrawler {
             get gotoOptions() {
                 throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
             },
-            enqueueLinks: async (enqueueOptions = {}) => {
-                return (await browserCrawlerEnqueueLinks({
-                    options: {
-                        ...enqueueOptions,
-                        limit: await this.calculateEnqueuedRequestLimit(enqueueOptions?.limit),
-                    },
-                    page,
-                    requestManager: await this.getRequestManager(),
-                    robotsTxtFile: await this.getRobotsTxtFileForUrl(crawlingContext.request.url),
-                    onSkippedRequest: this.handleSkippedRequest,
-                    originalRequestUrl: crawlingContext.request.url,
+            extractLinks,
+            enqueueLinks: async (options = {}) => {
+                const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
+                    enqueueStrategy: options.strategy,
                     finalRequestUrl: crawlingContext.request.loadedUrl,
-                    enqueueLinks: contextEnqueueLinks,
-                })); // TODO make this type safe
+                    originalRequestUrl: crawlingContext.request.url,
+                    userProvidedBaseUrl: options.baseUrl,
+                });
+                const urls = await extractLinks(options);
+                return addRequests(urls, {
+                    ...options,
+                    baseUrl,
+                    strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+                });
             },
         };
     }
-    async prepareNavigation(crawlingContext) {
+    async #prepareNavigation(crawlingContext) {
         if (crawlingContext.request.skipNavigation) {
             return {
                 request: new Proxy(crawlingContext.request, {
-                    get(target, propertyName, receiver) {
+                    get(target, propertyName) {
                         if (propertyName === 'loadedUrl') {
                             throw new NavigationSkippedError('The `request.loadedUrl` property is not available - `skipNavigation` was used');
                         }
-                        return Reflect.get(target, propertyName, receiver);
+                        // `target` as receiver and bound methods, or `#` members throw on the proxy
+                        const value = Reflect.get(target, propertyName, target);
+                        return typeof value === 'function' ? value.bind(target) : value;
                     },
+                    set: (target, propertyName, value) => Reflect.set(target, propertyName, value, target),
                 }),
                 get response() {
                     throw new NavigationSkippedError('The `response` property is not available - `skipNavigation` was used');
@@ -263,43 +255,44 @@ export class BrowserCrawler extends BasicCrawler {
         return {
             // Default to the full navigation timeout so a pre-navigation hook can read it; `navigate` narrows it
             // to the remaining shared window unless a hook overrode it (see there).
-            gotoOptions: { timeout: this.navigationTimeoutMillis },
-            [COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),
+            gotoOptions: { timeout: this.#navigationTimeoutMillis },
+            [COOKIES_BEFORE_HOOKS]: this.getCookieHeaderFromRequest(crawlingContext.request),
         };
     }
+    // oxlint-disable-next-line crawlee/prefer-private-fields -- patched by @crawlee/otel
     async navigate(crawlingContext) {
         tryCancel();
         const gotoOptions = crawlingContext.gotoOptions;
-        const remaining = remainingNavigationWindowMillis(crawlingContext, this.navigationTimeoutMillis);
+        const remaining = remainingNavigationWindowMillis(crawlingContext, this.#navigationTimeoutMillis);
         if (remaining <= 0) {
-            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
         // If a hook left the default `navigationTimeoutMillis` in place, bound the goto to whatever is left of the
         // shared navigation window. If it overrode the value - including `0`, Playwright's "no timeout" - honour
         // that verbatim as the goto's own timeout. The driver enforces this natively (so a timed-out goto is
         // aborted, not left lingering) and `handleNavigationTimeout` turns its error into our own message.
         const gotoTimeout = gotoOptions;
-        if (gotoTimeout.timeout === this.navigationTimeoutMillis) {
+        if (gotoTimeout.timeout === this.#navigationTimeoutMillis) {
             gotoTimeout.timeout = remaining;
         }
         const cookiesBeforeHooks = readContextField(crawlingContext, COOKIES_BEFORE_HOOKS);
-        const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);
-        await this.applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
+        const cookiesAfterHooks = this.getCookieHeaderFromRequest(crawlingContext.request);
+        await this.#applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
         let response;
         try {
-            response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
+            response = (await this.navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
         }
         catch (error) {
-            await this.handleNavigationTimeout(crawlingContext, error);
+            await this.#handleNavigationTimeout(crawlingContext, error);
             crawlingContext.request.state = RequestState.ERROR;
-            this.throwIfProxyError(error);
+            this.#throwIfProxyError(error);
             throw error;
         }
         tryCancel();
         crawlingContext.request.state = RequestState.AFTER_NAV;
         return { response };
     }
-    async finalizeNavigation(crawlingContext) {
+    async #finalizeNavigation(crawlingContext) {
         tryCancel();
         let response;
         try {
@@ -309,18 +302,18 @@ export class BrowserCrawler extends BasicCrawler {
             // `preparePage` installs a throwing getter for `response`; reaching this branch means
             // navigation produced no response and no hook overrode it. Treat as undefined.
         }
-        await this.processResponse(response, crawlingContext);
+        await this.#processResponse(response, crawlingContext);
         tryCancel();
         // Persist cookies from the navigation response before the user handler runs.
         // Cookies set during `requestHandler` are saved again afterwards.
-        await this.persistCookiesFromPage(crawlingContext);
+        await this.#persistCookiesFromPage(crawlingContext);
         return { request: crawlingContext.request };
     }
     /**
      * Copies cookies from the live browser page into the session cookie jar.
      */
-    async persistCookiesFromPage(crawlingContext) {
-        if (!this.saveResponseCookies || !crawlingContext.session) {
+    async #persistCookiesFromPage(crawlingContext) {
+        if (!this.#saveResponseCookies || !crawlingContext.session) {
             return;
         }
         const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
@@ -329,7 +322,7 @@ export class BrowserCrawler extends BasicCrawler {
         const url = (await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
         for (const cookie of cookies) {
             try {
-                crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
+                await crawlingContext.session.cookieJar.setCookie(browserPoolCookieToToughCookie(cookie), url, {
                     ignoreError: false,
                 });
             }
@@ -341,6 +334,7 @@ export class BrowserCrawler extends BasicCrawler {
     /**
      * Runs the user request handler, then re-reads browser cookies so login flows /
      * `page.setCookie` / XHR `Set-Cookie` updates are stored for later requests.
+     * @internal
      */
     async runRequestHandler(crawlingContext) {
         try {
@@ -349,7 +343,7 @@ export class BrowserCrawler extends BasicCrawler {
         finally {
             if (!crawlingContext.request.skipNavigation) {
                 try {
-                    await this.persistCookiesFromPage(crawlingContext);
+                    await this.#persistCookiesFromPage(crawlingContext);
                 }
                 catch {
                     // Page may already be closed on some failure paths; ignore.
@@ -357,20 +351,22 @@ export class BrowserCrawler extends BasicCrawler {
             }
         }
     }
-    async handleBlockedRequestByContent(crawlingContext) {
+    async #handleBlockedRequestByContent(crawlingContext) {
         if (this.retryOnBlocked) {
-            const error = await this.isRequestBlocked(crawlingContext);
+            const error = await this.#isRequestBlocked(crawlingContext);
             if (error)
                 throw new SessionError(error);
         }
         return {};
     }
-    async restoreRequestState(crawlingContext) {
+    async #restoreRequestState(crawlingContext) {
         crawlingContext.request.state = RequestState.REQUEST_HANDLER;
         return {};
     }
-    async applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
-        const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];
+    async #applyCookies({ session, request, page }, preHooksCookies, postHooksCookies) {
+        const sessionCookie = session
+            ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
+            : [];
         const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
@@ -381,7 +377,7 @@ export class BrowserCrawler extends BasicCrawler {
     /**
      * Marks session bad on navigation timeout, and stops in-flight page loading on any navigation error.
      */
-    async handleNavigationTimeout(crawlingContext, error) {
+    async #handleNavigationTimeout(crawlingContext, error) {
         const { session, page } = crawlingContext;
         // Fire-and-forget: no user code will run on this page after a failed navigation.
         // Swallow rejections: the page may already be detached.
@@ -390,22 +386,31 @@ export class BrowserCrawler extends BasicCrawler {
             session?.markBad();
             // The driver was handed the remaining window (usually shorter than `navigationTimeoutSecs` once the
             // hooks have run), so it names that value in its own error; report the configured window instead.
-            throw new TimeoutError(`Navigation timed out after ${this.navigationTimeoutMillis / 1000} seconds.`);
+            throw new TimeoutError(`Navigation timed out after ${this.#navigationTimeoutMillis / 1000} seconds.`);
         }
     }
     /**
      * Transforms proxy-related errors to `SessionError`.
      */
-    throwIfProxyError(error) {
+    #throwIfProxyError(error) {
         if (this.isProxyError(error)) {
-            throw new SessionError(this._getMessageFromError(error));
+            throw new SessionError(this.getMessageFromError(error));
         }
     }
-    async processResponse(response, crawlingContext) {
+    async #processResponse(response, crawlingContext) {
         const { session, request, page } = crawlingContext;
         if (typeof response === 'object' && typeof response.status === 'function') {
             const status = response.status();
-            this.stats.registerStatusCode(status);
+            this.statistics.registerStatusCode(status);
+            // Ahead of the error-status throw below: a 429 the user opted into treating as an error is still a
+            // rate limit the domain should back off from.
+            if (status === 429) {
+                // Both drivers lower-case header names and join duplicates, so a plain lookup is enough.
+                const retryAfter = response.headers?.()['retry-after'];
+                if (this.recordDomainRateLimit(crawlingContext.request.url, retryAfter)) {
+                    throw new RequestThrottledError(`${crawlingContext.request.url} responded with 429.`);
+                }
+            }
             if (this.isErrorStatusCode(status)) {
                 if (this.additionalHttpErrorStatusCodes.has(status)) {
                     throw new Error(`${status} - Error status code was set by user.`);
@@ -415,7 +420,7 @@ export class BrowserCrawler extends BasicCrawler {
         }
         if (this.sessionPool && response && session) {
             if (typeof response === 'object' && typeof response.status === 'function') {
-                this._throwOnBlockedRequest(response.status());
+                this.throwOnBlockedRequest(response.status());
             }
             else {
                 this.log.debug('Got a malformed Browser response.', { request, response });
@@ -424,43 +429,17 @@ export class BrowserCrawler extends BasicCrawler {
         request.loadedUrl = await page.url();
     }
     /**
-     * Function for cleaning up after all requests are processed.
-     * @ignore
+     * Closes the browsers of a pool the crawler owns, so a finished run leaves none behind. The pool itself is
+     * crawler-lifetime and survives — destroying it here would hand a repeated `run()` a dead pool.
      */
     async teardown() {
-        await this.browserPoolDep.ifOwned((pool) => pool.destroy());
+        await this.#browserPoolDep.ifOwned((pool) => pool.releaseAllBrowsers());
         await super.teardown();
     }
-}
-/** @internal */
-function containsEnqueueLinks(options) {
-    return !!options.enqueueLinks;
-}
-/** @internal */
-export async function browserCrawlerEnqueueLinks(options) {
-    const { options: enqueueLinksOptions, finalRequestUrl, originalRequestUrl, page } = options;
-    const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
-        enqueueStrategy: enqueueLinksOptions?.strategy,
-        finalRequestUrl,
-        originalRequestUrl,
-        userProvidedBaseUrl: enqueueLinksOptions?.baseUrl,
-    });
-    const urls = await extractUrlsFromPage(page, enqueueLinksOptions?.selector ?? 'a', enqueueLinksOptions?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
-    if (containsEnqueueLinks(options)) {
-        return options.enqueueLinks({
-            urls,
-            baseUrl,
-            ...enqueueLinksOptions,
-        });
+    async destroy() {
+        await super.destroy();
+        await this.#browserPoolDep.ifOwned((pool) => pool.destroy());
     }
-    return enqueueLinks({
-        requestManager: options.requestManager,
-        robotsTxtFile: options.robotsTxtFile,
-        onSkippedRequest: options.onSkippedRequest,
-        urls,
-        baseUrl,
-        ...enqueueLinksOptions,
-    });
 }
 /**
  * Extracts URLs from a given page.