@crawlee/playwright 4.0.0-rc.0 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/index.d.ts

@@ -1,7 +1,8 @@
 export * from '@crawlee/browser';
 export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
-export * from './internals/playwright-launcher.js';
+export { launchPlaywright } from './internals/playwright-launcher.js';
+export type { PlaywrightLaunchContext } from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';
 export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction.js';
 export * as playwrightUtils from './internals/utils/playwright-utils.js';

package/index.js

@@ -1,7 +1,7 @@
 export * from '@crawlee/browser';
 export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
-export * from './internals/playwright-launcher.js';
+export { launchPlaywright } from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';
 export { RenderingTypePredictor } from './internals/utils/rendering-type-prediction.js';
 export * as playwrightUtils from './internals/utils/playwright-utils.js';

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,15 +1,14 @@
-import type { BrowserHook, LoadedRequest, Request, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
-import type { BasicCrawlerOptions } from '@crawlee/basic';
+import type { BrowserHook, LoadedRequest, CrawlingRequest, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
+import type { BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StorageTransactionView } from '@crawlee/core';
+import type { StorageTransactionView } from '@crawlee/core';
 import type { Dictionary, Awaitable } from '@crawlee/types';
-import { type CheerioRoot } from '@crawlee/utils/internal';
-import { type Cheerio } from 'cheerio';
+import { type Cheerio, type CheerioAPI } from 'cheerio';
 import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
 import { z } from 'zod';
-import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
+import type { PlaywrightCrawlerOptions, PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
 import { type IRenderingTypePredictor } from './utils/rendering-type-prediction.js';
 declare const adaptiveStatisticStateSchema: z.ZodObject<{
     httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
@@ -35,7 +34,7 @@ export declare const adaptivePlaywrightCrawlerStatisticState: {
     }, z.core.$strip>;
 };
 export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
-    request: LoadedRequest<Request<UserData>>;
+    request: LoadedRequest<CrawlingRequest<UserData>>;
     /**
      * The HTTP response, either from the HTTP client or from the initial request from playwright's navigation.
      */
@@ -80,19 +79,19 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
      * });
      * ```
      */
-    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
+    parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioAPI>;
     enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
 }
 interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id' | 'session' | 'proxyInfo' | 'log'> {
     page?: Page;
-    request: Request;
+    request: CrawlingRequest;
     gotoOptions?: PlaywrightGotoOptions;
 }
 type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHookContext, ContextExtension>;
 type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
-    request: LoadedRequest<Request>;
+    request: LoadedRequest<CrawlingRequest>;
 }, ContextExtension>;
-export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks'> {
+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks' | 'contextPipelineBuilder'>, Pick<PlaywrightCrawlerOptions, 'launchContext' | 'headless' | 'browserPool' | 'remoteBrowser'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -186,29 +185,27 @@ export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<nev
     #private;
     constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
     protected init(): Promise<void>;
-    protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
-        readonly request: LoadedRequest<Request<Dictionary>>;
-        readonly response: Response;
-        readonly page: Page;
-        readonly querySelector: AdaptivePlaywrightCrawlerContext["querySelector"];
-        readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];
-        readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
-        readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
-        readonly enqueueLinks: AdaptivePlaywrightCrawlerContext["enqueueLinks"];
-    }>;
-    private adaptCheerioContext;
-    private adaptPlaywrightContext;
-    /**
-     * Runs one request handler attempt inside its own {@link StorageTransaction}, wrapping the inner
-     * (static or browser) context pipeline. The transaction is pushed to `transactions` *at creation
-     * time, before the `try`* - the `ok: false` branch of the returned {@link Result} carries no
-     * result, and failed attempts are routine here. The caller owns the outcome and disposal.
-     */
-    private crawlOne;
     protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
-    private enqueueLinks;
-    private createLogProxy;
+    /**
+     * Number of rendering type detections that have not settled yet, including results the predictor is
+     * still persisting.
+     */
+    get inFlightRenderingTypeDetectionCount(): number;
+    /**
+     * Waits for in-flight rendering type detections to settle, bounded by `timeoutMillis` (defaults to the
+     * internal timeout).
+     */
+    drainRenderingDetections({ timeoutMillis }?: {
+        timeoutMillis?: number;
+    }): Promise<void>;
+    /**
+     * Stops the crawler immediately, but not before rendering type detections already under way (and results
+     * the predictor is still persisting) have settled - see
+     * {@link AdaptivePlaywrightCrawler.drainRenderingDetections|`drainRenderingDetections()`}. Requests
+     * that are still running are not waited for, unlike {@link BasicCrawler.stop|`stop()`}.
+     */
     teardown(): Promise<void>;
+    destroy(): Promise<void>;
 }
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;

package/internals/adaptive-playwright-crawler.js

@@ -1,9 +1,10 @@
+import { setTimeout as delay } from 'node:timers/promises';
 import { isDeepStrictEqual } from 'node:util';
-import { BasicCrawler } from '@crawlee/basic';
+import { BasicCrawler, ContextPipeline, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
-import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected, parseArgument, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/core';
-import { extractUrlsFromCheerio } from '@crawlee/utils/internal';
+import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected } from '@crawlee/core';
+import { extractUrlsFromCheerio, parseArgument } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { PlaywrightCrawler } from './playwright-crawler.js';
@@ -76,9 +77,20 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
      * a discarded attempt's enqueues must never reach the queue.
      */
     #attemptWritePolicy;
-    #teardownHooks = [];
+    /** Owns the browser pool this crawler's runs use, so its per-run resources are released with ours. */
+    #browserCrawler;
+    /** Nothing of its state is per-run, but it owns a session pool that outlives one. */
+    #staticCrawler;
+    /**
+     * In-flight rendering type detections, plus the pending results of an asynchronous `storeResult`.
+     */
+    #activeDetections = new Set();
+    /**
+     * Set once `teardown()` starts, so that requests still in the pool stop opening new detections.
+     */
+    #shutDown = false;
     constructor(options = {}) {
-        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, transactionalStorage, ...rest } = options;
+        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, transactionalStorage, launchContext, headless, browserPool, remoteBrowser, ...rest } = options;
         // The user's value is replaced by `false` in the `super` call below — validate it separately,
         // wrapped in an object so the error still names the field.
         parseArgument({ transactionalStorage }, z.object({ transactionalStorage: BasicCrawler.optionsShape.transactionalStorage }), 'AdaptivePlaywrightCrawlerOptions');
@@ -114,7 +126,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     logMessage: `${AdaptivePlaywrightCrawler.name} request statistics:`,
                     stateExtension: adaptivePlaywrightCrawlerStatisticState,
                 }),
-            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+            contextPipelineBuilder: () => this.#buildContextPipeline(),
             // The base crawler must not wrap requests in a transaction of its own - this crawler opens
             // one per request handler attempt in `crawlOne` instead, forwarding the write policy of the
             // user-facing option (validated above) to those.
@@ -166,53 +178,54 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             preNavigationHooks: preNavigationHooks,
             postNavigationHooks: postNavigationHooks,
             extendContext,
+            launchContext,
+            headless,
+            browserPool,
+            remoteBrowser,
         });
-        this.#teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
-        this.#staticContextPipeline = staticCrawler.contextPipeline.compose({
-            action: this.adaptCheerioContext.bind(this),
-        });
-        this.#browserContextPipeline = browserCrawler.contextPipeline.compose({
-            action: this.adaptPlaywrightContext.bind(this),
-        });
+        this.#staticCrawler = staticCrawler;
+        this.#browserCrawler = browserCrawler;
+        this.#staticContextPipeline = staticCrawler.contextPipeline.compose(this.#adaptCheerioContext.bind(this));
+        this.#browserContextPipeline = browserCrawler.contextPipeline.compose(this.#adaptPlaywrightContext.bind(this));
     }
     async init() {
+        // A crawler can be run again after a teardown.
+        this.#shutDown = false;
         // Only the predictor we built ourselves is ours to initialize - an injected one is borrowed, so its
         // lifecycle (including restoring persisted state) stays with whoever created it.
         await this.#renderingTypePredictor.ifOwned((predictor) => predictor.initialize());
         return await super.init();
     }
-    buildContextPipeline() {
+    #buildContextPipeline() {
         const errorMessage = (prop) => `The \`${prop}\` property is not available on the outer context pipeline of AdaptivePlaywrightCrawler - it is provided by the inner (static/browser) pipelines`;
-        return super.buildContextPipeline().compose({
-            action: async ({ request }) => ({
-                get request() {
-                    return request;
-                },
-                get response() {
-                    throw new Error(errorMessage('response'));
-                },
-                get page() {
-                    throw new Error(errorMessage('page'));
-                },
-                get querySelector() {
-                    throw new Error(errorMessage('querySelector'));
-                },
-                get querySelectorAll() {
-                    throw new Error(errorMessage('querySelectorAll'));
-                },
-                get waitForSelector() {
-                    throw new Error(errorMessage('waitForSelector'));
-                },
-                get parseWithCheerio() {
-                    throw new Error(errorMessage('parseWithCheerio'));
-                },
-                get enqueueLinks() {
-                    throw new Error(errorMessage('enqueueLinks'));
-                },
-            }),
-        });
+        return ContextPipeline.create().compose(async ({ request }) => ({
+            get request() {
+                return request;
+            },
+            get response() {
+                throw new Error(errorMessage('response'));
+            },
+            get page() {
+                throw new Error(errorMessage('page'));
+            },
+            get querySelector() {
+                throw new Error(errorMessage('querySelector'));
+            },
+            get querySelectorAll() {
+                throw new Error(errorMessage('querySelectorAll'));
+            },
+            get waitForSelector() {
+                throw new Error(errorMessage('waitForSelector'));
+            },
+            get parseWithCheerio() {
+                throw new Error(errorMessage('parseWithCheerio'));
+            },
+            get enqueueLinks() {
+                throw new Error(errorMessage('enqueueLinks'));
+            },
+        }));
     }
-    async adaptCheerioContext(cheerioContext) {
+    async #adaptCheerioContext(cheerioContext) {
         return {
             get page() {
                 throw new Error('Page object was used in HTTP-only request handler');
@@ -225,12 +238,12 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
             enqueueLinks: async (options = {}) => {
                 const urls = extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
-                return (await this.enqueueLinks(urls, options, cheerioContext.request));
+                return (await this.#enqueueLinks(urls, options, cheerioContext.request));
             },
             response: cheerioContext.response,
         };
     }
-    async adaptPlaywrightContext(playwrightContext) {
+    async #adaptPlaywrightContext(playwrightContext) {
         // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const originalResponse = playwrightContext.response;
         return {
@@ -257,7 +270,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 const locator = playwrightContext.page.locator(selector).first();
                 await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
                 const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl);
-                return (await this.enqueueLinks(urls, options, playwrightContext.request));
+                return (await this.#enqueueLinks(urls, options, playwrightContext.request));
             },
         };
     }
@@ -267,7 +280,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
      * time, before the `try`* - the `ok: false` branch of the returned {@link Result} carries no
      * result, and failed attempts are routine here. The caller owns the outcome and disposal.
      */
-    async crawlOne(renderingType, context, useStateFunction, transactions) {
+    async #crawlOne(renderingType, context, useStateFunction, transactions) {
         const transaction = createStorageTransaction({
             policy: this.#attemptWritePolicy,
             commitTimeoutMillis: this.internalTimeoutMillis,
@@ -277,7 +290,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         const deferredCleanup = [];
         const attemptBoundContextHelpers = {
             useState: useStateFunction,
-            log: this.createLogProxy(context.log, logs),
+            log: this.#createLogProxy(context.log, logs),
             registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
         };
         const subCrawlerContext = Object.defineProperties({}, Object.getOwnPropertyDescriptors(context));
@@ -287,12 +300,16 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             Object.defineProperty(subCrawlerContext, key, { ...descriptor, configurable: false });
         }
         try {
+            // Any failure - middleware or handler - ends up in the `ok: false` branch below.
+            const rethrow = (error) => {
+                throw error;
+            };
             const callAdaptiveRequestHandler = async () => {
                 if (renderingType === 'static') {
-                    await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
+                    await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this), rethrow);
                 }
                 else if (renderingType === 'clientOnly') {
-                    await this.#browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
+                    await this.#browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this), rethrow);
                 }
             };
             // this crawler overrides `runRequestHandler` and times each rendering-type run itself, so it has
@@ -310,8 +327,8 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         }
     }
     async runRequestHandler(crawlingContext) {
-        const renderingTypePrediction = this.#renderingTypePredictor.value.predict(crawlingContext.request);
-        const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
+        const renderingTypePrediction = await this.#renderingTypePredictor.value.predict(crawlingContext.request);
+        const shouldDetectRenderingType = !this.#shutDown && Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
         if (!shouldDetectRenderingType) {
             crawlingContext.log.debug(`Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`);
         }
@@ -323,7 +340,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
                 crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
                 this.statistics.state.httpOnlyRequestHandlerRuns++;
-                const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
+                const plainHTTPRun = await this.#crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
                 if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {
                     crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
                     plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
@@ -332,10 +349,12 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 }
                 // Execution will "fall through" and try running the request handler in a browser
                 if (!plainHTTPRun.ok) {
-                    const actualError = plainHTTPRun.error instanceof RequestHandlerError
+                    const actualError = plainHTTPRun.error instanceof RequestHandlerError ||
+                        plainHTTPRun.error instanceof ContextPipelineInitializationError
                         ? plainHTTPRun.error.cause
                         : plainHTTPRun.error;
-                    if (await this.#shouldPropagateError(actualError, crawlingContext)) {
+                    if (actualError instanceof ContextPipelineInterruptedError ||
+                        (await this.#shouldPropagateError(actualError, crawlingContext))) {
                         throw actualError;
                     }
                     crawlingContext.log.exception(actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
@@ -367,34 +386,43 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     return this.stateCopy;
                 },
             };
-            const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker), transactions);
+            const browserRun = await this.#crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker), transactions);
             if (!browserRun.ok) {
                 throw browserRun.error;
             }
             browserRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
             await browserRun.result.commit();
             if (shouldDetectRenderingType) {
-                crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
-                // The detection attempt's transaction is never committed - its writes exist only for the
-                // result comparison.
-                const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker), transactions);
-                const detectionResult = (() => {
-                    if (!plainHTTPRun.ok) {
-                        return 'clientOnly';
-                    }
-                    const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
-                    if (comparisonResult === true || comparisonResult === 'equal') {
-                        return 'static';
-                    }
-                    if (comparisonResult === false || comparisonResult === 'different') {
-                        return 'clientOnly';
+                const detectionPromise = (async () => {
+                    crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
+                    // The detection attempt's transaction is never committed - its writes exist only for the
+                    // result comparison.
+                    const plainHTTPRun = await this.#crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker), transactions);
+                    const detectionResult = (() => {
+                        if (!plainHTTPRun.ok) {
+                            return 'clientOnly';
+                        }
+                        const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
+                        if (comparisonResult === true || comparisonResult === 'equal') {
+                            return 'static';
+                        }
+                        if (comparisonResult === false || comparisonResult === 'different') {
+                            return 'clientOnly';
+                        }
+                        return undefined;
+                    })();
+                    crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
+                    if (detectionResult !== undefined) {
+                        // Deliberately not awaited: a predictor that persists asynchronously gets to keep
+                        // batching its writes, and the drain below catches whatever is still pending.
+                        const stored = this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
+                        if (stored !== undefined) {
+                            // Nothing downstream awaits this, so a failed write would otherwise be silent.
+                            void this.#trackDetection(Promise.resolve(stored).catch((error) => this.log.exception(error, `Failed to store the rendering type detection result for ${crawlingContext.request.url}`)));
+                        }
                     }
-                    return undefined;
                 })();
-                crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
-                if (detectionResult !== undefined) {
-                    this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
-                }
+                await this.#trackDetection(detectionPromise);
             }
         }
         finally {
@@ -405,7 +433,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             }
         }
     }
-    async enqueueLinks(urls, options, request) {
+    async #enqueueLinks(urls, options, request) {
         const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
             enqueueStrategy: options?.strategy,
             finalRequestUrl: request.loadedUrl,
@@ -421,23 +449,94 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             strategy: options.strategy ?? EnqueueStrategy.SameHostname,
         });
     }
-    createLogProxy(log, logs) {
+    #createLogProxy(log, logs) {
         return new Proxy(log, {
-            get(target, propertyName, receiver) {
+            get(target, propertyName) {
                 if (proxyLogMethods.includes(propertyName)) {
                     return (...args) => {
                         logs.push([target, propertyName, ...args]);
                     };
                 }
-                return Reflect.get(target, propertyName, receiver);
+                const value = Reflect.get(target, propertyName, target);
+                // Bind non-intercepted methods to the target instance so private #-fields
+                // (e.g. BaseCrawleeLogger.#options, #warningsLogged) do not throw TypeError at runtime.
+                if (typeof value === 'function') {
+                    return value.bind(target);
+                }
+                return value;
             },
         });
     }
+    #trackDetection(promise) {
+        this.#activeDetections.add(promise);
+        // Not `finally()`: the promise it derives would reject on its own and go unhandled. A rejection here
+        // belongs to whoever awaits the original, or to `allSettled` in the drain.
+        void promise.catch(() => { }).then(() => this.#activeDetections.delete(promise));
+        return promise;
+    }
+    /**
+     * Number of rendering type detections that have not settled yet, including results the predictor is
+     * still persisting.
+     */
+    get inFlightRenderingTypeDetectionCount() {
+        return this.#activeDetections.size;
+    }
+    /**
+     * Waits for in-flight rendering type detections to settle, bounded by `timeoutMillis` (defaults to the
+     * internal timeout).
+     */
+    async drainRenderingDetections({ timeoutMillis } = {}) {
+        if (this.#activeDetections.size === 0) {
+            return;
+        }
+        const drained = (async () => {
+            while (this.#activeDetections.size > 0) {
+                await Promise.allSettled(Array.from(this.#activeDetections));
+            }
+        })();
+        const millis = timeoutMillis ?? this.internalTimeoutMillis;
+        // A caller opting out of the bound would otherwise get an immediate spurious timeout - `setTimeout`
+        // clamps a non-finite delay to 1ms.
+        if (!Number.isFinite(millis)) {
+            await drained;
+            return;
+        }
+        const abortTimer = new AbortController();
+        try {
+            const outcome = await Promise.race([
+                drained.then(() => 'drained'),
+                delay(millis, 'timedOut', { signal: abortTimer.signal }).catch(() => 'aborted'),
+            ]);
+            if (outcome === 'timedOut') {
+                this.log.warning(`Timed out after ${millis / 1e3} seconds waiting for ${this.#activeDetections.size} rendering type detection(s) to settle - their results may be lost.`);
+            }
+        }
+        finally {
+            abortTimer.abort();
+        }
+    }
+    /**
+     * Stops the crawler immediately, but not before rendering type detections already under way (and results
+     * the predictor is still persisting) have settled - see
+     * {@link AdaptivePlaywrightCrawler.drainRenderingDetections|`drainRenderingDetections()`}. Requests
+     * that are still running are not waited for, unlike {@link BasicCrawler.stop|`stop()`}.
+     */
     async teardown() {
+        // Called from outside `run()` - under `keepAlive`, say - the pool keeps dispatching until
+        // `super.teardown()` aborts it, and a request starting during the drain would open a detection the
+        // drain has already passed. Closing that first makes the drain a fence.
+        this.#shutDown = true;
+        await this.drainRenderingDetections();
         await super.teardown();
-        for (const hook of this.#teardownHooks) {
-            await hook();
-        }
+        // Mirrors the owned-only `initialize()` in `init()` - without this, the predictor we built keeps its
+        // PERSIST_STATE listener registered after the crawl and never gets a final write.
+        await this.#renderingTypePredictor.ifOwned((predictor) => predictor.teardown());
+        await this.#browserCrawler.teardown();
+    }
+    async destroy() {
+        await super.destroy();
+        await this.#staticCrawler.destroy();
+        await this.#browserCrawler.destroy();
     }
 }
 export function createAdaptivePlaywrightRouter(routesOrSchemas) {

package/internals/enqueue-links/click-elements.js

@@ -1,5 +1,6 @@
 import { URL } from 'node:url';
-import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, filterRequestOptionsByPatterns, parseArgument, urlPatternSchema, Request as CrawleeRequest, schemas, serviceLocator, } from '@crawlee/browser';
+import { applyRequestTransform, constructUrlPatternObjects, createRequestOptions, createSkippedRequestArgs, filterRequestOptionsByPatterns, urlPatternSchema, Request as CrawleeRequest, serviceLocator, } from '@crawlee/browser';
+import { parseArgument, schemas } from '@crawlee/utils/internal';
 import { z } from 'zod';
 const STARTING_Z_INDEX = 2147400000;
 const getLog = () => serviceLocator.getChildLog('Playwright Click Elements');
@@ -64,6 +65,7 @@ export async function enqueueLinksByClickingElements(options) {
     const { page, requestManager, selector, clickOptions, include, exclude, transformRequestFunction, waitForPageIdleSecs, maxWaitForPageIdleSecs, forefront, onSkippedRequest, } = parsedOptions;
     const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
     const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;
+    const hasOnSkippedRequest = onSkippedRequest !== undefined;
     const urlExcludePatternObjects = exclude?.length ? constructUrlPatternObjects(exclude) : [];
     const urlPatternObjects = include?.length ? constructUrlPatternObjects(include) : [];
     const interceptedRequests = await clickElementsAndInterceptNavigationRequests({
@@ -75,15 +77,15 @@ export async function enqueueLinksByClickingElements(options) {
     });
     const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);
     const skippedByFilters = [];
-    let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, (url) => skippedByFilters.push(url));
+    let filteredOptions = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects.length > 0 ? urlPatternObjects : undefined, urlExcludePatternObjects, undefined, hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined);
     if (onSkippedRequest && skippedByFilters.length > 0) {
-        await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));
+        await Promise.all(skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))));
     }
     if (transformRequestFunction) {
         const skippedByTransform = [];
-        filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) => skippedByTransform.push(r));
+        filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined);
         if (onSkippedRequest && skippedByTransform.length > 0) {
-            await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
+            await Promise.all(skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))));
         }
     }
     const requests = filteredOptions.map((opts) => new CrawleeRequest(opts));