@crawlee/playwright 4.0.0-beta.26 → 4.0.0-beta.260

package/internals/adaptive-playwright-crawler.js

@@ -1,47 +1,31 @@
+import { setTimeout as delay } from 'node:timers/promises';
 import { isDeepStrictEqual } from 'node:util';
-import { BasicCrawler } from '@crawlee/basic';
-import { extractUrlsFromPage } from '@crawlee/browser';
+import { BasicCrawler, ContextPipeline, ContextPipelineInitializationError, ContextPipelineInterruptedError, navigationDeadlineKey, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/basic';
+import { CrawlingRequest, extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
-import { Configuration, RequestHandlerError, RequestHandlerResult, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, withCheckedStorageAccess, } from '@crawlee/core';
-import { extractUrlsFromCheerio } from '@crawlee/utils';
+import { ResponseWithUrl } from '@crawlee/http-client';
+import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected } from '@crawlee/core';
+import { extractUrlsFromCheerio, MutationJournal, parseArgument } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { PlaywrightCrawler } from './playwright-crawler.js';
-import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
-class AdaptivePlaywrightCrawlerStatistics extends Statistics {
-    state = null; // this needs to be assigned for a valid override, but the initialization is done by a reset() call from the parent constructor
-    constructor(options = {}) {
-        super(options);
-        this.reset();
-    }
-    reset() {
-        super.reset();
-        this.state.httpOnlyRequestHandlerRuns = 0;
-        this.state.browserRequestHandlerRuns = 0;
-        this.state.renderingTypeMispredictions = 0;
-    }
-    async _maybeLoadStatistics() {
-        await super._maybeLoadStatistics();
-        const savedState = await this.keyValueStore?.getValue(this.persistStateKey);
-        if (!savedState) {
-            return;
-        }
-        this.state.httpOnlyRequestHandlerRuns = savedState.httpOnlyRequestHandlerRuns;
-        this.state.browserRequestHandlerRuns = savedState.browserRequestHandlerRuns;
-        this.state.renderingTypeMispredictions = savedState.renderingTypeMispredictions;
-    }
-    trackHttpOnlyRequestHandlerRun() {
-        this.state.httpOnlyRequestHandlerRuns ??= 0;
-        this.state.httpOnlyRequestHandlerRuns += 1;
-    }
-    trackBrowserRequestHandlerRun() {
-        this.state.browserRequestHandlerRuns ??= 0;
-        this.state.browserRequestHandlerRuns += 1;
-    }
-    trackRenderingTypeMisprediction() {
-        this.state.renderingTypeMispredictions ??= 0;
-        this.state.renderingTypeMispredictions += 1;
-    }
-}
+import { RenderingTypePredictor, } from './utils/rendering-type-prediction.js';
+const adaptiveStatisticStateSchema = z.object({
+    /** How many requests were handled by the HTTP-only request handler. */
+    httpOnlyRequestHandlerRuns: z.number().default(0),
+    /** How many requests were handled in a browser. */
+    browserRequestHandlerRuns: z.number().default(0),
+    /** How many times the HTTP-only handler produced a result the `resultChecker` rejected. */
+    renderingTypeMispredictions: z.number().default(0),
+});
+/**
+ * The {@link AdaptivePlaywrightCrawlerStatisticState} fields as a {@link Statistics} state extension, defaults
+ * and all. A {@link Statistics} instance to be injected into an {@link AdaptivePlaywrightCrawler} has to carry
+ * them - `deserialize.extend()` your own fields onto this one and pass the result as `stateExtension`.
+ */
+export const adaptivePlaywrightCrawlerStatisticState = {
+    deserialize: adaptiveStatisticStateSchema,
+};
 const proxyLogMethods = [
     'error',
     'exception',
@@ -60,9 +44,9 @@ const proxyLogMethods = [
  * ```javascript
  * const crawler = new AdaptivePlaywrightCrawler({
  *     renderingTypeDetectionRatio: 0.1,
- *     async requestHandler({ querySelector, pushData, enqueueLinks, request, log }) {
+ *     async requestHandler({ querySelectorAll, pushData, enqueueLinks, request, log }) {
  *         // This function is called to extract data from a single web page
- *         const $prices = await querySelector('span.price')
+ *         const $prices = await querySelectorAll('span.price')
  *
  *         await pushData({
  *             url: request.url,
@@ -78,46 +62,93 @@ const proxyLogMethods = [
  *     'http://www.example.com/page-2',
  * ]);
  * ```
- *
- * @experimental
  */
 export class AdaptivePlaywrightCrawler extends BasicCrawler {
-    config;
-    renderingTypePredictor;
-    resultChecker;
-    resultComparator;
-    preventDirectStorageAccess;
-    staticContextPipeline;
-    browserContextPipeline;
-    individualRequestHandlerTimeoutMillis;
-    resultObjects = new WeakMap();
-    teardownHooks = [];
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
-        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, resultComparator, statisticsOptions, preventDirectStorageAccess = true, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks, postNavigationHooks, extendContext, contextPipelineBuilder, ...rest } = options;
+    #renderingTypePredictor;
+    #resultChecker;
+    #shouldPropagateError;
+    #resultComparator;
+    #staticContextPipeline;
+    #browserContextPipeline;
+    #individualRequestHandlerTimeoutMillis;
+    /**
+     * The write policy of the per-attempt transactions. Defaults the request queue to `deferred`:
+     * a discarded attempt's enqueues must never reach the queue.
+     */
+    #attemptWritePolicy;
+    /** Owns the browser pool this crawler's runs use, so its per-run resources are released with ours. */
+    #browserCrawler;
+    /** Nothing of its state is per-run, but it owns a session pool that outlives one. */
+    #staticCrawler;
+    /**
+     * In-flight rendering type detections, plus the pending results of an asynchronous `storeResult`.
+     */
+    #activeDetections = new Set();
+    /**
+     * Set once `teardown()` starts, so that requests still in the pool stop opening new detections.
+     */
+    #shutDown = false;
+    constructor(options = {}) {
+        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, transactionalStorage, navigationTimeoutSecs, launchContext, headless, browserPool, remoteBrowser, ...rest } = options;
+        // The user's value is replaced by `false` in the `super` call below — validate it separately,
+        // wrapped in an object so the error still names the field.
+        parseArgument({ transactionalStorage }, z.object({ transactionalStorage: BasicCrawler.optionsShape.transactionalStorage }), 'AdaptivePlaywrightCrawlerOptions');
+        // The extra fields are only tracked if the injected instance was built with them - the types enforce that,
+        // but plain JS callers would otherwise silently increment `undefined` into a sticky `NaN`. Extend
+        // `adaptivePlaywrightCrawlerStatisticState` to satisfy this.
+        if (statistics !== undefined) {
+            parseArgument(statistics.state, z.object({
+                httpOnlyRequestHandlerRuns: z.number(),
+                browserRequestHandlerRuns: z.number(),
+                renderingTypeMispredictions: z.number(),
+            }), 'statistics.state');
+        }
+        // Per-attempt buffering is load-bearing here: the handler runs up to twice per request and the
+        // losing attempt's writes must be discardable.
+        if (transactionalStorage === false) {
+            throw new Error('AdaptivePlaywrightCrawler requires transactional storage - it runs the request handler ' +
+                'multiple times per request and must be able to discard the storage writes of losing ' +
+                'attempts. `transactionalStorage: false` is therefore not supported; a write policy ' +
+                'object is accepted and forwarded to the per-attempt transactions.');
+        }
         super({
             ...rest,
-            // Pass error handlers to the "main" crawler - we only pluck them from `rest` so that they don't go to the sub crawlers
             errorHandler,
             failedRequestHandler,
-            // Same for request handler
             requestHandler,
-            // The builder intentionally returns null so that it crashes the crawler when it tries to use this instead of one of two the specialized context pipelines
-            // (that would be a logical error in this class)
-            contextPipelineBuilder: () => null,
-        }, config);
-        this.config = config;
-        this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
-        this.renderingTypePredictor =
-            renderingTypePredictor ?? new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio });
-        this.resultChecker = resultChecker ?? (() => true);
+            requestHandlerTimeoutSecs,
+            // The base would build a `Statistics` without the adaptive fields, so provide a default that has them.
+            // The cast covers a `StatisticStateExtension` that adds further fields - those can only come from an
+            // injected instance, in which case this default is never built.
+            statistics: statistics ??
+                new Statistics({
+                    logMessage: `${AdaptivePlaywrightCrawler.name} request statistics:`,
+                    stateExtension: adaptivePlaywrightCrawlerStatisticState,
+                }),
+            contextPipelineBuilder: () => this.#buildContextPipeline(),
+            // The base crawler must not wrap requests in a transaction of its own - this crawler opens
+            // one per request handler attempt in `crawlOne` instead, forwarding the write policy of the
+            // user-facing option (validated above) to those.
+            transactionalStorage: false,
+        });
+        this.#individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+        // `renderingTypeDetectionRatio` only configures the default predictor - an injected one brings its own
+        // detection ratio (and its own state), so the option is ignored in that case.
+        this.#renderingTypePredictor = OwnedOrInjected.resolve(renderingTypePredictor, () => new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }));
+        this.#attemptWritePolicy = {
+            requestQueue: 'deferred',
+            ...(typeof transactionalStorage === 'object' ? transactionalStorage : {}),
+        };
+        this.#resultChecker = resultChecker ?? (() => true);
+        this.#shouldPropagateError = shouldPropagateError ?? (() => false);
         if (resultComparator !== undefined) {
-            this.resultComparator = resultComparator;
+            this.#resultComparator = resultComparator;
         }
         else if (resultChecker !== undefined) {
-            this.resultComparator = (resultA, resultB) => this.resultChecker(resultA) && this.resultChecker(resultB);
+            this.#resultComparator = (resultA, resultB, context) => this.#resultChecker(resultA, context) && this.#resultChecker(resultB, context);
         }
         else {
-            this.resultComparator = (resultA, resultB) => {
+            this.#resultComparator = (resultA, resultB) => {
                 return (resultA.datasetItems.length === resultB.datasetItems.length &&
                     resultA.datasetItems.every((itemA, i) => {
                         const itemB = resultB.datasetItems[i];
@@ -125,304 +156,469 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     }));
             };
         }
+        // `extendContext` is forwarded to the inner crawlers, which run it *before* navigation (see
+        // `BasicCrawler`), keeping the behavior consistent with the non-adaptive crawlers: the
+        // extension is visible to the pre/post-navigation hooks and the request handler, but cannot
+        // access navigation-dependent members (`page`, `response`, `$`, ...).
+        //
+        // The adaptive hooks target a subset context (`AdaptiveHookContext`); the casts to the inner
+        // crawlers' `PlaywrightHook` type relax that nominal difference. The `ContextPipeline` merges
+        // each hook's overrides at runtime regardless of the static type.
         const staticCrawler = new CheerioCrawler({
             ...rest,
-            useSessionPool: false,
-            statisticsOptions: {
-                persistenceOptions: { enable: false },
-            },
-            preNavigationHooks: [
-                async (context) => {
-                    for (const hook of preNavigationHooks ?? []) {
-                        await hook(context, undefined);
-                    }
-                },
-            ],
-            postNavigationHooks: [
-                async (context) => {
-                    for (const hook of postNavigationHooks ?? []) {
-                        await hook(context, undefined);
-                    }
-                },
-            ],
-        }, config);
+            statistics: new Statistics({ persistenceOptions: { enable: false } }),
+            preNavigationHooks,
+            postNavigationHooks,
+            extendContext,
+            navigationTimeoutSecs,
+        });
         const browserCrawler = new PlaywrightCrawler({
             ...rest,
-            useSessionPool: false,
-            statisticsOptions: {
-                persistenceOptions: { enable: false },
-            },
-            preNavigationHooks: [
-                async (context, gotoOptions) => {
-                    for (const hook of preNavigationHooks ?? []) {
-                        await hook(context, gotoOptions);
+            statistics: new Statistics({ persistenceOptions: { enable: false } }),
+            preNavigationHooks: preNavigationHooks,
+            postNavigationHooks: postNavigationHooks,
+            extendContext,
+            navigationTimeoutSecs,
+            launchContext,
+            headless,
+            browserPool,
+            remoteBrowser,
+        });
+        this.#staticCrawler = staticCrawler;
+        this.#browserCrawler = browserCrawler;
+        // Each attempt needs its own deferred cleanups, and they must share the attempt's cleanup stack: resources the
+        // pipeline defers releasing (the page) have to outlive the user's cleanups, which may still use them.
+        const deferredCleanup = ContextPipeline.create().compose((_context, onCleanup) => ({
+            registerDeferredCleanup: (cleanup) => {
+                onCleanup(async () => {
+                    try {
+                        await cleanup();
                     }
-                },
-            ],
-            postNavigationHooks: [
-                async (context, gotoOptions) => {
-                    for (const hook of postNavigationHooks ?? []) {
-                        await hook(context, gotoOptions);
+                    catch (error) {
+                        this.log.debug('Error in deferred cleanup', { error });
                     }
-                },
-            ],
-        }, config);
-        this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
-        this.staticContextPipeline = staticCrawler.contextPipeline
-            .compose({
-            action: this.adaptCheerioContext.bind(this),
-        })
-            .compose({
-            action: async (context) => extendContext ? await extendContext(context) : context,
-        });
-        this.browserContextPipeline = browserCrawler.contextPipeline
-            .compose({
-            action: this.adaptPlaywrightContext.bind(this),
-        })
-            .compose({
-            action: async (context) => extendContext ? await extendContext(context) : context,
-        });
-        this.stats = new AdaptivePlaywrightCrawlerStatistics({
-            logMessage: `${this.log.getOptions().prefix} request statistics:`,
-            config,
-            ...statisticsOptions,
-        });
-        this.preventDirectStorageAccess = preventDirectStorageAccess;
+                });
+            },
+        }));
+        this.#staticContextPipeline = deferredCleanup
+            .chain(staticCrawler.contextPipeline)
+            .compose(this.#adaptCheerioContext.bind(this));
+        this.#browserContextPipeline = deferredCleanup
+            .chain(browserCrawler.contextPipeline)
+            .compose(this.#adaptPlaywrightContext.bind(this));
     }
-    async _init() {
-        await this.renderingTypePredictor.initialize();
-        return await super._init();
+    async init() {
+        // A crawler can be run again after a teardown.
+        this.#shutDown = false;
+        // Only the predictor we built ourselves is ours to initialize - an injected one is borrowed, so its
+        // lifecycle (including restoring persisted state) stays with whoever created it.
+        await this.#renderingTypePredictor.ifOwned((predictor) => predictor.initialize());
+        return await super.init();
     }
-    async adaptCheerioContext(cheerioContext) {
-        // Capture the original response to avoid infinite recursion when the getter is copied to the context
-        const result = this.resultObjects.get(cheerioContext);
-        if (result === undefined) {
-            throw new Error('Logical error - `this.resultObjects` does not contain the result object');
-        }
+    #buildContextPipeline() {
+        const errorMessage = (prop) => `The \`${prop}\` property is not available on the outer context pipeline of AdaptivePlaywrightCrawler - it is provided by the inner (static/browser) pipelines`;
+        return ContextPipeline.create().compose(async ({ request }) => ({
+            get request() {
+                return request;
+            },
+            get response() {
+                throw new Error(errorMessage('response'));
+            },
+            get page() {
+                throw new Error(errorMessage('page'));
+            },
+            get querySelectorAll() {
+                throw new Error(errorMessage('querySelectorAll'));
+            },
+            get querySelectorOne() {
+                throw new Error(errorMessage('querySelectorOne'));
+            },
+            get waitForSelector() {
+                throw new Error(errorMessage('waitForSelector'));
+            },
+            get parseWithCheerio() {
+                throw new Error(errorMessage('parseWithCheerio'));
+            },
+            get enqueueLinks() {
+                throw new Error(errorMessage('enqueueLinks'));
+            },
+        }));
+    }
+    async #adaptCheerioContext(cheerioContext) {
         return {
             get page() {
                 throw new Error('Page object was used in HTTP-only request handler');
             },
-            async querySelector(selector) {
-                return cheerioContext.$(selector);
-            },
             enqueueLinks: async (options = {}) => {
-                const urls = options.urls ??
-                    extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
-                return (await this.enqueueLinks({ ...options, urls }, cheerioContext.request, result));
+                const urls = extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
+                return (await this.#enqueueLinks(urls, options, cheerioContext.request));
             },
-            response: cheerioContext.response,
+            // The HTTP crawler has already consumed the body; hand out a fresh copy, like the browser side does.
+            // HTML bodies are already decoded to UTF-8, so `arrayBuffer()` may disagree with the original headers.
+            response: new ResponseWithUrl(cheerioContext.body, cheerioContext.response),
         };
     }
-    async adaptPlaywrightContext(playwrightContext) {
+    async #adaptPlaywrightContext(playwrightContext) {
+        // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const originalResponse = playwrightContext.response;
-        const result = this.resultObjects.get(playwrightContext);
-        if (result === undefined) {
-            throw new Error('Logical error - `this.resultObjects` does not contain the result object');
+        // `headers()` can return repeated headers (e.g. `set-cookie`) joined with `\n`, which `Headers` rejects
+        const headers = new Headers();
+        for (const { name, value } of await originalResponse.headersArray()) {
+            headers.append(name, value);
         }
         return {
-            response: new Response(Uint8Array.from(await originalResponse.body()), {
-                headers: originalResponse.headers(),
+            response: new ResponseWithUrl(Uint8Array.from(await originalResponse.body()), {
+                headers,
                 status: originalResponse.status(),
                 statusText: originalResponse.statusText(),
+                url: originalResponse.url(),
             }),
-            async querySelector(selector, timeoutMs = 5000) {
-                const locator = playwrightContext.page.locator(selector).first();
-                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                const $ = await playwrightContext.parseWithCheerio();
-                return $(selector);
-            },
             enqueueLinks: async (options = {}, timeoutMs = 5000) => {
                 // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
-                let urls;
-                if (options.urls === undefined) {
-                    const selector = options.selector ?? 'a';
-                    const locator = playwrightContext.page.locator(selector).first();
-                    await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                    urls =
-                        options.urls ??
-                            (await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl));
-                }
-                else {
-                    urls = options.urls;
-                }
-                return (await this.enqueueLinks({ ...options, urls }, playwrightContext.request, result));
+                const selector = options.selector ?? 'a';
+                const locator = playwrightContext.page.locator(selector).first();
+                // Wait for client-rendered links, but like the HTTP side, enqueue nothing when there are none.
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' }).catch((error) => {
+                    if (error.name !== 'TimeoutError')
+                        throw error;
+                });
+                const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl);
+                return (await this.#enqueueLinks(urls, options, playwrightContext.request));
             },
         };
     }
-    async crawlOne(renderingType, context, useStateFunction) {
-        const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
+    /**
+     * Runs one request handler attempt inside its own {@link StorageTransaction}, wrapping the inner
+     * (static or browser) context pipeline. The transaction is pushed to `transactions` *at creation
+     * time, before the `try`*, as failed attempts are routine here. The caller owns the outcome and
+     * disposal, including closing or reverting the {@link ContextModifications}.
+     *
+     * `request` defaults to the context's; a detection rerun passes a copy so it sees the request as it
+     * was before the browser attempt.
+     */
+    async #crawlOne(renderingType, context, useStateFunction, transactions, request = context.request) {
+        const transaction = createStorageTransaction({
+            policy: this.#attemptWritePolicy,
+            commitTimeoutMillis: this.internalTimeoutMillis,
+        });
+        transactions.push(transaction);
         const logs = [];
-        const deferredCleanup = [];
-        const resultBoundContextHelpers = {
-            addRequests: result.addRequests,
-            pushData: result.pushData,
-            useState: this.allowStorageAccess(useStateFunction),
-            getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
-            enqueueLinks: async (options) => {
-                return await this.enqueueLinks(options, context.request, result);
+        const contextModifications = {
+            request: new MutationJournal(),
+            state: new MutationJournal(),
+        };
+        const subCrawlerContext = Object.defineProperties({}, Object.getOwnPropertyDescriptors(context));
+        const attemptBoundContextHelpers = {
+            request: contextModifications.request.track(request),
+            useState: async (defaultValue) => contextModifications.state.track(await useStateFunction(defaultValue)),
+            log: this.#createLogProxy(context.log, logs),
+            // The sub-crawler pipeline keeps this attempt's navigation deadline on `subCrawlerContext`, but the
+            // outer `extendTimeout` only pushes the deadline it finds on `context`, so push this one as well.
+            extendTimeout: (secs) => {
+                context.extendTimeout(secs);
+                if (subCrawlerContext[navigationDeadlineKey] !== undefined) {
+                    subCrawlerContext[navigationDeadlineKey] += secs * 1000;
+                }
             },
-            log: this.createLogProxy(context.log, logs),
-            registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
         };
-        const subCrawlerContext = { ...context, ...resultBoundContextHelpers };
-        this.resultObjects.set(subCrawlerContext, result);
+        // Mark attempt-bound helpers as non-configurable so they survive the sub-crawler context pipeline
+        // (which would otherwise override them with the sub-crawler's own versions, losing the binding).
+        for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(attemptBoundContextHelpers))) {
+            Object.defineProperty(subCrawlerContext, key, { ...descriptor, configurable: false });
+        }
         try {
+            // Any failure - middleware or handler - ends up in the `ok: false` branch below.
+            const rethrow = (error) => {
+                throw error;
+            };
             const callAdaptiveRequestHandler = async () => {
                 if (renderingType === 'static') {
-                    await this.staticContextPipeline.call(subCrawlerContext, async (finalContext) => await this.requestHandler(finalContext));
+                    await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this), rethrow);
                 }
                 else if (renderingType === 'clientOnly') {
-                    await this.browserContextPipeline.call(subCrawlerContext, async (finalContext) => await this.requestHandler(finalContext));
+                    await this.#browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this), rethrow);
                 }
             };
-            await addTimeoutToPromise(async () => withCheckedStorageAccess(() => {
-                if (this.preventDirectStorageAccess) {
-                    throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
-                }
-            }, callAdaptiveRequestHandler), this.individualRequestHandlerTimeoutMillis, 'Request handler timed out');
-            return { result, ok: true, logs };
+            // this crawler overrides `runRequestHandler` and times each rendering-type run itself, so it has
+            // to resolve any per-route override too - otherwise routes would be silently ignored here
+            const routeTimeoutSecs = this.requestHandler.getTimeoutSecs?.(context.request.label);
+            const timeoutMillis = routeTimeoutSecs === undefined ? this.#individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;
+            await addTimeoutToPromise(async () => transaction.run(callAdaptiveRequestHandler), timeoutMillis, 'Request handler timed out');
+            return { ok: true, transaction, contextModifications, logs };
         }
         catch (error) {
-            return { error, ok: false, logs };
-        }
-        finally {
-            await Promise.all(deferredCleanup.map((cleanup) => cleanup()));
+            return { ok: false, error, transaction, contextModifications, logs };
         }
     }
     async runRequestHandler(crawlingContext) {
-        const renderingTypePrediction = this.renderingTypePredictor.predict(crawlingContext.request);
-        const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
+        const renderingTypePrediction = await this.#renderingTypePredictor.value.predict(crawlingContext.request);
+        const shouldDetectRenderingType = !this.#shutDown && Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
         if (!shouldDetectRenderingType) {
             crawlingContext.log.debug(`Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`);
         }
-        if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
-            crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
-            this.stats.trackHttpOnlyRequestHandlerRun();
-            const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState);
-            if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
-                crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
-                plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
-                await this.commitResult(crawlingContext, plainHTTPRun.result);
-                return;
-            }
-            // Execution will "fall through" and try running the request handler in a browser
-            if (!plainHTTPRun.ok) {
-                const actualError = plainHTTPRun.error instanceof RequestHandlerError
-                    ? plainHTTPRun.error.cause
-                    : plainHTTPRun.error;
-                crawlingContext.log.exception(actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
-            }
-            else {
-                crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
-                this.stats.trackRenderingTypeMisprediction();
-            }
-        }
-        crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
-        this.stats.trackBrowserRequestHandlerRun();
-        // Run the request handler in a browser. The copy of the crawler state is kept so that we can perform
-        // a rendering type detection if necessary. Without this measure, the HTTP request handler would run
-        // under different conditions, which could change its behavior. Changes done to the crawler state by
-        // the HTTP request handler will not be committed to the actual storage.
-        const stateTracker = {
-            stateCopy: null,
-            async getLiveState(defaultValue = {}) {
-                const state = await crawlingContext.useState(defaultValue);
-                if (this.stateCopy === null) {
-                    this.stateCopy = JSON.parse(JSON.stringify(state));
-                }
-                return state;
-            },
-            async getStateCopy(defaultValue = {}) {
-                if (this.stateCopy === null) {
-                    return defaultValue;
+        // Every transaction created for this request - up to two, since the static-then-browser
+        // fall-through and the browser-then-detection pair are mutually exclusive. Disposed in the
+        // `finally` below, not earlier: the comparators read the journals after `crawlOne` returns.
+        const transactions = [];
+        try {
+            // A result the `resultChecker` rejected is as good as a detection saying the page needs a browser - but
+            // only once the browser run has succeeded. A thrown error says nothing about rendering.
+            let httpResultRejected = false;
+            if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
+                crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
+                this.statistics.state.httpOnlyRequestHandlerRuns++;
+                const plainHTTPRun = await this.#crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
+                if (plainHTTPRun.ok &&
+                    this.#resultChecker(plainHTTPRun.transaction, { request: crawlingContext.request })) {
+                    crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
+                    plainHTTPRun.logs.forEach(([log, method, ...args]) => log[method](...args));
+                    this.#keepContextModifications(plainHTTPRun.contextModifications);
+                    await plainHTTPRun.transaction.commit();
+                    return;
                 }
-                return this.stateCopy;
-            },
-        };
-        const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker));
-        if (!browserRun.ok) {
-            throw browserRun.error;
-        }
-        await this.commitResult(crawlingContext, browserRun.result);
-        if (shouldDetectRenderingType) {
-            crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
-            const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker));
-            const detectionResult = (() => {
+                // Execution will "fall through" and try running the request handler in a browser
                 if (!plainHTTPRun.ok) {
-                    return 'clientOnly';
-                }
-                const comparisonResult = this.resultComparator(plainHTTPRun.result, browserRun.result);
-                if (comparisonResult === true || comparisonResult === 'equal') {
-                    return 'static';
+                    const actualError = plainHTTPRun.error instanceof RequestHandlerError ||
+                        plainHTTPRun.error instanceof ContextPipelineInitializationError
+                        ? plainHTTPRun.error.cause
+                        : plainHTTPRun.error;
+                    if (actualError instanceof ContextPipelineInterruptedError ||
+                        (await this.#shouldPropagateError(actualError, crawlingContext))) {
+                        this.#keepContextModifications(plainHTTPRun.contextModifications);
+                        throw actualError;
+                    }
+                    crawlingContext.log.exception(actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
                 }
-                if (comparisonResult === false || comparisonResult === 'different') {
-                    return 'clientOnly';
+                else {
+                    crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
+                    this.statistics.state.renderingTypeMispredictions++;
+                    httpResultRejected = true;
                 }
-                return undefined;
-            })();
-            crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
-            if (detectionResult !== undefined) {
-                this.renderingTypePredictor.storeResult(crawlingContext.request, detectionResult);
+                // The browser attempt starts from the request and state as they were before the rejected one.
+                this.#discardContextModifications(plainHTTPRun.contextModifications, crawlingContext);
+            }
+            crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
+            this.statistics.state.browserRequestHandlerRuns++;
+            const browserRun = await this.#crawlOne('clientOnly', crawlingContext, crawlingContext.useState, transactions);
+            if (!browserRun.ok) {
+                // A failed browser attempt is handed to the retry logic with its writes - `userData` stashed for the
+                // next attempt must survive.
+                this.#keepContextModifications(browserRun.contextModifications);
+                throw browserRun.error;
+            }
+            try {
+                browserRun.logs.forEach(([log, method, ...args]) => log[method](...args));
+                // The `afterStorageCommit` callbacks belong to this attempt, so the detection rerun undoes their
+                // writes too - the journals stay open until they have run.
+                await browserRun.transaction.commit();
+            }
+            finally {
+                this.#keepContextModifications(browserRun.contextModifications);
+            }
+            // Past the shutdown fence, `teardown()` has already drained - and possibly torn down the predictor.
+            if (httpResultRejected && !this.#shutDown) {
+                this.#storeRenderingType(crawlingContext.request, 'clientOnly');
+            }
+            if (shouldDetectRenderingType) {
+                const detectionPromise = (async () => {
+                    crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
+                    // The detection attempt's transaction is never committed - its writes exist only for the
+                    // result comparison. It runs on copies of the request and the crawler state with the browser
+                    // attempt's writes undone, so both attempts start from the same conditions, and the copies go
+                    // away with it.
+                    const { request: requestJournal, state: stateJournal } = browserRun.contextModifications;
+                    let stateCopy;
+                    const plainHTTPRun = await this.#crawlOne('static', crawlingContext, async (defaultValue) => (stateCopy ??= stateJournal.undoOnto(JSON.parse(JSON.stringify(await crawlingContext.useState(defaultValue))))), transactions, requestJournal.undoOnto(CrawlingRequest.fromSchema(JSON.parse(JSON.stringify(crawlingContext.request)))));
+                    const detectionResult = (() => {
+                        if (!plainHTTPRun.ok) {
+                            return 'clientOnly';
+                        }
+                        const comparisonResult = this.#resultComparator(plainHTTPRun.transaction, browserRun.transaction, {
+                            request: crawlingContext.request,
+                        });
+                        if (comparisonResult === true || comparisonResult === 'equal') {
+                            return 'static';
+                        }
+                        if (comparisonResult === false || comparisonResult === 'different') {
+                            return 'clientOnly';
+                        }
+                        return undefined;
+                    })();
+                    crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
+                    if (detectionResult !== undefined) {
+                        this.#storeRenderingType(crawlingContext.request, detectionResult);
+                    }
+                })();
+                await this.#trackDetection(detectionPromise);
+            }
+        }
+        finally {
+            // A still-open transaction here belongs to a discarded attempt - roll it back, then release.
+            for (const transaction of transactions) {
+                transaction.rollback();
+                transaction.dispose();
             }
         }
     }
-    async commitResult(crawlingContext, { calls, keyValueStoreChanges }) {
-        await Promise.all([
-            ...calls.pushData.map(async (params) => crawlingContext.pushData(...params)),
-            ...calls.addRequests.map(async (params) => crawlingContext.addRequests(...params)),
-            ...Object.entries(keyValueStoreChanges).map(async ([storeIdOrName, changes]) => {
-                const store = await crawlingContext.getKeyValueStore(storeIdOrName);
-                await Promise.all(Object.entries(changes).map(async ([key, { changedValue, options }]) => store.setValue(key, changedValue, options)));
-            }),
-        ]);
+    #keepContextModifications(modifications) {
+        modifications.request.close();
+        modifications.state.close();
     }
-    allowStorageAccess(func) {
-        return async (...args) => withCheckedStorageAccess(() => { }, async () => func(...args));
+    #discardContextModifications(modifications, { request, log }) {
+        // Nothing else writes to a request while its handler runs, so a conflict means a write that bypassed
+        // the context (e.g. through a stashed reference) - it stays, but it is not what the retry logic expects.
+        modifications.request.revert({
+            onConflict: (key) => log.warning(`Request ${request.url}: '${String(key)}' was modified outside the request handler`),
+        });
+        // The state is shared with concurrent handlers; a conflict is one of them having written the same
+        // property meanwhile. Their value wins, and this attempt's write stays with it.
+        modifications.state.revert({
+            onConflict: (key) => log.debug(`Crawler state '${String(key)}' was changed concurrently; keeping the newer value`),
+        });
     }
-    async enqueueLinks(options, request, result) {
+    #storeRenderingType(request, renderingType) {
+        // Deliberately not awaited: a predictor that persists asynchronously gets to keep batching its writes,
+        // and `drainRenderingDetections()` catches whatever is still pending.
+        const stored = this.#renderingTypePredictor.value.storeResult(request, renderingType);
+        if (stored !== undefined) {
+            // Nothing downstream awaits this, so a failed write would otherwise be silent.
+            void this.#trackDetection(Promise.resolve(stored).catch((error) => this.log.exception(error, `Failed to store the rendering type detection result for ${request.url}`)));
+        }
+    }
+    async #enqueueLinks(urls, options, request) {
         const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
             enqueueStrategy: options?.strategy,
             finalRequestUrl: request.loadedUrl,
             originalRequestUrl: request.url,
             userProvidedBaseUrl: options?.baseUrl,
         });
-        const addRequestsBatched = async (requests) => {
-            await result.addRequests(requests);
-            return {
-                addedRequests: requests.map(({ uniqueKey, id }) => ({
-                    uniqueKey,
-                    requestId: id ?? '',
-                    wasAlreadyPresent: false,
-                    wasAlreadyHandled: false,
-                })),
-                waitForAllRequestsToBeAdded: Promise.resolve([]),
-            };
-        };
-        // We need to use a mock request queue implementation, in order to add the requests into our result object
-        const mockRequestQueue = { addRequestsBatched };
-        return await this.enqueueLinksWithCrawlDepth({ ...options, baseUrl }, request, mockRequestQueue);
+        const requestsWithDepth = this.addCrawlDepthRequestGenerator(urls, request.crawlDepth + 1);
+        // The per-attempt transaction buffers these (the queue policy defaults to `deferred` here),
+        // so a discarded attempt's enqueues never reach the queue.
+        return await this.addRequests(requestsWithDepth, {
+            ...options,
+            baseUrl,
+            strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+        });
     }
-    createLogProxy(log, logs) {
+    #createLogProxy(log, logs) {
         return new Proxy(log, {
-            get(target, propertyName, receiver) {
+            get(target, propertyName) {
                 if (proxyLogMethods.includes(propertyName)) {
                     return (...args) => {
                         logs.push([target, propertyName, ...args]);
                     };
                 }
-                return Reflect.get(target, propertyName, receiver);
+                const value = Reflect.get(target, propertyName, target);
+                // Bind non-intercepted methods to the target instance so private #-fields
+                // (e.g. BaseCrawleeLogger.#options, #warningsLogged) do not throw TypeError at runtime.
+                if (typeof value === 'function') {
+                    return value.bind(target);
+                }
+                return value;
             },
         });
     }
+    #trackDetection(promise) {
+        this.#activeDetections.add(promise);
+        // Not `finally()`: the promise it derives would reject on its own and go unhandled. A rejection here
+        // belongs to whoever awaits the original, or to `allSettled` in the drain.
+        void promise.catch(() => { }).then(() => this.#activeDetections.delete(promise));
+        return promise;
+    }
+    /**
+     * Number of rendering type detections that have not settled yet, including results the predictor is
+     * still persisting.
+     */
+    get inFlightRenderingTypeDetectionCount() {
+        return this.#activeDetections.size;
+    }
+    /**
+     * Waits for in-flight rendering type detections to settle, bounded by `timeoutMillis` (defaults to the
+     * internal timeout).
+     */
+    async drainRenderingDetections({ timeoutMillis } = {}) {
+        if (this.#activeDetections.size === 0) {
+            return;
+        }
+        const drained = (async () => {
+            while (this.#activeDetections.size > 0) {
+                await Promise.allSettled(Array.from(this.#activeDetections));
+            }
+        })();
+        const millis = timeoutMillis ?? this.internalTimeoutMillis;
+        // A caller opting out of the bound would otherwise get an immediate spurious timeout - `setTimeout`
+        // clamps a non-finite delay to 1ms.
+        if (!Number.isFinite(millis)) {
+            await drained;
+            return;
+        }
+        const abortTimer = new AbortController();
+        try {
+            const outcome = await Promise.race([
+                drained.then(() => 'drained'),
+                delay(millis, 'timedOut', { signal: abortTimer.signal }).catch(() => 'aborted'),
+            ]);
+            if (outcome === 'timedOut') {
+                this.log.warning(`Timed out after ${millis / 1e3} seconds waiting for ${this.#activeDetections.size} rendering type detection(s) to settle - their results may be lost.`);
+            }
+        }
+        finally {
+            abortTimer.abort();
+        }
+    }
+    /**
+     * Stops the crawler immediately, but not before rendering type detections already under way (and results
+     * the predictor is still persisting) have settled - see
+     * {@link AdaptivePlaywrightCrawler.drainRenderingDetections|`drainRenderingDetections()`}. Requests
+     * that are still running are not waited for, unlike {@link BasicCrawler.stop|`stop()`}.
+     */
     async teardown() {
+        // Called from outside `run()` - under `keepAlive`, say - the pool keeps dispatching until
+        // `super.teardown()` aborts it, and a request starting during the drain would open a detection the
+        // drain has already passed. Closing that first makes the drain a fence.
+        this.#shutDown = true;
+        await this.drainRenderingDetections();
         await super.teardown();
-        for (const hook of this.teardownHooks) {
-            await hook();
-        }
+        // Mirrors the owned-only `initialize()` in `init()` - without this, the predictor we built keeps its
+        // PERSIST_STATE listener registered after the crawl and never gets a final write.
+        await this.#renderingTypePredictor.ifOwned((predictor) => predictor.teardown());
+        await this.#browserCrawler.teardown();
     }
+    async destroy() {
+        await super.destroy();
+        await this.#staticCrawler.destroy();
+        await this.#browserCrawler.destroy();
+    }
+}
+export function createAdaptivePlaywrightRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-export function createAdaptivePlaywrightRouter(routes) {
-    return Router.create(routes);
+/**
+ * An opt-in {@link AdaptivePlaywrightCrawlerOptions.resultComparator|`resultComparator`} that considers two
+ * request handler results equal only if *all* of their observable effects match - the pushed dataset items, the
+ * enqueued requests, and the key-value store changes. This is stricter than the default comparator, which only
+ * compares dataset items.
+ *
+ * **Beware:** enqueued URLs are compared exactly. The same page rendered in a browser and via plain HTTP often
+ * yields links that differ only in tracking query parameters, for example:
+ * - `https://sdk.apify.com/docs/guides/getting-started`
+ * - `https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.173549427712`
+ *
+ * Such links are treated as *different*, which will make the crawler favor browser rendering for those pages.
+ *
+ * **Example usage:**
+ * ```ts
+ * const crawler = new AdaptivePlaywrightCrawler({
+ *     resultComparator: fullResultComparator,
+ *     async requestHandler({ pushData, enqueueLinks }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export function fullResultComparator(resultA, resultB) {
+    return (isDeepStrictEqual(resultA.datasetItems, resultB.datasetItems) &&
+        isDeepStrictEqual(resultA.enqueuedUrls, resultB.enqueuedUrls) &&
+        isDeepStrictEqual(resultA.keyValueStoreChanges, resultB.keyValueStoreChanges));
 }
-//# sourceMappingURL=adaptive-playwright-crawler.js.map