@crawlee/playwright 4.0.0-beta.1 → 4.0.0-beta.100

package/internals/adaptive-playwright-crawler.js

@@ -1,11 +1,12 @@
+import { isDeepStrictEqual } from 'node:util';
+import { BasicCrawler } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
-import { Configuration, RequestHandlerResult, Router, Statistics, withCheckedStorageAccess } from '@crawlee/core';
+import { CheerioCrawler } from '@crawlee/cheerio';
+import { OwnedOrInjected, RequestHandlerError, RequestHandlerResult, resolveBaseUrlForEnqueueLinksFiltering, Router, serviceLocator, Statistics, withCheckedStorageAccess, } from '@crawlee/core';
 import { extractUrlsFromCheerio } from '@crawlee/utils';
-import { load } from 'cheerio';
-import isEqual from 'lodash.isequal';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { PlaywrightCrawler } from './playwright-crawler.js';
-import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
+import { RenderingTypePredictor, } from './utils/rendering-type-prediction.js';
 class AdaptivePlaywrightCrawlerStatistics extends Statistics {
     state = null; // this needs to be assigned for a valid override, but the initialization is done by a reset() call from the parent constructor
     constructor(options = {}) {
@@ -18,8 +19,8 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics {
         this.state.browserRequestHandlerRuns = 0;
         this.state.renderingTypeMispredictions = 0;
     }
-    async _maybeLoadStatistics() {
-        await super._maybeLoadStatistics();
+    async maybeLoadStatistics() {
+        await super.maybeLoadStatistics();
         const savedState = await this.keyValueStore?.getValue(this.persistStateKey);
         if (!savedState) {
             return;
@@ -80,27 +81,34 @@ const proxyLogMethods = [
  *
  * @experimental
  */
-export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
-    config;
-    adaptiveRequestHandler;
+export class AdaptivePlaywrightCrawler extends BasicCrawler {
     renderingTypePredictor;
     resultChecker;
+    shouldPropagateError;
     resultComparator;
     preventDirectStorageAccess;
-    /**
-     * Default {@link Router} instance that will be used if we don't specify any {@link AdaptivePlaywrightCrawlerOptions.requestHandler|`requestHandler`}.
-     * See {@link Router.addHandler|`router.addHandler()`} and {@link Router.addDefaultHandler|`router.addDefaultHandler()`}.
-     */
-    // @ts-ignore
-    router = Router.create();
-    constructor(options = {}, config = Configuration.getGlobalConfig()) {
-        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, resultComparator, statisticsOptions, preventDirectStorageAccess = true, ...rest } = options;
-        super(rest, config);
-        this.config = config;
-        this.adaptiveRequestHandler = requestHandler ?? this.router;
-        this.renderingTypePredictor =
-            renderingTypePredictor ?? new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio });
+    staticContextPipeline;
+    browserContextPipeline;
+    individualRequestHandlerTimeoutMillis;
+    resultObjects = new WeakMap();
+    inFlightRenderingTypeDetections = 0;
+    teardownHooks = [];
+    constructor(options = {}) {
+        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statisticsOptions, preventDirectStorageAccess = true, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, ...rest } = options;
+        super({
+            ...rest,
+            errorHandler,
+            failedRequestHandler,
+            requestHandler,
+            requestHandlerTimeoutSecs,
+            contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+        });
+        this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+        // `renderingTypeDetectionRatio` only configures the default predictor - an injected one brings its own
+        // detection ratio (and its own state), so the option is ignored in that case.
+        this.renderingTypePredictor = OwnedOrInjected.resolve(renderingTypePredictor, () => new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }));
         this.resultChecker = resultChecker ?? (() => true);
+        this.shouldPropagateError = shouldPropagateError ?? (() => false);
         if (resultComparator !== undefined) {
             this.resultComparator = resultComparator;
         }
@@ -112,75 +120,292 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
                 return (resultA.datasetItems.length === resultB.datasetItems.length &&
                     resultA.datasetItems.every((itemA, i) => {
                         const itemB = resultB.datasetItems[i];
-                        return isEqual(itemA, itemB);
+                        return isDeepStrictEqual(itemA, itemB);
                     }));
             };
         }
+        // `extendContext` is forwarded to the inner crawlers, which run it *before* navigation (see
+        // `BasicCrawler`), keeping the behavior consistent with the non-adaptive crawlers: the
+        // extension is visible to the pre/post-navigation hooks and the request handler, but cannot
+        // access navigation-dependent members (`page`, `response`, `$`, ...).
+        //
+        // The adaptive hooks target a subset context (`AdaptiveHookContext`); the casts to the inner
+        // crawlers' `PlaywrightHook` type relax that nominal difference. The `ContextPipeline` merges
+        // each hook's overrides at runtime regardless of the static type.
+        const staticCrawler = new CheerioCrawler({
+            ...rest,
+            statisticsOptions: {
+                persistenceOptions: { enable: false },
+            },
+            preNavigationHooks,
+            postNavigationHooks,
+            extendContext,
+        });
+        const browserCrawler = new PlaywrightCrawler({
+            ...rest,
+            statisticsOptions: {
+                persistenceOptions: { enable: false },
+            },
+            preNavigationHooks: preNavigationHooks,
+            postNavigationHooks: postNavigationHooks,
+            extendContext,
+        });
+        this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
+        this.staticContextPipeline = staticCrawler.contextPipeline.compose({
+            action: this.adaptCheerioContext.bind(this),
+        });
+        this.browserContextPipeline = browserCrawler.contextPipeline.compose({
+            action: this.adaptPlaywrightContext.bind(this),
+        });
         this.stats = new AdaptivePlaywrightCrawlerStatistics({
             logMessage: `${this.log.getOptions().prefix} request statistics:`,
-            config,
             ...statisticsOptions,
         });
         this.preventDirectStorageAccess = preventDirectStorageAccess;
     }
-    async _runRequestHandler(crawlingContext) {
-        const renderingTypePrediction = this.renderingTypePredictor.predict(crawlingContext.request);
+    async _init() {
+        // Only the predictor we built ourselves is ours to initialize - an injected one is borrowed, so its
+        // lifecycle (including restoring persisted state) stays with whoever created it.
+        await this.renderingTypePredictor.ifOwned((predictor) => predictor.initialize());
+        return await super._init();
+    }
+    buildContextPipeline() {
+        const errorMessage = (prop) => `The \`${prop}\` property is not available on the outer context pipeline of AdaptivePlaywrightCrawler - it is provided by the inner (static/browser) pipelines`;
+        return super.buildContextPipeline().compose({
+            action: async ({ request }) => ({
+                get request() {
+                    return request;
+                },
+                get response() {
+                    throw new Error(errorMessage('response'));
+                },
+                get page() {
+                    throw new Error(errorMessage('page'));
+                },
+                get querySelector() {
+                    throw new Error(errorMessage('querySelector'));
+                },
+                get querySelectorAll() {
+                    throw new Error(errorMessage('querySelectorAll'));
+                },
+                get waitForSelector() {
+                    throw new Error(errorMessage('waitForSelector'));
+                },
+                get parseWithCheerio() {
+                    throw new Error(errorMessage('parseWithCheerio'));
+                },
+            }),
+        });
+    }
+    async adaptCheerioContext(cheerioContext) {
+        // Capture the original response to avoid infinite recursion when the getter is copied to the context
+        const result = this.resultObjects.get(cheerioContext);
+        if (result === undefined) {
+            throw new Error('Logical error - `this.resultObjects` does not contain the result object');
+        }
+        return {
+            get page() {
+                throw new Error('Page object was used in HTTP-only request handler');
+            },
+            async querySelector(selector) {
+                return cheerioContext.$(selector).first();
+            },
+            async querySelectorAll(selector) {
+                return cheerioContext.$(selector);
+            },
+            enqueueLinks: async (options = {}) => {
+                const urls = options.urls ??
+                    extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
+                return (await this.enqueueLinks({ ...options, urls }, cheerioContext.request, result));
+            },
+            response: cheerioContext.response,
+        };
+    }
+    async adaptPlaywrightContext(playwrightContext) {
+        const originalResponse = playwrightContext.response;
+        const result = this.resultObjects.get(playwrightContext);
+        if (result === undefined) {
+            throw new Error('Logical error - `this.resultObjects` does not contain the result object');
+        }
+        return {
+            response: new Response(Uint8Array.from(await originalResponse.body()), {
+                headers: originalResponse.headers(),
+                status: originalResponse.status(),
+                statusText: originalResponse.statusText(),
+            }),
+            async querySelector(selector, timeoutMs = 5000) {
+                const locator = playwrightContext.page.locator(selector).first();
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                const $ = await playwrightContext.parseWithCheerio();
+                return $(selector).first();
+            },
+            async querySelectorAll(selector, timeoutMs = 5000) {
+                const locator = playwrightContext.page.locator(selector).first();
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                const $ = await playwrightContext.parseWithCheerio();
+                return $(selector);
+            },
+            enqueueLinks: async (options = {}, timeoutMs = 5000) => {
+                // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
+                let urls;
+                if (options.urls === undefined) {
+                    const selector = options.selector ?? 'a';
+                    const locator = playwrightContext.page.locator(selector).first();
+                    await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                    urls =
+                        options.urls ??
+                            (await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl));
+                }
+                else {
+                    urls = options.urls;
+                }
+                return (await this.enqueueLinks({ ...options, urls }, playwrightContext.request, result));
+            },
+        };
+    }
+    async crawlOne(renderingType, context, useStateFunction) {
+        const result = new RequestHandlerResult(serviceLocator.getConfiguration(), AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
+        const logs = [];
+        const deferredCleanup = [];
+        const resultBoundContextHelpers = {
+            addRequests: result.addRequests,
+            pushData: result.pushData,
+            useState: this.allowStorageAccess(useStateFunction),
+            getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
+            log: this.createLogProxy(context.log, logs),
+            registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
+        };
+        const subCrawlerContext = Object.defineProperties({}, Object.getOwnPropertyDescriptors(context));
+        // Mark result-bound helpers as non-configurable so they survive the sub-crawler context pipeline
+        // (which would otherwise override them with the sub-crawler's own versions, losing the result binding).
+        for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(resultBoundContextHelpers))) {
+            Object.defineProperty(subCrawlerContext, key, { ...descriptor, configurable: false });
+        }
+        this.resultObjects.set(subCrawlerContext, result);
+        try {
+            const callAdaptiveRequestHandler = async () => {
+                if (renderingType === 'static') {
+                    await this.staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
+                }
+                else if (renderingType === 'clientOnly') {
+                    await this.browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
+                }
+            };
+            // this crawler overrides `runRequestHandler` and times each rendering-type run itself, so it has
+            // to resolve any per-route override too - otherwise routes would be silently ignored here
+            const routeTimeoutSecs = this.requestHandler.getTimeoutSecs?.(context.request.label);
+            const timeoutMillis = routeTimeoutSecs === undefined ? this.individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;
+            await addTimeoutToPromise(async () => withCheckedStorageAccess(() => {
+                if (this.preventDirectStorageAccess) {
+                    throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
+                }
+            }, callAdaptiveRequestHandler), timeoutMillis, 'Request handler timed out');
+            return { result, ok: true, logs };
+        }
+        catch (error) {
+            return { error, ok: false, logs };
+        }
+        finally {
+            await Promise.all(deferredCleanup.map((cleanup) => cleanup()));
+        }
+    }
+    async runRequestHandler(crawlingContext) {
+        const renderingTypePrediction = this.renderingTypePredictor.value.predict(crawlingContext.request);
         const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
         if (!shouldDetectRenderingType) {
             crawlingContext.log.debug(`Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`);
         }
-        if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
-            crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
-            this.stats.trackHttpOnlyRequestHandlerRun();
-            const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext);
-            if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
-                crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
-                plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
-                await this.commitResult(crawlingContext, plainHTTPRun.result);
-                return;
-            }
-            if (!plainHTTPRun.ok) {
-                crawlingContext.log.exception(plainHTTPRun.error, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
-            }
-            else {
-                crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
-                this.stats.trackRenderingTypeMisprediction();
-            }
-        }
-        crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
-        this.stats.trackBrowserRequestHandlerRun();
-        // Run the request handler in a browser. The copy of the crawler state is kept so that we can perform
-        // a rendering type detection if necessary. Without this measure, the HTTP request handler would run
-        // under different conditions, which could change its behavior. Changes done to the crawler state by
-        // the HTTP request handler will not be committed to the actual storage.
-        const { result: browserRun, initialStateCopy } = await this.runRequestHandlerInBrowser(crawlingContext);
-        if (!browserRun.ok) {
-            throw browserRun.error;
-        }
-        await this.commitResult(crawlingContext, browserRun.result);
         if (shouldDetectRenderingType) {
-            crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
-            const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext, initialStateCopy);
-            const detectionResult = (() => {
+            this.inFlightRenderingTypeDetections += 1;
+        }
+        try {
+            if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
+                crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
+                this.stats.trackHttpOnlyRequestHandlerRun();
+                const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState);
+                if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
+                    crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
+                    plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
+                    await this.commitResult(crawlingContext, plainHTTPRun.result);
+                    return;
+                }
+                // Execution will "fall through" and try running the request handler in a browser
                 if (!plainHTTPRun.ok) {
-                    return 'clientOnly';
+                    const actualError = plainHTTPRun.error instanceof RequestHandlerError
+                        ? plainHTTPRun.error.cause
+                        : plainHTTPRun.error;
+                    if (await this.shouldPropagateError(actualError, crawlingContext)) {
+                        throw actualError;
+                    }
+                    crawlingContext.log.exception(actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
+                }
+                else {
+                    crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
+                    this.stats.trackRenderingTypeMisprediction();
                 }
-                if (this.resultComparator(plainHTTPRun.result, browserRun.result)) {
-                    return 'static';
+            }
+            crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
+            this.stats.trackBrowserRequestHandlerRun();
+            // Run the request handler in a browser. The copy of the crawler state is kept so that we can perform
+            // a rendering type detection if necessary. Without this measure, the HTTP request handler would run
+            // under different conditions, which could change its behavior. Changes done to the crawler state by
+            // the HTTP request handler will not be committed to the actual storage.
+            const stateTracker = {
+                stateCopy: null,
+                async getLiveState(defaultValue = {}) {
+                    const state = await crawlingContext.useState(defaultValue);
+                    if (this.stateCopy === null) {
+                        this.stateCopy = JSON.parse(JSON.stringify(state));
+                    }
+                    return state;
+                },
+                async getStateCopy(defaultValue = {}) {
+                    if (this.stateCopy === null) {
+                        return defaultValue;
+                    }
+                    return this.stateCopy;
+                },
+            };
+            const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker));
+            if (!browserRun.ok) {
+                throw browserRun.error;
+            }
+            browserRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
+            await this.commitResult(crawlingContext, browserRun.result);
+            if (shouldDetectRenderingType) {
+                crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
+                const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker));
+                const detectionResult = (() => {
+                    if (!plainHTTPRun.ok) {
+                        return 'clientOnly';
+                    }
+                    const comparisonResult = this.resultComparator(plainHTTPRun.result, browserRun.result);
+                    if (comparisonResult === true || comparisonResult === 'equal') {
+                        return 'static';
+                    }
+                    if (comparisonResult === false || comparisonResult === 'different') {
+                        return 'clientOnly';
+                    }
+                    return undefined;
+                })();
+                crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
+                if (detectionResult !== undefined) {
+                    this.renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
                 }
-                return 'clientOnly';
-            })();
-            crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
-            this.renderingTypePredictor.storeResult(crawlingContext.request, detectionResult);
+            }
+        }
+        finally {
+            if (shouldDetectRenderingType) {
+                this.inFlightRenderingTypeDetections -= 1;
+            }
         }
     }
     async commitResult(crawlingContext, { calls, keyValueStoreChanges }) {
         await Promise.all([
             ...calls.pushData.map(async (params) => crawlingContext.pushData(...params)),
-            ...calls.enqueueLinks.map(async (params) => await crawlingContext.enqueueLinks(...params)),
             ...calls.addRequests.map(async (params) => crawlingContext.addRequests(...params)),
             ...Object.entries(keyValueStoreChanges).map(async ([storeIdOrName, changes]) => {
-                const store = await crawlingContext.getKeyValueStore(storeIdOrName);
+                const store = await crawlingContext.getKeyValueStore({ id: storeIdOrName });
                 await Promise.all(Object.entries(changes).map(async ([key, { changedValue, options }]) => store.setValue(key, changedValue, options)));
             }),
         ]);
@@ -188,151 +413,37 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
     allowStorageAccess(func) {
         return async (...args) => withCheckedStorageAccess(() => { }, async () => func(...args));
     }
-    async runRequestHandlerInBrowser(crawlingContext) {
-        const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
-        let initialStateCopy;
-        try {
-            await super._runRequestHandler.call(new Proxy(this, {
-                get: (target, propertyName, receiver) => {
-                    if (propertyName === 'userProvidedRequestHandler') {
-                        return async (playwrightContext) => withCheckedStorageAccess(() => {
-                            if (this.preventDirectStorageAccess) {
-                                throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
-                            }
-                        }, () => this.adaptiveRequestHandler({
-                            id: crawlingContext.id,
-                            session: crawlingContext.session,
-                            proxyInfo: crawlingContext.proxyInfo,
-                            request: crawlingContext.request,
-                            response: {
-                                url: crawlingContext.response.url(),
-                                statusCode: crawlingContext.response.status(),
-                                headers: crawlingContext.response.headers(),
-                                trailers: {},
-                                complete: true,
-                                redirectUrls: [],
-                            },
-                            log: crawlingContext.log,
-                            page: crawlingContext.page,
-                            querySelector: async (selector, timeoutMs = 5_000) => {
-                                const locator = playwrightContext.page.locator(selector).first();
-                                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                                const $ = await playwrightContext.parseWithCheerio();
-                                return $(selector);
-                            },
-                            async waitForSelector(selector, timeoutMs = 5_000) {
-                                const locator = playwrightContext.page.locator(selector).first();
-                                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                            },
-                            async parseWithCheerio(selector, timeoutMs = 5_000) {
-                                if (selector) {
-                                    const locator = playwrightContext.page.locator(selector).first();
-                                    await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                                }
-                                return playwrightContext.parseWithCheerio();
-                            },
-                            async enqueueLinks(options = {}, timeoutMs = 5_000) {
-                                const selector = options.selector ?? 'a';
-                                const locator = playwrightContext.page.locator(selector).first();
-                                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                                const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ??
-                                    playwrightContext.request.loadedUrl ??
-                                    playwrightContext.request.url);
-                                await result.enqueueLinks({ ...options, urls });
-                            },
-                            addRequests: result.addRequests,
-                            pushData: result.pushData,
-                            useState: this.allowStorageAccess(async (defaultValue) => {
-                                const state = await result.useState(defaultValue);
-                                if (initialStateCopy === undefined) {
-                                    initialStateCopy = JSON.parse(JSON.stringify(state));
-                                }
-                                return state;
-                            }),
-                            getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
-                        }));
-                    }
-                    return Reflect.get(target, propertyName, receiver);
-                },
-            }), crawlingContext);
-            return { result: { result, ok: true }, initialStateCopy };
-        }
-        catch (error) {
-            return { result: { error, ok: false }, initialStateCopy };
-        }
+    /**
+     * Reading the pending request count queries the underlying request manager, which counts as storage access.
+     * Since this is internal crawler bookkeeping used to compute the `enqueueLinks` limit (not user-initiated storage
+     * access), it must be allowed even while a request handler runs inside the storage-access guard.
+     */
+    async getPendingRequestCountApproximation() {
+        return this.allowStorageAccess(() => super.getPendingRequestCountApproximation())();
     }
-    async runRequestHandlerWithPlainHTTP(crawlingContext, oldStateCopy) {
-        const result = new RequestHandlerResult(this.config, AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
-        const logs = [];
-        const pageGotoOptions = { timeout: this.navigationTimeoutMillis }; // Irrelevant, but required by BrowserCrawler
-        try {
-            await withCheckedStorageAccess(() => {
-                if (this.preventDirectStorageAccess) {
-                    throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
-                }
-            }, async () => addTimeoutToPromise(async () => {
-                const hookContext = {
-                    id: crawlingContext.id,
-                    session: crawlingContext.session,
-                    proxyInfo: crawlingContext.proxyInfo,
-                    request: crawlingContext.request,
-                    log: this.createLogProxy(crawlingContext.log, logs),
-                };
-                await this._executeHooks(this.preNavigationHooks, {
-                    ...hookContext,
-                    get page() {
-                        throw new Error('Page object was used in HTTP-only pre-navigation hook');
-                    },
-                }, // This is safe because `executeHooks` just passes the context to the hooks which accept the partial context
-                pageGotoOptions);
-                const response = await crawlingContext.sendRequest({});
-                const loadedUrl = response.url;
-                crawlingContext.request.loadedUrl = loadedUrl;
-                const $ = load(response.body);
-                await this.adaptiveRequestHandler({
-                    ...hookContext,
-                    request: crawlingContext.request,
-                    response,
-                    get page() {
-                        throw new Error('Page object was used in HTTP-only request handler');
-                    },
-                    async querySelector(selector, _timeoutMs) {
-                        return $(selector);
-                    },
-                    async waitForSelector(selector, _timeoutMs) {
-                        if ($(selector).get().length === 0) {
-                            throw new Error(`Selector '${selector}' not found.`);
-                        }
-                    },
-                    async parseWithCheerio(selector, _timeoutMs) {
-                        if (selector && $(selector).get().length === 0) {
-                            throw new Error(`Selector '${selector}' not found.`);
-                        }
-                        return $;
-                    },
-                    async enqueueLinks(options = {}) {
-                        const urls = extractUrlsFromCheerio($, options.selector, options.baseUrl ?? loadedUrl);
-                        await result.enqueueLinks({ ...options, urls });
-                    },
-                    addRequests: result.addRequests,
-                    pushData: result.pushData,
-                    useState: async (defaultValue) => {
-                        // return the old state before the browser handler was executed
-                        // when rerunning the handler via HTTP for detection
-                        if (oldStateCopy !== undefined) {
-                            return oldStateCopy ?? defaultValue; // fallback to the default for `null`
-                        }
-                        return this.allowStorageAccess(result.useState)(defaultValue);
-                    },
-                    getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
-                });
-                await this._executeHooks(this.postNavigationHooks, crawlingContext, pageGotoOptions);
-            }, this.requestHandlerTimeoutInnerMillis, 'Request handler timed out'));
-            return { result, logs, ok: true };
-        }
-        catch (error) {
-            return { error, logs, ok: false };
-        }
+    async enqueueLinks(options, request, result) {
+        const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
+            enqueueStrategy: options?.strategy,
+            finalRequestUrl: request.loadedUrl,
+            originalRequestUrl: request.url,
+            userProvidedBaseUrl: options?.baseUrl,
+        });
+        const addRequestsBatched = async (requests) => {
+            await result.addRequests(requests);
+            return {
+                addedRequests: requests.map(({ uniqueKey, id }) => ({
+                    uniqueKey,
+                    requestId: id ?? '',
+                    wasAlreadyPresent: false,
+                    wasAlreadyHandled: false,
+                })),
+                waitForAllRequestsToBeAdded: Promise.resolve([]),
+                requestsOverLimit: [],
+            };
+        };
+        // We need to use a mock request queue implementation, in order to add the requests into our result object
+        const mockRequestQueue = { addRequestsBatched };
+        return await this.enqueueLinksWithCrawlDepth({ ...options, baseUrl }, request, mockRequestQueue);
     }
     createLogProxy(log, logs) {
         return new Proxy(log, {
@@ -346,8 +457,42 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
             },
         });
     }
+    async teardown() {
+        await super.teardown();
+        for (const hook of this.teardownHooks) {
+            await hook();
+        }
+    }
+}
+export function createAdaptivePlaywrightRouter(routesOrSchemas) {
+    return Router.create(routesOrSchemas);
 }
-export function createAdaptivePlaywrightRouter(routes) {
-    return Router.create(routes);
+/**
+ * An opt-in {@link AdaptivePlaywrightCrawlerOptions.resultComparator|`resultComparator`} that considers two
+ * request handler results equal only if *all* of their observable effects match - the pushed dataset items, the
+ * enqueued requests, and the key-value store changes. This is stricter than the default comparator, which only
+ * compares dataset items.
+ *
+ * **Beware:** enqueued URLs are compared exactly. The same page rendered in a browser and via plain HTTP often
+ * yields links that differ only in tracking query parameters, for example:
+ * - `https://sdk.apify.com/docs/guides/getting-started`
+ * - `https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.173549427712`
+ *
+ * Such links are treated as *different*, which will make the crawler favor browser rendering for those pages.
+ *
+ * **Example usage:**
+ * ```ts
+ * const crawler = new AdaptivePlaywrightCrawler({
+ *     resultComparator: fullResultComparator,
+ *     async requestHandler({ pushData, enqueueLinks }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export function fullResultComparator(resultA, resultB) {
+    return (isDeepStrictEqual(resultA.datasetItems, resultB.datasetItems) &&
+        isDeepStrictEqual(resultA.enqueuedUrls, resultB.enqueuedUrls) &&
+        isDeepStrictEqual(resultA.enqueuedUrlLists, resultB.enqueuedUrlLists) &&
+        isDeepStrictEqual(resultA.keyValueStoreChanges, resultB.keyValueStoreChanges));
 }
-//# sourceMappingURL=adaptive-playwright-crawler.js.map