@crawlee/playwright 4.0.0-beta.99 → 4.0.0-rc.0

package/internals/adaptive-playwright-crawler.js

@@ -2,46 +2,28 @@ import { isDeepStrictEqual } from 'node:util';
 import { BasicCrawler } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
-import { OwnedOrInjected, RequestHandlerError, RequestHandlerResult, resolveBaseUrlForEnqueueLinksFiltering, Router, serviceLocator, Statistics, withCheckedStorageAccess, } from '@crawlee/core';
-import { extractUrlsFromCheerio } from '@crawlee/utils';
+import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected, parseArgument, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/core';
+import { extractUrlsFromCheerio } from '@crawlee/utils/internal';
+import { z } from 'zod';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { PlaywrightCrawler } from './playwright-crawler.js';
 import { RenderingTypePredictor, } from './utils/rendering-type-prediction.js';
-class AdaptivePlaywrightCrawlerStatistics extends Statistics {
-    state = null; // this needs to be assigned for a valid override, but the initialization is done by a reset() call from the parent constructor
-    constructor(options = {}) {
-        super(options);
-        this.reset();
-    }
-    reset() {
-        super.reset();
-        this.state.httpOnlyRequestHandlerRuns = 0;
-        this.state.browserRequestHandlerRuns = 0;
-        this.state.renderingTypeMispredictions = 0;
-    }
-    async maybeLoadStatistics() {
-        await super.maybeLoadStatistics();
-        const savedState = await this.keyValueStore?.getValue(this.persistStateKey);
-        if (!savedState) {
-            return;
-        }
-        this.state.httpOnlyRequestHandlerRuns = savedState.httpOnlyRequestHandlerRuns;
-        this.state.browserRequestHandlerRuns = savedState.browserRequestHandlerRuns;
-        this.state.renderingTypeMispredictions = savedState.renderingTypeMispredictions;
-    }
-    trackHttpOnlyRequestHandlerRun() {
-        this.state.httpOnlyRequestHandlerRuns ??= 0;
-        this.state.httpOnlyRequestHandlerRuns += 1;
-    }
-    trackBrowserRequestHandlerRun() {
-        this.state.browserRequestHandlerRuns ??= 0;
-        this.state.browserRequestHandlerRuns += 1;
-    }
-    trackRenderingTypeMisprediction() {
-        this.state.renderingTypeMispredictions ??= 0;
-        this.state.renderingTypeMispredictions += 1;
-    }
-}
+const adaptiveStatisticStateSchema = z.object({
+    /** How many requests were handled by the HTTP-only request handler. */
+    httpOnlyRequestHandlerRuns: z.number().default(0),
+    /** How many requests were handled in a browser. */
+    browserRequestHandlerRuns: z.number().default(0),
+    /** How many times the HTTP-only handler produced a result the `resultChecker` rejected. */
+    renderingTypeMispredictions: z.number().default(0),
+});
+/**
+ * The {@link AdaptivePlaywrightCrawlerStatisticState} fields as a {@link Statistics} state extension, defaults
+ * and all. A {@link Statistics} instance to be injected into an {@link AdaptivePlaywrightCrawler} has to carry
+ * them - `deserialize.extend()` your own fields onto this one and pass the result as `stateExtension`.
+ */
+export const adaptivePlaywrightCrawlerStatisticState = {
+    deserialize: adaptiveStatisticStateSchema,
+};
 const proxyLogMethods = [
     'error',
     'exception',
@@ -82,41 +64,80 @@ const proxyLogMethods = [
  * @experimental
  */
 export class AdaptivePlaywrightCrawler extends BasicCrawler {
-    renderingTypePredictor;
-    resultChecker;
-    shouldPropagateError;
-    resultComparator;
-    preventDirectStorageAccess;
-    staticContextPipeline;
-    browserContextPipeline;
-    individualRequestHandlerTimeoutMillis;
-    resultObjects = new WeakMap();
-    inFlightRenderingTypeDetections = 0;
-    teardownHooks = [];
+    #renderingTypePredictor;
+    #resultChecker;
+    #shouldPropagateError;
+    #resultComparator;
+    #staticContextPipeline;
+    #browserContextPipeline;
+    #individualRequestHandlerTimeoutMillis;
+    /**
+     * The write policy of the per-attempt transactions. Defaults the request queue to `deferred`:
+     * a discarded attempt's enqueues must never reach the queue.
+     */
+    #attemptWritePolicy;
+    #teardownHooks = [];
     constructor(options = {}) {
-        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statisticsOptions, preventDirectStorageAccess = true, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, ...rest } = options;
+        const { requestHandler, renderingTypeDetectionRatio = 0.1, renderingTypePredictor, resultChecker, shouldPropagateError, resultComparator, statistics, requestHandlerTimeoutSecs = 60, errorHandler, failedRequestHandler, preNavigationHooks = [], postNavigationHooks = [], extendContext, contextPipelineBuilder, transactionalStorage, ...rest } = options;
+        // The user's value is replaced by `false` in the `super` call below — validate it separately,
+        // wrapped in an object so the error still names the field.
+        parseArgument({ transactionalStorage }, z.object({ transactionalStorage: BasicCrawler.optionsShape.transactionalStorage }), 'AdaptivePlaywrightCrawlerOptions');
+        // The extra fields are only tracked if the injected instance was built with them - the types enforce that,
+        // but plain JS callers would otherwise silently increment `undefined` into a sticky `NaN`. Extend
+        // `adaptivePlaywrightCrawlerStatisticState` to satisfy this.
+        if (statistics !== undefined) {
+            parseArgument(statistics.state, z.object({
+                httpOnlyRequestHandlerRuns: z.number(),
+                browserRequestHandlerRuns: z.number(),
+                renderingTypeMispredictions: z.number(),
+            }), 'statistics.state');
+        }
+        // Per-attempt buffering is load-bearing here: the handler runs up to twice per request and the
+        // losing attempt's writes must be discardable.
+        if (transactionalStorage === false) {
+            throw new Error('AdaptivePlaywrightCrawler requires transactional storage - it runs the request handler ' +
+                'multiple times per request and must be able to discard the storage writes of losing ' +
+                'attempts. `transactionalStorage: false` is therefore not supported; a write policy ' +
+                'object is accepted and forwarded to the per-attempt transactions.');
+        }
         super({
             ...rest,
             errorHandler,
             failedRequestHandler,
             requestHandler,
             requestHandlerTimeoutSecs,
+            // The base would build a `Statistics` without the adaptive fields, so provide a default that has them.
+            // The cast covers a `StatisticStateExtension` that adds further fields - those can only come from an
+            // injected instance, in which case this default is never built.
+            statistics: statistics ??
+                new Statistics({
+                    logMessage: `${AdaptivePlaywrightCrawler.name} request statistics:`,
+                    stateExtension: adaptivePlaywrightCrawlerStatisticState,
+                }),
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
+            // The base crawler must not wrap requests in a transaction of its own - this crawler opens
+            // one per request handler attempt in `crawlOne` instead, forwarding the write policy of the
+            // user-facing option (validated above) to those.
+            transactionalStorage: false,
         });
-        this.individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
+        this.#individualRequestHandlerTimeoutMillis = requestHandlerTimeoutSecs * 1000;
         // `renderingTypeDetectionRatio` only configures the default predictor - an injected one brings its own
         // detection ratio (and its own state), so the option is ignored in that case.
-        this.renderingTypePredictor = OwnedOrInjected.resolve(renderingTypePredictor, () => new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }));
-        this.resultChecker = resultChecker ?? (() => true);
-        this.shouldPropagateError = shouldPropagateError ?? (() => false);
+        this.#renderingTypePredictor = OwnedOrInjected.resolve(renderingTypePredictor, () => new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio }));
+        this.#attemptWritePolicy = {
+            requestQueue: 'deferred',
+            ...(typeof transactionalStorage === 'object' ? transactionalStorage : {}),
+        };
+        this.#resultChecker = resultChecker ?? (() => true);
+        this.#shouldPropagateError = shouldPropagateError ?? (() => false);
         if (resultComparator !== undefined) {
-            this.resultComparator = resultComparator;
+            this.#resultComparator = resultComparator;
         }
         else if (resultChecker !== undefined) {
-            this.resultComparator = (resultA, resultB) => this.resultChecker(resultA) && this.resultChecker(resultB);
+            this.#resultComparator = (resultA, resultB) => this.#resultChecker(resultA) && this.#resultChecker(resultB);
         }
         else {
-            this.resultComparator = (resultA, resultB) => {
+            this.#resultComparator = (resultA, resultB) => {
                 return (resultA.datasetItems.length === resultB.datasetItems.length &&
                     resultA.datasetItems.every((itemA, i) => {
                         const itemB = resultB.datasetItems[i];
@@ -134,40 +155,31 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         // each hook's overrides at runtime regardless of the static type.
         const staticCrawler = new CheerioCrawler({
             ...rest,
-            statisticsOptions: {
-                persistenceOptions: { enable: false },
-            },
+            statistics: new Statistics({ persistenceOptions: { enable: false } }),
             preNavigationHooks,
             postNavigationHooks,
             extendContext,
         });
         const browserCrawler = new PlaywrightCrawler({
             ...rest,
-            statisticsOptions: {
-                persistenceOptions: { enable: false },
-            },
+            statistics: new Statistics({ persistenceOptions: { enable: false } }),
             preNavigationHooks: preNavigationHooks,
             postNavigationHooks: postNavigationHooks,
             extendContext,
         });
-        this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
-        this.staticContextPipeline = staticCrawler.contextPipeline.compose({
+        this.#teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
+        this.#staticContextPipeline = staticCrawler.contextPipeline.compose({
             action: this.adaptCheerioContext.bind(this),
         });
-        this.browserContextPipeline = browserCrawler.contextPipeline.compose({
+        this.#browserContextPipeline = browserCrawler.contextPipeline.compose({
             action: this.adaptPlaywrightContext.bind(this),
         });
-        this.stats = new AdaptivePlaywrightCrawlerStatistics({
-            logMessage: `${this.log.getOptions().prefix} request statistics:`,
-            ...statisticsOptions,
-        });
-        this.preventDirectStorageAccess = preventDirectStorageAccess;
     }
-    async _init() {
+    async init() {
         // Only the predictor we built ourselves is ours to initialize - an injected one is borrowed, so its
         // lifecycle (including restoring persisted state) stays with whoever created it.
-        await this.renderingTypePredictor.ifOwned((predictor) => predictor.initialize());
-        return await super._init();
+        await this.#renderingTypePredictor.ifOwned((predictor) => predictor.initialize());
+        return await super.init();
     }
     buildContextPipeline() {
         const errorMessage = (prop) => `The \`${prop}\` property is not available on the outer context pipeline of AdaptivePlaywrightCrawler - it is provided by the inner (static/browser) pipelines`;
@@ -194,15 +206,13 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 get parseWithCheerio() {
                     throw new Error(errorMessage('parseWithCheerio'));
                 },
+                get enqueueLinks() {
+                    throw new Error(errorMessage('enqueueLinks'));
+                },
             }),
         });
     }
     async adaptCheerioContext(cheerioContext) {
-        // Capture the original response to avoid infinite recursion when the getter is copied to the context
-        const result = this.resultObjects.get(cheerioContext);
-        if (result === undefined) {
-            throw new Error('Logical error - `this.resultObjects` does not contain the result object');
-        }
         return {
             get page() {
                 throw new Error('Page object was used in HTTP-only request handler');
@@ -214,19 +224,15 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 return cheerioContext.$(selector);
             },
             enqueueLinks: async (options = {}) => {
-                const urls = options.urls ??
-                    extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
-                return (await this.enqueueLinks({ ...options, urls }, cheerioContext.request, result));
+                const urls = extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
+                return (await this.enqueueLinks(urls, options, cheerioContext.request));
             },
             response: cheerioContext.response,
         };
     }
     async adaptPlaywrightContext(playwrightContext) {
+        // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const originalResponse = playwrightContext.response;
-        const result = this.resultObjects.get(playwrightContext);
-        if (result === undefined) {
-            throw new Error('Logical error - `this.resultObjects` does not contain the result object');
-        }
         return {
             response: new Response(Uint8Array.from(await originalResponse.body()), {
                 headers: originalResponse.headers(),
@@ -247,60 +253,54 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
             enqueueLinks: async (options = {}, timeoutMs = 5000) => {
                 // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
-                let urls;
-                if (options.urls === undefined) {
-                    const selector = options.selector ?? 'a';
-                    const locator = playwrightContext.page.locator(selector).first();
-                    await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                    urls =
-                        options.urls ??
-                            (await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl));
-                }
-                else {
-                    urls = options.urls;
-                }
-                return (await this.enqueueLinks({ ...options, urls }, playwrightContext.request, result));
+                const selector = options.selector ?? 'a';
+                const locator = playwrightContext.page.locator(selector).first();
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl);
+                return (await this.enqueueLinks(urls, options, playwrightContext.request));
             },
         };
     }
-    async crawlOne(renderingType, context, useStateFunction) {
-        const result = new RequestHandlerResult(serviceLocator.getConfiguration(), AdaptivePlaywrightCrawler.CRAWLEE_STATE_KEY);
+    /**
+     * Runs one request handler attempt inside its own {@link StorageTransaction}, wrapping the inner
+     * (static or browser) context pipeline. The transaction is pushed to `transactions` *at creation
+     * time, before the `try`* - the `ok: false` branch of the returned {@link Result} carries no
+     * result, and failed attempts are routine here. The caller owns the outcome and disposal.
+     */
+    async crawlOne(renderingType, context, useStateFunction, transactions) {
+        const transaction = createStorageTransaction({
+            policy: this.#attemptWritePolicy,
+            commitTimeoutMillis: this.internalTimeoutMillis,
+        });
+        transactions.push(transaction);
         const logs = [];
         const deferredCleanup = [];
-        const resultBoundContextHelpers = {
-            addRequests: result.addRequests,
-            pushData: result.pushData,
-            useState: this.allowStorageAccess(useStateFunction),
-            getKeyValueStore: this.allowStorageAccess(result.getKeyValueStore),
+        const attemptBoundContextHelpers = {
+            useState: useStateFunction,
             log: this.createLogProxy(context.log, logs),
             registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
         };
         const subCrawlerContext = Object.defineProperties({}, Object.getOwnPropertyDescriptors(context));
-        // Mark result-bound helpers as non-configurable so they survive the sub-crawler context pipeline
-        // (which would otherwise override them with the sub-crawler's own versions, losing the result binding).
-        for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(resultBoundContextHelpers))) {
+        // Mark attempt-bound helpers as non-configurable so they survive the sub-crawler context pipeline
+        // (which would otherwise override them with the sub-crawler's own versions, losing the binding).
+        for (const [key, descriptor] of Object.entries(Object.getOwnPropertyDescriptors(attemptBoundContextHelpers))) {
             Object.defineProperty(subCrawlerContext, key, { ...descriptor, configurable: false });
         }
-        this.resultObjects.set(subCrawlerContext, result);
         try {
             const callAdaptiveRequestHandler = async () => {
                 if (renderingType === 'static') {
-                    await this.staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
+                    await this.#staticContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
                 }
                 else if (renderingType === 'clientOnly') {
-                    await this.browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
+                    await this.#browserContextPipeline.call(subCrawlerContext, this.requestHandler.bind(this));
                 }
             };
             // this crawler overrides `runRequestHandler` and times each rendering-type run itself, so it has
             // to resolve any per-route override too - otherwise routes would be silently ignored here
             const routeTimeoutSecs = this.requestHandler.getTimeoutSecs?.(context.request.label);
-            const timeoutMillis = routeTimeoutSecs === undefined ? this.individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;
-            await addTimeoutToPromise(async () => withCheckedStorageAccess(() => {
-                if (this.preventDirectStorageAccess) {
-                    throw new Error('Directly accessing storage in a request handler is not allowed in AdaptivePlaywrightCrawler');
-                }
-            }, callAdaptiveRequestHandler), timeoutMillis, 'Request handler timed out');
-            return { result, ok: true, logs };
+            const timeoutMillis = routeTimeoutSecs === undefined ? this.#individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;
+            await addTimeoutToPromise(async () => transaction.run(callAdaptiveRequestHandler), timeoutMillis, 'Request handler timed out');
+            return { result: transaction, ok: true, logs };
         }
         catch (error) {
             return { error, ok: false, logs };
@@ -310,23 +310,24 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         }
     }
     async runRequestHandler(crawlingContext) {
-        const renderingTypePrediction = this.renderingTypePredictor.value.predict(crawlingContext.request);
+        const renderingTypePrediction = this.#renderingTypePredictor.value.predict(crawlingContext.request);
         const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
         if (!shouldDetectRenderingType) {
             crawlingContext.log.debug(`Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`);
         }
-        if (shouldDetectRenderingType) {
-            this.inFlightRenderingTypeDetections += 1;
-        }
+        // Every transaction created for this request - up to two, since the static-then-browser
+        // fall-through and the browser-then-detection pair are mutually exclusive. Disposed in the
+        // `finally` below, not earlier: the comparators read the journals after `crawlOne` returns.
+        const transactions = [];
         try {
             if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
                 crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
-                this.stats.trackHttpOnlyRequestHandlerRun();
-                const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState);
-                if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
+                this.statistics.state.httpOnlyRequestHandlerRuns++;
+                const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
+                if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {
                     crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
                     plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
-                    await this.commitResult(crawlingContext, plainHTTPRun.result);
+                    await plainHTTPRun.result.commit();
                     return;
                 }
                 // Execution will "fall through" and try running the request handler in a browser
@@ -334,18 +335,18 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     const actualError = plainHTTPRun.error instanceof RequestHandlerError
                         ? plainHTTPRun.error.cause
                         : plainHTTPRun.error;
-                    if (await this.shouldPropagateError(actualError, crawlingContext)) {
+                    if (await this.#shouldPropagateError(actualError, crawlingContext)) {
                         throw actualError;
                     }
                     crawlingContext.log.exception(actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
                 }
                 else {
                     crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
-                    this.stats.trackRenderingTypeMisprediction();
+                    this.statistics.state.renderingTypeMispredictions++;
                 }
             }
             crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
-            this.stats.trackBrowserRequestHandlerRun();
+            this.statistics.state.browserRequestHandlerRuns++;
             // Run the request handler in a browser. The copy of the crawler state is kept so that we can perform
             // a rendering type detection if necessary. Without this measure, the HTTP request handler would run
             // under different conditions, which could change its behavior. Changes done to the crawler state by
@@ -366,20 +367,22 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     return this.stateCopy;
                 },
             };
-            const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker));
+            const browserRun = await this.crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker), transactions);
             if (!browserRun.ok) {
                 throw browserRun.error;
             }
             browserRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
-            await this.commitResult(crawlingContext, browserRun.result);
+            await browserRun.result.commit();
             if (shouldDetectRenderingType) {
                 crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
-                const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker));
+                // The detection attempt's transaction is never committed - its writes exist only for the
+                // result comparison.
+                const plainHTTPRun = await this.crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker), transactions);
                 const detectionResult = (() => {
                     if (!plainHTTPRun.ok) {
                         return 'clientOnly';
                     }
-                    const comparisonResult = this.resultComparator(plainHTTPRun.result, browserRun.result);
+                    const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
                     if (comparisonResult === true || comparisonResult === 'equal') {
                         return 'static';
                     }
@@ -390,60 +393,33 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 })();
                 crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
                 if (detectionResult !== undefined) {
-                    this.renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
+                    this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
                 }
             }
         }
         finally {
-            if (shouldDetectRenderingType) {
-                this.inFlightRenderingTypeDetections -= 1;
+            // A still-open transaction here belongs to a discarded attempt - roll it back, then release.
+            for (const transaction of transactions) {
+                transaction.rollback();
+                transaction.dispose();
             }
         }
     }
-    async commitResult(crawlingContext, { calls, keyValueStoreChanges }) {
-        await Promise.all([
-            ...calls.pushData.map(async (params) => crawlingContext.pushData(...params)),
-            ...calls.addRequests.map(async (params) => crawlingContext.addRequests(...params)),
-            ...Object.entries(keyValueStoreChanges).map(async ([storeIdOrName, changes]) => {
-                const store = await crawlingContext.getKeyValueStore({ id: storeIdOrName });
-                await Promise.all(Object.entries(changes).map(async ([key, { changedValue, options }]) => store.setValue(key, changedValue, options)));
-            }),
-        ]);
-    }
-    allowStorageAccess(func) {
-        return async (...args) => withCheckedStorageAccess(() => { }, async () => func(...args));
-    }
-    /**
-     * Reading the pending request count queries the underlying request manager, which counts as storage access.
-     * Since this is internal crawler bookkeeping used to compute the `enqueueLinks` limit (not user-initiated storage
-     * access), it must be allowed even while a request handler runs inside the storage-access guard.
-     */
-    async getPendingRequestCountApproximation() {
-        return this.allowStorageAccess(() => super.getPendingRequestCountApproximation())();
-    }
-    async enqueueLinks(options, request, result) {
+    async enqueueLinks(urls, options, request) {
         const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
             enqueueStrategy: options?.strategy,
             finalRequestUrl: request.loadedUrl,
             originalRequestUrl: request.url,
             userProvidedBaseUrl: options?.baseUrl,
         });
-        const addRequestsBatched = async (requests) => {
-            await result.addRequests(requests);
-            return {
-                addedRequests: requests.map(({ uniqueKey, id }) => ({
-                    uniqueKey,
-                    requestId: id ?? '',
-                    wasAlreadyPresent: false,
-                    wasAlreadyHandled: false,
-                })),
-                waitForAllRequestsToBeAdded: Promise.resolve([]),
-                requestsOverLimit: [],
-            };
-        };
-        // We need to use a mock request queue implementation, in order to add the requests into our result object
-        const mockRequestQueue = { addRequestsBatched };
-        return await this.enqueueLinksWithCrawlDepth({ ...options, baseUrl }, request, mockRequestQueue);
+        const requestsWithDepth = this.addCrawlDepthRequestGenerator(urls, request.crawlDepth + 1);
+        // The per-attempt transaction buffers these (the queue policy defaults to `deferred` here),
+        // so a discarded attempt's enqueues never reach the queue.
+        return await this.addRequests(requestsWithDepth, {
+            ...options,
+            baseUrl,
+            strategy: options.strategy ?? EnqueueStrategy.SameHostname,
+        });
     }
     createLogProxy(log, logs) {
         return new Proxy(log, {
@@ -459,7 +435,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
     }
     async teardown() {
         await super.teardown();
-        for (const hook of this.teardownHooks) {
+        for (const hook of this.#teardownHooks) {
             await hook();
         }
     }
@@ -493,6 +469,5 @@ export function createAdaptivePlaywrightRouter(routesOrSchemas) {
 export function fullResultComparator(resultA, resultB) {
     return (isDeepStrictEqual(resultA.datasetItems, resultB.datasetItems) &&
         isDeepStrictEqual(resultA.enqueuedUrls, resultB.enqueuedUrls) &&
-        isDeepStrictEqual(resultA.enqueuedUrlLists, resultB.enqueuedUrlLists) &&
         isDeepStrictEqual(resultA.keyValueStoreChanges, resultB.keyValueStoreChanges));
 }

package/internals/enqueue-links/click-elements.d.ts

@@ -1,4 +1,4 @@
-import type { GlobInput, IRequestManager, PseudoUrlInput, RegExpInput, RequestTransform, SkippedRequestCallback } from '@crawlee/browser';
+import type { IRequestManager, RequestTransform, SkippedRequestCallback, UrlPatternInput } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
@@ -26,64 +26,29 @@ export interface EnqueueLinksByClickingElementsOptions {
      */
     clickOptions?: ClickOptions;
     /**
-     * An array of glob pattern strings or plain objects
-     * containing glob pattern strings matching the URLs to be enqueued.
+     * An array of URL patterns that URLs must match to be enqueued.
      *
-     * The plain objects must include at least the `glob` property, which holds the glob pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
-     *
-     * The matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
-     *
-     * If `globs` is an empty array or `undefined`, then the function
-     * enqueues all the intercepted navigation requests produced by the page
-     * after clicking on elements matching the provided CSS selector.
-     */
-    globs?: GlobInput[];
-    /**
-     * An array of glob pattern strings, regexp patterns or plain objects
-     * containing patterns matching URLs that will **never** be enqueued.
-     *
-     * The plain objects must include either the `glob` property or the `regexp` property.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
      * Glob matching is always case-insensitive.
-     * If you need case-sensitive matching, provide a regexp.
-     */
-    exclude?: readonly (GlobInput | RegExpInput)[];
-    /**
-     * An array of regular expressions or plain objects
-     * containing regular expressions matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `regexp` property, which holds the regular expression.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * If you need case-sensitive matching, use a `RegExp`.
      *
-     * If `regexps` is an empty array or `undefined`, then the function
+     * If `include` is an empty array or `undefined`, then the function
      * enqueues all the intercepted navigation requests produced by the page
      * after clicking on elements matching the provided CSS selector.
      */
-    regexps?: RegExpInput[];
+    include?: UrlPatternInput[];
     /**
-     * *NOTE:* In future versions of SDK the options will be removed.
-     * Please use `globs` or `regexps` instead.
-     *
-     * An array of {@link PseudoUrl} strings or plain objects
-     * containing {@link PseudoUrl} strings matching the URLs to be enqueued.
-     *
-     * The plain objects must include at least the `purl` property, which holds the pseudo-URL pattern string.
-     * All remaining keys will be used as request options for the corresponding enqueued {@link Request} objects.
+     * An array of URL patterns. Matching URLs will **not** be enqueued.
      *
-     * With a pseudo-URL string, the matching is always case-insensitive.
-     * If you need case-sensitive matching, use `regexps` property directly.
+     * Accepts glob pattern strings, `{ glob: string }` objects, `RegExp` instances, or `{ regexp: RegExp }` objects.
      *
-     * If `pseudoUrls` is an empty array or `undefined`, then the function
-     * enqueues all the intercepted navigation requests produced by the page
-     * after clicking on elements matching the provided CSS selector.
-     *
-     * @deprecated prefer using `globs` or `regexps` instead
+     * Glob matching is always case-insensitive.
+     * If you need case-sensitive matching, use a `RegExp`.
      */
-    pseudoUrls?: PseudoUrlInput[];
+    exclude?: readonly UrlPatternInput[];
     /**
-     * After {@link Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
+     * After request options are filtered by `include`/`exclude` patterns,
      * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
      * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
      * but differ in methods or payloads, or to dynamically update or create `userData`.
@@ -98,8 +63,8 @@ export interface EnqueueLinksByClickingElementsOptions {
      * }
      * ```
      *
-     * Note that `transformRequestFunction` has the highest priority and can overwrite request options
-     * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+     * Note that `transformRequestFunction` has the highest priority and can overwrite
+     * the global `label` option.
      *
      * The function receives a {@link RequestOptions} object and can return either:
      * - The modified {@link RequestOptions} object
@@ -160,8 +125,7 @@ export interface EnqueueLinksByClickingElementsOptions {
  * in `href` elements, but rather navigations are triggered in click handlers.
  * If you're looking to find URLs in `href` attributes of the page, see {@link enqueueLinks}.
  *
- * Optionally, the function allows you to filter the target links' URLs using an array of {@link PseudoUrl} objects
- * and override settings of the enqueued {@link Request} objects.
+ * Optionally, the function allows you to filter the target links' URLs using an array of glob or regexp patterns.
  *
  * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,
  * such as changing the Z-index of elements being clicked and their visibility. Therefore,
@@ -183,9 +147,9 @@ export interface EnqueueLinksByClickingElementsOptions {
  *   page,
  *   requestManager,
  *   selector: 'a.product-detail',
- *   pseudoUrls: [
- *       'https://www.example.com/handbags/[.*]'
- *       'https://www.example.com/purses/[.*]'
+ *   include: [
+ *       'https://www.example.com/handbags/*',
+ *       'https://www.example.com/purses/*',
  *   ],
  * });
  * ```