@crawlee/playwright 4.0.0-beta.241 → 4.0.0-beta.242

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,6 +1,7 @@
-import type { BrowserHook, LoadedRequest, CrawlingRequest, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
+import type { BrowserHook, LoadedRequest, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import type { BasicCrawlerOptions, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
+import { CrawlingRequest } from '@crawlee/browser';
 import type { StorageTransactionView } from '@crawlee/core';
 import type { Dictionary, Awaitable } from '@crawlee/types';
 import { type Cheerio, type CheerioAPI } from 'cheerio';

package/internals/adaptive-playwright-crawler.js

@@ -1,11 +1,11 @@
 import { setTimeout as delay } from 'node:timers/promises';
 import { isDeepStrictEqual } from 'node:util';
 import { BasicCrawler, ContextPipeline, ContextPipelineInitializationError, ContextPipelineInterruptedError, navigationDeadlineKey, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/basic';
-import { extractUrlsFromPage } from '@crawlee/browser';
+import { CrawlingRequest, extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
 import { ResponseWithUrl } from '@crawlee/http-client';
 import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected } from '@crawlee/core';
-import { extractUrlsFromCheerio, parseArgument } from '@crawlee/utils/internal';
+import { extractUrlsFromCheerio, MutationJournal, parseArgument } from '@crawlee/utils/internal';
 import { z } from 'zod';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { PlaywrightCrawler } from './playwright-crawler.js';
@@ -267,10 +267,13 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
     /**
      * Runs one request handler attempt inside its own {@link StorageTransaction}, wrapping the inner
      * (static or browser) context pipeline. The transaction is pushed to `transactions` *at creation
-     * time, before the `try`* - the `ok: false` branch of the returned {@link Result} carries no
-     * result, and failed attempts are routine here. The caller owns the outcome and disposal.
+     * time, before the `try`*, as failed attempts are routine here. The caller owns the outcome and
+     * disposal, including closing or reverting the {@link ContextModifications}.
+     *
+     * `request` defaults to the context's; a detection rerun passes a copy so it sees the request as it
+     * was before the browser attempt.
      */
-    async #crawlOne(renderingType, context, useStateFunction, transactions) {
+    async #crawlOne(renderingType, context, useStateFunction, transactions, request = context.request) {
         const transaction = createStorageTransaction({
             policy: this.#attemptWritePolicy,
             commitTimeoutMillis: this.internalTimeoutMillis,
@@ -278,9 +281,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         transactions.push(transaction);
         const logs = [];
         const deferredCleanup = [];
+        const contextModifications = {
+            request: new MutationJournal(),
+            state: new MutationJournal(),
+        };
         const subCrawlerContext = Object.defineProperties({}, Object.getOwnPropertyDescriptors(context));
         const attemptBoundContextHelpers = {
-            useState: useStateFunction,
+            request: contextModifications.request.track(request),
+            useState: async (defaultValue) => contextModifications.state.track(await useStateFunction(defaultValue)),
             log: this.#createLogProxy(context.log, logs),
             registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
             // The sub-crawler pipeline keeps this attempt's navigation deadline on `subCrawlerContext`, but the
@@ -315,10 +323,10 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             const routeTimeoutSecs = this.requestHandler.getTimeoutSecs?.(context.request.label);
             const timeoutMillis = routeTimeoutSecs === undefined ? this.#individualRequestHandlerTimeoutMillis : routeTimeoutSecs * 1000;
             await addTimeoutToPromise(async () => transaction.run(callAdaptiveRequestHandler), timeoutMillis, 'Request handler timed out');
-            return { result: transaction, ok: true, logs };
+            return { ok: true, transaction, contextModifications, logs };
         }
         catch (error) {
-            return { error, ok: false, logs };
+            return { ok: false, error, transaction, contextModifications, logs };
         }
         finally {
             await Promise.all(deferredCleanup.map((cleanup) => cleanup()));
@@ -342,10 +350,12 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
                 this.statistics.state.httpOnlyRequestHandlerRuns++;
                 const plainHTTPRun = await this.#crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
-                if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result, { request: crawlingContext.request })) {
+                if (plainHTTPRun.ok &&
+                    this.#resultChecker(plainHTTPRun.transaction, { request: crawlingContext.request })) {
                     crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
-                    plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
-                    await plainHTTPRun.result.commit();
+                    plainHTTPRun.logs.forEach(([log, method, ...args]) => log[method](...args));
+                    this.#keepContextModifications(plainHTTPRun.contextModifications);
+                    await plainHTTPRun.transaction.commit();
                     return;
                 }
                 // Execution will "fall through" and try running the request handler in a browser
@@ -356,6 +366,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                         : plainHTTPRun.error;
                     if (actualError instanceof ContextPipelineInterruptedError ||
                         (await this.#shouldPropagateError(actualError, crawlingContext))) {
+                        this.#keepContextModifications(plainHTTPRun.contextModifications);
                         throw actualError;
                     }
                     crawlingContext.log.exception(actualError, `HTTP-only request handler failed for ${crawlingContext.request.url}`);
@@ -365,35 +376,20 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     this.statistics.state.renderingTypeMispredictions++;
                     httpResultRejected = true;
                 }
+                // The browser attempt starts from the request and state as they were before the rejected one.
+                this.#discardContextModifications(plainHTTPRun.contextModifications, crawlingContext);
             }
             crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
             this.statistics.state.browserRequestHandlerRuns++;
-            // Run the request handler in a browser. The copy of the crawler state is kept so that we can perform
-            // a rendering type detection if necessary. Without this measure, the HTTP request handler would run
-            // under different conditions, which could change its behavior. Changes done to the crawler state by
-            // the HTTP request handler will not be committed to the actual storage.
-            const stateTracker = {
-                stateCopy: null,
-                async getLiveState(defaultValue = {}) {
-                    const state = await crawlingContext.useState(defaultValue);
-                    if (this.stateCopy === null) {
-                        this.stateCopy = JSON.parse(JSON.stringify(state));
-                    }
-                    return state;
-                },
-                async getStateCopy(defaultValue = {}) {
-                    if (this.stateCopy === null) {
-                        return defaultValue;
-                    }
-                    return this.stateCopy;
-                },
-            };
-            const browserRun = await this.#crawlOne('clientOnly', crawlingContext, stateTracker.getLiveState.bind(stateTracker), transactions);
+            const browserRun = await this.#crawlOne('clientOnly', crawlingContext, crawlingContext.useState, transactions);
+            // A failed browser attempt is handed to the retry logic with its writes - `userData` stashed for the
+            // next attempt must survive.
+            this.#keepContextModifications(browserRun.contextModifications);
             if (!browserRun.ok) {
                 throw browserRun.error;
             }
-            browserRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
-            await browserRun.result.commit();
+            browserRun.logs.forEach(([log, method, ...args]) => log[method](...args));
+            await browserRun.transaction.commit();
             // Past the shutdown fence, `teardown()` has already drained - and possibly torn down the predictor.
             if (httpResultRejected && !this.#shutDown) {
                 this.#storeRenderingType(crawlingContext.request, 'clientOnly');
@@ -402,13 +398,17 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 const detectionPromise = (async () => {
                     crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
                     // The detection attempt's transaction is never committed - its writes exist only for the
-                    // result comparison.
-                    const plainHTTPRun = await this.#crawlOne('static', crawlingContext, stateTracker.getStateCopy.bind(stateTracker), transactions);
+                    // result comparison. It runs on copies of the request and the crawler state with the browser
+                    // attempt's writes undone, so both attempts start from the same conditions, and the copies go
+                    // away with it.
+                    const { request: requestJournal, state: stateJournal } = browserRun.contextModifications;
+                    let stateCopy;
+                    const plainHTTPRun = await this.#crawlOne('static', crawlingContext, async (defaultValue) => (stateCopy ??= stateJournal.undoOnto(JSON.parse(JSON.stringify(await crawlingContext.useState(defaultValue))))), transactions, requestJournal.undoOnto(CrawlingRequest.fromSchema(JSON.parse(JSON.stringify(crawlingContext.request)))));
                     const detectionResult = (() => {
                         if (!plainHTTPRun.ok) {
                             return 'clientOnly';
                         }
-                        const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result, {
+                        const comparisonResult = this.#resultComparator(plainHTTPRun.transaction, browserRun.transaction, {
                             request: crawlingContext.request,
                         });
                         if (comparisonResult === true || comparisonResult === 'equal') {
@@ -435,6 +435,22 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             }
         }
     }
+    #keepContextModifications(modifications) {
+        modifications.request.close();
+        modifications.state.close();
+    }
+    #discardContextModifications(modifications, { request, log }) {
+        // Nothing else writes to a request while its handler runs, so a conflict means a write that bypassed
+        // the context (e.g. through a stashed reference) - it stays, but it is not what the retry logic expects.
+        modifications.request.revert({
+            onConflict: (key) => log.warning(`Request ${request.url}: '${String(key)}' was modified outside the request handler`),
+        });
+        // The state is shared with concurrent handlers; a conflict is one of them having written the same
+        // property meanwhile. Their value wins, and this attempt's write stays with it.
+        modifications.state.revert({
+            onConflict: (key) => log.debug(`Crawler state '${String(key)}' was changed concurrently; keeping the newer value`),
+        });
+    }
     #storeRenderingType(request, renderingType) {
         // Deliberately not awaited: a predictor that persists asynchronously gets to keep batching its writes,
         // and `drainRenderingDetections()` catches whatever is still pending.

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.241",
+    "version": "4.0.0-beta.242",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -49,14 +49,14 @@
     "dependencies": {
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
-        "@crawlee/basic": "4.0.0-beta.241",
-        "@crawlee/browser": "4.0.0-beta.241",
-        "@crawlee/browser-pool": "4.0.0-beta.241",
-        "@crawlee/cheerio": "4.0.0-beta.241",
-        "@crawlee/core": "4.0.0-beta.241",
-        "@crawlee/http-client": "4.0.0-beta.241",
-        "@crawlee/types": "4.0.0-beta.241",
-        "@crawlee/utils": "4.0.0-beta.241",
+        "@crawlee/basic": "4.0.0-beta.242",
+        "@crawlee/browser": "4.0.0-beta.242",
+        "@crawlee/browser-pool": "4.0.0-beta.242",
+        "@crawlee/cheerio": "4.0.0-beta.242",
+        "@crawlee/core": "4.0.0-beta.242",
+        "@crawlee/http-client": "4.0.0-beta.242",
+        "@crawlee/types": "4.0.0-beta.242",
+        "@crawlee/utils": "4.0.0-beta.242",
         "cheerio": "^1.0.0",
         "domhandler": "^6.0.0",
         "jquery": "^3.7.1",
@@ -81,5 +81,5 @@
             }
         }
     },
-    "gitHead": "1c337d1ef50c3e28e25e2ae40bc38bb6de67c37e"
+    "gitHead": "9bd381d862366b6b874938fb0717f21fd430b9de"
 }