@crawlee/playwright 4.0.0-beta.251 → 4.0.0-beta.253

package/internals/adaptive-playwright-crawler.js

@@ -186,8 +186,26 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         });
         this.#staticCrawler = staticCrawler;
         this.#browserCrawler = browserCrawler;
-        this.#staticContextPipeline = staticCrawler.contextPipeline.compose(this.#adaptCheerioContext.bind(this));
-        this.#browserContextPipeline = browserCrawler.contextPipeline.compose(this.#adaptPlaywrightContext.bind(this));
+        // Each attempt needs its own deferred cleanups, and they must share the attempt's cleanup stack: resources the
+        // pipeline defers releasing (the page) have to outlive the user's cleanups, which may still use them.
+        const deferredCleanup = ContextPipeline.create().compose((_context, onCleanup) => ({
+            registerDeferredCleanup: (cleanup) => {
+                onCleanup(async () => {
+                    try {
+                        await cleanup();
+                    }
+                    catch (error) {
+                        this.log.debug('Error in deferred cleanup', { error });
+                    }
+                });
+            },
+        }));
+        this.#staticContextPipeline = deferredCleanup
+            .chain(staticCrawler.contextPipeline)
+            .compose(this.#adaptCheerioContext.bind(this));
+        this.#browserContextPipeline = deferredCleanup
+            .chain(browserCrawler.contextPipeline)
+            .compose(this.#adaptPlaywrightContext.bind(this));
     }
     async init() {
         // A crawler can be run again after a teardown.
@@ -243,9 +261,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
     async #adaptPlaywrightContext(playwrightContext) {
         // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const originalResponse = playwrightContext.response;
+        // `headers()` can return repeated headers (e.g. `set-cookie`) joined with `\n`, which `Headers` rejects
+        const headers = new Headers();
+        for (const { name, value } of await originalResponse.headersArray()) {
+            headers.append(name, value);
+        }
         return {
             response: new ResponseWithUrl(Uint8Array.from(await originalResponse.body()), {
-                headers: originalResponse.headers(),
+                headers,
                 status: originalResponse.status(),
                 statusText: originalResponse.statusText(),
                 url: originalResponse.url(),
@@ -280,7 +303,6 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         });
         transactions.push(transaction);
         const logs = [];
-        const deferredCleanup = [];
         const contextModifications = {
             request: new MutationJournal(),
             state: new MutationJournal(),
@@ -290,7 +312,6 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             request: contextModifications.request.track(request),
             useState: async (defaultValue) => contextModifications.state.track(await useStateFunction(defaultValue)),
             log: this.#createLogProxy(context.log, logs),
-            registerDeferredCleanup: (cleanup) => deferredCleanup.push(cleanup),
             // The sub-crawler pipeline keeps this attempt's navigation deadline on `subCrawlerContext`, but the
             // outer `extendTimeout` only pushes the deadline it finds on `context`, so push this one as well.
             extendTimeout: (secs) => {
@@ -328,9 +349,6 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         catch (error) {
             return { ok: false, error, transaction, contextModifications, logs };
         }
-        finally {
-            await Promise.all(deferredCleanup.map((cleanup) => cleanup()));
-        }
     }
     async runRequestHandler(crawlingContext) {
         const renderingTypePrediction = await this.#renderingTypePredictor.value.predict(crawlingContext.request);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.251",
+    "version": "4.0.0-beta.253",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -49,14 +49,14 @@
     "dependencies": {
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
-        "@crawlee/basic": "4.0.0-beta.251",
-        "@crawlee/browser": "4.0.0-beta.251",
-        "@crawlee/browser-pool": "4.0.0-beta.251",
-        "@crawlee/cheerio": "4.0.0-beta.251",
-        "@crawlee/core": "4.0.0-beta.251",
-        "@crawlee/http-client": "4.0.0-beta.251",
-        "@crawlee/types": "4.0.0-beta.251",
-        "@crawlee/utils": "4.0.0-beta.251",
+        "@crawlee/basic": "4.0.0-beta.253",
+        "@crawlee/browser": "4.0.0-beta.253",
+        "@crawlee/browser-pool": "4.0.0-beta.253",
+        "@crawlee/cheerio": "4.0.0-beta.253",
+        "@crawlee/core": "4.0.0-beta.253",
+        "@crawlee/http-client": "4.0.0-beta.253",
+        "@crawlee/types": "4.0.0-beta.253",
+        "@crawlee/utils": "4.0.0-beta.253",
         "cheerio": "^1.2.0",
         "domhandler": "^6.0.1",
         "jquery": "^3.7.1",
@@ -81,5 +81,5 @@
             }
         }
     },
-    "gitHead": "e8a0d8a538fabf624c73b6354dbf7194882300a5"
+    "gitHead": "f93aa3970468c422d9626db0d21175e8a39fdfed"
 }