@crawlee/playwright 4.0.0-beta.232 → 4.0.0-beta.233

package/internals/adaptive-playwright-crawler.js

@@ -3,6 +3,7 @@ import { isDeepStrictEqual } from 'node:util';
 import { BasicCrawler, ContextPipeline, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
+import { ResponseWithUrl } from '@crawlee/http-client';
 import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected } from '@crawlee/core';
 import { extractUrlsFromCheerio, parseArgument } from '@crawlee/utils/internal';
 import { z } from 'zod';
@@ -234,23 +235,30 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 const urls = extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
                 return (await this.#enqueueLinks(urls, options, cheerioContext.request));
             },
-            response: cheerioContext.response,
+            // The HTTP crawler has already consumed the body; hand out a fresh copy, like the browser side does.
+            // HTML bodies are already decoded to UTF-8, so `arrayBuffer()` may disagree with the original headers.
+            response: new ResponseWithUrl(cheerioContext.body, cheerioContext.response),
         };
     }
     async #adaptPlaywrightContext(playwrightContext) {
         // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const originalResponse = playwrightContext.response;
         return {
-            response: new Response(Uint8Array.from(await originalResponse.body()), {
+            response: new ResponseWithUrl(Uint8Array.from(await originalResponse.body()), {
                 headers: originalResponse.headers(),
                 status: originalResponse.status(),
                 statusText: originalResponse.statusText(),
+                url: originalResponse.url(),
             }),
             enqueueLinks: async (options = {}, timeoutMs = 5000) => {
                 // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
                 const selector = options.selector ?? 'a';
                 const locator = playwrightContext.page.locator(selector).first();
-                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                // Wait for client-rendered links, but like the HTTP side, enqueue nothing when there are none.
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' }).catch((error) => {
+                    if (error.name !== 'TimeoutError')
+                        throw error;
+                });
                 const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl);
                 return (await this.#enqueueLinks(urls, options, playwrightContext.request));
             },

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.232",
+    "version": "4.0.0-beta.233",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -49,13 +49,14 @@
     "dependencies": {
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
-        "@crawlee/basic": "4.0.0-beta.232",
-        "@crawlee/browser": "4.0.0-beta.232",
-        "@crawlee/browser-pool": "4.0.0-beta.232",
-        "@crawlee/cheerio": "4.0.0-beta.232",
-        "@crawlee/core": "4.0.0-beta.232",
-        "@crawlee/types": "4.0.0-beta.232",
-        "@crawlee/utils": "4.0.0-beta.232",
+        "@crawlee/basic": "4.0.0-beta.233",
+        "@crawlee/browser": "4.0.0-beta.233",
+        "@crawlee/browser-pool": "4.0.0-beta.233",
+        "@crawlee/cheerio": "4.0.0-beta.233",
+        "@crawlee/core": "4.0.0-beta.233",
+        "@crawlee/http-client": "4.0.0-beta.233",
+        "@crawlee/types": "4.0.0-beta.233",
+        "@crawlee/utils": "4.0.0-beta.233",
         "cheerio": "^1.0.0",
         "domhandler": "^6.0.0",
         "jquery": "^3.7.1",
@@ -80,5 +81,5 @@
             }
         }
     },
-    "gitHead": "5eab8f877242d46d7cae13b640d8e1adbf194911"
+    "gitHead": "3f074fb4fc8bf016aa1740cac0feac5ae84eca36"
 }