@crawlee/playwright 4.0.0-beta.232 → 4.0.0-beta.235

package/internals/adaptive-playwright-crawler.d.ts

@@ -184,8 +184,6 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
  *     'http://www.example.com/page-2',
  * ]);
  * ```
- *
- * @experimental
  */
 export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     #private;

package/internals/adaptive-playwright-crawler.js

@@ -3,6 +3,7 @@ import { isDeepStrictEqual } from 'node:util';
 import { BasicCrawler, ContextPipeline, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
+import { ResponseWithUrl } from '@crawlee/http-client';
 import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected } from '@crawlee/core';
 import { extractUrlsFromCheerio, parseArgument } from '@crawlee/utils/internal';
 import { z } from 'zod';
@@ -61,8 +62,6 @@ const proxyLogMethods = [
  *     'http://www.example.com/page-2',
  * ]);
  * ```
- *
- * @experimental
  */
 export class AdaptivePlaywrightCrawler extends BasicCrawler {
     #renderingTypePredictor;
@@ -234,23 +233,30 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 const urls = extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
                 return (await this.#enqueueLinks(urls, options, cheerioContext.request));
             },
-            response: cheerioContext.response,
+            // The HTTP crawler has already consumed the body; hand out a fresh copy, like the browser side does.
+            // HTML bodies are already decoded to UTF-8, so `arrayBuffer()` may disagree with the original headers.
+            response: new ResponseWithUrl(cheerioContext.body, cheerioContext.response),
         };
     }
     async #adaptPlaywrightContext(playwrightContext) {
         // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const originalResponse = playwrightContext.response;
         return {
-            response: new Response(Uint8Array.from(await originalResponse.body()), {
+            response: new ResponseWithUrl(Uint8Array.from(await originalResponse.body()), {
                 headers: originalResponse.headers(),
                 status: originalResponse.status(),
                 statusText: originalResponse.statusText(),
+                url: originalResponse.url(),
             }),
             enqueueLinks: async (options = {}, timeoutMs = 5000) => {
                 // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
                 const selector = options.selector ?? 'a';
                 const locator = playwrightContext.page.locator(selector).first();
-                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                // Wait for client-rendered links, but like the HTTP side, enqueue nothing when there are none.
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' }).catch((error) => {
+                    if (error.name !== 'TimeoutError')
+                        throw error;
+                });
                 const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl);
                 return (await this.#enqueueLinks(urls, options, playwrightContext.request));
             },

package/internals/utils/rendering-type-prediction.d.ts

@@ -9,8 +9,6 @@ export interface RenderingTypePredictorOptions {
 /**
  * Minimal contract that any object passed to {@link AdaptivePlaywrightCrawler} as its
  * `renderingTypePredictor` option must satisfy.
- *
- * @experimental
  */
 export interface IRenderingTypePredictor {
     /**
@@ -35,8 +33,6 @@ export interface IRenderingTypePredictor {
 }
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
- *
- * @experimental
  */
 export declare class RenderingTypePredictor implements IRenderingTypePredictor {
     #private;

package/internals/utils/rendering-type-prediction.js

@@ -62,8 +62,6 @@ const stateCodec = z.codec(persistedState, predictorState, {
 });
 /**
  * Stores rendering type information for previously crawled URLs and predicts the rendering type for URLs that have yet to be crawled and recommends when rendering type detection should be performed.
- *
- * @experimental
  */
 export class RenderingTypePredictor {
     #detectionRatio;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.232",
+    "version": "4.0.0-beta.235",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -49,13 +49,14 @@
     "dependencies": {
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
-        "@crawlee/basic": "4.0.0-beta.232",
-        "@crawlee/browser": "4.0.0-beta.232",
-        "@crawlee/browser-pool": "4.0.0-beta.232",
-        "@crawlee/cheerio": "4.0.0-beta.232",
-        "@crawlee/core": "4.0.0-beta.232",
-        "@crawlee/types": "4.0.0-beta.232",
-        "@crawlee/utils": "4.0.0-beta.232",
+        "@crawlee/basic": "4.0.0-beta.235",
+        "@crawlee/browser": "4.0.0-beta.235",
+        "@crawlee/browser-pool": "4.0.0-beta.235",
+        "@crawlee/cheerio": "4.0.0-beta.235",
+        "@crawlee/core": "4.0.0-beta.235",
+        "@crawlee/http-client": "4.0.0-beta.235",
+        "@crawlee/types": "4.0.0-beta.235",
+        "@crawlee/utils": "4.0.0-beta.235",
         "cheerio": "^1.0.0",
         "domhandler": "^6.0.0",
         "jquery": "^3.7.1",
@@ -80,5 +81,5 @@
             }
         }
     },
-    "gitHead": "5eab8f877242d46d7cae13b640d8e1adbf194911"
+    "gitHead": "06b640b70f6c6c912a19fd6643574274bdc1a953"
 }