@crawlee/playwright 4.0.0-beta.231 → 4.0.0-beta.233

package/internals/adaptive-playwright-crawler.d.ts

@@ -43,16 +43,17 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
      * Playwright Page object. If accessed in HTTP-only rendering, this will throw an error and make the AdaptivePlaywrightCrawlerContext retry the request in a browser.
      */
     page: Page;
-    /**
-     * Wait for an element matching the selector to appear and return a Cheerio object of the first matched element.
-     * Timeout defaults to 5s.
-     */
-    querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
     /**
      * Wait for an element matching the selector to appear and return a Cheerio object of all matched elements.
+     * Throws when nothing matches - in the HTTP-only run, that makes the crawler retry the request in a browser.
      * Timeout defaults to 5s.
      */
     querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+    /**
+     * Like {@link AdaptivePlaywrightCrawlerContext.querySelectorAll|`querySelectorAll`}, but returns only the
+     * first match.
+     */
+    querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
     /**
      * Wait for an element matching the selector to appear.
      * Timeout defaults to 5s.
@@ -91,6 +92,11 @@ type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHo
 type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
     request: LoadedRequest<CrawlingRequest>;
 }, ContextExtension>;
+/** Passed to {@link AdaptivePlaywrightCrawlerOptions.resultChecker} and `resultComparator` alongside the results. */
+export interface AdaptiveResultContext {
+    /** The request whose handler produced the results. */
+    request: CrawlingRequest;
+}
 export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks' | 'contextPipelineBuilder'>, Pick<PlaywrightCrawlerOptions, 'launchContext' | 'headless' | 'browserPool' | 'remoteBrowser'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
@@ -121,7 +127,7 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
      * If it returns false, the request is retried in a browser.
      * If no callback is specified, every result is considered valid.
      */
-    resultChecker?: (result: StorageTransactionView) => boolean;
+    resultChecker?: (result: StorageTransactionView, context: AdaptiveResultContext) => boolean;
     /**
      * An optional callback that decides whether an error thrown during the plain HTTP request handler
      * should be propagated (instead of falling back to browser navigation).
@@ -144,7 +150,7 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
      *
      * For a stricter, ready-made comparator that also takes enqueued requests and key-value store changes into account, see {@link fullResultComparator}.
      */
-    resultComparator?: (resultA: StorageTransactionView, resultB: StorageTransactionView) => boolean | 'equal' | 'different' | 'inconclusive';
+    resultComparator?: (resultA: StorageTransactionView, resultB: StorageTransactionView, context: AdaptiveResultContext) => boolean | 'equal' | 'different' | 'inconclusive';
     /**
      * A custom rendering type predictor. A predictor passed here is borrowed - the crawler never drives its
      * lifecycle, so set it up yourself (the built-in {@link RenderingTypePredictor} needs `initialize()`).
@@ -160,9 +166,9 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
  * ```javascript
  * const crawler = new AdaptivePlaywrightCrawler({
  *     renderingTypeDetectionRatio: 0.1,
- *     async requestHandler({ querySelector, pushData, enqueueLinks, request, log }) {
+ *     async requestHandler({ querySelectorAll, pushData, enqueueLinks, request, log }) {
  *         // This function is called to extract data from a single web page
- *         const $prices = await querySelector('span.price')
+ *         const $prices = await querySelectorAll('span.price')
  *
  *         await pushData({
  *             url: request.url,

package/internals/adaptive-playwright-crawler.js

@@ -3,6 +3,7 @@ import { isDeepStrictEqual } from 'node:util';
 import { BasicCrawler, ContextPipeline, ContextPipelineInitializationError, ContextPipelineInterruptedError, RequestHandlerError, resolveBaseUrlForEnqueueLinksFiltering, Router, Statistics, } from '@crawlee/basic';
 import { extractUrlsFromPage } from '@crawlee/browser';
 import { CheerioCrawler } from '@crawlee/cheerio';
+import { ResponseWithUrl } from '@crawlee/http-client';
 import { createStorageTransaction, EnqueueStrategy, OwnedOrInjected } from '@crawlee/core';
 import { extractUrlsFromCheerio, parseArgument } from '@crawlee/utils/internal';
 import { z } from 'zod';
@@ -43,9 +44,9 @@ const proxyLogMethods = [
  * ```javascript
  * const crawler = new AdaptivePlaywrightCrawler({
  *     renderingTypeDetectionRatio: 0.1,
- *     async requestHandler({ querySelector, pushData, enqueueLinks, request, log }) {
+ *     async requestHandler({ querySelectorAll, pushData, enqueueLinks, request, log }) {
  *         // This function is called to extract data from a single web page
- *         const $prices = await querySelector('span.price')
+ *         const $prices = await querySelectorAll('span.price')
  *
  *         await pushData({
  *             url: request.url,
@@ -146,7 +147,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             this.#resultComparator = resultComparator;
         }
         else if (resultChecker !== undefined) {
-            this.#resultComparator = (resultA, resultB) => this.#resultChecker(resultA) && this.#resultChecker(resultB);
+            this.#resultComparator = (resultA, resultB, context) => this.#resultChecker(resultA, context) && this.#resultChecker(resultB, context);
         }
         else {
             this.#resultComparator = (resultA, resultB) => {
@@ -208,12 +209,12 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             get page() {
                 throw new Error(errorMessage('page'));
             },
-            get querySelector() {
-                throw new Error(errorMessage('querySelector'));
-            },
             get querySelectorAll() {
                 throw new Error(errorMessage('querySelectorAll'));
             },
+            get querySelectorOne() {
+                throw new Error(errorMessage('querySelectorOne'));
+            },
             get waitForSelector() {
                 throw new Error(errorMessage('waitForSelector'));
             },
@@ -230,45 +231,34 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             get page() {
                 throw new Error('Page object was used in HTTP-only request handler');
             },
-            async querySelector(selector) {
-                return cheerioContext.$(selector).first();
-            },
-            async querySelectorAll(selector) {
-                return cheerioContext.$(selector);
-            },
             enqueueLinks: async (options = {}) => {
                 const urls = extractUrlsFromCheerio(cheerioContext.$, options.selector, options.baseUrl ?? cheerioContext.request.loadedUrl);
                 return (await this.#enqueueLinks(urls, options, cheerioContext.request));
             },
-            response: cheerioContext.response,
+            // The HTTP crawler has already consumed the body; hand out a fresh copy, like the browser side does.
+            // HTML bodies are already decoded to UTF-8, so `arrayBuffer()` may disagree with the original headers.
+            response: new ResponseWithUrl(cheerioContext.body, cheerioContext.response),
         };
     }
     async #adaptPlaywrightContext(playwrightContext) {
         // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const originalResponse = playwrightContext.response;
         return {
-            response: new Response(Uint8Array.from(await originalResponse.body()), {
+            response: new ResponseWithUrl(Uint8Array.from(await originalResponse.body()), {
                 headers: originalResponse.headers(),
                 status: originalResponse.status(),
                 statusText: originalResponse.statusText(),
+                url: originalResponse.url(),
             }),
-            async querySelector(selector, timeoutMs = 5000) {
-                const locator = playwrightContext.page.locator(selector).first();
-                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                const $ = await playwrightContext.parseWithCheerio();
-                return $(selector).first();
-            },
-            async querySelectorAll(selector, timeoutMs = 5000) {
-                const locator = playwrightContext.page.locator(selector).first();
-                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
-                const $ = await playwrightContext.parseWithCheerio();
-                return $(selector);
-            },
             enqueueLinks: async (options = {}, timeoutMs = 5000) => {
                 // TODO consider using `context.parseWithCheerio` to make this universal and avoid code duplication
                 const selector = options.selector ?? 'a';
                 const locator = playwrightContext.page.locator(selector).first();
-                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                // Wait for client-rendered links, but like the HTTP side, enqueue nothing when there are none.
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' }).catch((error) => {
+                    if (error.name !== 'TimeoutError')
+                        throw error;
+                });
                 const urls = await extractUrlsFromPage(playwrightContext.page, selector, options.baseUrl ?? playwrightContext.request.loadedUrl);
                 return (await this.#enqueueLinks(urls, options, playwrightContext.request));
             },
@@ -337,11 +327,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         // `finally` below, not earlier: the comparators read the journals after `crawlOne` returns.
         const transactions = [];
         try {
+            // A result the `resultChecker` rejected is as good as a detection saying the page needs a browser - but
+            // only once the browser run has succeeded. A thrown error says nothing about rendering.
+            let httpResultRejected = false;
             if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
                 crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
                 this.statistics.state.httpOnlyRequestHandlerRuns++;
                 const plainHTTPRun = await this.#crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
-                if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {
+                if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result, { request: crawlingContext.request })) {
                     crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
                     plainHTTPRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
                     await plainHTTPRun.result.commit();
@@ -362,6 +355,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 else {
                     crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
                     this.statistics.state.renderingTypeMispredictions++;
+                    httpResultRejected = true;
                 }
             }
             crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
@@ -392,6 +386,10 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             }
             browserRun.logs?.forEach(([log, method, ...args]) => log[method](...args));
             await browserRun.result.commit();
+            // Past the shutdown fence, `teardown()` has already drained - and possibly torn down the predictor.
+            if (httpResultRejected && !this.#shutDown) {
+                this.#storeRenderingType(crawlingContext.request, 'clientOnly');
+            }
             if (shouldDetectRenderingType) {
                 const detectionPromise = (async () => {
                     crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
@@ -402,7 +400,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                         if (!plainHTTPRun.ok) {
                             return 'clientOnly';
                         }
-                        const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
+                        const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result, {
+                            request: crawlingContext.request,
+                        });
                         if (comparisonResult === true || comparisonResult === 'equal') {
                             return 'static';
                         }
@@ -413,13 +413,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     })();
                     crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
                     if (detectionResult !== undefined) {
-                        // Deliberately not awaited: a predictor that persists asynchronously gets to keep
-                        // batching its writes, and the drain below catches whatever is still pending.
-                        const stored = this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
-                        if (stored !== undefined) {
-                            // Nothing downstream awaits this, so a failed write would otherwise be silent.
-                            void this.#trackDetection(Promise.resolve(stored).catch((error) => this.log.exception(error, `Failed to store the rendering type detection result for ${crawlingContext.request.url}`)));
-                        }
+                        this.#storeRenderingType(crawlingContext.request, detectionResult);
                     }
                 })();
                 await this.#trackDetection(detectionPromise);
@@ -433,6 +427,15 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             }
         }
     }
+    #storeRenderingType(request, renderingType) {
+        // Deliberately not awaited: a predictor that persists asynchronously gets to keep batching its writes,
+        // and `drainRenderingDetections()` catches whatever is still pending.
+        const stored = this.#renderingTypePredictor.value.storeResult(request, renderingType);
+        if (stored !== undefined) {
+            // Nothing downstream awaits this, so a failed write would otherwise be silent.
+            void this.#trackDetection(Promise.resolve(stored).catch((error) => this.log.exception(error, `Failed to store the rendering type detection result for ${request.url}`)));
+        }
+    }
     async #enqueueLinks(urls, options, request) {
         const baseUrl = resolveBaseUrlForEnqueueLinksFiltering({
             enqueueStrategy: options?.strategy,

package/internals/playwright-crawler.js

@@ -114,6 +114,21 @@ export class PlaywrightCrawler extends BrowserCrawler {
             const locator = context.page.locator(selector).first();
             await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
         };
+        const parseWithCheerioFn = async (selector, timeoutMs = 5_000) => {
+            if (selector) {
+                await waitForSelector(selector, timeoutMs);
+            }
+            return parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
+        };
+        const querySelectorAll = async (selector, timeoutMs) => {
+            const matches = (await parseWithCheerioFn(selector, timeoutMs))(selector);
+            // The browser located it but the Cheerio snapshot does not have it - a Playwright-only selector,
+            // or an element under a shadow root/iframe the snapshot was told to ignore.
+            if (matches.length === 0) {
+                throw new Error(`Selector '${selector}' not found in the page snapshot.`);
+            }
+            return matches;
+        };
         const downloads = [];
         context.page.on('download', (download) => downloads.push(download));
         return {
@@ -128,12 +143,9 @@ export class PlaywrightCrawler extends BrowserCrawler {
             },
             blockRequests: async (options) => blockRequests(context.page, options),
             waitForSelector,
-            parseWithCheerio: async (selector, timeoutMs = 5_000) => {
-                if (selector) {
-                    await waitForSelector(selector, timeoutMs);
-                }
-                return parseWithCheerio(context.page, this.ignoreShadowRoots, this.ignoreIframes);
-            },
+            parseWithCheerio: parseWithCheerioFn,
+            querySelectorAll,
+            querySelectorOne: async (selector, timeoutMs) => (await querySelectorAll(selector, timeoutMs)).first(),
             infiniteScroll: async (options) => infiniteScroll(context.page, options),
             listDownloads: async () => downloads,
             saveSnapshot: async (options) => saveSnapshot(context.page, {

package/internals/utils/playwright-utils.d.ts

@@ -19,7 +19,8 @@
  */
 import { Configuration, type Request } from '@crawlee/browser';
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import type { CheerioAPI } from 'cheerio';
+import type { Cheerio, CheerioAPI } from 'cheerio';
+import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Download, Page, Response } from 'playwright';
 import type { EnqueueLinksByClickingElementsOptions } from '../enqueue-links/click-elements.js';
@@ -420,6 +421,15 @@ export interface PlaywrightContextUtils {
      * ```
      */
     waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+    /**
+     * Like {@link PlaywrightContextUtils.waitForSelector|`waitForSelector`}, but returns the matched elements
+     * as a Cheerio object. Throws when nothing matches.
+     */
+    querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+    /**
+     * Like {@link PlaywrightContextUtils.querySelectorAll|`querySelectorAll`}, but returns only the first match.
+     */
+    querySelectorOne(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
     /**
      * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@link CheerioCrawler}.
      * When provided with the `selector` argument, it waits for it to be available first.

package/internals/utils/rendering-type-prediction.d.ts

@@ -24,7 +24,8 @@ export interface IRenderingTypePredictor {
         detectionProbabilityRecommendation: number;
     }>;
     /**
-     * Report a detected rendering type, so that future predictions can take it into account.
+     * Report a rendering type established by a detection, or `clientOnly` when the `resultChecker` rejected an
+     * HTTP-only result, so that future predictions can take it into account.
      *
      * The crawler does not await this per detection - it collects the returned promises and drains them in
      * `teardown()`, so an implementation that persists results asynchronously can keep batching its writes

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.231",
+    "version": "4.0.0-beta.233",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.13.0"
@@ -49,14 +49,16 @@
     "dependencies": {
         "@apify/datastructures": "^3.0.1",
         "@apify/timeout": "^1.0.1",
-        "@crawlee/basic": "4.0.0-beta.231",
-        "@crawlee/browser": "4.0.0-beta.231",
-        "@crawlee/browser-pool": "4.0.0-beta.231",
-        "@crawlee/cheerio": "4.0.0-beta.231",
-        "@crawlee/core": "4.0.0-beta.231",
-        "@crawlee/types": "4.0.0-beta.231",
-        "@crawlee/utils": "4.0.0-beta.231",
+        "@crawlee/basic": "4.0.0-beta.233",
+        "@crawlee/browser": "4.0.0-beta.233",
+        "@crawlee/browser-pool": "4.0.0-beta.233",
+        "@crawlee/cheerio": "4.0.0-beta.233",
+        "@crawlee/core": "4.0.0-beta.233",
+        "@crawlee/http-client": "4.0.0-beta.233",
+        "@crawlee/types": "4.0.0-beta.233",
+        "@crawlee/utils": "4.0.0-beta.233",
         "cheerio": "^1.0.0",
+        "domhandler": "^6.0.0",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
         "ml-matrix": "^6.12.1",
@@ -79,5 +81,5 @@
             }
         }
     },
-    "gitHead": "bc264d72aeef35d0915047359e96ca3635af4cea"
+    "gitHead": "3f074fb4fc8bf016aa1740cac0feac5ae84eca36"
 }