@crawlee/playwright 4.0.0-beta.80 → 4.0.0-beta.81

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,15 +1,14 @@
 import type { BrowserHook, LoadedRequest, Request, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import type { BasicCrawlerOptions } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RestrictedCrawlingContext, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
+import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
 import { RequestHandlerResult, Statistics } from '@crawlee/core';
-import type { BatchAddRequestsResult, Dictionary, Awaitable } from '@crawlee/types';
+import type { Dictionary, Awaitable } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils';
 import { type Cheerio } from 'cheerio';
 import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
-import type { SetRequired } from 'type-fest';
 import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
 import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
 interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
@@ -21,7 +20,7 @@ declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
     state: AdaptivePlaywrightCrawlerStatisticState;
     constructor(options?: StatisticsOptions);
     reset(): void;
-    protected _maybeLoadStatistics(): Promise<void>;
+    protected maybeLoadStatistics(): Promise<void>;
     trackHttpOnlyRequestHandlerRun(): void;
     trackBrowserRequestHandlerRun(): void;
     trackRenderingTypeMisprediction(): void;
@@ -37,10 +36,15 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
      */
     page: Page;
     /**
-     * Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
+     * Wait for an element matching the selector to appear and return a Cheerio object of the first matched element.
      * Timeout defaults to 5s.
      */
     querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+    /**
+     * Wait for an element matching the selector to appear and return a Cheerio object of all matched elements.
+     * Timeout defaults to 5s.
+     */
+    querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
     /**
      * Wait for an element matching the selector to appear.
      * Timeout defaults to 5s.
@@ -130,6 +134,8 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
      *   If it returns 'inconclusive', the detection result won't be used.
      * If no result comparator is specified, but there is a `resultChecker`, any site where the `resultChecker` returns true is considered static.
      * If neither `resultComparator` nor `resultChecker` are specified, a deep comparison of returned dataset items is used as a default.
+     *
+     * For a stricter, ready-made comparator that also takes enqueued requests and key-value store changes into account, see {@link fullResultComparator}.
      */
     resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
     /**
@@ -191,6 +197,7 @@ export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptiveP
         readonly response: Response;
         readonly page: Page;
         readonly querySelector: AdaptivePlaywrightCrawlerContext["querySelector"];
+        readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];
         readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
         readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
     }>;
@@ -198,19 +205,43 @@ export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptiveP
     private adaptPlaywrightContext;
     private crawlOne;
     protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
-    protected commitResult(crawlingContext: CrawlingContext, { calls, keyValueStoreChanges }: RequestHandlerResult): Promise<void>;
-    protected allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>): (...args: TArgs) => Promise<R>;
+    private commitResult;
+    private allowStorageAccess;
     /**
      * Reading the pending request count queries the underlying request manager, which counts as storage access.
      * Since this is internal crawler bookkeeping used to compute the `enqueueLinks` limit (not user-initiated storage
      * access), it must be allowed even while a request handler runs inside the storage-access guard.
      */
     protected getPendingRequestCountApproximation(): Promise<number>;
-    protected enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: RestrictedCrawlingContext['request'], result: RequestHandlerResult): Promise<BatchAddRequestsResult>;
+    private enqueueLinks;
     private createLogProxy;
     teardown(): Promise<void>;
 }
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
+/**
+ * An opt-in {@link AdaptivePlaywrightCrawlerOptions.resultComparator|`resultComparator`} that considers two
+ * request handler results equal only if *all* of their observable effects match - the pushed dataset items, the
+ * enqueued requests, and the key-value store changes. This is stricter than the default comparator, which only
+ * compares dataset items.
+ *
+ * **Beware:** enqueued URLs are compared exactly. The same page rendered in a browser and via plain HTTP often
+ * yields links that differ only in tracking query parameters, for example:
+ * - `https://sdk.apify.com/docs/guides/getting-started`
+ * - `https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.173549427712`
+ *
+ * Such links are treated as *different*, which will make the crawler favor browser rendering for those pages.
+ *
+ * **Example usage:**
+ * ```ts
+ * const crawler = new AdaptivePlaywrightCrawler({
+ *     resultComparator: fullResultComparator,
+ *     async requestHandler({ pushData, enqueueLinks }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export declare function fullResultComparator(resultA: RequestHandlerResult, resultB: RequestHandlerResult): boolean;
 export {};

package/internals/adaptive-playwright-crawler.js

@@ -19,8 +19,8 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics {
         this.state.browserRequestHandlerRuns = 0;
         this.state.renderingTypeMispredictions = 0;
     }
-    async _maybeLoadStatistics() {
-        await super._maybeLoadStatistics();
+    async maybeLoadStatistics() {
+        await super.maybeLoadStatistics();
         const savedState = await this.keyValueStore?.getValue(this.persistStateKey);
         if (!savedState) {
             return;
@@ -183,6 +183,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 get querySelector() {
                     throw new Error(errorMessage('querySelector'));
                 },
+                get querySelectorAll() {
+                    throw new Error(errorMessage('querySelectorAll'));
+                },
                 get waitForSelector() {
                     throw new Error(errorMessage('waitForSelector'));
                 },
@@ -203,6 +206,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 throw new Error('Page object was used in HTTP-only request handler');
             },
             async querySelector(selector) {
+                return cheerioContext.$(selector).first();
+            },
+            async querySelectorAll(selector) {
                 return cheerioContext.$(selector);
             },
             enqueueLinks: async (options = {}) => {
@@ -226,6 +232,12 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 statusText: originalResponse.statusText(),
             }),
             async querySelector(selector, timeoutMs = 5000) {
+                const locator = playwrightContext.page.locator(selector).first();
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                const $ = await playwrightContext.parseWithCheerio();
+                return $(selector).first();
+            },
+            async querySelectorAll(selector, timeoutMs = 5000) {
                 const locator = playwrightContext.page.locator(selector).first();
                 await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
                 const $ = await playwrightContext.parseWithCheerio();
@@ -449,3 +461,32 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
 export function createAdaptivePlaywrightRouter(routesOrSchemas) {
     return Router.create(routesOrSchemas);
 }
+/**
+ * An opt-in {@link AdaptivePlaywrightCrawlerOptions.resultComparator|`resultComparator`} that considers two
+ * request handler results equal only if *all* of their observable effects match - the pushed dataset items, the
+ * enqueued requests, and the key-value store changes. This is stricter than the default comparator, which only
+ * compares dataset items.
+ *
+ * **Beware:** enqueued URLs are compared exactly. The same page rendered in a browser and via plain HTTP often
+ * yields links that differ only in tracking query parameters, for example:
+ * - `https://sdk.apify.com/docs/guides/getting-started`
+ * - `https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.173549427712`
+ *
+ * Such links are treated as *different*, which will make the crawler favor browser rendering for those pages.
+ *
+ * **Example usage:**
+ * ```ts
+ * const crawler = new AdaptivePlaywrightCrawler({
+ *     resultComparator: fullResultComparator,
+ *     async requestHandler({ pushData, enqueueLinks }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export function fullResultComparator(resultA, resultB) {
+    return (isDeepStrictEqual(resultA.datasetItems, resultB.datasetItems) &&
+        isDeepStrictEqual(resultA.enqueuedUrls, resultB.enqueuedUrls) &&
+        isDeepStrictEqual(resultA.enqueuedUrlLists, resultB.enqueuedUrlLists) &&
+        isDeepStrictEqual(resultA.keyValueStoreChanges, resultB.keyValueStoreChanges));
+}

package/internals/utils/rendering-type-prediction.d.ts

@@ -1,7 +1,5 @@
 import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
 export type RenderingType = 'clientOnly' | 'static';
-type URLComponents = string[];
-type FeatureVector = [staticResultsSimilarity: number, clientOnlyResultsSimilarity: number];
 export interface RenderingTypePredictorOptions {
     /** A number between 0 and 1 that determines the desired ratio of rendering type detections */
     detectionRatio: number;
@@ -32,7 +30,6 @@ export declare class RenderingTypePredictor {
      */
     storeResult(requests: Request | Request[], renderingType: RenderingType): void;
     private resultCount;
-    protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
-    protected retrain(): void;
+    private calculateFeatureVector;
+    private retrain;
 }
-export {};

package/internals/utils/rendering-type-prediction.js

@@ -10,10 +10,17 @@ const calculateUrlSimilarity = (a, b) => {
     if (a[0] !== b[0]) {
         return 0;
     }
-    for (let i = 1; i < Math.max(a.length, b.length); i++) {
+    const maxLength = Math.max(a.length, b.length);
+    // Only the hostname is present (no path components to compare) - the hosts already match.
+    if (maxLength <= 1) {
+        return 1;
+    }
+    for (let i = 1; i < maxLength; i++) {
         values.push(stringComparison.jaroWinkler.similarity(a[i] ?? '', b[i] ?? '') > 0.8 ? 1 : 0);
     }
-    return sum(values) / Math.max(a.length, b.length);
+    // The first component (index 0, the hostname) is excluded from the comparison above,
+    // so it must also be excluded from the denominator of the weighted average.
+    return sum(values) / (maxLength - 1);
 };
 const sum = (values) => values.reduce((acc, value) => acc + value);
 const mean = (values) => (values.length > 0 ? sum(values) / values.length : undefined);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.80",
+    "version": "4.0.0-beta.81",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.3.2",
-        "@crawlee/basic": "4.0.0-beta.80",
-        "@crawlee/browser": "4.0.0-beta.80",
-        "@crawlee/browser-pool": "4.0.0-beta.80",
-        "@crawlee/cheerio": "4.0.0-beta.80",
-        "@crawlee/core": "4.0.0-beta.80",
-        "@crawlee/types": "4.0.0-beta.80",
-        "@crawlee/utils": "4.0.0-beta.80",
+        "@crawlee/basic": "4.0.0-beta.81",
+        "@crawlee/browser": "4.0.0-beta.81",
+        "@crawlee/browser-pool": "4.0.0-beta.81",
+        "@crawlee/cheerio": "4.0.0-beta.81",
+        "@crawlee/core": "4.0.0-beta.81",
+        "@crawlee/types": "4.0.0-beta.81",
+        "@crawlee/utils": "4.0.0-beta.81",
         "cheerio": "^1.0.0",
         "idcac-playwright": "^0.1.3",
         "jquery": "^3.7.1",
@@ -85,5 +85,5 @@
             }
         }
     },
-    "gitHead": "96c57b4a0c999e4b2bd198792490af28db7aa42d"
+    "gitHead": "80dc6b4fc82237e63a51a71153809ec8dfd0cc50"
 }