@crawlee/playwright 4.0.0-beta.80 → 4.0.0-beta.82

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,15 +1,14 @@
 import type { BrowserHook, LoadedRequest, Request, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import type { BasicCrawlerOptions } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RestrictedCrawlingContext, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
+import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticsOptions, StatisticState } from '@crawlee/core';
 import { RequestHandlerResult, Statistics } from '@crawlee/core';
-import type { BatchAddRequestsResult, Dictionary, Awaitable } from '@crawlee/types';
+import type { Dictionary, Awaitable } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils';
 import { type Cheerio } from 'cheerio';
 import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
-import type { SetRequired } from 'type-fest';
 import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
 import { RenderingTypePredictor } from './utils/rendering-type-prediction.js';
 interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
@@ -21,7 +20,7 @@ declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
     state: AdaptivePlaywrightCrawlerStatisticState;
     constructor(options?: StatisticsOptions);
     reset(): void;
-    protected _maybeLoadStatistics(): Promise<void>;
+    protected maybeLoadStatistics(): Promise<void>;
     trackHttpOnlyRequestHandlerRun(): void;
     trackBrowserRequestHandlerRun(): void;
     trackRenderingTypeMisprediction(): void;
@@ -37,10 +36,15 @@ export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary =
      */
     page: Page;
     /**
-     * Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
+     * Wait for an element matching the selector to appear and return a Cheerio object of the first matched element.
      * Timeout defaults to 5s.
      */
     querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
+    /**
+     * Wait for an element matching the selector to appear and return a Cheerio object of all matched elements.
+     * Timeout defaults to 5s.
+     */
+    querySelectorAll(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
     /**
      * Wait for an element matching the selector to appear.
      * Timeout defaults to 5s.
@@ -75,13 +79,11 @@ interface AdaptiveHookContext extends Pick<AdaptivePlaywrightCrawlerContext, 'id
     request: Request;
     gotoOptions?: PlaywrightGotoOptions;
 }
-interface AdaptiveHook extends BrowserHook<AdaptiveHookContext> {
-}
-interface AdaptivePostNavigationHook extends BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
+type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHookContext, ContextExtension>;
+type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
     request: LoadedRequest<Request>;
-}> {
-}
-export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {
+}, ContextExtension>;
+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -90,7 +92,7 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
      * A hook may optionally return a partial object whose properties are merged into the crawling context,
      * allowing the hook to override context members for subsequent hooks and pipeline stages.
      */
-    preNavigationHooks?: AdaptiveHook[];
+    preNavigationHooks?: AdaptiveHook<ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -99,7 +101,7 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
      * A hook may optionally return a partial object whose properties are merged into the crawling context
      * (e.g. to override `response` after solving a challenge).
      */
-    postNavigationHooks?: AdaptivePostNavigationHook[];
+    postNavigationHooks?: AdaptivePostNavigationHook<ContextExtension>[];
     /**
      * Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
      * Defaults to 0.1 (so 10%).
@@ -130,6 +132,8 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
      *   If it returns 'inconclusive', the detection result won't be used.
      * If no result comparator is specified, but there is a `resultChecker`, any site where the `resultChecker` returns true is considered static.
      * If neither `resultComparator` nor `resultChecker` are specified, a deep comparison of returned dataset items is used as a default.
+     *
+     * For a stricter, ready-made comparator that also takes enqueued requests and key-value store changes into account, see {@link fullResultComparator}.
      */
     resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
     /**
@@ -171,7 +175,7 @@ export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends Adapti
  *
  * @experimental
  */
-export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {
+export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext> {
     private renderingTypePredictor;
     private resultChecker;
     private shouldPropagateError;
@@ -184,13 +188,14 @@ export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptiveP
     private resultObjects;
     private inFlightRenderingTypeDetections;
     private teardownHooks;
-    constructor(options?: AdaptivePlaywrightCrawlerOptions<ExtendedContext>);
+    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext>);
     protected _init(): Promise<void>;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
         readonly request: LoadedRequest<Request<Dictionary>>;
         readonly response: Response;
         readonly page: Page;
         readonly querySelector: AdaptivePlaywrightCrawlerContext["querySelector"];
+        readonly querySelectorAll: AdaptivePlaywrightCrawlerContext["querySelectorAll"];
         readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
         readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
     }>;
@@ -198,19 +203,43 @@ export declare class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptiveP
     private adaptPlaywrightContext;
     private crawlOne;
     protected runRequestHandler(crawlingContext: CrawlingContext): Promise<void>;
-    protected commitResult(crawlingContext: CrawlingContext, { calls, keyValueStoreChanges }: RequestHandlerResult): Promise<void>;
-    protected allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>): (...args: TArgs) => Promise<R>;
+    private commitResult;
+    private allowStorageAccess;
     /**
      * Reading the pending request count queries the underlying request manager, which counts as storage access.
      * Since this is internal crawler bookkeeping used to compute the `enqueueLinks` limit (not user-initiated storage
      * access), it must be allowed even while a request handler runs inside the storage-access guard.
      */
     protected getPendingRequestCountApproximation(): Promise<number>;
-    protected enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: RestrictedCrawlingContext['request'], result: RequestHandlerResult): Promise<BatchAddRequestsResult>;
+    private enqueueLinks;
     private createLogProxy;
     teardown(): Promise<void>;
 }
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
 export declare function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;
+/**
+ * An opt-in {@link AdaptivePlaywrightCrawlerOptions.resultComparator|`resultComparator`} that considers two
+ * request handler results equal only if *all* of their observable effects match - the pushed dataset items, the
+ * enqueued requests, and the key-value store changes. This is stricter than the default comparator, which only
+ * compares dataset items.
+ *
+ * **Beware:** enqueued URLs are compared exactly. The same page rendered in a browser and via plain HTTP often
+ * yields links that differ only in tracking query parameters, for example:
+ * - `https://sdk.apify.com/docs/guides/getting-started`
+ * - `https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.173549427712`
+ *
+ * Such links are treated as *different*, which will make the crawler favor browser rendering for those pages.
+ *
+ * **Example usage:**
+ * ```ts
+ * const crawler = new AdaptivePlaywrightCrawler({
+ *     resultComparator: fullResultComparator,
+ *     async requestHandler({ pushData, enqueueLinks }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export declare function fullResultComparator(resultA: RequestHandlerResult, resultB: RequestHandlerResult): boolean;
 export {};

package/internals/adaptive-playwright-crawler.js

@@ -19,8 +19,8 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics {
         this.state.browserRequestHandlerRuns = 0;
         this.state.renderingTypeMispredictions = 0;
     }
-    async _maybeLoadStatistics() {
-        await super._maybeLoadStatistics();
+    async maybeLoadStatistics() {
+        await super.maybeLoadStatistics();
         const savedState = await this.keyValueStore?.getValue(this.persistStateKey);
         if (!savedState) {
             return;
@@ -122,10 +122,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                     }));
             };
         }
-        // Each adaptive hook is registered as its own static/browser hook so the underlying
-        // `ContextPipeline` handles override merging between hooks for free. The hook signatures
-        // are structurally compatible with the underlying crawlers' contexts (subset of fields);
-        // the casts just relax the nominal type difference.
+        // `extendContext` is forwarded to the inner crawlers, which run it *before* navigation (see
+        // `BasicCrawler`), keeping the behavior consistent with the non-adaptive crawlers: the
+        // extension is visible to the pre/post-navigation hooks and the request handler, but cannot
+        // access navigation-dependent members (`page`, `response`, `$`, ...).
+        //
+        // The adaptive hooks target a subset context (`AdaptiveHookContext`); the casts to the inner
+        // crawlers' `PlaywrightHook` type relax that nominal difference. The `ContextPipeline` merges
+        // each hook's overrides at runtime regardless of the static type.
         const staticCrawler = new CheerioCrawler({
             ...rest,
             statisticsOptions: {
@@ -133,6 +137,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
             preNavigationHooks,
             postNavigationHooks,
+            extendContext,
         });
         const browserCrawler = new PlaywrightCrawler({
             ...rest,
@@ -141,21 +146,14 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
             },
             preNavigationHooks: preNavigationHooks,
             postNavigationHooks: postNavigationHooks,
+            extendContext,
         });
         this.teardownHooks.push(browserCrawler.teardown.bind(browserCrawler));
-        this.staticContextPipeline = staticCrawler.contextPipeline
-            .compose({
+        this.staticContextPipeline = staticCrawler.contextPipeline.compose({
             action: this.adaptCheerioContext.bind(this),
-        })
-            .compose({
-            action: async (context) => extendContext ? await extendContext(context) : context,
         });
-        this.browserContextPipeline = browserCrawler.contextPipeline
-            .compose({
+        this.browserContextPipeline = browserCrawler.contextPipeline.compose({
             action: this.adaptPlaywrightContext.bind(this),
-        })
-            .compose({
-            action: async (context) => extendContext ? await extendContext(context) : context,
         });
         this.stats = new AdaptivePlaywrightCrawlerStatistics({
             logMessage: `${this.log.getOptions().prefix} request statistics:`,
@@ -183,6 +181,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 get querySelector() {
                     throw new Error(errorMessage('querySelector'));
                 },
+                get querySelectorAll() {
+                    throw new Error(errorMessage('querySelectorAll'));
+                },
                 get waitForSelector() {
                     throw new Error(errorMessage('waitForSelector'));
                 },
@@ -203,6 +204,9 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 throw new Error('Page object was used in HTTP-only request handler');
             },
             async querySelector(selector) {
+                return cheerioContext.$(selector).first();
+            },
+            async querySelectorAll(selector) {
                 return cheerioContext.$(selector);
             },
             enqueueLinks: async (options = {}) => {
@@ -226,6 +230,12 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 statusText: originalResponse.statusText(),
             }),
             async querySelector(selector, timeoutMs = 5000) {
+                const locator = playwrightContext.page.locator(selector).first();
+                await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+                const $ = await playwrightContext.parseWithCheerio();
+                return $(selector).first();
+            },
+            async querySelectorAll(selector, timeoutMs = 5000) {
                 const locator = playwrightContext.page.locator(selector).first();
                 await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
                 const $ = await playwrightContext.parseWithCheerio();
@@ -449,3 +459,32 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
 export function createAdaptivePlaywrightRouter(routesOrSchemas) {
     return Router.create(routesOrSchemas);
 }
+/**
+ * An opt-in {@link AdaptivePlaywrightCrawlerOptions.resultComparator|`resultComparator`} that considers two
+ * request handler results equal only if *all* of their observable effects match - the pushed dataset items, the
+ * enqueued requests, and the key-value store changes. This is stricter than the default comparator, which only
+ * compares dataset items.
+ *
+ * **Beware:** enqueued URLs are compared exactly. The same page rendered in a browser and via plain HTTP often
+ * yields links that differ only in tracking query parameters, for example:
+ * - `https://sdk.apify.com/docs/guides/getting-started`
+ * - `https://sdk.apify.com/docs/guides/getting-started?__hsfp=1136113150&__hssc=7591405.1.173549427712`
+ *
+ * Such links are treated as *different*, which will make the crawler favor browser rendering for those pages.
+ *
+ * **Example usage:**
+ * ```ts
+ * const crawler = new AdaptivePlaywrightCrawler({
+ *     resultComparator: fullResultComparator,
+ *     async requestHandler({ pushData, enqueueLinks }) {
+ *         // ...
+ *     },
+ * });
+ * ```
+ */
+export function fullResultComparator(resultA, resultB) {
+    return (isDeepStrictEqual(resultA.datasetItems, resultB.datasetItems) &&
+        isDeepStrictEqual(resultA.enqueuedUrls, resultB.enqueuedUrls) &&
+        isDeepStrictEqual(resultA.enqueuedUrlLists, resultB.enqueuedUrlLists) &&
+        isDeepStrictEqual(resultA.keyValueStoreChanges, resultB.keyValueStoreChanges));
+}

package/internals/playwright-crawler.d.ts

@@ -59,7 +59,7 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
      * ]
      * ```
      */
-    preNavigationHooks?: PlaywrightHook[];
+    preNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter. A hook may optionally return a partial object
@@ -76,7 +76,7 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
      * ]
      * ```
      */
-    postNavigationHooks?: PlaywrightHook[];
+    postNavigationHooks?: BrowserHook<PlaywrightCrawlingContext, ContextExtension>[];
 }
 /**
  * Provides a simple framework for parallel crawling of web pages
@@ -243,7 +243,7 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options?: PlaywrightCrawlerOptions<ExtendedContext>);
+    constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext>);
 // @ts-ignore optional peer dependency or compatibility with es2022
     protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, Response, Dictionary, Dictionary> & {
         injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;

package/internals/utils/rendering-type-prediction.d.ts

@@ -1,7 +1,5 @@
 import type { RecoverableStatePersistenceOptions, Request } from '@crawlee/core';
 export type RenderingType = 'clientOnly' | 'static';
-type URLComponents = string[];
-type FeatureVector = [staticResultsSimilarity: number, clientOnlyResultsSimilarity: number];
 export interface RenderingTypePredictorOptions {
     /** A number between 0 and 1 that determines the desired ratio of rendering type detections */
     detectionRatio: number;
@@ -32,7 +30,6 @@ export declare class RenderingTypePredictor {
      */
     storeResult(requests: Request | Request[], renderingType: RenderingType): void;
     private resultCount;
-    protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
-    protected retrain(): void;
+    private calculateFeatureVector;
+    private retrain;
 }
-export {};

package/internals/utils/rendering-type-prediction.js

@@ -10,10 +10,17 @@ const calculateUrlSimilarity = (a, b) => {
     if (a[0] !== b[0]) {
         return 0;
     }
-    for (let i = 1; i < Math.max(a.length, b.length); i++) {
+    const maxLength = Math.max(a.length, b.length);
+    // Only the hostname is present (no path components to compare) - the hosts already match.
+    if (maxLength <= 1) {
+        return 1;
+    }
+    for (let i = 1; i < maxLength; i++) {
         values.push(stringComparison.jaroWinkler.similarity(a[i] ?? '', b[i] ?? '') > 0.8 ? 1 : 0);
     }
-    return sum(values) / Math.max(a.length, b.length);
+    // The first component (index 0, the hostname) is excluded from the comparison above,
+    // so it must also be excluded from the denominator of the weighted average.
+    return sum(values) / (maxLength - 1);
 };
 const sum = (values) => values.reduce((acc, value) => acc + value);
 const mean = (values) => (values.length > 0 ? sum(values) / values.length : undefined);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.80",
+    "version": "4.0.0-beta.82",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.3.2",
-        "@crawlee/basic": "4.0.0-beta.80",
-        "@crawlee/browser": "4.0.0-beta.80",
-        "@crawlee/browser-pool": "4.0.0-beta.80",
-        "@crawlee/cheerio": "4.0.0-beta.80",
-        "@crawlee/core": "4.0.0-beta.80",
-        "@crawlee/types": "4.0.0-beta.80",
-        "@crawlee/utils": "4.0.0-beta.80",
+        "@crawlee/basic": "4.0.0-beta.82",
+        "@crawlee/browser": "4.0.0-beta.82",
+        "@crawlee/browser-pool": "4.0.0-beta.82",
+        "@crawlee/cheerio": "4.0.0-beta.82",
+        "@crawlee/core": "4.0.0-beta.82",
+        "@crawlee/types": "4.0.0-beta.82",
+        "@crawlee/utils": "4.0.0-beta.82",
         "cheerio": "^1.0.0",
         "idcac-playwright": "^0.1.3",
         "jquery": "^3.7.1",
@@ -85,5 +85,5 @@
             }
         }
     },
-    "gitHead": "96c57b4a0c999e4b2bd198792490af28db7aa42d"
+    "gitHead": "eb1096f7c7743d124375ef011fbbadb19476822e"
 }