@crawlee/playwright 4.0.0-beta.124 → 4.0.0-beta.125

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,34 +1,39 @@
 import type { BrowserHook, LoadedRequest, Request, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import type { BasicCrawlerOptions } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticPersistedState, StatisticState, StorageTransactionView } from '@crawlee/core';
-import { Statistics } from '@crawlee/core';
+import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StorageTransactionView } from '@crawlee/core';
 import type { Dictionary, Awaitable } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils/internal';
 import { type Cheerio } from 'cheerio';
 import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
+import { z } from 'zod';
 import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
 import { type IRenderingTypePredictor } from './utils/rendering-type-prediction.js';
-interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
-    httpOnlyRequestHandlerRuns?: number;
-    browserRequestHandlerRuns?: number;
-    renderingTypeMispredictions?: number;
-}
-interface AdaptivePlaywrightCrawlerPersistedStatisticState extends StatisticPersistedState {
-    httpOnlyRequestHandlerRuns?: number;
-    browserRequestHandlerRuns?: number;
-    renderingTypeMispredictions?: number;
-}
-declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
-    get state(): AdaptivePlaywrightCrawlerStatisticState;
-    protected defaultState(): AdaptivePlaywrightCrawlerStatisticState;
-    protected deserializeState(persistedState: AdaptivePlaywrightCrawlerPersistedStatisticState): AdaptivePlaywrightCrawlerStatisticState;
-    trackHttpOnlyRequestHandlerRun(): void;
-    trackBrowserRequestHandlerRun(): void;
-    trackRenderingTypeMisprediction(): void;
-}
+declare const adaptiveStatisticStateSchema: z.ZodObject<{
+    httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+    browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+    renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+/**
+ * The extra statistics fields {@link AdaptivePlaywrightCrawler} tracks on top of the built-in
+ * {@link StatisticState} ones. They are available on `crawler.stats.state` and are persisted with the rest of
+ * the statistics.
+ */
+export type AdaptivePlaywrightCrawlerStatisticState = z.infer<typeof adaptiveStatisticStateSchema>;
+/**
+ * The {@link AdaptivePlaywrightCrawlerStatisticState} fields as a {@link Statistics} state extension, defaults
+ * and all. A {@link Statistics} instance to be injected into an {@link AdaptivePlaywrightCrawler} has to carry
+ * them - `deserialize.extend()` your own fields onto this one and pass the result as `stateExtension`.
+ */
+export declare const adaptivePlaywrightCrawlerStatisticState: {
+    deserialize: z.ZodObject<{
+        httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+        browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+        renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strip>;
+};
 export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
     request: LoadedRequest<Request<UserData>>;
     /**
@@ -87,7 +92,7 @@ type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHo
 type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
     request: LoadedRequest<Request>;
 }, ContextExtension>;
-export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes>, 'preNavigationHooks' | 'postNavigationHooks'> {
+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -177,10 +182,9 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
  *
  * @experimental
  */
-export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes> {
+export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     #private;
-    get stats(): AdaptivePlaywrightCrawlerStatistics;
-    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
+    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
     protected init(): Promise<void>;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
         readonly request: LoadedRequest<Request<Dictionary>>;

package/internals/adaptive-playwright-crawler.js

@@ -8,39 +8,22 @@ import { z } from 'zod';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { PlaywrightCrawler } from './playwright-crawler.js';
 import { RenderingTypePredictor, } from './utils/rendering-type-prediction.js';
-class AdaptivePlaywrightCrawlerStatistics extends Statistics {
-    get state() {
-        return super.state;
-    }
-    defaultState() {
-        return {
-            ...super.defaultState(),
-            httpOnlyRequestHandlerRuns: 0,
-            browserRequestHandlerRuns: 0,
-            renderingTypeMispredictions: 0,
-        };
-    }
-    deserializeState(persistedState) {
-        return {
-            ...super.deserializeState(persistedState),
-            httpOnlyRequestHandlerRuns: persistedState.httpOnlyRequestHandlerRuns,
-            browserRequestHandlerRuns: persistedState.browserRequestHandlerRuns,
-            renderingTypeMispredictions: persistedState.renderingTypeMispredictions,
-        };
-    }
-    trackHttpOnlyRequestHandlerRun() {
-        this.state.httpOnlyRequestHandlerRuns ??= 0;
-        this.state.httpOnlyRequestHandlerRuns += 1;
-    }
-    trackBrowserRequestHandlerRun() {
-        this.state.browserRequestHandlerRuns ??= 0;
-        this.state.browserRequestHandlerRuns += 1;
-    }
-    trackRenderingTypeMisprediction() {
-        this.state.renderingTypeMispredictions ??= 0;
-        this.state.renderingTypeMispredictions += 1;
-    }
-}
+const adaptiveStatisticStateSchema = z.object({
+    /** How many requests were handled by the HTTP-only request handler. */
+    httpOnlyRequestHandlerRuns: z.number().default(0),
+    /** How many requests were handled in a browser. */
+    browserRequestHandlerRuns: z.number().default(0),
+    /** How many times the HTTP-only handler produced a result the `resultChecker` rejected. */
+    renderingTypeMispredictions: z.number().default(0),
+});
+/**
+ * The {@link AdaptivePlaywrightCrawlerStatisticState} fields as a {@link Statistics} state extension, defaults
+ * and all. A {@link Statistics} instance to be injected into an {@link AdaptivePlaywrightCrawler} has to carry
+ * them - `deserialize.extend()` your own fields onto this one and pass the result as `stateExtension`.
+ */
+export const adaptivePlaywrightCrawlerStatisticState = {
+    deserialize: adaptiveStatisticStateSchema,
+};
 const proxyLogMethods = [
     'error',
     'exception',
@@ -88,10 +71,6 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
     #staticContextPipeline;
     #browserContextPipeline;
     #individualRequestHandlerTimeoutMillis;
-    // The constructor always injects an `AdaptivePlaywrightCrawlerStatistics`, so narrowing the cast is sound.
-    get stats() {
-        return super.stats;
-    }
     /**
      * The write policy of the per-attempt transactions. Defaults the request queue to `deferred`:
      * a discarded attempt's enqueues must never reach the queue.
@@ -103,6 +82,16 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         // The user's value is replaced by `false` in the `super` call below — validate it separately,
         // wrapped in an object so the error still names the field.
         parseArgument({ transactionalStorage }, z.object({ transactionalStorage: BasicCrawler.optionsShape.transactionalStorage }), 'AdaptivePlaywrightCrawlerOptions');
+        // The extra fields are only tracked if the injected instance was built with them - the types enforce that,
+        // but plain JS callers would otherwise silently increment `undefined` into a sticky `NaN`. Extend
+        // `adaptivePlaywrightCrawlerStatisticState` to satisfy this.
+        if (statistics !== undefined) {
+            parseArgument(statistics.state, z.object({
+                httpOnlyRequestHandlerRuns: z.number(),
+                browserRequestHandlerRuns: z.number(),
+                renderingTypeMispredictions: z.number(),
+            }), 'statistics.state');
+        }
         // Per-attempt buffering is load-bearing here: the handler runs up to twice per request and the
         // losing attempt's writes must be discardable.
         if (transactionalStorage === false) {
@@ -111,20 +100,19 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 'attempts. `transactionalStorage: false` is therefore not supported; a write policy ' +
                 'object is accepted and forwarded to the per-attempt transactions.');
         }
-        if (statistics !== undefined && !(statistics instanceof AdaptivePlaywrightCrawlerStatistics)) {
-            throw new Error('AdaptivePlaywrightCrawler tracks extra fields on its own Statistics subclass and cannot use a ' +
-                'plain `statistics` instance. Omit the option to let the crawler build its own.');
-        }
         super({
             ...rest,
             errorHandler,
             failedRequestHandler,
             requestHandler,
             requestHandlerTimeoutSecs,
-            // Inject our subclass so the base tracks the extra adaptive fields instead of building a plain `Statistics`.
+            // The base would build a `Statistics` without the adaptive fields, so provide a default that has them.
+            // The cast covers a `StatisticStateExtension` that adds further fields - those can only come from an
+            // injected instance, in which case this default is never built.
             statistics: statistics ??
-                new AdaptivePlaywrightCrawlerStatistics({
+                new Statistics({
                     logMessage: `${AdaptivePlaywrightCrawler.name} request statistics:`,
+                    stateExtension: adaptivePlaywrightCrawlerStatisticState,
                 }),
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
             // The base crawler must not wrap requests in a transaction of its own - this crawler opens
@@ -334,7 +322,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         try {
             if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
                 crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
-                this.stats.trackHttpOnlyRequestHandlerRun();
+                this.stats.state.httpOnlyRequestHandlerRuns++;
                 const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
                 if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {
                     crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
@@ -354,11 +342,11 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 }
                 else {
                     crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
-                    this.stats.trackRenderingTypeMisprediction();
+                    this.stats.state.renderingTypeMispredictions++;
                 }
             }
             crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
-            this.stats.trackBrowserRequestHandlerRun();
+            this.stats.state.browserRequestHandlerRuns++;
             // Run the request handler in a browser. The copy of the crawler state is kept so that we can perform
             // a rendering type detection if necessary. Without this measure, the HTTP request handler would run
             // under different conditions, which could change its behavior. Changes done to the crawler state by

package/internals/playwright-crawler.d.ts

@@ -12,9 +12,9 @@ export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
 export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
 }
 export type PlaywrightHook<UserData extends Dictionary = any> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
-export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
+export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
     browserPlugins: [PlaywrightPlugin];
-}, Routes> {
+}, Routes, StatisticStateExtension> {
     /**
      * The same options as used by {@link launchPlaywright}.
      */
@@ -142,9 +142,9 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
  * ```
  * @category Crawlers
  */
-export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>> extends BrowserCrawler<Page, Response, {
+export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, {
     browserPlugins: [PlaywrightPlugin];
-}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     protected static optionsShape: {
         browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
@@ -266,7 +266,7 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
+    constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
 // @ts-ignore optional peer dependency or compatibility with es2022
     protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, Response, Dictionary, Dictionary> & {
         injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.124",
+    "version": "4.0.0-beta.125",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.124",
-        "@crawlee/browser": "4.0.0-beta.124",
-        "@crawlee/browser-pool": "4.0.0-beta.124",
-        "@crawlee/cheerio": "4.0.0-beta.124",
-        "@crawlee/core": "4.0.0-beta.124",
-        "@crawlee/types": "4.0.0-beta.124",
-        "@crawlee/utils": "4.0.0-beta.124",
+        "@crawlee/basic": "4.0.0-beta.125",
+        "@crawlee/browser": "4.0.0-beta.125",
+        "@crawlee/browser-pool": "4.0.0-beta.125",
+        "@crawlee/cheerio": "4.0.0-beta.125",
+        "@crawlee/core": "4.0.0-beta.125",
+        "@crawlee/types": "4.0.0-beta.125",
+        "@crawlee/utils": "4.0.0-beta.125",
         "cheerio": "^1.0.0",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
@@ -84,5 +84,5 @@
             }
         }
     },
-    "gitHead": "0694ee1b94c755b98141671baa93cc363f2bf8e3"
+    "gitHead": "04a7212dd4abe4a5f94f5fc9632acb6089819c8b"
 }