@crawlee/playwright 4.0.0-beta.124 → 4.0.0-beta.126

package/index.d.ts

@@ -1,4 +1,5 @@
 export * from '@crawlee/browser';
+export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
 export * from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';

package/index.js

@@ -1,4 +1,5 @@
 export * from '@crawlee/browser';
+export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
 export * from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';

package/internals/adaptive-playwright-crawler.d.ts

@@ -1,34 +1,39 @@
 import type { BrowserHook, LoadedRequest, Request, RouterHandler, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import type { BasicCrawlerOptions } from '@crawlee/basic';
 import { BasicCrawler } from '@crawlee/basic';
-import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StatisticPersistedState, StatisticState, StorageTransactionView } from '@crawlee/core';
-import { Statistics } from '@crawlee/core';
+import type { ContextPipeline, CrawlingContext, EnqueueLinksOptions, GetUserDataFromRequest, RouterRoutes, StorageTransactionView } from '@crawlee/core';
 import type { Dictionary, Awaitable } from '@crawlee/types';
 import { type CheerioRoot } from '@crawlee/utils/internal';
 import { type Cheerio } from 'cheerio';
 import type { AnyNode } from 'domhandler';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Page } from 'playwright';
+import { z } from 'zod';
 import type { PlaywrightCrawlingContext, PlaywrightGotoOptions } from './playwright-crawler.js';
 import { type IRenderingTypePredictor } from './utils/rendering-type-prediction.js';
-interface AdaptivePlaywrightCrawlerStatisticState extends StatisticState {
-    httpOnlyRequestHandlerRuns?: number;
-    browserRequestHandlerRuns?: number;
-    renderingTypeMispredictions?: number;
-}
-interface AdaptivePlaywrightCrawlerPersistedStatisticState extends StatisticPersistedState {
-    httpOnlyRequestHandlerRuns?: number;
-    browserRequestHandlerRuns?: number;
-    renderingTypeMispredictions?: number;
-}
-declare class AdaptivePlaywrightCrawlerStatistics extends Statistics {
-    get state(): AdaptivePlaywrightCrawlerStatisticState;
-    protected defaultState(): AdaptivePlaywrightCrawlerStatisticState;
-    protected deserializeState(persistedState: AdaptivePlaywrightCrawlerPersistedStatisticState): AdaptivePlaywrightCrawlerStatisticState;
-    trackHttpOnlyRequestHandlerRun(): void;
-    trackBrowserRequestHandlerRun(): void;
-    trackRenderingTypeMisprediction(): void;
-}
+declare const adaptiveStatisticStateSchema: z.ZodObject<{
+    httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+    browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+    renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
+}, z.core.$strip>;
+/**
+ * The extra statistics fields {@link AdaptivePlaywrightCrawler} tracks on top of the built-in
+ * {@link StatisticState} ones. They are available on `crawler.stats.state` and are persisted with the rest of
+ * the statistics.
+ */
+export type AdaptivePlaywrightCrawlerStatisticState = z.infer<typeof adaptiveStatisticStateSchema>;
+/**
+ * The {@link AdaptivePlaywrightCrawlerStatisticState} fields as a {@link Statistics} state extension, defaults
+ * and all. A {@link Statistics} instance to be injected into an {@link AdaptivePlaywrightCrawler} has to carry
+ * them - `deserialize.extend()` your own fields onto this one and pass the result as `stateExtension`.
+ */
+export declare const adaptivePlaywrightCrawlerStatisticState: {
+    deserialize: z.ZodObject<{
+        httpOnlyRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+        browserRequestHandlerRuns: z.ZodDefault<z.ZodNumber>;
+        renderingTypeMispredictions: z.ZodDefault<z.ZodNumber>;
+    }, z.core.$strip>;
+};
 export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary = any> extends CrawlingContext<UserData> {
     request: LoadedRequest<Request<UserData>>;
     /**
@@ -87,7 +92,7 @@ type AdaptiveHook<ContextExtension = Dictionary<never>> = BrowserHook<AdaptiveHo
 type AdaptivePostNavigationHook<ContextExtension = Dictionary<never>> = BrowserHook<Omit<AdaptiveHookContext, 'request'> & {
     request: LoadedRequest<Request>;
 }, ContextExtension>;
-export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes>, 'preNavigationHooks' | 'postNavigationHooks'> {
+export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension>, 'preNavigationHooks' | 'postNavigationHooks'> {
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies.
      * The function accepts a subset of the crawling context. If you attempt to access the `page` property during HTTP-only crawling,
@@ -177,10 +182,9 @@ export interface AdaptivePlaywrightCrawlerOptions<ContextExtension = Dictionary<
  *
  * @experimental
  */
-export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes> {
+export declare class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<AdaptivePlaywrightCrawlerContext['request']>>, StatisticStateExtension extends AdaptivePlaywrightCrawlerStatisticState = AdaptivePlaywrightCrawlerStatisticState> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     #private;
-    get stats(): AdaptivePlaywrightCrawlerStatistics;
-    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
+    constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
     protected init(): Promise<void>;
     protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, CrawlingContext<Dictionary> & {
         readonly request: LoadedRequest<Request<Dictionary>>;

package/internals/adaptive-playwright-crawler.js

@@ -8,39 +8,22 @@ import { z } from 'zod';
 import { addTimeoutToPromise } from '@apify/timeout';
 import { PlaywrightCrawler } from './playwright-crawler.js';
 import { RenderingTypePredictor, } from './utils/rendering-type-prediction.js';
-class AdaptivePlaywrightCrawlerStatistics extends Statistics {
-    get state() {
-        return super.state;
-    }
-    defaultState() {
-        return {
-            ...super.defaultState(),
-            httpOnlyRequestHandlerRuns: 0,
-            browserRequestHandlerRuns: 0,
-            renderingTypeMispredictions: 0,
-        };
-    }
-    deserializeState(persistedState) {
-        return {
-            ...super.deserializeState(persistedState),
-            httpOnlyRequestHandlerRuns: persistedState.httpOnlyRequestHandlerRuns,
-            browserRequestHandlerRuns: persistedState.browserRequestHandlerRuns,
-            renderingTypeMispredictions: persistedState.renderingTypeMispredictions,
-        };
-    }
-    trackHttpOnlyRequestHandlerRun() {
-        this.state.httpOnlyRequestHandlerRuns ??= 0;
-        this.state.httpOnlyRequestHandlerRuns += 1;
-    }
-    trackBrowserRequestHandlerRun() {
-        this.state.browserRequestHandlerRuns ??= 0;
-        this.state.browserRequestHandlerRuns += 1;
-    }
-    trackRenderingTypeMisprediction() {
-        this.state.renderingTypeMispredictions ??= 0;
-        this.state.renderingTypeMispredictions += 1;
-    }
-}
+const adaptiveStatisticStateSchema = z.object({
+    /** How many requests were handled by the HTTP-only request handler. */
+    httpOnlyRequestHandlerRuns: z.number().default(0),
+    /** How many requests were handled in a browser. */
+    browserRequestHandlerRuns: z.number().default(0),
+    /** How many times the HTTP-only handler produced a result the `resultChecker` rejected. */
+    renderingTypeMispredictions: z.number().default(0),
+});
+/**
+ * The {@link AdaptivePlaywrightCrawlerStatisticState} fields as a {@link Statistics} state extension, defaults
+ * and all. A {@link Statistics} instance to be injected into an {@link AdaptivePlaywrightCrawler} has to carry
+ * them - `deserialize.extend()` your own fields onto this one and pass the result as `stateExtension`.
+ */
+export const adaptivePlaywrightCrawlerStatisticState = {
+    deserialize: adaptiveStatisticStateSchema,
+};
 const proxyLogMethods = [
     'error',
     'exception',
@@ -88,10 +71,6 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
     #staticContextPipeline;
     #browserContextPipeline;
     #individualRequestHandlerTimeoutMillis;
-    // The constructor always injects an `AdaptivePlaywrightCrawlerStatistics`, so narrowing the cast is sound.
-    get stats() {
-        return super.stats;
-    }
     /**
      * The write policy of the per-attempt transactions. Defaults the request queue to `deferred`:
      * a discarded attempt's enqueues must never reach the queue.
@@ -103,6 +82,16 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         // The user's value is replaced by `false` in the `super` call below — validate it separately,
         // wrapped in an object so the error still names the field.
         parseArgument({ transactionalStorage }, z.object({ transactionalStorage: BasicCrawler.optionsShape.transactionalStorage }), 'AdaptivePlaywrightCrawlerOptions');
+        // The extra fields are only tracked if the injected instance was built with them - the types enforce that,
+        // but plain JS callers would otherwise silently increment `undefined` into a sticky `NaN`. Extend
+        // `adaptivePlaywrightCrawlerStatisticState` to satisfy this.
+        if (statistics !== undefined) {
+            parseArgument(statistics.state, z.object({
+                httpOnlyRequestHandlerRuns: z.number(),
+                browserRequestHandlerRuns: z.number(),
+                renderingTypeMispredictions: z.number(),
+            }), 'statistics.state');
+        }
         // Per-attempt buffering is load-bearing here: the handler runs up to twice per request and the
         // losing attempt's writes must be discardable.
         if (transactionalStorage === false) {
@@ -111,20 +100,19 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 'attempts. `transactionalStorage: false` is therefore not supported; a write policy ' +
                 'object is accepted and forwarded to the per-attempt transactions.');
         }
-        if (statistics !== undefined && !(statistics instanceof AdaptivePlaywrightCrawlerStatistics)) {
-            throw new Error('AdaptivePlaywrightCrawler tracks extra fields on its own Statistics subclass and cannot use a ' +
-                'plain `statistics` instance. Omit the option to let the crawler build its own.');
-        }
         super({
             ...rest,
             errorHandler,
             failedRequestHandler,
             requestHandler,
             requestHandlerTimeoutSecs,
-            // Inject our subclass so the base tracks the extra adaptive fields instead of building a plain `Statistics`.
+            // The base would build a `Statistics` without the adaptive fields, so provide a default that has them.
+            // The cast covers a `StatisticStateExtension` that adds further fields - those can only come from an
+            // injected instance, in which case this default is never built.
             statistics: statistics ??
-                new AdaptivePlaywrightCrawlerStatistics({
+                new Statistics({
                     logMessage: `${AdaptivePlaywrightCrawler.name} request statistics:`,
+                    stateExtension: adaptivePlaywrightCrawlerStatisticState,
                 }),
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
             // The base crawler must not wrap requests in a transaction of its own - this crawler opens
@@ -334,7 +322,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         try {
             if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
                 crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
-                this.stats.trackHttpOnlyRequestHandlerRun();
+                this.stats.state.httpOnlyRequestHandlerRuns++;
                 const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
                 if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {
                     crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
@@ -354,11 +342,11 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 }
                 else {
                     crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
-                    this.stats.trackRenderingTypeMisprediction();
+                    this.stats.state.renderingTypeMispredictions++;
                 }
             }
             crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
-            this.stats.trackBrowserRequestHandlerRun();
+            this.stats.state.browserRequestHandlerRuns++;
             // Run the request handler in a browser. The copy of the crawler state is kept so that we can perform
             // a rendering type detection if necessary. Without this measure, the HTTP request handler would run
             // under different conditions, which could change its behavior. Changes done to the crawler state by

package/internals/playwright-browser-pool.d.ts

@@ -0,0 +1,71 @@
+import type { Configuration } from '@crawlee/browser';
+import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PlaywrightPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
+// @ts-ignore optional peer dependency or compatibility with es2022
+import type { Page } from 'playwright';
+import type { PlaywrightLaunchContext } from './playwright-launcher.js';
+/** A {@link BrowserPool} of Playwright browsers, as built by {@link playwrightBrowserPool}. */
+export type PlaywrightBrowserPool = BrowserPool<{
+    browserPlugins: [PlaywrightPlugin];
+}, [PlaywrightPlugin]>;
+export interface PlaywrightBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PlaywrightPlugin['createController']>, ReturnType<PlaywrightPlugin['createLaunchContext']>, Page> {
+    /** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */
+    launchContext?: PlaywrightLaunchContext;
+    /**
+     * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
+     * Defaults to `true`, and can also be set via {@link Configuration}.
+     */
+    headless?: boolean;
+    /** Configuration to read the browser defaults from. Defaults to the global configuration. */
+    configuration?: Configuration;
+}
+export interface RemotePlaywrightBrowserPoolOptions extends Pick<PlaywrightBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
+}
+/**
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: playwrightBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         launchContext: { launcher: firefox },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function playwrightBrowserPool(options?: PlaywrightBrowserPoolOptions): PlaywrightBrowserPool;
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
+ *
+ * A {@link PlaywrightCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: remotePlaywrightBrowserPool({
+ *         endpoint: 'wss://production-sfo.browserless.io?token=xxx',
+ *         maxOpenBrowsers: 2,
+ *         browserPoolOptions: { useFingerprints: false },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * @category Browser management
+ */
+export declare function remotePlaywrightBrowserPool(options: RemotePlaywrightBrowserPoolOptions): RemoteBrowserPool<Page>;

package/internals/playwright-browser-pool.js

@@ -0,0 +1,61 @@
+import { PlaywrightLauncher } from './playwright-launcher.js';
+/**
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: playwrightBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         launchContext: { launcher: firefox },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export function playwrightBrowserPool(options = {}) {
+    const { launchContext, headless, configuration, ...poolOptions } = options;
+    return playwrightLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
+}
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
+ *
+ * A {@link PlaywrightCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: remotePlaywrightBrowserPool({
+ *         endpoint: 'wss://production-sfo.browserless.io?token=xxx',
+ *         maxOpenBrowsers: 2,
+ *         browserPoolOptions: { useFingerprints: false },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * @category Browser management
+ */
+export function remotePlaywrightBrowserPool(options) {
+    const { launchContext, headless, configuration, ...remoteOptions } = options;
+    return playwrightLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
+}
+function playwrightLauncher(launchContext = {}, headless, configuration) {
+    return new PlaywrightLauncher(headless == null
+        ? launchContext
+        : { ...launchContext, launchOptions: { ...launchContext.launchOptions, headless } }, configuration);
+}

package/internals/playwright-crawler.d.ts

@@ -1,6 +1,5 @@
 import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import { BrowserCrawler } from '@crawlee/browser';
-import type { PlaywrightPlugin } from '@crawlee/browser-pool';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Download, LaunchOptions, Page, Response } from 'playwright';
@@ -12,13 +11,16 @@ export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
 export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
 }
 export type PlaywrightHook<UserData extends Dictionary = any> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
-export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
-    browserPlugins: [PlaywrightPlugin];
-}, Routes> {
+export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     /**
      * The same options as used by {@link launchPlaywright}.
      */
     launchContext?: PlaywrightLaunchContext;
+    /**
+     * Whether to run browser in headless mode. Defaults to `true`.
+     * Can be also set via {@link Configuration}.
+     */
+    headless?: boolean;
     /**
      * Function that is called to process each request.
      *
@@ -142,18 +144,16 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
  * ```
  * @category Crawlers
  */
-export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>> extends BrowserCrawler<Page, Response, {
-    browserPlugins: [PlaywrightPlugin];
-}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     protected static optionsShape: {
-        browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        headless: z.ZodOptional<z.ZodBoolean>;
         launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
-        headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
         browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
         proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -205,14 +205,14 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
         id: z.ZodOptional<z.ZodString>;
     };
     protected static optionsSchema: z.ZodObject<{
-        browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        headless: z.ZodOptional<z.ZodBoolean>;
         launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
-        headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
         browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
         proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -266,7 +266,7 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
     /**
      * All `PlaywrightCrawler` parameters are passed via an options object.
      */
-    constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
+    constructor(options?: PlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
 // @ts-ignore optional peer dependency or compatibility with es2022
     protected buildContextPipeline(): import("@crawlee/browser").ContextPipeline<import("@crawlee/browser").CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, Response, Dictionary, Dictionary> & {
         injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;

package/internals/playwright-crawler.js

@@ -1,6 +1,6 @@
-import { BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator } from '@crawlee/browser';
+import { assertBrowserPoolNotConfigured, BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator, } from '@crawlee/browser';
 import { z } from 'zod';
-import { PlaywrightLauncher } from './playwright-launcher.js';
+import { playwrightBrowserPool, remotePlaywrightBrowserPool } from './playwright-browser-pool.js';
 import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
 /**
  * Provides a simple framework for parallel crawling of web pages
@@ -70,7 +70,7 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
 export class PlaywrightCrawler extends BrowserCrawler {
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
-        browserPoolOptions: schemas.anyObject.optional(),
+        headless: z.boolean().optional(),
         launcher: schemas.anyObject.optional(),
     };
     static optionsSchema = z.strictObject(PlaywrightCrawler.optionsShape);
@@ -79,29 +79,25 @@ export class PlaywrightCrawler extends BrowserCrawler {
      */
     constructor(options = {}) {
         const parsedOptions = parseArgument(options, PlaywrightCrawler.optionsSchema, 'PlaywrightCrawlerOptions');
-        const { launchContext, headless, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
-        const browserPoolOptions = {
-            ...parsedOptions.browserPoolOptions,
-        };
+        const { launchContext, headless, configuration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
         if (launchContext.proxyUrl) {
             throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.' +
                 'Use PlaywrightCrawlerOptions.proxyConfiguration');
         }
-        // `browserPlugins` is working when it's not overridden by `launchContext`,
-        // which for crawlers it is always overridden. Hence the error to use the other option.
-        if (browserPoolOptions.browserPlugins) {
-            throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');
-        }
-        if (headless != null) {
-            launchContext.launchOptions ??= {};
-            launchContext.launchOptions.headless = headless;
+        if (options.browserPool) {
+            // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
+            assertBrowserPoolNotConfigured(new.target.name, {
+                launchContext: options.launchContext,
+                headless: options.headless,
+            });
         }
-        const playwrightLauncher = new PlaywrightLauncher(launchContext, parsedOptions.configuration);
-        browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
         super({
             ...browserCrawlerOptions,
             launchContext,
-            browserPoolOptions,
+            configuration,
+            browserPoolBuilder: (remoteBrowser) => remoteBrowser
+                ? remotePlaywrightBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
+                : playwrightBrowserPool({ launchContext, headless, configuration }),
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.124",
+    "version": "4.0.0-beta.126",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.124",
-        "@crawlee/browser": "4.0.0-beta.124",
-        "@crawlee/browser-pool": "4.0.0-beta.124",
-        "@crawlee/cheerio": "4.0.0-beta.124",
-        "@crawlee/core": "4.0.0-beta.124",
-        "@crawlee/types": "4.0.0-beta.124",
-        "@crawlee/utils": "4.0.0-beta.124",
+        "@crawlee/basic": "4.0.0-beta.126",
+        "@crawlee/browser": "4.0.0-beta.126",
+        "@crawlee/browser-pool": "4.0.0-beta.126",
+        "@crawlee/cheerio": "4.0.0-beta.126",
+        "@crawlee/core": "4.0.0-beta.126",
+        "@crawlee/types": "4.0.0-beta.126",
+        "@crawlee/utils": "4.0.0-beta.126",
         "cheerio": "^1.0.0",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
@@ -84,5 +84,5 @@
             }
         }
     },
-    "gitHead": "0694ee1b94c755b98141671baa93cc363f2bf8e3"
+    "gitHead": "5f0a8c1a43d4587f8640881fc55836a23e1c4f30"
 }