@crawlee/playwright 4.0.0-beta.125 → 4.0.0-beta.127

package/index.d.ts

@@ -1,4 +1,5 @@
 export * from '@crawlee/browser';
+export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
 export * from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';

package/index.js

@@ -1,4 +1,5 @@
 export * from '@crawlee/browser';
+export * from './internals/playwright-browser-pool.js';
 export * from './internals/playwright-crawler.js';
 export * from './internals/playwright-launcher.js';
 export * from './internals/adaptive-playwright-crawler.js';

package/internals/adaptive-playwright-crawler.d.ts

@@ -18,7 +18,7 @@ declare const adaptiveStatisticStateSchema: z.ZodObject<{
 }, z.core.$strip>;
 /**
  * The extra statistics fields {@link AdaptivePlaywrightCrawler} tracks on top of the built-in
- * {@link StatisticState} ones. They are available on `crawler.stats.state` and are persisted with the rest of
+ * {@link StatisticState} ones. They are available on `crawler.statistics.state` and are persisted with the rest of
  * the statistics.
  */
 export type AdaptivePlaywrightCrawlerStatisticState = z.infer<typeof adaptiveStatisticStateSchema>;

package/internals/adaptive-playwright-crawler.js

@@ -322,7 +322,7 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
         try {
             if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
                 crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
-                this.stats.state.httpOnlyRequestHandlerRuns++;
+                this.statistics.state.httpOnlyRequestHandlerRuns++;
                 const plainHTTPRun = await this.crawlOne('static', crawlingContext, crawlingContext.useState, transactions);
                 if (plainHTTPRun.ok && this.#resultChecker(plainHTTPRun.result)) {
                     crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
@@ -342,11 +342,11 @@ export class AdaptivePlaywrightCrawler extends BasicCrawler {
                 }
                 else {
                     crawlingContext.log.warning(`HTTP-only request handler returned a suspicious result for ${crawlingContext.request.url}`);
-                    this.stats.state.renderingTypeMispredictions++;
+                    this.statistics.state.renderingTypeMispredictions++;
                 }
             }
             crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
-            this.stats.state.browserRequestHandlerRuns++;
+            this.statistics.state.browserRequestHandlerRuns++;
             // Run the request handler in a browser. The copy of the crawler state is kept so that we can perform
             // a rendering type detection if necessary. Without this measure, the HTTP request handler would run
             // under different conditions, which could change its behavior. Changes done to the crawler state by

package/internals/playwright-browser-pool.d.ts

@@ -0,0 +1,71 @@
+import type { Configuration } from '@crawlee/browser';
+import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, PlaywrightPlugin, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
+// @ts-ignore optional peer dependency or compatibility with es2022
+import type { Page } from 'playwright';
+import type { PlaywrightLaunchContext } from './playwright-launcher.js';
+/** A {@link BrowserPool} of Playwright browsers, as built by {@link playwrightBrowserPool}. */
+export type PlaywrightBrowserPool = BrowserPool<{
+    browserPlugins: [PlaywrightPlugin];
+}, [PlaywrightPlugin]>;
+export interface PlaywrightBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<PlaywrightPlugin['createController']>, ReturnType<PlaywrightPlugin['createLaunchContext']>, Page> {
+    /** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */
+    launchContext?: PlaywrightLaunchContext;
+    /**
+     * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
+     * Defaults to `true`, and can also be set via {@link Configuration}.
+     */
+    headless?: boolean;
+    /** Configuration to read the browser defaults from. Defaults to the global configuration. */
+    configuration?: Configuration;
+}
+export interface RemotePlaywrightBrowserPoolOptions extends Pick<PlaywrightBrowserPoolOptions, 'launchContext' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
+}
+/**
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: playwrightBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         launchContext: { launcher: firefox },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function playwrightBrowserPool(options?: PlaywrightBrowserPoolOptions): PlaywrightBrowserPool;
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
+ *
+ * A {@link PlaywrightCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: remotePlaywrightBrowserPool({
+ *         endpoint: 'wss://production-sfo.browserless.io?token=xxx',
+ *         maxOpenBrowsers: 2,
+ *         browserPoolOptions: { useFingerprints: false },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * @category Browser management
+ */
+export declare function remotePlaywrightBrowserPool(options: RemotePlaywrightBrowserPoolOptions): RemoteBrowserPool<Page>;

package/internals/playwright-browser-pool.js

@@ -0,0 +1,61 @@
+import { PlaywrightLauncher } from './playwright-launcher.js';
+/**
+ * Builds a {@link BrowserPool} of Playwright browsers to pass to a {@link PlaywrightCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext` and
+ * `headless`, and derives the browser plugin from them - so a pool built here always matches the crawler it is
+ * given to, and configuring one never means assembling a {@link PlaywrightPlugin} by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: playwrightBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         launchContext: { launcher: firefox },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export function playwrightBrowserPool(options = {}) {
+    const { launchContext, headless, configuration, ...poolOptions } = options;
+    return playwrightLauncher(launchContext, headless, configuration).createBrowserPool(poolOptions);
+}
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link playwrightBrowserPool}: connects to a remote browser
+ * service (Browserbase, Browserless, Steel, ...) with a Playwright plugin derived from `launchContext`.
+ *
+ * A {@link PlaywrightCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new PlaywrightCrawler({
+ *     browserPool: remotePlaywrightBrowserPool({
+ *         endpoint: 'wss://production-sfo.browserless.io?token=xxx',
+ *         maxOpenBrowsers: 2,
+ *         browserPoolOptions: { useFingerprints: false },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * @category Browser management
+ */
+export function remotePlaywrightBrowserPool(options) {
+    const { launchContext, headless, configuration, ...remoteOptions } = options;
+    return playwrightLauncher(launchContext, headless, configuration).createRemoteBrowserPool(remoteOptions);
+}
+function playwrightLauncher(launchContext = {}, headless, configuration) {
+    return new PlaywrightLauncher(headless == null
+        ? launchContext
+        : { ...launchContext, launchOptions: { ...launchContext.launchOptions, headless } }, configuration);
+}

package/internals/playwright-crawler.d.ts

@@ -1,6 +1,5 @@
 import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import { BrowserCrawler } from '@crawlee/browser';
-import type { PlaywrightPlugin } from '@crawlee/browser-pool';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
 import type { Download, LaunchOptions, Page, Response } from 'playwright';
@@ -12,13 +11,16 @@ export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
 export interface PlaywrightCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<Page, Response, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
 }
 export type PlaywrightHook<UserData extends Dictionary = any> = BrowserHook<PlaywrightCrawlingContext<UserData>>;
-export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
-    browserPlugins: [PlaywrightPlugin];
-}, Routes, StatisticStateExtension> {
+export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<Page, Response, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     /**
      * The same options as used by {@link launchPlaywright}.
      */
     launchContext?: PlaywrightLaunchContext;
+    /**
+     * Whether to run browser in headless mode. Defaults to `true`.
+     * Can be also set via {@link Configuration}.
+     */
+    headless?: boolean;
     /**
      * Function that is called to process each request.
      *
@@ -142,18 +144,16 @@ export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary<never>,
  * ```
  * @category Crawlers
  */
-export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, {
-    browserPlugins: [PlaywrightPlugin];
-}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
+export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<PlaywrightCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<Page, Response, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     protected static optionsShape: {
-        browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        headless: z.ZodOptional<z.ZodBoolean>;
         launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
-        headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
         browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
         proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -205,14 +205,14 @@ export declare class PlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
         id: z.ZodOptional<z.ZodString>;
     };
     protected static optionsSchema: z.ZodObject<{
-        browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        headless: z.ZodOptional<z.ZodBoolean>;
         launcher: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
-        headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
         browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
         proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;

package/internals/playwright-crawler.js

@@ -1,6 +1,6 @@
-import { BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator } from '@crawlee/browser';
+import { assertBrowserPoolNotConfigured, BrowserCrawler, parseArgument, RequestState, Router, schemas, serviceLocator, } from '@crawlee/browser';
 import { z } from 'zod';
-import { PlaywrightLauncher } from './playwright-launcher.js';
+import { playwrightBrowserPool, remotePlaywrightBrowserPool } from './playwright-browser-pool.js';
 import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
 /**
  * Provides a simple framework for parallel crawling of web pages
@@ -70,7 +70,7 @@ import { gotoExtended, playwrightUtils } from './utils/playwright-utils.js';
 export class PlaywrightCrawler extends BrowserCrawler {
     static optionsShape = {
         ...BrowserCrawler.optionsShape,
-        browserPoolOptions: schemas.anyObject.optional(),
+        headless: z.boolean().optional(),
         launcher: schemas.anyObject.optional(),
     };
     static optionsSchema = z.strictObject(PlaywrightCrawler.optionsShape);
@@ -79,29 +79,25 @@ export class PlaywrightCrawler extends BrowserCrawler {
      */
     constructor(options = {}) {
         const parsedOptions = parseArgument(options, PlaywrightCrawler.optionsSchema, 'PlaywrightCrawlerOptions');
-        const { launchContext, headless, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
-        const browserPoolOptions = {
-            ...parsedOptions.browserPoolOptions,
-        };
+        const { launchContext, headless, configuration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
         if (launchContext.proxyUrl) {
             throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.' +
                 'Use PlaywrightCrawlerOptions.proxyConfiguration');
         }
-        // `browserPlugins` is working when it's not overridden by `launchContext`,
-        // which for crawlers it is always overridden. Hence the error to use the other option.
-        if (browserPoolOptions.browserPlugins) {
-            throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');
-        }
-        if (headless != null) {
-            launchContext.launchOptions ??= {};
-            launchContext.launchOptions.headless = headless;
+        if (options.browserPool) {
+            // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
+            assertBrowserPoolNotConfigured(new.target.name, {
+                launchContext: options.launchContext,
+                headless: options.headless,
+            });
         }
-        const playwrightLauncher = new PlaywrightLauncher(launchContext, parsedOptions.configuration);
-        browserPoolOptions.browserPlugins = [playwrightLauncher.createBrowserPlugin()];
         super({
             ...browserCrawlerOptions,
             launchContext,
-            browserPoolOptions,
+            configuration,
+            browserPoolBuilder: (remoteBrowser) => remoteBrowser
+                ? remotePlaywrightBrowserPool({ ...remoteBrowser, launchContext, headless, configuration })
+                : playwrightBrowserPool({ launchContext, headless, configuration }),
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/playwright",
-    "version": "4.0.0-beta.125",
+    "version": "4.0.0-beta.127",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
         "node": ">=22.0.0"
@@ -49,13 +49,13 @@
     "dependencies": {
         "@apify/datastructures": "^2.0.3",
         "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.125",
-        "@crawlee/browser": "4.0.0-beta.125",
-        "@crawlee/browser-pool": "4.0.0-beta.125",
-        "@crawlee/cheerio": "4.0.0-beta.125",
-        "@crawlee/core": "4.0.0-beta.125",
-        "@crawlee/types": "4.0.0-beta.125",
-        "@crawlee/utils": "4.0.0-beta.125",
+        "@crawlee/basic": "4.0.0-beta.127",
+        "@crawlee/browser": "4.0.0-beta.127",
+        "@crawlee/browser-pool": "4.0.0-beta.127",
+        "@crawlee/cheerio": "4.0.0-beta.127",
+        "@crawlee/core": "4.0.0-beta.127",
+        "@crawlee/types": "4.0.0-beta.127",
+        "@crawlee/utils": "4.0.0-beta.127",
         "cheerio": "^1.0.0",
         "jquery": "^3.7.1",
         "ml-logistic-regression": "^2.0.0",
@@ -84,5 +84,5 @@
             }
         }
     },
-    "gitHead": "04a7212dd4abe4a5f94f5fc9632acb6089819c8b"
+    "gitHead": "4aa4a4d8d105bb530649ac5cd9197a3169106bd9"
 }