@crawlee/browser 4.0.0-beta.99 → 4.0.0-rc.1

package/internals/browser-launcher.d.ts

@@ -1,6 +1,8 @@
 import { Configuration } from '@crawlee/basic';
-import type { BrowserPlugin, BrowserPluginOptions } from '@crawlee/browser-pool';
+import type { BrowserPlugin, BrowserPluginOptions, BrowserPoolHooks, BrowserPoolOptions, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
+import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
 import type { Constructor, Dictionary } from '@crawlee/types';
+import { z } from 'zod';
 export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginOptions<TOptions> {
     /**
      * URL to an HTTP proxy server. It must define the port number,
@@ -64,11 +66,25 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
      */
     launcher?: Launcher;
 }
+/**
+ * The {@link BrowserPool} options a launcher-built pool accepts: everything the pool itself takes except
+ * `browserPlugins`, which the launcher derives from its launch context. The hooks are deliberately unconstrained -
+ * the browser they run against is only known to the concrete `*BrowserPool()` factory, which is where the
+ * caller-facing types are pinned down.
+ */
+type LauncherBrowserPoolOptions = Omit<BrowserPoolOptions, 'browserPlugins'> & {
+    [Hook in keyof BrowserPoolHooks<any, any, any>]?: readonly ((...args: any[]) => unknown)[];
+};
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link LauncherBrowserPoolOptions}.
+ */
+type LauncherRemoteBrowserPoolOptions = Omit<RemoteBrowserPoolOptions, 'browserPlugins'>;
 /**
  * Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
  * @ignore
  */
 export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Launcher = Plugin['library'], T extends Constructor<Plugin> = Constructor<Plugin>, LaunchOptions extends Dictionary<any> | undefined = Partial<Parameters<Plugin['launch']>[0]>, LaunchResult extends ReturnType<Plugin['launch']> = ReturnType<Plugin['launch']>> {
+    #private;
     readonly configuration: Configuration;
     launcher: Launcher;
     proxyUrl?: string;
@@ -77,24 +93,30 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
     otherLaunchContextProps: Dictionary;
     Plugin: T;
     userAgent?: string;
+    /**
+     * @internal
+     */
     protected static optionsShape: {
-// @ts-ignore optional peer dependency or compatibility with es2022
-        proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useChrome: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        userDataDir: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        launchOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
-// @ts-ignore optional peer dependency or compatibility with es2022
-        userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
+        launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
     };
+    /** @internal */
+    protected static optionsSchema: z.ZodObject<{
+        proxyUrl: z.ZodOptional<z.ZodURL>;
+        useChrome: z.ZodOptional<z.ZodBoolean>;
+        useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
+        browserPerProxy: z.ZodOptional<z.ZodBoolean>;
+        ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
+        userDataDir: z.ZodOptional<z.ZodString>;
+        launchOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        userAgent: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
     static requireLauncherOrThrow<T>(launcher: string, apifyImageName: string): T;
     /**
      * All `BrowserLauncher` parameters are passed via an launchContext object.
@@ -104,6 +126,21 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
      * @ignore
      */
     createBrowserPlugin(): Plugin;
+    /**
+     * Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
+     * per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
+     * a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
+     * @internal
+     */
+    createBrowserPool(options?: LauncherBrowserPoolOptions): BrowserPool<{
+        browserPlugins: [Plugin];
+    }, [Plugin]>;
+    /**
+     * The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
+     * supplies the plugin, the caller supplies the remote connection details.
+     * @internal
+     */
+    createRemoteBrowserPool<Page>(options: LauncherRemoteBrowserPoolOptions): RemoteBrowserPool<Page>;
     /**
      * Launches a browser instance based on the plugin.
      * @returns Browser instance.
@@ -111,10 +148,5 @@ export declare abstract class BrowserLauncher<Plugin extends BrowserPlugin, Laun
     launch(): LaunchResult;
     createLaunchOptions(): Dictionary;
     protected getDefaultHeadlessOption(): boolean;
-    private getChromeExecutablePath;
-    /**
-     * Gets a typical path to Chrome executable, depending on the current operating system.
-     */
-    private getTypicalChromeExecutablePath;
-    private validateProxyUrlProtocol;
 }
+export {};

package/internals/browser-launcher.js

@@ -1,8 +1,10 @@
 import fs from 'node:fs';
 import { createRequire } from 'node:module';
 import os from 'node:os';
-import { Configuration } from '@crawlee/basic';
-import ow from 'ow';
+import { Configuration, serviceLocator } from '@crawlee/basic';
+import { BrowserPool, RemoteBrowserPool } from '@crawlee/browser-pool';
+import { schemas } from '@crawlee/utils/internal';
+import { z } from 'zod';
 const DEFAULT_VIEWPORT = {
     width: 1366,
     height: 768,
@@ -22,16 +24,21 @@ export class BrowserLauncher {
     // to be provided by child classes;
     Plugin;
     userAgent;
+    /**
+     * @internal
+     */
     static optionsShape = {
-        proxyUrl: ow.optional.string.url,
-        useChrome: ow.optional.boolean,
-        useIncognitoPages: ow.optional.boolean,
-        browserPerProxy: ow.optional.boolean,
-        ignoreProxyCertificate: ow.optional.boolean,
-        userDataDir: ow.optional.string,
-        launchOptions: ow.optional.object,
-        userAgent: ow.optional.string,
+        proxyUrl: z.url().optional(),
+        useChrome: z.boolean().optional(),
+        useIncognitoPages: z.boolean().optional(),
+        browserPerProxy: z.boolean().optional(),
+        ignoreProxyCertificate: z.boolean().optional(),
+        userDataDir: z.string().optional(),
+        launchOptions: schemas.anyObject.optional(),
+        userAgent: z.string().optional(),
     };
+    /** @internal */
+    static optionsSchema = z.strictObject(BrowserLauncher.optionsShape);
     static requireLauncherOrThrow(launcher, apifyImageName) {
         try {
             return require(launcher); // eslint-disable-line
@@ -54,7 +61,7 @@ export class BrowserLauncher {
     constructor(launchContext, configuration = Configuration.getGlobalConfiguration()) {
         this.configuration = configuration;
         const { launcher, proxyUrl, useChrome, userAgent, launchOptions = {}, ...otherLaunchContextProps } = launchContext;
-        this.validateProxyUrlProtocol(proxyUrl);
+        this.#validateProxyUrlProtocol(proxyUrl);
         // those need to be reassigned otherwise they are {} in types
         this.launcher = launcher;
         this.proxyUrl = proxyUrl;
@@ -73,6 +80,49 @@ export class BrowserLauncher {
             ...this.otherLaunchContextProps,
         });
     }
+    /**
+     * Builds a {@link BrowserPool} running a single plugin for this launcher's browser. Shared body of the
+     * per-library `*BrowserPool()` factories, which exist so that configuring a pool never requires assembling
+     * a plugin by hand — and therefore never lets the plugin drift away from the crawler it is used with.
+     * @internal
+     */
+    createBrowserPool(options = {}) {
+        // The hook types `BrowserPool` derives from `Plugin` are unresolvable while `Plugin` is still a free type
+        // parameter, so the argument cannot be checked here. The concrete `*BrowserPool()` factories are where the
+        // caller-facing hook types get pinned down.
+        return new BrowserPool({
+            ...this.#resolveFingerprinting(options),
+            browserPlugins: [this.createBrowserPlugin()],
+        });
+    }
+    /**
+     * The {@link RemoteBrowserPool} counterpart of {@link BrowserLauncher.createBrowserPool}: the launcher
+     * supplies the plugin, the caller supplies the remote connection details.
+     * @internal
+     */
+    createRemoteBrowserPool(options) {
+        return new RemoteBrowserPool({
+            ...options,
+            browserPlugins: [this.createBrowserPlugin()],
+            browserPoolOptions: this.#resolveFingerprinting(options.browserPoolOptions ?? {}),
+        });
+    }
+    /**
+     * A custom `userAgent` and Crawlee's fingerprint injection would both write the same headers, so an
+     * explicitly requested user agent wins.
+     */
+    #resolveFingerprinting(options) {
+        if (!this.userAgent) {
+            return options;
+        }
+        if (options.useFingerprints) {
+            serviceLocator
+                .getLogger()
+                .child({ prefix: 'BrowserLauncher' })
+                .info('Custom user agent provided, disabling automatic browser fingerprint injection!');
+        }
+        return { ...options, useFingerprints: false };
+    }
     /**
      * Launches a browser instance based on the plugin.
      * @returns Browser instance.
@@ -98,20 +148,20 @@ export class BrowserLauncher {
             launchOptions.headless = this.getDefaultHeadlessOption();
         }
         if (this.useChrome && !launchOptions.executablePath) {
-            launchOptions.executablePath = this.getChromeExecutablePath();
+            launchOptions.executablePath = this.#getChromeExecutablePath();
         }
         return launchOptions;
     }
     getDefaultHeadlessOption() {
         return this.configuration.headless && !this.configuration.xvfb;
     }
-    getChromeExecutablePath() {
-        return this.configuration.chromeExecutablePath ?? this.getTypicalChromeExecutablePath();
+    #getChromeExecutablePath() {
+        return this.configuration.chromeExecutablePath ?? this.#getTypicalChromeExecutablePath();
     }
     /**
      * Gets a typical path to Chrome executable, depending on the current operating system.
      */
-    getTypicalChromeExecutablePath() {
+    #getTypicalChromeExecutablePath() {
         /**
          * Returns path of Chrome executable by its OS environment variable to deal with non-english language OS.
          * Taking also into account the old [chrome 380177 issue](https://bugs.chromium.org/p/chromium/issues/detail?id=380177).
@@ -139,7 +189,7 @@ export class BrowserLauncher {
                 return '/usr/bin/google-chrome';
         }
     }
-    validateProxyUrlProtocol(proxyUrl) {
+    #validateProxyUrlProtocol(proxyUrl) {
         if (!proxyUrl)
             return;
         if (!/^(http|https|socks4|socks5)/i.test(proxyUrl)) {

package/package.json

@@ -1,9 +1,9 @@
 {
     "name": "@crawlee/browser",
-    "version": "4.0.0-beta.99",
+    "version": "4.0.0-rc.1",
     "description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
     "engines": {
-        "node": ">=22.0.0"
+        "node": ">=22.13.0"
     },
     "type": "module",
     "exports": {
@@ -47,14 +47,13 @@
         "access": "public"
     },
     "dependencies": {
-        "@apify/timeout": "^0.4.4",
-        "@crawlee/basic": "4.0.0-beta.99",
-        "@crawlee/browser-pool": "4.0.0-beta.99",
-        "@crawlee/types": "4.0.0-beta.99",
-        "@crawlee/utils": "4.0.0-beta.99",
-        "ow": "^2.0.0",
+        "@apify/timeout": "^1.0.1",
+        "@crawlee/basic": "4.0.0-rc.1",
+        "@crawlee/browser-pool": "4.0.0-rc.1",
+        "@crawlee/types": "4.0.0-rc.1",
+        "@crawlee/utils": "4.0.0-rc.1",
         "tslib": "^2.8.1",
-        "type-fest": "^4.41.0"
+        "zod": "^4.5.4"
     },
     "peerDependencies": {
         "playwright": "*",
@@ -75,5 +74,5 @@
             }
         }
     },
-    "gitHead": "ad2748380941842bb10cff100f4b4caad92049e3"
+    "gitHead": "f354ca5e943bed1a5c657a1fce1974c053f9fff0"
 }