@crawlee/stagehand 4.0.0-beta.124 → 4.0.0-beta.126

package/index.d.ts

@@ -51,6 +51,8 @@
  */
 export * from '@crawlee/browser';
 export { StagehandCrawler, createStagehandRouter } from './internals/stagehand-crawler';
+export { stagehandBrowserPool, remoteStagehandBrowserPool } from './internals/stagehand-browser-pool';
+export type { StagehandBrowserPool, StagehandBrowserPoolOptions, RemoteStagehandBrowserPoolOptions, } from './internals/stagehand-browser-pool';
 export type { StagehandOptions, StagehandPage, StagehandCrawlingContext, StagehandHook, StagehandRequestHandler, StagehandGotoOptions, StagehandCrawlerOptions, } from './internals/stagehand-crawler';
 export type { StagehandLaunchContext } from './internals/stagehand-launcher';
 export * as stagehandUtils from './internals/utils/stagehand-utils';

package/index.js

@@ -51,7 +51,7 @@
  * @module @crawlee/stagehand
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.stagehandUtils = exports.createStagehandRouter = exports.StagehandCrawler = void 0;
+exports.stagehandUtils = exports.remoteStagehandBrowserPool = exports.stagehandBrowserPool = exports.createStagehandRouter = exports.StagehandCrawler = void 0;
 const tslib_1 = require("tslib");
 // Re-export everything from @crawlee/browser for convenience
 tslib_1.__exportStar(require("@crawlee/browser"), exports);
@@ -59,5 +59,9 @@ tslib_1.__exportStar(require("@crawlee/browser"), exports);
 var stagehand_crawler_1 = require("./internals/stagehand-crawler");
 Object.defineProperty(exports, "StagehandCrawler", { enumerable: true, get: function () { return stagehand_crawler_1.StagehandCrawler; } });
 Object.defineProperty(exports, "createStagehandRouter", { enumerable: true, get: function () { return stagehand_crawler_1.createStagehandRouter; } });
+// Export the browser pool factories, which are how a pool with non-default options reaches the crawler
+var stagehand_browser_pool_1 = require("./internals/stagehand-browser-pool");
+Object.defineProperty(exports, "stagehandBrowserPool", { enumerable: true, get: function () { return stagehand_browser_pool_1.stagehandBrowserPool; } });
+Object.defineProperty(exports, "remoteStagehandBrowserPool", { enumerable: true, get: function () { return stagehand_browser_pool_1.remoteStagehandBrowserPool; } });
 // Export utilities as namespace
 exports.stagehandUtils = tslib_1.__importStar(require("./internals/utils/stagehand-utils"));

package/index.mjs

@@ -83,6 +83,7 @@ export const SystemStatus = mod.SystemStatus;
 export const ThrottlingRequestManager = mod.ThrottlingRequestManager;
 export const activeStorageTransaction = mod.activeStorageTransaction;
 export const applyRequestTransform = mod.applyRequestTransform;
+export const assertBrowserPoolNotConfigured = mod.assertBrowserPoolNotConfigured;
 export const assertJsonSerializable = mod.assertJsonSerializable;
 export const bindMethodsToServiceLocator = mod.bindMethodsToServiceLocator;
 export const browserPoolCookieToToughCookie = mod.browserPoolCookieToToughCookie;
@@ -92,6 +93,7 @@ export const coerceNumber = mod.coerceNumber;
 export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
 export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
 export const constructUrlPatternObjects = mod.constructUrlPatternObjects;
+export const convertStateSync = mod.convertStateSync;
 export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
 export const crawleeConfigFields = mod.crawleeConfigFields;
 export const createBasicRouter = mod.createBasicRouter;
@@ -123,6 +125,7 @@ export const parseValue = mod.parseValue;
 export const purgeDefaultStorages = mod.purgeDefaultStorages;
 export const rejectOperationInTransaction = mod.rejectOperationInTransaction;
 export const remainingNavigationWindowMillis = mod.remainingNavigationWindowMillis;
+export const remoteStagehandBrowserPool = mod.remoteStagehandBrowserPool;
 export const resolveBaseUrlForEnqueueLinksFiltering = mod.resolveBaseUrlForEnqueueLinksFiltering;
 export const resolveStorageIdentifier = mod.resolveStorageIdentifier;
 export const schemas = mod.schemas;
@@ -130,6 +133,7 @@ export const serializeArray = mod.serializeArray;
 export const serializeValue = mod.serializeValue;
 export const serviceLocator = mod.serviceLocator;
 export const snapshotValue = mod.snapshotValue;
+export const stagehandBrowserPool = mod.stagehandBrowserPool;
 export const stagehandUtils = mod.stagehandUtils;
 export const supportsDomainThrottling = mod.supportsDomainThrottling;
 export const toughCookieToBrowserPoolCookie = mod.toughCookieToBrowserPoolCookie;

package/internals/stagehand-browser-pool.d.ts

@@ -0,0 +1,63 @@
+import type { Configuration } from '@crawlee/browser';
+import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
+// @ts-ignore optional peer dependency or compatibility with es2022
+import type { Page } from 'playwright';
+import type { StagehandOptions } from './stagehand-crawler';
+import type { StagehandLaunchContext } from './stagehand-launcher';
+import type { StagehandPlugin } from './stagehand-plugin';
+/** A {@link BrowserPool} of Stagehand browsers, as built by {@link stagehandBrowserPool}. */
+export type StagehandBrowserPool = BrowserPool<{
+    browserPlugins: [StagehandPlugin];
+}, [StagehandPlugin]>;
+export interface StagehandBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<StagehandPlugin['createController']>, ReturnType<StagehandPlugin['createLaunchContext']>, Page> {
+    /** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */
+    launchContext?: StagehandLaunchContext;
+    /** Stagehand configuration for the AI behavior and Browserbase integration. */
+    stagehandOptions?: StagehandOptions;
+    /**
+     * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
+     * Defaults to `true`, and can also be set via {@link Configuration}.
+     */
+    headless?: boolean;
+    /** Configuration to read the browser defaults from. Defaults to the global configuration. */
+    configuration?: Configuration;
+}
+export interface RemoteStagehandBrowserPoolOptions extends Pick<StagehandBrowserPoolOptions, 'launchContext' | 'stagehandOptions' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
+}
+/**
+ * Builds a {@link BrowserPool} of Stagehand browsers to pass to a {@link StagehandCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext`,
+ * `stagehandOptions` and `headless`, and derives the browser plugin from them - so a pool built here always
+ * matches the crawler it is given to, and configuring one never means assembling a {@link StagehandPlugin}
+ * by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new StagehandCrawler({
+ *     browserPool: stagehandBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         stagehandOptions: { env: 'LOCAL', model: 'openai/gpt-4.1-mini' },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function stagehandBrowserPool(options?: StagehandBrowserPoolOptions): StagehandBrowserPool;
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link stagehandBrowserPool}: connects to a remote browser
+ * service with a Stagehand plugin derived from `launchContext` and `stagehandOptions`.
+ *
+ * A {@link StagehandCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function remoteStagehandBrowserPool(options: RemoteStagehandBrowserPoolOptions): RemoteBrowserPool<Page>;

package/internals/stagehand-browser-pool.js

@@ -0,0 +1,55 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stagehandBrowserPool = stagehandBrowserPool;
+exports.remoteStagehandBrowserPool = remoteStagehandBrowserPool;
+const stagehand_launcher_1 = require("./stagehand-launcher");
+/**
+ * Builds a {@link BrowserPool} of Stagehand browsers to pass to a {@link StagehandCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext`,
+ * `stagehandOptions` and `headless`, and derives the browser plugin from them - so a pool built here always
+ * matches the crawler it is given to, and configuring one never means assembling a {@link StagehandPlugin}
+ * by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new StagehandCrawler({
+ *     browserPool: stagehandBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         stagehandOptions: { env: 'LOCAL', model: 'openai/gpt-4.1-mini' },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+function stagehandBrowserPool(options = {}) {
+    const { launchContext, stagehandOptions, headless, configuration, ...poolOptions } = options;
+    return stagehandLauncher(launchContext, stagehandOptions, headless, configuration).createBrowserPool(poolOptions);
+}
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link stagehandBrowserPool}: connects to a remote browser
+ * service with a Stagehand plugin derived from `launchContext` and `stagehandOptions`.
+ *
+ * A {@link StagehandCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * @category Browser management
+ */
+function remoteStagehandBrowserPool(options) {
+    const { launchContext, stagehandOptions, headless, configuration, ...remoteOptions } = options;
+    return stagehandLauncher(launchContext, stagehandOptions, headless, configuration).createRemoteBrowserPool(remoteOptions);
+}
+function stagehandLauncher(launchContext = {}, stagehandOptions = {}, headless, configuration) {
+    return new stagehand_launcher_1.StagehandLauncher({
+        ...launchContext,
+        stagehandOptions,
+        ...(headless == null ? {} : { launchOptions: { ...launchContext.launchOptions, headless } }),
+    }, configuration);
+}

package/internals/stagehand-crawler.d.ts

@@ -6,7 +6,6 @@ import type { Dictionary } from '@crawlee/types';
 import type { LaunchOptions, Page, Response } from 'playwright';
 import { z } from 'zod';
 import type { StagehandLaunchContext } from './stagehand-launcher';
-import type { StagehandPlugin } from './stagehand-plugin';
 /**
  * Stagehand-specific configuration options.
  */
@@ -196,9 +195,7 @@ export interface StagehandRequestHandler extends RequestHandler<LoadedContext<St
 /**
  * Options for StagehandCrawler.
  */
-export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>> extends BrowserCrawlerOptions<StagehandPage, Response, StagehandCrawlingContext, ContextExtension, ExtendedContext, {
-    browserPlugins: [StagehandPlugin];
-}, Routes> {
+export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<StagehandPage, Response, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     /**
      * Stagehand-specific configuration options.
      * These options configure the AI behavior and Browserbase integration.
@@ -208,6 +205,11 @@ export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, E
      * Launch context with Stagehand-specific options.
      */
     launchContext?: StagehandLaunchContext;
+    /**
+     * Whether to run browser in headless mode. Defaults to `true`.
+     * Can be also set via {@link Configuration}.
+     */
+    headless?: boolean;
     /**
      * Function that is called to process each request.
      *
@@ -311,18 +313,16 @@ export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, E
  * await crawler.run(['https://example.com']);
  * ```
  */
-export declare class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>> extends BrowserCrawler<StagehandPage, Response, {
-    browserPlugins: [StagehandPlugin];
-}, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes> {
+export declare class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<StagehandPage, Response, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
     protected static optionsShape: {
         stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        headless: z.ZodOptional<z.ZodBoolean>;
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
-        headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
         browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
         proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -375,13 +375,13 @@ export declare class StagehandCrawler<ContextExtension = Dictionary<never>, Exte
     };
     protected static optionsSchema: z.ZodObject<{
         stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
-        browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
+        headless: z.ZodOptional<z.ZodBoolean>;
         navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
         preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
         launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
-        headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
         browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
+        browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
         remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
         saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
         proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
@@ -437,7 +437,7 @@ export declare class StagehandCrawler<ContextExtension = Dictionary<never>, Exte
      *
      * @param options - Crawler configuration options
      */
-    constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
+    constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
     protected buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext>;
     /**
      * Resolves the {@link StagehandController} that owns the given page, or

package/internals/stagehand-crawler.js

@@ -4,7 +4,7 @@ exports.StagehandCrawler = void 0;
 exports.createStagehandRouter = createStagehandRouter;
 const browser_1 = require("@crawlee/browser");
 const zod_1 = require("zod");
-const stagehand_launcher_1 = require("./stagehand-launcher");
+const stagehand_browser_pool_1 = require("./stagehand-browser-pool");
 const stagehand_utils_1 = require("./utils/stagehand-utils");
 /**
  * StagehandCrawler provides AI-powered web crawling using Browserbase's Stagehand library.
@@ -59,7 +59,7 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
     static optionsShape = {
         ...browser_1.BrowserCrawler.optionsShape,
         stagehandOptions: browser_1.schemas.anyObject.optional(),
-        browserPoolOptions: browser_1.schemas.anyObject.optional(),
+        headless: zod_1.z.boolean().optional(),
     };
     static optionsSchema = zod_1.z.strictObject(StagehandCrawler.optionsShape);
     /**
@@ -69,21 +69,35 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
      */
     constructor(options = {}) {
         const parsedOptions = (0, browser_1.parseArgument)(options, StagehandCrawler.optionsSchema, 'StagehandCrawlerOptions');
-        const { stagehandOptions = {}, launchContext, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
-        const browserPoolOptions = {
-            ...parsedOptions.browserPoolOptions,
-        };
-        // Create launcher with Stagehand plugin
-        const launcher = new stagehand_launcher_1.StagehandLauncher({
-            ...launchContext,
-            stagehandOptions,
-        });
-        browserPoolOptions.browserPlugins = [launcher.createBrowserPlugin()];
-        // Initialize BrowserCrawler with Stagehand plugin and fingerprinting enabled
+        const { stagehandOptions, launchContext, headless, configuration, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
+        if (options.browserPool) {
+            // The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
+            (0, browser_1.assertBrowserPoolNotConfigured)(new.target.name, {
+                launchContext: options.launchContext,
+                stagehandOptions: options.stagehandOptions,
+                headless: options.headless,
+            });
+        }
         super({
             ...browserCrawlerOptions,
             launchContext,
-            browserPoolOptions,
+            configuration,
+            // The pool serves plain Playwright pages - a page only becomes a `StagehandPage` further down the
+            // pipeline, once `setUpStagehand` enhances it - so its page type is narrower than the crawler's.
+            browserPoolBuilder: (remoteBrowser) => (remoteBrowser
+                ? (0, stagehand_browser_pool_1.remoteStagehandBrowserPool)({
+                    ...remoteBrowser,
+                    launchContext,
+                    stagehandOptions,
+                    headless,
+                    configuration,
+                })
+                : (0, stagehand_browser_pool_1.stagehandBrowserPool)({
+                    launchContext,
+                    stagehandOptions,
+                    headless,
+                    configuration,
+                })),
             contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/stagehand",
-    "version": "4.0.0-beta.124",
+    "version": "4.0.0-beta.126",
     "description": "AI-powered web crawling with Stagehand integration for Crawlee - enables natural language browser automation with act(), extract(), and observe() methods.",
     "engines": {
         "node": ">=22.0.0"
@@ -57,11 +57,11 @@
     },
     "dependencies": {
         "@apify/timeout": "^0.4.4",
-        "@crawlee/browser": "4.0.0-beta.124",
-        "@crawlee/browser-pool": "4.0.0-beta.124",
-        "@crawlee/core": "4.0.0-beta.124",
-        "@crawlee/types": "4.0.0-beta.124",
-        "@crawlee/utils": "4.0.0-beta.124",
+        "@crawlee/browser": "4.0.0-beta.126",
+        "@crawlee/browser-pool": "4.0.0-beta.126",
+        "@crawlee/core": "4.0.0-beta.126",
+        "@crawlee/types": "4.0.0-beta.126",
+        "@crawlee/utils": "4.0.0-beta.126",
         "tslib": "^2.8.1",
         "zod": "^4.4.3"
     },
@@ -88,5 +88,5 @@
             }
         }
     },
-    "gitHead": "0694ee1b94c755b98141671baa93cc363f2bf8e3"
+    "gitHead": "5f0a8c1a43d4587f8640881fc55836a23e1c4f30"
 }