@crawlee/stagehand 4.0.0-beta.99 → 4.0.0-rc.1

package/README.md

@@ -34,7 +34,7 @@ Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) N
 
 We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
 
-> Crawlee requires **Node.js 16 or higher**.
+> Crawlee requires **Node.js 22.13 or higher**.
 
 ### With Crawlee CLI
 

package/index.d.ts

@@ -51,7 +51,8 @@
  */
 export * from '@crawlee/browser';
 export { StagehandCrawler, createStagehandRouter } from './internals/stagehand-crawler';
-export type { StagehandOptions, StagehandPage, StagehandCrawlingContext, StagehandHook, StagehandRequestHandler, StagehandGotoOptions, StagehandCrawlerOptions, } from './internals/stagehand-crawler';
+export { stagehandBrowserPool, remoteStagehandBrowserPool } from './internals/stagehand-browser-pool';
+export type { StagehandBrowserPool, StagehandBrowserPoolOptions, RemoteStagehandBrowserPoolOptions, } from './internals/stagehand-browser-pool';
+export type { StagehandOptions, StagehandPage, StagehandCrawlingContext, StagehandHook, StagehandGotoOptions, StagehandCrawlerOptions, } from './internals/stagehand-crawler';
 export type { StagehandLaunchContext } from './internals/stagehand-launcher';
-export * as stagehandUtils from './internals/utils/stagehand-utils';
-export type { ActOptions, ActResult, Action, AgentConfig, AgentResult, ExtractOptions, ModelConfiguration, ObserveOptions, Stagehand, } from '@browserbasehq/stagehand';
+export type { ActOptions, ActResult, Action, AgentConfig, ExtractOptions, ModelConfiguration, ObserveOptions, Stagehand, } from '@browserbasehq/stagehand';

package/index.js

@@ -51,7 +51,7 @@
  * @module @crawlee/stagehand
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.stagehandUtils = exports.createStagehandRouter = exports.StagehandCrawler = void 0;
+exports.remoteStagehandBrowserPool = exports.stagehandBrowserPool = exports.createStagehandRouter = exports.StagehandCrawler = void 0;
 const tslib_1 = require("tslib");
 // Re-export everything from @crawlee/browser for convenience
 tslib_1.__exportStar(require("@crawlee/browser"), exports);
@@ -59,5 +59,7 @@ tslib_1.__exportStar(require("@crawlee/browser"), exports);
 var stagehand_crawler_1 = require("./internals/stagehand-crawler");
 Object.defineProperty(exports, "StagehandCrawler", { enumerable: true, get: function () { return stagehand_crawler_1.StagehandCrawler; } });
 Object.defineProperty(exports, "createStagehandRouter", { enumerable: true, get: function () { return stagehand_crawler_1.createStagehandRouter; } });
-// Export utilities as namespace
-exports.stagehandUtils = tslib_1.__importStar(require("./internals/utils/stagehand-utils"));
+// Export the browser pool factories, which are how a pool with non-default options reaches the crawler
+var stagehand_browser_pool_1 = require("./internals/stagehand-browser-pool");
+Object.defineProperty(exports, "stagehandBrowserPool", { enumerable: true, get: function () { return stagehand_browser_pool_1.stagehandBrowserPool; } });
+Object.defineProperty(exports, "remoteStagehandBrowserPool", { enumerable: true, get: function () { return stagehand_browser_pool_1.remoteStagehandBrowserPool; } });

package/index.mjs

@@ -2,14 +2,15 @@ import mod from "./index.js";
 
 export default mod;
 export const API_PROCESSED_REQUESTS_DELAY_MILLIS = mod.API_PROCESSED_REQUESTS_DELAY_MILLIS;
+export const AfterCommitError = mod.AfterCommitError;
 export const ApifyLogAdapter = mod.ApifyLogAdapter;
+export const ArgumentValidationError = mod.ArgumentValidationError;
 export const AutoscaledPool = mod.AutoscaledPool;
 export const BLOCKED_STATUS_CODES = mod.BLOCKED_STATUS_CODES;
 export const BaseCrawleeLogger = mod.BaseCrawleeLogger;
 export const BasicCrawler = mod.BasicCrawler;
 export const BrowserCrawler = mod.BrowserCrawler;
 export const BrowserLauncher = mod.BrowserLauncher;
-export const ClientLoadSignal = mod.ClientLoadSignal;
 export const ConcurrencySystem = mod.ConcurrencySystem;
 export const Configuration = mod.Configuration;
 export const ContextPipeline = mod.ContextPipeline;
@@ -18,13 +19,13 @@ export const ContextPipelineInitializationError = mod.ContextPipelineInitializat
 export const ContextPipelineInterruptedError = mod.ContextPipelineInterruptedError;
 export const CookieParseError = mod.CookieParseError;
 export const CpuLoadSignal = mod.CpuLoadSignal;
+export const CrawlingRequest = mod.CrawlingRequest;
 export const CriticalError = mod.CriticalError;
 export const DATASET_ITERATORS_DEFAULT_LIMIT = mod.DATASET_ITERATORS_DEFAULT_LIMIT;
 export const DEFAULT_CURRENT_HISTORY_SECS = mod.DEFAULT_CURRENT_HISTORY_SECS;
 export const DEFAULT_SNAPSHOT_HISTORY_SECS = mod.DEFAULT_SNAPSHOT_HISTORY_SECS;
 export const Dataset = mod.Dataset;
 export const EnqueueStrategy = mod.EnqueueStrategy;
-export const ErrorSnapshotter = mod.ErrorSnapshotter;
 export const ErrorTracker = mod.ErrorTracker;
 export const EventLoopLoadSignal = mod.EventLoopLoadSignal;
 export const EventManager = mod.EventManager;
@@ -36,7 +37,6 @@ export const LogLevel = mod.LogLevel;
 export const Logger = mod.Logger;
 export const LoggerJson = mod.LoggerJson;
 export const LoggerText = mod.LoggerText;
-export const MAX_POOL_SIZE = mod.MAX_POOL_SIZE;
 export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
 export const MemoryLoadSignal = mod.MemoryLoadSignal;
 export const MemoryStorageBackend = mod.MemoryStorageBackend;
@@ -45,20 +45,19 @@ export const MissingSessionError = mod.MissingSessionError;
 export const NavigationSkippedError = mod.NavigationSkippedError;
 export const NonRetryableError = mod.NonRetryableError;
 export const OwnedOrInjected = mod.OwnedOrInjected;
-export const PERSIST_STATE_KEY = mod.PERSIST_STATE_KEY;
+export const PersistentRateLimitError = mod.PersistentRateLimitError;
 export const ProxyConfiguration = mod.ProxyConfiguration;
-export const PseudoUrl = mod.PseudoUrl;
 export const QUERY_HEAD_BUFFER = mod.QUERY_HEAD_BUFFER;
 export const QUERY_HEAD_MIN_LENGTH = mod.QUERY_HEAD_MIN_LENGTH;
 export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
 export const RecoverableState = mod.RecoverableState;
 export const Request = mod.Request;
 export const RequestHandlerError = mod.RequestHandlerError;
-export const RequestHandlerResult = mod.RequestHandlerResult;
 export const RequestList = mod.RequestList;
 export const RequestManagerTandem = mod.RequestManagerTandem;
 export const RequestQueue = mod.RequestQueue;
 export const RequestState = mod.RequestState;
+export const RequestThrottledError = mod.RequestThrottledError;
 export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
@@ -69,25 +68,30 @@ export const ServiceLocator = mod.ServiceLocator;
 export const Session = mod.Session;
 export const SessionError = mod.SessionError;
 export const SessionPool = mod.SessionPool;
+export const SessionRetiredError = mod.SessionRetiredError;
 export const SitemapRequestLoader = mod.SitemapRequestLoader;
 export const SnapshotStore = mod.SnapshotStore;
 export const Snapshotter = mod.Snapshotter;
 export const StagehandCrawler = mod.StagehandCrawler;
+export const StateValidationError = mod.StateValidationError;
 export const Statistics = mod.Statistics;
+export const StorageBackendLoadSignal = mod.StorageBackendLoadSignal;
 export const StorageInstanceManager = mod.StorageInstanceManager;
-export const StorageStatsTracker = mod.StorageStatsTracker;
+export const StorageTransaction = mod.StorageTransaction;
 export const SystemStatus = mod.SystemStatus;
+export const ThrottlingRequestManager = mod.ThrottlingRequestManager;
+export const activeStorageTransaction = mod.activeStorageTransaction;
 export const applyRequestTransform = mod.applyRequestTransform;
 export const assertJsonSerializable = mod.assertJsonSerializable;
 export const bindMethodsToServiceLocator = mod.bindMethodsToServiceLocator;
-export const browserCrawlerEnqueueLinks = mod.browserCrawlerEnqueueLinks;
 export const browserPoolCookieToToughCookie = mod.browserPoolCookieToToughCookie;
-export const checkStorageAccess = mod.checkStorageAccess;
+export const buildEnqueueStrategyPatterns = mod.buildEnqueueStrategyPatterns;
 export const coerceBoolean = mod.coerceBoolean;
 export const coerceNumber = mod.coerceNumber;
 export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
-export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFromPseudoUrls;
 export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
+export const constructUrlPatternObjects = mod.constructUrlPatternObjects;
+export const convertStateSync = mod.convertStateSync;
 export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
 export const crawleeConfigFields = mod.crawleeConfigFields;
 export const createBasicRouter = mod.createBasicRouter;
@@ -95,10 +99,12 @@ export const createDeserialize = mod.createDeserialize;
 export const createDualIterable = mod.createDualIterable;
 export const createRequestDebugInfo = mod.createRequestDebugInfo;
 export const createRequestOptions = mod.createRequestOptions;
+export const createSkippedRequestArgs = mod.createSkippedRequestArgs;
 export const createStagehandRouter = mod.createStagehandRouter;
+export const createStorageTransaction = mod.createStorageTransaction;
+export const currentStorageTransaction = mod.currentStorageTransaction;
 export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
-export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
 export const extractUrlsFromPage = mod.extractUrlsFromPage;
 export const field = mod.field;
@@ -111,20 +117,25 @@ export const inspectValue = mod.inspectValue;
 export const log = mod.log;
 export const mergeCookies = mod.mergeCookies;
 export const navigationDeadlineKey = mod.navigationDeadlineKey;
+export const operationRejectedInTransaction = mod.operationRejectedInTransaction;
+export const parseRetryAfterHeader = mod.parseRetryAfterHeader;
 export const parseValue = mod.parseValue;
 export const purgeDefaultStorages = mod.purgeDefaultStorages;
+export const rejectOperationInTransaction = mod.rejectOperationInTransaction;
 export const remainingNavigationWindowMillis = mod.remainingNavigationWindowMillis;
+export const remoteStagehandBrowserPool = mod.remoteStagehandBrowserPool;
 export const resolveBaseUrlForEnqueueLinksFiltering = mod.resolveBaseUrlForEnqueueLinksFiltering;
-export const resolveStorageIdentifier = mod.resolveStorageIdentifier;
 export const serializeArray = mod.serializeArray;
 export const serializeValue = mod.serializeValue;
 export const serviceLocator = mod.serviceLocator;
-export const stagehandUtils = mod.stagehandUtils;
+export const snapshotValue = mod.snapshotValue;
+export const stagehandBrowserPool = mod.stagehandBrowserPool;
 export const toughCookieToBrowserPoolCookie = mod.toughCookieToBrowserPoolCookie;
-export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
+export const urlPatternSchema = mod.urlPatternSchema;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
 export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
-export const withCheckedStorageAccess = mod.withCheckedStorageAccess;
+export const withDirectStorageAccess = mod.withDirectStorageAccess;
+export const withStorageTransaction = mod.withStorageTransaction;

package/internals/stagehand-browser-pool.d.ts

@@ -0,0 +1,63 @@
+import type { Configuration } from '@crawlee/browser';
+import type { BrowserPool, BrowserPoolHooks, BrowserPoolOptions, RemoteBrowserPool, RemoteBrowserPoolOptions } from '@crawlee/browser-pool';
+// @ts-ignore optional peer dependency or compatibility with es2022
+import type { Page } from 'playwright';
+import type { StagehandOptions } from './stagehand-crawler';
+import type { StagehandLaunchContext } from './stagehand-launcher';
+import type { StagehandPlugin } from './stagehand-plugin';
+/** A {@link BrowserPool} of Stagehand browsers, as built by {@link stagehandBrowserPool}. */
+export type StagehandBrowserPool = BrowserPool<{
+    browserPlugins: [StagehandPlugin];
+}, [StagehandPlugin]>;
+export interface StagehandBrowserPoolOptions extends Omit<BrowserPoolOptions, 'browserPlugins'>, BrowserPoolHooks<ReturnType<StagehandPlugin['createController']>, ReturnType<StagehandPlugin['createLaunchContext']>, Page> {
+    /** How to launch the browser: which Playwright browser type, proxy, user data dir, ... */
+    launchContext?: StagehandLaunchContext;
+    /** Stagehand configuration for the AI behavior and Browserbase integration. */
+    stagehandOptions?: StagehandOptions;
+    /**
+     * Whether to run the browser in headless mode. Shorthand for `launchContext.launchOptions.headless`.
+     * Defaults to `true`, and can also be set via {@link Configuration}.
+     */
+    headless?: boolean;
+    /** Configuration to read the browser defaults from. Defaults to the global configuration. */
+    configuration?: Configuration;
+}
+export interface RemoteStagehandBrowserPoolOptions extends Pick<StagehandBrowserPoolOptions, 'launchContext' | 'stagehandOptions' | 'headless' | 'configuration'>, Omit<RemoteBrowserPoolOptions, 'browserPlugins'> {
+}
+/**
+ * Builds a {@link BrowserPool} of Stagehand browsers to pass to a {@link StagehandCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext`,
+ * `stagehandOptions` and `headless`, and derives the browser plugin from them - so a pool built here always
+ * matches the crawler it is given to, and configuring one never means assembling a {@link StagehandPlugin}
+ * by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new StagehandCrawler({
+ *     browserPool: stagehandBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         stagehandOptions: { env: 'LOCAL', model: 'openai/gpt-4.1-mini' },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function stagehandBrowserPool(options?: StagehandBrowserPoolOptions): StagehandBrowserPool;
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link stagehandBrowserPool}: connects to a remote browser
+ * service with a Stagehand plugin derived from `launchContext` and `stagehandOptions`.
+ *
+ * A {@link StagehandCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * @category Browser management
+ */
+export declare function remoteStagehandBrowserPool(options: RemoteStagehandBrowserPoolOptions): RemoteBrowserPool<Page>;

package/internals/stagehand-browser-pool.js

@@ -0,0 +1,55 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.stagehandBrowserPool = stagehandBrowserPool;
+exports.remoteStagehandBrowserPool = remoteStagehandBrowserPool;
+const stagehand_launcher_1 = require("./stagehand-launcher");
+/**
+ * Builds a {@link BrowserPool} of Stagehand browsers to pass to a {@link StagehandCrawler} as its
+ * {@link BrowserCrawlerOptions.browserPool|`browserPool`}.
+ *
+ * It accepts every {@link BrowserPoolOptions|`BrowserPool` option} plus the crawler's own `launchContext`,
+ * `stagehandOptions` and `headless`, and derives the browser plugin from them - so a pool built here always
+ * matches the crawler it is given to, and configuring one never means assembling a {@link StagehandPlugin}
+ * by hand.
+ *
+ * **Example usage:**
+ *
+ * ```javascript
+ * const crawler = new StagehandCrawler({
+ *     browserPool: stagehandBrowserPool({
+ *         maxOpenPagesPerBrowser: 1,
+ *         stagehandOptions: { env: 'LOCAL', model: 'openai/gpt-4.1-mini' },
+ *     }),
+ *     requestHandler: async ({ page }) => { ... },
+ * });
+ * ```
+ *
+ * The returned pool is *not* torn down by the crawler, which is what makes it shareable between crawlers.
+ *
+ * @category Browser management
+ */
+function stagehandBrowserPool(options = {}) {
+    const { launchContext, stagehandOptions, headless, configuration, ...poolOptions } = options;
+    return stagehandLauncher(launchContext, stagehandOptions, headless, configuration).createBrowserPool(poolOptions);
+}
+/**
+ * The {@link RemoteBrowserPool} counterpart of {@link stagehandBrowserPool}: connects to a remote browser
+ * service with a Stagehand plugin derived from `launchContext` and `stagehandOptions`.
+ *
+ * A {@link StagehandCrawler} accepts the same connection details directly via
+ * {@link BrowserCrawlerOptions.remoteBrowser|`remoteBrowser`}; reach for this factory when you also need to
+ * tune the wrapping pool, or to share one remote pool between crawlers.
+ *
+ * @category Browser management
+ */
+function remoteStagehandBrowserPool(options) {
+    const { launchContext, stagehandOptions, headless, configuration, ...remoteOptions } = options;
+    return stagehandLauncher(launchContext, stagehandOptions, headless, configuration).createRemoteBrowserPool(remoteOptions);
+}
+function stagehandLauncher(launchContext = {}, stagehandOptions = {}, headless, configuration) {
+    return new stagehand_launcher_1.StagehandLauncher({
+        ...launchContext,
+        stagehandOptions,
+        ...(headless == null ? {} : { launchOptions: { ...launchContext.launchOptions, headless } }),
+    }, configuration);
+}

package/internals/stagehand-controller.d.ts

@@ -18,8 +18,7 @@ import type { StagehandPlugin } from './stagehand-plugin';
  * @ignore
  */
 export declare class StagehandController extends BrowserController<BrowserType, LaunchOptions, PlaywrightBrowser> {
-    private stagehand;
-    private readonly stagehandInstances;
+    #private;
     constructor(browserPlugin: StagehandPlugin, stagehandInstances: WeakMap<PlaywrightBrowser, Stagehand>);
     /**
      * Gets the Stagehand instance associated with this controller's browser.
@@ -30,20 +29,6 @@ export declare class StagehandController extends BrowserController<BrowserType,
      * We use Playwright's browser API directly since we connected via CDP.
      */
     protected _newPage(_contextOptions?: unknown): Promise<Page>;
-    /**
-     * Waits until Stagehand knows about the page, so that `act()`/`extract()`/`observe()` can resolve it.
-     *
-     * Stagehand only learns about pages from CDP `Target.attachedToTarget` events on its own connection,
-     * and it maps them by main frame id. We create pages through a separate `connectOverCDP()` handle, so
-     * `context.newPage()` resolves before Stagehand has processed that event — the page is unresolvable for
-     * a short window, and the AI methods fail with 'Failed to resolve V3 Page from Playwright page'.
-     * Stagehand's own `newPage()` polls for the same reason.
-     */
-    private waitForStagehandToRegisterPage;
-    /**
-     * Reads the page's main frame id, which is the key Stagehand resolves pages by.
-     */
-    private getMainFrameId;
     /**
      * Normalizes proxy options for Playwright.
      */

package/internals/stagehand-controller.js

@@ -18,23 +18,23 @@ const utils_1 = require("@crawlee/utils");
  * @ignore
  */
 class StagehandController extends browser_pool_1.BrowserController {
-    stagehand = null;
-    stagehandInstances;
+    #stagehand = null;
+    #stagehandInstances;
     constructor(browserPlugin, stagehandInstances) {
         super(browserPlugin);
-        this.stagehandInstances = stagehandInstances;
+        this.#stagehandInstances = stagehandInstances;
     }
     /**
      * Gets the Stagehand instance associated with this controller's browser.
      */
     getStagehand() {
-        if (!this.stagehand) {
-            this.stagehand = this.stagehandInstances.get(this.browser);
-            if (!this.stagehand) {
+        if (!this.#stagehand) {
+            this.#stagehand = this.#stagehandInstances.get(this.browser);
+            if (!this.#stagehand) {
                 throw new Error('Stagehand instance not found for browser');
             }
         }
-        return this.stagehand;
+        return this.#stagehand;
     }
     /**
      * Creates a new page using the browser's default context.
@@ -51,10 +51,10 @@ class StagehandController extends browser_pool_1.BrowserController {
             const page = await context.newPage();
             // Track active pages
             page.once('close', () => {
-                this.activePages--;
+                this.registerPageClosed(page);
             });
             try {
-                await this.waitForStagehandToRegisterPage(page);
+                await this.#waitForStagehandToRegisterPage(page);
             }
             catch (error) {
                 await page.close().catch(() => { });
@@ -77,9 +77,9 @@ class StagehandController extends browser_pool_1.BrowserController {
      * a short window, and the AI methods fail with 'Failed to resolve V3 Page from Playwright page'.
      * Stagehand's own `newPage()` polls for the same reason.
      */
-    async waitForStagehandToRegisterPage(page, timeoutMs = 10_000) {
+    async #waitForStagehandToRegisterPage(page, timeoutMs = 10_000) {
         const stagehand = this.getStagehand();
-        const mainFrameId = await this.getMainFrameId(page);
+        const mainFrameId = await this.#getMainFrameId(page);
         const deadline = Date.now() + timeoutMs;
         while (Date.now() < deadline) {
             if (stagehand.context.resolvePageByMainFrameId(mainFrameId)) {
@@ -94,7 +94,7 @@ class StagehandController extends browser_pool_1.BrowserController {
     /**
      * Reads the page's main frame id, which is the key Stagehand resolves pages by.
      */
-    async getMainFrameId(page) {
+    async #getMainFrameId(page) {
         const cdpSession = await page.context().newCDPSession(page);
         try {
             const { frameTree } = await cdpSession.send('Page.getFrameTree');