@crawlee/stagehand 3.17.1-beta.9 → 3.18.0

package/index.mjs

@@ -50,6 +50,7 @@ export const RequestQueue = mod.RequestQueue;
 export const RequestQueueV1 = mod.RequestQueueV1;
 export const RequestQueueV2 = mod.RequestQueueV2;
 export const RequestState = mod.RequestState;
+export const RequestValidationError = mod.RequestValidationError;
 export const RetryRequestError = mod.RetryRequestError;
 export const Router = mod.Router;
 export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
@@ -81,6 +82,7 @@ export const createEventLoopLoadSignal = mod.createEventLoopLoadSignal;
 export const createRequestOptions = mod.createRequestOptions;
 export const createRequests = mod.createRequests;
 export const createStagehandRouter = mod.createStagehandRouter;
+export const defaultRoute = mod.defaultRoute;
 export const deserializeArray = mod.deserializeArray;
 export const enqueueLinks = mod.enqueueLinks;
 export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
@@ -103,5 +105,6 @@ export const tryAbsoluteURL = mod.tryAbsoluteURL;
 export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
 export const useState = mod.useState;
 export const validateGlobPattern = mod.validateGlobPattern;
+export const validateUserData = mod.validateUserData;
 export const validators = mod.validators;
 export const withCheckedStorageAccess = mod.withCheckedStorageAccess;

package/internals/stagehand-controller.d.ts

@@ -30,6 +30,20 @@ export declare class StagehandController extends BrowserController<BrowserType,
      * We use Playwright's browser API directly since we connected via CDP.
      */
     protected _newPage(_contextOptions?: unknown): Promise<Page>;
+    /**
+     * Waits until Stagehand knows about the page, so that `act()`/`extract()`/`observe()` can resolve it.
+     *
+     * Stagehand only learns about pages from CDP `Target.attachedToTarget` events on its own connection,
+     * and it maps them by main frame id. We create pages through a separate `connectOverCDP()` handle, so
+     * `context.newPage()` resolves before Stagehand has processed that event — the page is unresolvable for
+     * a short window, and the AI methods fail with 'Failed to resolve V3 Page from Playwright page'.
+     * Stagehand's own `newPage()` polls for the same reason.
+     */
+    private waitForStagehandToRegisterPage;
+    /**
+     * Reads the page's main frame id, which is the key Stagehand resolves pages by.
+     */
+    private getMainFrameId;
     /**
      * Normalizes proxy options for Playwright.
      */

package/internals/stagehand-controller.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.StagehandController = void 0;
 const tslib_1 = require("tslib");
 const browser_pool_1 = require("@crawlee/browser-pool");
+const utils_1 = require("@crawlee/utils");
 const log_1 = tslib_1.__importDefault(require("@apify/log"));
 /**
  * StagehandController manages the lifecycle of a Stagehand-controlled browser for Crawlee's BrowserPool.
@@ -63,6 +64,13 @@ class StagehandController extends browser_pool_1.BrowserController {
             page.once('close', () => {
                 this.activePages--;
             });
+            try {
+                await this.waitForStagehandToRegisterPage(page);
+            }
+            catch (error) {
+                await page.close().catch(() => { });
+                throw error;
+            }
             return page;
         }
         catch (error) {
@@ -71,6 +79,42 @@ class StagehandController extends browser_pool_1.BrowserController {
             });
         }
     }
+    /**
+     * Waits until Stagehand knows about the page, so that `act()`/`extract()`/`observe()` can resolve it.
+     *
+     * Stagehand only learns about pages from CDP `Target.attachedToTarget` events on its own connection,
+     * and it maps them by main frame id. We create pages through a separate `connectOverCDP()` handle, so
+     * `context.newPage()` resolves before Stagehand has processed that event — the page is unresolvable for
+     * a short window, and the AI methods fail with 'Failed to resolve V3 Page from Playwright page'.
+     * Stagehand's own `newPage()` polls for the same reason.
+     */
+    async waitForStagehandToRegisterPage(page, timeoutMs = 10000) {
+        const stagehand = this.getStagehand();
+        const mainFrameId = await this.getMainFrameId(page);
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            if (stagehand.context.resolvePageByMainFrameId(mainFrameId)) {
+                return;
+            }
+            await (0, utils_1.sleep)(25);
+        }
+        // Stagehand skips registration entirely when it cannot install its helper script into the page's CDP
+        // session, so this is not always just a slow attach - it can never resolve.
+        throw new Error(`Stagehand did not register the page within ${timeoutMs}ms`);
+    }
+    /**
+     * Reads the page's main frame id, which is the key Stagehand resolves pages by.
+     */
+    async getMainFrameId(page) {
+        const cdpSession = await page.context().newCDPSession(page);
+        try {
+            const { frameTree } = await cdpSession.send('Page.getFrameTree');
+            return frameTree.frame.id;
+        }
+        finally {
+            await cdpSession.detach().catch(() => { });
+        }
+    }
     /**
      * Normalizes proxy options for Playwright.
      */

package/internals/stagehand-crawler.d.ts

@@ -1,5 +1,5 @@
 import type { Action, ActOptions, ActResult, AgentConfig, ExtractOptions, LLMClient, ModelConfiguration, NonStreamingAgentInstance, ObserveOptions, Stagehand, StreamingAgentInstance } from '@browserbasehq/stagehand';
-import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, BrowserRequestHandler, GetUserDataFromRequest, LoadedContext, RouterRoutes } from '@crawlee/browser';
+import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, BrowserRequestHandler, GetUserDataFromRequest, LoadedContext, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
 import { BrowserCrawler, Configuration } from '@crawlee/browser';
 import type { Dictionary } from '@crawlee/types';
 // @ts-ignore optional peer dependency or compatibility with es2022
@@ -408,6 +408,9 @@ export declare class StagehandCrawler extends BrowserCrawler<{
 // @ts-ignore optional peer dependency or compatibility with es2022
         statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
     };
+    /** Set while `_runRequestHandler` has swapped `userProvidedRequestHandler` for its page-enhancing wrapper. */
+    private unwrappedRequestHandler?;
+    protected get userRequestHandler(): BrowserRequestHandler<StagehandCrawlingContext>;
     /**
      * Creates a new instance of StagehandCrawler.
      *
@@ -458,5 +461,6 @@ export declare class StagehandCrawler extends BrowserCrawler<{
  * });
  * ```
  */
-// @ts-ignore optional peer dependency or compatibility with es2022
-export declare function createStagehandRouter<Context extends StagehandCrawlingContext = StagehandCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): import("@crawlee/browser").RouterHandler<Context>;
+export declare function createStagehandRouter<Context extends StagehandCrawlingContext = StagehandCrawlingContext, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<Context['request']>>>(routes?: RouterRoutes<Context, Routes>): RouterHandler<Context, Routes>;
+export declare function createStagehandRouter<Context extends StagehandCrawlingContext = StagehandCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, Record<string, UserData>>): RouterHandler<Context, Record<string, UserData>>;
+export declare function createStagehandRouter<Context extends StagehandCrawlingContext = StagehandCrawlingContext, const Schemas extends RouteSchemas = RouteSchemas>(schemas: Schemas): RouterHandler<Context, RoutesFromSchemas<Schemas>>;

package/internals/stagehand-crawler.js

@@ -57,6 +57,9 @@ const stagehand_utils_1 = require("./utils/stagehand-utils");
  * ```
  */
 class StagehandCrawler extends browser_1.BrowserCrawler {
+    get userRequestHandler() {
+        return this.unwrappedRequestHandler ?? super.userRequestHandler;
+    }
     /**
      * Creates a new instance of StagehandCrawler.
      *
@@ -88,6 +91,13 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
             writable: true,
             value: config
         });
+        /** Set while `_runRequestHandler` has swapped `userProvidedRequestHandler` for its page-enhancing wrapper. */
+        Object.defineProperty(this, "unwrappedRequestHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
     }
     /**
      * Overrides the request handler to enhance the page with Stagehand AI methods.
@@ -105,6 +115,9 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
     async _runRequestHandler(crawlingContext) {
         // Store the original handler (could be this.requestHandler or this.router)
         const originalHandler = this.userProvidedRequestHandler;
+        // Keep `userRequestHandler` pointing at the unwrapped handler: the swap below is live while the user's
+        // handler runs, which is exactly when `context.addRequests`/`enqueueLinks` resolve router metadata.
+        this.unwrappedRequestHandler = originalHandler;
         // Replace with a wrapper that enhances the page before calling the user's handler
         this.userProvidedRequestHandler = async (ctx) => {
             // Get Stagehand instance from controller
@@ -122,6 +135,7 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
         finally {
             // Restore original handler
             this.userProvidedRequestHandler = originalHandler;
+            this.unwrappedRequestHandler = undefined;
         }
     }
     /**
@@ -144,30 +158,6 @@ Object.defineProperty(StagehandCrawler, "optionsShape", {
         browserPoolOptions: ow_1.default.optional.object,
     }
 });
-/**
- * Creates a new router for StagehandCrawler with type-safe route handlers.
- *
- * @param options - Router options
- * @returns Configured router instance
- *
- * @example
- * ```typescript
- * const router = createStagehandRouter();
- *
- * router.addHandler('product', async ({ page, request, log }) => {
- *   log.info(`Processing product: ${request.url}`);
- *   const data = await page.extract('Get product info', schema);
- * });
- *
- * router.addDefaultHandler(async ({ page, enqueueLinks }) => {
- *   await enqueueLinks({ globs: ['https://example.com/products/*'] });
- * });
- *
- * const crawler = new StagehandCrawler({
- *   requestHandler: router,
- * });
- * ```
- */
-function createStagehandRouter(routes) {
-    return browser_1.Router.create(routes);
+function createStagehandRouter(routesOrSchemas) {
+    return browser_1.Router.create(routesOrSchemas);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@crawlee/stagehand",
-    "version": "3.17.1-beta.9",
+    "version": "3.18.0",
     "description": "AI-powered web crawling with Stagehand integration for Crawlee - enables natural language browser automation with act(), extract(), and observe() methods.",
     "engines": {
         "node": ">=16.0.0"
@@ -57,12 +57,12 @@
     },
     "dependencies": {
         "@apify/log": "^2.4.0",
-        "@apify/timeout": "^0.3.1",
-        "@crawlee/browser": "3.17.1-beta.9",
-        "@crawlee/browser-pool": "3.17.1-beta.9",
-        "@crawlee/core": "3.17.1-beta.9",
-        "@crawlee/types": "3.17.1-beta.9",
-        "@crawlee/utils": "3.17.1-beta.9",
+        "@apify/timeout": "^0.4.0",
+        "@crawlee/browser": "3.18.0",
+        "@crawlee/browser-pool": "3.18.0",
+        "@crawlee/core": "3.18.0",
+        "@crawlee/types": "3.18.0",
+        "@crawlee/utils": "3.18.0",
         "ow": "^0.28.1",
         "tslib": "^2.4.0"
     },
@@ -94,5 +94,5 @@
             }
         }
     },
-    "gitHead": "c547ce4489d8617620a51516cc885187503f3136"
+    "gitHead": "49c115e1ce3b3fbf2bef61f16bffeadfdcf5bd19"
 }