@crawlee/stagehand 4.0.0-beta.99 → 4.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.d.ts +4 -3
- package/index.js +5 -3
- package/index.mjs +26 -15
- package/internals/stagehand-browser-pool.d.ts +63 -0
- package/internals/stagehand-browser-pool.js +55 -0
- package/internals/stagehand-controller.d.ts +1 -16
- package/internals/stagehand-controller.js +12 -12
- package/internals/stagehand-crawler.d.ts +140 -128
- package/internals/stagehand-crawler.js +45 -26
- package/internals/stagehand-launcher.d.ts +34 -36
- package/internals/stagehand-launcher.js +22 -13
- package/internals/stagehand-plugin.d.ts +3 -17
- package/internals/stagehand-plugin.js +20 -26
- package/package.json +12 -18
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import type { Action, ActOptions, ActResult, AgentConfig, ExtractOptions, LLMClient, ModelConfiguration, NonStreamingAgentInstance, ObserveOptions, Stagehand, StreamingAgentInstance } from '@browserbasehq/stagehand';
|
|
2
|
-
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook,
|
|
2
|
+
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, GetUserDataFromRequest, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
|
|
3
3
|
import { BrowserCrawler } from '@crawlee/browser';
|
|
4
4
|
import type { Dictionary } from '@crawlee/types';
|
|
5
5
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
|
-
import type {
|
|
7
|
-
import
|
|
6
|
+
import type { Page, Response } from 'playwright';
|
|
7
|
+
import { z } from 'zod';
|
|
8
8
|
import type { StagehandLaunchContext } from './stagehand-launcher';
|
|
9
|
-
import type { StagehandPlugin } from './stagehand-plugin';
|
|
10
9
|
/**
|
|
11
10
|
* Stagehand-specific configuration options.
|
|
12
11
|
*/
|
|
@@ -172,7 +171,7 @@ export interface StagehandPage extends Page {
|
|
|
172
171
|
* Goto options for StagehandCrawler navigation.
|
|
173
172
|
*/
|
|
174
173
|
export type StagehandGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
|
175
|
-
export interface StagehandCrawlingContext<UserData extends Dictionary =
|
|
174
|
+
export interface StagehandCrawlingContext<UserData extends Dictionary = any> extends BrowserCrawlingContext<StagehandPage, Response, UserData, StagehandGotoOptions> {
|
|
176
175
|
/**
|
|
177
176
|
* Enhanced Playwright page with Stagehand AI methods.
|
|
178
177
|
* Use page.act(), page.extract(), page.observe(), page.agent() for AI-powered operations.
|
|
@@ -187,19 +186,11 @@ export interface StagehandCrawlingContext<UserData extends Dictionary = Dictiona
|
|
|
187
186
|
/**
|
|
188
187
|
* Hook function for StagehandCrawler.
|
|
189
188
|
*/
|
|
190
|
-
export
|
|
191
|
-
}
|
|
192
|
-
/**
|
|
193
|
-
* Request handler for StagehandCrawler.
|
|
194
|
-
*/
|
|
195
|
-
export interface StagehandRequestHandler extends RequestHandler<LoadedContext<StagehandCrawlingContext>> {
|
|
196
|
-
}
|
|
189
|
+
export type StagehandHook<UserData extends Dictionary = any> = BrowserHook<StagehandCrawlingContext<UserData>>;
|
|
197
190
|
/**
|
|
198
191
|
* Options for StagehandCrawler.
|
|
199
192
|
*/
|
|
200
|
-
export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']
|
|
201
|
-
browserPlugins: [StagehandPlugin];
|
|
202
|
-
}, Routes> {
|
|
193
|
+
export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawlerOptions<StagehandPage, Response, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
203
194
|
/**
|
|
204
195
|
* Stagehand-specific configuration options.
|
|
205
196
|
* These options configure the AI behavior and Browserbase integration.
|
|
@@ -209,6 +200,11 @@ export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, E
|
|
|
209
200
|
* Launch context with Stagehand-specific options.
|
|
210
201
|
*/
|
|
211
202
|
launchContext?: StagehandLaunchContext;
|
|
203
|
+
/**
|
|
204
|
+
* Whether to run browser in headless mode. Defaults to `true`.
|
|
205
|
+
* Can be also set via {@link Configuration}.
|
|
206
|
+
*/
|
|
207
|
+
headless?: boolean;
|
|
212
208
|
/**
|
|
213
209
|
* Function that is called to process each request.
|
|
214
210
|
*
|
|
@@ -257,11 +253,11 @@ export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, E
|
|
|
257
253
|
/**
|
|
258
254
|
* Async functions that are sequentially evaluated before the navigation.
|
|
259
255
|
*/
|
|
260
|
-
preNavigationHooks?:
|
|
256
|
+
preNavigationHooks?: BrowserHook<StagehandCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
|
|
261
257
|
/**
|
|
262
258
|
* Async functions that are sequentially evaluated after the navigation.
|
|
263
259
|
*/
|
|
264
|
-
postNavigationHooks?:
|
|
260
|
+
postNavigationHooks?: BrowserHook<StagehandCrawlingContext<GetUserDataFromRequest<ExtendedContext['request']>>, ContextExtension>[];
|
|
265
261
|
}
|
|
266
262
|
/**
|
|
267
263
|
* StagehandCrawler provides AI-powered web crawling using Browserbase's Stagehand library.
|
|
@@ -312,127 +308,143 @@ export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, E
|
|
|
312
308
|
* await crawler.run(['https://example.com']);
|
|
313
309
|
* ```
|
|
314
310
|
*/
|
|
315
|
-
export declare class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']
|
|
316
|
-
|
|
317
|
-
|
|
311
|
+
export declare class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>, StatisticStateExtension extends object = {}> extends BrowserCrawler<StagehandPage, Response, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes, StatisticStateExtension> {
|
|
312
|
+
#private;
|
|
313
|
+
/**
|
|
314
|
+
* @internal
|
|
315
|
+
*/
|
|
318
316
|
protected static optionsShape: {
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
350
|
-
|
|
351
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
378
|
-
ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
379
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
380
|
-
blockedStatusCodes: import("ow").ArrayPredicate<number>;
|
|
381
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
382
|
-
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
383
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
384
|
-
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
385
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
386
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
387
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
388
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
389
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
390
|
-
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
391
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
392
|
-
storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
393
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
394
|
-
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
395
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
396
|
-
logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
397
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
398
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
399
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
400
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
401
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
402
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
403
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
404
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
405
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
406
|
-
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
407
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
408
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
317
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
318
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
319
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
320
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
321
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
322
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
323
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
324
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
325
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
326
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
327
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
328
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
329
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
330
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
331
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
332
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
333
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
334
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
335
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
336
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
337
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
338
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
339
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
340
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
341
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
342
|
+
deferred: "deferred";
|
|
343
|
+
writeThrough: "writeThrough";
|
|
344
|
+
}>>;
|
|
345
|
+
}, z.core.$strict>]>>;
|
|
346
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
347
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
348
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
349
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
350
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
|
|
351
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
352
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
353
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
|
|
354
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
355
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
356
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
357
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
358
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
359
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
360
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
361
|
+
id: z.ZodOptional<z.ZodString>;
|
|
362
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
363
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
364
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
365
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
366
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
367
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
368
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
369
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
370
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
371
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
372
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
373
|
+
stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
374
|
+
headless: z.ZodOptional<z.ZodBoolean>;
|
|
409
375
|
};
|
|
376
|
+
/** @internal */
|
|
377
|
+
protected static optionsSchema: z.ZodObject<{
|
|
378
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
379
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
380
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
381
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
382
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
383
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
384
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
385
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
386
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
387
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
388
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
389
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
390
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
391
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
392
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
393
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
394
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
395
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
396
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
397
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
398
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
399
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
400
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
401
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
402
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
403
|
+
deferred: "deferred";
|
|
404
|
+
writeThrough: "writeThrough";
|
|
405
|
+
}>>;
|
|
406
|
+
}, z.core.$strict>]>>;
|
|
407
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
408
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
409
|
+
httpClient: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/http-client").BaseHttpClient>>;
|
|
410
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
411
|
+
configuration: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").Configuration>>;
|
|
412
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
413
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
414
|
+
eventManager: z.ZodOptional<z.ZodInstanceOf<import("@crawlee/browser").EventManager>>;
|
|
415
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
416
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
417
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
418
|
+
initialConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
419
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
420
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
421
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
422
|
+
id: z.ZodOptional<z.ZodString>;
|
|
423
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
424
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
425
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
426
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
427
|
+
browserPoolBuilder: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
428
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
429
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
430
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
431
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
432
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
433
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
434
|
+
stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
435
|
+
headless: z.ZodOptional<z.ZodBoolean>;
|
|
436
|
+
}, z.core.$strict>;
|
|
410
437
|
/**
|
|
411
438
|
* Creates a new instance of StagehandCrawler.
|
|
412
439
|
*
|
|
413
440
|
* @param options - Crawler configuration options
|
|
414
441
|
*/
|
|
415
|
-
constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
|
|
416
|
-
protected buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext>;
|
|
417
|
-
/**
|
|
418
|
-
* Resolves the {@link StagehandController} that owns the given page, or
|
|
419
|
-
* `undefined` when the pool does not expose controllers (e.g. a custom
|
|
420
|
-
* {@link IBrowserPool} implementation).
|
|
421
|
-
*
|
|
422
|
-
* Stagehand needs direct controller access to reach the `Stagehand`
|
|
423
|
-
* instance bound to the page's browser, which is why it reaches past the
|
|
424
|
-
* {@link IBrowserPool} abstraction here.
|
|
425
|
-
*/
|
|
426
|
-
private getBrowserControllerByPage;
|
|
427
|
-
/**
|
|
428
|
-
* Enhance the page with Stagehand AI methods.
|
|
429
|
-
*/
|
|
430
|
-
private setUpStagehand;
|
|
442
|
+
constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
|
|
431
443
|
/**
|
|
432
444
|
* Navigation handler for Stagehand crawler.
|
|
433
445
|
* Uses standard Playwright navigation.
|
|
434
446
|
*/
|
|
435
|
-
protected
|
|
447
|
+
protected navigationHandler(crawlingContext: StagehandCrawlingContext, gotoOptions: StagehandGotoOptions): Promise<Response | null>;
|
|
436
448
|
}
|
|
437
449
|
/**
|
|
438
450
|
* Creates a new router for StagehandCrawler with type-safe route handlers.
|
|
@@ -450,7 +462,7 @@ export declare class StagehandCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
450
462
|
* });
|
|
451
463
|
*
|
|
452
464
|
* router.addDefaultHandler(async ({ page, enqueueLinks }) => {
|
|
453
|
-
* await enqueueLinks({
|
|
465
|
+
* await enqueueLinks({ include: ['https://example.com/products/*'] });
|
|
454
466
|
* });
|
|
455
467
|
*
|
|
456
468
|
* const crawler = new StagehandCrawler({
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.StagehandCrawler = void 0;
|
|
4
4
|
exports.createStagehandRouter = createStagehandRouter;
|
|
5
|
-
const tslib_1 = require("tslib");
|
|
6
5
|
const browser_1 = require("@crawlee/browser");
|
|
7
|
-
const
|
|
8
|
-
const
|
|
6
|
+
const internal_1 = require("@crawlee/utils/internal");
|
|
7
|
+
const zod_1 = require("zod");
|
|
8
|
+
const stagehand_browser_pool_1 = require("./stagehand-browser-pool");
|
|
9
9
|
const stagehand_utils_1 = require("./utils/stagehand-utils");
|
|
10
10
|
/**
|
|
11
11
|
* StagehandCrawler provides AI-powered web crawling using Browserbase's Stagehand library.
|
|
@@ -57,38 +57,57 @@ const stagehand_utils_1 = require("./utils/stagehand-utils");
|
|
|
57
57
|
* ```
|
|
58
58
|
*/
|
|
59
59
|
class StagehandCrawler extends browser_1.BrowserCrawler {
|
|
60
|
+
/**
|
|
61
|
+
* @internal
|
|
62
|
+
*/
|
|
60
63
|
static optionsShape = {
|
|
61
64
|
...browser_1.BrowserCrawler.optionsShape,
|
|
62
|
-
|
|
63
|
-
|
|
65
|
+
launchContext: internal_1.schemas.anyObject.default(() => ({})),
|
|
66
|
+
stagehandOptions: internal_1.schemas.anyObject.optional(),
|
|
67
|
+
headless: zod_1.z.boolean().optional(),
|
|
64
68
|
};
|
|
69
|
+
/** @internal */
|
|
70
|
+
static optionsSchema = zod_1.z.strictObject(StagehandCrawler.optionsShape);
|
|
65
71
|
/**
|
|
66
72
|
* Creates a new instance of StagehandCrawler.
|
|
67
73
|
*
|
|
68
74
|
* @param options - Crawler configuration options
|
|
69
75
|
*/
|
|
70
76
|
constructor(options = {}) {
|
|
71
|
-
(0,
|
|
72
|
-
const { stagehandOptions
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
}
|
|
81
|
-
browserPoolOptions.browserPlugins = [launcher.createBrowserPlugin()];
|
|
82
|
-
// Initialize BrowserCrawler with Stagehand plugin and fingerprinting enabled
|
|
77
|
+
const parsedOptions = (0, internal_1.parseArgument)(options, StagehandCrawler.optionsSchema, 'StagehandCrawlerOptions');
|
|
78
|
+
const { stagehandOptions, launchContext, headless, configuration, ...browserCrawlerOptions } = parsedOptions;
|
|
79
|
+
if (options.browserPool) {
|
|
80
|
+
// The raw options, not the parsed ones: `launchContext` has a default, so by now it is always set.
|
|
81
|
+
(0, internal_1.assertBrowserPoolNotConfigured)(new.target.name, {
|
|
82
|
+
launchContext: options.launchContext,
|
|
83
|
+
stagehandOptions: options.stagehandOptions,
|
|
84
|
+
headless: options.headless,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
83
87
|
super({
|
|
84
88
|
...browserCrawlerOptions,
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
89
|
+
configuration,
|
|
90
|
+
// The pool serves plain Playwright pages - a page only becomes a `StagehandPage` further down the
|
|
91
|
+
// pipeline, once `setUpStagehand` enhances it - so its page type is narrower than the crawler's.
|
|
92
|
+
browserPoolBuilder: (remoteBrowser) => (remoteBrowser
|
|
93
|
+
? (0, stagehand_browser_pool_1.remoteStagehandBrowserPool)({
|
|
94
|
+
...remoteBrowser,
|
|
95
|
+
launchContext,
|
|
96
|
+
stagehandOptions,
|
|
97
|
+
headless,
|
|
98
|
+
configuration,
|
|
99
|
+
})
|
|
100
|
+
: (0, stagehand_browser_pool_1.stagehandBrowserPool)({
|
|
101
|
+
launchContext,
|
|
102
|
+
stagehandOptions,
|
|
103
|
+
headless,
|
|
104
|
+
configuration,
|
|
105
|
+
})),
|
|
106
|
+
contextPipelineBuilder: () => this.#buildContextPipeline(),
|
|
88
107
|
});
|
|
89
108
|
}
|
|
90
|
-
buildContextPipeline() {
|
|
91
|
-
return
|
|
109
|
+
#buildContextPipeline() {
|
|
110
|
+
return this.buildContextPipeline().compose(this.#setUpStagehand.bind(this));
|
|
92
111
|
}
|
|
93
112
|
/**
|
|
94
113
|
* Resolves the {@link StagehandController} that owns the given page, or
|
|
@@ -99,7 +118,7 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
|
|
|
99
118
|
* instance bound to the page's browser, which is why it reaches past the
|
|
100
119
|
* {@link IBrowserPool} abstraction here.
|
|
101
120
|
*/
|
|
102
|
-
getBrowserControllerByPage(page) {
|
|
121
|
+
#getBrowserControllerByPage(page) {
|
|
103
122
|
if ('getBrowserControllerByPage' in this.browserPool) {
|
|
104
123
|
return this.browserPool.getBrowserControllerByPage(page);
|
|
105
124
|
}
|
|
@@ -108,8 +127,8 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
|
|
|
108
127
|
/**
|
|
109
128
|
* Enhance the page with Stagehand AI methods.
|
|
110
129
|
*/
|
|
111
|
-
async setUpStagehand(crawlingContext) {
|
|
112
|
-
const controller = this
|
|
130
|
+
async #setUpStagehand(crawlingContext) {
|
|
131
|
+
const controller = this.#getBrowserControllerByPage(crawlingContext.page);
|
|
113
132
|
if (!controller) {
|
|
114
133
|
throw new Error('Could not resolve StagehandController for page — is the browser pool configured correctly?');
|
|
115
134
|
}
|
|
@@ -123,7 +142,7 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
|
|
|
123
142
|
* Navigation handler for Stagehand crawler.
|
|
124
143
|
* Uses standard Playwright navigation.
|
|
125
144
|
*/
|
|
126
|
-
async
|
|
145
|
+
async navigationHandler(crawlingContext, gotoOptions) {
|
|
127
146
|
// Use standard page.goto for navigation
|
|
128
147
|
return crawlingContext.page.goto(crawlingContext.request.url, gotoOptions);
|
|
129
148
|
}
|
|
@@ -2,6 +2,7 @@ import type { BrowserLaunchContext } from '@crawlee/browser';
|
|
|
2
2
|
import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
3
3
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
4
|
import type { BrowserType, LaunchOptions } from 'playwright';
|
|
5
|
+
import { z } from 'zod';
|
|
5
6
|
import type { StagehandOptions } from './stagehand-crawler';
|
|
6
7
|
import { StagehandPlugin } from './stagehand-plugin';
|
|
7
8
|
/**
|
|
@@ -13,10 +14,6 @@ export interface StagehandLaunchContext extends BrowserLaunchContext<LaunchOptio
|
|
|
13
14
|
* These will be passed to Stagehand's localBrowserLaunchOptions after fingerprinting is applied.
|
|
14
15
|
*/
|
|
15
16
|
launchOptions?: LaunchOptions & Parameters<BrowserType['launchPersistentContext']>[1];
|
|
16
|
-
/**
|
|
17
|
-
* Stagehand-specific configuration for AI operations.
|
|
18
|
-
*/
|
|
19
|
-
stagehandOptions?: StagehandOptions;
|
|
20
17
|
/**
|
|
21
18
|
* URL to a HTTP proxy server. It must define the port number,
|
|
22
19
|
* and it may also contain proxy username and password.
|
|
@@ -31,21 +28,6 @@ export interface StagehandLaunchContext extends BrowserLaunchContext<LaunchOptio
|
|
|
31
28
|
* @default false
|
|
32
29
|
*/
|
|
33
30
|
useChrome?: boolean;
|
|
34
|
-
/**
|
|
35
|
-
* With this option selected, all pages will be opened in a new incognito browser context.
|
|
36
|
-
* @default false
|
|
37
|
-
*/
|
|
38
|
-
useIncognitoPages?: boolean;
|
|
39
|
-
/**
|
|
40
|
-
* Sets the User Data Directory path.
|
|
41
|
-
* The user data directory contains profile data such as history, bookmarks, and cookies.
|
|
42
|
-
*/
|
|
43
|
-
userDataDir?: string;
|
|
44
|
-
/**
|
|
45
|
-
* By default this function uses `require("playwright").chromium`.
|
|
46
|
-
* If you want to use a different browser you can pass it by this property.
|
|
47
|
-
*/
|
|
48
|
-
launcher?: BrowserType;
|
|
49
31
|
}
|
|
50
32
|
/**
|
|
51
33
|
* StagehandLauncher is based on BrowserLauncher and creates StagehandPlugin instances.
|
|
@@ -54,36 +36,52 @@ export interface StagehandLaunchContext extends BrowserLaunchContext<LaunchOptio
|
|
|
54
36
|
* @ignore
|
|
55
37
|
*/
|
|
56
38
|
export declare class StagehandLauncher extends BrowserLauncher<StagehandPlugin> {
|
|
39
|
+
#private;
|
|
57
40
|
readonly configuration: Configuration;
|
|
41
|
+
/**
|
|
42
|
+
* @internal
|
|
43
|
+
*/
|
|
58
44
|
protected static optionsShape: {
|
|
45
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
46
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
47
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
48
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
49
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
50
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
59
51
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
launchContextOptions: import("ow", { with: { "resolution-mode": "import" } }).ObjectPredicate<object> & import("ow", { with: { "resolution-mode": "import" } }).BasePredicate<object | undefined>;
|
|
52
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
53
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
63
54
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
64
|
-
|
|
55
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
65
56
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
66
|
-
|
|
57
|
+
launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
67
58
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
59
|
+
stagehandOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
60
|
+
};
|
|
61
|
+
/** @internal */
|
|
62
|
+
protected static optionsSchema: z.ZodObject<{
|
|
63
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
64
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
65
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
66
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
67
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
68
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
73
69
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
74
|
-
|
|
70
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
71
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
75
72
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
76
|
-
|
|
73
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
77
74
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
78
|
-
|
|
75
|
+
launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
79
76
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
80
|
-
|
|
81
|
-
}
|
|
82
|
-
private readonly stagehandOptions;
|
|
77
|
+
stagehandOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
78
|
+
}, z.core.$strict>;
|
|
83
79
|
/**
|
|
84
80
|
* All StagehandLauncher parameters are passed via the launchContext object.
|
|
85
81
|
*/
|
|
86
|
-
constructor(launchContext?: StagehandLaunchContext
|
|
82
|
+
constructor(launchContext?: StagehandLaunchContext & {
|
|
83
|
+
stagehandOptions?: StagehandOptions;
|
|
84
|
+
}, configuration?: Configuration);
|
|
87
85
|
/**
|
|
88
86
|
* Creates a new StagehandPlugin instance with resolved options.
|
|
89
87
|
*/
|