@crawlee/stagehand 4.0.0-beta.121 → 4.0.0-beta.122
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.mjs +4 -0
- package/internals/stagehand-crawler.d.ts +117 -95
- package/internals/stagehand-crawler.js +7 -7
- package/internals/stagehand-launcher.d.ts +26 -15
- package/internals/stagehand-launcher.js +10 -9
- package/package.json +11 -16
package/index.mjs
CHANGED
|
@@ -3,6 +3,7 @@ import mod from "./index.js";
|
|
|
3
3
|
export default mod;
|
|
4
4
|
export const API_PROCESSED_REQUESTS_DELAY_MILLIS = mod.API_PROCESSED_REQUESTS_DELAY_MILLIS;
|
|
5
5
|
export const ApifyLogAdapter = mod.ApifyLogAdapter;
|
|
6
|
+
export const ArgumentValidationError = mod.ArgumentValidationError;
|
|
6
7
|
export const AutoscaledPool = mod.AutoscaledPool;
|
|
7
8
|
export const BLOCKED_STATUS_CODES = mod.BLOCKED_STATUS_CODES;
|
|
8
9
|
export const BaseCrawleeLogger = mod.BaseCrawleeLogger;
|
|
@@ -117,6 +118,7 @@ export const log = mod.log;
|
|
|
117
118
|
export const mergeCookies = mod.mergeCookies;
|
|
118
119
|
export const navigationDeadlineKey = mod.navigationDeadlineKey;
|
|
119
120
|
export const operationRejectedInTransaction = mod.operationRejectedInTransaction;
|
|
121
|
+
export const parseArgument = mod.parseArgument;
|
|
120
122
|
export const parseRetryAfterHeader = mod.parseRetryAfterHeader;
|
|
121
123
|
export const parseValue = mod.parseValue;
|
|
122
124
|
export const purgeDefaultStorages = mod.purgeDefaultStorages;
|
|
@@ -124,6 +126,7 @@ export const rejectOperationInTransaction = mod.rejectOperationInTransaction;
|
|
|
124
126
|
export const remainingNavigationWindowMillis = mod.remainingNavigationWindowMillis;
|
|
125
127
|
export const resolveBaseUrlForEnqueueLinksFiltering = mod.resolveBaseUrlForEnqueueLinksFiltering;
|
|
126
128
|
export const resolveStorageIdentifier = mod.resolveStorageIdentifier;
|
|
129
|
+
export const schemas = mod.schemas;
|
|
127
130
|
export const serializeArray = mod.serializeArray;
|
|
128
131
|
export const serializeValue = mod.serializeValue;
|
|
129
132
|
export const serviceLocator = mod.serviceLocator;
|
|
@@ -133,6 +136,7 @@ export const supportsDomainThrottling = mod.supportsDomainThrottling;
|
|
|
133
136
|
export const toughCookieToBrowserPoolCookie = mod.toughCookieToBrowserPoolCookie;
|
|
134
137
|
export const tryAbsoluteURL = mod.tryAbsoluteURL;
|
|
135
138
|
export const updateEnqueueLinksPatternCache = mod.updateEnqueueLinksPatternCache;
|
|
139
|
+
export const urlPatternSchema = mod.urlPatternSchema;
|
|
136
140
|
export const useState = mod.useState;
|
|
137
141
|
export const validateGlobPattern = mod.validateGlobPattern;
|
|
138
142
|
export const validateUserData = mod.validateUserData;
|
|
@@ -4,7 +4,7 @@ import { BrowserCrawler } from '@crawlee/browser';
|
|
|
4
4
|
import type { Dictionary } from '@crawlee/types';
|
|
5
5
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
6
|
import type { LaunchOptions, Page, Response } from 'playwright';
|
|
7
|
-
import
|
|
7
|
+
import { z } from 'zod';
|
|
8
8
|
import type { StagehandLaunchContext } from './stagehand-launcher';
|
|
9
9
|
import type { StagehandPlugin } from './stagehand-plugin';
|
|
10
10
|
/**
|
|
@@ -315,101 +315,123 @@ export declare class StagehandCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
315
315
|
browserPlugins: [StagehandPlugin];
|
|
316
316
|
}, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes> {
|
|
317
317
|
protected static optionsShape: {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
376
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
377
|
-
additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
378
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
379
|
-
ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
380
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
381
|
-
blockedStatusCodes: import("ow").ArrayPredicate<number>;
|
|
382
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
383
|
-
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
384
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
385
|
-
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
386
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
387
|
-
transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/browser").StorageWritePolicy> | undefined>;
|
|
388
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
389
|
-
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
390
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
391
|
-
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
392
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
393
|
-
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
394
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
395
|
-
storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
396
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
397
|
-
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
398
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
399
|
-
logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
400
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
401
|
-
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
402
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
403
|
-
maxConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
404
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
405
|
-
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
406
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
407
|
-
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
408
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
409
|
-
statistics: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
410
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
411
|
-
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
318
|
+
stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
319
|
+
browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
320
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
321
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
322
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
323
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
324
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
325
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
326
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
327
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
328
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
329
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
330
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
331
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
332
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
333
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
334
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
335
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
336
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
337
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
338
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
339
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
340
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
341
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
342
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
343
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
344
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
345
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
346
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
347
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
348
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
349
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
350
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
351
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
352
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
353
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
354
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
355
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
356
|
+
deferred: "deferred";
|
|
357
|
+
writeThrough: "writeThrough";
|
|
358
|
+
}>>;
|
|
359
|
+
}, z.core.$strict>]>>;
|
|
360
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
361
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
362
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
363
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
364
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
|
|
365
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
366
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
367
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
|
|
368
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
369
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
370
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
371
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
372
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
373
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
374
|
+
id: z.ZodOptional<z.ZodString>;
|
|
412
375
|
};
|
|
376
|
+
protected static optionsSchema: z.ZodObject<{
|
|
377
|
+
stagehandOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
378
|
+
browserPoolOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
379
|
+
navigationTimeoutSecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
380
|
+
preNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
381
|
+
postNavigationHooks: z.ZodDefault<z.ZodCustom<unknown[], unknown[]>>;
|
|
382
|
+
launchContext: z.ZodDefault<z.ZodCustom<Dictionary, Dictionary>>;
|
|
383
|
+
headless: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodString]>>;
|
|
384
|
+
browserPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
385
|
+
remoteBrowser: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
386
|
+
saveResponseCookies: z.ZodDefault<z.ZodBoolean>;
|
|
387
|
+
proxyConfiguration: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
388
|
+
ignoreIframes: z.ZodDefault<z.ZodBoolean>;
|
|
389
|
+
ignoreShadowRoots: z.ZodDefault<z.ZodBoolean>;
|
|
390
|
+
contextPipelineBuilder: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
391
|
+
extendContext: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
392
|
+
requestList: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
393
|
+
requestQueue: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
394
|
+
requestManager: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
395
|
+
requestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
396
|
+
requestHandlerTimeoutSecs: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
397
|
+
errorHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
398
|
+
failedRequestHandler: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
399
|
+
maxRequestRetries: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
400
|
+
sameDomainDelaySecs: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
401
|
+
maxRequestsPerCrawl: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
402
|
+
maxCrawlDepth: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
403
|
+
taskLoopOptions: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
404
|
+
concurrencySystem: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
405
|
+
sessionPool: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
406
|
+
statusMessageLoggingInterval: z.ZodDefault<z.ZodCustom<number, number>>;
|
|
407
|
+
statusMessageCallback: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
408
|
+
additionalHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
409
|
+
ignoreHttpErrorStatusCodes: z.ZodDefault<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
410
|
+
blockedStatusCodes: z.ZodOptional<z.ZodArray<z.ZodCustom<number, number>>>;
|
|
411
|
+
retryOnBlocked: z.ZodDefault<z.ZodBoolean>;
|
|
412
|
+
respectRobotsTxtFile: z.ZodDefault<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<Dictionary, Dictionary>]>>;
|
|
413
|
+
transactionalStorage: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodObject<{
|
|
414
|
+
requestQueue: z.ZodOptional<z.ZodEnum<{
|
|
415
|
+
deferred: "deferred";
|
|
416
|
+
writeThrough: "writeThrough";
|
|
417
|
+
}>>;
|
|
418
|
+
}, z.core.$strict>]>>;
|
|
419
|
+
onSkippedRequest: z.ZodOptional<z.ZodCustom<(...args: any[]) => unknown, (...args: any[]) => unknown>>;
|
|
420
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
421
|
+
httpClient: z.ZodOptional<z.ZodCustom<import("@crawlee/http-client").BaseHttpClient, import("@crawlee/http-client").BaseHttpClient>>;
|
|
422
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
423
|
+
configuration: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").Configuration, import("@crawlee/browser").Configuration>>;
|
|
424
|
+
storageBackend: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
425
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
426
|
+
eventManager: z.ZodOptional<z.ZodCustom<import("@crawlee/browser").EventManager, import("@crawlee/browser").EventManager>>;
|
|
427
|
+
logger: z.ZodOptional<z.ZodType<Dictionary<any>, unknown, z.core.$ZodTypeInternals<Dictionary<any>, unknown>>>;
|
|
428
|
+
minConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
429
|
+
maxConcurrency: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
430
|
+
maxRequestsPerMinute: z.ZodOptional<z.ZodCustom<number, number>>;
|
|
431
|
+
keepAlive: z.ZodOptional<z.ZodBoolean>;
|
|
432
|
+
statistics: z.ZodOptional<z.ZodCustom<Dictionary, Dictionary>>;
|
|
433
|
+
id: z.ZodOptional<z.ZodString>;
|
|
434
|
+
}, z.core.$strict>;
|
|
413
435
|
/**
|
|
414
436
|
* Creates a new instance of StagehandCrawler.
|
|
415
437
|
*
|
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.StagehandCrawler = void 0;
|
|
4
4
|
exports.createStagehandRouter = createStagehandRouter;
|
|
5
|
-
const tslib_1 = require("tslib");
|
|
6
5
|
const browser_1 = require("@crawlee/browser");
|
|
7
|
-
const
|
|
6
|
+
const zod_1 = require("zod");
|
|
8
7
|
const stagehand_launcher_1 = require("./stagehand-launcher");
|
|
9
8
|
const stagehand_utils_1 = require("./utils/stagehand-utils");
|
|
10
9
|
/**
|
|
@@ -59,19 +58,20 @@ const stagehand_utils_1 = require("./utils/stagehand-utils");
|
|
|
59
58
|
class StagehandCrawler extends browser_1.BrowserCrawler {
|
|
60
59
|
static optionsShape = {
|
|
61
60
|
...browser_1.BrowserCrawler.optionsShape,
|
|
62
|
-
stagehandOptions:
|
|
63
|
-
browserPoolOptions:
|
|
61
|
+
stagehandOptions: browser_1.schemas.anyObject.optional(),
|
|
62
|
+
browserPoolOptions: browser_1.schemas.anyObject.optional(),
|
|
64
63
|
};
|
|
64
|
+
static optionsSchema = zod_1.z.strictObject(StagehandCrawler.optionsShape);
|
|
65
65
|
/**
|
|
66
66
|
* Creates a new instance of StagehandCrawler.
|
|
67
67
|
*
|
|
68
68
|
* @param options - Crawler configuration options
|
|
69
69
|
*/
|
|
70
70
|
constructor(options = {}) {
|
|
71
|
-
(0,
|
|
72
|
-
const { stagehandOptions = {}, launchContext
|
|
71
|
+
const parsedOptions = (0, browser_1.parseArgument)(options, StagehandCrawler.optionsSchema, 'StagehandCrawlerOptions');
|
|
72
|
+
const { stagehandOptions = {}, launchContext, contextPipelineBuilder, ...browserCrawlerOptions } = parsedOptions;
|
|
73
73
|
const browserPoolOptions = {
|
|
74
|
-
...
|
|
74
|
+
...parsedOptions.browserPoolOptions,
|
|
75
75
|
};
|
|
76
76
|
// Create launcher with Stagehand plugin
|
|
77
77
|
const launcher = new stagehand_launcher_1.StagehandLauncher({
|
|
@@ -2,6 +2,7 @@ import type { BrowserLaunchContext } from '@crawlee/browser';
|
|
|
2
2
|
import { BrowserLauncher, Configuration } from '@crawlee/browser';
|
|
3
3
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
4
4
|
import type { BrowserType, LaunchOptions } from 'playwright';
|
|
5
|
+
import { z } from 'zod';
|
|
5
6
|
import type { StagehandOptions } from './stagehand-crawler';
|
|
6
7
|
import { StagehandPlugin } from './stagehand-plugin';
|
|
7
8
|
/**
|
|
@@ -58,28 +59,38 @@ export declare class StagehandLauncher extends BrowserLauncher<StagehandPlugin>
|
|
|
58
59
|
readonly configuration: Configuration;
|
|
59
60
|
protected static optionsShape: {
|
|
60
61
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
61
|
-
launcher: import("
|
|
62
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
62
63
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
63
|
-
launchContextOptions: import("
|
|
64
|
+
launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
64
65
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
65
|
-
stagehandOptions: import("
|
|
66
|
+
stagehandOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
67
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
68
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
69
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
70
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
71
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
72
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
66
73
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
72
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
73
|
-
browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
74
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
75
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
76
|
+
};
|
|
77
|
+
protected static optionsSchema: z.ZodObject<{
|
|
74
78
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
75
|
-
|
|
79
|
+
launcher: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
76
80
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
77
|
-
|
|
81
|
+
launchContextOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
78
82
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
79
|
-
|
|
83
|
+
stagehandOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
84
|
+
proxyUrl: z.ZodOptional<z.ZodURL>;
|
|
85
|
+
useChrome: z.ZodOptional<z.ZodBoolean>;
|
|
86
|
+
useIncognitoPages: z.ZodOptional<z.ZodBoolean>;
|
|
87
|
+
browserPerProxy: z.ZodOptional<z.ZodBoolean>;
|
|
88
|
+
ignoreProxyCertificate: z.ZodOptional<z.ZodBoolean>;
|
|
89
|
+
userDataDir: z.ZodOptional<z.ZodString>;
|
|
80
90
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
81
|
-
|
|
82
|
-
|
|
91
|
+
launchOptions: z.ZodOptional<z.ZodCustom<import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary, import("@crawlee/types", { with: { "resolution-mode": "import" } }).Dictionary>>;
|
|
92
|
+
userAgent: z.ZodOptional<z.ZodString>;
|
|
93
|
+
}, z.core.$strict>;
|
|
83
94
|
/**
|
|
84
95
|
* All StagehandLauncher parameters are passed via the launchContext object.
|
|
85
96
|
*/
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.StagehandLauncher = void 0;
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
4
|
const browser_1 = require("@crawlee/browser");
|
|
6
|
-
const
|
|
5
|
+
const zod_1 = require("zod");
|
|
7
6
|
const stagehand_plugin_1 = require("./stagehand-plugin");
|
|
8
7
|
/**
|
|
9
8
|
* StagehandLauncher is based on BrowserLauncher and creates StagehandPlugin instances.
|
|
@@ -15,24 +14,26 @@ class StagehandLauncher extends browser_1.BrowserLauncher {
|
|
|
15
14
|
configuration;
|
|
16
15
|
static optionsShape = {
|
|
17
16
|
...browser_1.BrowserLauncher.optionsShape,
|
|
18
|
-
launcher
|
|
19
|
-
|
|
20
|
-
|
|
17
|
+
// Passthrough schemas — the launcher module object must keep its prototype through parsing.
|
|
18
|
+
launcher: browser_1.schemas.anyObject.optional(),
|
|
19
|
+
launchContextOptions: browser_1.schemas.anyObject.optional(),
|
|
20
|
+
stagehandOptions: browser_1.schemas.anyObject.optional(),
|
|
21
21
|
};
|
|
22
|
+
static optionsSchema = zod_1.z.strictObject(StagehandLauncher.optionsShape);
|
|
22
23
|
#stagehandOptions;
|
|
23
24
|
/**
|
|
24
25
|
* All StagehandLauncher parameters are passed via the launchContext object.
|
|
25
26
|
*/
|
|
26
27
|
constructor(launchContext = {}, configuration = browser_1.Configuration.getGlobalConfiguration()) {
|
|
27
|
-
(0,
|
|
28
|
-
const { launcher = browser_1.BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, stagehandOptions = {}, } =
|
|
29
|
-
const { launchOptions = {}, ...rest } =
|
|
28
|
+
const parsedContext = (0, browser_1.parseArgument)(launchContext, StagehandLauncher.optionsSchema, 'StagehandLaunchContext');
|
|
29
|
+
const { launcher = browser_1.BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, stagehandOptions = {}, } = parsedContext;
|
|
30
|
+
const { launchOptions = {}, ...rest } = parsedContext;
|
|
30
31
|
// Call super first before initializing properties
|
|
31
32
|
super({
|
|
32
33
|
...rest,
|
|
33
34
|
launchOptions: {
|
|
34
35
|
...launchOptions,
|
|
35
|
-
executablePath: getDefaultExecutablePath(
|
|
36
|
+
executablePath: getDefaultExecutablePath(parsedContext, configuration),
|
|
36
37
|
},
|
|
37
38
|
launcher,
|
|
38
39
|
}, configuration);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/stagehand",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.122",
|
|
4
4
|
"description": "AI-powered web crawling with Stagehand integration for Crawlee - enables natural language browser automation with act(), extract(), and observe() methods.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -57,18 +57,17 @@
|
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
59
|
"@apify/timeout": "^0.4.4",
|
|
60
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
61
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
62
|
-
"@crawlee/core": "4.0.0-beta.
|
|
63
|
-
"@crawlee/types": "4.0.0-beta.
|
|
64
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
65
|
-
"
|
|
66
|
-
"
|
|
60
|
+
"@crawlee/browser": "4.0.0-beta.122",
|
|
61
|
+
"@crawlee/browser-pool": "4.0.0-beta.122",
|
|
62
|
+
"@crawlee/core": "4.0.0-beta.122",
|
|
63
|
+
"@crawlee/types": "4.0.0-beta.122",
|
|
64
|
+
"@crawlee/utils": "4.0.0-beta.122",
|
|
65
|
+
"tslib": "^2.8.1",
|
|
66
|
+
"zod": "^4.4.3"
|
|
67
67
|
},
|
|
68
68
|
"peerDependencies": {
|
|
69
69
|
"@browserbasehq/stagehand": "^3.0.0",
|
|
70
|
-
"playwright": "*"
|
|
71
|
-
"zod": "^3.0.0 || ^4.0.0"
|
|
70
|
+
"playwright": "*"
|
|
72
71
|
},
|
|
73
72
|
"peerDependenciesMeta": {
|
|
74
73
|
"@browserbasehq/stagehand": {
|
|
@@ -76,15 +75,11 @@
|
|
|
76
75
|
},
|
|
77
76
|
"playwright": {
|
|
78
77
|
"optional": true
|
|
79
|
-
},
|
|
80
|
-
"zod": {
|
|
81
|
-
"optional": true
|
|
82
78
|
}
|
|
83
79
|
},
|
|
84
80
|
"devDependencies": {
|
|
85
81
|
"@browserbasehq/stagehand": "3.0.7",
|
|
86
|
-
"playwright": "^1.60.0"
|
|
87
|
-
"zod": "^4.3.5"
|
|
82
|
+
"playwright": "^1.60.0"
|
|
88
83
|
},
|
|
89
84
|
"lerna": {
|
|
90
85
|
"command": {
|
|
@@ -93,5 +88,5 @@
|
|
|
93
88
|
}
|
|
94
89
|
}
|
|
95
90
|
},
|
|
96
|
-
"gitHead": "
|
|
91
|
+
"gitHead": "2c3e87fefdb9e1fca4c144f03167d8524cfdc2e5"
|
|
97
92
|
}
|