@crawlee/stagehand 3.18.0 → 4.0.0-beta.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/index.d.ts +1 -1
- package/index.mjs +41 -22
- package/internals/stagehand-controller.js +5 -16
- package/internals/stagehand-crawler.d.ts +57 -60
- package/internals/stagehand-crawler.js +40 -69
- package/internals/stagehand-launcher.d.ts +5 -7
- package/internals/stagehand-launcher.js +14 -28
- package/internals/stagehand-plugin.d.ts +1 -3
- package/internals/stagehand-plugin.js +6 -19
- package/package.json +12 -13
package/README.md
CHANGED
|
@@ -1,23 +1,23 @@
|
|
|
1
1
|
<h1 align="center">
|
|
2
2
|
<a href="https://crawlee.dev">
|
|
3
3
|
<picture>
|
|
4
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true"
|
|
5
|
-
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500"
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true" />
|
|
5
|
+
<img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500" />
|
|
6
6
|
</picture>
|
|
7
7
|
</a>
|
|
8
|
-
<br
|
|
8
|
+
<br />
|
|
9
9
|
<small>A web scraping and browser automation library</small>
|
|
10
10
|
</h1>
|
|
11
11
|
|
|
12
|
-
<p align=center>
|
|
13
|
-
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift"
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" width="250" height="55"/></a>
|
|
14
14
|
</p>
|
|
15
15
|
|
|
16
|
-
<p align=center>
|
|
17
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg"
|
|
18
|
-
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg"
|
|
19
|
-
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord"
|
|
20
|
-
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status"
|
|
16
|
+
<p align="center">
|
|
17
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" /></a>
|
|
18
|
+
<a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" /></a>
|
|
19
|
+
<a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" /></a>
|
|
20
|
+
<a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" /></a>
|
|
21
21
|
</p>
|
|
22
22
|
|
|
23
23
|
Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
|
package/index.d.ts
CHANGED
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
* @module @crawlee/stagehand
|
|
51
51
|
*/
|
|
52
52
|
export * from '@crawlee/browser';
|
|
53
|
-
export { StagehandCrawler, createStagehandRouter
|
|
53
|
+
export { StagehandCrawler, createStagehandRouter } from './internals/stagehand-crawler';
|
|
54
54
|
export type { StagehandOptions, StagehandPage, StagehandCrawlingContext, StagehandHook, StagehandRequestHandler, StagehandGotoOptions, StagehandCrawlerOptions, } from './internals/stagehand-crawler';
|
|
55
55
|
export type { StagehandLaunchContext } from './internals/stagehand-launcher';
|
|
56
56
|
export * as stagehandUtils from './internals/utils/stagehand-utils';
|
package/index.mjs
CHANGED
|
@@ -2,25 +2,33 @@ import mod from "./index.js";
|
|
|
2
2
|
|
|
3
3
|
export default mod;
|
|
4
4
|
export const API_PROCESSED_REQUESTS_DELAY_MILLIS = mod.API_PROCESSED_REQUESTS_DELAY_MILLIS;
|
|
5
|
+
export const ApifyLogAdapter = mod.ApifyLogAdapter;
|
|
5
6
|
export const AutoscaledPool = mod.AutoscaledPool;
|
|
6
|
-
export const BASIC_CRAWLER_TIMEOUT_BUFFER_SECS = mod.BASIC_CRAWLER_TIMEOUT_BUFFER_SECS;
|
|
7
7
|
export const BLOCKED_STATUS_CODES = mod.BLOCKED_STATUS_CODES;
|
|
8
|
+
export const BaseCrawleeLogger = mod.BaseCrawleeLogger;
|
|
8
9
|
export const BasicCrawler = mod.BasicCrawler;
|
|
9
10
|
export const BrowserCrawler = mod.BrowserCrawler;
|
|
10
11
|
export const BrowserLauncher = mod.BrowserLauncher;
|
|
12
|
+
export const ClientLoadSignal = mod.ClientLoadSignal;
|
|
13
|
+
export const ConcurrencySystem = mod.ConcurrencySystem;
|
|
11
14
|
export const Configuration = mod.Configuration;
|
|
15
|
+
export const ContextPipeline = mod.ContextPipeline;
|
|
16
|
+
export const ContextPipelineCleanupError = mod.ContextPipelineCleanupError;
|
|
17
|
+
export const ContextPipelineInitializationError = mod.ContextPipelineInitializationError;
|
|
18
|
+
export const ContextPipelineInterruptedError = mod.ContextPipelineInterruptedError;
|
|
12
19
|
export const CookieParseError = mod.CookieParseError;
|
|
13
|
-
export const
|
|
20
|
+
export const CpuLoadSignal = mod.CpuLoadSignal;
|
|
14
21
|
export const CriticalError = mod.CriticalError;
|
|
15
22
|
export const DATASET_ITERATORS_DEFAULT_LIMIT = mod.DATASET_ITERATORS_DEFAULT_LIMIT;
|
|
23
|
+
export const DEFAULT_CURRENT_HISTORY_SECS = mod.DEFAULT_CURRENT_HISTORY_SECS;
|
|
24
|
+
export const DEFAULT_SNAPSHOT_HISTORY_SECS = mod.DEFAULT_SNAPSHOT_HISTORY_SECS;
|
|
16
25
|
export const Dataset = mod.Dataset;
|
|
17
|
-
export const EVENT_SESSION_RETIRED = mod.EVENT_SESSION_RETIRED;
|
|
18
26
|
export const EnqueueStrategy = mod.EnqueueStrategy;
|
|
19
27
|
export const ErrorSnapshotter = mod.ErrorSnapshotter;
|
|
20
28
|
export const ErrorTracker = mod.ErrorTracker;
|
|
29
|
+
export const EventLoopLoadSignal = mod.EventLoopLoadSignal;
|
|
21
30
|
export const EventManager = mod.EventManager;
|
|
22
31
|
export const EventType = mod.EventType;
|
|
23
|
-
export const GotScrapingHttpClient = mod.GotScrapingHttpClient;
|
|
24
32
|
export const KeyValueStore = mod.KeyValueStore;
|
|
25
33
|
export const LocalEventManager = mod.LocalEventManager;
|
|
26
34
|
export const Log = mod.Log;
|
|
@@ -31,74 +39,85 @@ export const LoggerText = mod.LoggerText;
|
|
|
31
39
|
export const MAX_POOL_SIZE = mod.MAX_POOL_SIZE;
|
|
32
40
|
export const MAX_QUERIES_FOR_CONSISTENCY = mod.MAX_QUERIES_FOR_CONSISTENCY;
|
|
33
41
|
export const MemoryLoadSignal = mod.MemoryLoadSignal;
|
|
42
|
+
export const MemoryStorageBackend = mod.MemoryStorageBackend;
|
|
34
43
|
export const MissingRouteError = mod.MissingRouteError;
|
|
44
|
+
export const MissingSessionError = mod.MissingSessionError;
|
|
45
|
+
export const NavigationSkippedError = mod.NavigationSkippedError;
|
|
35
46
|
export const NonRetryableError = mod.NonRetryableError;
|
|
47
|
+
export const OwnedOrInjected = mod.OwnedOrInjected;
|
|
36
48
|
export const PERSIST_STATE_KEY = mod.PERSIST_STATE_KEY;
|
|
37
49
|
export const ProxyConfiguration = mod.ProxyConfiguration;
|
|
38
|
-
export const PseudoUrl = mod.PseudoUrl;
|
|
39
50
|
export const QUERY_HEAD_BUFFER = mod.QUERY_HEAD_BUFFER;
|
|
40
51
|
export const QUERY_HEAD_MIN_LENGTH = mod.QUERY_HEAD_MIN_LENGTH;
|
|
41
52
|
export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
|
|
42
53
|
export const RecoverableState = mod.RecoverableState;
|
|
43
54
|
export const Request = mod.Request;
|
|
55
|
+
export const RequestHandlerError = mod.RequestHandlerError;
|
|
44
56
|
export const RequestHandlerResult = mod.RequestHandlerResult;
|
|
45
57
|
export const RequestList = mod.RequestList;
|
|
46
|
-
export const RequestListAdapter = mod.RequestListAdapter;
|
|
47
58
|
export const RequestManagerTandem = mod.RequestManagerTandem;
|
|
48
|
-
export const RequestProvider = mod.RequestProvider;
|
|
49
59
|
export const RequestQueue = mod.RequestQueue;
|
|
50
|
-
export const RequestQueueV1 = mod.RequestQueueV1;
|
|
51
|
-
export const RequestQueueV2 = mod.RequestQueueV2;
|
|
52
60
|
export const RequestState = mod.RequestState;
|
|
53
61
|
export const RequestValidationError = mod.RequestValidationError;
|
|
54
62
|
export const RetryRequestError = mod.RetryRequestError;
|
|
55
63
|
export const Router = mod.Router;
|
|
56
64
|
export const STATE_PERSISTENCE_KEY = mod.STATE_PERSISTENCE_KEY;
|
|
57
65
|
export const STORAGE_CONSISTENCY_DELAY_MILLIS = mod.STORAGE_CONSISTENCY_DELAY_MILLIS;
|
|
66
|
+
export const ServiceConflictError = mod.ServiceConflictError;
|
|
67
|
+
export const ServiceLocator = mod.ServiceLocator;
|
|
58
68
|
export const Session = mod.Session;
|
|
59
69
|
export const SessionError = mod.SessionError;
|
|
60
70
|
export const SessionPool = mod.SessionPool;
|
|
61
|
-
export const
|
|
71
|
+
export const SitemapRequestLoader = mod.SitemapRequestLoader;
|
|
62
72
|
export const SnapshotStore = mod.SnapshotStore;
|
|
63
73
|
export const Snapshotter = mod.Snapshotter;
|
|
64
74
|
export const StagehandCrawler = mod.StagehandCrawler;
|
|
65
75
|
export const Statistics = mod.Statistics;
|
|
66
|
-
export const
|
|
76
|
+
export const StorageInstanceManager = mod.StorageInstanceManager;
|
|
77
|
+
export const StorageStatsTracker = mod.StorageStatsTracker;
|
|
67
78
|
export const SystemStatus = mod.SystemStatus;
|
|
79
|
+
export const applyRequestTransform = mod.applyRequestTransform;
|
|
80
|
+
export const assertJsonSerializable = mod.assertJsonSerializable;
|
|
81
|
+
export const bindMethodsToServiceLocator = mod.bindMethodsToServiceLocator;
|
|
68
82
|
export const browserCrawlerEnqueueLinks = mod.browserCrawlerEnqueueLinks;
|
|
69
83
|
export const browserPoolCookieToToughCookie = mod.browserPoolCookieToToughCookie;
|
|
70
|
-
export const checkAndSerialize = mod.checkAndSerialize;
|
|
71
84
|
export const checkStorageAccess = mod.checkStorageAccess;
|
|
72
|
-
export const
|
|
85
|
+
export const coerceBoolean = mod.coerceBoolean;
|
|
86
|
+
export const coerceNumber = mod.coerceNumber;
|
|
73
87
|
export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
|
|
74
|
-
export const constructRegExpObjectsFromPseudoUrls = mod.constructRegExpObjectsFromPseudoUrls;
|
|
75
88
|
export const constructRegExpObjectsFromRegExps = mod.constructRegExpObjectsFromRegExps;
|
|
89
|
+
export const constructUrlPatternObjects = mod.constructUrlPatternObjects;
|
|
76
90
|
export const cookieStringToToughCookie = mod.cookieStringToToughCookie;
|
|
91
|
+
export const crawleeConfigFields = mod.crawleeConfigFields;
|
|
77
92
|
export const createBasicRouter = mod.createBasicRouter;
|
|
78
|
-
export const createClientLoadSignal = mod.createClientLoadSignal;
|
|
79
|
-
export const createCpuLoadSignal = mod.createCpuLoadSignal;
|
|
80
93
|
export const createDeserialize = mod.createDeserialize;
|
|
81
|
-
export const
|
|
94
|
+
export const createDualIterable = mod.createDualIterable;
|
|
95
|
+
export const createRequestDebugInfo = mod.createRequestDebugInfo;
|
|
82
96
|
export const createRequestOptions = mod.createRequestOptions;
|
|
83
|
-
export const createRequests = mod.createRequests;
|
|
84
97
|
export const createStagehandRouter = mod.createStagehandRouter;
|
|
85
98
|
export const defaultRoute = mod.defaultRoute;
|
|
86
99
|
export const deserializeArray = mod.deserializeArray;
|
|
87
100
|
export const enqueueLinks = mod.enqueueLinks;
|
|
88
101
|
export const evaluateLoadSignalSample = mod.evaluateLoadSignalSample;
|
|
89
102
|
export const extractUrlsFromPage = mod.extractUrlsFromPage;
|
|
90
|
-
export const
|
|
103
|
+
export const field = mod.field;
|
|
104
|
+
export const filterRequestOptionsByPatterns = mod.filterRequestOptionsByPatterns;
|
|
91
105
|
export const getCookiesFromResponse = mod.getCookiesFromResponse;
|
|
92
106
|
export const getDefaultCookieExpirationDate = mod.getDefaultCookieExpirationDate;
|
|
107
|
+
export const getObjectType = mod.getObjectType;
|
|
93
108
|
export const getRequestId = mod.getRequestId;
|
|
94
|
-
export const
|
|
109
|
+
export const inspectValue = mod.inspectValue;
|
|
95
110
|
export const log = mod.log;
|
|
96
|
-
export const maybeStringify = mod.maybeStringify;
|
|
97
111
|
export const mergeCookies = mod.mergeCookies;
|
|
98
|
-
export const
|
|
112
|
+
export const navigationDeadlineKey = mod.navigationDeadlineKey;
|
|
113
|
+
export const parseValue = mod.parseValue;
|
|
99
114
|
export const purgeDefaultStorages = mod.purgeDefaultStorages;
|
|
115
|
+
export const remainingNavigationWindowMillis = mod.remainingNavigationWindowMillis;
|
|
100
116
|
export const resolveBaseUrlForEnqueueLinksFiltering = mod.resolveBaseUrlForEnqueueLinksFiltering;
|
|
117
|
+
export const resolveStorageIdentifier = mod.resolveStorageIdentifier;
|
|
101
118
|
export const serializeArray = mod.serializeArray;
|
|
119
|
+
export const serializeValue = mod.serializeValue;
|
|
120
|
+
export const serviceLocator = mod.serviceLocator;
|
|
102
121
|
export const stagehandUtils = mod.stagehandUtils;
|
|
103
122
|
export const toughCookieToBrowserPoolCookie = mod.toughCookieToBrowserPoolCookie;
|
|
104
123
|
export const tryAbsoluteURL = mod.tryAbsoluteURL;
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.StagehandController = void 0;
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
4
|
const browser_pool_1 = require("@crawlee/browser-pool");
|
|
5
|
+
const core_1 = require("@crawlee/core");
|
|
6
6
|
const utils_1 = require("@crawlee/utils");
|
|
7
|
-
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
8
7
|
/**
|
|
9
8
|
* StagehandController manages the lifecycle of a Stagehand-controlled browser for Crawlee's BrowserPool.
|
|
10
9
|
*
|
|
@@ -19,20 +18,10 @@ const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
|
19
18
|
* @ignore
|
|
20
19
|
*/
|
|
21
20
|
class StagehandController extends browser_pool_1.BrowserController {
|
|
21
|
+
stagehand = null;
|
|
22
|
+
stagehandInstances;
|
|
22
23
|
constructor(browserPlugin, stagehandInstances) {
|
|
23
24
|
super(browserPlugin);
|
|
24
|
-
Object.defineProperty(this, "stagehand", {
|
|
25
|
-
enumerable: true,
|
|
26
|
-
configurable: true,
|
|
27
|
-
writable: true,
|
|
28
|
-
value: null
|
|
29
|
-
});
|
|
30
|
-
Object.defineProperty(this, "stagehandInstances", {
|
|
31
|
-
enumerable: true,
|
|
32
|
-
configurable: true,
|
|
33
|
-
writable: true,
|
|
34
|
-
value: void 0
|
|
35
|
-
});
|
|
36
25
|
this.stagehandInstances = stagehandInstances;
|
|
37
26
|
}
|
|
38
27
|
/**
|
|
@@ -88,7 +77,7 @@ class StagehandController extends browser_pool_1.BrowserController {
|
|
|
88
77
|
* a short window, and the AI methods fail with 'Failed to resolve V3 Page from Playwright page'.
|
|
89
78
|
* Stagehand's own `newPage()` polls for the same reason.
|
|
90
79
|
*/
|
|
91
|
-
async waitForStagehandToRegisterPage(page, timeoutMs =
|
|
80
|
+
async waitForStagehandToRegisterPage(page, timeoutMs = 10_000) {
|
|
92
81
|
const stagehand = this.getStagehand();
|
|
93
82
|
const mainFrameId = await this.getMainFrameId(page);
|
|
94
83
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -170,7 +159,7 @@ class StagehandController extends browser_pool_1.BrowserController {
|
|
|
170
159
|
await stagehand.close();
|
|
171
160
|
}
|
|
172
161
|
catch (error) {
|
|
173
|
-
|
|
162
|
+
core_1.serviceLocator.getLogger().error('Error closing Stagehand', { error });
|
|
174
163
|
}
|
|
175
164
|
}
|
|
176
165
|
/**
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import type { Action, ActOptions, ActResult, AgentConfig, ExtractOptions, LLMClient, ModelConfiguration, NonStreamingAgentInstance, ObserveOptions, Stagehand, StreamingAgentInstance } from '@browserbasehq/stagehand';
|
|
2
|
-
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook,
|
|
3
|
-
import { BrowserCrawler
|
|
2
|
+
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserHook, ContextPipeline, CrawlingContext, GetUserDataFromRequest, LoadedContext, RequestHandler, RouterHandler, RouterRoutes, RouteSchemas, RoutesFromSchemas } from '@crawlee/browser';
|
|
3
|
+
import { BrowserCrawler } from '@crawlee/browser';
|
|
4
4
|
import type { Dictionary } from '@crawlee/types';
|
|
5
5
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
6
6
|
import type { LaunchOptions, Page, Response } from 'playwright';
|
|
7
7
|
import type { z } from 'zod';
|
|
8
|
-
import type { StagehandController } from './stagehand-controller';
|
|
9
8
|
import type { StagehandLaunchContext } from './stagehand-launcher';
|
|
10
9
|
import type { StagehandPlugin } from './stagehand-plugin';
|
|
11
10
|
/**
|
|
@@ -133,7 +132,7 @@ export interface StagehandPage extends Page {
|
|
|
133
132
|
* );
|
|
134
133
|
* ```
|
|
135
134
|
*/
|
|
136
|
-
extract<T>(instruction: string, schema: z.
|
|
135
|
+
extract<T>(instruction: string, schema: z.ZodType<T>, options?: Omit<ExtractOptions, 'page'>): Promise<T>;
|
|
137
136
|
/**
|
|
138
137
|
* Observe the page and get AI-suggested actions.
|
|
139
138
|
*
|
|
@@ -169,7 +168,11 @@ export interface StagehandPage extends Page {
|
|
|
169
168
|
/**
|
|
170
169
|
* Crawling context for StagehandCrawler with enhanced page object.
|
|
171
170
|
*/
|
|
172
|
-
|
|
171
|
+
/**
|
|
172
|
+
* Goto options for StagehandCrawler navigation.
|
|
173
|
+
*/
|
|
174
|
+
export type StagehandGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
|
175
|
+
export interface StagehandCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<StagehandPage, Response, UserData, StagehandGotoOptions> {
|
|
173
176
|
/**
|
|
174
177
|
* Enhanced Playwright page with Stagehand AI methods.
|
|
175
178
|
* Use page.act(), page.extract(), page.observe(), page.agent() for AI-powered operations.
|
|
@@ -184,23 +187,19 @@ export interface StagehandCrawlingContext<UserData extends Dictionary = Dictiona
|
|
|
184
187
|
/**
|
|
185
188
|
* Hook function for StagehandCrawler.
|
|
186
189
|
*/
|
|
187
|
-
export interface StagehandHook extends BrowserHook<StagehandCrawlingContext
|
|
190
|
+
export interface StagehandHook extends BrowserHook<StagehandCrawlingContext> {
|
|
188
191
|
}
|
|
189
192
|
/**
|
|
190
193
|
* Request handler for StagehandCrawler.
|
|
191
194
|
*/
|
|
192
|
-
export interface StagehandRequestHandler extends
|
|
195
|
+
export interface StagehandRequestHandler extends RequestHandler<LoadedContext<StagehandCrawlingContext>> {
|
|
193
196
|
}
|
|
194
|
-
/**
|
|
195
|
-
* Goto options for StagehandCrawler navigation.
|
|
196
|
-
*/
|
|
197
|
-
export type StagehandGotoOptions = Dictionary & Parameters<Page['goto']>[1];
|
|
198
197
|
/**
|
|
199
198
|
* Options for StagehandCrawler.
|
|
200
199
|
*/
|
|
201
|
-
export interface StagehandCrawlerOptions extends BrowserCrawlerOptions<StagehandCrawlingContext, {
|
|
200
|
+
export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>> extends BrowserCrawlerOptions<StagehandPage, Response, StagehandCrawlingContext, ContextExtension, ExtendedContext, {
|
|
202
201
|
browserPlugins: [StagehandPlugin];
|
|
203
|
-
}> {
|
|
202
|
+
}, Routes> {
|
|
204
203
|
/**
|
|
205
204
|
* Stagehand-specific configuration options.
|
|
206
205
|
* These options configure the AI behavior and Browserbase integration.
|
|
@@ -216,7 +215,6 @@ export interface StagehandCrawlerOptions extends BrowserCrawlerOptions<Stagehand
|
|
|
216
215
|
* The function receives the {@link StagehandCrawlingContext} as an argument, where:
|
|
217
216
|
* - `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
|
|
218
217
|
* - `page` is an enhanced Playwright [`Page`](https://playwright.dev/docs/api/class-page) with AI methods
|
|
219
|
-
* - `browserController` is an instance of {@link StagehandController}
|
|
220
218
|
* - `response` is the main resource response as returned by `page.goto(request.url)`
|
|
221
219
|
* - `stagehand` is the Stagehand instance for advanced control
|
|
222
220
|
*
|
|
@@ -255,11 +253,7 @@ export interface StagehandCrawlerOptions extends BrowserCrawlerOptions<Stagehand
|
|
|
255
253
|
* }
|
|
256
254
|
* ```
|
|
257
255
|
*/
|
|
258
|
-
requestHandler?:
|
|
259
|
-
/**
|
|
260
|
-
* Function called when request handling fails after all retries.
|
|
261
|
-
*/
|
|
262
|
-
failedRequestHandler?: StagehandRequestHandler;
|
|
256
|
+
requestHandler?: RouterHandler<ExtendedContext, Routes> | RequestHandler<ExtendedContext>;
|
|
263
257
|
/**
|
|
264
258
|
* Async functions that are sequentially evaluated before the navigation.
|
|
265
259
|
*/
|
|
@@ -318,17 +312,14 @@ export interface StagehandCrawlerOptions extends BrowserCrawlerOptions<Stagehand
|
|
|
318
312
|
* await crawler.run(['https://example.com']);
|
|
319
313
|
* ```
|
|
320
314
|
*/
|
|
321
|
-
export declare class StagehandCrawler extends BrowserCrawler<{
|
|
315
|
+
export declare class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension, Routes extends Record<keyof Routes, Dictionary> = Record<string, GetUserDataFromRequest<StagehandCrawlingContext['request']>>> extends BrowserCrawler<StagehandPage, Response, {
|
|
322
316
|
browserPlugins: [StagehandPlugin];
|
|
323
|
-
}, LaunchOptions, StagehandCrawlingContext> {
|
|
324
|
-
readonly config: Configuration;
|
|
317
|
+
}, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext, Routes> {
|
|
325
318
|
protected static optionsShape: {
|
|
326
319
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
327
|
-
stagehandOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
328
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
329
|
-
browserPoolOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
320
|
+
stagehandOptions: import("ow", { with: { "resolution-mode": "import" } }).ObjectPredicate<object> & import("ow", { with: { "resolution-mode": "import" } }).BasePredicate<object | undefined>;
|
|
330
321
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
331
|
-
|
|
322
|
+
browserPoolOptions: import("ow", { with: { "resolution-mode": "import" } }).ObjectPredicate<object> & import("ow", { with: { "resolution-mode": "import" } }).BasePredicate<object | undefined>;
|
|
332
323
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
333
324
|
navigationTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
334
325
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -340,51 +331,53 @@ export declare class StagehandCrawler extends BrowserCrawler<{
|
|
|
340
331
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
341
332
|
headless: import("ow").AnyPredicate<string | boolean>;
|
|
342
333
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
343
|
-
|
|
334
|
+
browserPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
344
335
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
345
|
-
|
|
336
|
+
remoteBrowser: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
346
337
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
347
|
-
|
|
338
|
+
saveResponseCookies: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
348
339
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
349
340
|
proxyConfiguration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
350
341
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
351
|
-
|
|
342
|
+
contextPipelineBuilder: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
352
343
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
353
|
-
|
|
344
|
+
extendContext: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
354
345
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
355
346
|
requestList: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
356
347
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
357
348
|
requestQueue: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
358
349
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
359
350
|
requestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
360
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
361
|
-
handleRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
362
351
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
363
352
|
requestHandlerTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
364
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
365
|
-
handleRequestTimeoutSecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
366
353
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
367
354
|
errorHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
368
355
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
369
356
|
failedRequestHandler: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
370
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
371
|
-
handleFailedRequestFunction: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
372
357
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
373
358
|
maxRequestRetries: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
374
359
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
375
360
|
sameDomainDelaySecs: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
376
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
377
|
-
maxSessionRotations: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
378
361
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
379
362
|
maxRequestsPerCrawl: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
380
363
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
381
364
|
maxCrawlDepth: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
382
365
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
383
|
-
|
|
366
|
+
taskLoopOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
367
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
368
|
+
concurrencySystem: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
369
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
370
|
+
sessionPool: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
384
371
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
385
372
|
statusMessageLoggingInterval: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
386
373
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
387
374
|
statusMessageCallback: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
375
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
376
|
+
additionalHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
377
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
378
|
+
ignoreHttpErrorStatusCodes: import("ow").ArrayPredicate<number>;
|
|
379
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
380
|
+
blockedStatusCodes: import("ow").ArrayPredicate<number>;
|
|
388
381
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
389
382
|
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
390
383
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -393,6 +386,14 @@ export declare class StagehandCrawler extends BrowserCrawler<{
|
|
|
393
386
|
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
394
387
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
395
388
|
httpClient: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
389
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
390
|
+
configuration: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
391
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
392
|
+
storageBackend: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
393
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
394
|
+
eventManager: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
395
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
396
|
+
logger: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
396
397
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
397
398
|
minConcurrency: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
398
399
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -401,36 +402,32 @@ export declare class StagehandCrawler extends BrowserCrawler<{
|
|
|
401
402
|
maxRequestsPerMinute: import("ow").NumberPredicate & import("ow").BasePredicate<number | undefined>;
|
|
402
403
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
403
404
|
keepAlive: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
404
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
405
|
-
log: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
406
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
407
|
-
experiments: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
408
405
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
409
406
|
statisticsOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
407
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
408
|
+
id: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
410
409
|
};
|
|
411
|
-
/** Set while `_runRequestHandler` has swapped `userProvidedRequestHandler` for its page-enhancing wrapper. */
|
|
412
|
-
private unwrappedRequestHandler?;
|
|
413
|
-
protected get userRequestHandler(): BrowserRequestHandler<StagehandCrawlingContext>;
|
|
414
410
|
/**
|
|
415
411
|
* Creates a new instance of StagehandCrawler.
|
|
416
412
|
*
|
|
417
413
|
* @param options - Crawler configuration options
|
|
418
414
|
*/
|
|
419
|
-
constructor(options?: StagehandCrawlerOptions,
|
|
415
|
+
constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext, Routes>);
|
|
416
|
+
protected buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext>;
|
|
420
417
|
/**
|
|
421
|
-
*
|
|
418
|
+
* Resolves the {@link StagehandController} that owns the given page, or
|
|
419
|
+
* `undefined` when the pool does not expose controllers (e.g. a custom
|
|
420
|
+
* {@link IBrowserPool} implementation).
|
|
422
421
|
*
|
|
423
|
-
*
|
|
424
|
-
*
|
|
425
|
-
*
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
*
|
|
430
|
-
* This is similar to how PlaywrightCrawler adds utility methods via registerUtilsToContext,
|
|
431
|
-
* but we need to actually transform the page object to add Stagehand AI methods.
|
|
422
|
+
* Stagehand needs direct controller access to reach the `Stagehand`
|
|
423
|
+
* instance bound to the page's browser, which is why it reaches past the
|
|
424
|
+
* {@link IBrowserPool} abstraction here.
|
|
425
|
+
*/
|
|
426
|
+
private getBrowserControllerByPage;
|
|
427
|
+
/**
|
|
428
|
+
* Enhance the page with Stagehand AI methods.
|
|
432
429
|
*/
|
|
433
|
-
|
|
430
|
+
private setUpStagehand;
|
|
434
431
|
/**
|
|
435
432
|
* Navigation handler for Stagehand crawler.
|
|
436
433
|
* Uses standard Playwright navigation.
|
|
@@ -453,7 +450,7 @@ export declare class StagehandCrawler extends BrowserCrawler<{
|
|
|
453
450
|
* });
|
|
454
451
|
*
|
|
455
452
|
* router.addDefaultHandler(async ({ page, enqueueLinks }) => {
|
|
456
|
-
* await enqueueLinks({
|
|
453
|
+
* await enqueueLinks({ include: ['https://example.com/products/*'] });
|
|
457
454
|
* });
|
|
458
455
|
*
|
|
459
456
|
* const crawler = new StagehandCrawler({
|
|
@@ -57,86 +57,67 @@ const stagehand_utils_1 = require("./utils/stagehand-utils");
|
|
|
57
57
|
* ```
|
|
58
58
|
*/
|
|
59
59
|
class StagehandCrawler extends browser_1.BrowserCrawler {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
60
|
+
static optionsShape = {
|
|
61
|
+
...browser_1.BrowserCrawler.optionsShape,
|
|
62
|
+
stagehandOptions: ow_1.default.optional.object,
|
|
63
|
+
browserPoolOptions: ow_1.default.optional.object,
|
|
64
|
+
};
|
|
63
65
|
/**
|
|
64
66
|
* Creates a new instance of StagehandCrawler.
|
|
65
67
|
*
|
|
66
68
|
* @param options - Crawler configuration options
|
|
67
69
|
*/
|
|
68
|
-
constructor(options = {}
|
|
69
|
-
const { stagehandOptions = {}, launchContext = {}, browserPoolOptions = {}, ...browserCrawlerOptions } = options;
|
|
70
|
-
// Validate options
|
|
70
|
+
constructor(options = {}) {
|
|
71
71
|
(0, ow_1.default)(options, 'StagehandCrawlerOptions', ow_1.default.object.exactShape(StagehandCrawler.optionsShape));
|
|
72
|
+
const { stagehandOptions = {}, launchContext = {}, contextPipelineBuilder, ...browserCrawlerOptions } = options;
|
|
73
|
+
const browserPoolOptions = {
|
|
74
|
+
...options.browserPoolOptions,
|
|
75
|
+
};
|
|
72
76
|
// Create launcher with Stagehand plugin
|
|
73
77
|
const launcher = new stagehand_launcher_1.StagehandLauncher({
|
|
74
78
|
...launchContext,
|
|
75
79
|
stagehandOptions,
|
|
76
|
-
}
|
|
80
|
+
});
|
|
81
|
+
browserPoolOptions.browserPlugins = [launcher.createBrowserPlugin()];
|
|
77
82
|
// Initialize BrowserCrawler with Stagehand plugin and fingerprinting enabled
|
|
78
83
|
super({
|
|
79
84
|
...browserCrawlerOptions,
|
|
80
85
|
launchContext,
|
|
81
|
-
browserPoolOptions
|
|
82
|
-
|
|
83
|
-
browserPlugins: [launcher.createBrowserPlugin()],
|
|
84
|
-
// Enable fingerprinting by default for anti-blocking
|
|
85
|
-
useFingerprints: browserPoolOptions.useFingerprints ?? true,
|
|
86
|
-
},
|
|
87
|
-
}, config);
|
|
88
|
-
Object.defineProperty(this, "config", {
|
|
89
|
-
enumerable: true,
|
|
90
|
-
configurable: true,
|
|
91
|
-
writable: true,
|
|
92
|
-
value: config
|
|
93
|
-
});
|
|
94
|
-
/** Set while `_runRequestHandler` has swapped `userProvidedRequestHandler` for its page-enhancing wrapper. */
|
|
95
|
-
Object.defineProperty(this, "unwrappedRequestHandler", {
|
|
96
|
-
enumerable: true,
|
|
97
|
-
configurable: true,
|
|
98
|
-
writable: true,
|
|
99
|
-
value: void 0
|
|
86
|
+
browserPoolOptions,
|
|
87
|
+
contextPipelineBuilder: contextPipelineBuilder ?? (() => this.buildContextPipeline()),
|
|
100
88
|
});
|
|
101
89
|
}
|
|
90
|
+
buildContextPipeline() {
|
|
91
|
+
return super.buildContextPipeline().compose({ action: this.setUpStagehand.bind(this) });
|
|
92
|
+
}
|
|
102
93
|
/**
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
* 1. Store the original userProvidedRequestHandler
|
|
107
|
-
* 2. Replace it with a wrapper that enhances the page first
|
|
108
|
-
* 3. Call super (which creates page/browserController, then calls our wrapper)
|
|
109
|
-
* 4. Our wrapper enhances the page and calls the original handler
|
|
110
|
-
* 5. Restore the original handler
|
|
94
|
+
* Resolves the {@link StagehandController} that owns the given page, or
|
|
95
|
+
* `undefined` when the pool does not expose controllers (e.g. a custom
|
|
96
|
+
* {@link IBrowserPool} implementation).
|
|
111
97
|
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
98
|
+
* Stagehand needs direct controller access to reach the `Stagehand`
|
|
99
|
+
* instance bound to the page's browser, which is why it reaches past the
|
|
100
|
+
* {@link IBrowserPool} abstraction here.
|
|
114
101
|
*/
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
// Keep `userRequestHandler` pointing at the unwrapped handler: the swap below is live while the user's
|
|
119
|
-
// handler runs, which is exactly when `context.addRequests`/`enqueueLinks` resolve router metadata.
|
|
120
|
-
this.unwrappedRequestHandler = originalHandler;
|
|
121
|
-
// Replace with a wrapper that enhances the page before calling the user's handler
|
|
122
|
-
this.userProvidedRequestHandler = async (ctx) => {
|
|
123
|
-
// Get Stagehand instance from controller
|
|
124
|
-
const stagehand = ctx.browserController.getStagehand();
|
|
125
|
-
ctx.stagehand = stagehand;
|
|
126
|
-
// Enhance page with AI methods (page.act(), page.extract(), etc.)
|
|
127
|
-
ctx.page = (0, stagehand_utils_1.enhancePageWithStagehand)(ctx.page, stagehand);
|
|
128
|
-
// Call the original user handler
|
|
129
|
-
return originalHandler(ctx);
|
|
130
|
-
};
|
|
131
|
-
try {
|
|
132
|
-
// Call parent - this creates the page and eventually calls our wrapped handler
|
|
133
|
-
await super._runRequestHandler(crawlingContext);
|
|
102
|
+
getBrowserControllerByPage(page) {
|
|
103
|
+
if ('getBrowserControllerByPage' in this.browserPool) {
|
|
104
|
+
return this.browserPool.getBrowserControllerByPage(page);
|
|
134
105
|
}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Enhance the page with Stagehand AI methods.
|
|
110
|
+
*/
|
|
111
|
+
async setUpStagehand(crawlingContext) {
|
|
112
|
+
const controller = this.getBrowserControllerByPage(crawlingContext.page);
|
|
113
|
+
if (!controller) {
|
|
114
|
+
throw new Error('Could not resolve StagehandController for page — is the browser pool configured correctly?');
|
|
139
115
|
}
|
|
116
|
+
const stagehand = controller.getStagehand();
|
|
117
|
+
return {
|
|
118
|
+
stagehand,
|
|
119
|
+
page: (0, stagehand_utils_1.enhancePageWithStagehand)(crawlingContext.page, stagehand),
|
|
120
|
+
};
|
|
140
121
|
}
|
|
141
122
|
/**
|
|
142
123
|
* Navigation handler for Stagehand crawler.
|
|
@@ -148,16 +129,6 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
|
|
|
148
129
|
}
|
|
149
130
|
}
|
|
150
131
|
exports.StagehandCrawler = StagehandCrawler;
|
|
151
|
-
Object.defineProperty(StagehandCrawler, "optionsShape", {
|
|
152
|
-
enumerable: true,
|
|
153
|
-
configurable: true,
|
|
154
|
-
writable: true,
|
|
155
|
-
value: {
|
|
156
|
-
...browser_1.BrowserCrawler.optionsShape,
|
|
157
|
-
stagehandOptions: ow_1.default.optional.object,
|
|
158
|
-
browserPoolOptions: ow_1.default.optional.object,
|
|
159
|
-
}
|
|
160
|
-
});
|
|
161
132
|
function createStagehandRouter(routesOrSchemas) {
|
|
162
133
|
return browser_1.Router.create(routesOrSchemas);
|
|
163
134
|
}
|
|
@@ -54,14 +54,14 @@ export interface StagehandLaunchContext extends BrowserLaunchContext<LaunchOptio
|
|
|
54
54
|
* @ignore
|
|
55
55
|
*/
|
|
56
56
|
export declare class StagehandLauncher extends BrowserLauncher<StagehandPlugin> {
|
|
57
|
-
readonly
|
|
57
|
+
readonly configuration: Configuration;
|
|
58
58
|
protected static optionsShape: {
|
|
59
59
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
60
|
-
launcher: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
60
|
+
launcher: import("ow", { with: { "resolution-mode": "import" } }).ObjectPredicate<object> & import("ow", { with: { "resolution-mode": "import" } }).BasePredicate<object | undefined>;
|
|
61
61
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
62
|
-
launchContextOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
62
|
+
launchContextOptions: import("ow", { with: { "resolution-mode": "import" } }).ObjectPredicate<object> & import("ow", { with: { "resolution-mode": "import" } }).BasePredicate<object | undefined>;
|
|
63
63
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
64
|
-
stagehandOptions: import("ow").ObjectPredicate<object> & import("ow").BasePredicate<object | undefined>;
|
|
64
|
+
stagehandOptions: import("ow", { with: { "resolution-mode": "import" } }).ObjectPredicate<object> & import("ow", { with: { "resolution-mode": "import" } }).BasePredicate<object | undefined>;
|
|
65
65
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
66
66
|
proxyUrl: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
67
67
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -70,8 +70,6 @@ export declare class StagehandLauncher extends BrowserLauncher<StagehandPlugin>
|
|
|
70
70
|
useIncognitoPages: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
71
71
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
72
72
|
browserPerProxy: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
73
|
-
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
74
|
-
experimentalContainers: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
75
73
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
76
74
|
ignoreProxyCertificate: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
77
75
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -85,7 +83,7 @@ export declare class StagehandLauncher extends BrowserLauncher<StagehandPlugin>
|
|
|
85
83
|
/**
|
|
86
84
|
* All StagehandLauncher parameters are passed via the launchContext object.
|
|
87
85
|
*/
|
|
88
|
-
constructor(launchContext?: StagehandLaunchContext,
|
|
86
|
+
constructor(launchContext?: StagehandLaunchContext, configuration?: Configuration);
|
|
89
87
|
/**
|
|
90
88
|
* Creates a new StagehandPlugin instance with resolved options.
|
|
91
89
|
*/
|
|
@@ -12,10 +12,18 @@ const stagehand_plugin_1 = require("./stagehand-plugin");
|
|
|
12
12
|
* @ignore
|
|
13
13
|
*/
|
|
14
14
|
class StagehandLauncher extends browser_1.BrowserLauncher {
|
|
15
|
+
configuration;
|
|
16
|
+
static optionsShape = {
|
|
17
|
+
...browser_1.BrowserLauncher.optionsShape,
|
|
18
|
+
launcher: ow_1.default.optional.object,
|
|
19
|
+
launchContextOptions: ow_1.default.optional.object,
|
|
20
|
+
stagehandOptions: ow_1.default.optional.object,
|
|
21
|
+
};
|
|
22
|
+
stagehandOptions;
|
|
15
23
|
/**
|
|
16
24
|
* All StagehandLauncher parameters are passed via the launchContext object.
|
|
17
25
|
*/
|
|
18
|
-
constructor(launchContext = {},
|
|
26
|
+
constructor(launchContext = {}, configuration = browser_1.Configuration.getGlobalConfiguration()) {
|
|
19
27
|
(0, ow_1.default)(launchContext, 'StagehandLaunchContext', ow_1.default.object.exactShape(StagehandLauncher.optionsShape));
|
|
20
28
|
const { launcher = browser_1.BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium, stagehandOptions = {}, } = launchContext;
|
|
21
29
|
const { launchOptions = {}, ...rest } = launchContext;
|
|
@@ -24,22 +32,11 @@ class StagehandLauncher extends browser_1.BrowserLauncher {
|
|
|
24
32
|
...rest,
|
|
25
33
|
launchOptions: {
|
|
26
34
|
...launchOptions,
|
|
27
|
-
executablePath: getDefaultExecutablePath(launchContext,
|
|
35
|
+
executablePath: getDefaultExecutablePath(launchContext, configuration),
|
|
28
36
|
},
|
|
29
37
|
launcher,
|
|
30
|
-
},
|
|
31
|
-
|
|
32
|
-
enumerable: true,
|
|
33
|
-
configurable: true,
|
|
34
|
-
writable: true,
|
|
35
|
-
value: config
|
|
36
|
-
});
|
|
37
|
-
Object.defineProperty(this, "stagehandOptions", {
|
|
38
|
-
enumerable: true,
|
|
39
|
-
configurable: true,
|
|
40
|
-
writable: true,
|
|
41
|
-
value: void 0
|
|
42
|
-
});
|
|
38
|
+
}, configuration);
|
|
39
|
+
this.configuration = configuration;
|
|
43
40
|
this.stagehandOptions = {
|
|
44
41
|
env: 'LOCAL',
|
|
45
42
|
model: 'openai/gpt-4.1-mini',
|
|
@@ -60,23 +57,12 @@ class StagehandLauncher extends browser_1.BrowserLauncher {
|
|
|
60
57
|
}
|
|
61
58
|
}
|
|
62
59
|
exports.StagehandLauncher = StagehandLauncher;
|
|
63
|
-
Object.defineProperty(StagehandLauncher, "optionsShape", {
|
|
64
|
-
enumerable: true,
|
|
65
|
-
configurable: true,
|
|
66
|
-
writable: true,
|
|
67
|
-
value: {
|
|
68
|
-
...browser_1.BrowserLauncher.optionsShape,
|
|
69
|
-
launcher: ow_1.default.optional.object,
|
|
70
|
-
launchContextOptions: ow_1.default.optional.object,
|
|
71
|
-
stagehandOptions: ow_1.default.optional.object,
|
|
72
|
-
}
|
|
73
|
-
});
|
|
74
60
|
/**
|
|
75
61
|
* Gets the default executable path for the browser.
|
|
76
62
|
* @ignore
|
|
77
63
|
*/
|
|
78
|
-
function getDefaultExecutablePath(launchContext,
|
|
79
|
-
const pathFromPlaywrightImage =
|
|
64
|
+
function getDefaultExecutablePath(launchContext, configuration) {
|
|
65
|
+
const pathFromPlaywrightImage = configuration.defaultBrowserPath;
|
|
80
66
|
const { launchOptions = {} } = launchContext;
|
|
81
67
|
if (launchOptions.executablePath) {
|
|
82
68
|
return launchOptions.executablePath;
|
|
@@ -25,8 +25,6 @@ export interface StagehandPluginOptions extends BrowserPluginOptions<LaunchOptio
|
|
|
25
25
|
* Limitations:
|
|
26
26
|
* - Only Chromium is supported (Stagehand uses CDP)
|
|
27
27
|
* - Some fingerprinting options may not be fully applied (Stagehand controls browser launch)
|
|
28
|
-
*
|
|
29
|
-
* @ignore
|
|
30
28
|
*/
|
|
31
29
|
export declare class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, PlaywrightBrowser> {
|
|
32
30
|
readonly stagehandOptions: StagehandOptions;
|
|
@@ -43,7 +41,7 @@ export declare class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOp
|
|
|
43
41
|
/**
|
|
44
42
|
* Creates a controller for the Stagehand browser.
|
|
45
43
|
*/
|
|
46
|
-
|
|
44
|
+
createController(): BrowserController<BrowserType, LaunchOptions, PlaywrightBrowser>;
|
|
47
45
|
/**
|
|
48
46
|
* Adds proxy configuration to launch options.
|
|
49
47
|
*/
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.StagehandPlugin = void 0;
|
|
4
|
-
const tslib_1 = require("tslib");
|
|
5
4
|
const browser_pool_1 = require("@crawlee/browser-pool");
|
|
5
|
+
const core_1 = require("@crawlee/core");
|
|
6
6
|
// Stagehand is built on CDP (Chrome DevTools Protocol), which only works with Chromium-based browsers.
|
|
7
7
|
// Firefox and WebKit are not supported by Stagehand.
|
|
8
8
|
const playwright_1 = require("playwright");
|
|
9
|
-
const log_1 = tslib_1.__importDefault(require("@apify/log"));
|
|
10
9
|
const stagehand_controller_1 = require("./stagehand-controller");
|
|
11
10
|
/**
|
|
12
11
|
* StagehandPlugin integrates Stagehand with Crawlee's BrowserPool.
|
|
@@ -20,24 +19,12 @@ const stagehand_controller_1 = require("./stagehand-controller");
|
|
|
20
19
|
* Limitations:
|
|
21
20
|
* - Only Chromium is supported (Stagehand uses CDP)
|
|
22
21
|
* - Some fingerprinting options may not be fully applied (Stagehand controls browser launch)
|
|
23
|
-
*
|
|
24
|
-
* @ignore
|
|
25
22
|
*/
|
|
26
23
|
class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
24
|
+
stagehandOptions;
|
|
25
|
+
stagehandInstances = new WeakMap();
|
|
27
26
|
constructor(library, options = {}) {
|
|
28
27
|
super(library, options);
|
|
29
|
-
Object.defineProperty(this, "stagehandOptions", {
|
|
30
|
-
enumerable: true,
|
|
31
|
-
configurable: true,
|
|
32
|
-
writable: true,
|
|
33
|
-
value: void 0
|
|
34
|
-
});
|
|
35
|
-
Object.defineProperty(this, "stagehandInstances", {
|
|
36
|
-
enumerable: true,
|
|
37
|
-
configurable: true,
|
|
38
|
-
writable: true,
|
|
39
|
-
value: new WeakMap()
|
|
40
|
-
});
|
|
41
28
|
this.stagehandOptions = options.stagehandOptions ?? {};
|
|
42
29
|
}
|
|
43
30
|
/**
|
|
@@ -102,7 +89,7 @@ class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
|
102
89
|
await stagehand.close().catch(() => { });
|
|
103
90
|
await closeAnonymizedProxy();
|
|
104
91
|
const augmentedError = this._augmentLaunchError(error, launchContext);
|
|
105
|
-
|
|
92
|
+
core_1.serviceLocator.getLogger().error('Stagehand browser launch failed', { message: augmentedError.message });
|
|
106
93
|
throw augmentedError;
|
|
107
94
|
}
|
|
108
95
|
}
|
|
@@ -124,14 +111,14 @@ class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
|
124
111
|
/**
|
|
125
112
|
* Creates a controller for the Stagehand browser.
|
|
126
113
|
*/
|
|
127
|
-
|
|
114
|
+
createController() {
|
|
128
115
|
return new stagehand_controller_1.StagehandController(this, this.stagehandInstances);
|
|
129
116
|
}
|
|
130
117
|
/**
|
|
131
118
|
* Adds proxy configuration to launch options.
|
|
132
119
|
*/
|
|
133
120
|
async _addProxyToLaunchOptions(launchContext) {
|
|
134
|
-
launchContext.launchOptions
|
|
121
|
+
launchContext.launchOptions ??= {};
|
|
135
122
|
const { launchOptions, proxyUrl } = launchContext;
|
|
136
123
|
if (proxyUrl) {
|
|
137
124
|
const url = new URL(proxyUrl);
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/stagehand",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "4.0.0-beta.100",
|
|
4
4
|
"description": "AI-powered web crawling with Stagehand integration for Crawlee - enables natural language browser automation with act(), extract(), and observe() methods.",
|
|
5
5
|
"engines": {
|
|
6
|
-
"node": ">=
|
|
6
|
+
"node": ">=22.0.0"
|
|
7
7
|
},
|
|
8
8
|
"main": "./index.js",
|
|
9
9
|
"module": "./index.mjs",
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
},
|
|
48
48
|
"homepage": "https://crawlee.dev",
|
|
49
49
|
"scripts": {
|
|
50
|
-
"build": "
|
|
50
|
+
"build": "pnpm clean && pnpm compile && pnpm copy",
|
|
51
51
|
"clean": "rimraf ./dist",
|
|
52
52
|
"compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./index.js ./index.mjs",
|
|
53
53
|
"copy": "tsx ../../scripts/copy.ts"
|
|
@@ -56,15 +56,14 @@
|
|
|
56
56
|
"access": "public"
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
|
-
"@apify/
|
|
60
|
-
"@
|
|
61
|
-
"@crawlee/browser": "
|
|
62
|
-
"@crawlee/
|
|
63
|
-
"@crawlee/
|
|
64
|
-
"@crawlee/
|
|
65
|
-
"
|
|
66
|
-
"
|
|
67
|
-
"tslib": "^2.4.0"
|
|
59
|
+
"@apify/timeout": "^0.4.4",
|
|
60
|
+
"@crawlee/browser": "4.0.0-beta.100",
|
|
61
|
+
"@crawlee/browser-pool": "4.0.0-beta.100",
|
|
62
|
+
"@crawlee/core": "4.0.0-beta.100",
|
|
63
|
+
"@crawlee/types": "4.0.0-beta.100",
|
|
64
|
+
"@crawlee/utils": "4.0.0-beta.100",
|
|
65
|
+
"ow": "^2.0.0",
|
|
66
|
+
"tslib": "^2.8.1"
|
|
68
67
|
},
|
|
69
68
|
"peerDependencies": {
|
|
70
69
|
"@browserbasehq/stagehand": "^3.0.0",
|
|
@@ -94,5 +93,5 @@
|
|
|
94
93
|
}
|
|
95
94
|
}
|
|
96
95
|
},
|
|
97
|
-
"gitHead": "
|
|
96
|
+
"gitHead": "109b4c2097f7b9a26bfe761b3c9969dd626f8e38"
|
|
98
97
|
}
|