@crawlee/stagehand 4.0.0-beta.104 → 4.0.0-beta.106
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.mjs +9 -3
- package/internals/stagehand-controller.d.ts +1 -2
- package/internals/stagehand-controller.js +7 -7
- package/internals/stagehand-crawler.d.ts +3 -1
- package/internals/stagehand-crawler.js +1 -1
- package/internals/stagehand-launcher.d.ts +1 -1
- package/internals/stagehand-launcher.js +3 -3
- package/internals/stagehand-plugin.d.ts +5 -5
- package/internals/stagehand-plugin.js +12 -12
- package/package.json +7 -7
package/index.mjs
CHANGED
|
@@ -53,7 +53,6 @@ export const REQUESTS_PERSISTENCE_KEY = mod.REQUESTS_PERSISTENCE_KEY;
|
|
|
53
53
|
export const RecoverableState = mod.RecoverableState;
|
|
54
54
|
export const Request = mod.Request;
|
|
55
55
|
export const RequestHandlerError = mod.RequestHandlerError;
|
|
56
|
-
export const RequestHandlerResult = mod.RequestHandlerResult;
|
|
57
56
|
export const RequestList = mod.RequestList;
|
|
58
57
|
export const RequestManagerTandem = mod.RequestManagerTandem;
|
|
59
58
|
export const RequestQueue = mod.RequestQueue;
|
|
@@ -75,13 +74,14 @@ export const StagehandCrawler = mod.StagehandCrawler;
|
|
|
75
74
|
export const Statistics = mod.Statistics;
|
|
76
75
|
export const StorageInstanceManager = mod.StorageInstanceManager;
|
|
77
76
|
export const StorageStatsTracker = mod.StorageStatsTracker;
|
|
77
|
+
export const StorageTransaction = mod.StorageTransaction;
|
|
78
78
|
export const SystemStatus = mod.SystemStatus;
|
|
79
|
+
export const activeStorageTransaction = mod.activeStorageTransaction;
|
|
79
80
|
export const applyRequestTransform = mod.applyRequestTransform;
|
|
80
81
|
export const assertJsonSerializable = mod.assertJsonSerializable;
|
|
81
82
|
export const bindMethodsToServiceLocator = mod.bindMethodsToServiceLocator;
|
|
82
83
|
export const browserCrawlerEnqueueLinks = mod.browserCrawlerEnqueueLinks;
|
|
83
84
|
export const browserPoolCookieToToughCookie = mod.browserPoolCookieToToughCookie;
|
|
84
|
-
export const checkStorageAccess = mod.checkStorageAccess;
|
|
85
85
|
export const coerceBoolean = mod.coerceBoolean;
|
|
86
86
|
export const coerceNumber = mod.coerceNumber;
|
|
87
87
|
export const constructGlobObjectsFromGlobs = mod.constructGlobObjectsFromGlobs;
|
|
@@ -95,6 +95,8 @@ export const createDualIterable = mod.createDualIterable;
|
|
|
95
95
|
export const createRequestDebugInfo = mod.createRequestDebugInfo;
|
|
96
96
|
export const createRequestOptions = mod.createRequestOptions;
|
|
97
97
|
export const createStagehandRouter = mod.createStagehandRouter;
|
|
98
|
+
export const createStorageTransaction = mod.createStorageTransaction;
|
|
99
|
+
export const currentStorageTransaction = mod.currentStorageTransaction;
|
|
98
100
|
export const defaultRoute = mod.defaultRoute;
|
|
99
101
|
export const deserializeArray = mod.deserializeArray;
|
|
100
102
|
export const enqueueLinks = mod.enqueueLinks;
|
|
@@ -110,14 +112,17 @@ export const inspectValue = mod.inspectValue;
|
|
|
110
112
|
export const log = mod.log;
|
|
111
113
|
export const mergeCookies = mod.mergeCookies;
|
|
112
114
|
export const navigationDeadlineKey = mod.navigationDeadlineKey;
|
|
115
|
+
export const operationRejectedInTransaction = mod.operationRejectedInTransaction;
|
|
113
116
|
export const parseValue = mod.parseValue;
|
|
114
117
|
export const purgeDefaultStorages = mod.purgeDefaultStorages;
|
|
118
|
+
export const rejectOperationInTransaction = mod.rejectOperationInTransaction;
|
|
115
119
|
export const remainingNavigationWindowMillis = mod.remainingNavigationWindowMillis;
|
|
116
120
|
export const resolveBaseUrlForEnqueueLinksFiltering = mod.resolveBaseUrlForEnqueueLinksFiltering;
|
|
117
121
|
export const resolveStorageIdentifier = mod.resolveStorageIdentifier;
|
|
118
122
|
export const serializeArray = mod.serializeArray;
|
|
119
123
|
export const serializeValue = mod.serializeValue;
|
|
120
124
|
export const serviceLocator = mod.serviceLocator;
|
|
125
|
+
export const snapshotValue = mod.snapshotValue;
|
|
121
126
|
export const stagehandUtils = mod.stagehandUtils;
|
|
122
127
|
export const toughCookieToBrowserPoolCookie = mod.toughCookieToBrowserPoolCookie;
|
|
123
128
|
export const tryAbsoluteURL = mod.tryAbsoluteURL;
|
|
@@ -126,4 +131,5 @@ export const useState = mod.useState;
|
|
|
126
131
|
export const validateGlobPattern = mod.validateGlobPattern;
|
|
127
132
|
export const validateUserData = mod.validateUserData;
|
|
128
133
|
export const validators = mod.validators;
|
|
129
|
-
export const
|
|
134
|
+
export const withDirectStorageAccess = mod.withDirectStorageAccess;
|
|
135
|
+
export const withStorageTransaction = mod.withStorageTransaction;
|
|
@@ -18,8 +18,7 @@ import type { StagehandPlugin } from './stagehand-plugin';
|
|
|
18
18
|
* @ignore
|
|
19
19
|
*/
|
|
20
20
|
export declare class StagehandController extends BrowserController<BrowserType, LaunchOptions, PlaywrightBrowser> {
|
|
21
|
-
private
|
|
22
|
-
private readonly stagehandInstances;
|
|
21
|
+
#private;
|
|
23
22
|
constructor(browserPlugin: StagehandPlugin, stagehandInstances: WeakMap<PlaywrightBrowser, Stagehand>);
|
|
24
23
|
/**
|
|
25
24
|
* Gets the Stagehand instance associated with this controller's browser.
|
|
@@ -18,23 +18,23 @@ const utils_1 = require("@crawlee/utils");
|
|
|
18
18
|
* @ignore
|
|
19
19
|
*/
|
|
20
20
|
class StagehandController extends browser_pool_1.BrowserController {
|
|
21
|
-
stagehand = null;
|
|
22
|
-
stagehandInstances;
|
|
21
|
+
#stagehand = null;
|
|
22
|
+
#stagehandInstances;
|
|
23
23
|
constructor(browserPlugin, stagehandInstances) {
|
|
24
24
|
super(browserPlugin);
|
|
25
|
-
this
|
|
25
|
+
this.#stagehandInstances = stagehandInstances;
|
|
26
26
|
}
|
|
27
27
|
/**
|
|
28
28
|
* Gets the Stagehand instance associated with this controller's browser.
|
|
29
29
|
*/
|
|
30
30
|
getStagehand() {
|
|
31
|
-
if (!this
|
|
32
|
-
this
|
|
33
|
-
if (!this
|
|
31
|
+
if (!this.#stagehand) {
|
|
32
|
+
this.#stagehand = this.#stagehandInstances.get(this.browser);
|
|
33
|
+
if (!this.#stagehand) {
|
|
34
34
|
throw new Error('Stagehand instance not found for browser');
|
|
35
35
|
}
|
|
36
36
|
}
|
|
37
|
-
return this
|
|
37
|
+
return this.#stagehand;
|
|
38
38
|
}
|
|
39
39
|
/**
|
|
40
40
|
* Creates a new page using the browser's default context.
|
|
@@ -381,6 +381,8 @@ export declare class StagehandCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
381
381
|
retryOnBlocked: import("ow").BooleanPredicate & import("ow").BasePredicate<boolean | undefined>;
|
|
382
382
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
383
383
|
respectRobotsTxtFile: import("ow").AnyPredicate<boolean | object>;
|
|
384
|
+
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
385
|
+
transactionalStorage: import("ow").BasePredicate<boolean | Partial<import("@crawlee/browser").StorageWritePolicy> | undefined>;
|
|
384
386
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
385
387
|
onSkippedRequest: import("ow").Predicate<Function> & import("ow").BasePredicate<Function | undefined>;
|
|
386
388
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -431,7 +433,7 @@ export declare class StagehandCrawler<ContextExtension = Dictionary<never>, Exte
|
|
|
431
433
|
* Navigation handler for Stagehand crawler.
|
|
432
434
|
* Uses standard Playwright navigation.
|
|
433
435
|
*/
|
|
434
|
-
protected
|
|
436
|
+
protected navigationHandler(crawlingContext: StagehandCrawlingContext, gotoOptions: StagehandGotoOptions): Promise<Response | null>;
|
|
435
437
|
}
|
|
436
438
|
/**
|
|
437
439
|
* Creates a new router for StagehandCrawler with type-safe route handlers.
|
|
@@ -123,7 +123,7 @@ class StagehandCrawler extends browser_1.BrowserCrawler {
|
|
|
123
123
|
* Navigation handler for Stagehand crawler.
|
|
124
124
|
* Uses standard Playwright navigation.
|
|
125
125
|
*/
|
|
126
|
-
async
|
|
126
|
+
async navigationHandler(crawlingContext, gotoOptions) {
|
|
127
127
|
// Use standard page.goto for navigation
|
|
128
128
|
return crawlingContext.page.goto(crawlingContext.request.url, gotoOptions);
|
|
129
129
|
}
|
|
@@ -54,6 +54,7 @@ export interface StagehandLaunchContext extends BrowserLaunchContext<LaunchOptio
|
|
|
54
54
|
* @ignore
|
|
55
55
|
*/
|
|
56
56
|
export declare class StagehandLauncher extends BrowserLauncher<StagehandPlugin> {
|
|
57
|
+
#private;
|
|
57
58
|
readonly configuration: Configuration;
|
|
58
59
|
protected static optionsShape: {
|
|
59
60
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
@@ -79,7 +80,6 @@ export declare class StagehandLauncher extends BrowserLauncher<StagehandPlugin>
|
|
|
79
80
|
// @ts-ignore optional peer dependency or compatibility with es2022
|
|
80
81
|
userAgent: import("ow").StringPredicate & import("ow").BasePredicate<string | undefined>;
|
|
81
82
|
};
|
|
82
|
-
private readonly stagehandOptions;
|
|
83
83
|
/**
|
|
84
84
|
* All StagehandLauncher parameters are passed via the launchContext object.
|
|
85
85
|
*/
|
|
@@ -19,7 +19,7 @@ class StagehandLauncher extends browser_1.BrowserLauncher {
|
|
|
19
19
|
launchContextOptions: ow_1.default.optional.object,
|
|
20
20
|
stagehandOptions: ow_1.default.optional.object,
|
|
21
21
|
};
|
|
22
|
-
stagehandOptions;
|
|
22
|
+
#stagehandOptions;
|
|
23
23
|
/**
|
|
24
24
|
* All StagehandLauncher parameters are passed via the launchContext object.
|
|
25
25
|
*/
|
|
@@ -37,7 +37,7 @@ class StagehandLauncher extends browser_1.BrowserLauncher {
|
|
|
37
37
|
launcher,
|
|
38
38
|
}, configuration);
|
|
39
39
|
this.configuration = configuration;
|
|
40
|
-
this
|
|
40
|
+
this.#stagehandOptions = {
|
|
41
41
|
env: 'LOCAL',
|
|
42
42
|
model: 'openai/gpt-4.1-mini',
|
|
43
43
|
...stagehandOptions,
|
|
@@ -52,7 +52,7 @@ class StagehandLauncher extends browser_1.BrowserLauncher {
|
|
|
52
52
|
...this.otherLaunchContextProps,
|
|
53
53
|
proxyUrl: this.proxyUrl,
|
|
54
54
|
launchOptions: this.createLaunchOptions(),
|
|
55
|
-
stagehandOptions: this
|
|
55
|
+
stagehandOptions: this.#stagehandOptions, // Set AFTER to override any unresolved options
|
|
56
56
|
});
|
|
57
57
|
}
|
|
58
58
|
}
|
|
@@ -27,8 +27,8 @@ export interface StagehandPluginOptions extends BrowserPluginOptions<LaunchOptio
|
|
|
27
27
|
* - Some fingerprinting options may not be fully applied (Stagehand controls browser launch)
|
|
28
28
|
*/
|
|
29
29
|
export declare class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOptions, PlaywrightBrowser> {
|
|
30
|
+
#private;
|
|
30
31
|
readonly stagehandOptions: StagehandOptions;
|
|
31
|
-
private readonly stagehandInstances;
|
|
32
32
|
constructor(library: BrowserType, options?: StagehandPluginOptions);
|
|
33
33
|
/**
|
|
34
34
|
* Launches a browser using Stagehand and connects Playwright to it via CDP.
|
|
@@ -37,7 +37,7 @@ export declare class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOp
|
|
|
37
37
|
/**
|
|
38
38
|
* Cleans up Stagehand instance when browser disconnects.
|
|
39
39
|
*/
|
|
40
|
-
private
|
|
40
|
+
private cleanupStagehand;
|
|
41
41
|
/**
|
|
42
42
|
* Creates a controller for the Stagehand browser.
|
|
43
43
|
*/
|
|
@@ -45,15 +45,15 @@ export declare class StagehandPlugin extends BrowserPlugin<BrowserType, LaunchOp
|
|
|
45
45
|
/**
|
|
46
46
|
* Adds proxy configuration to launch options.
|
|
47
47
|
*/
|
|
48
|
-
protected
|
|
48
|
+
protected addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
|
|
49
49
|
/**
|
|
50
50
|
* Determines if this is a Chromium-based browser.
|
|
51
51
|
*/
|
|
52
|
-
protected
|
|
52
|
+
protected isChromiumBasedBrowser(): boolean;
|
|
53
53
|
/**
|
|
54
54
|
* Augments launch errors with helpful context.
|
|
55
55
|
*/
|
|
56
|
-
private
|
|
56
|
+
private augmentLaunchError;
|
|
57
57
|
/**
|
|
58
58
|
* Gets the Stagehand instance for a given browser.
|
|
59
59
|
*/
|
|
@@ -22,7 +22,7 @@ const stagehand_controller_1 = require("./stagehand-controller");
|
|
|
22
22
|
*/
|
|
23
23
|
class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
24
24
|
stagehandOptions;
|
|
25
|
-
stagehandInstances = new WeakMap();
|
|
25
|
+
#stagehandInstances = new WeakMap();
|
|
26
26
|
constructor(library, options = {}) {
|
|
27
27
|
super(library, options);
|
|
28
28
|
this.stagehandOptions = options.stagehandOptions ?? {};
|
|
@@ -76,10 +76,10 @@ class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
|
76
76
|
}
|
|
77
77
|
const browser = await playwright_1.chromium.connectOverCDP(cdpUrl);
|
|
78
78
|
// Store the Stagehand instance for AI operations
|
|
79
|
-
this
|
|
79
|
+
this.#stagehandInstances.set(browser, stagehand);
|
|
80
80
|
// Handle browser disconnection - cleanup both Stagehand and anonymized proxy
|
|
81
81
|
browser.on('disconnected', async () => {
|
|
82
|
-
await this.
|
|
82
|
+
await this.cleanupStagehand(browser);
|
|
83
83
|
await closeAnonymizedProxy();
|
|
84
84
|
});
|
|
85
85
|
return browser;
|
|
@@ -88,7 +88,7 @@ class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
|
88
88
|
// Clean up on failure
|
|
89
89
|
await stagehand.close().catch(() => { });
|
|
90
90
|
await closeAnonymizedProxy();
|
|
91
|
-
const augmentedError = this.
|
|
91
|
+
const augmentedError = this.augmentLaunchError(error, launchContext);
|
|
92
92
|
core_1.serviceLocator.getLogger().error('Stagehand browser launch failed', { message: augmentedError.message });
|
|
93
93
|
throw augmentedError;
|
|
94
94
|
}
|
|
@@ -96,8 +96,8 @@ class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
|
96
96
|
/**
|
|
97
97
|
* Cleans up Stagehand instance when browser disconnects.
|
|
98
98
|
*/
|
|
99
|
-
async
|
|
100
|
-
const stagehand = this
|
|
99
|
+
async cleanupStagehand(browser) {
|
|
100
|
+
const stagehand = this.#stagehandInstances.get(browser);
|
|
101
101
|
if (stagehand) {
|
|
102
102
|
try {
|
|
103
103
|
await stagehand.close();
|
|
@@ -105,19 +105,19 @@ class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
|
105
105
|
catch {
|
|
106
106
|
// Ignore cleanup errors
|
|
107
107
|
}
|
|
108
|
-
this
|
|
108
|
+
this.#stagehandInstances.delete(browser);
|
|
109
109
|
}
|
|
110
110
|
}
|
|
111
111
|
/**
|
|
112
112
|
* Creates a controller for the Stagehand browser.
|
|
113
113
|
*/
|
|
114
114
|
createController() {
|
|
115
|
-
return new stagehand_controller_1.StagehandController(this, this
|
|
115
|
+
return new stagehand_controller_1.StagehandController(this, this.#stagehandInstances);
|
|
116
116
|
}
|
|
117
117
|
/**
|
|
118
118
|
* Adds proxy configuration to launch options.
|
|
119
119
|
*/
|
|
120
|
-
async
|
|
120
|
+
async addProxyToLaunchOptions(launchContext) {
|
|
121
121
|
launchContext.launchOptions ??= {};
|
|
122
122
|
const { launchOptions, proxyUrl } = launchContext;
|
|
123
123
|
if (proxyUrl) {
|
|
@@ -132,14 +132,14 @@ class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
|
132
132
|
/**
|
|
133
133
|
* Determines if this is a Chromium-based browser.
|
|
134
134
|
*/
|
|
135
|
-
|
|
135
|
+
isChromiumBasedBrowser() {
|
|
136
136
|
const name = this.library?.name?.();
|
|
137
137
|
return name === 'chromium';
|
|
138
138
|
}
|
|
139
139
|
/**
|
|
140
140
|
* Augments launch errors with helpful context.
|
|
141
141
|
*/
|
|
142
|
-
|
|
142
|
+
augmentLaunchError(error, launchContext) {
|
|
143
143
|
const message = error instanceof Error ? error.message : String(error);
|
|
144
144
|
const model = this.stagehandOptions.model;
|
|
145
145
|
let helpText = '';
|
|
@@ -163,7 +163,7 @@ class StagehandPlugin extends browser_pool_1.BrowserPlugin {
|
|
|
163
163
|
* Gets the Stagehand instance for a given browser.
|
|
164
164
|
*/
|
|
165
165
|
getStagehandForBrowser(browser) {
|
|
166
|
-
return this
|
|
166
|
+
return this.#stagehandInstances.get(browser);
|
|
167
167
|
}
|
|
168
168
|
}
|
|
169
169
|
exports.StagehandPlugin = StagehandPlugin;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crawlee/stagehand",
|
|
3
|
-
"version": "4.0.0-beta.
|
|
3
|
+
"version": "4.0.0-beta.106",
|
|
4
4
|
"description": "AI-powered web crawling with Stagehand integration for Crawlee - enables natural language browser automation with act(), extract(), and observe() methods.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0"
|
|
@@ -57,11 +57,11 @@
|
|
|
57
57
|
},
|
|
58
58
|
"dependencies": {
|
|
59
59
|
"@apify/timeout": "^0.4.4",
|
|
60
|
-
"@crawlee/browser": "4.0.0-beta.
|
|
61
|
-
"@crawlee/browser-pool": "4.0.0-beta.
|
|
62
|
-
"@crawlee/core": "4.0.0-beta.
|
|
63
|
-
"@crawlee/types": "4.0.0-beta.
|
|
64
|
-
"@crawlee/utils": "4.0.0-beta.
|
|
60
|
+
"@crawlee/browser": "4.0.0-beta.106",
|
|
61
|
+
"@crawlee/browser-pool": "4.0.0-beta.106",
|
|
62
|
+
"@crawlee/core": "4.0.0-beta.106",
|
|
63
|
+
"@crawlee/types": "4.0.0-beta.106",
|
|
64
|
+
"@crawlee/utils": "4.0.0-beta.106",
|
|
65
65
|
"ow": "^2.0.0",
|
|
66
66
|
"tslib": "^2.8.1"
|
|
67
67
|
},
|
|
@@ -93,5 +93,5 @@
|
|
|
93
93
|
}
|
|
94
94
|
}
|
|
95
95
|
},
|
|
96
|
-
"gitHead": "
|
|
96
|
+
"gitHead": "c622f1fc65e65221ea245817c58ecc0ffb4a5cb0"
|
|
97
97
|
}
|