@browserbasehq/stagehand 1.1.1 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dom/build/index.js +186 -46
- package/dist/dom/build/process.js +186 -46
- package/dist/dom/build/xpathUtils.js +482 -0
- package/dist/index.d.ts +6 -6
- package/dist/index.js +1398 -705
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -8,6 +8,7 @@ var __getOwnPropSymbols = Object.getOwnPropertySymbols;
|
|
|
8
8
|
var __getProtoOf = Object.getPrototypeOf;
|
|
9
9
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
10
10
|
var __propIsEnum = Object.prototype.propertyIsEnumerable;
|
|
11
|
+
var __reflectGet = Reflect.get;
|
|
11
12
|
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
12
13
|
var __spreadValues = (a, b) => {
|
|
13
14
|
for (var prop in b || (b = {}))
|
|
@@ -54,6 +55,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
54
55
|
mod
|
|
55
56
|
));
|
|
56
57
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
58
|
+
var __superGet = (cls, obj, key) => __reflectGet(__getProtoOf(cls), key, obj);
|
|
57
59
|
var __async = (__this, __arguments, generator) => {
|
|
58
60
|
return new Promise((resolve, reject) => {
|
|
59
61
|
var fulfilled = (value) => {
|
|
@@ -82,7 +84,6 @@ __export(lib_exports, {
|
|
|
82
84
|
});
|
|
83
85
|
module.exports = __toCommonJS(lib_exports);
|
|
84
86
|
var import_test = require("@playwright/test");
|
|
85
|
-
var import_crypto = __toESM(require("crypto"));
|
|
86
87
|
var import_fs2 = __toESM(require("fs"));
|
|
87
88
|
var import_sdk2 = require("@browserbasehq/sdk");
|
|
88
89
|
|
|
@@ -95,6 +96,7 @@ You are given:
|
|
|
95
96
|
1. the user's overall goal
|
|
96
97
|
2. the steps that you've taken so far
|
|
97
98
|
3. a list of active DOM elements in this chunk to consider to get closer to the goal.
|
|
99
|
+
4. Optionally, a list of variable names that the user has provided that you may use to accomplish the goal. To use the variables, you must use the special <|VARIABLE_NAME|> syntax.
|
|
98
100
|
|
|
99
101
|
You have 2 tools that you can call: doAction, and skipSection. Do action only performs Playwright actions. Do not perform any other actions.
|
|
100
102
|
|
|
@@ -156,8 +158,8 @@ function buildActSystemPrompt() {
|
|
|
156
158
|
content: actSystemPrompt
|
|
157
159
|
};
|
|
158
160
|
}
|
|
159
|
-
function buildActUserPrompt(action, steps = "None", domElements) {
|
|
160
|
-
|
|
161
|
+
function buildActUserPrompt(action, steps = "None", domElements, variables) {
|
|
162
|
+
let actUserPrompt = `
|
|
161
163
|
# My Goal
|
|
162
164
|
${action}
|
|
163
165
|
|
|
@@ -167,6 +169,12 @@ ${steps}
|
|
|
167
169
|
# Current Active Dom Elements
|
|
168
170
|
${domElements}
|
|
169
171
|
`;
|
|
172
|
+
if (variables) {
|
|
173
|
+
actUserPrompt += `
|
|
174
|
+
# Variables
|
|
175
|
+
${Object.entries(variables).map(([key, value]) => `<|${key.toUpperCase()}|>`).join("\n")}
|
|
176
|
+
`;
|
|
177
|
+
}
|
|
170
178
|
return {
|
|
171
179
|
role: "user",
|
|
172
180
|
content: actUserPrompt
|
|
@@ -392,6 +400,14 @@ function verifyActCompletion(_0) {
|
|
|
392
400
|
return response.completed;
|
|
393
401
|
});
|
|
394
402
|
}
|
|
403
|
+
function fillInVariables(text, variables) {
|
|
404
|
+
let processedText = text;
|
|
405
|
+
Object.entries(variables).forEach(([key, value]) => {
|
|
406
|
+
const placeholder = `<|${key.toUpperCase()}|>`;
|
|
407
|
+
processedText = processedText.replace(placeholder, value);
|
|
408
|
+
});
|
|
409
|
+
return processedText;
|
|
410
|
+
}
|
|
395
411
|
function act(_0) {
|
|
396
412
|
return __async(this, arguments, function* ({
|
|
397
413
|
action,
|
|
@@ -402,12 +418,13 @@ function act(_0) {
|
|
|
402
418
|
screenshot,
|
|
403
419
|
retries = 0,
|
|
404
420
|
logger,
|
|
405
|
-
requestId
|
|
421
|
+
requestId,
|
|
422
|
+
variables
|
|
406
423
|
}) {
|
|
407
424
|
const llmClient = llmProvider.getClient(modelName, requestId);
|
|
408
425
|
const messages = [
|
|
409
426
|
buildActSystemPrompt(),
|
|
410
|
-
buildActUserPrompt(action, steps, domElements)
|
|
427
|
+
buildActUserPrompt(action, steps, domElements, variables)
|
|
411
428
|
];
|
|
412
429
|
const response = yield llmClient.createChatCompletion({
|
|
413
430
|
model: modelName,
|
|
@@ -596,7 +613,18 @@ var OpenAIClient = class {
|
|
|
596
613
|
if (this.enableCaching) {
|
|
597
614
|
const cachedResponse = yield this.cache.get(cacheOptions, this.requestId);
|
|
598
615
|
if (cachedResponse) {
|
|
616
|
+
this.logger({
|
|
617
|
+
category: "llm_cache",
|
|
618
|
+
message: `LLM Cache hit - returning cached response`,
|
|
619
|
+
level: 1
|
|
620
|
+
});
|
|
599
621
|
return cachedResponse;
|
|
622
|
+
} else {
|
|
623
|
+
this.logger({
|
|
624
|
+
category: "llm_cache",
|
|
625
|
+
message: `LLM Cache miss - no cached response found`,
|
|
626
|
+
level: 1
|
|
627
|
+
});
|
|
600
628
|
}
|
|
601
629
|
}
|
|
602
630
|
if (options.image) {
|
|
@@ -673,7 +701,18 @@ var AnthropicClient = class {
|
|
|
673
701
|
if (this.enableCaching) {
|
|
674
702
|
const cachedResponse = yield this.cache.get(cacheOptions, this.requestId);
|
|
675
703
|
if (cachedResponse) {
|
|
704
|
+
this.logger({
|
|
705
|
+
category: "llm_cache",
|
|
706
|
+
message: `LLM Cache hit - returning cached response`,
|
|
707
|
+
level: 1
|
|
708
|
+
});
|
|
676
709
|
return cachedResponse;
|
|
710
|
+
} else {
|
|
711
|
+
this.logger({
|
|
712
|
+
category: "llm_cache",
|
|
713
|
+
message: `LLM Cache miss - no cached response found`,
|
|
714
|
+
level: 1
|
|
715
|
+
});
|
|
677
716
|
}
|
|
678
717
|
}
|
|
679
718
|
const systemMessage = options.messages.find((msg) => msg.role === "system");
|
|
@@ -801,24 +840,24 @@ var AnthropicClient = class {
|
|
|
801
840
|
}
|
|
802
841
|
};
|
|
803
842
|
|
|
804
|
-
// lib/
|
|
843
|
+
// lib/cache/BaseCache.ts
|
|
805
844
|
var fs = __toESM(require("fs"));
|
|
806
845
|
var path = __toESM(require("path"));
|
|
807
846
|
var crypto = __toESM(require("crypto"));
|
|
808
|
-
var
|
|
809
|
-
constructor(logger, cacheDir = path.join(process.cwd(), "tmp", ".cache"), cacheFile = "
|
|
847
|
+
var BaseCache = class {
|
|
848
|
+
constructor(logger, cacheDir = path.join(process.cwd(), "tmp", ".cache"), cacheFile = "cache.json") {
|
|
810
849
|
this.CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
811
850
|
// 1 week in milliseconds
|
|
812
851
|
this.CLEANUP_PROBABILITY = 0.01;
|
|
813
|
-
// 1% chance
|
|
814
852
|
this.LOCK_TIMEOUT_MS = 1e3;
|
|
815
|
-
this.
|
|
816
|
-
this.
|
|
817
|
-
|
|
853
|
+
this.lockAcquired = false;
|
|
854
|
+
this.lockAcquireFailures = 0;
|
|
855
|
+
// Added for request ID tracking
|
|
856
|
+
this.requestIdToUsedHashes = {};
|
|
818
857
|
this.logger = logger;
|
|
819
858
|
this.cacheDir = cacheDir;
|
|
820
859
|
this.cacheFile = path.join(cacheDir, cacheFile);
|
|
821
|
-
this.lockFile = path.join(cacheDir, "
|
|
860
|
+
this.lockFile = path.join(cacheDir, "cache.lock");
|
|
822
861
|
this.ensureCacheDirectory();
|
|
823
862
|
this.setupProcessHandlers();
|
|
824
863
|
}
|
|
@@ -832,11 +871,11 @@ var LLMCache = class {
|
|
|
832
871
|
process.on("SIGTERM", releaseLockAndExit);
|
|
833
872
|
process.on("uncaughtException", (err) => {
|
|
834
873
|
this.logger({
|
|
835
|
-
category: "
|
|
874
|
+
category: "base_cache",
|
|
836
875
|
message: `Uncaught exception: ${err}`,
|
|
837
876
|
level: 2
|
|
838
877
|
});
|
|
839
|
-
if (this.
|
|
878
|
+
if (this.lockAcquired) {
|
|
840
879
|
releaseLockAndExit();
|
|
841
880
|
}
|
|
842
881
|
});
|
|
@@ -844,6 +883,11 @@ var LLMCache = class {
|
|
|
844
883
|
ensureCacheDirectory() {
|
|
845
884
|
if (!fs.existsSync(this.cacheDir)) {
|
|
846
885
|
fs.mkdirSync(this.cacheDir, { recursive: true });
|
|
886
|
+
this.logger({
|
|
887
|
+
category: "base_cache",
|
|
888
|
+
message: `Created cache directory at ${this.cacheDir}`,
|
|
889
|
+
level: 1
|
|
890
|
+
});
|
|
847
891
|
}
|
|
848
892
|
}
|
|
849
893
|
createHash(data) {
|
|
@@ -862,25 +906,35 @@ var LLMCache = class {
|
|
|
862
906
|
const lockAge = Date.now() - fs.statSync(this.lockFile).mtimeMs;
|
|
863
907
|
if (lockAge > this.LOCK_TIMEOUT_MS) {
|
|
864
908
|
fs.unlinkSync(this.lockFile);
|
|
909
|
+
this.logger({
|
|
910
|
+
category: "base_cache",
|
|
911
|
+
message: "Stale lock file removed",
|
|
912
|
+
level: 1
|
|
913
|
+
});
|
|
865
914
|
}
|
|
866
915
|
}
|
|
867
916
|
fs.writeFileSync(this.lockFile, process.pid.toString(), { flag: "wx" });
|
|
868
|
-
this.
|
|
869
|
-
this.
|
|
917
|
+
this.lockAcquireFailures = 0;
|
|
918
|
+
this.lockAcquired = true;
|
|
919
|
+
this.logger({
|
|
920
|
+
category: "base_cache",
|
|
921
|
+
message: "Lock acquired",
|
|
922
|
+
level: 1
|
|
923
|
+
});
|
|
870
924
|
return true;
|
|
871
925
|
} catch (error) {
|
|
872
926
|
yield this.sleep(5);
|
|
873
927
|
}
|
|
874
928
|
}
|
|
875
929
|
this.logger({
|
|
876
|
-
category: "
|
|
930
|
+
category: "base_cache",
|
|
877
931
|
message: "Failed to acquire lock after timeout",
|
|
878
932
|
level: 2
|
|
879
933
|
});
|
|
880
|
-
this.
|
|
881
|
-
if (this.
|
|
934
|
+
this.lockAcquireFailures++;
|
|
935
|
+
if (this.lockAcquireFailures >= 3) {
|
|
882
936
|
this.logger({
|
|
883
|
-
category: "
|
|
937
|
+
category: "base_cache",
|
|
884
938
|
message: "Failed to acquire lock 3 times in a row. Releasing lock manually.",
|
|
885
939
|
level: 1
|
|
886
940
|
});
|
|
@@ -893,111 +947,125 @@ var LLMCache = class {
|
|
|
893
947
|
try {
|
|
894
948
|
if (fs.existsSync(this.lockFile)) {
|
|
895
949
|
fs.unlinkSync(this.lockFile);
|
|
950
|
+
this.logger({
|
|
951
|
+
category: "base_cache",
|
|
952
|
+
message: "Lock released",
|
|
953
|
+
level: 1
|
|
954
|
+
});
|
|
896
955
|
}
|
|
897
|
-
this.
|
|
956
|
+
this.lockAcquired = false;
|
|
898
957
|
} catch (error) {
|
|
899
958
|
this.logger({
|
|
900
|
-
category: "
|
|
959
|
+
category: "base_cache",
|
|
901
960
|
message: `Error releasing lock: ${error}`,
|
|
902
961
|
level: 2
|
|
903
962
|
});
|
|
904
963
|
}
|
|
905
964
|
}
|
|
965
|
+
/**
|
|
966
|
+
* Cleans up stale cache entries that exceed the maximum age.
|
|
967
|
+
*/
|
|
968
|
+
cleanupStaleEntries() {
|
|
969
|
+
return __async(this, null, function* () {
|
|
970
|
+
if (!(yield this.acquireLock())) {
|
|
971
|
+
this.logger({
|
|
972
|
+
category: "llm_cache",
|
|
973
|
+
message: "Failed to acquire lock for cleanup",
|
|
974
|
+
level: 2
|
|
975
|
+
});
|
|
976
|
+
return;
|
|
977
|
+
}
|
|
978
|
+
try {
|
|
979
|
+
const cache = this.readCache();
|
|
980
|
+
const now = Date.now();
|
|
981
|
+
let entriesRemoved = 0;
|
|
982
|
+
for (const [hash, entry] of Object.entries(cache)) {
|
|
983
|
+
if (now - entry.timestamp > this.CACHE_MAX_AGE_MS) {
|
|
984
|
+
delete cache[hash];
|
|
985
|
+
entriesRemoved++;
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
if (entriesRemoved > 0) {
|
|
989
|
+
this.writeCache(cache);
|
|
990
|
+
this.logger({
|
|
991
|
+
category: "llm_cache",
|
|
992
|
+
message: `Cleaned up ${entriesRemoved} stale cache entries`,
|
|
993
|
+
level: 1
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
} catch (error) {
|
|
997
|
+
this.logger({
|
|
998
|
+
category: "llm_cache",
|
|
999
|
+
message: `Error during cache cleanup: ${error}`,
|
|
1000
|
+
level: 2
|
|
1001
|
+
});
|
|
1002
|
+
} finally {
|
|
1003
|
+
this.releaseLock();
|
|
1004
|
+
}
|
|
1005
|
+
});
|
|
1006
|
+
}
|
|
906
1007
|
readCache() {
|
|
907
1008
|
if (fs.existsSync(this.cacheFile)) {
|
|
908
|
-
|
|
1009
|
+
try {
|
|
1010
|
+
const data = fs.readFileSync(this.cacheFile, "utf-8");
|
|
1011
|
+
return JSON.parse(data);
|
|
1012
|
+
} catch (error) {
|
|
1013
|
+
this.logger({
|
|
1014
|
+
category: "base_cache",
|
|
1015
|
+
message: `Error reading cache file: ${error}. Resetting cache.`,
|
|
1016
|
+
level: 1
|
|
1017
|
+
});
|
|
1018
|
+
this.resetCache();
|
|
1019
|
+
return {};
|
|
1020
|
+
}
|
|
909
1021
|
}
|
|
910
1022
|
return {};
|
|
911
1023
|
}
|
|
912
1024
|
writeCache(cache) {
|
|
913
1025
|
try {
|
|
914
|
-
if (Math.random() < this.CLEANUP_PROBABILITY) {
|
|
915
|
-
this.cleanupStaleEntries(cache);
|
|
916
|
-
}
|
|
917
1026
|
fs.writeFileSync(this.cacheFile, JSON.stringify(cache, null, 2));
|
|
918
|
-
} finally {
|
|
919
|
-
this.releaseLock();
|
|
920
|
-
}
|
|
921
|
-
}
|
|
922
|
-
cleanupStaleEntries(cache) {
|
|
923
|
-
if (!this.acquireLock()) {
|
|
924
|
-
this.logger({
|
|
925
|
-
category: "llm_cache",
|
|
926
|
-
message: "Failed to acquire lock for cleaning up cache",
|
|
927
|
-
level: 2
|
|
928
|
-
});
|
|
929
|
-
return;
|
|
930
|
-
}
|
|
931
|
-
try {
|
|
932
|
-
const now = Date.now();
|
|
933
|
-
let entriesRemoved = 0;
|
|
934
|
-
for (const [hash, entry] of Object.entries(cache)) {
|
|
935
|
-
if (now - entry.timestamp > this.CACHE_MAX_AGE_MS) {
|
|
936
|
-
delete cache[hash];
|
|
937
|
-
entriesRemoved++;
|
|
938
|
-
}
|
|
939
|
-
}
|
|
940
|
-
if (entriesRemoved > 0) {
|
|
941
|
-
this.logger({
|
|
942
|
-
category: "llm_cache",
|
|
943
|
-
message: `Cleaned up ${entriesRemoved} stale cache entries`,
|
|
944
|
-
level: 1
|
|
945
|
-
});
|
|
946
|
-
}
|
|
947
|
-
} catch (error) {
|
|
948
1027
|
this.logger({
|
|
949
|
-
category: "
|
|
950
|
-
message:
|
|
1028
|
+
category: "base_cache",
|
|
1029
|
+
message: "Cache written to file",
|
|
951
1030
|
level: 1
|
|
952
1031
|
});
|
|
953
|
-
}
|
|
954
|
-
this.releaseLock();
|
|
955
|
-
}
|
|
956
|
-
}
|
|
957
|
-
resetCache() {
|
|
958
|
-
if (!this.acquireLock()) {
|
|
1032
|
+
} catch (error) {
|
|
959
1033
|
this.logger({
|
|
960
|
-
category: "
|
|
961
|
-
message:
|
|
1034
|
+
category: "base_cache",
|
|
1035
|
+
message: `Error writing cache file: ${error}`,
|
|
962
1036
|
level: 2
|
|
963
1037
|
});
|
|
964
|
-
return;
|
|
965
|
-
}
|
|
966
|
-
try {
|
|
967
|
-
this.ensureCacheDirectory();
|
|
968
|
-
fs.writeFileSync(this.cacheFile, "{}");
|
|
969
1038
|
} finally {
|
|
970
1039
|
this.releaseLock();
|
|
971
1040
|
}
|
|
972
1041
|
}
|
|
973
|
-
|
|
1042
|
+
/**
|
|
1043
|
+
* Retrieves data from the cache based on the provided options.
|
|
1044
|
+
* @param hashObj - The options used to generate the cache key.
|
|
1045
|
+
* @param requestId - The identifier for the current request.
|
|
1046
|
+
* @returns The cached data if available, otherwise null.
|
|
1047
|
+
*/
|
|
1048
|
+
get(hashObj, requestId) {
|
|
974
1049
|
return __async(this, null, function* () {
|
|
975
|
-
var _a, _b;
|
|
976
1050
|
if (!(yield this.acquireLock())) {
|
|
977
1051
|
this.logger({
|
|
978
|
-
category: "
|
|
1052
|
+
category: "base_cache",
|
|
979
1053
|
message: "Failed to acquire lock for getting cache",
|
|
980
1054
|
level: 2
|
|
981
1055
|
});
|
|
982
1056
|
return null;
|
|
983
1057
|
}
|
|
984
1058
|
try {
|
|
985
|
-
const hash = this.createHash(
|
|
1059
|
+
const hash = this.createHash(hashObj);
|
|
986
1060
|
const cache = this.readCache();
|
|
987
1061
|
if (cache[hash]) {
|
|
988
|
-
this.
|
|
989
|
-
|
|
990
|
-
message: "Cache hit",
|
|
991
|
-
level: 1
|
|
992
|
-
});
|
|
993
|
-
(_b = (_a = this.request_id_to_used_hashes)[requestId]) != null ? _b : _a[requestId] = [];
|
|
994
|
-
this.request_id_to_used_hashes[requestId].push(hash);
|
|
995
|
-
return cache[hash].response;
|
|
1062
|
+
this.trackRequestIdUsage(requestId, hash);
|
|
1063
|
+
return cache[hash].data;
|
|
996
1064
|
}
|
|
997
1065
|
return null;
|
|
998
1066
|
} catch (error) {
|
|
999
1067
|
this.logger({
|
|
1000
|
-
category: "
|
|
1068
|
+
category: "base_cache",
|
|
1001
1069
|
message: `Error getting cache: ${error}. Resetting cache.`,
|
|
1002
1070
|
level: 1
|
|
1003
1071
|
});
|
|
@@ -1008,82 +1076,189 @@ var LLMCache = class {
|
|
|
1008
1076
|
}
|
|
1009
1077
|
});
|
|
1010
1078
|
}
|
|
1011
|
-
|
|
1079
|
+
/**
|
|
1080
|
+
* Stores data in the cache based on the provided options and requestId.
|
|
1081
|
+
* @param hashObj - The options used to generate the cache key.
|
|
1082
|
+
* @param data - The data to be cached.
|
|
1083
|
+
* @param requestId - The identifier for the cache entry.
|
|
1084
|
+
*/
|
|
1085
|
+
set(hashObj, data, requestId) {
|
|
1012
1086
|
return __async(this, null, function* () {
|
|
1013
|
-
var _a;
|
|
1014
1087
|
if (!(yield this.acquireLock())) {
|
|
1015
1088
|
this.logger({
|
|
1016
|
-
category: "
|
|
1017
|
-
message: "Failed to acquire lock for
|
|
1089
|
+
category: "base_cache",
|
|
1090
|
+
message: "Failed to acquire lock for setting cache",
|
|
1018
1091
|
level: 2
|
|
1019
1092
|
});
|
|
1020
1093
|
return;
|
|
1021
1094
|
}
|
|
1022
1095
|
try {
|
|
1096
|
+
const hash = this.createHash(hashObj);
|
|
1023
1097
|
const cache = this.readCache();
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
}
|
|
1030
|
-
}
|
|
1031
|
-
this.logger({
|
|
1032
|
-
category: "llm_cache",
|
|
1033
|
-
message: `Deleted ${entriesRemoved.length} cache entries for requestId ${requestId}`,
|
|
1034
|
-
level: 1
|
|
1035
|
-
});
|
|
1098
|
+
cache[hash] = {
|
|
1099
|
+
data,
|
|
1100
|
+
timestamp: Date.now(),
|
|
1101
|
+
requestId
|
|
1102
|
+
};
|
|
1036
1103
|
this.writeCache(cache);
|
|
1037
|
-
|
|
1104
|
+
this.trackRequestIdUsage(requestId, hash);
|
|
1105
|
+
} catch (error) {
|
|
1038
1106
|
this.logger({
|
|
1039
|
-
category: "
|
|
1040
|
-
message: `Error
|
|
1107
|
+
category: "base_cache",
|
|
1108
|
+
message: `Error setting cache: ${error}. Resetting cache.`,
|
|
1041
1109
|
level: 1
|
|
1042
1110
|
});
|
|
1111
|
+
this.resetCache();
|
|
1043
1112
|
} finally {
|
|
1044
1113
|
this.releaseLock();
|
|
1114
|
+
if (Math.random() < this.CLEANUP_PROBABILITY) {
|
|
1115
|
+
this.cleanupStaleEntries();
|
|
1116
|
+
}
|
|
1045
1117
|
}
|
|
1046
1118
|
});
|
|
1047
1119
|
}
|
|
1048
|
-
|
|
1120
|
+
delete(hashObj) {
|
|
1049
1121
|
return __async(this, null, function* () {
|
|
1050
|
-
var _a, _b;
|
|
1051
1122
|
if (!(yield this.acquireLock())) {
|
|
1052
1123
|
this.logger({
|
|
1053
|
-
category: "
|
|
1054
|
-
message: "Failed to acquire lock for
|
|
1124
|
+
category: "base_cache",
|
|
1125
|
+
message: "Failed to acquire lock for removing cache entry",
|
|
1055
1126
|
level: 2
|
|
1056
1127
|
});
|
|
1057
1128
|
return;
|
|
1058
1129
|
}
|
|
1059
1130
|
try {
|
|
1060
|
-
const hash = this.createHash(
|
|
1131
|
+
const hash = this.createHash(hashObj);
|
|
1061
1132
|
const cache = this.readCache();
|
|
1062
|
-
cache[hash]
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1133
|
+
if (cache[hash]) {
|
|
1134
|
+
delete cache[hash];
|
|
1135
|
+
this.writeCache(cache);
|
|
1136
|
+
} else {
|
|
1137
|
+
this.logger({
|
|
1138
|
+
category: "base_cache",
|
|
1139
|
+
message: "Cache entry not found to delete",
|
|
1140
|
+
level: 1
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
} catch (error) {
|
|
1070
1144
|
this.logger({
|
|
1071
|
-
category: "
|
|
1072
|
-
message:
|
|
1073
|
-
level:
|
|
1145
|
+
category: "base_cache",
|
|
1146
|
+
message: `Error removing cache entry: ${error}`,
|
|
1147
|
+
level: 2
|
|
1148
|
+
});
|
|
1149
|
+
} finally {
|
|
1150
|
+
this.releaseLock();
|
|
1151
|
+
}
|
|
1152
|
+
});
|
|
1153
|
+
}
|
|
1154
|
+
/**
|
|
1155
|
+
* Tracks the usage of a hash with a specific requestId.
|
|
1156
|
+
* @param requestId - The identifier for the current request.
|
|
1157
|
+
* @param hash - The cache key hash.
|
|
1158
|
+
*/
|
|
1159
|
+
trackRequestIdUsage(requestId, hash) {
|
|
1160
|
+
var _a, _b;
|
|
1161
|
+
(_b = (_a = this.requestIdToUsedHashes)[requestId]) != null ? _b : _a[requestId] = [];
|
|
1162
|
+
this.requestIdToUsedHashes[requestId].push(hash);
|
|
1163
|
+
}
|
|
1164
|
+
/**
|
|
1165
|
+
* Deletes all cache entries associated with a specific requestId.
|
|
1166
|
+
* @param requestId - The identifier for the request whose cache entries should be deleted.
|
|
1167
|
+
*/
|
|
1168
|
+
deleteCacheForRequestId(requestId) {
|
|
1169
|
+
return __async(this, null, function* () {
|
|
1170
|
+
var _a;
|
|
1171
|
+
if (!(yield this.acquireLock())) {
|
|
1172
|
+
this.logger({
|
|
1173
|
+
category: "base_cache",
|
|
1174
|
+
message: "Failed to acquire lock for deleting cache",
|
|
1175
|
+
level: 2
|
|
1074
1176
|
});
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
try {
|
|
1180
|
+
const cache = this.readCache();
|
|
1181
|
+
const hashes = (_a = this.requestIdToUsedHashes[requestId]) != null ? _a : [];
|
|
1182
|
+
let entriesRemoved = 0;
|
|
1183
|
+
for (const hash of hashes) {
|
|
1184
|
+
if (cache[hash]) {
|
|
1185
|
+
delete cache[hash];
|
|
1186
|
+
entriesRemoved++;
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
if (entriesRemoved > 0) {
|
|
1190
|
+
this.writeCache(cache);
|
|
1191
|
+
} else {
|
|
1192
|
+
this.logger({
|
|
1193
|
+
category: "base_cache",
|
|
1194
|
+
message: `No cache entries found for requestId ${requestId}`,
|
|
1195
|
+
level: 1
|
|
1196
|
+
});
|
|
1197
|
+
}
|
|
1198
|
+
delete this.requestIdToUsedHashes[requestId];
|
|
1075
1199
|
} catch (error) {
|
|
1076
1200
|
this.logger({
|
|
1077
|
-
category: "
|
|
1078
|
-
message: `Error
|
|
1079
|
-
level:
|
|
1201
|
+
category: "base_cache",
|
|
1202
|
+
message: `Error deleting cache for requestId ${requestId}: ${error}`,
|
|
1203
|
+
level: 2
|
|
1080
1204
|
});
|
|
1081
|
-
this.resetCache();
|
|
1082
1205
|
} finally {
|
|
1083
1206
|
this.releaseLock();
|
|
1084
1207
|
}
|
|
1085
1208
|
});
|
|
1086
1209
|
}
|
|
1210
|
+
/**
|
|
1211
|
+
* Resets the entire cache by clearing the cache file.
|
|
1212
|
+
*/
|
|
1213
|
+
resetCache() {
|
|
1214
|
+
try {
|
|
1215
|
+
fs.writeFileSync(this.cacheFile, "{}");
|
|
1216
|
+
this.requestIdToUsedHashes = {};
|
|
1217
|
+
} catch (error) {
|
|
1218
|
+
this.logger({
|
|
1219
|
+
category: "base_cache",
|
|
1220
|
+
message: `Error resetting cache: ${error}`,
|
|
1221
|
+
level: 2
|
|
1222
|
+
});
|
|
1223
|
+
} finally {
|
|
1224
|
+
this.releaseLock();
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
};
|
|
1228
|
+
|
|
1229
|
+
// lib/cache/LLMCache.ts
|
|
1230
|
+
var LLMCache = class _LLMCache extends BaseCache {
|
|
1231
|
+
constructor(logger, cacheDir, cacheFile) {
|
|
1232
|
+
super(logger, cacheDir, cacheFile || "llm_calls.json");
|
|
1233
|
+
}
|
|
1234
|
+
/**
|
|
1235
|
+
* Overrides the get method to track used hashes by requestId.
|
|
1236
|
+
* @param options - The options used to generate the cache key.
|
|
1237
|
+
* @param requestId - The identifier for the current request.
|
|
1238
|
+
* @returns The cached data if available, otherwise null.
|
|
1239
|
+
*/
|
|
1240
|
+
get(options, requestId) {
|
|
1241
|
+
return __async(this, null, function* () {
|
|
1242
|
+
const data = yield __superGet(_LLMCache.prototype, this, "get").call(this, options, requestId);
|
|
1243
|
+
return data;
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
/**
|
|
1247
|
+
* Overrides the set method to include cache cleanup logic.
|
|
1248
|
+
* @param options - The options used to generate the cache key.
|
|
1249
|
+
* @param data - The data to be cached.
|
|
1250
|
+
* @param requestId - The identifier for the current request.
|
|
1251
|
+
*/
|
|
1252
|
+
set(options, data, requestId) {
|
|
1253
|
+
return __async(this, null, function* () {
|
|
1254
|
+
yield __superGet(_LLMCache.prototype, this, "set").call(this, options, data, requestId);
|
|
1255
|
+
this.logger({
|
|
1256
|
+
category: "llm_cache",
|
|
1257
|
+
message: "Cache miss - saved new response",
|
|
1258
|
+
level: 1
|
|
1259
|
+
});
|
|
1260
|
+
});
|
|
1261
|
+
}
|
|
1087
1262
|
};
|
|
1088
1263
|
|
|
1089
1264
|
// lib/llm/LLMProvider.ts
|
|
@@ -1203,8 +1378,8 @@ var ScreenshotService = class _ScreenshotService {
|
|
|
1203
1378
|
const { width, height } = yield image.metadata();
|
|
1204
1379
|
const svgAnnotations = yield Promise.all(
|
|
1205
1380
|
Object.entries(this.selectorMap).map(
|
|
1206
|
-
(_0) => __async(this, [_0], function* ([id,
|
|
1207
|
-
return this.createElementAnnotation(id,
|
|
1381
|
+
(_0) => __async(this, [_0], function* ([id, selectors]) {
|
|
1382
|
+
return this.createElementAnnotation(id, selectors);
|
|
1208
1383
|
})
|
|
1209
1384
|
)
|
|
1210
1385
|
);
|
|
@@ -1226,18 +1401,25 @@ var ScreenshotService = class _ScreenshotService {
|
|
|
1226
1401
|
return annotatedScreenshot;
|
|
1227
1402
|
});
|
|
1228
1403
|
}
|
|
1229
|
-
createElementAnnotation(id,
|
|
1404
|
+
createElementAnnotation(id, selectors) {
|
|
1230
1405
|
return __async(this, null, function* () {
|
|
1231
1406
|
try {
|
|
1232
|
-
|
|
1233
|
-
const
|
|
1407
|
+
let element = null;
|
|
1408
|
+
const selectorPromises = selectors.map(
|
|
1409
|
+
(selector) => __async(this, null, function* () {
|
|
1410
|
+
try {
|
|
1411
|
+
element = yield this.page.locator(`xpath=${selector}`).first();
|
|
1412
|
+
const box2 = yield element.boundingBox({ timeout: 5e3 });
|
|
1413
|
+
return box2;
|
|
1414
|
+
} catch (e) {
|
|
1415
|
+
return null;
|
|
1416
|
+
}
|
|
1417
|
+
})
|
|
1418
|
+
);
|
|
1419
|
+
const boxes = yield Promise.all(selectorPromises);
|
|
1420
|
+
const box = boxes.find((b) => b !== null);
|
|
1234
1421
|
if (!box) {
|
|
1235
|
-
|
|
1236
|
-
category: "Debug",
|
|
1237
|
-
message: `No bounding box for element ${id}`,
|
|
1238
|
-
level: 2
|
|
1239
|
-
});
|
|
1240
|
-
return "";
|
|
1422
|
+
throw new Error(`Unable to create annotation for element ${id}`);
|
|
1241
1423
|
}
|
|
1242
1424
|
const scrollPosition = yield this.page.evaluate(() => ({
|
|
1243
1425
|
scrollX: window.scrollX,
|
|
@@ -1264,8 +1446,8 @@ var ScreenshotService = class _ScreenshotService {
|
|
|
1264
1446
|
`;
|
|
1265
1447
|
} catch (error) {
|
|
1266
1448
|
this.log({
|
|
1267
|
-
category: "
|
|
1268
|
-
message: `Failed to create annotation for element ${id}: ${error}`,
|
|
1449
|
+
category: "Vision",
|
|
1450
|
+
message: `Warning: Failed to create annotation for element ${id}: ${error}, trace: ${error.stack}`,
|
|
1269
1451
|
level: 0
|
|
1270
1452
|
});
|
|
1271
1453
|
return "";
|
|
@@ -1317,40 +1499,981 @@ var ScreenshotService = class _ScreenshotService {
|
|
|
1317
1499
|
}
|
|
1318
1500
|
};
|
|
1319
1501
|
|
|
1320
|
-
// lib/
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1502
|
+
// lib/types.ts
|
|
1503
|
+
var PlaywrightCommandException = class extends Error {
|
|
1504
|
+
constructor(message) {
|
|
1505
|
+
super(message);
|
|
1506
|
+
this.name = "PlaywrightCommandException";
|
|
1507
|
+
}
|
|
1508
|
+
};
|
|
1509
|
+
var PlaywrightCommandMethodNotSupportedException = class extends Error {
|
|
1510
|
+
constructor(message) {
|
|
1511
|
+
super(message);
|
|
1512
|
+
this.name = "PlaywrightCommandMethodNotSupportedException";
|
|
1513
|
+
}
|
|
1514
|
+
};
|
|
1515
|
+
|
|
1516
|
+
// lib/cache/ActionCache.ts
|
|
1517
|
+
var ActionCache = class _ActionCache extends BaseCache {
|
|
1518
|
+
constructor(logger, cacheDir, cacheFile) {
|
|
1519
|
+
super(logger, cacheDir, cacheFile || "action_cache.json");
|
|
1520
|
+
}
|
|
1521
|
+
addActionStep(_0) {
|
|
1522
|
+
return __async(this, arguments, function* ({
|
|
1523
|
+
url,
|
|
1524
|
+
action,
|
|
1525
|
+
previousSelectors,
|
|
1526
|
+
playwrightCommand,
|
|
1527
|
+
componentString,
|
|
1528
|
+
xpaths,
|
|
1529
|
+
newStepString,
|
|
1530
|
+
completed,
|
|
1531
|
+
requestId
|
|
1532
|
+
}) {
|
|
1533
|
+
this.logger({
|
|
1534
|
+
category: "action_cache",
|
|
1535
|
+
message: `Adding action step to cache: ${action}, requestId: ${requestId}, url: ${url}, previousSelectors: ${previousSelectors}`,
|
|
1536
|
+
level: 1
|
|
1537
|
+
});
|
|
1538
|
+
yield this.set(
|
|
1539
|
+
{ url, action, previousSelectors },
|
|
1540
|
+
{
|
|
1541
|
+
playwrightCommand,
|
|
1542
|
+
componentString,
|
|
1543
|
+
xpaths,
|
|
1544
|
+
newStepString,
|
|
1545
|
+
completed,
|
|
1546
|
+
previousSelectors,
|
|
1547
|
+
action
|
|
1548
|
+
},
|
|
1549
|
+
requestId
|
|
1550
|
+
);
|
|
1551
|
+
});
|
|
1552
|
+
}
|
|
1553
|
+
/**
|
|
1554
|
+
* Retrieves all actions for a specific trajectory.
|
|
1555
|
+
* @param trajectoryId - Unique identifier for the trajectory.
|
|
1556
|
+
* @param requestId - The identifier for the current request.
|
|
1557
|
+
* @returns An array of TrajectoryEntry objects or null if not found.
|
|
1558
|
+
*/
|
|
1559
|
+
getActionStep(_0) {
|
|
1560
|
+
return __async(this, arguments, function* ({
|
|
1561
|
+
url,
|
|
1562
|
+
action,
|
|
1563
|
+
previousSelectors,
|
|
1564
|
+
requestId
|
|
1565
|
+
}) {
|
|
1566
|
+
const data = yield __superGet(_ActionCache.prototype, this, "get").call(this, { url, action, previousSelectors }, requestId);
|
|
1567
|
+
if (!data) {
|
|
1568
|
+
return null;
|
|
1569
|
+
}
|
|
1570
|
+
return data;
|
|
1571
|
+
});
|
|
1572
|
+
}
|
|
1573
|
+
removeActionStep(cacheHashObj) {
|
|
1574
|
+
return __async(this, null, function* () {
|
|
1575
|
+
yield __superGet(_ActionCache.prototype, this, "delete").call(this, cacheHashObj);
|
|
1576
|
+
});
|
|
1577
|
+
}
|
|
1578
|
+
/**
|
|
1579
|
+
* Clears all actions for a specific trajectory.
|
|
1580
|
+
* @param trajectoryId - Unique identifier for the trajectory.
|
|
1581
|
+
* @param requestId - The identifier for the current request.
|
|
1582
|
+
*/
|
|
1583
|
+
clearAction(requestId) {
|
|
1584
|
+
return __async(this, null, function* () {
|
|
1585
|
+
yield __superGet(_ActionCache.prototype, this, "deleteCacheForRequestId").call(this, requestId);
|
|
1586
|
+
this.logger({
|
|
1587
|
+
category: "action_cache",
|
|
1588
|
+
message: `Cleared action for ID: ${requestId}`,
|
|
1589
|
+
level: 1
|
|
1590
|
+
});
|
|
1591
|
+
});
|
|
1592
|
+
}
|
|
1593
|
+
/**
|
|
1594
|
+
* Resets the entire action cache.
|
|
1595
|
+
*/
|
|
1596
|
+
resetCache() {
|
|
1597
|
+
return __async(this, null, function* () {
|
|
1598
|
+
yield __superGet(_ActionCache.prototype, this, "resetCache").call(this);
|
|
1599
|
+
this.logger({
|
|
1600
|
+
category: "action_cache",
|
|
1601
|
+
message: "Action cache has been reset.",
|
|
1602
|
+
level: 1
|
|
1603
|
+
});
|
|
1604
|
+
});
|
|
1605
|
+
}
|
|
1606
|
+
};
|
|
1607
|
+
|
|
1608
|
+
// lib/utils.ts
|
|
1609
|
+
var import_crypto = __toESM(require("crypto"));
|
|
1610
|
+
function generateId(operation) {
|
|
1611
|
+
return import_crypto.default.createHash("sha256").update(operation).digest("hex");
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
// lib/handlers/actHandler.ts
|
|
1615
|
+
var StagehandActHandler = class {
|
|
1616
|
+
constructor({
|
|
1617
|
+
stagehand,
|
|
1618
|
+
verbose,
|
|
1619
|
+
llmProvider,
|
|
1620
|
+
enableCaching,
|
|
1621
|
+
logger,
|
|
1622
|
+
waitForSettledDom,
|
|
1623
|
+
defaultModelName,
|
|
1624
|
+
startDomDebug,
|
|
1625
|
+
cleanupDomDebug
|
|
1626
|
+
}) {
|
|
1627
|
+
this.stagehand = stagehand;
|
|
1628
|
+
this.verbose = verbose;
|
|
1629
|
+
this.llmProvider = llmProvider;
|
|
1630
|
+
this.enableCaching = enableCaching;
|
|
1631
|
+
this.logger = logger;
|
|
1632
|
+
this.waitForSettledDom = waitForSettledDom;
|
|
1633
|
+
this.actionCache = new ActionCache(this.logger);
|
|
1634
|
+
this.defaultModelName = defaultModelName;
|
|
1635
|
+
this.startDomDebug = startDomDebug;
|
|
1636
|
+
this.cleanupDomDebug = cleanupDomDebug;
|
|
1637
|
+
this.actions = {};
|
|
1638
|
+
}
|
|
1639
|
+
_recordAction(action, result) {
|
|
1640
|
+
return __async(this, null, function* () {
|
|
1641
|
+
const id = generateId(action);
|
|
1642
|
+
this.actions[id] = { result, action };
|
|
1643
|
+
return id;
|
|
1644
|
+
});
|
|
1645
|
+
}
|
|
1646
|
+
_verifyActionCompletion(_0) {
|
|
1647
|
+
return __async(this, arguments, function* ({
|
|
1648
|
+
completed,
|
|
1649
|
+
verifierUseVision,
|
|
1650
|
+
requestId,
|
|
1651
|
+
action,
|
|
1652
|
+
steps,
|
|
1653
|
+
model,
|
|
1654
|
+
domSettleTimeoutMs
|
|
1655
|
+
}) {
|
|
1656
|
+
yield this.waitForSettledDom(domSettleTimeoutMs);
|
|
1657
|
+
const { selectorMap } = yield this.stagehand.page.evaluate(() => {
|
|
1658
|
+
return window.processAllOfDom();
|
|
1659
|
+
});
|
|
1660
|
+
let actionCompleted = false;
|
|
1661
|
+
if (completed) {
|
|
1662
|
+
this.stagehand.log({
|
|
1663
|
+
category: "action",
|
|
1664
|
+
message: `Action marked as completed, Verifying if this is true...`,
|
|
1665
|
+
level: 1
|
|
1666
|
+
});
|
|
1667
|
+
let domElements = void 0;
|
|
1668
|
+
let fullpageScreenshot = void 0;
|
|
1669
|
+
if (verifierUseVision) {
|
|
1670
|
+
try {
|
|
1671
|
+
const screenshotService = new ScreenshotService(
|
|
1672
|
+
this.stagehand.page,
|
|
1673
|
+
selectorMap,
|
|
1674
|
+
this.verbose
|
|
1675
|
+
);
|
|
1676
|
+
fullpageScreenshot = yield screenshotService.getScreenshot(true, 15);
|
|
1677
|
+
} catch (e) {
|
|
1678
|
+
this.stagehand.log({
|
|
1679
|
+
category: "action",
|
|
1680
|
+
message: `Error getting full page screenshot: ${e.message}
|
|
1681
|
+
. Trying again...`,
|
|
1682
|
+
level: 1
|
|
1683
|
+
});
|
|
1684
|
+
const screenshotService = new ScreenshotService(
|
|
1685
|
+
this.stagehand.page,
|
|
1686
|
+
selectorMap,
|
|
1687
|
+
this.verbose
|
|
1688
|
+
);
|
|
1689
|
+
fullpageScreenshot = yield screenshotService.getScreenshot(true, 15);
|
|
1690
|
+
}
|
|
1691
|
+
} else {
|
|
1692
|
+
({ outputString: domElements } = yield this.stagehand.page.evaluate(
|
|
1693
|
+
() => {
|
|
1694
|
+
return window.processAllOfDom();
|
|
1695
|
+
}
|
|
1696
|
+
));
|
|
1697
|
+
}
|
|
1698
|
+
actionCompleted = yield verifyActCompletion({
|
|
1699
|
+
goal: action,
|
|
1700
|
+
steps,
|
|
1701
|
+
llmProvider: this.llmProvider,
|
|
1702
|
+
modelName: model,
|
|
1703
|
+
screenshot: fullpageScreenshot,
|
|
1704
|
+
domElements,
|
|
1705
|
+
logger: this.logger,
|
|
1706
|
+
requestId
|
|
1707
|
+
});
|
|
1708
|
+
this.stagehand.log({
|
|
1709
|
+
category: "action",
|
|
1710
|
+
message: `Action completion verification result: ${actionCompleted}`,
|
|
1711
|
+
level: 1
|
|
1712
|
+
});
|
|
1713
|
+
}
|
|
1714
|
+
return actionCompleted;
|
|
1715
|
+
});
|
|
1716
|
+
}
|
|
1717
|
+
_performPlaywrightMethod(method, args, xpath, domSettleTimeoutMs) {
|
|
1718
|
+
return __async(this, null, function* () {
|
|
1719
|
+
const locator = this.stagehand.page.locator(`xpath=${xpath}`).first();
|
|
1720
|
+
const initialUrl = this.stagehand.page.url();
|
|
1721
|
+
if (method === "scrollIntoView") {
|
|
1722
|
+
this.stagehand.log({
|
|
1723
|
+
category: "action",
|
|
1724
|
+
message: `Scrolling element into view`,
|
|
1725
|
+
level: 2
|
|
1726
|
+
});
|
|
1727
|
+
try {
|
|
1728
|
+
yield locator.evaluate((element) => {
|
|
1729
|
+
element.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
1730
|
+
}).catch((e) => {
|
|
1731
|
+
this.stagehand.log({
|
|
1732
|
+
category: "action",
|
|
1733
|
+
message: `Error scrolling element into view: ${e.message}
|
|
1734
|
+
Trace: ${e.stack}`,
|
|
1735
|
+
level: 1
|
|
1736
|
+
});
|
|
1737
|
+
});
|
|
1738
|
+
} catch (e) {
|
|
1739
|
+
this.stagehand.log({
|
|
1740
|
+
category: "action",
|
|
1741
|
+
message: `Error scrolling element into view: ${e.message}
|
|
1742
|
+
Trace: ${e.stack}`,
|
|
1743
|
+
level: 1
|
|
1744
|
+
});
|
|
1745
|
+
throw new PlaywrightCommandException(e.message);
|
|
1746
|
+
}
|
|
1747
|
+
} else if (method === "fill" || method === "type") {
|
|
1748
|
+
try {
|
|
1749
|
+
yield locator.fill("");
|
|
1750
|
+
yield locator.click();
|
|
1751
|
+
const text = args[0];
|
|
1752
|
+
for (const char of text) {
|
|
1753
|
+
yield this.stagehand.page.keyboard.type(char, {
|
|
1754
|
+
delay: Math.random() * 50 + 25
|
|
1755
|
+
});
|
|
1756
|
+
}
|
|
1757
|
+
} catch (e) {
|
|
1758
|
+
this.logger({
|
|
1759
|
+
category: "action",
|
|
1760
|
+
message: `Error filling element: ${e.message}
|
|
1761
|
+
Trace: ${e.stack}`,
|
|
1762
|
+
level: 1
|
|
1763
|
+
});
|
|
1764
|
+
throw new PlaywrightCommandException(e.message);
|
|
1765
|
+
}
|
|
1766
|
+
} else if (method === "press") {
|
|
1767
|
+
try {
|
|
1768
|
+
const key = args[0];
|
|
1769
|
+
yield this.stagehand.page.keyboard.press(key);
|
|
1770
|
+
} catch (e) {
|
|
1771
|
+
this.logger({
|
|
1772
|
+
category: "action",
|
|
1773
|
+
message: `Error pressing key: ${e.message}
|
|
1774
|
+
Trace: ${e.stack}`,
|
|
1775
|
+
level: 1
|
|
1776
|
+
});
|
|
1777
|
+
throw new PlaywrightCommandException(e.message);
|
|
1778
|
+
}
|
|
1779
|
+
} else if (typeof locator[method] === "function") {
|
|
1780
|
+
this.logger({
|
|
1781
|
+
category: "action",
|
|
1782
|
+
message: `Page URL before action: ${this.stagehand.page.url()}`,
|
|
1783
|
+
level: 2
|
|
1784
|
+
});
|
|
1785
|
+
try {
|
|
1786
|
+
yield locator[method](...args);
|
|
1787
|
+
} catch (e) {
|
|
1788
|
+
this.logger({
|
|
1789
|
+
category: "action",
|
|
1790
|
+
message: `Error performing method ${method} with args ${JSON.stringify(
|
|
1791
|
+
args
|
|
1792
|
+
)}: ${e.message}
|
|
1793
|
+
Trace: ${e.stack}`,
|
|
1794
|
+
level: 1
|
|
1795
|
+
});
|
|
1796
|
+
throw new PlaywrightCommandException(e.message);
|
|
1797
|
+
}
|
|
1798
|
+
if (method === "click") {
|
|
1799
|
+
this.logger({
|
|
1800
|
+
category: "action",
|
|
1801
|
+
message: `Clicking element, checking for page navigation`,
|
|
1802
|
+
level: 1
|
|
1803
|
+
});
|
|
1804
|
+
const newOpenedTab = yield Promise.race([
|
|
1805
|
+
new Promise((resolve) => {
|
|
1806
|
+
this.stagehand.context.once("page", (page) => resolve(page));
|
|
1807
|
+
setTimeout(() => resolve(null), 1500);
|
|
1808
|
+
})
|
|
1809
|
+
]);
|
|
1810
|
+
this.logger({
|
|
1811
|
+
category: "action",
|
|
1812
|
+
message: `Clicked element, ${newOpenedTab ? "opened a new tab" : "no new tabs opened"}`,
|
|
1813
|
+
level: 1
|
|
1814
|
+
});
|
|
1815
|
+
if (newOpenedTab) {
|
|
1816
|
+
this.logger({
|
|
1817
|
+
category: "action",
|
|
1818
|
+
message: `New page detected (new tab) with URL: ${newOpenedTab.url()}`,
|
|
1819
|
+
level: 1
|
|
1820
|
+
});
|
|
1821
|
+
yield newOpenedTab.close();
|
|
1822
|
+
yield this.stagehand.page.goto(newOpenedTab.url());
|
|
1823
|
+
yield this.stagehand.page.waitForLoadState("domcontentloaded");
|
|
1824
|
+
yield this.waitForSettledDom(domSettleTimeoutMs);
|
|
1825
|
+
}
|
|
1826
|
+
yield Promise.race([
|
|
1827
|
+
this.stagehand.page.waitForLoadState("networkidle"),
|
|
1828
|
+
new Promise((resolve) => setTimeout(resolve, 5e3))
|
|
1829
|
+
]).catch((e) => {
|
|
1830
|
+
this.logger({
|
|
1831
|
+
category: "action",
|
|
1832
|
+
message: `Network idle timeout hit`,
|
|
1833
|
+
level: 1
|
|
1834
|
+
});
|
|
1835
|
+
});
|
|
1836
|
+
this.logger({
|
|
1837
|
+
category: "action",
|
|
1838
|
+
message: `Finished waiting for (possible) page navigation`,
|
|
1839
|
+
level: 1
|
|
1840
|
+
});
|
|
1841
|
+
if (this.stagehand.page.url() !== initialUrl) {
|
|
1842
|
+
this.logger({
|
|
1843
|
+
category: "action",
|
|
1844
|
+
message: `New page detected with URL: ${this.stagehand.page.url()}`,
|
|
1845
|
+
level: 1
|
|
1846
|
+
});
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
} else {
|
|
1850
|
+
this.logger({
|
|
1851
|
+
category: "action",
|
|
1852
|
+
message: `Chosen method ${method} is invalid`,
|
|
1853
|
+
level: 1
|
|
1854
|
+
});
|
|
1855
|
+
throw new PlaywrightCommandMethodNotSupportedException(
|
|
1856
|
+
`Method ${method} not supported`
|
|
1857
|
+
);
|
|
1858
|
+
}
|
|
1859
|
+
yield this.waitForSettledDom(domSettleTimeoutMs);
|
|
1860
|
+
});
|
|
1861
|
+
}
|
|
1862
|
+
_getComponentString(locator) {
|
|
1863
|
+
return __async(this, null, function* () {
|
|
1864
|
+
return yield locator.evaluate((el) => {
|
|
1865
|
+
const clone = el.cloneNode(true);
|
|
1866
|
+
const attributesToKeep = [
|
|
1867
|
+
"type",
|
|
1868
|
+
"name",
|
|
1869
|
+
"placeholder",
|
|
1870
|
+
"aria-label",
|
|
1871
|
+
"role",
|
|
1872
|
+
"href",
|
|
1873
|
+
"title",
|
|
1874
|
+
"alt"
|
|
1875
|
+
];
|
|
1876
|
+
Array.from(clone.attributes).forEach((attr) => {
|
|
1877
|
+
if (!attributesToKeep.includes(attr.name)) {
|
|
1878
|
+
clone.removeAttribute(attr.name);
|
|
1879
|
+
}
|
|
1880
|
+
});
|
|
1881
|
+
const outerHtml = clone.outerHTML;
|
|
1882
|
+
return outerHtml.trim().replace(/\s+/g, " ");
|
|
1883
|
+
});
|
|
1884
|
+
});
|
|
1885
|
+
}
|
|
1886
|
+
getElement(xpath, timeout = 5e3) {
|
|
1887
|
+
return __async(this, null, function* () {
|
|
1888
|
+
try {
|
|
1889
|
+
const element = this.stagehand.page.locator(`xpath=${xpath}`).first();
|
|
1890
|
+
yield element.waitFor({ state: "attached", timeout });
|
|
1891
|
+
return element;
|
|
1892
|
+
} catch (e) {
|
|
1893
|
+
this.logger({
|
|
1894
|
+
category: "action",
|
|
1895
|
+
message: `Element with XPath ${xpath} not found within ${timeout}ms.`,
|
|
1896
|
+
level: 1
|
|
1897
|
+
});
|
|
1898
|
+
return null;
|
|
1899
|
+
}
|
|
1900
|
+
});
|
|
1901
|
+
}
|
|
1902
|
+
_checkIfCachedStepIsValid_oneXpath(cachedStep) {
|
|
1903
|
+
return __async(this, null, function* () {
|
|
1904
|
+
this.logger({
|
|
1905
|
+
category: "action",
|
|
1906
|
+
message: `Checking if cached step is valid: ${cachedStep.xpath}, ${cachedStep.savedComponentString}`,
|
|
1907
|
+
level: 1
|
|
1908
|
+
});
|
|
1909
|
+
try {
|
|
1910
|
+
const locator = yield this.getElement(cachedStep.xpath);
|
|
1911
|
+
if (!locator) {
|
|
1912
|
+
this.logger({
|
|
1913
|
+
category: "action",
|
|
1914
|
+
message: `Locator not found for xpath: ${cachedStep.xpath}`,
|
|
1915
|
+
level: 1
|
|
1916
|
+
});
|
|
1917
|
+
return false;
|
|
1918
|
+
}
|
|
1919
|
+
this.logger({
|
|
1920
|
+
category: "action",
|
|
1921
|
+
message: `locator element: ${yield this._getComponentString(locator)}`,
|
|
1922
|
+
level: 1
|
|
1923
|
+
});
|
|
1924
|
+
let currentComponent = yield this._getComponentString(locator);
|
|
1925
|
+
this.logger({
|
|
1926
|
+
category: "action",
|
|
1927
|
+
message: `Current text: ${currentComponent}`,
|
|
1928
|
+
level: 1
|
|
1929
|
+
});
|
|
1930
|
+
if (!currentComponent || !cachedStep.savedComponentString) {
|
|
1931
|
+
this.logger({
|
|
1932
|
+
category: "action",
|
|
1933
|
+
message: `Current text or cached text is undefined`,
|
|
1934
|
+
level: 1
|
|
1935
|
+
});
|
|
1936
|
+
return false;
|
|
1937
|
+
}
|
|
1938
|
+
const normalizedCurrentText = currentComponent.trim().replace(/\s+/g, " ");
|
|
1939
|
+
const normalizedCachedText = cachedStep.savedComponentString.trim().replace(/\s+/g, " ");
|
|
1940
|
+
if (normalizedCurrentText !== normalizedCachedText) {
|
|
1941
|
+
this.logger({
|
|
1942
|
+
category: "action",
|
|
1943
|
+
message: `Current text and cached text do not match: ${normalizedCurrentText} !== ${normalizedCachedText}`,
|
|
1944
|
+
level: 1
|
|
1945
|
+
});
|
|
1946
|
+
return false;
|
|
1947
|
+
}
|
|
1948
|
+
return true;
|
|
1949
|
+
} catch (e) {
|
|
1950
|
+
this.logger({
|
|
1951
|
+
category: "action",
|
|
1952
|
+
message: `Error checking if cached step is valid: ${e.message}
|
|
1953
|
+
Trace: ${e.stack}`,
|
|
1954
|
+
level: 1
|
|
1955
|
+
});
|
|
1956
|
+
return false;
|
|
1957
|
+
}
|
|
1958
|
+
});
|
|
1959
|
+
}
|
|
1960
|
+
_getValidCachedStepXpath(cachedStep) {
|
|
1961
|
+
return __async(this, null, function* () {
|
|
1962
|
+
const reversedXpaths = [...cachedStep.xpaths].reverse();
|
|
1963
|
+
for (const xpath of reversedXpaths) {
|
|
1964
|
+
const isValid = yield this._checkIfCachedStepIsValid_oneXpath({
|
|
1965
|
+
xpath,
|
|
1966
|
+
savedComponentString: cachedStep.savedComponentString
|
|
1967
|
+
});
|
|
1968
|
+
if (isValid) {
|
|
1969
|
+
return xpath;
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
return null;
|
|
1973
|
+
});
|
|
1974
|
+
}
|
|
1975
|
+
_runCachedActionIfAvailable(_0) {
|
|
1976
|
+
return __async(this, arguments, function* ({
|
|
1977
|
+
action,
|
|
1978
|
+
previousSelectors,
|
|
1979
|
+
requestId,
|
|
1980
|
+
steps,
|
|
1981
|
+
chunksSeen,
|
|
1982
|
+
modelName,
|
|
1983
|
+
useVision,
|
|
1984
|
+
verifierUseVision,
|
|
1985
|
+
retries,
|
|
1986
|
+
variables,
|
|
1987
|
+
model,
|
|
1988
|
+
domSettleTimeoutMs
|
|
1989
|
+
}) {
|
|
1990
|
+
const cacheObj = {
|
|
1991
|
+
url: this.stagehand.page.url(),
|
|
1992
|
+
action,
|
|
1993
|
+
previousSelectors,
|
|
1994
|
+
requestId
|
|
1995
|
+
};
|
|
1996
|
+
this.logger({
|
|
1997
|
+
category: "action",
|
|
1998
|
+
message: `Checking action cache for: ${JSON.stringify(cacheObj)}`,
|
|
1999
|
+
level: 1
|
|
2000
|
+
});
|
|
2001
|
+
const cachedStep = yield this.actionCache.getActionStep(cacheObj);
|
|
2002
|
+
if (!cachedStep) {
|
|
2003
|
+
this.logger({
|
|
2004
|
+
category: "action",
|
|
2005
|
+
message: `Action cache miss: ${JSON.stringify(cacheObj)}`,
|
|
2006
|
+
level: 1
|
|
2007
|
+
});
|
|
2008
|
+
return null;
|
|
2009
|
+
}
|
|
2010
|
+
this.logger({
|
|
2011
|
+
category: "action",
|
|
2012
|
+
message: `Action cache semi-hit: ${cachedStep.playwrightCommand.method} with args: ${JSON.stringify(
|
|
2013
|
+
cachedStep.playwrightCommand.args
|
|
2014
|
+
)}`,
|
|
2015
|
+
level: 1
|
|
2016
|
+
});
|
|
2017
|
+
try {
|
|
2018
|
+
const validXpath = yield this._getValidCachedStepXpath({
|
|
2019
|
+
xpaths: cachedStep.xpaths,
|
|
2020
|
+
savedComponentString: cachedStep.componentString
|
|
2021
|
+
});
|
|
2022
|
+
this.logger({
|
|
2023
|
+
category: "action",
|
|
2024
|
+
message: `Cached action step is valid: ${validXpath !== null}`,
|
|
2025
|
+
level: 1
|
|
2026
|
+
});
|
|
2027
|
+
if (!validXpath) {
|
|
2028
|
+
this.logger({
|
|
2029
|
+
category: "action",
|
|
2030
|
+
message: `Cached action step is invalid, removing...`,
|
|
2031
|
+
level: 1
|
|
2032
|
+
});
|
|
2033
|
+
yield this.actionCache.removeActionStep(cacheObj);
|
|
2034
|
+
return null;
|
|
2035
|
+
}
|
|
2036
|
+
this.logger({
|
|
2037
|
+
category: "action",
|
|
2038
|
+
message: `Action Cache Hit: ${cachedStep.playwrightCommand.method} with args: ${JSON.stringify(
|
|
2039
|
+
cachedStep.playwrightCommand.args
|
|
2040
|
+
)}`,
|
|
2041
|
+
level: 1
|
|
2042
|
+
});
|
|
2043
|
+
cachedStep.playwrightCommand.args = cachedStep.playwrightCommand.args.map(
|
|
2044
|
+
(arg) => {
|
|
2045
|
+
return fillInVariables(arg, variables);
|
|
2046
|
+
}
|
|
2047
|
+
);
|
|
2048
|
+
yield this._performPlaywrightMethod(
|
|
2049
|
+
cachedStep.playwrightCommand.method,
|
|
2050
|
+
cachedStep.playwrightCommand.args,
|
|
2051
|
+
validXpath,
|
|
2052
|
+
domSettleTimeoutMs
|
|
2053
|
+
);
|
|
2054
|
+
steps = steps + cachedStep.newStepString;
|
|
2055
|
+
const { outputString, selectorMap } = yield this.stagehand.page.evaluate(
|
|
2056
|
+
({ chunksSeen: chunksSeen2 }) => {
|
|
2057
|
+
return window.processDom(chunksSeen2);
|
|
2058
|
+
},
|
|
2059
|
+
{ chunksSeen }
|
|
2060
|
+
);
|
|
2061
|
+
if (cachedStep.completed) {
|
|
2062
|
+
let actionCompleted = yield this._verifyActionCompletion({
|
|
2063
|
+
completed: true,
|
|
2064
|
+
verifierUseVision,
|
|
2065
|
+
model,
|
|
2066
|
+
steps,
|
|
2067
|
+
requestId,
|
|
2068
|
+
action,
|
|
2069
|
+
domSettleTimeoutMs
|
|
2070
|
+
});
|
|
2071
|
+
this.logger({
|
|
2072
|
+
category: "action",
|
|
2073
|
+
message: `Action completion verification result from cache: ${actionCompleted}`,
|
|
2074
|
+
level: 1
|
|
2075
|
+
});
|
|
2076
|
+
if (actionCompleted) {
|
|
2077
|
+
return {
|
|
2078
|
+
success: true,
|
|
2079
|
+
message: "Action completed successfully using cached step",
|
|
2080
|
+
action
|
|
2081
|
+
};
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
return this.act({
|
|
2085
|
+
action,
|
|
2086
|
+
steps,
|
|
2087
|
+
chunksSeen,
|
|
2088
|
+
modelName,
|
|
2089
|
+
useVision,
|
|
2090
|
+
verifierUseVision,
|
|
2091
|
+
retries,
|
|
2092
|
+
requestId,
|
|
2093
|
+
variables,
|
|
2094
|
+
previousSelectors: [...previousSelectors, cachedStep.xpaths[0]],
|
|
2095
|
+
skipActionCacheForThisStep: false,
|
|
2096
|
+
domSettleTimeoutMs
|
|
2097
|
+
});
|
|
2098
|
+
} catch (exception) {
|
|
2099
|
+
this.logger({
|
|
2100
|
+
category: "action",
|
|
2101
|
+
message: `Error performing cached action step: ${exception.message}
|
|
2102
|
+
Trace: ${exception.stack}`,
|
|
2103
|
+
level: 1
|
|
2104
|
+
});
|
|
2105
|
+
yield this.actionCache.removeActionStep(cacheObj);
|
|
2106
|
+
return null;
|
|
2107
|
+
}
|
|
2108
|
+
});
|
|
2109
|
+
}
|
|
2110
|
+
act(_0) {
|
|
2111
|
+
return __async(this, arguments, function* ({
|
|
2112
|
+
action,
|
|
2113
|
+
steps = "",
|
|
2114
|
+
chunksSeen,
|
|
2115
|
+
modelName,
|
|
2116
|
+
useVision,
|
|
2117
|
+
verifierUseVision,
|
|
2118
|
+
retries = 0,
|
|
2119
|
+
requestId,
|
|
2120
|
+
variables,
|
|
2121
|
+
previousSelectors,
|
|
2122
|
+
skipActionCacheForThisStep = false,
|
|
2123
|
+
domSettleTimeoutMs
|
|
2124
|
+
}) {
|
|
2125
|
+
var _a;
|
|
2126
|
+
try {
|
|
2127
|
+
yield this.waitForSettledDom(domSettleTimeoutMs);
|
|
2128
|
+
yield this.startDomDebug();
|
|
2129
|
+
const model = modelName != null ? modelName : this.defaultModelName;
|
|
2130
|
+
if (this.enableCaching && !skipActionCacheForThisStep) {
|
|
2131
|
+
const response2 = yield this._runCachedActionIfAvailable({
|
|
2132
|
+
action,
|
|
2133
|
+
previousSelectors,
|
|
2134
|
+
requestId,
|
|
2135
|
+
steps,
|
|
2136
|
+
chunksSeen,
|
|
2137
|
+
modelName: model,
|
|
2138
|
+
useVision,
|
|
2139
|
+
verifierUseVision,
|
|
2140
|
+
retries,
|
|
2141
|
+
variables,
|
|
2142
|
+
model,
|
|
2143
|
+
domSettleTimeoutMs
|
|
2144
|
+
});
|
|
2145
|
+
if (response2 !== null) {
|
|
2146
|
+
return response2;
|
|
2147
|
+
} else {
|
|
2148
|
+
return this.act({
|
|
2149
|
+
action,
|
|
2150
|
+
steps,
|
|
2151
|
+
chunksSeen,
|
|
2152
|
+
modelName,
|
|
2153
|
+
useVision,
|
|
2154
|
+
verifierUseVision,
|
|
2155
|
+
retries,
|
|
2156
|
+
requestId,
|
|
2157
|
+
variables,
|
|
2158
|
+
previousSelectors,
|
|
2159
|
+
skipActionCacheForThisStep: true,
|
|
2160
|
+
domSettleTimeoutMs
|
|
2161
|
+
});
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2164
|
+
if (!modelsWithVision.includes(model) && (useVision !== false || verifierUseVision)) {
|
|
2165
|
+
this.logger({
|
|
2166
|
+
category: "action",
|
|
2167
|
+
message: `${model} does not support vision, but useVision was set to ${useVision}. Defaulting to false.`,
|
|
2168
|
+
level: 1
|
|
2169
|
+
});
|
|
2170
|
+
useVision = false;
|
|
2171
|
+
verifierUseVision = false;
|
|
2172
|
+
}
|
|
2173
|
+
this.logger({
|
|
2174
|
+
category: "action",
|
|
2175
|
+
message: `Running / Continuing action: ${action} on page: ${this.stagehand.page.url()}`,
|
|
2176
|
+
level: 2
|
|
2177
|
+
});
|
|
2178
|
+
this.logger({
|
|
2179
|
+
category: "action",
|
|
2180
|
+
message: `Processing DOM...`,
|
|
2181
|
+
level: 2
|
|
2182
|
+
});
|
|
2183
|
+
const { outputString, selectorMap, chunk, chunks } = yield this.stagehand.page.evaluate(
|
|
2184
|
+
({ chunksSeen: chunksSeen2 }) => {
|
|
2185
|
+
return window.processDom(chunksSeen2);
|
|
2186
|
+
},
|
|
2187
|
+
{ chunksSeen }
|
|
2188
|
+
);
|
|
2189
|
+
this.logger({
|
|
2190
|
+
category: "action",
|
|
2191
|
+
message: `Looking at chunk ${chunk}. Chunks left: ${chunks.length - chunksSeen.length}`,
|
|
2192
|
+
level: 1
|
|
2193
|
+
});
|
|
2194
|
+
let annotatedScreenshot;
|
|
2195
|
+
if (useVision === true) {
|
|
2196
|
+
if (!modelsWithVision.includes(model)) {
|
|
2197
|
+
this.logger({
|
|
2198
|
+
category: "action",
|
|
2199
|
+
message: `${model} does not support vision. Skipping vision processing.`,
|
|
2200
|
+
level: 1
|
|
2201
|
+
});
|
|
2202
|
+
} else {
|
|
2203
|
+
const screenshotService = new ScreenshotService(
|
|
2204
|
+
this.stagehand.page,
|
|
2205
|
+
selectorMap,
|
|
2206
|
+
this.verbose
|
|
2207
|
+
);
|
|
2208
|
+
annotatedScreenshot = yield screenshotService.getAnnotatedScreenshot(false);
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
const response = yield act({
|
|
2212
|
+
action,
|
|
2213
|
+
domElements: outputString,
|
|
2214
|
+
steps,
|
|
2215
|
+
llmProvider: this.llmProvider,
|
|
2216
|
+
modelName: model,
|
|
2217
|
+
screenshot: annotatedScreenshot,
|
|
2218
|
+
logger: this.logger,
|
|
2219
|
+
requestId,
|
|
2220
|
+
variables
|
|
2221
|
+
});
|
|
2222
|
+
this.logger({
|
|
2223
|
+
category: "action",
|
|
2224
|
+
message: `Received response from LLM: ${JSON.stringify(response)}`,
|
|
2225
|
+
level: 1
|
|
2226
|
+
});
|
|
2227
|
+
yield this.cleanupDomDebug();
|
|
2228
|
+
if (!response) {
|
|
2229
|
+
if (chunksSeen.length + 1 < chunks.length) {
|
|
2230
|
+
chunksSeen.push(chunk);
|
|
2231
|
+
this.logger({
|
|
2232
|
+
category: "action",
|
|
2233
|
+
message: `No action found in current chunk. Chunks seen: ${chunksSeen.length}.`,
|
|
2234
|
+
level: 1
|
|
2235
|
+
});
|
|
2236
|
+
return this.act({
|
|
2237
|
+
action,
|
|
2238
|
+
steps: steps + (!steps.endsWith("\n") ? "\n" : "") + "## Step: Scrolled to another section\n",
|
|
2239
|
+
chunksSeen,
|
|
2240
|
+
modelName,
|
|
2241
|
+
useVision,
|
|
2242
|
+
verifierUseVision,
|
|
2243
|
+
requestId,
|
|
2244
|
+
variables,
|
|
2245
|
+
previousSelectors,
|
|
2246
|
+
skipActionCacheForThisStep,
|
|
2247
|
+
domSettleTimeoutMs
|
|
2248
|
+
});
|
|
2249
|
+
} else if (useVision === "fallback") {
|
|
2250
|
+
this.logger({
|
|
2251
|
+
category: "action",
|
|
2252
|
+
message: `Switching to vision-based processing`,
|
|
2253
|
+
level: 1
|
|
2254
|
+
});
|
|
2255
|
+
yield this.stagehand.page.evaluate(() => window.scrollToHeight(0));
|
|
2256
|
+
return yield this.act({
|
|
2257
|
+
action,
|
|
2258
|
+
steps,
|
|
2259
|
+
chunksSeen,
|
|
2260
|
+
modelName,
|
|
2261
|
+
useVision: true,
|
|
2262
|
+
verifierUseVision,
|
|
2263
|
+
requestId,
|
|
2264
|
+
variables,
|
|
2265
|
+
previousSelectors,
|
|
2266
|
+
skipActionCacheForThisStep,
|
|
2267
|
+
domSettleTimeoutMs
|
|
2268
|
+
});
|
|
2269
|
+
} else {
|
|
2270
|
+
if (this.enableCaching) {
|
|
2271
|
+
this.llmProvider.cleanRequestCache(requestId);
|
|
2272
|
+
this.actionCache.deleteCacheForRequestId(requestId);
|
|
2273
|
+
}
|
|
2274
|
+
return {
|
|
2275
|
+
success: false,
|
|
2276
|
+
message: `Action was not able to be completed.`,
|
|
2277
|
+
action
|
|
2278
|
+
};
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
const elementId = response["element"];
|
|
2282
|
+
const xpaths = selectorMap[elementId];
|
|
2283
|
+
const method = response["method"];
|
|
2284
|
+
const args = response["args"];
|
|
2285
|
+
const elementLines = outputString.split("\n");
|
|
2286
|
+
const elementText = ((_a = elementLines.find((line) => line.startsWith(`${elementId}:`))) == null ? void 0 : _a.split(":")[1]) || "Element not found";
|
|
2287
|
+
this.logger({
|
|
2288
|
+
category: "action",
|
|
2289
|
+
message: `Executing method: ${method} on element: ${elementId} (xpaths: ${xpaths.join(
|
|
2290
|
+
", "
|
|
2291
|
+
)}) with args: ${JSON.stringify(args)}`,
|
|
2292
|
+
level: 1
|
|
2293
|
+
});
|
|
2294
|
+
try {
|
|
2295
|
+
const initialUrl = this.stagehand.page.url();
|
|
2296
|
+
const locator = this.stagehand.page.locator(`xpath=${xpaths[0]}`).first();
|
|
2297
|
+
const originalUrl = this.stagehand.page.url();
|
|
2298
|
+
const componentString = yield this._getComponentString(locator);
|
|
2299
|
+
const responseArgs = [...args];
|
|
2300
|
+
if (variables) {
|
|
2301
|
+
responseArgs.forEach((arg, index) => {
|
|
2302
|
+
if (typeof arg === "string") {
|
|
2303
|
+
args[index] = fillInVariables(arg, variables);
|
|
2304
|
+
}
|
|
2305
|
+
});
|
|
2306
|
+
}
|
|
2307
|
+
yield this._performPlaywrightMethod(
|
|
2308
|
+
method,
|
|
2309
|
+
args,
|
|
2310
|
+
xpaths[0],
|
|
2311
|
+
domSettleTimeoutMs
|
|
2312
|
+
);
|
|
2313
|
+
const newStepString = (!steps.endsWith("\n") ? "\n" : "") + `## Step: ${response.step}
|
|
2314
|
+
Element: ${elementText}
|
|
2315
|
+
Action: ${response.method}
|
|
2316
|
+
Reasoning: ${response.why}
|
|
2317
|
+
`;
|
|
2318
|
+
steps += newStepString;
|
|
2319
|
+
if (this.enableCaching) {
|
|
2320
|
+
this.actionCache.addActionStep({
|
|
2321
|
+
action,
|
|
2322
|
+
url: originalUrl,
|
|
2323
|
+
previousSelectors,
|
|
2324
|
+
playwrightCommand: {
|
|
2325
|
+
method,
|
|
2326
|
+
args: responseArgs
|
|
2327
|
+
},
|
|
2328
|
+
componentString,
|
|
2329
|
+
requestId,
|
|
2330
|
+
xpaths,
|
|
2331
|
+
newStepString,
|
|
2332
|
+
completed: response.completed
|
|
2333
|
+
}).catch((e) => {
|
|
2334
|
+
this.logger({
|
|
2335
|
+
category: "action",
|
|
2336
|
+
message: `Error adding action step to cache: ${e.message}
|
|
2337
|
+
Trace: ${e.stack}`,
|
|
2338
|
+
level: 1
|
|
2339
|
+
});
|
|
2340
|
+
});
|
|
2341
|
+
}
|
|
2342
|
+
if (this.stagehand.page.url() !== initialUrl) {
|
|
2343
|
+
steps += ` Result (Important): Page URL changed from ${initialUrl} to ${this.stagehand.page.url()}
|
|
2344
|
+
|
|
2345
|
+
`;
|
|
2346
|
+
}
|
|
2347
|
+
const actionCompleted = yield this._verifyActionCompletion({
|
|
2348
|
+
completed: response.completed,
|
|
2349
|
+
verifierUseVision,
|
|
2350
|
+
requestId,
|
|
2351
|
+
action,
|
|
2352
|
+
steps,
|
|
2353
|
+
model,
|
|
2354
|
+
domSettleTimeoutMs
|
|
2355
|
+
});
|
|
2356
|
+
if (!actionCompleted) {
|
|
2357
|
+
this.logger({
|
|
2358
|
+
category: "action",
|
|
2359
|
+
message: `Continuing to next action step`,
|
|
2360
|
+
level: 1
|
|
2361
|
+
});
|
|
2362
|
+
return this.act({
|
|
2363
|
+
action,
|
|
2364
|
+
steps,
|
|
2365
|
+
modelName,
|
|
2366
|
+
chunksSeen,
|
|
2367
|
+
useVision,
|
|
2368
|
+
verifierUseVision,
|
|
2369
|
+
requestId,
|
|
2370
|
+
variables,
|
|
2371
|
+
previousSelectors: [...previousSelectors, xpaths[0]],
|
|
2372
|
+
skipActionCacheForThisStep: false,
|
|
2373
|
+
domSettleTimeoutMs
|
|
2374
|
+
});
|
|
2375
|
+
} else {
|
|
2376
|
+
this.logger({
|
|
2377
|
+
category: "action",
|
|
2378
|
+
message: `Action completed successfully`,
|
|
2379
|
+
level: 1
|
|
2380
|
+
});
|
|
2381
|
+
yield this._recordAction(action, response.step);
|
|
2382
|
+
return {
|
|
2383
|
+
success: true,
|
|
2384
|
+
message: `Action completed successfully: ${steps}${response.step}`,
|
|
2385
|
+
action
|
|
2386
|
+
};
|
|
2387
|
+
}
|
|
2388
|
+
} catch (error) {
|
|
2389
|
+
this.logger({
|
|
2390
|
+
category: "action",
|
|
2391
|
+
message: `Error performing action - D (Retries: ${retries}): ${error.message}
|
|
2392
|
+
Trace: ${error.stack}`,
|
|
2393
|
+
level: 1
|
|
2394
|
+
});
|
|
2395
|
+
if (retries < 2) {
|
|
2396
|
+
return this.act({
|
|
2397
|
+
action,
|
|
2398
|
+
steps,
|
|
2399
|
+
modelName,
|
|
2400
|
+
useVision,
|
|
2401
|
+
verifierUseVision,
|
|
2402
|
+
retries: retries + 1,
|
|
2403
|
+
chunksSeen,
|
|
2404
|
+
requestId,
|
|
2405
|
+
variables,
|
|
2406
|
+
previousSelectors,
|
|
2407
|
+
skipActionCacheForThisStep,
|
|
2408
|
+
domSettleTimeoutMs
|
|
2409
|
+
});
|
|
2410
|
+
}
|
|
2411
|
+
yield this._recordAction(action, "");
|
|
2412
|
+
if (this.enableCaching) {
|
|
2413
|
+
this.llmProvider.cleanRequestCache(requestId);
|
|
2414
|
+
this.actionCache.deleteCacheForRequestId(requestId);
|
|
2415
|
+
}
|
|
2416
|
+
return {
|
|
2417
|
+
success: false,
|
|
2418
|
+
message: `Error performing action - A: ${error.message}`,
|
|
2419
|
+
action
|
|
2420
|
+
};
|
|
2421
|
+
}
|
|
2422
|
+
} catch (error) {
|
|
2423
|
+
this.logger({
|
|
2424
|
+
category: "action",
|
|
2425
|
+
message: `Error performing action - B: ${error.message}
|
|
2426
|
+
Trace: ${error.stack}`,
|
|
2427
|
+
level: 1
|
|
2428
|
+
});
|
|
2429
|
+
if (this.enableCaching) {
|
|
2430
|
+
this.llmProvider.cleanRequestCache(requestId);
|
|
2431
|
+
this.actionCache.deleteCacheForRequestId(requestId);
|
|
2432
|
+
}
|
|
2433
|
+
return {
|
|
2434
|
+
success: false,
|
|
2435
|
+
message: `Error performing action - C: ${error.message}`,
|
|
2436
|
+
action
|
|
2437
|
+
};
|
|
2438
|
+
}
|
|
2439
|
+
});
|
|
2440
|
+
}
|
|
2441
|
+
};
|
|
2442
|
+
|
|
2443
|
+
// lib/index.ts
|
|
2444
|
+
require("dotenv").config({ path: ".env" });
|
|
2445
|
+
function getBrowser(apiKey, projectId, env = "LOCAL", headless = false, logger, browserbaseSessionCreateParams, browserbaseResumeSessionID) {
|
|
2446
|
+
return __async(this, null, function* () {
|
|
2447
|
+
if (env === "BROWSERBASE") {
|
|
2448
|
+
if (!apiKey) {
|
|
2449
|
+
logger({
|
|
2450
|
+
category: "Init",
|
|
2451
|
+
message: "BROWSERBASE_API_KEY is required to use BROWSERBASE env. Defaulting to LOCAL.",
|
|
2452
|
+
level: 0
|
|
2453
|
+
});
|
|
2454
|
+
env = "LOCAL";
|
|
2455
|
+
}
|
|
2456
|
+
if (!projectId) {
|
|
2457
|
+
logger({
|
|
2458
|
+
category: "Init",
|
|
2459
|
+
message: "BROWSERBASE_PROJECT_ID is required for some Browserbase features that may not work without it.",
|
|
2460
|
+
level: 1
|
|
2461
|
+
});
|
|
2462
|
+
}
|
|
2463
|
+
}
|
|
2464
|
+
if (env === "BROWSERBASE") {
|
|
2465
|
+
if (!apiKey) {
|
|
2466
|
+
throw new Error("BROWSERBASE_API_KEY is required.");
|
|
2467
|
+
}
|
|
2468
|
+
let debugUrl = void 0;
|
|
2469
|
+
let sessionUrl = void 0;
|
|
2470
|
+
let sessionId;
|
|
2471
|
+
let connectUrl;
|
|
2472
|
+
const browserbase = new import_sdk2.Browserbase({
|
|
2473
|
+
apiKey
|
|
2474
|
+
});
|
|
2475
|
+
if (browserbaseResumeSessionID) {
|
|
2476
|
+
try {
|
|
1354
2477
|
const sessionStatus = yield browserbase.sessions.retrieve(
|
|
1355
2478
|
browserbaseResumeSessionID
|
|
1356
2479
|
);
|
|
@@ -1507,20 +2630,31 @@ var Stagehand = class {
|
|
|
1507
2630
|
this.llmProvider = llmProvider || new LLMProvider(this.logger, this.enableCaching);
|
|
1508
2631
|
this.env = env;
|
|
1509
2632
|
this.observations = {};
|
|
1510
|
-
this.apiKey = apiKey;
|
|
1511
|
-
this.projectId = projectId;
|
|
1512
|
-
this.actions = {};
|
|
2633
|
+
this.apiKey = apiKey != null ? apiKey : process.env.BROWSERBASE_API_KEY;
|
|
2634
|
+
this.projectId = projectId != null ? projectId : process.env.BROWSERBASE_PROJECT_ID;
|
|
1513
2635
|
this.verbose = verbose != null ? verbose : 0;
|
|
1514
2636
|
this.debugDom = debugDom != null ? debugDom : false;
|
|
1515
2637
|
this.defaultModelName = "gpt-4o";
|
|
1516
2638
|
this.domSettleTimeoutMs = domSettleTimeoutMs != null ? domSettleTimeoutMs : 3e4;
|
|
1517
2639
|
this.headless = headless != null ? headless : false;
|
|
1518
2640
|
this.browserBaseSessionCreateParams = browserBaseSessionCreateParams;
|
|
2641
|
+
this.actHandler = new StagehandActHandler({
|
|
2642
|
+
stagehand: this,
|
|
2643
|
+
verbose: this.verbose,
|
|
2644
|
+
llmProvider: this.llmProvider,
|
|
2645
|
+
enableCaching: this.enableCaching,
|
|
2646
|
+
logger: this.logger,
|
|
2647
|
+
waitForSettledDom: this._waitForSettledDom.bind(this),
|
|
2648
|
+
defaultModelName: this.defaultModelName,
|
|
2649
|
+
startDomDebug: this.startDomDebug.bind(this),
|
|
2650
|
+
cleanupDomDebug: this.cleanupDomDebug.bind(this)
|
|
2651
|
+
});
|
|
1519
2652
|
this.browserbaseResumeSessionID = browserbaseResumeSessionID;
|
|
1520
2653
|
}
|
|
1521
2654
|
init() {
|
|
1522
2655
|
return __async(this, arguments, function* ({
|
|
1523
|
-
modelName = "gpt-4o"
|
|
2656
|
+
modelName = "gpt-4o",
|
|
2657
|
+
domSettleTimeoutMs
|
|
1524
2658
|
} = {}) {
|
|
1525
2659
|
const { context, debugUrl, sessionUrl } = yield getBrowser(
|
|
1526
2660
|
this.apiKey,
|
|
@@ -1536,7 +2670,10 @@ var Stagehand = class {
|
|
|
1536
2670
|
});
|
|
1537
2671
|
this.context = context;
|
|
1538
2672
|
this.page = context.pages()[0];
|
|
2673
|
+
yield this.page.waitForLoadState("domcontentloaded");
|
|
2674
|
+
yield this._waitForSettledDom();
|
|
1539
2675
|
this.defaultModelName = modelName;
|
|
2676
|
+
this.domSettleTimeoutMs = domSettleTimeoutMs != null ? domSettleTimeoutMs : this.domSettleTimeoutMs;
|
|
1540
2677
|
const originalGoto = this.page.goto.bind(this.page);
|
|
1541
2678
|
this.page.goto = (url, options) => __async(this, null, function* () {
|
|
1542
2679
|
const result = yield originalGoto(url, options);
|
|
@@ -1547,14 +2684,33 @@ var Stagehand = class {
|
|
|
1547
2684
|
if (this.headless) {
|
|
1548
2685
|
yield this.page.setViewportSize({ width: 1280, height: 720 });
|
|
1549
2686
|
}
|
|
1550
|
-
yield this.
|
|
1551
|
-
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "
|
|
2687
|
+
yield this.context.addInitScript({
|
|
2688
|
+
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "xpathUtils.js"),
|
|
2689
|
+
content: import_fs2.default.readFileSync(
|
|
2690
|
+
import_path2.default.join(__dirname, "..", "dist", "dom", "build", "xpathUtils.js"),
|
|
2691
|
+
"utf8"
|
|
2692
|
+
)
|
|
2693
|
+
});
|
|
2694
|
+
yield this.context.addInitScript({
|
|
2695
|
+
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "process.js"),
|
|
2696
|
+
content: import_fs2.default.readFileSync(
|
|
2697
|
+
import_path2.default.join(__dirname, "..", "dist", "dom", "build", "process.js"),
|
|
2698
|
+
"utf8"
|
|
2699
|
+
)
|
|
1552
2700
|
});
|
|
1553
|
-
yield this.
|
|
1554
|
-
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "utils.js")
|
|
2701
|
+
yield this.context.addInitScript({
|
|
2702
|
+
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "utils.js"),
|
|
2703
|
+
content: import_fs2.default.readFileSync(
|
|
2704
|
+
import_path2.default.join(__dirname, "..", "dist", "dom", "build", "utils.js"),
|
|
2705
|
+
"utf8"
|
|
2706
|
+
)
|
|
1555
2707
|
});
|
|
1556
|
-
yield this.
|
|
1557
|
-
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "debug.js")
|
|
2708
|
+
yield this.context.addInitScript({
|
|
2709
|
+
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "debug.js"),
|
|
2710
|
+
content: import_fs2.default.readFileSync(
|
|
2711
|
+
import_path2.default.join(__dirname, "..", "dist", "dom", "build", "debug.js"),
|
|
2712
|
+
"utf8"
|
|
2713
|
+
)
|
|
1558
2714
|
});
|
|
1559
2715
|
return { debugUrl, sessionUrl };
|
|
1560
2716
|
});
|
|
@@ -1574,14 +2730,33 @@ var Stagehand = class {
|
|
|
1574
2730
|
if (this.headless) {
|
|
1575
2731
|
yield this.page.setViewportSize({ width: 1280, height: 720 });
|
|
1576
2732
|
}
|
|
1577
|
-
yield this.
|
|
1578
|
-
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "
|
|
2733
|
+
yield this.context.addInitScript({
|
|
2734
|
+
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "xpathUtils.js"),
|
|
2735
|
+
content: import_fs2.default.readFileSync(
|
|
2736
|
+
import_path2.default.join(__dirname, "..", "dist", "dom", "build", "xpathUtils.js"),
|
|
2737
|
+
"utf8"
|
|
2738
|
+
)
|
|
2739
|
+
});
|
|
2740
|
+
yield this.context.addInitScript({
|
|
2741
|
+
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "process.js"),
|
|
2742
|
+
content: import_fs2.default.readFileSync(
|
|
2743
|
+
import_path2.default.join(__dirname, "..", "dist", "dom", "build", "process.js"),
|
|
2744
|
+
"utf8"
|
|
2745
|
+
)
|
|
1579
2746
|
});
|
|
1580
|
-
yield this.
|
|
1581
|
-
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "utils.js")
|
|
2747
|
+
yield this.context.addInitScript({
|
|
2748
|
+
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "utils.js"),
|
|
2749
|
+
content: import_fs2.default.readFileSync(
|
|
2750
|
+
import_path2.default.join(__dirname, "..", "dist", "dom", "build", "utils.js"),
|
|
2751
|
+
"utf8"
|
|
2752
|
+
)
|
|
1582
2753
|
});
|
|
1583
|
-
yield this.
|
|
1584
|
-
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "debug.js")
|
|
2754
|
+
yield this.context.addInitScript({
|
|
2755
|
+
path: import_path2.default.join(__dirname, "..", "dist", "dom", "build", "debug.js"),
|
|
2756
|
+
content: import_fs2.default.readFileSync(
|
|
2757
|
+
import_path2.default.join(__dirname, "..", "dist", "dom", "build", "debug.js"),
|
|
2758
|
+
"utf8"
|
|
2759
|
+
)
|
|
1585
2760
|
});
|
|
1586
2761
|
return { context: this.context };
|
|
1587
2762
|
});
|
|
@@ -1720,24 +2895,13 @@ Trace: ${e.stack}`,
|
|
|
1720
2895
|
}
|
|
1721
2896
|
});
|
|
1722
2897
|
}
|
|
1723
|
-
// Recording
|
|
1724
|
-
_generateId(operation) {
|
|
1725
|
-
return import_crypto.default.createHash("sha256").update(operation).digest("hex");
|
|
1726
|
-
}
|
|
1727
2898
|
_recordObservation(instruction, result) {
|
|
1728
2899
|
return __async(this, null, function* () {
|
|
1729
|
-
const id =
|
|
2900
|
+
const id = generateId(instruction);
|
|
1730
2901
|
this.observations[id] = { result, instruction };
|
|
1731
2902
|
return id;
|
|
1732
2903
|
});
|
|
1733
2904
|
}
|
|
1734
|
-
_recordAction(action, result) {
|
|
1735
|
-
return __async(this, null, function* () {
|
|
1736
|
-
const id = this._generateId(action);
|
|
1737
|
-
this.actions[id] = { result, action };
|
|
1738
|
-
return id;
|
|
1739
|
-
});
|
|
1740
|
-
}
|
|
1741
2905
|
// Main methods
|
|
1742
2906
|
_extract(_0) {
|
|
1743
2907
|
return __async(this, arguments, function* ({
|
|
@@ -1826,116 +2990,25 @@ Trace: ${e.stack}`,
|
|
|
1826
2990
|
domSettleTimeoutMs
|
|
1827
2991
|
}) {
|
|
1828
2992
|
if (!instruction) {
|
|
1829
|
-
instruction = `Find elements that can be used for any future actions in the page. These may be navigation links, related pages, section/subsection links, buttons, or other interactive elements. Be comprehensive: if there are multiple elements that may be relevant for future actions, return all of them.`;
|
|
1830
|
-
}
|
|
1831
|
-
const model = modelName != null ? modelName : this.defaultModelName;
|
|
1832
|
-
this.log({
|
|
1833
|
-
category: "observation",
|
|
1834
|
-
message: `starting observation: ${instruction}`,
|
|
1835
|
-
level: 1
|
|
1836
|
-
});
|
|
1837
|
-
yield this._waitForSettledDom(domSettleTimeoutMs);
|
|
1838
|
-
yield this.startDomDebug();
|
|
1839
|
-
let { outputString, selectorMap } = yield this.page.evaluate(
|
|
1840
|
-
(fullPage2) => fullPage2 ? window.processAllOfDom() : window.processDom([]),
|
|
1841
|
-
fullPage
|
|
1842
|
-
);
|
|
1843
|
-
let annotatedScreenshot;
|
|
1844
|
-
if (useVision === true) {
|
|
1845
|
-
if (!modelsWithVision.includes(model)) {
|
|
1846
|
-
this.log({
|
|
1847
|
-
category: "observation",
|
|
1848
|
-
message: `${model} does not support vision. Skipping vision processing.`,
|
|
1849
|
-
level: 1
|
|
1850
|
-
});
|
|
1851
|
-
} else {
|
|
1852
|
-
const screenshotService = new ScreenshotService(
|
|
1853
|
-
this.page,
|
|
1854
|
-
selectorMap,
|
|
1855
|
-
this.verbose
|
|
1856
|
-
);
|
|
1857
|
-
annotatedScreenshot = yield screenshotService.getAnnotatedScreenshot(fullPage);
|
|
1858
|
-
outputString = "n/a. use the image to find the elements.";
|
|
1859
|
-
}
|
|
1860
|
-
}
|
|
1861
|
-
const observationResponse = yield observe({
|
|
1862
|
-
instruction,
|
|
1863
|
-
domElements: outputString,
|
|
1864
|
-
llmProvider: this.llmProvider,
|
|
1865
|
-
modelName: modelName || this.defaultModelName,
|
|
1866
|
-
image: annotatedScreenshot,
|
|
1867
|
-
requestId
|
|
1868
|
-
});
|
|
1869
|
-
const elementsWithSelectors = observationResponse.elements.map(
|
|
1870
|
-
(element) => {
|
|
1871
|
-
const _a = element, { elementId } = _a, rest = __objRest(_a, ["elementId"]);
|
|
1872
|
-
return __spreadProps(__spreadValues({}, rest), {
|
|
1873
|
-
selector: `xpath=${selectorMap[elementId]}`
|
|
1874
|
-
});
|
|
1875
|
-
}
|
|
1876
|
-
);
|
|
1877
|
-
yield this.cleanupDomDebug();
|
|
1878
|
-
this._recordObservation(instruction, elementsWithSelectors);
|
|
1879
|
-
this.log({
|
|
1880
|
-
category: "observation",
|
|
1881
|
-
message: `found element ${JSON.stringify(elementsWithSelectors)}`,
|
|
1882
|
-
level: 1
|
|
1883
|
-
});
|
|
1884
|
-
yield this._recordObservation(instruction, elementsWithSelectors);
|
|
1885
|
-
return elementsWithSelectors;
|
|
1886
|
-
});
|
|
1887
|
-
}
|
|
1888
|
-
_act(_0) {
|
|
1889
|
-
return __async(this, arguments, function* ({
|
|
1890
|
-
action,
|
|
1891
|
-
steps = "",
|
|
1892
|
-
chunksSeen,
|
|
1893
|
-
modelName,
|
|
1894
|
-
useVision,
|
|
1895
|
-
verifierUseVision,
|
|
1896
|
-
retries = 0,
|
|
1897
|
-
requestId,
|
|
1898
|
-
domSettleTimeoutMs
|
|
1899
|
-
}) {
|
|
1900
|
-
var _a;
|
|
1901
|
-
const model = modelName != null ? modelName : this.defaultModelName;
|
|
1902
|
-
if (!modelsWithVision.includes(model) && (useVision !== false || verifierUseVision)) {
|
|
1903
|
-
this.log({
|
|
1904
|
-
category: "action",
|
|
1905
|
-
message: `${model} does not support vision, but useVision was set to ${useVision}. Defaulting to false.`,
|
|
1906
|
-
level: 1
|
|
1907
|
-
});
|
|
1908
|
-
useVision = false;
|
|
1909
|
-
verifierUseVision = false;
|
|
2993
|
+
instruction = `Find elements that can be used for any future actions in the page. These may be navigation links, related pages, section/subsection links, buttons, or other interactive elements. Be comprehensive: if there are multiple elements that may be relevant for future actions, return all of them.`;
|
|
1910
2994
|
}
|
|
2995
|
+
const model = modelName != null ? modelName : this.defaultModelName;
|
|
1911
2996
|
this.log({
|
|
1912
|
-
category: "
|
|
1913
|
-
message: `
|
|
1914
|
-
level:
|
|
2997
|
+
category: "observation",
|
|
2998
|
+
message: `starting observation: ${instruction}`,
|
|
2999
|
+
level: 1
|
|
1915
3000
|
});
|
|
1916
3001
|
yield this._waitForSettledDom(domSettleTimeoutMs);
|
|
1917
3002
|
yield this.startDomDebug();
|
|
1918
|
-
this.
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
level: 2
|
|
1922
|
-
});
|
|
1923
|
-
const { outputString, selectorMap, chunk, chunks } = yield this.page.evaluate(
|
|
1924
|
-
({ chunksSeen: chunksSeen2 }) => {
|
|
1925
|
-
return window.processDom(chunksSeen2);
|
|
1926
|
-
},
|
|
1927
|
-
{ chunksSeen }
|
|
3003
|
+
let { outputString, selectorMap } = yield this.page.evaluate(
|
|
3004
|
+
(fullPage2) => fullPage2 ? window.processAllOfDom() : window.processDom([]),
|
|
3005
|
+
fullPage
|
|
1928
3006
|
);
|
|
1929
|
-
this.log({
|
|
1930
|
-
category: "action",
|
|
1931
|
-
message: `Looking at chunk ${chunk}. Chunks left: ${chunks.length - chunksSeen.length}`,
|
|
1932
|
-
level: 1
|
|
1933
|
-
});
|
|
1934
3007
|
let annotatedScreenshot;
|
|
1935
3008
|
if (useVision === true) {
|
|
1936
3009
|
if (!modelsWithVision.includes(model)) {
|
|
1937
3010
|
this.log({
|
|
1938
|
-
category: "
|
|
3011
|
+
category: "observation",
|
|
1939
3012
|
message: `${model} does not support vision. Skipping vision processing.`,
|
|
1940
3013
|
level: 1
|
|
1941
3014
|
});
|
|
@@ -1945,419 +3018,35 @@ Trace: ${e.stack}`,
|
|
|
1945
3018
|
selectorMap,
|
|
1946
3019
|
this.verbose
|
|
1947
3020
|
);
|
|
1948
|
-
annotatedScreenshot = yield screenshotService.getAnnotatedScreenshot(
|
|
3021
|
+
annotatedScreenshot = yield screenshotService.getAnnotatedScreenshot(fullPage);
|
|
3022
|
+
outputString = "n/a. use the image to find the elements.";
|
|
1949
3023
|
}
|
|
1950
3024
|
}
|
|
1951
|
-
const
|
|
1952
|
-
|
|
3025
|
+
const observationResponse = yield observe({
|
|
3026
|
+
instruction,
|
|
1953
3027
|
domElements: outputString,
|
|
1954
|
-
steps,
|
|
1955
3028
|
llmProvider: this.llmProvider,
|
|
1956
|
-
modelName:
|
|
1957
|
-
|
|
1958
|
-
logger: this.logger,
|
|
3029
|
+
modelName: modelName || this.defaultModelName,
|
|
3030
|
+
image: annotatedScreenshot,
|
|
1959
3031
|
requestId
|
|
1960
3032
|
});
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
yield this.cleanupDomDebug();
|
|
1967
|
-
if (!response) {
|
|
1968
|
-
if (chunksSeen.length + 1 < chunks.length) {
|
|
1969
|
-
chunksSeen.push(chunk);
|
|
1970
|
-
this.log({
|
|
1971
|
-
category: "action",
|
|
1972
|
-
message: `No action found in current chunk. Chunks seen: ${chunksSeen.length}.`,
|
|
1973
|
-
level: 1
|
|
1974
|
-
});
|
|
1975
|
-
return this._act({
|
|
1976
|
-
action,
|
|
1977
|
-
steps: steps + (!steps.endsWith("\n") ? "\n" : "") + "## Step: Scrolled to another section\n",
|
|
1978
|
-
chunksSeen,
|
|
1979
|
-
modelName,
|
|
1980
|
-
useVision,
|
|
1981
|
-
verifierUseVision,
|
|
1982
|
-
requestId,
|
|
1983
|
-
domSettleTimeoutMs
|
|
1984
|
-
});
|
|
1985
|
-
} else if (useVision === "fallback") {
|
|
1986
|
-
this.log({
|
|
1987
|
-
category: "action",
|
|
1988
|
-
message: `Switching to vision-based processing`,
|
|
1989
|
-
level: 1
|
|
1990
|
-
});
|
|
1991
|
-
yield this.page.evaluate(() => window.scrollToHeight(0));
|
|
1992
|
-
return yield this._act({
|
|
1993
|
-
action,
|
|
1994
|
-
steps,
|
|
1995
|
-
chunksSeen,
|
|
1996
|
-
modelName,
|
|
1997
|
-
useVision: true,
|
|
1998
|
-
verifierUseVision,
|
|
1999
|
-
requestId,
|
|
2000
|
-
domSettleTimeoutMs
|
|
3033
|
+
const elementsWithSelectors = observationResponse.elements.map(
|
|
3034
|
+
(element) => {
|
|
3035
|
+
const _a = element, { elementId } = _a, rest = __objRest(_a, ["elementId"]);
|
|
3036
|
+
return __spreadProps(__spreadValues({}, rest), {
|
|
3037
|
+
selector: `xpath=${selectorMap[elementId][0]}`
|
|
2001
3038
|
});
|
|
2002
|
-
} else {
|
|
2003
|
-
if (this.enableCaching) {
|
|
2004
|
-
this.llmProvider.cleanRequestCache(requestId);
|
|
2005
|
-
}
|
|
2006
|
-
return {
|
|
2007
|
-
success: false,
|
|
2008
|
-
message: `Action was not able to be completed.`,
|
|
2009
|
-
action
|
|
2010
|
-
};
|
|
2011
3039
|
}
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
const method = response["method"];
|
|
2016
|
-
const args = response["args"];
|
|
2017
|
-
const elementLines = outputString.split("\n");
|
|
2018
|
-
const elementText = ((_a = elementLines.find((line) => line.startsWith(`${elementId}:`))) == null ? void 0 : _a.split(":")[1]) || "Element not found";
|
|
3040
|
+
);
|
|
3041
|
+
yield this.cleanupDomDebug();
|
|
3042
|
+
this._recordObservation(instruction, elementsWithSelectors);
|
|
2019
3043
|
this.log({
|
|
2020
|
-
category: "
|
|
2021
|
-
message: `
|
|
2022
|
-
args
|
|
2023
|
-
)}`,
|
|
3044
|
+
category: "observation",
|
|
3045
|
+
message: `found element ${JSON.stringify(elementsWithSelectors)}`,
|
|
2024
3046
|
level: 1
|
|
2025
3047
|
});
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
try {
|
|
2029
|
-
const initialUrl = this.page.url();
|
|
2030
|
-
if (method === "scrollIntoView") {
|
|
2031
|
-
this.log({
|
|
2032
|
-
category: "action",
|
|
2033
|
-
message: `Scrolling element into view`,
|
|
2034
|
-
level: 2
|
|
2035
|
-
});
|
|
2036
|
-
try {
|
|
2037
|
-
yield locator.evaluate((element) => {
|
|
2038
|
-
element.scrollIntoView({ behavior: "smooth", block: "center" });
|
|
2039
|
-
}).catch((e) => {
|
|
2040
|
-
this.log({
|
|
2041
|
-
category: "action",
|
|
2042
|
-
message: `Error scrolling element into view: ${e.message}
|
|
2043
|
-
Trace: ${e.stack}`,
|
|
2044
|
-
level: 1
|
|
2045
|
-
});
|
|
2046
|
-
});
|
|
2047
|
-
} catch (e) {
|
|
2048
|
-
this.log({
|
|
2049
|
-
category: "action",
|
|
2050
|
-
message: `Error scrolling element into view (Retries ${retries}): ${e.message}
|
|
2051
|
-
Trace: ${e.stack}`,
|
|
2052
|
-
level: 1
|
|
2053
|
-
});
|
|
2054
|
-
if (retries < 2) {
|
|
2055
|
-
return this._act({
|
|
2056
|
-
action,
|
|
2057
|
-
steps,
|
|
2058
|
-
modelName,
|
|
2059
|
-
useVision,
|
|
2060
|
-
verifierUseVision,
|
|
2061
|
-
retries: retries + 1,
|
|
2062
|
-
chunksSeen,
|
|
2063
|
-
requestId,
|
|
2064
|
-
domSettleTimeoutMs
|
|
2065
|
-
});
|
|
2066
|
-
}
|
|
2067
|
-
}
|
|
2068
|
-
} else if (method === "fill" || method === "type") {
|
|
2069
|
-
try {
|
|
2070
|
-
yield locator.fill("");
|
|
2071
|
-
yield locator.click();
|
|
2072
|
-
const text = args[0];
|
|
2073
|
-
for (const char of text) {
|
|
2074
|
-
yield this.page.keyboard.type(char, {
|
|
2075
|
-
delay: Math.random() * 50 + 25
|
|
2076
|
-
});
|
|
2077
|
-
}
|
|
2078
|
-
} catch (e) {
|
|
2079
|
-
this.log({
|
|
2080
|
-
category: "action",
|
|
2081
|
-
message: `Error filling element (Retries ${retries}): ${e.message}
|
|
2082
|
-
Trace: ${e.stack}`,
|
|
2083
|
-
level: 1
|
|
2084
|
-
});
|
|
2085
|
-
if (retries < 2) {
|
|
2086
|
-
return this._act({
|
|
2087
|
-
action,
|
|
2088
|
-
steps,
|
|
2089
|
-
modelName,
|
|
2090
|
-
useVision,
|
|
2091
|
-
verifierUseVision,
|
|
2092
|
-
retries: retries + 1,
|
|
2093
|
-
chunksSeen,
|
|
2094
|
-
requestId,
|
|
2095
|
-
domSettleTimeoutMs
|
|
2096
|
-
});
|
|
2097
|
-
}
|
|
2098
|
-
}
|
|
2099
|
-
} else if (method === "press") {
|
|
2100
|
-
try {
|
|
2101
|
-
const key = args[0];
|
|
2102
|
-
yield this.page.keyboard.press(key);
|
|
2103
|
-
} catch (e) {
|
|
2104
|
-
this.log({
|
|
2105
|
-
category: "action",
|
|
2106
|
-
message: `Error pressing key (Retries ${retries}): ${e.message}
|
|
2107
|
-
Trace: ${e.stack}`,
|
|
2108
|
-
level: 1
|
|
2109
|
-
});
|
|
2110
|
-
if (retries < 2) {
|
|
2111
|
-
return this._act({
|
|
2112
|
-
action,
|
|
2113
|
-
steps,
|
|
2114
|
-
modelName,
|
|
2115
|
-
useVision,
|
|
2116
|
-
verifierUseVision,
|
|
2117
|
-
retries: retries + 1,
|
|
2118
|
-
chunksSeen,
|
|
2119
|
-
requestId,
|
|
2120
|
-
domSettleTimeoutMs
|
|
2121
|
-
});
|
|
2122
|
-
}
|
|
2123
|
-
}
|
|
2124
|
-
} else if (typeof locator[method] === "function") {
|
|
2125
|
-
this.log({
|
|
2126
|
-
category: "action",
|
|
2127
|
-
message: `Page URL before action: ${this.page.url()}`,
|
|
2128
|
-
level: 2
|
|
2129
|
-
});
|
|
2130
|
-
try {
|
|
2131
|
-
yield locator[method](...args);
|
|
2132
|
-
} catch (e) {
|
|
2133
|
-
this.log({
|
|
2134
|
-
category: "action",
|
|
2135
|
-
message: `Error performing method ${method} with args ${JSON.stringify(
|
|
2136
|
-
args
|
|
2137
|
-
)} (Retries: ${retries}): ${e.message}
|
|
2138
|
-
Trace: ${e.stack}`,
|
|
2139
|
-
level: 1
|
|
2140
|
-
});
|
|
2141
|
-
if (retries < 2) {
|
|
2142
|
-
return this._act({
|
|
2143
|
-
action,
|
|
2144
|
-
steps,
|
|
2145
|
-
modelName,
|
|
2146
|
-
useVision,
|
|
2147
|
-
verifierUseVision,
|
|
2148
|
-
retries: retries + 1,
|
|
2149
|
-
chunksSeen,
|
|
2150
|
-
requestId,
|
|
2151
|
-
domSettleTimeoutMs
|
|
2152
|
-
});
|
|
2153
|
-
}
|
|
2154
|
-
}
|
|
2155
|
-
if (method === "click") {
|
|
2156
|
-
this.log({
|
|
2157
|
-
category: "action",
|
|
2158
|
-
message: `Clicking element, checking for page navigation`,
|
|
2159
|
-
level: 1
|
|
2160
|
-
});
|
|
2161
|
-
const newOpenedTab = yield Promise.race([
|
|
2162
|
-
new Promise((resolve) => {
|
|
2163
|
-
this.context.once("page", (page) => resolve(page));
|
|
2164
|
-
setTimeout(() => resolve(null), 1500);
|
|
2165
|
-
})
|
|
2166
|
-
]);
|
|
2167
|
-
this.log({
|
|
2168
|
-
category: "action",
|
|
2169
|
-
message: `Clicked element, ${newOpenedTab ? "opened a new tab" : "no new tabs opened"}`,
|
|
2170
|
-
level: 1
|
|
2171
|
-
});
|
|
2172
|
-
if (newOpenedTab) {
|
|
2173
|
-
this.log({
|
|
2174
|
-
category: "action",
|
|
2175
|
-
message: `New page detected (new tab) with URL: ${newOpenedTab.url()}`,
|
|
2176
|
-
level: 1
|
|
2177
|
-
});
|
|
2178
|
-
yield newOpenedTab.close();
|
|
2179
|
-
yield this.page.goto(newOpenedTab.url());
|
|
2180
|
-
yield this.page.waitForLoadState("domcontentloaded");
|
|
2181
|
-
yield this._waitForSettledDom(domSettleTimeoutMs);
|
|
2182
|
-
}
|
|
2183
|
-
yield Promise.race([
|
|
2184
|
-
this.page.waitForLoadState("networkidle"),
|
|
2185
|
-
new Promise((resolve) => setTimeout(resolve, 5e3))
|
|
2186
|
-
]).catch((e) => {
|
|
2187
|
-
this.log({
|
|
2188
|
-
category: "action",
|
|
2189
|
-
message: `Network idle timeout hit`,
|
|
2190
|
-
level: 1
|
|
2191
|
-
});
|
|
2192
|
-
});
|
|
2193
|
-
this.log({
|
|
2194
|
-
category: "action",
|
|
2195
|
-
message: `Finished waiting for (possible) page navigation`,
|
|
2196
|
-
level: 1
|
|
2197
|
-
});
|
|
2198
|
-
if (this.page.url() !== initialUrl) {
|
|
2199
|
-
this.log({
|
|
2200
|
-
category: "action",
|
|
2201
|
-
message: `New page detected with URL: ${this.page.url()}`,
|
|
2202
|
-
level: 1
|
|
2203
|
-
});
|
|
2204
|
-
}
|
|
2205
|
-
}
|
|
2206
|
-
} else {
|
|
2207
|
-
this.log({
|
|
2208
|
-
category: "action",
|
|
2209
|
-
message: `Chosen method ${method} is invalid`,
|
|
2210
|
-
level: 1
|
|
2211
|
-
});
|
|
2212
|
-
if (retries < 2) {
|
|
2213
|
-
return this._act({
|
|
2214
|
-
action,
|
|
2215
|
-
steps,
|
|
2216
|
-
modelName: model,
|
|
2217
|
-
useVision,
|
|
2218
|
-
verifierUseVision,
|
|
2219
|
-
retries: retries + 1,
|
|
2220
|
-
chunksSeen,
|
|
2221
|
-
requestId
|
|
2222
|
-
});
|
|
2223
|
-
} else {
|
|
2224
|
-
if (this.enableCaching) {
|
|
2225
|
-
this.llmProvider.cleanRequestCache(requestId);
|
|
2226
|
-
}
|
|
2227
|
-
return {
|
|
2228
|
-
success: false,
|
|
2229
|
-
message: `Internal error: Chosen method ${method} is invalid`,
|
|
2230
|
-
action
|
|
2231
|
-
};
|
|
2232
|
-
}
|
|
2233
|
-
}
|
|
2234
|
-
let newSteps = steps + (!steps.endsWith("\n") ? "\n" : "") + `## Step: ${response.step}
|
|
2235
|
-
Element: ${elementText}
|
|
2236
|
-
Action: ${response.method}
|
|
2237
|
-
Reasoning: ${response.why}
|
|
2238
|
-
`;
|
|
2239
|
-
if (urlChangeString) {
|
|
2240
|
-
newSteps += ` Result (Important): ${urlChangeString}
|
|
2241
|
-
|
|
2242
|
-
`;
|
|
2243
|
-
}
|
|
2244
|
-
let actionComplete = false;
|
|
2245
|
-
if (response.completed) {
|
|
2246
|
-
this.log({
|
|
2247
|
-
category: "action",
|
|
2248
|
-
message: `Action marked as completed, Verifying if this is true...`,
|
|
2249
|
-
level: 1
|
|
2250
|
-
});
|
|
2251
|
-
let domElements = void 0;
|
|
2252
|
-
let fullpageScreenshot = void 0;
|
|
2253
|
-
if (verifierUseVision) {
|
|
2254
|
-
try {
|
|
2255
|
-
const screenshotService = new ScreenshotService(
|
|
2256
|
-
this.page,
|
|
2257
|
-
selectorMap,
|
|
2258
|
-
this.verbose
|
|
2259
|
-
);
|
|
2260
|
-
fullpageScreenshot = yield screenshotService.getScreenshot(
|
|
2261
|
-
true,
|
|
2262
|
-
15
|
|
2263
|
-
);
|
|
2264
|
-
} catch (e) {
|
|
2265
|
-
this.log({
|
|
2266
|
-
category: "action",
|
|
2267
|
-
message: `Error getting full page screenshot: ${e.message}
|
|
2268
|
-
. Trying again...`,
|
|
2269
|
-
level: 1
|
|
2270
|
-
});
|
|
2271
|
-
const screenshotService = new ScreenshotService(
|
|
2272
|
-
this.page,
|
|
2273
|
-
selectorMap,
|
|
2274
|
-
this.verbose
|
|
2275
|
-
);
|
|
2276
|
-
fullpageScreenshot = yield screenshotService.getScreenshot(
|
|
2277
|
-
true,
|
|
2278
|
-
15
|
|
2279
|
-
);
|
|
2280
|
-
}
|
|
2281
|
-
} else {
|
|
2282
|
-
({ outputString: domElements } = yield this.page.evaluate(() => {
|
|
2283
|
-
return window.processAllOfDom();
|
|
2284
|
-
}));
|
|
2285
|
-
}
|
|
2286
|
-
actionComplete = yield verifyActCompletion({
|
|
2287
|
-
goal: action,
|
|
2288
|
-
steps: newSteps,
|
|
2289
|
-
llmProvider: this.llmProvider,
|
|
2290
|
-
modelName: model,
|
|
2291
|
-
screenshot: fullpageScreenshot,
|
|
2292
|
-
domElements,
|
|
2293
|
-
logger: this.logger,
|
|
2294
|
-
requestId
|
|
2295
|
-
});
|
|
2296
|
-
this.log({
|
|
2297
|
-
category: "action",
|
|
2298
|
-
message: `Action completion verification result: ${actionComplete}`,
|
|
2299
|
-
level: 1
|
|
2300
|
-
});
|
|
2301
|
-
}
|
|
2302
|
-
if (!actionComplete) {
|
|
2303
|
-
this.log({
|
|
2304
|
-
category: "action",
|
|
2305
|
-
message: `Continuing to next action step`,
|
|
2306
|
-
level: 1
|
|
2307
|
-
});
|
|
2308
|
-
return this._act({
|
|
2309
|
-
action,
|
|
2310
|
-
steps: newSteps,
|
|
2311
|
-
modelName,
|
|
2312
|
-
chunksSeen,
|
|
2313
|
-
useVision,
|
|
2314
|
-
verifierUseVision,
|
|
2315
|
-
requestId,
|
|
2316
|
-
domSettleTimeoutMs
|
|
2317
|
-
});
|
|
2318
|
-
} else {
|
|
2319
|
-
this.log({
|
|
2320
|
-
category: "action",
|
|
2321
|
-
message: `Action completed successfully`,
|
|
2322
|
-
level: 1
|
|
2323
|
-
});
|
|
2324
|
-
yield this._recordAction(action, response.step);
|
|
2325
|
-
return {
|
|
2326
|
-
success: true,
|
|
2327
|
-
message: `Action completed successfully: ${steps}${response.step}`,
|
|
2328
|
-
action
|
|
2329
|
-
};
|
|
2330
|
-
}
|
|
2331
|
-
} catch (error) {
|
|
2332
|
-
this.log({
|
|
2333
|
-
category: "action",
|
|
2334
|
-
message: `Error performing action (Retries: ${retries}): ${error.message}
|
|
2335
|
-
Trace: ${error.stack}`,
|
|
2336
|
-
level: 1
|
|
2337
|
-
});
|
|
2338
|
-
if (retries < 2) {
|
|
2339
|
-
return this._act({
|
|
2340
|
-
action,
|
|
2341
|
-
steps,
|
|
2342
|
-
modelName,
|
|
2343
|
-
useVision,
|
|
2344
|
-
verifierUseVision,
|
|
2345
|
-
retries: retries + 1,
|
|
2346
|
-
chunksSeen,
|
|
2347
|
-
requestId,
|
|
2348
|
-
domSettleTimeoutMs
|
|
2349
|
-
});
|
|
2350
|
-
}
|
|
2351
|
-
yield this._recordAction(action, "");
|
|
2352
|
-
if (this.enableCaching) {
|
|
2353
|
-
this.llmProvider.cleanRequestCache(requestId);
|
|
2354
|
-
}
|
|
2355
|
-
return {
|
|
2356
|
-
success: false,
|
|
2357
|
-
message: `Error performing action: ${error.message}`,
|
|
2358
|
-
action
|
|
2359
|
-
};
|
|
2360
|
-
}
|
|
3048
|
+
yield this._recordObservation(instruction, elementsWithSelectors);
|
|
3049
|
+
return elementsWithSelectors;
|
|
2361
3050
|
});
|
|
2362
3051
|
}
|
|
2363
3052
|
act(_0) {
|
|
@@ -2365,6 +3054,7 @@ Trace: ${error.stack}`,
|
|
|
2365
3054
|
action,
|
|
2366
3055
|
modelName,
|
|
2367
3056
|
useVision = "fallback",
|
|
3057
|
+
variables = {},
|
|
2368
3058
|
domSettleTimeoutMs
|
|
2369
3059
|
}) {
|
|
2370
3060
|
useVision = useVision != null ? useVision : "fallback";
|
|
@@ -2373,13 +3063,19 @@ Trace: ${error.stack}`,
|
|
|
2373
3063
|
category: "act",
|
|
2374
3064
|
message: `Running act with action: ${action}, requestId: ${requestId}`
|
|
2375
3065
|
});
|
|
2376
|
-
|
|
3066
|
+
if (variables) {
|
|
3067
|
+
this.variables = __spreadValues(__spreadValues({}, this.variables), variables);
|
|
3068
|
+
}
|
|
3069
|
+
return this.actHandler.act({
|
|
2377
3070
|
action,
|
|
2378
3071
|
modelName,
|
|
2379
3072
|
chunksSeen: [],
|
|
2380
3073
|
useVision,
|
|
2381
3074
|
verifierUseVision: useVision !== false,
|
|
2382
3075
|
requestId,
|
|
3076
|
+
variables,
|
|
3077
|
+
previousSelectors: [],
|
|
3078
|
+
skipActionCacheForThisStep: false,
|
|
2383
3079
|
domSettleTimeoutMs
|
|
2384
3080
|
}).catch((e) => {
|
|
2385
3081
|
this.logger({
|
|
@@ -2387,9 +3083,6 @@ Trace: ${error.stack}`,
|
|
|
2387
3083
|
message: `Error acting: ${e.message}
|
|
2388
3084
|
Trace: ${e.stack}`
|
|
2389
3085
|
});
|
|
2390
|
-
if (this.enableCaching) {
|
|
2391
|
-
this.llmProvider.cleanRequestCache(requestId);
|
|
2392
|
-
}
|
|
2393
3086
|
return {
|
|
2394
3087
|
success: false,
|
|
2395
3088
|
message: `Internal error: Error acting: ${e.message}`,
|