@midscene/web 0.7.1 → 0.7.2-beta-20241024094141.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/playground.js +8438 -0
- package/dist/browser/types/playground.d.ts +313 -0
- package/dist/es/appium.js +680 -604
- package/dist/es/debug.js +95 -73
- package/dist/es/index.js +939 -797
- package/dist/es/midscene-playground.js +678 -609
- package/dist/es/playground.js +593 -1024
- package/dist/es/playwright-report.js +29 -11
- package/dist/es/playwright.js +705 -597
- package/dist/es/puppeteer.js +636 -552
- package/dist/lib/appium.js +688 -609
- package/dist/lib/debug.js +95 -73
- package/dist/lib/index.js +950 -804
- package/dist/lib/midscene-playground.js +687 -615
- package/dist/lib/playground.js +586 -1007
- package/dist/lib/playwright-report.js +30 -9
- package/dist/lib/playwright.js +713 -602
- package/dist/lib/puppeteer.js +644 -557
- package/dist/script/htmlElement.js +11 -10
- package/dist/script/htmlElementDebug.js +11 -10
- package/dist/types/appium.d.ts +2 -3
- package/dist/types/debug.d.ts +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/{page-ad820b3c.d.ts → page-8117b0ad.d.ts} +8 -7
- package/dist/types/playground.d.ts +6 -21
- package/dist/types/playwright.d.ts +3 -4
- package/dist/types/puppeteer.d.ts +2 -3
- package/dist/types/{tasks-82c1054b.d.ts → tasks-cb6bf758.d.ts} +6 -6
- package/package.json +11 -5
- package/static/index.html +1 -1
package/dist/lib/appium.js
CHANGED
|
@@ -46,6 +46,26 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
46
46
|
mod
|
|
47
47
|
));
|
|
48
48
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
49
|
+
var __async = (__this, __arguments, generator) => {
|
|
50
|
+
return new Promise((resolve, reject) => {
|
|
51
|
+
var fulfilled = (value) => {
|
|
52
|
+
try {
|
|
53
|
+
step(generator.next(value));
|
|
54
|
+
} catch (e) {
|
|
55
|
+
reject(e);
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
var rejected = (value) => {
|
|
59
|
+
try {
|
|
60
|
+
step(generator.throw(value));
|
|
61
|
+
} catch (e) {
|
|
62
|
+
reject(e);
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
|
|
66
|
+
step((generator = generator.apply(__this, __arguments)).next());
|
|
67
|
+
});
|
|
68
|
+
};
|
|
49
69
|
|
|
50
70
|
// ../../node_modules/.pnpm/dayjs@1.11.11/node_modules/dayjs/dayjs.min.js
|
|
51
71
|
var require_dayjs_min = __commonJS({
|
|
@@ -870,29 +890,29 @@ __export(appium_exports, {
|
|
|
870
890
|
module.exports = __toCommonJS(appium_exports);
|
|
871
891
|
|
|
872
892
|
// src/common/agent.ts
|
|
873
|
-
var
|
|
893
|
+
var import_utils7 = require("@midscene/core/utils");
|
|
874
894
|
|
|
875
895
|
// src/common/tasks.ts
|
|
876
896
|
var import_node_assert2 = __toESM(require("assert"));
|
|
877
897
|
var import_core = require("@midscene/core");
|
|
878
|
-
var
|
|
879
|
-
var import_img3 = require("@midscene/shared/img");
|
|
898
|
+
var import_utils5 = require("@midscene/core/utils");
|
|
880
899
|
|
|
881
900
|
// src/common/task-cache.ts
|
|
882
901
|
var import_node_fs2 = require("fs");
|
|
883
902
|
var import_node_path2 = require("path");
|
|
884
|
-
var
|
|
903
|
+
var import_utils2 = require("@midscene/core/utils");
|
|
885
904
|
var import_fs2 = require("@midscene/shared/fs");
|
|
905
|
+
var import_utils3 = require("@midscene/shared/utils");
|
|
886
906
|
|
|
887
907
|
// src/common/utils.ts
|
|
888
908
|
var import_node_assert = __toESM(require("assert"));
|
|
889
|
-
var import_node_crypto = require("crypto");
|
|
890
909
|
var import_node_fs = require("fs");
|
|
891
910
|
var import_node_path = __toESM(require("path"));
|
|
892
911
|
var import_constants = require("@midscene/shared/constants");
|
|
893
912
|
var import_fs = require("@midscene/shared/fs");
|
|
894
913
|
var import_img = require("@midscene/shared/img");
|
|
895
914
|
var import_img2 = require("@midscene/shared/img");
|
|
915
|
+
var import_utils = require("@midscene/shared/utils");
|
|
896
916
|
var import_dayjs = __toESM(require_dayjs_min());
|
|
897
917
|
|
|
898
918
|
// src/web-element.ts
|
|
@@ -921,56 +941,59 @@ var WebElementInfo = class {
|
|
|
921
941
|
};
|
|
922
942
|
|
|
923
943
|
// src/common/utils.ts
|
|
924
|
-
|
|
925
|
-
(
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
const url = page.url();
|
|
930
|
-
const file = await page.screenshot();
|
|
931
|
-
const screenshotBase64 = (0, import_img.base64Encoded)(file);
|
|
932
|
-
const captureElementSnapshot = await page.getElementInfos();
|
|
933
|
-
const elementsInfo = await alignElements(captureElementSnapshot, page);
|
|
934
|
-
const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
|
|
935
|
-
if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
|
|
936
|
-
return false;
|
|
944
|
+
function parseContextFromWebPage(page, _opt) {
|
|
945
|
+
return __async(this, null, function* () {
|
|
946
|
+
(0, import_node_assert.default)(page, "page is required");
|
|
947
|
+
if (page._forceUsePageContext) {
|
|
948
|
+
return yield page._forceUsePageContext();
|
|
937
949
|
}
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
950
|
+
const url = page.url();
|
|
951
|
+
const screenshotBase64 = yield page.screenshotBase64();
|
|
952
|
+
const captureElementSnapshot = yield page.getElementInfos();
|
|
953
|
+
const elementsInfo = yield alignElements(captureElementSnapshot, page);
|
|
954
|
+
const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
|
|
955
|
+
if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
|
|
956
|
+
return false;
|
|
957
|
+
}
|
|
958
|
+
return true;
|
|
959
|
+
});
|
|
960
|
+
const size = yield (0, import_img.imageInfoOfBase64)(screenshotBase64);
|
|
961
|
+
const screenshotBase64WithElementMarker = yield (0, import_img2.compositeElementInfoImg)({
|
|
962
|
+
inputImgBase64: screenshotBase64,
|
|
963
|
+
elementsPositionInfo: elementsPositionInfoWithoutText
|
|
964
|
+
});
|
|
965
|
+
return {
|
|
966
|
+
content: elementsInfo,
|
|
967
|
+
size,
|
|
968
|
+
screenshotBase64,
|
|
969
|
+
screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
|
|
970
|
+
url
|
|
971
|
+
};
|
|
944
972
|
});
|
|
945
|
-
return {
|
|
946
|
-
content: elementsInfo,
|
|
947
|
-
size,
|
|
948
|
-
screenshotBase64,
|
|
949
|
-
screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
|
|
950
|
-
url
|
|
951
|
-
};
|
|
952
973
|
}
|
|
953
974
|
var sizeThreshold = 3;
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
975
|
+
function alignElements(elements, page) {
|
|
976
|
+
return __async(this, null, function* () {
|
|
977
|
+
const validElements = elements.filter((item) => {
|
|
978
|
+
return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
|
|
979
|
+
});
|
|
980
|
+
const textsAligned = [];
|
|
981
|
+
for (const item of validElements) {
|
|
982
|
+
const { rect, id, content, attributes, locator, indexId } = item;
|
|
983
|
+
textsAligned.push(
|
|
984
|
+
new WebElementInfo({
|
|
985
|
+
rect,
|
|
986
|
+
locator,
|
|
987
|
+
id,
|
|
988
|
+
content,
|
|
989
|
+
attributes,
|
|
990
|
+
page,
|
|
991
|
+
indexId
|
|
992
|
+
})
|
|
993
|
+
);
|
|
994
|
+
}
|
|
995
|
+
return textsAligned;
|
|
957
996
|
});
|
|
958
|
-
const textsAligned = [];
|
|
959
|
-
for (const item of validElements) {
|
|
960
|
-
const { rect, id, content, attributes, locator, indexId } = item;
|
|
961
|
-
textsAligned.push(
|
|
962
|
-
new WebElementInfo({
|
|
963
|
-
rect,
|
|
964
|
-
locator,
|
|
965
|
-
id,
|
|
966
|
-
content,
|
|
967
|
-
attributes,
|
|
968
|
-
page,
|
|
969
|
-
indexId
|
|
970
|
-
})
|
|
971
|
-
);
|
|
972
|
-
}
|
|
973
|
-
return textsAligned;
|
|
974
997
|
}
|
|
975
998
|
function reportFileName(tag = "web") {
|
|
976
999
|
const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
|
|
@@ -1001,7 +1024,7 @@ var testFileIndex = /* @__PURE__ */ new Map();
|
|
|
1001
1024
|
function generateCacheId(fileName) {
|
|
1002
1025
|
let taskFile = fileName || getCurrentExecutionFile();
|
|
1003
1026
|
if (!taskFile) {
|
|
1004
|
-
taskFile = (0,
|
|
1027
|
+
taskFile = (0, import_utils.uuid)();
|
|
1005
1028
|
console.warn(
|
|
1006
1029
|
"Midscene - using random UUID for cache id. Cache may be invalid."
|
|
1007
1030
|
);
|
|
@@ -1020,7 +1043,7 @@ function generateCacheId(fileName) {
|
|
|
1020
1043
|
// src/common/task-cache.ts
|
|
1021
1044
|
var TaskCache = class {
|
|
1022
1045
|
constructor(opts) {
|
|
1023
|
-
this.midscenePkgInfo = (0, import_fs2.
|
|
1046
|
+
this.midscenePkgInfo = (0, import_fs2.getRunningPkgInfo)();
|
|
1024
1047
|
this.cacheId = generateCacheId(opts == null ? void 0 : opts.fileName);
|
|
1025
1048
|
this.cache = this.readCacheFromFile() || {
|
|
1026
1049
|
aiTasks: []
|
|
@@ -1102,11 +1125,17 @@ var TaskCache = class {
|
|
|
1102
1125
|
return this.newCache;
|
|
1103
1126
|
}
|
|
1104
1127
|
readCacheFromFile() {
|
|
1105
|
-
|
|
1128
|
+
if (import_utils3.ifInBrowser) {
|
|
1129
|
+
return void 0;
|
|
1130
|
+
}
|
|
1131
|
+
const cacheFile = (0, import_node_path2.join)((0, import_utils2.getLogDirByType)("cache"), `${this.cacheId}.json`);
|
|
1106
1132
|
if (process.env.MIDSCENE_CACHE === "true" && (0, import_node_fs2.existsSync)(cacheFile)) {
|
|
1107
1133
|
try {
|
|
1108
1134
|
const data = (0, import_node_fs2.readFileSync)(cacheFile, "utf8");
|
|
1109
1135
|
const jsonData = JSON.parse(data);
|
|
1136
|
+
if (!this.midscenePkgInfo) {
|
|
1137
|
+
return void 0;
|
|
1138
|
+
}
|
|
1110
1139
|
if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
|
|
1111
1140
|
return void 0;
|
|
1112
1141
|
}
|
|
@@ -1118,11 +1147,14 @@ var TaskCache = class {
|
|
|
1118
1147
|
return void 0;
|
|
1119
1148
|
}
|
|
1120
1149
|
writeCacheToFile() {
|
|
1121
|
-
const midscenePkgInfo = (0, import_fs2.
|
|
1122
|
-
(
|
|
1150
|
+
const midscenePkgInfo = (0, import_fs2.getRunningPkgInfo)();
|
|
1151
|
+
if (!midscenePkgInfo) {
|
|
1152
|
+
return;
|
|
1153
|
+
}
|
|
1154
|
+
(0, import_utils2.writeLogFile)({
|
|
1123
1155
|
fileName: `${this.cacheId}`,
|
|
1124
1156
|
fileExt: "json",
|
|
1125
|
-
fileContent: (0,
|
|
1157
|
+
fileContent: (0, import_utils2.stringifyDumpData)(
|
|
1126
1158
|
__spreadValues({
|
|
1127
1159
|
pkgName: midscenePkgInfo.name,
|
|
1128
1160
|
pkgVersion: midscenePkgInfo.version,
|
|
@@ -1139,423 +1171,435 @@ var TaskCache = class {
|
|
|
1139
1171
|
var PageTaskExecutor = class {
|
|
1140
1172
|
constructor(page, opts) {
|
|
1141
1173
|
this.page = page;
|
|
1142
|
-
this.insight = new import_core.Insight(
|
|
1143
|
-
return
|
|
1144
|
-
});
|
|
1174
|
+
this.insight = new import_core.Insight(() => __async(this, null, function* () {
|
|
1175
|
+
return yield parseContextFromWebPage(page);
|
|
1176
|
+
}));
|
|
1145
1177
|
this.taskCache = new TaskCache({
|
|
1146
1178
|
fileName: opts == null ? void 0 : opts.cacheId
|
|
1147
1179
|
});
|
|
1148
1180
|
}
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1181
|
+
recordScreenshot(timing) {
|
|
1182
|
+
return __async(this, null, function* () {
|
|
1183
|
+
const base64 = yield this.page.screenshotBase64();
|
|
1184
|
+
const item = {
|
|
1185
|
+
type: "screenshot",
|
|
1186
|
+
ts: Date.now(),
|
|
1187
|
+
screenshot: base64,
|
|
1188
|
+
timing
|
|
1189
|
+
};
|
|
1190
|
+
return item;
|
|
1191
|
+
});
|
|
1158
1192
|
}
|
|
1159
1193
|
wrapExecutorWithScreenshot(taskApply) {
|
|
1160
1194
|
const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
|
|
1161
|
-
executor:
|
|
1195
|
+
executor: (param, context, ...args) => __async(this, null, function* () {
|
|
1162
1196
|
const recorder = [];
|
|
1163
1197
|
const { task } = context;
|
|
1164
1198
|
task.recorder = recorder;
|
|
1165
|
-
const shot =
|
|
1199
|
+
const shot = yield this.recordScreenshot(`before ${task.type}`);
|
|
1166
1200
|
recorder.push(shot);
|
|
1167
|
-
const result =
|
|
1201
|
+
const result = yield taskApply.executor(param, context, ...args);
|
|
1168
1202
|
if (taskApply.type === "Action") {
|
|
1169
|
-
|
|
1170
|
-
const shot2 =
|
|
1203
|
+
yield (0, import_utils5.sleep)(1e3);
|
|
1204
|
+
const shot2 = yield this.recordScreenshot("after Action");
|
|
1171
1205
|
recorder.push(shot2);
|
|
1172
1206
|
}
|
|
1173
1207
|
return result;
|
|
1174
|
-
}
|
|
1208
|
+
})
|
|
1175
1209
|
});
|
|
1176
1210
|
return taskWithScreenshot;
|
|
1177
1211
|
}
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1192
|
-
const pageContext = await this.insight.contextRetrieverFn();
|
|
1193
|
-
const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
|
|
1194
|
-
pageContext,
|
|
1195
|
-
"locate",
|
|
1196
|
-
param.prompt
|
|
1197
|
-
);
|
|
1198
|
-
let locateResult;
|
|
1199
|
-
const callAI = this.insight.aiVendorFn;
|
|
1200
|
-
const element = await this.insight.locate(param.prompt, {
|
|
1201
|
-
quickAnswer: plan2.quickAnswer,
|
|
1202
|
-
callAI: async (...message) => {
|
|
1203
|
-
if (locateCache) {
|
|
1204
|
-
locateResult = locateCache;
|
|
1205
|
-
return Promise.resolve(locateCache);
|
|
1206
|
-
}
|
|
1207
|
-
locateResult = await callAI(...message);
|
|
1208
|
-
(0, import_node_assert2.default)(locateResult);
|
|
1209
|
-
return locateResult;
|
|
1210
|
-
}
|
|
1211
|
-
});
|
|
1212
|
-
if (locateResult) {
|
|
1213
|
-
cacheGroup == null ? void 0 : cacheGroup.saveCache({
|
|
1214
|
-
type: "locate",
|
|
1215
|
-
pageContext: {
|
|
1216
|
-
url: pageContext.url,
|
|
1217
|
-
size: pageContext.size
|
|
1218
|
-
},
|
|
1219
|
-
prompt: param.prompt,
|
|
1220
|
-
response: locateResult
|
|
1221
|
-
});
|
|
1222
|
-
}
|
|
1223
|
-
if (!element) {
|
|
1224
|
-
task.log = {
|
|
1225
|
-
dump: insightDump
|
|
1212
|
+
convertPlanToExecutable(plans, cacheGroup) {
|
|
1213
|
+
return __async(this, null, function* () {
|
|
1214
|
+
const tasks = plans.map((plan2) => {
|
|
1215
|
+
if (plan2.type === "Locate") {
|
|
1216
|
+
const taskFind = {
|
|
1217
|
+
type: "Insight",
|
|
1218
|
+
subType: "Locate",
|
|
1219
|
+
param: plan2.param,
|
|
1220
|
+
executor: (param, taskContext) => __async(this, null, function* () {
|
|
1221
|
+
const { task } = taskContext;
|
|
1222
|
+
let insightDump;
|
|
1223
|
+
const dumpCollector = (dump) => {
|
|
1224
|
+
insightDump = dump;
|
|
1226
1225
|
};
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1226
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1227
|
+
const pageContext = yield this.insight.contextRetrieverFn();
|
|
1228
|
+
const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
|
|
1229
|
+
pageContext,
|
|
1230
|
+
"locate",
|
|
1231
|
+
param.prompt
|
|
1232
|
+
);
|
|
1233
|
+
let locateResult;
|
|
1234
|
+
const callAI = this.insight.aiVendorFn;
|
|
1235
|
+
const element = yield this.insight.locate(param.prompt, {
|
|
1236
|
+
quickAnswer: plan2.quickAnswer,
|
|
1237
|
+
callAI: (...message) => __async(this, null, function* () {
|
|
1238
|
+
if (locateCache) {
|
|
1239
|
+
locateResult = locateCache;
|
|
1240
|
+
return Promise.resolve(locateCache);
|
|
1241
|
+
}
|
|
1242
|
+
locateResult = yield callAI(...message);
|
|
1243
|
+
(0, import_node_assert2.default)(locateResult);
|
|
1244
|
+
return locateResult;
|
|
1245
|
+
})
|
|
1246
|
+
});
|
|
1247
|
+
if (locateResult) {
|
|
1248
|
+
cacheGroup == null ? void 0 : cacheGroup.saveCache({
|
|
1249
|
+
type: "locate",
|
|
1250
|
+
pageContext: {
|
|
1251
|
+
url: pageContext.url,
|
|
1252
|
+
size: pageContext.size
|
|
1253
|
+
},
|
|
1254
|
+
prompt: param.prompt,
|
|
1255
|
+
response: locateResult
|
|
1256
|
+
});
|
|
1238
1257
|
}
|
|
1239
|
-
|
|
1240
|
-
}
|
|
1241
|
-
};
|
|
1242
|
-
return taskFind;
|
|
1243
|
-
}
|
|
1244
|
-
if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
|
|
1245
|
-
const assertPlan = plan2;
|
|
1246
|
-
const taskAssert = {
|
|
1247
|
-
type: "Insight",
|
|
1248
|
-
subType: "Assert",
|
|
1249
|
-
param: assertPlan.param,
|
|
1250
|
-
executor: async (param, taskContext) => {
|
|
1251
|
-
const { task } = taskContext;
|
|
1252
|
-
let insightDump;
|
|
1253
|
-
const dumpCollector = (dump) => {
|
|
1254
|
-
insightDump = dump;
|
|
1255
|
-
};
|
|
1256
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1257
|
-
const assertion = await this.insight.assert(
|
|
1258
|
-
assertPlan.param.assertion
|
|
1259
|
-
);
|
|
1260
|
-
if (!assertion.pass) {
|
|
1261
|
-
if (plan2.type === "Assert") {
|
|
1262
|
-
task.output = assertion;
|
|
1258
|
+
if (!element) {
|
|
1263
1259
|
task.log = {
|
|
1264
1260
|
dump: insightDump
|
|
1265
1261
|
};
|
|
1266
|
-
throw new Error(
|
|
1267
|
-
assertion.thought || "Assertion failed without reason"
|
|
1268
|
-
);
|
|
1262
|
+
throw new Error(`Element not found: ${param.prompt}`);
|
|
1269
1263
|
}
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1264
|
+
return {
|
|
1265
|
+
output: {
|
|
1266
|
+
element
|
|
1267
|
+
},
|
|
1268
|
+
log: {
|
|
1269
|
+
dump: insightDump
|
|
1270
|
+
},
|
|
1271
|
+
cache: {
|
|
1272
|
+
hit: Boolean(locateCache)
|
|
1273
|
+
}
|
|
1274
|
+
};
|
|
1275
|
+
})
|
|
1276
|
+
};
|
|
1277
|
+
return taskFind;
|
|
1278
|
+
}
|
|
1279
|
+
if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
|
|
1280
|
+
const assertPlan = plan2;
|
|
1281
|
+
const taskAssert = {
|
|
1282
|
+
type: "Insight",
|
|
1283
|
+
subType: "Assert",
|
|
1284
|
+
param: assertPlan.param,
|
|
1285
|
+
executor: (param, taskContext) => __async(this, null, function* () {
|
|
1286
|
+
const { task } = taskContext;
|
|
1287
|
+
let insightDump;
|
|
1288
|
+
const dumpCollector = (dump) => {
|
|
1289
|
+
insightDump = dump;
|
|
1290
|
+
};
|
|
1291
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1292
|
+
const assertion = yield this.insight.assert(
|
|
1293
|
+
assertPlan.param.assertion
|
|
1294
|
+
);
|
|
1295
|
+
if (!assertion.pass) {
|
|
1296
|
+
if (plan2.type === "Assert") {
|
|
1297
|
+
task.output = assertion;
|
|
1298
|
+
task.log = {
|
|
1299
|
+
dump: insightDump
|
|
1300
|
+
};
|
|
1301
|
+
throw new Error(
|
|
1302
|
+
assertion.thought || "Assertion failed without reason"
|
|
1303
|
+
);
|
|
1304
|
+
}
|
|
1305
|
+
task.error = assertion.thought;
|
|
1276
1306
|
}
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1307
|
+
return {
|
|
1308
|
+
output: assertion,
|
|
1309
|
+
log: {
|
|
1310
|
+
dump: insightDump
|
|
1311
|
+
}
|
|
1312
|
+
};
|
|
1313
|
+
})
|
|
1314
|
+
};
|
|
1315
|
+
return taskAssert;
|
|
1316
|
+
}
|
|
1317
|
+
if (plan2.type === "Input") {
|
|
1318
|
+
const taskActionInput = {
|
|
1319
|
+
type: "Action",
|
|
1320
|
+
subType: "Input",
|
|
1321
|
+
param: plan2.param,
|
|
1322
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (taskParam, { element }) {
|
|
1323
|
+
if (element) {
|
|
1324
|
+
yield this.page.clearInput(element);
|
|
1325
|
+
if (taskParam.value === "") {
|
|
1326
|
+
return;
|
|
1327
|
+
}
|
|
1328
|
+
yield this.page.keyboard.type(taskParam.value);
|
|
1292
1329
|
}
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
}
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
}
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
}
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
}
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
}
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
}
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
)
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1394
|
-
}
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
return
|
|
1330
|
+
})
|
|
1331
|
+
};
|
|
1332
|
+
return taskActionInput;
|
|
1333
|
+
}
|
|
1334
|
+
if (plan2.type === "KeyboardPress") {
|
|
1335
|
+
const taskActionKeyboardPress = {
|
|
1336
|
+
type: "Action",
|
|
1337
|
+
subType: "KeyboardPress",
|
|
1338
|
+
param: plan2.param,
|
|
1339
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
1340
|
+
(0, import_node_assert2.default)(taskParam.value, "No key to press");
|
|
1341
|
+
yield this.page.keyboard.press(taskParam.value);
|
|
1342
|
+
})
|
|
1343
|
+
};
|
|
1344
|
+
return taskActionKeyboardPress;
|
|
1345
|
+
}
|
|
1346
|
+
if (plan2.type === "Tap") {
|
|
1347
|
+
const taskActionTap = {
|
|
1348
|
+
type: "Action",
|
|
1349
|
+
subType: "Tap",
|
|
1350
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
|
|
1351
|
+
(0, import_node_assert2.default)(element, "Element not found, cannot tap");
|
|
1352
|
+
yield this.page.mouse.click(
|
|
1353
|
+
element.center[0],
|
|
1354
|
+
element.center[1]
|
|
1355
|
+
);
|
|
1356
|
+
})
|
|
1357
|
+
};
|
|
1358
|
+
return taskActionTap;
|
|
1359
|
+
}
|
|
1360
|
+
if (plan2.type === "Hover") {
|
|
1361
|
+
const taskActionHover = {
|
|
1362
|
+
type: "Action",
|
|
1363
|
+
subType: "Hover",
|
|
1364
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
|
|
1365
|
+
(0, import_node_assert2.default)(element, "Element not found, cannot hover");
|
|
1366
|
+
yield this.page.mouse.move(
|
|
1367
|
+
element.center[0],
|
|
1368
|
+
element.center[1]
|
|
1369
|
+
);
|
|
1370
|
+
})
|
|
1371
|
+
};
|
|
1372
|
+
return taskActionHover;
|
|
1373
|
+
}
|
|
1374
|
+
if (plan2.type === "Scroll") {
|
|
1375
|
+
const taskActionScroll = {
|
|
1376
|
+
type: "Action",
|
|
1377
|
+
subType: "Scroll",
|
|
1378
|
+
param: plan2.param,
|
|
1379
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
1380
|
+
const scrollToEventName = taskParam.scrollType;
|
|
1381
|
+
switch (scrollToEventName) {
|
|
1382
|
+
case "scrollUntilTop":
|
|
1383
|
+
yield this.page.scrollUntilTop();
|
|
1384
|
+
break;
|
|
1385
|
+
case "scrollUntilBottom":
|
|
1386
|
+
yield this.page.scrollUntilBottom();
|
|
1387
|
+
break;
|
|
1388
|
+
case "scrollUpOneScreen":
|
|
1389
|
+
yield this.page.scrollUpOneScreen();
|
|
1390
|
+
break;
|
|
1391
|
+
case "scrollDownOneScreen":
|
|
1392
|
+
yield this.page.scrollDownOneScreen();
|
|
1393
|
+
break;
|
|
1394
|
+
default:
|
|
1395
|
+
console.error(
|
|
1396
|
+
"Unknown scroll event type:",
|
|
1397
|
+
scrollToEventName
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
})
|
|
1401
|
+
};
|
|
1402
|
+
return taskActionScroll;
|
|
1403
|
+
}
|
|
1404
|
+
if (plan2.type === "Sleep") {
|
|
1405
|
+
const taskActionSleep = {
|
|
1406
|
+
type: "Action",
|
|
1407
|
+
subType: "Sleep",
|
|
1408
|
+
param: plan2.param,
|
|
1409
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
1410
|
+
yield (0, import_utils5.sleep)(taskParam.timeMs || 3e3);
|
|
1411
|
+
})
|
|
1412
|
+
};
|
|
1413
|
+
return taskActionSleep;
|
|
1414
|
+
}
|
|
1415
|
+
if (plan2.type === "Error") {
|
|
1416
|
+
const taskActionError = {
|
|
1417
|
+
type: "Action",
|
|
1418
|
+
subType: "Error",
|
|
1419
|
+
param: plan2.param,
|
|
1420
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
1421
|
+
(0, import_node_assert2.default)(
|
|
1422
|
+
taskParam.thought,
|
|
1423
|
+
"An error occurred, but no thought provided"
|
|
1424
|
+
);
|
|
1425
|
+
throw new Error(taskParam.thought);
|
|
1426
|
+
})
|
|
1427
|
+
};
|
|
1428
|
+
return taskActionError;
|
|
1429
|
+
}
|
|
1430
|
+
throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
|
|
1431
|
+
}).map((task) => {
|
|
1432
|
+
return this.wrapExecutorWithScreenshot(task);
|
|
1433
|
+
});
|
|
1434
|
+
return tasks;
|
|
1398
1435
|
});
|
|
1399
|
-
return tasks;
|
|
1400
1436
|
}
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1437
|
+
action(userPrompt) {
|
|
1438
|
+
return __async(this, null, function* () {
|
|
1439
|
+
const taskExecutor = new import_core.Executor(userPrompt);
|
|
1440
|
+
const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
|
|
1441
|
+
let plans = [];
|
|
1442
|
+
const planningTask = {
|
|
1443
|
+
type: "Planning",
|
|
1444
|
+
param: {
|
|
1445
|
+
userPrompt
|
|
1446
|
+
},
|
|
1447
|
+
executor: (param) => __async(this, null, function* () {
|
|
1448
|
+
const pageContext = yield this.insight.contextRetrieverFn();
|
|
1449
|
+
let planResult;
|
|
1450
|
+
const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
|
|
1451
|
+
if (planCache) {
|
|
1452
|
+
planResult = planCache;
|
|
1453
|
+
} else {
|
|
1454
|
+
planResult = yield (0, import_core.plan)(param.userPrompt, {
|
|
1455
|
+
context: pageContext
|
|
1456
|
+
});
|
|
1457
|
+
}
|
|
1458
|
+
(0, import_node_assert2.default)(planResult.plans.length > 0, "No plans found");
|
|
1459
|
+
plans = planResult.plans;
|
|
1460
|
+
cacheGroup.saveCache({
|
|
1461
|
+
type: "plan",
|
|
1462
|
+
pageContext: {
|
|
1463
|
+
url: pageContext.url,
|
|
1464
|
+
size: pageContext.size
|
|
1465
|
+
},
|
|
1466
|
+
prompt: userPrompt,
|
|
1467
|
+
response: planResult
|
|
1419
1468
|
});
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
});
|
|
1469
|
+
return {
|
|
1470
|
+
output: planResult,
|
|
1471
|
+
cache: {
|
|
1472
|
+
hit: Boolean(planCache)
|
|
1473
|
+
}
|
|
1474
|
+
};
|
|
1475
|
+
})
|
|
1476
|
+
};
|
|
1477
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
|
|
1478
|
+
let output = yield taskExecutor.flush();
|
|
1479
|
+
if (taskExecutor.isInErrorState()) {
|
|
1432
1480
|
return {
|
|
1433
|
-
output
|
|
1434
|
-
|
|
1435
|
-
hit: Boolean(planCache)
|
|
1436
|
-
}
|
|
1481
|
+
output,
|
|
1482
|
+
executor: taskExecutor
|
|
1437
1483
|
};
|
|
1438
1484
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
if (taskExecutor.isInErrorState()) {
|
|
1485
|
+
const executables = yield this.convertPlanToExecutable(plans, cacheGroup);
|
|
1486
|
+
yield taskExecutor.append(executables);
|
|
1487
|
+
output = yield taskExecutor.flush();
|
|
1443
1488
|
return {
|
|
1444
1489
|
output,
|
|
1445
1490
|
executor: taskExecutor
|
|
1446
1491
|
};
|
|
1447
|
-
}
|
|
1448
|
-
const executables = await this.convertPlanToExecutable(plans, cacheGroup);
|
|
1449
|
-
await taskExecutor.append(executables);
|
|
1450
|
-
output = await taskExecutor.flush();
|
|
1451
|
-
return {
|
|
1452
|
-
output,
|
|
1453
|
-
executor: taskExecutor
|
|
1454
|
-
};
|
|
1455
|
-
}
|
|
1456
|
-
async query(demand) {
|
|
1457
|
-
const description = typeof demand === "string" ? demand : JSON.stringify(demand);
|
|
1458
|
-
const taskExecutor = new import_core.Executor(description);
|
|
1459
|
-
const queryTask = {
|
|
1460
|
-
type: "Insight",
|
|
1461
|
-
subType: "Query",
|
|
1462
|
-
param: {
|
|
1463
|
-
dataDemand: demand
|
|
1464
|
-
},
|
|
1465
|
-
executor: async (param) => {
|
|
1466
|
-
let insightDump;
|
|
1467
|
-
const dumpCollector = (dump) => {
|
|
1468
|
-
insightDump = dump;
|
|
1469
|
-
};
|
|
1470
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1471
|
-
const data = await this.insight.extract(param.dataDemand);
|
|
1472
|
-
return {
|
|
1473
|
-
output: data,
|
|
1474
|
-
log: { dump: insightDump }
|
|
1475
|
-
};
|
|
1476
|
-
}
|
|
1477
|
-
};
|
|
1478
|
-
await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
|
|
1479
|
-
const output = await taskExecutor.flush();
|
|
1480
|
-
return {
|
|
1481
|
-
output,
|
|
1482
|
-
executor: taskExecutor
|
|
1483
|
-
};
|
|
1492
|
+
});
|
|
1484
1493
|
}
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1494
|
+
query(demand) {
|
|
1495
|
+
return __async(this, null, function* () {
|
|
1496
|
+
const description = typeof demand === "string" ? demand : JSON.stringify(demand);
|
|
1497
|
+
const taskExecutor = new import_core.Executor(description);
|
|
1498
|
+
const queryTask = {
|
|
1499
|
+
type: "Insight",
|
|
1500
|
+
subType: "Query",
|
|
1501
|
+
param: {
|
|
1502
|
+
dataDemand: demand
|
|
1503
|
+
},
|
|
1504
|
+
executor: (param) => __async(this, null, function* () {
|
|
1505
|
+
let insightDump;
|
|
1506
|
+
const dumpCollector = (dump) => {
|
|
1507
|
+
insightDump = dump;
|
|
1508
|
+
};
|
|
1509
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1510
|
+
const data = yield this.insight.extract(param.dataDemand);
|
|
1511
|
+
return {
|
|
1512
|
+
output: data,
|
|
1513
|
+
log: { dump: insightDump }
|
|
1514
|
+
};
|
|
1515
|
+
})
|
|
1516
|
+
};
|
|
1517
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
|
|
1518
|
+
const output = yield taskExecutor.flush();
|
|
1519
|
+
return {
|
|
1520
|
+
output,
|
|
1521
|
+
executor: taskExecutor
|
|
1522
|
+
};
|
|
1523
|
+
});
|
|
1501
1524
|
}
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
(0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
|
|
1509
|
-
const overallStartTime = Date.now();
|
|
1510
|
-
let startTime = Date.now();
|
|
1511
|
-
let errorThought = "";
|
|
1512
|
-
while (Date.now() - overallStartTime < timeoutMs) {
|
|
1513
|
-
startTime = Date.now();
|
|
1514
|
-
const assertPlan = {
|
|
1515
|
-
type: "AssertWithoutThrow",
|
|
1525
|
+
assert(assertion) {
|
|
1526
|
+
return __async(this, null, function* () {
|
|
1527
|
+
const description = `assert: ${assertion}`;
|
|
1528
|
+
const taskExecutor = new import_core.Executor(description);
|
|
1529
|
+
const assertionPlan = {
|
|
1530
|
+
type: "Assert",
|
|
1516
1531
|
param: {
|
|
1517
1532
|
assertion
|
|
1518
1533
|
}
|
|
1519
1534
|
};
|
|
1520
|
-
const assertTask =
|
|
1521
|
-
|
|
1522
|
-
const output =
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
+
const assertTask = yield this.convertPlanToExecutable([assertionPlan]);
|
|
1536
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
|
|
1537
|
+
const output = yield taskExecutor.flush();
|
|
1538
|
+
return {
|
|
1539
|
+
output,
|
|
1540
|
+
executor: taskExecutor
|
|
1541
|
+
};
|
|
1542
|
+
});
|
|
1543
|
+
}
|
|
1544
|
+
waitFor(assertion, opt) {
|
|
1545
|
+
return __async(this, null, function* () {
|
|
1546
|
+
const description = `waitFor: ${assertion}`;
|
|
1547
|
+
const taskExecutor = new import_core.Executor(description);
|
|
1548
|
+
const { timeoutMs, checkIntervalMs } = opt;
|
|
1549
|
+
(0, import_node_assert2.default)(assertion, "No assertion for waitFor");
|
|
1550
|
+
(0, import_node_assert2.default)(timeoutMs, "No timeoutMs for waitFor");
|
|
1551
|
+
(0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
|
|
1552
|
+
const overallStartTime = Date.now();
|
|
1553
|
+
let startTime = Date.now();
|
|
1554
|
+
let errorThought = "";
|
|
1555
|
+
while (Date.now() - overallStartTime < timeoutMs) {
|
|
1556
|
+
startTime = Date.now();
|
|
1557
|
+
const assertPlan = {
|
|
1558
|
+
type: "AssertWithoutThrow",
|
|
1535
1559
|
param: {
|
|
1536
|
-
|
|
1560
|
+
assertion
|
|
1537
1561
|
}
|
|
1538
1562
|
};
|
|
1539
|
-
const
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
)
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1563
|
+
const assertTask = yield this.convertPlanToExecutable([assertPlan]);
|
|
1564
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
|
|
1565
|
+
const output = yield taskExecutor.flush();
|
|
1566
|
+
if (output == null ? void 0 : output.pass) {
|
|
1567
|
+
return {
|
|
1568
|
+
output: void 0,
|
|
1569
|
+
executor: taskExecutor
|
|
1570
|
+
};
|
|
1571
|
+
}
|
|
1572
|
+
errorThought = (output == null ? void 0 : output.thought) || "unknown error";
|
|
1573
|
+
const now = Date.now();
|
|
1574
|
+
if (now - startTime < checkIntervalMs) {
|
|
1575
|
+
const timeRemaining = checkIntervalMs - (now - startTime);
|
|
1576
|
+
const sleepPlan = {
|
|
1577
|
+
type: "Sleep",
|
|
1578
|
+
param: {
|
|
1579
|
+
timeMs: timeRemaining
|
|
1580
|
+
}
|
|
1581
|
+
};
|
|
1582
|
+
const sleepTask = yield this.convertPlanToExecutable([sleepPlan]);
|
|
1583
|
+
yield taskExecutor.append(
|
|
1584
|
+
this.wrapExecutorWithScreenshot(sleepTask[0])
|
|
1585
|
+
);
|
|
1586
|
+
yield taskExecutor.flush();
|
|
1587
|
+
}
|
|
1550
1588
|
}
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1589
|
+
const errorPlan = {
|
|
1590
|
+
type: "Error",
|
|
1591
|
+
param: {
|
|
1592
|
+
thought: `waitFor timeout: ${errorThought}`
|
|
1593
|
+
}
|
|
1594
|
+
};
|
|
1595
|
+
const errorTask = yield this.convertPlanToExecutable([errorPlan]);
|
|
1596
|
+
yield taskExecutor.append(errorTask[0]);
|
|
1597
|
+
yield taskExecutor.flush();
|
|
1598
|
+
return {
|
|
1599
|
+
output: void 0,
|
|
1600
|
+
executor: taskExecutor
|
|
1601
|
+
};
|
|
1602
|
+
});
|
|
1559
1603
|
}
|
|
1560
1604
|
};
|
|
1561
1605
|
|
|
@@ -1589,90 +1633,101 @@ var PageAgent = class {
|
|
|
1589
1633
|
dumpDataString() {
|
|
1590
1634
|
this.dump.groupName = this.opts.groupName;
|
|
1591
1635
|
this.dump.groupDescription = this.opts.groupDescription;
|
|
1592
|
-
return (0,
|
|
1636
|
+
return (0, import_utils7.stringifyDumpData)(this.dump);
|
|
1593
1637
|
}
|
|
1594
1638
|
writeOutActionDumps() {
|
|
1595
1639
|
const { generateReport, autoPrintReportMsg } = this.opts;
|
|
1596
|
-
this.reportFile = (0,
|
|
1640
|
+
this.reportFile = (0, import_utils7.writeLogFile)({
|
|
1597
1641
|
fileName: this.reportFileName,
|
|
1598
|
-
fileExt:
|
|
1642
|
+
fileExt: import_utils7.groupedActionDumpFileExt,
|
|
1599
1643
|
fileContent: this.dumpDataString(),
|
|
1600
1644
|
type: "dump",
|
|
1601
1645
|
generateReport
|
|
1602
1646
|
});
|
|
1603
|
-
if (generateReport && autoPrintReportMsg) {
|
|
1647
|
+
if (generateReport && autoPrintReportMsg && this.reportFile) {
|
|
1604
1648
|
printReportMsg(this.reportFile);
|
|
1605
1649
|
}
|
|
1606
1650
|
}
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1651
|
+
aiAction(taskPrompt) {
|
|
1652
|
+
return __async(this, null, function* () {
|
|
1653
|
+
const { executor } = yield this.taskExecutor.action(taskPrompt);
|
|
1654
|
+
this.appendExecutionDump(executor.dump());
|
|
1655
|
+
this.writeOutActionDumps();
|
|
1656
|
+
if (executor.isInErrorState()) {
|
|
1657
|
+
const errorTask = executor.latestErrorTask();
|
|
1658
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1614
1659
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1615
|
-
|
|
1660
|
+
}
|
|
1661
|
+
});
|
|
1616
1662
|
}
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1663
|
+
aiQuery(demand) {
|
|
1664
|
+
return __async(this, null, function* () {
|
|
1665
|
+
const { output, executor } = yield this.taskExecutor.query(demand);
|
|
1666
|
+
this.appendExecutionDump(executor.dump());
|
|
1667
|
+
this.writeOutActionDumps();
|
|
1668
|
+
if (executor.isInErrorState()) {
|
|
1669
|
+
const errorTask = executor.latestErrorTask();
|
|
1670
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1624
1671
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1625
|
-
|
|
1626
|
-
return output;
|
|
1627
|
-
}
|
|
1628
|
-
async aiAssert(assertion, msg, opt) {
|
|
1629
|
-
const { output, executor } = await this.taskExecutor.assert(assertion);
|
|
1630
|
-
this.appendExecutionDump(executor.dump());
|
|
1631
|
-
this.writeOutActionDumps();
|
|
1632
|
-
if (opt == null ? void 0 : opt.keepRawResponse) {
|
|
1672
|
+
}
|
|
1633
1673
|
return output;
|
|
1634
|
-
}
|
|
1635
|
-
if (!(output == null ? void 0 : output.pass)) {
|
|
1636
|
-
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1637
|
-
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
|
|
1638
|
-
throw new Error(`${errMsg}
|
|
1639
|
-
${reasonMsg}`);
|
|
1640
|
-
}
|
|
1674
|
+
});
|
|
1641
1675
|
}
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1676
|
+
aiAssert(assertion, msg, opt) {
|
|
1677
|
+
return __async(this, null, function* () {
|
|
1678
|
+
var _a;
|
|
1679
|
+
const { output, executor } = yield this.taskExecutor.assert(assertion);
|
|
1680
|
+
this.appendExecutionDump(executor.dump());
|
|
1681
|
+
this.writeOutActionDumps();
|
|
1682
|
+
if (opt == null ? void 0 : opt.keepRawResponse) {
|
|
1683
|
+
return output;
|
|
1684
|
+
}
|
|
1685
|
+
if (!(output == null ? void 0 : output.pass)) {
|
|
1686
|
+
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1687
|
+
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || ((_a = executor.latestErrorTask()) == null ? void 0 : _a.error) || "(no_reason)"}`;
|
|
1688
|
+
throw new Error(`${errMsg}
|
|
1689
|
+
${reasonMsg}`);
|
|
1690
|
+
}
|
|
1647
1691
|
});
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
const
|
|
1652
|
-
|
|
1692
|
+
}
|
|
1693
|
+
aiWaitFor(assertion, opt) {
|
|
1694
|
+
return __async(this, null, function* () {
|
|
1695
|
+
const { executor } = yield this.taskExecutor.waitFor(assertion, {
|
|
1696
|
+
timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
|
|
1697
|
+
checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
|
|
1698
|
+
assertion
|
|
1699
|
+
});
|
|
1700
|
+
this.appendExecutionDump(executor.dump());
|
|
1701
|
+
this.writeOutActionDumps();
|
|
1702
|
+
if (executor.isInErrorState()) {
|
|
1703
|
+
const errorTask = executor.latestErrorTask();
|
|
1704
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1653
1705
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1654
|
-
|
|
1706
|
+
}
|
|
1707
|
+
});
|
|
1655
1708
|
}
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1709
|
+
ai(taskPrompt, type = "action") {
|
|
1710
|
+
return __async(this, null, function* () {
|
|
1711
|
+
if (type === "action") {
|
|
1712
|
+
return this.aiAction(taskPrompt);
|
|
1713
|
+
}
|
|
1714
|
+
if (type === "query") {
|
|
1715
|
+
return this.aiQuery(taskPrompt);
|
|
1716
|
+
}
|
|
1717
|
+
if (type === "assert") {
|
|
1718
|
+
return this.aiAssert(taskPrompt);
|
|
1719
|
+
}
|
|
1720
|
+
throw new Error(
|
|
1721
|
+
`Unknown type: ${type}, only support 'action', 'query', 'assert'`
|
|
1722
|
+
);
|
|
1723
|
+
});
|
|
1669
1724
|
}
|
|
1670
1725
|
};
|
|
1671
1726
|
|
|
1672
1727
|
// src/appium/page.ts
|
|
1673
1728
|
var import_node_fs3 = __toESM(require("fs"));
|
|
1674
|
-
var
|
|
1675
|
-
var
|
|
1729
|
+
var import_utils9 = require("@midscene/core/utils");
|
|
1730
|
+
var import_img3 = require("@midscene/shared/img");
|
|
1676
1731
|
var import_xmldom = require("@xmldom/xmldom");
|
|
1677
1732
|
|
|
1678
1733
|
// src/extractor/web-extractor.ts
|
|
@@ -1867,26 +1922,30 @@ var Page = class {
|
|
|
1867
1922
|
this.pageType = "appium";
|
|
1868
1923
|
this.browser = browser;
|
|
1869
1924
|
}
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1925
|
+
getElementInfos() {
|
|
1926
|
+
return __async(this, null, function* () {
|
|
1927
|
+
const pageSource = yield this.browser.getPageSource();
|
|
1928
|
+
const { width, height } = yield this.browser.getWindowSize();
|
|
1929
|
+
const parser = new import_xmldom.DOMParser();
|
|
1930
|
+
const doc = parser.parseFromString(pageSource, "text/xml");
|
|
1931
|
+
const infos = extractTextWithPosition2(doc).filter(
|
|
1932
|
+
(element) => element.rect.height !== height && element.rect.width !== width && element.rect.left !== 0 && element.rect.top !== 0 && element.attributes.visible === "true"
|
|
1933
|
+
);
|
|
1934
|
+
return infos;
|
|
1935
|
+
});
|
|
1879
1936
|
}
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1937
|
+
screenshotBase64() {
|
|
1938
|
+
return __async(this, null, function* () {
|
|
1939
|
+
const { width, height } = yield this.browser.getWindowSize();
|
|
1940
|
+
const path2 = (0, import_utils9.getTmpFile)("png");
|
|
1941
|
+
const screenshotBuffer = yield this.browser.saveScreenshot(path2);
|
|
1942
|
+
const resizedScreenshotBuffer = yield (0, import_img3.resizeImg)(screenshotBuffer, {
|
|
1943
|
+
width,
|
|
1944
|
+
height
|
|
1945
|
+
});
|
|
1946
|
+
import_node_fs3.default.writeFileSync(path2, resizedScreenshotBuffer);
|
|
1947
|
+
return (0, import_img3.base64Encoded)(path2);
|
|
1887
1948
|
});
|
|
1888
|
-
import_node_fs3.default.writeFileSync(path2, resizedScreenshotBuffer);
|
|
1889
|
-
return path2;
|
|
1890
1949
|
}
|
|
1891
1950
|
get mouse() {
|
|
1892
1951
|
return {
|
|
@@ -1902,14 +1961,16 @@ var Page = class {
|
|
|
1902
1961
|
press: (key) => this.keyboardPress(key)
|
|
1903
1962
|
};
|
|
1904
1963
|
}
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1964
|
+
clearInput(element) {
|
|
1965
|
+
return __async(this, null, function* () {
|
|
1966
|
+
if (!element) {
|
|
1967
|
+
return;
|
|
1968
|
+
}
|
|
1969
|
+
const ele = yield this.browser.$(element.locator);
|
|
1970
|
+
const blank = " ";
|
|
1971
|
+
yield this.keyboardType(blank);
|
|
1972
|
+
yield ele.clearValue();
|
|
1973
|
+
});
|
|
1913
1974
|
}
|
|
1914
1975
|
url() {
|
|
1915
1976
|
var _a;
|
|
@@ -1925,114 +1986,132 @@ var Page = class {
|
|
|
1925
1986
|
return "";
|
|
1926
1987
|
}
|
|
1927
1988
|
// Scroll to top element
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1989
|
+
scrollUntilTop() {
|
|
1990
|
+
return __async(this, null, function* () {
|
|
1991
|
+
const { height } = yield this.browser.getWindowSize();
|
|
1992
|
+
yield this.mouseWheel(0, height, 100);
|
|
1993
|
+
});
|
|
1931
1994
|
}
|
|
1932
1995
|
// Scroll to bottom element
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1996
|
+
scrollUntilBottom() {
|
|
1997
|
+
return __async(this, null, function* () {
|
|
1998
|
+
const { height } = yield this.browser.getWindowSize();
|
|
1999
|
+
yield this.mouseWheel(0, -height, 100);
|
|
2000
|
+
});
|
|
1936
2001
|
}
|
|
1937
2002
|
// Scroll up one screen
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
2003
|
+
scrollUpOneScreen() {
|
|
2004
|
+
return __async(this, null, function* () {
|
|
2005
|
+
const { height } = yield this.browser.getWindowSize();
|
|
2006
|
+
yield this.mouseWheel(0, height, 1e3);
|
|
2007
|
+
});
|
|
1941
2008
|
}
|
|
1942
2009
|
// Scroll down one screen
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
2010
|
+
scrollDownOneScreen() {
|
|
2011
|
+
return __async(this, null, function* () {
|
|
2012
|
+
const { height } = yield this.browser.getWindowSize();
|
|
2013
|
+
yield this.mouseWheel(0, -height, 1e3);
|
|
2014
|
+
});
|
|
1946
2015
|
}
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
if (!actions.length) {
|
|
1954
|
-
return;
|
|
1955
|
-
}
|
|
1956
|
-
await this.browser.performActions([
|
|
1957
|
-
{
|
|
1958
|
-
type: "key",
|
|
1959
|
-
id: "keyboard",
|
|
1960
|
-
actions
|
|
2016
|
+
keyboardType(text) {
|
|
2017
|
+
return __async(this, null, function* () {
|
|
2018
|
+
const actions = [];
|
|
2019
|
+
for (const char of text) {
|
|
2020
|
+
actions.push({ type: "keyDown", value: char });
|
|
2021
|
+
actions.push({ type: "keyUp", value: char });
|
|
1961
2022
|
}
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
async keyboardPress(key) {
|
|
1965
|
-
await this.browser.performActions([
|
|
1966
|
-
{
|
|
1967
|
-
type: "key",
|
|
1968
|
-
id: "keyboard",
|
|
1969
|
-
actions: [
|
|
1970
|
-
{ type: "keyDown", value: key },
|
|
1971
|
-
{ type: "keyUp", value: key }
|
|
1972
|
-
]
|
|
2023
|
+
if (!actions.length) {
|
|
2024
|
+
return;
|
|
1973
2025
|
}
|
|
1974
|
-
|
|
2026
|
+
yield this.browser.performActions([
|
|
2027
|
+
{
|
|
2028
|
+
type: "key",
|
|
2029
|
+
id: "keyboard",
|
|
2030
|
+
actions
|
|
2031
|
+
}
|
|
2032
|
+
]);
|
|
2033
|
+
});
|
|
1975
2034
|
}
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
}
|
|
1990
|
-
]);
|
|
2035
|
+
keyboardPress(key) {
|
|
2036
|
+
return __async(this, null, function* () {
|
|
2037
|
+
yield this.browser.performActions([
|
|
2038
|
+
{
|
|
2039
|
+
type: "key",
|
|
2040
|
+
id: "keyboard",
|
|
2041
|
+
actions: [
|
|
2042
|
+
{ type: "keyDown", value: key },
|
|
2043
|
+
{ type: "keyUp", value: key }
|
|
2044
|
+
]
|
|
2045
|
+
}
|
|
2046
|
+
]);
|
|
2047
|
+
});
|
|
1991
2048
|
}
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2049
|
+
mouseClick(x, y, button = "left") {
|
|
2050
|
+
return __async(this, null, function* () {
|
|
2051
|
+
yield this.mouseMove(x, y);
|
|
2052
|
+
yield this.browser.performActions([
|
|
2053
|
+
{
|
|
2054
|
+
type: "pointer",
|
|
2055
|
+
id: "mouse",
|
|
2056
|
+
parameters: { pointerType: "mouse" },
|
|
2057
|
+
actions: [
|
|
2058
|
+
{ type: "pointerMove", duration: 0, x, y },
|
|
2059
|
+
{ type: "pointerDown", button: buttonToNumber(button) },
|
|
2060
|
+
{ type: "pause", duration: 100 },
|
|
2061
|
+
{ type: "pointerUp", button: buttonToNumber(button) }
|
|
2062
|
+
]
|
|
2063
|
+
}
|
|
2064
|
+
]);
|
|
2065
|
+
});
|
|
2001
2066
|
}
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2067
|
+
mouseMove(x, y) {
|
|
2068
|
+
return __async(this, null, function* () {
|
|
2069
|
+
yield this.browser.performActions([
|
|
2070
|
+
{
|
|
2071
|
+
type: "pointer",
|
|
2072
|
+
id: "mouse",
|
|
2073
|
+
parameters: { pointerType: "mouse" },
|
|
2074
|
+
actions: [{ type: "pointerMove", duration: 0, x, y }]
|
|
2075
|
+
}
|
|
2076
|
+
]);
|
|
2077
|
+
});
|
|
2078
|
+
}
|
|
2079
|
+
mouseWheel(deltaX, deltaY, duration = 1e3) {
|
|
2080
|
+
return __async(this, null, function* () {
|
|
2081
|
+
const n = 4;
|
|
2082
|
+
const windowSize = yield this.browser.getWindowSize();
|
|
2083
|
+
const startX = deltaX < 0 ? (n - 1) * (windowSize.width / n) : windowSize.width / n;
|
|
2084
|
+
const startY = deltaY < 0 ? (n - 1) * (windowSize.height / n) : windowSize.height / n;
|
|
2085
|
+
const maxNegativeDeltaX = startX;
|
|
2086
|
+
const maxPositiveDeltaX = (n - 1) * (windowSize.width / n);
|
|
2087
|
+
const maxNegativeDeltaY = startY;
|
|
2088
|
+
const maxPositiveDeltaY = (n - 1) * (windowSize.height / n);
|
|
2089
|
+
deltaX = Math.max(-maxNegativeDeltaX, Math.min(deltaX, maxPositiveDeltaX));
|
|
2090
|
+
deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
|
|
2091
|
+
yield this.browser.performActions([
|
|
2092
|
+
{
|
|
2093
|
+
type: "pointer",
|
|
2094
|
+
id: "finger1",
|
|
2095
|
+
parameters: { pointerType: "touch" },
|
|
2096
|
+
actions: [
|
|
2097
|
+
{ type: "pointerMove", duration: 0, x: startX, y: startY },
|
|
2098
|
+
{ type: "pointerDown", button: 0 },
|
|
2099
|
+
{ type: "pause", duration },
|
|
2100
|
+
{
|
|
2101
|
+
type: "pointerMove",
|
|
2102
|
+
duration,
|
|
2103
|
+
origin: "pointer",
|
|
2104
|
+
// Use 'pointer' as the starting point
|
|
2105
|
+
x: deltaX,
|
|
2106
|
+
// X offset relative to the starting point
|
|
2107
|
+
y: deltaY
|
|
2108
|
+
// Y offset relative to the starting point
|
|
2109
|
+
},
|
|
2110
|
+
{ type: "pointerUp", button: 0 }
|
|
2111
|
+
]
|
|
2112
|
+
}
|
|
2113
|
+
]);
|
|
2114
|
+
});
|
|
2036
2115
|
}
|
|
2037
2116
|
};
|
|
2038
2117
|
// Annotate the CommonJS export names for ESM import in node:
|