@midscene/web 0.7.1 → 0.7.2-beta-20241024094141.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/playground.js +8438 -0
- package/dist/browser/types/playground.d.ts +313 -0
- package/dist/es/appium.js +680 -604
- package/dist/es/debug.js +95 -73
- package/dist/es/index.js +939 -797
- package/dist/es/midscene-playground.js +678 -609
- package/dist/es/playground.js +593 -1024
- package/dist/es/playwright-report.js +29 -11
- package/dist/es/playwright.js +705 -597
- package/dist/es/puppeteer.js +636 -552
- package/dist/lib/appium.js +688 -609
- package/dist/lib/debug.js +95 -73
- package/dist/lib/index.js +950 -804
- package/dist/lib/midscene-playground.js +687 -615
- package/dist/lib/playground.js +586 -1007
- package/dist/lib/playwright-report.js +30 -9
- package/dist/lib/playwright.js +713 -602
- package/dist/lib/puppeteer.js +644 -557
- package/dist/script/htmlElement.js +11 -10
- package/dist/script/htmlElementDebug.js +11 -10
- package/dist/types/appium.d.ts +2 -3
- package/dist/types/debug.d.ts +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/{page-ad820b3c.d.ts → page-8117b0ad.d.ts} +8 -7
- package/dist/types/playground.d.ts +6 -21
- package/dist/types/playwright.d.ts +3 -4
- package/dist/types/puppeteer.d.ts +2 -3
- package/dist/types/{tasks-82c1054b.d.ts → tasks-cb6bf758.d.ts} +6 -6
- package/package.json +11 -5
- package/static/index.html +1 -1
|
@@ -41,6 +41,26 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
41
41
|
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
42
42
|
mod
|
|
43
43
|
));
|
|
44
|
+
var __async = (__this, __arguments, generator) => {
|
|
45
|
+
return new Promise((resolve, reject) => {
|
|
46
|
+
var fulfilled = (value) => {
|
|
47
|
+
try {
|
|
48
|
+
step(generator.next(value));
|
|
49
|
+
} catch (e) {
|
|
50
|
+
reject(e);
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
var rejected = (value) => {
|
|
54
|
+
try {
|
|
55
|
+
step(generator.throw(value));
|
|
56
|
+
} catch (e) {
|
|
57
|
+
reject(e);
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
|
|
61
|
+
step((generator = generator.apply(__this, __arguments)).next());
|
|
62
|
+
});
|
|
63
|
+
};
|
|
44
64
|
|
|
45
65
|
// ../../node_modules/.pnpm/dayjs@1.11.11/node_modules/dayjs/dayjs.min.js
|
|
46
66
|
var require_dayjs_min = __commonJS({
|
|
@@ -678,19 +698,19 @@ var require_main = __commonJS({
|
|
|
678
698
|
|
|
679
699
|
// src/playground/server.ts
|
|
680
700
|
var import_node_assert3 = __toESM(require("assert"));
|
|
681
|
-
var
|
|
701
|
+
var import_node_crypto = require("crypto");
|
|
682
702
|
var import_node_fs3 = require("fs");
|
|
683
703
|
var import_node_path3 = require("path");
|
|
684
704
|
|
|
685
705
|
// src/common/utils.ts
|
|
686
706
|
var import_node_assert = __toESM(require("assert"));
|
|
687
|
-
var import_node_crypto = require("crypto");
|
|
688
707
|
var import_node_fs = require("fs");
|
|
689
708
|
var import_node_path = __toESM(require("path"));
|
|
690
709
|
var import_constants = require("@midscene/shared/constants");
|
|
691
710
|
var import_fs = require("@midscene/shared/fs");
|
|
692
711
|
var import_img = require("@midscene/shared/img");
|
|
693
712
|
var import_img2 = require("@midscene/shared/img");
|
|
713
|
+
var import_utils = require("@midscene/shared/utils");
|
|
694
714
|
var import_dayjs = __toESM(require_dayjs_min());
|
|
695
715
|
|
|
696
716
|
// src/web-element.ts
|
|
@@ -719,56 +739,59 @@ var WebElementInfo = class {
|
|
|
719
739
|
};
|
|
720
740
|
|
|
721
741
|
// src/common/utils.ts
|
|
722
|
-
|
|
723
|
-
(
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
const url = page.url();
|
|
728
|
-
const file = await page.screenshot();
|
|
729
|
-
const screenshotBase64 = (0, import_img.base64Encoded)(file);
|
|
730
|
-
const captureElementSnapshot = await page.getElementInfos();
|
|
731
|
-
const elementsInfo = await alignElements(captureElementSnapshot, page);
|
|
732
|
-
const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
|
|
733
|
-
if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
|
|
734
|
-
return false;
|
|
742
|
+
function parseContextFromWebPage(page, _opt) {
|
|
743
|
+
return __async(this, null, function* () {
|
|
744
|
+
(0, import_node_assert.default)(page, "page is required");
|
|
745
|
+
if (page._forceUsePageContext) {
|
|
746
|
+
return yield page._forceUsePageContext();
|
|
735
747
|
}
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
748
|
+
const url = page.url();
|
|
749
|
+
const screenshotBase64 = yield page.screenshotBase64();
|
|
750
|
+
const captureElementSnapshot = yield page.getElementInfos();
|
|
751
|
+
const elementsInfo = yield alignElements(captureElementSnapshot, page);
|
|
752
|
+
const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
|
|
753
|
+
if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
|
|
754
|
+
return false;
|
|
755
|
+
}
|
|
756
|
+
return true;
|
|
757
|
+
});
|
|
758
|
+
const size = yield (0, import_img.imageInfoOfBase64)(screenshotBase64);
|
|
759
|
+
const screenshotBase64WithElementMarker = yield (0, import_img2.compositeElementInfoImg)({
|
|
760
|
+
inputImgBase64: screenshotBase64,
|
|
761
|
+
elementsPositionInfo: elementsPositionInfoWithoutText
|
|
762
|
+
});
|
|
763
|
+
return {
|
|
764
|
+
content: elementsInfo,
|
|
765
|
+
size,
|
|
766
|
+
screenshotBase64,
|
|
767
|
+
screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
|
|
768
|
+
url
|
|
769
|
+
};
|
|
742
770
|
});
|
|
743
|
-
return {
|
|
744
|
-
content: elementsInfo,
|
|
745
|
-
size,
|
|
746
|
-
screenshotBase64,
|
|
747
|
-
screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
|
|
748
|
-
url
|
|
749
|
-
};
|
|
750
771
|
}
|
|
751
772
|
var sizeThreshold = 3;
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
773
|
+
function alignElements(elements, page) {
|
|
774
|
+
return __async(this, null, function* () {
|
|
775
|
+
const validElements = elements.filter((item) => {
|
|
776
|
+
return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
|
|
777
|
+
});
|
|
778
|
+
const textsAligned = [];
|
|
779
|
+
for (const item of validElements) {
|
|
780
|
+
const { rect, id, content, attributes, locator, indexId } = item;
|
|
781
|
+
textsAligned.push(
|
|
782
|
+
new WebElementInfo({
|
|
783
|
+
rect,
|
|
784
|
+
locator,
|
|
785
|
+
id,
|
|
786
|
+
content,
|
|
787
|
+
attributes,
|
|
788
|
+
page,
|
|
789
|
+
indexId
|
|
790
|
+
})
|
|
791
|
+
);
|
|
792
|
+
}
|
|
793
|
+
return textsAligned;
|
|
755
794
|
});
|
|
756
|
-
const textsAligned = [];
|
|
757
|
-
for (const item of validElements) {
|
|
758
|
-
const { rect, id, content, attributes, locator, indexId } = item;
|
|
759
|
-
textsAligned.push(
|
|
760
|
-
new WebElementInfo({
|
|
761
|
-
rect,
|
|
762
|
-
locator,
|
|
763
|
-
id,
|
|
764
|
-
content,
|
|
765
|
-
attributes,
|
|
766
|
-
page,
|
|
767
|
-
indexId
|
|
768
|
-
})
|
|
769
|
-
);
|
|
770
|
-
}
|
|
771
|
-
return textsAligned;
|
|
772
795
|
}
|
|
773
796
|
function reportFileName(tag = "web") {
|
|
774
797
|
const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
|
|
@@ -799,7 +822,7 @@ var testFileIndex = /* @__PURE__ */ new Map();
|
|
|
799
822
|
function generateCacheId(fileName) {
|
|
800
823
|
let taskFile = fileName || getCurrentExecutionFile();
|
|
801
824
|
if (!taskFile) {
|
|
802
|
-
taskFile = (0,
|
|
825
|
+
taskFile = (0, import_utils.uuid)();
|
|
803
826
|
console.warn(
|
|
804
827
|
"Midscene - using random UUID for cache id. Cache may be invalid."
|
|
805
828
|
);
|
|
@@ -817,28 +840,28 @@ function generateCacheId(fileName) {
|
|
|
817
840
|
var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
|
|
818
841
|
|
|
819
842
|
// src/playground/server.ts
|
|
820
|
-
var
|
|
843
|
+
var import_utils11 = require("@midscene/core/utils");
|
|
844
|
+
var import_utils12 = require("@midscene/shared/utils");
|
|
821
845
|
var import_cors = __toESM(require("cors"));
|
|
822
|
-
var import_dotenv = __toESM(require_main());
|
|
823
846
|
var import_express = __toESM(require("express"));
|
|
824
847
|
|
|
825
848
|
// src/common/agent.ts
|
|
826
|
-
var
|
|
849
|
+
var import_utils7 = require("@midscene/core/utils");
|
|
827
850
|
|
|
828
851
|
// src/common/tasks.ts
|
|
829
852
|
var import_node_assert2 = __toESM(require("assert"));
|
|
830
853
|
var import_core = require("@midscene/core");
|
|
831
|
-
var
|
|
832
|
-
var import_img3 = require("@midscene/shared/img");
|
|
854
|
+
var import_utils5 = require("@midscene/core/utils");
|
|
833
855
|
|
|
834
856
|
// src/common/task-cache.ts
|
|
835
857
|
var import_node_fs2 = require("fs");
|
|
836
858
|
var import_node_path2 = require("path");
|
|
837
|
-
var
|
|
859
|
+
var import_utils2 = require("@midscene/core/utils");
|
|
838
860
|
var import_fs2 = require("@midscene/shared/fs");
|
|
861
|
+
var import_utils3 = require("@midscene/shared/utils");
|
|
839
862
|
var TaskCache = class {
|
|
840
863
|
constructor(opts) {
|
|
841
|
-
this.midscenePkgInfo = (0, import_fs2.
|
|
864
|
+
this.midscenePkgInfo = (0, import_fs2.getRunningPkgInfo)();
|
|
842
865
|
this.cacheId = generateCacheId(opts == null ? void 0 : opts.fileName);
|
|
843
866
|
this.cache = this.readCacheFromFile() || {
|
|
844
867
|
aiTasks: []
|
|
@@ -920,11 +943,17 @@ var TaskCache = class {
|
|
|
920
943
|
return this.newCache;
|
|
921
944
|
}
|
|
922
945
|
readCacheFromFile() {
|
|
923
|
-
|
|
946
|
+
if (import_utils3.ifInBrowser) {
|
|
947
|
+
return void 0;
|
|
948
|
+
}
|
|
949
|
+
const cacheFile = (0, import_node_path2.join)((0, import_utils2.getLogDirByType)("cache"), `${this.cacheId}.json`);
|
|
924
950
|
if (process.env.MIDSCENE_CACHE === "true" && (0, import_node_fs2.existsSync)(cacheFile)) {
|
|
925
951
|
try {
|
|
926
952
|
const data = (0, import_node_fs2.readFileSync)(cacheFile, "utf8");
|
|
927
953
|
const jsonData = JSON.parse(data);
|
|
954
|
+
if (!this.midscenePkgInfo) {
|
|
955
|
+
return void 0;
|
|
956
|
+
}
|
|
928
957
|
if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
|
|
929
958
|
return void 0;
|
|
930
959
|
}
|
|
@@ -936,11 +965,14 @@ var TaskCache = class {
|
|
|
936
965
|
return void 0;
|
|
937
966
|
}
|
|
938
967
|
writeCacheToFile() {
|
|
939
|
-
const midscenePkgInfo = (0, import_fs2.
|
|
940
|
-
(
|
|
968
|
+
const midscenePkgInfo = (0, import_fs2.getRunningPkgInfo)();
|
|
969
|
+
if (!midscenePkgInfo) {
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
972
|
+
(0, import_utils2.writeLogFile)({
|
|
941
973
|
fileName: `${this.cacheId}`,
|
|
942
974
|
fileExt: "json",
|
|
943
|
-
fileContent: (0,
|
|
975
|
+
fileContent: (0, import_utils2.stringifyDumpData)(
|
|
944
976
|
__spreadValues({
|
|
945
977
|
pkgName: midscenePkgInfo.name,
|
|
946
978
|
pkgVersion: midscenePkgInfo.version,
|
|
@@ -957,423 +989,435 @@ var TaskCache = class {
|
|
|
957
989
|
var PageTaskExecutor = class {
|
|
958
990
|
constructor(page, opts) {
|
|
959
991
|
this.page = page;
|
|
960
|
-
this.insight = new import_core.Insight(
|
|
961
|
-
return
|
|
962
|
-
});
|
|
992
|
+
this.insight = new import_core.Insight(() => __async(this, null, function* () {
|
|
993
|
+
return yield parseContextFromWebPage(page);
|
|
994
|
+
}));
|
|
963
995
|
this.taskCache = new TaskCache({
|
|
964
996
|
fileName: opts == null ? void 0 : opts.cacheId
|
|
965
997
|
});
|
|
966
998
|
}
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
999
|
+
recordScreenshot(timing) {
|
|
1000
|
+
return __async(this, null, function* () {
|
|
1001
|
+
const base64 = yield this.page.screenshotBase64();
|
|
1002
|
+
const item = {
|
|
1003
|
+
type: "screenshot",
|
|
1004
|
+
ts: Date.now(),
|
|
1005
|
+
screenshot: base64,
|
|
1006
|
+
timing
|
|
1007
|
+
};
|
|
1008
|
+
return item;
|
|
1009
|
+
});
|
|
976
1010
|
}
|
|
977
1011
|
wrapExecutorWithScreenshot(taskApply) {
|
|
978
1012
|
const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
|
|
979
|
-
executor:
|
|
1013
|
+
executor: (param, context, ...args) => __async(this, null, function* () {
|
|
980
1014
|
const recorder = [];
|
|
981
1015
|
const { task } = context;
|
|
982
1016
|
task.recorder = recorder;
|
|
983
|
-
const shot =
|
|
1017
|
+
const shot = yield this.recordScreenshot(`before ${task.type}`);
|
|
984
1018
|
recorder.push(shot);
|
|
985
|
-
const result =
|
|
1019
|
+
const result = yield taskApply.executor(param, context, ...args);
|
|
986
1020
|
if (taskApply.type === "Action") {
|
|
987
|
-
|
|
988
|
-
const shot2 =
|
|
1021
|
+
yield (0, import_utils5.sleep)(1e3);
|
|
1022
|
+
const shot2 = yield this.recordScreenshot("after Action");
|
|
989
1023
|
recorder.push(shot2);
|
|
990
1024
|
}
|
|
991
1025
|
return result;
|
|
992
|
-
}
|
|
1026
|
+
})
|
|
993
1027
|
});
|
|
994
1028
|
return taskWithScreenshot;
|
|
995
1029
|
}
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1010
|
-
const pageContext = await this.insight.contextRetrieverFn();
|
|
1011
|
-
const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
|
|
1012
|
-
pageContext,
|
|
1013
|
-
"locate",
|
|
1014
|
-
param.prompt
|
|
1015
|
-
);
|
|
1016
|
-
let locateResult;
|
|
1017
|
-
const callAI = this.insight.aiVendorFn;
|
|
1018
|
-
const element = await this.insight.locate(param.prompt, {
|
|
1019
|
-
quickAnswer: plan2.quickAnswer,
|
|
1020
|
-
callAI: async (...message) => {
|
|
1021
|
-
if (locateCache) {
|
|
1022
|
-
locateResult = locateCache;
|
|
1023
|
-
return Promise.resolve(locateCache);
|
|
1024
|
-
}
|
|
1025
|
-
locateResult = await callAI(...message);
|
|
1026
|
-
(0, import_node_assert2.default)(locateResult);
|
|
1027
|
-
return locateResult;
|
|
1028
|
-
}
|
|
1029
|
-
});
|
|
1030
|
-
if (locateResult) {
|
|
1031
|
-
cacheGroup == null ? void 0 : cacheGroup.saveCache({
|
|
1032
|
-
type: "locate",
|
|
1033
|
-
pageContext: {
|
|
1034
|
-
url: pageContext.url,
|
|
1035
|
-
size: pageContext.size
|
|
1036
|
-
},
|
|
1037
|
-
prompt: param.prompt,
|
|
1038
|
-
response: locateResult
|
|
1039
|
-
});
|
|
1040
|
-
}
|
|
1041
|
-
if (!element) {
|
|
1042
|
-
task.log = {
|
|
1043
|
-
dump: insightDump
|
|
1030
|
+
convertPlanToExecutable(plans, cacheGroup) {
|
|
1031
|
+
return __async(this, null, function* () {
|
|
1032
|
+
const tasks = plans.map((plan2) => {
|
|
1033
|
+
if (plan2.type === "Locate") {
|
|
1034
|
+
const taskFind = {
|
|
1035
|
+
type: "Insight",
|
|
1036
|
+
subType: "Locate",
|
|
1037
|
+
param: plan2.param,
|
|
1038
|
+
executor: (param, taskContext) => __async(this, null, function* () {
|
|
1039
|
+
const { task } = taskContext;
|
|
1040
|
+
let insightDump;
|
|
1041
|
+
const dumpCollector = (dump) => {
|
|
1042
|
+
insightDump = dump;
|
|
1044
1043
|
};
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1044
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1045
|
+
const pageContext = yield this.insight.contextRetrieverFn();
|
|
1046
|
+
const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
|
|
1047
|
+
pageContext,
|
|
1048
|
+
"locate",
|
|
1049
|
+
param.prompt
|
|
1050
|
+
);
|
|
1051
|
+
let locateResult;
|
|
1052
|
+
const callAI = this.insight.aiVendorFn;
|
|
1053
|
+
const element = yield this.insight.locate(param.prompt, {
|
|
1054
|
+
quickAnswer: plan2.quickAnswer,
|
|
1055
|
+
callAI: (...message) => __async(this, null, function* () {
|
|
1056
|
+
if (locateCache) {
|
|
1057
|
+
locateResult = locateCache;
|
|
1058
|
+
return Promise.resolve(locateCache);
|
|
1059
|
+
}
|
|
1060
|
+
locateResult = yield callAI(...message);
|
|
1061
|
+
(0, import_node_assert2.default)(locateResult);
|
|
1062
|
+
return locateResult;
|
|
1063
|
+
})
|
|
1064
|
+
});
|
|
1065
|
+
if (locateResult) {
|
|
1066
|
+
cacheGroup == null ? void 0 : cacheGroup.saveCache({
|
|
1067
|
+
type: "locate",
|
|
1068
|
+
pageContext: {
|
|
1069
|
+
url: pageContext.url,
|
|
1070
|
+
size: pageContext.size
|
|
1071
|
+
},
|
|
1072
|
+
prompt: param.prompt,
|
|
1073
|
+
response: locateResult
|
|
1074
|
+
});
|
|
1056
1075
|
}
|
|
1057
|
-
|
|
1058
|
-
}
|
|
1059
|
-
};
|
|
1060
|
-
return taskFind;
|
|
1061
|
-
}
|
|
1062
|
-
if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
|
|
1063
|
-
const assertPlan = plan2;
|
|
1064
|
-
const taskAssert = {
|
|
1065
|
-
type: "Insight",
|
|
1066
|
-
subType: "Assert",
|
|
1067
|
-
param: assertPlan.param,
|
|
1068
|
-
executor: async (param, taskContext) => {
|
|
1069
|
-
const { task } = taskContext;
|
|
1070
|
-
let insightDump;
|
|
1071
|
-
const dumpCollector = (dump) => {
|
|
1072
|
-
insightDump = dump;
|
|
1073
|
-
};
|
|
1074
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1075
|
-
const assertion = await this.insight.assert(
|
|
1076
|
-
assertPlan.param.assertion
|
|
1077
|
-
);
|
|
1078
|
-
if (!assertion.pass) {
|
|
1079
|
-
if (plan2.type === "Assert") {
|
|
1080
|
-
task.output = assertion;
|
|
1076
|
+
if (!element) {
|
|
1081
1077
|
task.log = {
|
|
1082
1078
|
dump: insightDump
|
|
1083
1079
|
};
|
|
1084
|
-
throw new Error(
|
|
1085
|
-
assertion.thought || "Assertion failed without reason"
|
|
1086
|
-
);
|
|
1080
|
+
throw new Error(`Element not found: ${param.prompt}`);
|
|
1087
1081
|
}
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1082
|
+
return {
|
|
1083
|
+
output: {
|
|
1084
|
+
element
|
|
1085
|
+
},
|
|
1086
|
+
log: {
|
|
1087
|
+
dump: insightDump
|
|
1088
|
+
},
|
|
1089
|
+
cache: {
|
|
1090
|
+
hit: Boolean(locateCache)
|
|
1091
|
+
}
|
|
1092
|
+
};
|
|
1093
|
+
})
|
|
1094
|
+
};
|
|
1095
|
+
return taskFind;
|
|
1096
|
+
}
|
|
1097
|
+
if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
|
|
1098
|
+
const assertPlan = plan2;
|
|
1099
|
+
const taskAssert = {
|
|
1100
|
+
type: "Insight",
|
|
1101
|
+
subType: "Assert",
|
|
1102
|
+
param: assertPlan.param,
|
|
1103
|
+
executor: (param, taskContext) => __async(this, null, function* () {
|
|
1104
|
+
const { task } = taskContext;
|
|
1105
|
+
let insightDump;
|
|
1106
|
+
const dumpCollector = (dump) => {
|
|
1107
|
+
insightDump = dump;
|
|
1108
|
+
};
|
|
1109
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1110
|
+
const assertion = yield this.insight.assert(
|
|
1111
|
+
assertPlan.param.assertion
|
|
1112
|
+
);
|
|
1113
|
+
if (!assertion.pass) {
|
|
1114
|
+
if (plan2.type === "Assert") {
|
|
1115
|
+
task.output = assertion;
|
|
1116
|
+
task.log = {
|
|
1117
|
+
dump: insightDump
|
|
1118
|
+
};
|
|
1119
|
+
throw new Error(
|
|
1120
|
+
assertion.thought || "Assertion failed without reason"
|
|
1121
|
+
);
|
|
1122
|
+
}
|
|
1123
|
+
task.error = assertion.thought;
|
|
1094
1124
|
}
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1125
|
+
return {
|
|
1126
|
+
output: assertion,
|
|
1127
|
+
log: {
|
|
1128
|
+
dump: insightDump
|
|
1129
|
+
}
|
|
1130
|
+
};
|
|
1131
|
+
})
|
|
1132
|
+
};
|
|
1133
|
+
return taskAssert;
|
|
1134
|
+
}
|
|
1135
|
+
if (plan2.type === "Input") {
|
|
1136
|
+
const taskActionInput = {
|
|
1137
|
+
type: "Action",
|
|
1138
|
+
subType: "Input",
|
|
1139
|
+
param: plan2.param,
|
|
1140
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (taskParam, { element }) {
|
|
1141
|
+
if (element) {
|
|
1142
|
+
yield this.page.clearInput(element);
|
|
1143
|
+
if (taskParam.value === "") {
|
|
1144
|
+
return;
|
|
1145
|
+
}
|
|
1146
|
+
yield this.page.keyboard.type(taskParam.value);
|
|
1110
1147
|
}
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
}
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
}
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
}
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
}
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
}
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
)
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
}
|
|
1211
|
-
|
|
1212
|
-
}
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
return
|
|
1148
|
+
})
|
|
1149
|
+
};
|
|
1150
|
+
return taskActionInput;
|
|
1151
|
+
}
|
|
1152
|
+
if (plan2.type === "KeyboardPress") {
|
|
1153
|
+
const taskActionKeyboardPress = {
|
|
1154
|
+
type: "Action",
|
|
1155
|
+
subType: "KeyboardPress",
|
|
1156
|
+
param: plan2.param,
|
|
1157
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
1158
|
+
(0, import_node_assert2.default)(taskParam.value, "No key to press");
|
|
1159
|
+
yield this.page.keyboard.press(taskParam.value);
|
|
1160
|
+
})
|
|
1161
|
+
};
|
|
1162
|
+
return taskActionKeyboardPress;
|
|
1163
|
+
}
|
|
1164
|
+
if (plan2.type === "Tap") {
|
|
1165
|
+
const taskActionTap = {
|
|
1166
|
+
type: "Action",
|
|
1167
|
+
subType: "Tap",
|
|
1168
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
|
|
1169
|
+
(0, import_node_assert2.default)(element, "Element not found, cannot tap");
|
|
1170
|
+
yield this.page.mouse.click(
|
|
1171
|
+
element.center[0],
|
|
1172
|
+
element.center[1]
|
|
1173
|
+
);
|
|
1174
|
+
})
|
|
1175
|
+
};
|
|
1176
|
+
return taskActionTap;
|
|
1177
|
+
}
|
|
1178
|
+
if (plan2.type === "Hover") {
|
|
1179
|
+
const taskActionHover = {
|
|
1180
|
+
type: "Action",
|
|
1181
|
+
subType: "Hover",
|
|
1182
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
|
|
1183
|
+
(0, import_node_assert2.default)(element, "Element not found, cannot hover");
|
|
1184
|
+
yield this.page.mouse.move(
|
|
1185
|
+
element.center[0],
|
|
1186
|
+
element.center[1]
|
|
1187
|
+
);
|
|
1188
|
+
})
|
|
1189
|
+
};
|
|
1190
|
+
return taskActionHover;
|
|
1191
|
+
}
|
|
1192
|
+
if (plan2.type === "Scroll") {
|
|
1193
|
+
const taskActionScroll = {
|
|
1194
|
+
type: "Action",
|
|
1195
|
+
subType: "Scroll",
|
|
1196
|
+
param: plan2.param,
|
|
1197
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
1198
|
+
const scrollToEventName = taskParam.scrollType;
|
|
1199
|
+
switch (scrollToEventName) {
|
|
1200
|
+
case "scrollUntilTop":
|
|
1201
|
+
yield this.page.scrollUntilTop();
|
|
1202
|
+
break;
|
|
1203
|
+
case "scrollUntilBottom":
|
|
1204
|
+
yield this.page.scrollUntilBottom();
|
|
1205
|
+
break;
|
|
1206
|
+
case "scrollUpOneScreen":
|
|
1207
|
+
yield this.page.scrollUpOneScreen();
|
|
1208
|
+
break;
|
|
1209
|
+
case "scrollDownOneScreen":
|
|
1210
|
+
yield this.page.scrollDownOneScreen();
|
|
1211
|
+
break;
|
|
1212
|
+
default:
|
|
1213
|
+
console.error(
|
|
1214
|
+
"Unknown scroll event type:",
|
|
1215
|
+
scrollToEventName
|
|
1216
|
+
);
|
|
1217
|
+
}
|
|
1218
|
+
})
|
|
1219
|
+
};
|
|
1220
|
+
return taskActionScroll;
|
|
1221
|
+
}
|
|
1222
|
+
if (plan2.type === "Sleep") {
|
|
1223
|
+
const taskActionSleep = {
|
|
1224
|
+
type: "Action",
|
|
1225
|
+
subType: "Sleep",
|
|
1226
|
+
param: plan2.param,
|
|
1227
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
1228
|
+
yield (0, import_utils5.sleep)(taskParam.timeMs || 3e3);
|
|
1229
|
+
})
|
|
1230
|
+
};
|
|
1231
|
+
return taskActionSleep;
|
|
1232
|
+
}
|
|
1233
|
+
if (plan2.type === "Error") {
|
|
1234
|
+
const taskActionError = {
|
|
1235
|
+
type: "Action",
|
|
1236
|
+
subType: "Error",
|
|
1237
|
+
param: plan2.param,
|
|
1238
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
1239
|
+
(0, import_node_assert2.default)(
|
|
1240
|
+
taskParam.thought,
|
|
1241
|
+
"An error occurred, but no thought provided"
|
|
1242
|
+
);
|
|
1243
|
+
throw new Error(taskParam.thought);
|
|
1244
|
+
})
|
|
1245
|
+
};
|
|
1246
|
+
return taskActionError;
|
|
1247
|
+
}
|
|
1248
|
+
throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
|
|
1249
|
+
}).map((task) => {
|
|
1250
|
+
return this.wrapExecutorWithScreenshot(task);
|
|
1251
|
+
});
|
|
1252
|
+
return tasks;
|
|
1216
1253
|
});
|
|
1217
|
-
return tasks;
|
|
1218
1254
|
}
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1255
|
+
action(userPrompt) {
|
|
1256
|
+
return __async(this, null, function* () {
|
|
1257
|
+
const taskExecutor = new import_core.Executor(userPrompt);
|
|
1258
|
+
const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
|
|
1259
|
+
let plans = [];
|
|
1260
|
+
const planningTask = {
|
|
1261
|
+
type: "Planning",
|
|
1262
|
+
param: {
|
|
1263
|
+
userPrompt
|
|
1264
|
+
},
|
|
1265
|
+
executor: (param) => __async(this, null, function* () {
|
|
1266
|
+
const pageContext = yield this.insight.contextRetrieverFn();
|
|
1267
|
+
let planResult;
|
|
1268
|
+
const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
|
|
1269
|
+
if (planCache) {
|
|
1270
|
+
planResult = planCache;
|
|
1271
|
+
} else {
|
|
1272
|
+
planResult = yield (0, import_core.plan)(param.userPrompt, {
|
|
1273
|
+
context: pageContext
|
|
1274
|
+
});
|
|
1275
|
+
}
|
|
1276
|
+
(0, import_node_assert2.default)(planResult.plans.length > 0, "No plans found");
|
|
1277
|
+
plans = planResult.plans;
|
|
1278
|
+
cacheGroup.saveCache({
|
|
1279
|
+
type: "plan",
|
|
1280
|
+
pageContext: {
|
|
1281
|
+
url: pageContext.url,
|
|
1282
|
+
size: pageContext.size
|
|
1283
|
+
},
|
|
1284
|
+
prompt: userPrompt,
|
|
1285
|
+
response: planResult
|
|
1237
1286
|
});
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
});
|
|
1287
|
+
return {
|
|
1288
|
+
output: planResult,
|
|
1289
|
+
cache: {
|
|
1290
|
+
hit: Boolean(planCache)
|
|
1291
|
+
}
|
|
1292
|
+
};
|
|
1293
|
+
})
|
|
1294
|
+
};
|
|
1295
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
|
|
1296
|
+
let output = yield taskExecutor.flush();
|
|
1297
|
+
if (taskExecutor.isInErrorState()) {
|
|
1250
1298
|
return {
|
|
1251
|
-
output
|
|
1252
|
-
|
|
1253
|
-
hit: Boolean(planCache)
|
|
1254
|
-
}
|
|
1299
|
+
output,
|
|
1300
|
+
executor: taskExecutor
|
|
1255
1301
|
};
|
|
1256
1302
|
}
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
if (taskExecutor.isInErrorState()) {
|
|
1303
|
+
const executables = yield this.convertPlanToExecutable(plans, cacheGroup);
|
|
1304
|
+
yield taskExecutor.append(executables);
|
|
1305
|
+
output = yield taskExecutor.flush();
|
|
1261
1306
|
return {
|
|
1262
1307
|
output,
|
|
1263
1308
|
executor: taskExecutor
|
|
1264
1309
|
};
|
|
1265
|
-
}
|
|
1266
|
-
const executables = await this.convertPlanToExecutable(plans, cacheGroup);
|
|
1267
|
-
await taskExecutor.append(executables);
|
|
1268
|
-
output = await taskExecutor.flush();
|
|
1269
|
-
return {
|
|
1270
|
-
output,
|
|
1271
|
-
executor: taskExecutor
|
|
1272
|
-
};
|
|
1273
|
-
}
|
|
1274
|
-
async query(demand) {
|
|
1275
|
-
const description = typeof demand === "string" ? demand : JSON.stringify(demand);
|
|
1276
|
-
const taskExecutor = new import_core.Executor(description);
|
|
1277
|
-
const queryTask = {
|
|
1278
|
-
type: "Insight",
|
|
1279
|
-
subType: "Query",
|
|
1280
|
-
param: {
|
|
1281
|
-
dataDemand: demand
|
|
1282
|
-
},
|
|
1283
|
-
executor: async (param) => {
|
|
1284
|
-
let insightDump;
|
|
1285
|
-
const dumpCollector = (dump) => {
|
|
1286
|
-
insightDump = dump;
|
|
1287
|
-
};
|
|
1288
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1289
|
-
const data = await this.insight.extract(param.dataDemand);
|
|
1290
|
-
return {
|
|
1291
|
-
output: data,
|
|
1292
|
-
log: { dump: insightDump }
|
|
1293
|
-
};
|
|
1294
|
-
}
|
|
1295
|
-
};
|
|
1296
|
-
await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
|
|
1297
|
-
const output = await taskExecutor.flush();
|
|
1298
|
-
return {
|
|
1299
|
-
output,
|
|
1300
|
-
executor: taskExecutor
|
|
1301
|
-
};
|
|
1310
|
+
});
|
|
1302
1311
|
}
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1312
|
+
query(demand) {
|
|
1313
|
+
return __async(this, null, function* () {
|
|
1314
|
+
const description = typeof demand === "string" ? demand : JSON.stringify(demand);
|
|
1315
|
+
const taskExecutor = new import_core.Executor(description);
|
|
1316
|
+
const queryTask = {
|
|
1317
|
+
type: "Insight",
|
|
1318
|
+
subType: "Query",
|
|
1319
|
+
param: {
|
|
1320
|
+
dataDemand: demand
|
|
1321
|
+
},
|
|
1322
|
+
executor: (param) => __async(this, null, function* () {
|
|
1323
|
+
let insightDump;
|
|
1324
|
+
const dumpCollector = (dump) => {
|
|
1325
|
+
insightDump = dump;
|
|
1326
|
+
};
|
|
1327
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1328
|
+
const data = yield this.insight.extract(param.dataDemand);
|
|
1329
|
+
return {
|
|
1330
|
+
output: data,
|
|
1331
|
+
log: { dump: insightDump }
|
|
1332
|
+
};
|
|
1333
|
+
})
|
|
1334
|
+
};
|
|
1335
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
|
|
1336
|
+
const output = yield taskExecutor.flush();
|
|
1337
|
+
return {
|
|
1338
|
+
output,
|
|
1339
|
+
executor: taskExecutor
|
|
1340
|
+
};
|
|
1341
|
+
});
|
|
1319
1342
|
}
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
(0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
|
|
1327
|
-
const overallStartTime = Date.now();
|
|
1328
|
-
let startTime = Date.now();
|
|
1329
|
-
let errorThought = "";
|
|
1330
|
-
while (Date.now() - overallStartTime < timeoutMs) {
|
|
1331
|
-
startTime = Date.now();
|
|
1332
|
-
const assertPlan = {
|
|
1333
|
-
type: "AssertWithoutThrow",
|
|
1343
|
+
assert(assertion) {
|
|
1344
|
+
return __async(this, null, function* () {
|
|
1345
|
+
const description = `assert: ${assertion}`;
|
|
1346
|
+
const taskExecutor = new import_core.Executor(description);
|
|
1347
|
+
const assertionPlan = {
|
|
1348
|
+
type: "Assert",
|
|
1334
1349
|
param: {
|
|
1335
1350
|
assertion
|
|
1336
1351
|
}
|
|
1337
1352
|
};
|
|
1338
|
-
const assertTask =
|
|
1339
|
-
|
|
1340
|
-
const output =
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
+
const assertTask = yield this.convertPlanToExecutable([assertionPlan]);
|
|
1354
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
|
|
1355
|
+
const output = yield taskExecutor.flush();
|
|
1356
|
+
return {
|
|
1357
|
+
output,
|
|
1358
|
+
executor: taskExecutor
|
|
1359
|
+
};
|
|
1360
|
+
});
|
|
1361
|
+
}
|
|
1362
|
+
waitFor(assertion, opt) {
|
|
1363
|
+
return __async(this, null, function* () {
|
|
1364
|
+
const description = `waitFor: ${assertion}`;
|
|
1365
|
+
const taskExecutor = new import_core.Executor(description);
|
|
1366
|
+
const { timeoutMs, checkIntervalMs } = opt;
|
|
1367
|
+
(0, import_node_assert2.default)(assertion, "No assertion for waitFor");
|
|
1368
|
+
(0, import_node_assert2.default)(timeoutMs, "No timeoutMs for waitFor");
|
|
1369
|
+
(0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
|
|
1370
|
+
const overallStartTime = Date.now();
|
|
1371
|
+
let startTime = Date.now();
|
|
1372
|
+
let errorThought = "";
|
|
1373
|
+
while (Date.now() - overallStartTime < timeoutMs) {
|
|
1374
|
+
startTime = Date.now();
|
|
1375
|
+
const assertPlan = {
|
|
1376
|
+
type: "AssertWithoutThrow",
|
|
1353
1377
|
param: {
|
|
1354
|
-
|
|
1378
|
+
assertion
|
|
1355
1379
|
}
|
|
1356
1380
|
};
|
|
1357
|
-
const
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
)
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1381
|
+
const assertTask = yield this.convertPlanToExecutable([assertPlan]);
|
|
1382
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
|
|
1383
|
+
const output = yield taskExecutor.flush();
|
|
1384
|
+
if (output == null ? void 0 : output.pass) {
|
|
1385
|
+
return {
|
|
1386
|
+
output: void 0,
|
|
1387
|
+
executor: taskExecutor
|
|
1388
|
+
};
|
|
1389
|
+
}
|
|
1390
|
+
errorThought = (output == null ? void 0 : output.thought) || "unknown error";
|
|
1391
|
+
const now = Date.now();
|
|
1392
|
+
if (now - startTime < checkIntervalMs) {
|
|
1393
|
+
const timeRemaining = checkIntervalMs - (now - startTime);
|
|
1394
|
+
const sleepPlan = {
|
|
1395
|
+
type: "Sleep",
|
|
1396
|
+
param: {
|
|
1397
|
+
timeMs: timeRemaining
|
|
1398
|
+
}
|
|
1399
|
+
};
|
|
1400
|
+
const sleepTask = yield this.convertPlanToExecutable([sleepPlan]);
|
|
1401
|
+
yield taskExecutor.append(
|
|
1402
|
+
this.wrapExecutorWithScreenshot(sleepTask[0])
|
|
1403
|
+
);
|
|
1404
|
+
yield taskExecutor.flush();
|
|
1405
|
+
}
|
|
1368
1406
|
}
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1407
|
+
const errorPlan = {
|
|
1408
|
+
type: "Error",
|
|
1409
|
+
param: {
|
|
1410
|
+
thought: `waitFor timeout: ${errorThought}`
|
|
1411
|
+
}
|
|
1412
|
+
};
|
|
1413
|
+
const errorTask = yield this.convertPlanToExecutable([errorPlan]);
|
|
1414
|
+
yield taskExecutor.append(errorTask[0]);
|
|
1415
|
+
yield taskExecutor.flush();
|
|
1416
|
+
return {
|
|
1417
|
+
output: void 0,
|
|
1418
|
+
executor: taskExecutor
|
|
1419
|
+
};
|
|
1420
|
+
});
|
|
1377
1421
|
}
|
|
1378
1422
|
};
|
|
1379
1423
|
|
|
@@ -1407,83 +1451,94 @@ var PageAgent = class {
|
|
|
1407
1451
|
dumpDataString() {
|
|
1408
1452
|
this.dump.groupName = this.opts.groupName;
|
|
1409
1453
|
this.dump.groupDescription = this.opts.groupDescription;
|
|
1410
|
-
return (0,
|
|
1454
|
+
return (0, import_utils7.stringifyDumpData)(this.dump);
|
|
1411
1455
|
}
|
|
1412
1456
|
writeOutActionDumps() {
|
|
1413
1457
|
const { generateReport, autoPrintReportMsg } = this.opts;
|
|
1414
|
-
this.reportFile = (0,
|
|
1458
|
+
this.reportFile = (0, import_utils7.writeLogFile)({
|
|
1415
1459
|
fileName: this.reportFileName,
|
|
1416
|
-
fileExt:
|
|
1460
|
+
fileExt: import_utils7.groupedActionDumpFileExt,
|
|
1417
1461
|
fileContent: this.dumpDataString(),
|
|
1418
1462
|
type: "dump",
|
|
1419
1463
|
generateReport
|
|
1420
1464
|
});
|
|
1421
|
-
if (generateReport && autoPrintReportMsg) {
|
|
1465
|
+
if (generateReport && autoPrintReportMsg && this.reportFile) {
|
|
1422
1466
|
printReportMsg(this.reportFile);
|
|
1423
1467
|
}
|
|
1424
1468
|
}
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1469
|
+
aiAction(taskPrompt) {
|
|
1470
|
+
return __async(this, null, function* () {
|
|
1471
|
+
const { executor } = yield this.taskExecutor.action(taskPrompt);
|
|
1472
|
+
this.appendExecutionDump(executor.dump());
|
|
1473
|
+
this.writeOutActionDumps();
|
|
1474
|
+
if (executor.isInErrorState()) {
|
|
1475
|
+
const errorTask = executor.latestErrorTask();
|
|
1476
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1432
1477
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1433
|
-
|
|
1478
|
+
}
|
|
1479
|
+
});
|
|
1434
1480
|
}
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1481
|
+
aiQuery(demand) {
|
|
1482
|
+
return __async(this, null, function* () {
|
|
1483
|
+
const { output, executor } = yield this.taskExecutor.query(demand);
|
|
1484
|
+
this.appendExecutionDump(executor.dump());
|
|
1485
|
+
this.writeOutActionDumps();
|
|
1486
|
+
if (executor.isInErrorState()) {
|
|
1487
|
+
const errorTask = executor.latestErrorTask();
|
|
1488
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1442
1489
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1443
|
-
|
|
1444
|
-
return output;
|
|
1445
|
-
}
|
|
1446
|
-
async aiAssert(assertion, msg, opt) {
|
|
1447
|
-
const { output, executor } = await this.taskExecutor.assert(assertion);
|
|
1448
|
-
this.appendExecutionDump(executor.dump());
|
|
1449
|
-
this.writeOutActionDumps();
|
|
1450
|
-
if (opt == null ? void 0 : opt.keepRawResponse) {
|
|
1490
|
+
}
|
|
1451
1491
|
return output;
|
|
1452
|
-
}
|
|
1453
|
-
if (!(output == null ? void 0 : output.pass)) {
|
|
1454
|
-
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1455
|
-
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
|
|
1456
|
-
throw new Error(`${errMsg}
|
|
1457
|
-
${reasonMsg}`);
|
|
1458
|
-
}
|
|
1492
|
+
});
|
|
1459
1493
|
}
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1494
|
+
aiAssert(assertion, msg, opt) {
|
|
1495
|
+
return __async(this, null, function* () {
|
|
1496
|
+
var _a;
|
|
1497
|
+
const { output, executor } = yield this.taskExecutor.assert(assertion);
|
|
1498
|
+
this.appendExecutionDump(executor.dump());
|
|
1499
|
+
this.writeOutActionDumps();
|
|
1500
|
+
if (opt == null ? void 0 : opt.keepRawResponse) {
|
|
1501
|
+
return output;
|
|
1502
|
+
}
|
|
1503
|
+
if (!(output == null ? void 0 : output.pass)) {
|
|
1504
|
+
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1505
|
+
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || ((_a = executor.latestErrorTask()) == null ? void 0 : _a.error) || "(no_reason)"}`;
|
|
1506
|
+
throw new Error(`${errMsg}
|
|
1507
|
+
${reasonMsg}`);
|
|
1508
|
+
}
|
|
1465
1509
|
});
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
const
|
|
1470
|
-
|
|
1510
|
+
}
|
|
1511
|
+
aiWaitFor(assertion, opt) {
|
|
1512
|
+
return __async(this, null, function* () {
|
|
1513
|
+
const { executor } = yield this.taskExecutor.waitFor(assertion, {
|
|
1514
|
+
timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
|
|
1515
|
+
checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
|
|
1516
|
+
assertion
|
|
1517
|
+
});
|
|
1518
|
+
this.appendExecutionDump(executor.dump());
|
|
1519
|
+
this.writeOutActionDumps();
|
|
1520
|
+
if (executor.isInErrorState()) {
|
|
1521
|
+
const errorTask = executor.latestErrorTask();
|
|
1522
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1471
1523
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1472
|
-
|
|
1524
|
+
}
|
|
1525
|
+
});
|
|
1473
1526
|
}
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1527
|
+
ai(taskPrompt, type = "action") {
|
|
1528
|
+
return __async(this, null, function* () {
|
|
1529
|
+
if (type === "action") {
|
|
1530
|
+
return this.aiAction(taskPrompt);
|
|
1531
|
+
}
|
|
1532
|
+
if (type === "query") {
|
|
1533
|
+
return this.aiQuery(taskPrompt);
|
|
1534
|
+
}
|
|
1535
|
+
if (type === "assert") {
|
|
1536
|
+
return this.aiAssert(taskPrompt);
|
|
1537
|
+
}
|
|
1538
|
+
throw new Error(
|
|
1539
|
+
`Unknown type: ${type}, only support 'action', 'query', 'assert'`
|
|
1540
|
+
);
|
|
1541
|
+
});
|
|
1487
1542
|
}
|
|
1488
1543
|
};
|
|
1489
1544
|
|
|
@@ -1495,8 +1550,6 @@ var StaticPageAgent = class extends PageAgent {
|
|
|
1495
1550
|
};
|
|
1496
1551
|
|
|
1497
1552
|
// src/playground/static-page.ts
|
|
1498
|
-
var import_utils8 = require("@midscene/core/utils");
|
|
1499
|
-
var import_img4 = require("@midscene/shared/img");
|
|
1500
1553
|
var ThrowNotImplemented = (methodName) => {
|
|
1501
1554
|
throw new Error(
|
|
1502
1555
|
`The method "${methodName}" is not implemented as designed since this is a static UI context. (${ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED})`
|
|
@@ -1516,38 +1569,52 @@ var StaticPage = class {
|
|
|
1516
1569
|
};
|
|
1517
1570
|
this.uiContext = uiContext;
|
|
1518
1571
|
}
|
|
1519
|
-
|
|
1520
|
-
return
|
|
1572
|
+
getElementInfos() {
|
|
1573
|
+
return __async(this, null, function* () {
|
|
1574
|
+
return ThrowNotImplemented("getElementInfos");
|
|
1575
|
+
});
|
|
1521
1576
|
}
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1577
|
+
screenshotBase64() {
|
|
1578
|
+
return __async(this, null, function* () {
|
|
1579
|
+
const base64 = this.uiContext.screenshotBase64;
|
|
1580
|
+
if (!base64) {
|
|
1581
|
+
throw new Error("screenshot base64 is empty");
|
|
1582
|
+
}
|
|
1583
|
+
return base64;
|
|
1584
|
+
});
|
|
1530
1585
|
}
|
|
1531
1586
|
url() {
|
|
1532
1587
|
return this.uiContext.url;
|
|
1533
1588
|
}
|
|
1534
|
-
|
|
1535
|
-
return
|
|
1589
|
+
scrollUntilTop() {
|
|
1590
|
+
return __async(this, null, function* () {
|
|
1591
|
+
return ThrowNotImplemented("scrollUntilTop");
|
|
1592
|
+
});
|
|
1536
1593
|
}
|
|
1537
|
-
|
|
1538
|
-
return
|
|
1594
|
+
scrollUntilBottom() {
|
|
1595
|
+
return __async(this, null, function* () {
|
|
1596
|
+
return ThrowNotImplemented("scrollUntilBottom");
|
|
1597
|
+
});
|
|
1539
1598
|
}
|
|
1540
|
-
|
|
1541
|
-
return
|
|
1599
|
+
scrollUpOneScreen() {
|
|
1600
|
+
return __async(this, null, function* () {
|
|
1601
|
+
return ThrowNotImplemented("scrollUpOneScreen");
|
|
1602
|
+
});
|
|
1542
1603
|
}
|
|
1543
|
-
|
|
1544
|
-
return
|
|
1604
|
+
scrollDownOneScreen() {
|
|
1605
|
+
return __async(this, null, function* () {
|
|
1606
|
+
return ThrowNotImplemented("scrollDownOneScreen");
|
|
1607
|
+
});
|
|
1545
1608
|
}
|
|
1546
|
-
|
|
1547
|
-
return
|
|
1609
|
+
clearInput() {
|
|
1610
|
+
return __async(this, null, function* () {
|
|
1611
|
+
return ThrowNotImplemented("clearInput");
|
|
1612
|
+
});
|
|
1548
1613
|
}
|
|
1549
|
-
|
|
1550
|
-
return this
|
|
1614
|
+
_forceUsePageContext() {
|
|
1615
|
+
return __async(this, null, function* () {
|
|
1616
|
+
return this.uiContext;
|
|
1617
|
+
});
|
|
1551
1618
|
}
|
|
1552
1619
|
};
|
|
1553
1620
|
|
|
@@ -1561,123 +1628,128 @@ var errorHandler = (err, req, res, next) => {
|
|
|
1561
1628
|
error: err.message
|
|
1562
1629
|
});
|
|
1563
1630
|
};
|
|
1564
|
-
var setup = () => {
|
|
1565
|
-
|
|
1566
|
-
|
|
1631
|
+
var setup = () => __async(void 0, null, function* () {
|
|
1632
|
+
if (!import_utils12.ifInBrowser) {
|
|
1633
|
+
const dotenv = yield Promise.resolve().then(() => __toESM(require_main()));
|
|
1634
|
+
dotenv.config();
|
|
1635
|
+
}
|
|
1636
|
+
});
|
|
1567
1637
|
var PlaygroundServer = class {
|
|
1568
1638
|
constructor() {
|
|
1569
1639
|
this.app = (0, import_express.default)();
|
|
1570
|
-
this.tmpDir = (0,
|
|
1640
|
+
this.tmpDir = (0, import_utils11.getTmpDir)();
|
|
1571
1641
|
setup();
|
|
1572
1642
|
}
|
|
1573
|
-
filePathForUuid(
|
|
1574
|
-
return (0, import_node_path3.join)(this.tmpDir, `${
|
|
1643
|
+
filePathForUuid(uuid2) {
|
|
1644
|
+
return (0, import_node_path3.join)(this.tmpDir, `${uuid2}.json`);
|
|
1575
1645
|
}
|
|
1576
|
-
saveContextFile(
|
|
1577
|
-
const tmpFile = this.filePathForUuid(
|
|
1646
|
+
saveContextFile(uuid2, context) {
|
|
1647
|
+
const tmpFile = this.filePathForUuid(uuid2);
|
|
1578
1648
|
console.log(`save context file: ${tmpFile}`);
|
|
1579
1649
|
(0, import_node_fs3.writeFileSync)(tmpFile, context);
|
|
1580
1650
|
return tmpFile;
|
|
1581
1651
|
}
|
|
1582
|
-
|
|
1583
|
-
this
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
this.app.get("/playground/:uuid", async (req, res) => {
|
|
1599
|
-
res.sendFile((0, import_node_path3.join)(staticPath, "index.html"));
|
|
1600
|
-
});
|
|
1601
|
-
this.app.get("/context/:uuid", async (req, res) => {
|
|
1602
|
-
const { uuid } = req.params;
|
|
1603
|
-
const contextFile = this.filePathForUuid(uuid);
|
|
1604
|
-
(0, import_node_assert3.default)((0, import_node_fs3.existsSync)(contextFile), "Context not found");
|
|
1605
|
-
const context = (0, import_node_fs3.readFileSync)(contextFile, "utf8");
|
|
1606
|
-
res.json({
|
|
1607
|
-
context
|
|
1652
|
+
launch() {
|
|
1653
|
+
return __async(this, null, function* () {
|
|
1654
|
+
this.app.use(errorHandler);
|
|
1655
|
+
this.app.use(
|
|
1656
|
+
(0, import_cors.default)({
|
|
1657
|
+
origin: "*",
|
|
1658
|
+
credentials: true
|
|
1659
|
+
})
|
|
1660
|
+
);
|
|
1661
|
+
this.app.get("/status", (0, import_cors.default)(), (req, res) => __async(this, null, function* () {
|
|
1662
|
+
res.send({
|
|
1663
|
+
status: "ok"
|
|
1664
|
+
});
|
|
1665
|
+
}));
|
|
1666
|
+
this.app.get("/", (req, res) => {
|
|
1667
|
+
res.sendFile((0, import_node_path3.join)(staticPath, "index.html"));
|
|
1608
1668
|
});
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
const
|
|
1615
|
-
(0, import_node_assert3.default)(
|
|
1616
|
-
const
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
location: `/playground/${uuid}`,
|
|
1620
|
-
uuid
|
|
1669
|
+
this.app.get("/playground/:uuid", (req, res) => __async(this, null, function* () {
|
|
1670
|
+
res.sendFile((0, import_node_path3.join)(staticPath, "index.html"));
|
|
1671
|
+
}));
|
|
1672
|
+
this.app.get("/context/:uuid", (req, res) => __async(this, null, function* () {
|
|
1673
|
+
const { uuid: uuid2 } = req.params;
|
|
1674
|
+
const contextFile = this.filePathForUuid(uuid2);
|
|
1675
|
+
(0, import_node_assert3.default)((0, import_node_fs3.existsSync)(contextFile), "Context not found");
|
|
1676
|
+
const context = (0, import_node_fs3.readFileSync)(contextFile, "utf8");
|
|
1677
|
+
res.json({
|
|
1678
|
+
context
|
|
1621
1679
|
});
|
|
1622
|
-
}
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1680
|
+
}));
|
|
1681
|
+
this.app.post(
|
|
1682
|
+
"/playground-with-context",
|
|
1683
|
+
import_express.default.json({ limit: "50mb" }),
|
|
1684
|
+
(req, res) => __async(this, null, function* () {
|
|
1685
|
+
const context = req.body.context;
|
|
1686
|
+
(0, import_node_assert3.default)(context, "context is required");
|
|
1687
|
+
const uuid2 = (0, import_node_crypto.randomUUID)();
|
|
1688
|
+
this.saveContextFile(uuid2, context);
|
|
1689
|
+
return res.json({
|
|
1690
|
+
location: `/playground/${uuid2}`,
|
|
1691
|
+
uuid: uuid2
|
|
1692
|
+
});
|
|
1693
|
+
})
|
|
1694
|
+
);
|
|
1695
|
+
this.app.post(
|
|
1696
|
+
"/execute",
|
|
1697
|
+
import_express.default.json({ limit: "30mb" }),
|
|
1698
|
+
(req, res) => __async(this, null, function* () {
|
|
1699
|
+
const { context, type, prompt } = req.body;
|
|
1700
|
+
(0, import_node_assert3.default)(context, "context is required");
|
|
1701
|
+
(0, import_node_assert3.default)(type, "type is required");
|
|
1702
|
+
(0, import_node_assert3.default)(prompt, "prompt is required");
|
|
1703
|
+
const requestId = agentRequestCount++;
|
|
1704
|
+
console.log(`handle request: #${requestId}, ${type}, ${prompt}`);
|
|
1705
|
+
const page = new StaticPage(context);
|
|
1706
|
+
const agent = new StaticPageAgent(page);
|
|
1707
|
+
const response = {
|
|
1708
|
+
result: null,
|
|
1709
|
+
dump: null,
|
|
1710
|
+
error: null
|
|
1711
|
+
};
|
|
1712
|
+
try {
|
|
1713
|
+
if (type === "aiQuery") {
|
|
1714
|
+
response.result = yield agent.aiQuery(prompt);
|
|
1715
|
+
} else if (type === "aiAction") {
|
|
1716
|
+
response.result = yield agent.aiAction(prompt);
|
|
1717
|
+
} else if (type === "aiAssert") {
|
|
1718
|
+
response.result = yield agent.aiAssert(prompt, void 0, {
|
|
1719
|
+
keepRawResponse: true
|
|
1720
|
+
});
|
|
1721
|
+
} else {
|
|
1722
|
+
response.error = `Unknown type: ${type}`;
|
|
1723
|
+
}
|
|
1724
|
+
} catch (error) {
|
|
1725
|
+
if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {
|
|
1726
|
+
response.error = error.message;
|
|
1727
|
+
}
|
|
1652
1728
|
}
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1729
|
+
try {
|
|
1730
|
+
response.dump = JSON.parse(agent.dumpDataString());
|
|
1731
|
+
agent.writeOutActionDumps();
|
|
1732
|
+
} catch (error) {
|
|
1733
|
+
console.error(
|
|
1734
|
+
`write out dump failed: #${requestId}, ${error.message}`
|
|
1735
|
+
);
|
|
1656
1736
|
}
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
`
|
|
1664
|
-
|
|
1665
|
-
}
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
}
|
|
1674
|
-
}
|
|
1675
|
-
);
|
|
1676
|
-
return new Promise((resolve, reject) => {
|
|
1677
|
-
const port = this.port || defaultPort;
|
|
1678
|
-
this.server = this.app.listen(port, () => {
|
|
1679
|
-
this.port = port;
|
|
1680
|
-
resolve(this);
|
|
1737
|
+
res.send(response);
|
|
1738
|
+
if (response.error) {
|
|
1739
|
+
console.error(
|
|
1740
|
+
`handle request failed: #${requestId}, ${response.error}`
|
|
1741
|
+
);
|
|
1742
|
+
} else {
|
|
1743
|
+
console.log(`handle request done: #${requestId}`);
|
|
1744
|
+
}
|
|
1745
|
+
})
|
|
1746
|
+
);
|
|
1747
|
+
return new Promise((resolve, reject) => {
|
|
1748
|
+
const port = this.port || defaultPort;
|
|
1749
|
+
this.server = this.app.listen(port, () => {
|
|
1750
|
+
this.port = port;
|
|
1751
|
+
resolve(this);
|
|
1752
|
+
});
|
|
1681
1753
|
});
|
|
1682
1754
|
});
|
|
1683
1755
|
}
|