@midscene/web 0.7.1 → 0.7.2-beta-20241024094141.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -41,6 +41,26 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
41
41
  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
42
42
  mod
43
43
  ));
44
+ var __async = (__this, __arguments, generator) => {
45
+ return new Promise((resolve, reject) => {
46
+ var fulfilled = (value) => {
47
+ try {
48
+ step(generator.next(value));
49
+ } catch (e) {
50
+ reject(e);
51
+ }
52
+ };
53
+ var rejected = (value) => {
54
+ try {
55
+ step(generator.throw(value));
56
+ } catch (e) {
57
+ reject(e);
58
+ }
59
+ };
60
+ var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
61
+ step((generator = generator.apply(__this, __arguments)).next());
62
+ });
63
+ };
44
64
 
45
65
  // ../../node_modules/.pnpm/dayjs@1.11.11/node_modules/dayjs/dayjs.min.js
46
66
  var require_dayjs_min = __commonJS({
@@ -678,19 +698,19 @@ var require_main = __commonJS({
678
698
 
679
699
  // src/playground/server.ts
680
700
  var import_node_assert3 = __toESM(require("assert"));
681
- var import_node_crypto2 = require("crypto");
701
+ var import_node_crypto = require("crypto");
682
702
  var import_node_fs3 = require("fs");
683
703
  var import_node_path3 = require("path");
684
704
 
685
705
  // src/common/utils.ts
686
706
  var import_node_assert = __toESM(require("assert"));
687
- var import_node_crypto = require("crypto");
688
707
  var import_node_fs = require("fs");
689
708
  var import_node_path = __toESM(require("path"));
690
709
  var import_constants = require("@midscene/shared/constants");
691
710
  var import_fs = require("@midscene/shared/fs");
692
711
  var import_img = require("@midscene/shared/img");
693
712
  var import_img2 = require("@midscene/shared/img");
713
+ var import_utils = require("@midscene/shared/utils");
694
714
  var import_dayjs = __toESM(require_dayjs_min());
695
715
 
696
716
  // src/web-element.ts
@@ -719,56 +739,59 @@ var WebElementInfo = class {
719
739
  };
720
740
 
721
741
  // src/common/utils.ts
722
- async function parseContextFromWebPage(page, _opt) {
723
- (0, import_node_assert.default)(page, "page is required");
724
- if (page._forceUsePageContext) {
725
- return await page._forceUsePageContext();
726
- }
727
- const url = page.url();
728
- const file = await page.screenshot();
729
- const screenshotBase64 = (0, import_img.base64Encoded)(file);
730
- const captureElementSnapshot = await page.getElementInfos();
731
- const elementsInfo = await alignElements(captureElementSnapshot, page);
732
- const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
733
- if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
734
- return false;
742
+ function parseContextFromWebPage(page, _opt) {
743
+ return __async(this, null, function* () {
744
+ (0, import_node_assert.default)(page, "page is required");
745
+ if (page._forceUsePageContext) {
746
+ return yield page._forceUsePageContext();
735
747
  }
736
- return true;
737
- });
738
- const size = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
739
- const screenshotBase64WithElementMarker = await (0, import_img2.compositeElementInfoImg)({
740
- inputImgBase64: screenshotBase64.split(";base64,").pop(),
741
- elementsPositionInfo: elementsPositionInfoWithoutText
748
+ const url = page.url();
749
+ const screenshotBase64 = yield page.screenshotBase64();
750
+ const captureElementSnapshot = yield page.getElementInfos();
751
+ const elementsInfo = yield alignElements(captureElementSnapshot, page);
752
+ const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
753
+ if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
754
+ return false;
755
+ }
756
+ return true;
757
+ });
758
+ const size = yield (0, import_img.imageInfoOfBase64)(screenshotBase64);
759
+ const screenshotBase64WithElementMarker = yield (0, import_img2.compositeElementInfoImg)({
760
+ inputImgBase64: screenshotBase64,
761
+ elementsPositionInfo: elementsPositionInfoWithoutText
762
+ });
763
+ return {
764
+ content: elementsInfo,
765
+ size,
766
+ screenshotBase64,
767
+ screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
768
+ url
769
+ };
742
770
  });
743
- return {
744
- content: elementsInfo,
745
- size,
746
- screenshotBase64,
747
- screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
748
- url
749
- };
750
771
  }
751
772
  var sizeThreshold = 3;
752
- async function alignElements(elements, page) {
753
- const validElements = elements.filter((item) => {
754
- return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
773
+ function alignElements(elements, page) {
774
+ return __async(this, null, function* () {
775
+ const validElements = elements.filter((item) => {
776
+ return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
777
+ });
778
+ const textsAligned = [];
779
+ for (const item of validElements) {
780
+ const { rect, id, content, attributes, locator, indexId } = item;
781
+ textsAligned.push(
782
+ new WebElementInfo({
783
+ rect,
784
+ locator,
785
+ id,
786
+ content,
787
+ attributes,
788
+ page,
789
+ indexId
790
+ })
791
+ );
792
+ }
793
+ return textsAligned;
755
794
  });
756
- const textsAligned = [];
757
- for (const item of validElements) {
758
- const { rect, id, content, attributes, locator, indexId } = item;
759
- textsAligned.push(
760
- new WebElementInfo({
761
- rect,
762
- locator,
763
- id,
764
- content,
765
- attributes,
766
- page,
767
- indexId
768
- })
769
- );
770
- }
771
- return textsAligned;
772
795
  }
773
796
  function reportFileName(tag = "web") {
774
797
  const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
@@ -799,7 +822,7 @@ var testFileIndex = /* @__PURE__ */ new Map();
799
822
  function generateCacheId(fileName) {
800
823
  let taskFile = fileName || getCurrentExecutionFile();
801
824
  if (!taskFile) {
802
- taskFile = (0, import_node_crypto.randomUUID)();
825
+ taskFile = (0, import_utils.uuid)();
803
826
  console.warn(
804
827
  "Midscene - using random UUID for cache id. Cache may be invalid."
805
828
  );
@@ -817,28 +840,28 @@ function generateCacheId(fileName) {
817
840
  var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
818
841
 
819
842
  // src/playground/server.ts
820
- var import_utils10 = require("@midscene/core/utils");
843
+ var import_utils11 = require("@midscene/core/utils");
844
+ var import_utils12 = require("@midscene/shared/utils");
821
845
  var import_cors = __toESM(require("cors"));
822
- var import_dotenv = __toESM(require_main());
823
846
  var import_express = __toESM(require("express"));
824
847
 
825
848
  // src/common/agent.ts
826
- var import_utils5 = require("@midscene/core/utils");
849
+ var import_utils7 = require("@midscene/core/utils");
827
850
 
828
851
  // src/common/tasks.ts
829
852
  var import_node_assert2 = __toESM(require("assert"));
830
853
  var import_core = require("@midscene/core");
831
- var import_utils3 = require("@midscene/core/utils");
832
- var import_img3 = require("@midscene/shared/img");
854
+ var import_utils5 = require("@midscene/core/utils");
833
855
 
834
856
  // src/common/task-cache.ts
835
857
  var import_node_fs2 = require("fs");
836
858
  var import_node_path2 = require("path");
837
- var import_utils = require("@midscene/core/utils");
859
+ var import_utils2 = require("@midscene/core/utils");
838
860
  var import_fs2 = require("@midscene/shared/fs");
861
+ var import_utils3 = require("@midscene/shared/utils");
839
862
  var TaskCache = class {
840
863
  constructor(opts) {
841
- this.midscenePkgInfo = (0, import_fs2.getMidscenePkgInfo)(__dirname);
864
+ this.midscenePkgInfo = (0, import_fs2.getRunningPkgInfo)();
842
865
  this.cacheId = generateCacheId(opts == null ? void 0 : opts.fileName);
843
866
  this.cache = this.readCacheFromFile() || {
844
867
  aiTasks: []
@@ -920,11 +943,17 @@ var TaskCache = class {
920
943
  return this.newCache;
921
944
  }
922
945
  readCacheFromFile() {
923
- const cacheFile = (0, import_node_path2.join)((0, import_utils.getLogDirByType)("cache"), `${this.cacheId}.json`);
946
+ if (import_utils3.ifInBrowser) {
947
+ return void 0;
948
+ }
949
+ const cacheFile = (0, import_node_path2.join)((0, import_utils2.getLogDirByType)("cache"), `${this.cacheId}.json`);
924
950
  if (process.env.MIDSCENE_CACHE === "true" && (0, import_node_fs2.existsSync)(cacheFile)) {
925
951
  try {
926
952
  const data = (0, import_node_fs2.readFileSync)(cacheFile, "utf8");
927
953
  const jsonData = JSON.parse(data);
954
+ if (!this.midscenePkgInfo) {
955
+ return void 0;
956
+ }
928
957
  if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
929
958
  return void 0;
930
959
  }
@@ -936,11 +965,14 @@ var TaskCache = class {
936
965
  return void 0;
937
966
  }
938
967
  writeCacheToFile() {
939
- const midscenePkgInfo = (0, import_fs2.getMidscenePkgInfo)(__dirname);
940
- (0, import_utils.writeLogFile)({
968
+ const midscenePkgInfo = (0, import_fs2.getRunningPkgInfo)();
969
+ if (!midscenePkgInfo) {
970
+ return;
971
+ }
972
+ (0, import_utils2.writeLogFile)({
941
973
  fileName: `${this.cacheId}`,
942
974
  fileExt: "json",
943
- fileContent: (0, import_utils.stringifyDumpData)(
975
+ fileContent: (0, import_utils2.stringifyDumpData)(
944
976
  __spreadValues({
945
977
  pkgName: midscenePkgInfo.name,
946
978
  pkgVersion: midscenePkgInfo.version,
@@ -957,423 +989,435 @@ var TaskCache = class {
957
989
  var PageTaskExecutor = class {
958
990
  constructor(page, opts) {
959
991
  this.page = page;
960
- this.insight = new import_core.Insight(async () => {
961
- return await parseContextFromWebPage(page);
962
- });
992
+ this.insight = new import_core.Insight(() => __async(this, null, function* () {
993
+ return yield parseContextFromWebPage(page);
994
+ }));
963
995
  this.taskCache = new TaskCache({
964
996
  fileName: opts == null ? void 0 : opts.cacheId
965
997
  });
966
998
  }
967
- async recordScreenshot(timing) {
968
- const file = await this.page.screenshot();
969
- const item = {
970
- type: "screenshot",
971
- ts: Date.now(),
972
- screenshot: (0, import_img3.base64Encoded)(file),
973
- timing
974
- };
975
- return item;
999
+ recordScreenshot(timing) {
1000
+ return __async(this, null, function* () {
1001
+ const base64 = yield this.page.screenshotBase64();
1002
+ const item = {
1003
+ type: "screenshot",
1004
+ ts: Date.now(),
1005
+ screenshot: base64,
1006
+ timing
1007
+ };
1008
+ return item;
1009
+ });
976
1010
  }
977
1011
  wrapExecutorWithScreenshot(taskApply) {
978
1012
  const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
979
- executor: async (param, context, ...args) => {
1013
+ executor: (param, context, ...args) => __async(this, null, function* () {
980
1014
  const recorder = [];
981
1015
  const { task } = context;
982
1016
  task.recorder = recorder;
983
- const shot = await this.recordScreenshot(`before ${task.type}`);
1017
+ const shot = yield this.recordScreenshot(`before ${task.type}`);
984
1018
  recorder.push(shot);
985
- const result = await taskApply.executor(param, context, ...args);
1019
+ const result = yield taskApply.executor(param, context, ...args);
986
1020
  if (taskApply.type === "Action") {
987
- await (0, import_utils3.sleep)(1e3);
988
- const shot2 = await this.recordScreenshot("after Action");
1021
+ yield (0, import_utils5.sleep)(1e3);
1022
+ const shot2 = yield this.recordScreenshot("after Action");
989
1023
  recorder.push(shot2);
990
1024
  }
991
1025
  return result;
992
- }
1026
+ })
993
1027
  });
994
1028
  return taskWithScreenshot;
995
1029
  }
996
- async convertPlanToExecutable(plans, cacheGroup) {
997
- const tasks = plans.map((plan2) => {
998
- if (plan2.type === "Locate") {
999
- const taskFind = {
1000
- type: "Insight",
1001
- subType: "Locate",
1002
- param: plan2.param,
1003
- executor: async (param, taskContext) => {
1004
- const { task } = taskContext;
1005
- let insightDump;
1006
- const dumpCollector = (dump) => {
1007
- insightDump = dump;
1008
- };
1009
- this.insight.onceDumpUpdatedFn = dumpCollector;
1010
- const pageContext = await this.insight.contextRetrieverFn();
1011
- const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
1012
- pageContext,
1013
- "locate",
1014
- param.prompt
1015
- );
1016
- let locateResult;
1017
- const callAI = this.insight.aiVendorFn;
1018
- const element = await this.insight.locate(param.prompt, {
1019
- quickAnswer: plan2.quickAnswer,
1020
- callAI: async (...message) => {
1021
- if (locateCache) {
1022
- locateResult = locateCache;
1023
- return Promise.resolve(locateCache);
1024
- }
1025
- locateResult = await callAI(...message);
1026
- (0, import_node_assert2.default)(locateResult);
1027
- return locateResult;
1028
- }
1029
- });
1030
- if (locateResult) {
1031
- cacheGroup == null ? void 0 : cacheGroup.saveCache({
1032
- type: "locate",
1033
- pageContext: {
1034
- url: pageContext.url,
1035
- size: pageContext.size
1036
- },
1037
- prompt: param.prompt,
1038
- response: locateResult
1039
- });
1040
- }
1041
- if (!element) {
1042
- task.log = {
1043
- dump: insightDump
1030
+ convertPlanToExecutable(plans, cacheGroup) {
1031
+ return __async(this, null, function* () {
1032
+ const tasks = plans.map((plan2) => {
1033
+ if (plan2.type === "Locate") {
1034
+ const taskFind = {
1035
+ type: "Insight",
1036
+ subType: "Locate",
1037
+ param: plan2.param,
1038
+ executor: (param, taskContext) => __async(this, null, function* () {
1039
+ const { task } = taskContext;
1040
+ let insightDump;
1041
+ const dumpCollector = (dump) => {
1042
+ insightDump = dump;
1044
1043
  };
1045
- throw new Error(`Element not found: ${param.prompt}`);
1046
- }
1047
- return {
1048
- output: {
1049
- element
1050
- },
1051
- log: {
1052
- dump: insightDump
1053
- },
1054
- cache: {
1055
- hit: Boolean(locateCache)
1044
+ this.insight.onceDumpUpdatedFn = dumpCollector;
1045
+ const pageContext = yield this.insight.contextRetrieverFn();
1046
+ const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
1047
+ pageContext,
1048
+ "locate",
1049
+ param.prompt
1050
+ );
1051
+ let locateResult;
1052
+ const callAI = this.insight.aiVendorFn;
1053
+ const element = yield this.insight.locate(param.prompt, {
1054
+ quickAnswer: plan2.quickAnswer,
1055
+ callAI: (...message) => __async(this, null, function* () {
1056
+ if (locateCache) {
1057
+ locateResult = locateCache;
1058
+ return Promise.resolve(locateCache);
1059
+ }
1060
+ locateResult = yield callAI(...message);
1061
+ (0, import_node_assert2.default)(locateResult);
1062
+ return locateResult;
1063
+ })
1064
+ });
1065
+ if (locateResult) {
1066
+ cacheGroup == null ? void 0 : cacheGroup.saveCache({
1067
+ type: "locate",
1068
+ pageContext: {
1069
+ url: pageContext.url,
1070
+ size: pageContext.size
1071
+ },
1072
+ prompt: param.prompt,
1073
+ response: locateResult
1074
+ });
1056
1075
  }
1057
- };
1058
- }
1059
- };
1060
- return taskFind;
1061
- }
1062
- if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
1063
- const assertPlan = plan2;
1064
- const taskAssert = {
1065
- type: "Insight",
1066
- subType: "Assert",
1067
- param: assertPlan.param,
1068
- executor: async (param, taskContext) => {
1069
- const { task } = taskContext;
1070
- let insightDump;
1071
- const dumpCollector = (dump) => {
1072
- insightDump = dump;
1073
- };
1074
- this.insight.onceDumpUpdatedFn = dumpCollector;
1075
- const assertion = await this.insight.assert(
1076
- assertPlan.param.assertion
1077
- );
1078
- if (!assertion.pass) {
1079
- if (plan2.type === "Assert") {
1080
- task.output = assertion;
1076
+ if (!element) {
1081
1077
  task.log = {
1082
1078
  dump: insightDump
1083
1079
  };
1084
- throw new Error(
1085
- assertion.thought || "Assertion failed without reason"
1086
- );
1080
+ throw new Error(`Element not found: ${param.prompt}`);
1087
1081
  }
1088
- task.error = assertion.thought;
1089
- }
1090
- return {
1091
- output: assertion,
1092
- log: {
1093
- dump: insightDump
1082
+ return {
1083
+ output: {
1084
+ element
1085
+ },
1086
+ log: {
1087
+ dump: insightDump
1088
+ },
1089
+ cache: {
1090
+ hit: Boolean(locateCache)
1091
+ }
1092
+ };
1093
+ })
1094
+ };
1095
+ return taskFind;
1096
+ }
1097
+ if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
1098
+ const assertPlan = plan2;
1099
+ const taskAssert = {
1100
+ type: "Insight",
1101
+ subType: "Assert",
1102
+ param: assertPlan.param,
1103
+ executor: (param, taskContext) => __async(this, null, function* () {
1104
+ const { task } = taskContext;
1105
+ let insightDump;
1106
+ const dumpCollector = (dump) => {
1107
+ insightDump = dump;
1108
+ };
1109
+ this.insight.onceDumpUpdatedFn = dumpCollector;
1110
+ const assertion = yield this.insight.assert(
1111
+ assertPlan.param.assertion
1112
+ );
1113
+ if (!assertion.pass) {
1114
+ if (plan2.type === "Assert") {
1115
+ task.output = assertion;
1116
+ task.log = {
1117
+ dump: insightDump
1118
+ };
1119
+ throw new Error(
1120
+ assertion.thought || "Assertion failed without reason"
1121
+ );
1122
+ }
1123
+ task.error = assertion.thought;
1094
1124
  }
1095
- };
1096
- }
1097
- };
1098
- return taskAssert;
1099
- }
1100
- if (plan2.type === "Input") {
1101
- const taskActionInput = {
1102
- type: "Action",
1103
- subType: "Input",
1104
- param: plan2.param,
1105
- executor: async (taskParam, { element }) => {
1106
- if (element) {
1107
- await this.page.clearInput(element);
1108
- if (taskParam.value === "") {
1109
- return;
1125
+ return {
1126
+ output: assertion,
1127
+ log: {
1128
+ dump: insightDump
1129
+ }
1130
+ };
1131
+ })
1132
+ };
1133
+ return taskAssert;
1134
+ }
1135
+ if (plan2.type === "Input") {
1136
+ const taskActionInput = {
1137
+ type: "Action",
1138
+ subType: "Input",
1139
+ param: plan2.param,
1140
+ executor: (_0, _1) => __async(this, [_0, _1], function* (taskParam, { element }) {
1141
+ if (element) {
1142
+ yield this.page.clearInput(element);
1143
+ if (taskParam.value === "") {
1144
+ return;
1145
+ }
1146
+ yield this.page.keyboard.type(taskParam.value);
1110
1147
  }
1111
- await this.page.keyboard.type(taskParam.value);
1112
- }
1113
- }
1114
- };
1115
- return taskActionInput;
1116
- }
1117
- if (plan2.type === "KeyboardPress") {
1118
- const taskActionKeyboardPress = {
1119
- type: "Action",
1120
- subType: "KeyboardPress",
1121
- param: plan2.param,
1122
- executor: async (taskParam) => {
1123
- (0, import_node_assert2.default)(taskParam.value, "No key to press");
1124
- await this.page.keyboard.press(taskParam.value);
1125
- }
1126
- };
1127
- return taskActionKeyboardPress;
1128
- }
1129
- if (plan2.type === "Tap") {
1130
- const taskActionTap = {
1131
- type: "Action",
1132
- subType: "Tap",
1133
- executor: async (param, { element }) => {
1134
- (0, import_node_assert2.default)(element, "Element not found, cannot tap");
1135
- await this.page.mouse.click(
1136
- element.center[0],
1137
- element.center[1]
1138
- );
1139
- }
1140
- };
1141
- return taskActionTap;
1142
- }
1143
- if (plan2.type === "Hover") {
1144
- const taskActionHover = {
1145
- type: "Action",
1146
- subType: "Hover",
1147
- executor: async (param, { element }) => {
1148
- (0, import_node_assert2.default)(element, "Element not found, cannot hover");
1149
- await this.page.mouse.move(
1150
- element.center[0],
1151
- element.center[1]
1152
- );
1153
- }
1154
- };
1155
- return taskActionHover;
1156
- }
1157
- if (plan2.type === "Scroll") {
1158
- const taskActionScroll = {
1159
- type: "Action",
1160
- subType: "Scroll",
1161
- param: plan2.param,
1162
- executor: async (taskParam) => {
1163
- const scrollToEventName = taskParam.scrollType;
1164
- switch (scrollToEventName) {
1165
- case "scrollUntilTop":
1166
- await this.page.scrollUntilTop();
1167
- break;
1168
- case "scrollUntilBottom":
1169
- await this.page.scrollUntilBottom();
1170
- break;
1171
- case "scrollUpOneScreen":
1172
- await this.page.scrollUpOneScreen();
1173
- break;
1174
- case "scrollDownOneScreen":
1175
- await this.page.scrollDownOneScreen();
1176
- break;
1177
- default:
1178
- console.error(
1179
- "Unknown scroll event type:",
1180
- scrollToEventName
1181
- );
1182
- }
1183
- }
1184
- };
1185
- return taskActionScroll;
1186
- }
1187
- if (plan2.type === "Sleep") {
1188
- const taskActionSleep = {
1189
- type: "Action",
1190
- subType: "Sleep",
1191
- param: plan2.param,
1192
- executor: async (taskParam) => {
1193
- await (0, import_utils3.sleep)(taskParam.timeMs || 3e3);
1194
- }
1195
- };
1196
- return taskActionSleep;
1197
- }
1198
- if (plan2.type === "Error") {
1199
- const taskActionError = {
1200
- type: "Action",
1201
- subType: "Error",
1202
- param: plan2.param,
1203
- executor: async (taskParam) => {
1204
- (0, import_node_assert2.default)(
1205
- taskParam.thought,
1206
- "An error occurred, but no thought provided"
1207
- );
1208
- throw new Error(taskParam.thought);
1209
- }
1210
- };
1211
- return taskActionError;
1212
- }
1213
- throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
1214
- }).map((task) => {
1215
- return this.wrapExecutorWithScreenshot(task);
1148
+ })
1149
+ };
1150
+ return taskActionInput;
1151
+ }
1152
+ if (plan2.type === "KeyboardPress") {
1153
+ const taskActionKeyboardPress = {
1154
+ type: "Action",
1155
+ subType: "KeyboardPress",
1156
+ param: plan2.param,
1157
+ executor: (taskParam) => __async(this, null, function* () {
1158
+ (0, import_node_assert2.default)(taskParam.value, "No key to press");
1159
+ yield this.page.keyboard.press(taskParam.value);
1160
+ })
1161
+ };
1162
+ return taskActionKeyboardPress;
1163
+ }
1164
+ if (plan2.type === "Tap") {
1165
+ const taskActionTap = {
1166
+ type: "Action",
1167
+ subType: "Tap",
1168
+ executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
1169
+ (0, import_node_assert2.default)(element, "Element not found, cannot tap");
1170
+ yield this.page.mouse.click(
1171
+ element.center[0],
1172
+ element.center[1]
1173
+ );
1174
+ })
1175
+ };
1176
+ return taskActionTap;
1177
+ }
1178
+ if (plan2.type === "Hover") {
1179
+ const taskActionHover = {
1180
+ type: "Action",
1181
+ subType: "Hover",
1182
+ executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
1183
+ (0, import_node_assert2.default)(element, "Element not found, cannot hover");
1184
+ yield this.page.mouse.move(
1185
+ element.center[0],
1186
+ element.center[1]
1187
+ );
1188
+ })
1189
+ };
1190
+ return taskActionHover;
1191
+ }
1192
+ if (plan2.type === "Scroll") {
1193
+ const taskActionScroll = {
1194
+ type: "Action",
1195
+ subType: "Scroll",
1196
+ param: plan2.param,
1197
+ executor: (taskParam) => __async(this, null, function* () {
1198
+ const scrollToEventName = taskParam.scrollType;
1199
+ switch (scrollToEventName) {
1200
+ case "scrollUntilTop":
1201
+ yield this.page.scrollUntilTop();
1202
+ break;
1203
+ case "scrollUntilBottom":
1204
+ yield this.page.scrollUntilBottom();
1205
+ break;
1206
+ case "scrollUpOneScreen":
1207
+ yield this.page.scrollUpOneScreen();
1208
+ break;
1209
+ case "scrollDownOneScreen":
1210
+ yield this.page.scrollDownOneScreen();
1211
+ break;
1212
+ default:
1213
+ console.error(
1214
+ "Unknown scroll event type:",
1215
+ scrollToEventName
1216
+ );
1217
+ }
1218
+ })
1219
+ };
1220
+ return taskActionScroll;
1221
+ }
1222
+ if (plan2.type === "Sleep") {
1223
+ const taskActionSleep = {
1224
+ type: "Action",
1225
+ subType: "Sleep",
1226
+ param: plan2.param,
1227
+ executor: (taskParam) => __async(this, null, function* () {
1228
+ yield (0, import_utils5.sleep)(taskParam.timeMs || 3e3);
1229
+ })
1230
+ };
1231
+ return taskActionSleep;
1232
+ }
1233
+ if (plan2.type === "Error") {
1234
+ const taskActionError = {
1235
+ type: "Action",
1236
+ subType: "Error",
1237
+ param: plan2.param,
1238
+ executor: (taskParam) => __async(this, null, function* () {
1239
+ (0, import_node_assert2.default)(
1240
+ taskParam.thought,
1241
+ "An error occurred, but no thought provided"
1242
+ );
1243
+ throw new Error(taskParam.thought);
1244
+ })
1245
+ };
1246
+ return taskActionError;
1247
+ }
1248
+ throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
1249
+ }).map((task) => {
1250
+ return this.wrapExecutorWithScreenshot(task);
1251
+ });
1252
+ return tasks;
1216
1253
  });
1217
- return tasks;
1218
1254
  }
1219
- async action(userPrompt) {
1220
- const taskExecutor = new import_core.Executor(userPrompt);
1221
- const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
1222
- let plans = [];
1223
- const planningTask = {
1224
- type: "Planning",
1225
- param: {
1226
- userPrompt
1227
- },
1228
- executor: async (param) => {
1229
- const pageContext = await this.insight.contextRetrieverFn();
1230
- let planResult;
1231
- const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
1232
- if (planCache) {
1233
- planResult = planCache;
1234
- } else {
1235
- planResult = await (0, import_core.plan)(param.userPrompt, {
1236
- context: pageContext
1255
+ action(userPrompt) {
1256
+ return __async(this, null, function* () {
1257
+ const taskExecutor = new import_core.Executor(userPrompt);
1258
+ const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
1259
+ let plans = [];
1260
+ const planningTask = {
1261
+ type: "Planning",
1262
+ param: {
1263
+ userPrompt
1264
+ },
1265
+ executor: (param) => __async(this, null, function* () {
1266
+ const pageContext = yield this.insight.contextRetrieverFn();
1267
+ let planResult;
1268
+ const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
1269
+ if (planCache) {
1270
+ planResult = planCache;
1271
+ } else {
1272
+ planResult = yield (0, import_core.plan)(param.userPrompt, {
1273
+ context: pageContext
1274
+ });
1275
+ }
1276
+ (0, import_node_assert2.default)(planResult.plans.length > 0, "No plans found");
1277
+ plans = planResult.plans;
1278
+ cacheGroup.saveCache({
1279
+ type: "plan",
1280
+ pageContext: {
1281
+ url: pageContext.url,
1282
+ size: pageContext.size
1283
+ },
1284
+ prompt: userPrompt,
1285
+ response: planResult
1237
1286
  });
1238
- }
1239
- (0, import_node_assert2.default)(planResult.plans.length > 0, "No plans found");
1240
- plans = planResult.plans;
1241
- cacheGroup.saveCache({
1242
- type: "plan",
1243
- pageContext: {
1244
- url: pageContext.url,
1245
- size: pageContext.size
1246
- },
1247
- prompt: userPrompt,
1248
- response: planResult
1249
- });
1287
+ return {
1288
+ output: planResult,
1289
+ cache: {
1290
+ hit: Boolean(planCache)
1291
+ }
1292
+ };
1293
+ })
1294
+ };
1295
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
1296
+ let output = yield taskExecutor.flush();
1297
+ if (taskExecutor.isInErrorState()) {
1250
1298
  return {
1251
- output: planResult,
1252
- cache: {
1253
- hit: Boolean(planCache)
1254
- }
1299
+ output,
1300
+ executor: taskExecutor
1255
1301
  };
1256
1302
  }
1257
- };
1258
- await taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
1259
- let output = await taskExecutor.flush();
1260
- if (taskExecutor.isInErrorState()) {
1303
+ const executables = yield this.convertPlanToExecutable(plans, cacheGroup);
1304
+ yield taskExecutor.append(executables);
1305
+ output = yield taskExecutor.flush();
1261
1306
  return {
1262
1307
  output,
1263
1308
  executor: taskExecutor
1264
1309
  };
1265
- }
1266
- const executables = await this.convertPlanToExecutable(plans, cacheGroup);
1267
- await taskExecutor.append(executables);
1268
- output = await taskExecutor.flush();
1269
- return {
1270
- output,
1271
- executor: taskExecutor
1272
- };
1273
- }
1274
- async query(demand) {
1275
- const description = typeof demand === "string" ? demand : JSON.stringify(demand);
1276
- const taskExecutor = new import_core.Executor(description);
1277
- const queryTask = {
1278
- type: "Insight",
1279
- subType: "Query",
1280
- param: {
1281
- dataDemand: demand
1282
- },
1283
- executor: async (param) => {
1284
- let insightDump;
1285
- const dumpCollector = (dump) => {
1286
- insightDump = dump;
1287
- };
1288
- this.insight.onceDumpUpdatedFn = dumpCollector;
1289
- const data = await this.insight.extract(param.dataDemand);
1290
- return {
1291
- output: data,
1292
- log: { dump: insightDump }
1293
- };
1294
- }
1295
- };
1296
- await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
1297
- const output = await taskExecutor.flush();
1298
- return {
1299
- output,
1300
- executor: taskExecutor
1301
- };
1310
+ });
1302
1311
  }
1303
- async assert(assertion) {
1304
- const description = `assert: ${assertion}`;
1305
- const taskExecutor = new import_core.Executor(description);
1306
- const assertionPlan = {
1307
- type: "Assert",
1308
- param: {
1309
- assertion
1310
- }
1311
- };
1312
- const assertTask = await this.convertPlanToExecutable([assertionPlan]);
1313
- await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1314
- const output = await taskExecutor.flush();
1315
- return {
1316
- output,
1317
- executor: taskExecutor
1318
- };
1312
+ query(demand) {
1313
+ return __async(this, null, function* () {
1314
+ const description = typeof demand === "string" ? demand : JSON.stringify(demand);
1315
+ const taskExecutor = new import_core.Executor(description);
1316
+ const queryTask = {
1317
+ type: "Insight",
1318
+ subType: "Query",
1319
+ param: {
1320
+ dataDemand: demand
1321
+ },
1322
+ executor: (param) => __async(this, null, function* () {
1323
+ let insightDump;
1324
+ const dumpCollector = (dump) => {
1325
+ insightDump = dump;
1326
+ };
1327
+ this.insight.onceDumpUpdatedFn = dumpCollector;
1328
+ const data = yield this.insight.extract(param.dataDemand);
1329
+ return {
1330
+ output: data,
1331
+ log: { dump: insightDump }
1332
+ };
1333
+ })
1334
+ };
1335
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
1336
+ const output = yield taskExecutor.flush();
1337
+ return {
1338
+ output,
1339
+ executor: taskExecutor
1340
+ };
1341
+ });
1319
1342
  }
1320
- async waitFor(assertion, opt) {
1321
- const description = `waitFor: ${assertion}`;
1322
- const taskExecutor = new import_core.Executor(description);
1323
- const { timeoutMs, checkIntervalMs } = opt;
1324
- (0, import_node_assert2.default)(assertion, "No assertion for waitFor");
1325
- (0, import_node_assert2.default)(timeoutMs, "No timeoutMs for waitFor");
1326
- (0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
1327
- const overallStartTime = Date.now();
1328
- let startTime = Date.now();
1329
- let errorThought = "";
1330
- while (Date.now() - overallStartTime < timeoutMs) {
1331
- startTime = Date.now();
1332
- const assertPlan = {
1333
- type: "AssertWithoutThrow",
1343
+ assert(assertion) {
1344
+ return __async(this, null, function* () {
1345
+ const description = `assert: ${assertion}`;
1346
+ const taskExecutor = new import_core.Executor(description);
1347
+ const assertionPlan = {
1348
+ type: "Assert",
1334
1349
  param: {
1335
1350
  assertion
1336
1351
  }
1337
1352
  };
1338
- const assertTask = await this.convertPlanToExecutable([assertPlan]);
1339
- await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1340
- const output = await taskExecutor.flush();
1341
- if (output == null ? void 0 : output.pass) {
1342
- return {
1343
- output: void 0,
1344
- executor: taskExecutor
1345
- };
1346
- }
1347
- errorThought = (output == null ? void 0 : output.thought) || "unknown error";
1348
- const now = Date.now();
1349
- if (now - startTime < checkIntervalMs) {
1350
- const timeRemaining = checkIntervalMs - (now - startTime);
1351
- const sleepPlan = {
1352
- type: "Sleep",
1353
+ const assertTask = yield this.convertPlanToExecutable([assertionPlan]);
1354
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1355
+ const output = yield taskExecutor.flush();
1356
+ return {
1357
+ output,
1358
+ executor: taskExecutor
1359
+ };
1360
+ });
1361
+ }
1362
+ waitFor(assertion, opt) {
1363
+ return __async(this, null, function* () {
1364
+ const description = `waitFor: ${assertion}`;
1365
+ const taskExecutor = new import_core.Executor(description);
1366
+ const { timeoutMs, checkIntervalMs } = opt;
1367
+ (0, import_node_assert2.default)(assertion, "No assertion for waitFor");
1368
+ (0, import_node_assert2.default)(timeoutMs, "No timeoutMs for waitFor");
1369
+ (0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
1370
+ const overallStartTime = Date.now();
1371
+ let startTime = Date.now();
1372
+ let errorThought = "";
1373
+ while (Date.now() - overallStartTime < timeoutMs) {
1374
+ startTime = Date.now();
1375
+ const assertPlan = {
1376
+ type: "AssertWithoutThrow",
1353
1377
  param: {
1354
- timeMs: timeRemaining
1378
+ assertion
1355
1379
  }
1356
1380
  };
1357
- const sleepTask = await this.convertPlanToExecutable([sleepPlan]);
1358
- await taskExecutor.append(
1359
- this.wrapExecutorWithScreenshot(sleepTask[0])
1360
- );
1361
- await taskExecutor.flush();
1362
- }
1363
- }
1364
- const errorPlan = {
1365
- type: "Error",
1366
- param: {
1367
- thought: `waitFor timeout: ${errorThought}`
1381
+ const assertTask = yield this.convertPlanToExecutable([assertPlan]);
1382
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1383
+ const output = yield taskExecutor.flush();
1384
+ if (output == null ? void 0 : output.pass) {
1385
+ return {
1386
+ output: void 0,
1387
+ executor: taskExecutor
1388
+ };
1389
+ }
1390
+ errorThought = (output == null ? void 0 : output.thought) || "unknown error";
1391
+ const now = Date.now();
1392
+ if (now - startTime < checkIntervalMs) {
1393
+ const timeRemaining = checkIntervalMs - (now - startTime);
1394
+ const sleepPlan = {
1395
+ type: "Sleep",
1396
+ param: {
1397
+ timeMs: timeRemaining
1398
+ }
1399
+ };
1400
+ const sleepTask = yield this.convertPlanToExecutable([sleepPlan]);
1401
+ yield taskExecutor.append(
1402
+ this.wrapExecutorWithScreenshot(sleepTask[0])
1403
+ );
1404
+ yield taskExecutor.flush();
1405
+ }
1368
1406
  }
1369
- };
1370
- const errorTask = await this.convertPlanToExecutable([errorPlan]);
1371
- await taskExecutor.append(errorTask[0]);
1372
- await taskExecutor.flush();
1373
- return {
1374
- output: void 0,
1375
- executor: taskExecutor
1376
- };
1407
+ const errorPlan = {
1408
+ type: "Error",
1409
+ param: {
1410
+ thought: `waitFor timeout: ${errorThought}`
1411
+ }
1412
+ };
1413
+ const errorTask = yield this.convertPlanToExecutable([errorPlan]);
1414
+ yield taskExecutor.append(errorTask[0]);
1415
+ yield taskExecutor.flush();
1416
+ return {
1417
+ output: void 0,
1418
+ executor: taskExecutor
1419
+ };
1420
+ });
1377
1421
  }
1378
1422
  };
1379
1423
 
@@ -1407,83 +1451,94 @@ var PageAgent = class {
1407
1451
  dumpDataString() {
1408
1452
  this.dump.groupName = this.opts.groupName;
1409
1453
  this.dump.groupDescription = this.opts.groupDescription;
1410
- return (0, import_utils5.stringifyDumpData)(this.dump);
1454
+ return (0, import_utils7.stringifyDumpData)(this.dump);
1411
1455
  }
1412
1456
  writeOutActionDumps() {
1413
1457
  const { generateReport, autoPrintReportMsg } = this.opts;
1414
- this.reportFile = (0, import_utils5.writeLogFile)({
1458
+ this.reportFile = (0, import_utils7.writeLogFile)({
1415
1459
  fileName: this.reportFileName,
1416
- fileExt: import_utils5.groupedActionDumpFileExt,
1460
+ fileExt: import_utils7.groupedActionDumpFileExt,
1417
1461
  fileContent: this.dumpDataString(),
1418
1462
  type: "dump",
1419
1463
  generateReport
1420
1464
  });
1421
- if (generateReport && autoPrintReportMsg) {
1465
+ if (generateReport && autoPrintReportMsg && this.reportFile) {
1422
1466
  printReportMsg(this.reportFile);
1423
1467
  }
1424
1468
  }
1425
- async aiAction(taskPrompt) {
1426
- const { executor } = await this.taskExecutor.action(taskPrompt);
1427
- this.appendExecutionDump(executor.dump());
1428
- this.writeOutActionDumps();
1429
- if (executor.isInErrorState()) {
1430
- const errorTask = executor.latestErrorTask();
1431
- throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1469
+ aiAction(taskPrompt) {
1470
+ return __async(this, null, function* () {
1471
+ const { executor } = yield this.taskExecutor.action(taskPrompt);
1472
+ this.appendExecutionDump(executor.dump());
1473
+ this.writeOutActionDumps();
1474
+ if (executor.isInErrorState()) {
1475
+ const errorTask = executor.latestErrorTask();
1476
+ throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1432
1477
  ${errorTask == null ? void 0 : errorTask.errorStack}`);
1433
- }
1478
+ }
1479
+ });
1434
1480
  }
1435
- async aiQuery(demand) {
1436
- const { output, executor } = await this.taskExecutor.query(demand);
1437
- this.appendExecutionDump(executor.dump());
1438
- this.writeOutActionDumps();
1439
- if (executor.isInErrorState()) {
1440
- const errorTask = executor.latestErrorTask();
1441
- throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1481
+ aiQuery(demand) {
1482
+ return __async(this, null, function* () {
1483
+ const { output, executor } = yield this.taskExecutor.query(demand);
1484
+ this.appendExecutionDump(executor.dump());
1485
+ this.writeOutActionDumps();
1486
+ if (executor.isInErrorState()) {
1487
+ const errorTask = executor.latestErrorTask();
1488
+ throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1442
1489
  ${errorTask == null ? void 0 : errorTask.errorStack}`);
1443
- }
1444
- return output;
1445
- }
1446
- async aiAssert(assertion, msg, opt) {
1447
- const { output, executor } = await this.taskExecutor.assert(assertion);
1448
- this.appendExecutionDump(executor.dump());
1449
- this.writeOutActionDumps();
1450
- if (opt == null ? void 0 : opt.keepRawResponse) {
1490
+ }
1451
1491
  return output;
1452
- }
1453
- if (!(output == null ? void 0 : output.pass)) {
1454
- const errMsg = msg || `Assertion failed: ${assertion}`;
1455
- const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
1456
- throw new Error(`${errMsg}
1457
- ${reasonMsg}`);
1458
- }
1492
+ });
1459
1493
  }
1460
- async aiWaitFor(assertion, opt) {
1461
- const { executor } = await this.taskExecutor.waitFor(assertion, {
1462
- timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
1463
- checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
1464
- assertion
1494
+ aiAssert(assertion, msg, opt) {
1495
+ return __async(this, null, function* () {
1496
+ var _a;
1497
+ const { output, executor } = yield this.taskExecutor.assert(assertion);
1498
+ this.appendExecutionDump(executor.dump());
1499
+ this.writeOutActionDumps();
1500
+ if (opt == null ? void 0 : opt.keepRawResponse) {
1501
+ return output;
1502
+ }
1503
+ if (!(output == null ? void 0 : output.pass)) {
1504
+ const errMsg = msg || `Assertion failed: ${assertion}`;
1505
+ const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || ((_a = executor.latestErrorTask()) == null ? void 0 : _a.error) || "(no_reason)"}`;
1506
+ throw new Error(`${errMsg}
1507
+ ${reasonMsg}`);
1508
+ }
1465
1509
  });
1466
- this.appendExecutionDump(executor.dump());
1467
- this.writeOutActionDumps();
1468
- if (executor.isInErrorState()) {
1469
- const errorTask = executor.latestErrorTask();
1470
- throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1510
+ }
1511
+ aiWaitFor(assertion, opt) {
1512
+ return __async(this, null, function* () {
1513
+ const { executor } = yield this.taskExecutor.waitFor(assertion, {
1514
+ timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
1515
+ checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
1516
+ assertion
1517
+ });
1518
+ this.appendExecutionDump(executor.dump());
1519
+ this.writeOutActionDumps();
1520
+ if (executor.isInErrorState()) {
1521
+ const errorTask = executor.latestErrorTask();
1522
+ throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1471
1523
  ${errorTask == null ? void 0 : errorTask.errorStack}`);
1472
- }
1524
+ }
1525
+ });
1473
1526
  }
1474
- async ai(taskPrompt, type = "action") {
1475
- if (type === "action") {
1476
- return this.aiAction(taskPrompt);
1477
- }
1478
- if (type === "query") {
1479
- return this.aiQuery(taskPrompt);
1480
- }
1481
- if (type === "assert") {
1482
- return this.aiAssert(taskPrompt);
1483
- }
1484
- throw new Error(
1485
- `Unknown type: ${type}, only support 'action', 'query', 'assert'`
1486
- );
1527
+ ai(taskPrompt, type = "action") {
1528
+ return __async(this, null, function* () {
1529
+ if (type === "action") {
1530
+ return this.aiAction(taskPrompt);
1531
+ }
1532
+ if (type === "query") {
1533
+ return this.aiQuery(taskPrompt);
1534
+ }
1535
+ if (type === "assert") {
1536
+ return this.aiAssert(taskPrompt);
1537
+ }
1538
+ throw new Error(
1539
+ `Unknown type: ${type}, only support 'action', 'query', 'assert'`
1540
+ );
1541
+ });
1487
1542
  }
1488
1543
  };
1489
1544
 
@@ -1495,8 +1550,6 @@ var StaticPageAgent = class extends PageAgent {
1495
1550
  };
1496
1551
 
1497
1552
  // src/playground/static-page.ts
1498
- var import_utils8 = require("@midscene/core/utils");
1499
- var import_img4 = require("@midscene/shared/img");
1500
1553
  var ThrowNotImplemented = (methodName) => {
1501
1554
  throw new Error(
1502
1555
  `The method "${methodName}" is not implemented as designed since this is a static UI context. (${ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED})`
@@ -1516,38 +1569,52 @@ var StaticPage = class {
1516
1569
  };
1517
1570
  this.uiContext = uiContext;
1518
1571
  }
1519
- async getElementInfos() {
1520
- return ThrowNotImplemented("getElementInfos");
1572
+ getElementInfos() {
1573
+ return __async(this, null, function* () {
1574
+ return ThrowNotImplemented("getElementInfos");
1575
+ });
1521
1576
  }
1522
- async screenshot() {
1523
- const base64 = this.uiContext.screenshotBase64;
1524
- if (!base64) {
1525
- throw new Error("screenshot base64 is empty");
1526
- }
1527
- const tmpFilePath = (0, import_utils8.getTmpFile)("png");
1528
- await (0, import_img4.saveBase64Image)({ base64Data: base64, outputPath: tmpFilePath });
1529
- return tmpFilePath;
1577
+ screenshotBase64() {
1578
+ return __async(this, null, function* () {
1579
+ const base64 = this.uiContext.screenshotBase64;
1580
+ if (!base64) {
1581
+ throw new Error("screenshot base64 is empty");
1582
+ }
1583
+ return base64;
1584
+ });
1530
1585
  }
1531
1586
  url() {
1532
1587
  return this.uiContext.url;
1533
1588
  }
1534
- async scrollUntilTop() {
1535
- return ThrowNotImplemented("scrollUntilTop");
1589
+ scrollUntilTop() {
1590
+ return __async(this, null, function* () {
1591
+ return ThrowNotImplemented("scrollUntilTop");
1592
+ });
1536
1593
  }
1537
- async scrollUntilBottom() {
1538
- return ThrowNotImplemented("scrollUntilBottom");
1594
+ scrollUntilBottom() {
1595
+ return __async(this, null, function* () {
1596
+ return ThrowNotImplemented("scrollUntilBottom");
1597
+ });
1539
1598
  }
1540
- async scrollUpOneScreen() {
1541
- return ThrowNotImplemented("scrollUpOneScreen");
1599
+ scrollUpOneScreen() {
1600
+ return __async(this, null, function* () {
1601
+ return ThrowNotImplemented("scrollUpOneScreen");
1602
+ });
1542
1603
  }
1543
- async scrollDownOneScreen() {
1544
- return ThrowNotImplemented("scrollDownOneScreen");
1604
+ scrollDownOneScreen() {
1605
+ return __async(this, null, function* () {
1606
+ return ThrowNotImplemented("scrollDownOneScreen");
1607
+ });
1545
1608
  }
1546
- async clearInput() {
1547
- return ThrowNotImplemented("clearInput");
1609
+ clearInput() {
1610
+ return __async(this, null, function* () {
1611
+ return ThrowNotImplemented("clearInput");
1612
+ });
1548
1613
  }
1549
- async _forceUsePageContext() {
1550
- return this.uiContext;
1614
+ _forceUsePageContext() {
1615
+ return __async(this, null, function* () {
1616
+ return this.uiContext;
1617
+ });
1551
1618
  }
1552
1619
  };
1553
1620
 
@@ -1561,123 +1628,128 @@ var errorHandler = (err, req, res, next) => {
1561
1628
  error: err.message
1562
1629
  });
1563
1630
  };
1564
- var setup = () => {
1565
- import_dotenv.default.config();
1566
- };
1631
+ var setup = () => __async(void 0, null, function* () {
1632
+ if (!import_utils12.ifInBrowser) {
1633
+ const dotenv = yield Promise.resolve().then(() => __toESM(require_main()));
1634
+ dotenv.config();
1635
+ }
1636
+ });
1567
1637
  var PlaygroundServer = class {
1568
1638
  constructor() {
1569
1639
  this.app = (0, import_express.default)();
1570
- this.tmpDir = (0, import_utils10.getTmpDir)();
1640
+ this.tmpDir = (0, import_utils11.getTmpDir)();
1571
1641
  setup();
1572
1642
  }
1573
- filePathForUuid(uuid) {
1574
- return (0, import_node_path3.join)(this.tmpDir, `${uuid}.json`);
1643
+ filePathForUuid(uuid2) {
1644
+ return (0, import_node_path3.join)(this.tmpDir, `${uuid2}.json`);
1575
1645
  }
1576
- saveContextFile(uuid, context) {
1577
- const tmpFile = this.filePathForUuid(uuid);
1646
+ saveContextFile(uuid2, context) {
1647
+ const tmpFile = this.filePathForUuid(uuid2);
1578
1648
  console.log(`save context file: ${tmpFile}`);
1579
1649
  (0, import_node_fs3.writeFileSync)(tmpFile, context);
1580
1650
  return tmpFile;
1581
1651
  }
1582
- async launch() {
1583
- this.app.use(errorHandler);
1584
- this.app.use(
1585
- (0, import_cors.default)({
1586
- origin: "*",
1587
- credentials: true
1588
- })
1589
- );
1590
- this.app.get("/status", (0, import_cors.default)(), async (req, res) => {
1591
- res.send({
1592
- status: "ok"
1593
- });
1594
- });
1595
- this.app.get("/", (req, res) => {
1596
- res.sendFile((0, import_node_path3.join)(staticPath, "index.html"));
1597
- });
1598
- this.app.get("/playground/:uuid", async (req, res) => {
1599
- res.sendFile((0, import_node_path3.join)(staticPath, "index.html"));
1600
- });
1601
- this.app.get("/context/:uuid", async (req, res) => {
1602
- const { uuid } = req.params;
1603
- const contextFile = this.filePathForUuid(uuid);
1604
- (0, import_node_assert3.default)((0, import_node_fs3.existsSync)(contextFile), "Context not found");
1605
- const context = (0, import_node_fs3.readFileSync)(contextFile, "utf8");
1606
- res.json({
1607
- context
1652
+ launch() {
1653
+ return __async(this, null, function* () {
1654
+ this.app.use(errorHandler);
1655
+ this.app.use(
1656
+ (0, import_cors.default)({
1657
+ origin: "*",
1658
+ credentials: true
1659
+ })
1660
+ );
1661
+ this.app.get("/status", (0, import_cors.default)(), (req, res) => __async(this, null, function* () {
1662
+ res.send({
1663
+ status: "ok"
1664
+ });
1665
+ }));
1666
+ this.app.get("/", (req, res) => {
1667
+ res.sendFile((0, import_node_path3.join)(staticPath, "index.html"));
1608
1668
  });
1609
- });
1610
- this.app.post(
1611
- "/playground-with-context",
1612
- import_express.default.json({ limit: "50mb" }),
1613
- async (req, res) => {
1614
- const context = req.body.context;
1615
- (0, import_node_assert3.default)(context, "context is required");
1616
- const uuid = (0, import_node_crypto2.randomUUID)();
1617
- this.saveContextFile(uuid, context);
1618
- return res.json({
1619
- location: `/playground/${uuid}`,
1620
- uuid
1669
+ this.app.get("/playground/:uuid", (req, res) => __async(this, null, function* () {
1670
+ res.sendFile((0, import_node_path3.join)(staticPath, "index.html"));
1671
+ }));
1672
+ this.app.get("/context/:uuid", (req, res) => __async(this, null, function* () {
1673
+ const { uuid: uuid2 } = req.params;
1674
+ const contextFile = this.filePathForUuid(uuid2);
1675
+ (0, import_node_assert3.default)((0, import_node_fs3.existsSync)(contextFile), "Context not found");
1676
+ const context = (0, import_node_fs3.readFileSync)(contextFile, "utf8");
1677
+ res.json({
1678
+ context
1621
1679
  });
1622
- }
1623
- );
1624
- this.app.post(
1625
- "/execute",
1626
- import_express.default.json({ limit: "30mb" }),
1627
- async (req, res) => {
1628
- const { context, type, prompt } = req.body;
1629
- (0, import_node_assert3.default)(context, "context is required");
1630
- (0, import_node_assert3.default)(type, "type is required");
1631
- (0, import_node_assert3.default)(prompt, "prompt is required");
1632
- const requestId = agentRequestCount++;
1633
- console.log(`handle request: #${requestId}, ${type}, ${prompt}`);
1634
- const page = new StaticPage(context);
1635
- const agent = new StaticPageAgent(page);
1636
- const response = {
1637
- result: null,
1638
- dump: null,
1639
- error: null
1640
- };
1641
- try {
1642
- if (type === "aiQuery") {
1643
- response.result = await agent.aiQuery(prompt);
1644
- } else if (type === "aiAction") {
1645
- response.result = await agent.aiAction(prompt);
1646
- } else if (type === "aiAssert") {
1647
- response.result = await agent.aiAssert(prompt, void 0, {
1648
- keepRawResponse: true
1649
- });
1650
- } else {
1651
- response.error = `Unknown type: ${type}`;
1680
+ }));
1681
+ this.app.post(
1682
+ "/playground-with-context",
1683
+ import_express.default.json({ limit: "50mb" }),
1684
+ (req, res) => __async(this, null, function* () {
1685
+ const context = req.body.context;
1686
+ (0, import_node_assert3.default)(context, "context is required");
1687
+ const uuid2 = (0, import_node_crypto.randomUUID)();
1688
+ this.saveContextFile(uuid2, context);
1689
+ return res.json({
1690
+ location: `/playground/${uuid2}`,
1691
+ uuid: uuid2
1692
+ });
1693
+ })
1694
+ );
1695
+ this.app.post(
1696
+ "/execute",
1697
+ import_express.default.json({ limit: "30mb" }),
1698
+ (req, res) => __async(this, null, function* () {
1699
+ const { context, type, prompt } = req.body;
1700
+ (0, import_node_assert3.default)(context, "context is required");
1701
+ (0, import_node_assert3.default)(type, "type is required");
1702
+ (0, import_node_assert3.default)(prompt, "prompt is required");
1703
+ const requestId = agentRequestCount++;
1704
+ console.log(`handle request: #${requestId}, ${type}, ${prompt}`);
1705
+ const page = new StaticPage(context);
1706
+ const agent = new StaticPageAgent(page);
1707
+ const response = {
1708
+ result: null,
1709
+ dump: null,
1710
+ error: null
1711
+ };
1712
+ try {
1713
+ if (type === "aiQuery") {
1714
+ response.result = yield agent.aiQuery(prompt);
1715
+ } else if (type === "aiAction") {
1716
+ response.result = yield agent.aiAction(prompt);
1717
+ } else if (type === "aiAssert") {
1718
+ response.result = yield agent.aiAssert(prompt, void 0, {
1719
+ keepRawResponse: true
1720
+ });
1721
+ } else {
1722
+ response.error = `Unknown type: ${type}`;
1723
+ }
1724
+ } catch (error) {
1725
+ if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {
1726
+ response.error = error.message;
1727
+ }
1652
1728
  }
1653
- } catch (error) {
1654
- if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {
1655
- response.error = error.message;
1729
+ try {
1730
+ response.dump = JSON.parse(agent.dumpDataString());
1731
+ agent.writeOutActionDumps();
1732
+ } catch (error) {
1733
+ console.error(
1734
+ `write out dump failed: #${requestId}, ${error.message}`
1735
+ );
1656
1736
  }
1657
- }
1658
- try {
1659
- response.dump = JSON.parse(agent.dumpDataString());
1660
- agent.writeOutActionDumps();
1661
- } catch (error) {
1662
- console.error(
1663
- `write out dump failed: #${requestId}, ${error.message}`
1664
- );
1665
- }
1666
- res.send(response);
1667
- if (response.error) {
1668
- console.error(
1669
- `handle request failed: #${requestId}, ${response.error}`
1670
- );
1671
- } else {
1672
- console.log(`handle request done: #${requestId}`);
1673
- }
1674
- }
1675
- );
1676
- return new Promise((resolve, reject) => {
1677
- const port = this.port || defaultPort;
1678
- this.server = this.app.listen(port, () => {
1679
- this.port = port;
1680
- resolve(this);
1737
+ res.send(response);
1738
+ if (response.error) {
1739
+ console.error(
1740
+ `handle request failed: #${requestId}, ${response.error}`
1741
+ );
1742
+ } else {
1743
+ console.log(`handle request done: #${requestId}`);
1744
+ }
1745
+ })
1746
+ );
1747
+ return new Promise((resolve, reject) => {
1748
+ const port = this.port || defaultPort;
1749
+ this.server = this.app.listen(port, () => {
1750
+ this.port = port;
1751
+ resolve(this);
1752
+ });
1681
1753
  });
1682
1754
  });
1683
1755
  }