@midscene/web 0.7.1 → 0.7.2-beta-20241024094141.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -46,6 +46,26 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
46
46
  mod
47
47
  ));
48
48
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
49
+ var __async = (__this, __arguments, generator) => {
50
+ return new Promise((resolve, reject) => {
51
+ var fulfilled = (value) => {
52
+ try {
53
+ step(generator.next(value));
54
+ } catch (e) {
55
+ reject(e);
56
+ }
57
+ };
58
+ var rejected = (value) => {
59
+ try {
60
+ step(generator.throw(value));
61
+ } catch (e) {
62
+ reject(e);
63
+ }
64
+ };
65
+ var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
66
+ step((generator = generator.apply(__this, __arguments)).next());
67
+ });
68
+ };
49
69
 
50
70
  // ../../node_modules/.pnpm/dayjs@1.11.11/node_modules/dayjs/dayjs.min.js
51
71
  var require_dayjs_min = __commonJS({
@@ -352,32 +372,32 @@ __export(playwright_exports, {
352
372
  module.exports = __toCommonJS(playwright_exports);
353
373
 
354
374
  // src/playwright/ai-fixture.ts
355
- var import_node_crypto2 = require("crypto");
375
+ var import_node_crypto = require("crypto");
356
376
 
357
377
  // src/common/agent.ts
358
- var import_utils5 = require("@midscene/core/utils");
378
+ var import_utils7 = require("@midscene/core/utils");
359
379
 
360
380
  // src/common/tasks.ts
361
381
  var import_node_assert2 = __toESM(require("assert"));
362
382
  var import_core = require("@midscene/core");
363
- var import_utils3 = require("@midscene/core/utils");
364
- var import_img3 = require("@midscene/shared/img");
383
+ var import_utils5 = require("@midscene/core/utils");
365
384
 
366
385
  // src/common/task-cache.ts
367
386
  var import_node_fs2 = require("fs");
368
387
  var import_node_path2 = require("path");
369
- var import_utils = require("@midscene/core/utils");
388
+ var import_utils2 = require("@midscene/core/utils");
370
389
  var import_fs2 = require("@midscene/shared/fs");
390
+ var import_utils3 = require("@midscene/shared/utils");
371
391
 
372
392
  // src/common/utils.ts
373
393
  var import_node_assert = __toESM(require("assert"));
374
- var import_node_crypto = require("crypto");
375
394
  var import_node_fs = require("fs");
376
395
  var import_node_path = __toESM(require("path"));
377
396
  var import_constants = require("@midscene/shared/constants");
378
397
  var import_fs = require("@midscene/shared/fs");
379
398
  var import_img = require("@midscene/shared/img");
380
399
  var import_img2 = require("@midscene/shared/img");
400
+ var import_utils = require("@midscene/shared/utils");
381
401
  var import_dayjs = __toESM(require_dayjs_min());
382
402
 
383
403
  // src/web-element.ts
@@ -406,63 +426,68 @@ var WebElementInfo = class {
406
426
  };
407
427
 
408
428
  // src/common/utils.ts
409
- async function parseContextFromWebPage(page, _opt) {
410
- (0, import_node_assert.default)(page, "page is required");
411
- if (page._forceUsePageContext) {
412
- return await page._forceUsePageContext();
413
- }
414
- const url = page.url();
415
- const file = await page.screenshot();
416
- const screenshotBase64 = (0, import_img.base64Encoded)(file);
417
- const captureElementSnapshot = await page.getElementInfos();
418
- const elementsInfo = await alignElements(captureElementSnapshot, page);
419
- const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
420
- if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
421
- return false;
429
+ function parseContextFromWebPage(page, _opt) {
430
+ return __async(this, null, function* () {
431
+ (0, import_node_assert.default)(page, "page is required");
432
+ if (page._forceUsePageContext) {
433
+ return yield page._forceUsePageContext();
422
434
  }
423
- return true;
424
- });
425
- const size = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
426
- const screenshotBase64WithElementMarker = await (0, import_img2.compositeElementInfoImg)({
427
- inputImgBase64: screenshotBase64.split(";base64,").pop(),
428
- elementsPositionInfo: elementsPositionInfoWithoutText
435
+ const url = page.url();
436
+ const screenshotBase64 = yield page.screenshotBase64();
437
+ const captureElementSnapshot = yield page.getElementInfos();
438
+ const elementsInfo = yield alignElements(captureElementSnapshot, page);
439
+ const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
440
+ if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
441
+ return false;
442
+ }
443
+ return true;
444
+ });
445
+ const size = yield (0, import_img.imageInfoOfBase64)(screenshotBase64);
446
+ const screenshotBase64WithElementMarker = yield (0, import_img2.compositeElementInfoImg)({
447
+ inputImgBase64: screenshotBase64,
448
+ elementsPositionInfo: elementsPositionInfoWithoutText
449
+ });
450
+ return {
451
+ content: elementsInfo,
452
+ size,
453
+ screenshotBase64,
454
+ screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
455
+ url
456
+ };
429
457
  });
430
- return {
431
- content: elementsInfo,
432
- size,
433
- screenshotBase64,
434
- screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
435
- url
436
- };
437
458
  }
438
- async function getExtraReturnLogic() {
439
- const pathDir = (0, import_fs.findNearestPackageJson)(__dirname);
440
- (0, import_node_assert.default)(pathDir, `can't find pathDir, with ${__dirname}`);
441
- const scriptPath = import_node_path.default.join(pathDir, "./dist/script/htmlElement.js");
442
- const elementInfosScriptContent = (0, import_node_fs.readFileSync)(scriptPath, "utf-8");
443
- return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
459
+ function getExtraReturnLogic() {
460
+ return __async(this, null, function* () {
461
+ const pathDir = (0, import_fs.findNearestPackageJson)(__dirname);
462
+ (0, import_node_assert.default)(pathDir, `can't find pathDir, with ${__dirname}`);
463
+ const scriptPath = import_node_path.default.join(pathDir, "./dist/script/htmlElement.js");
464
+ const elementInfosScriptContent = (0, import_node_fs.readFileSync)(scriptPath, "utf-8");
465
+ return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
466
+ });
444
467
  }
445
468
  var sizeThreshold = 3;
446
- async function alignElements(elements, page) {
447
- const validElements = elements.filter((item) => {
448
- return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
469
+ function alignElements(elements, page) {
470
+ return __async(this, null, function* () {
471
+ const validElements = elements.filter((item) => {
472
+ return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
473
+ });
474
+ const textsAligned = [];
475
+ for (const item of validElements) {
476
+ const { rect, id, content, attributes, locator, indexId } = item;
477
+ textsAligned.push(
478
+ new WebElementInfo({
479
+ rect,
480
+ locator,
481
+ id,
482
+ content,
483
+ attributes,
484
+ page,
485
+ indexId
486
+ })
487
+ );
488
+ }
489
+ return textsAligned;
449
490
  });
450
- const textsAligned = [];
451
- for (const item of validElements) {
452
- const { rect, id, content, attributes, locator, indexId } = item;
453
- textsAligned.push(
454
- new WebElementInfo({
455
- rect,
456
- locator,
457
- id,
458
- content,
459
- attributes,
460
- page,
461
- indexId
462
- })
463
- );
464
- }
465
- return textsAligned;
466
491
  }
467
492
  function reportFileName(tag = "web") {
468
493
  const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
@@ -493,7 +518,7 @@ var testFileIndex = /* @__PURE__ */ new Map();
493
518
  function generateCacheId(fileName) {
494
519
  let taskFile = fileName || getCurrentExecutionFile();
495
520
  if (!taskFile) {
496
- taskFile = (0, import_node_crypto.randomUUID)();
521
+ taskFile = (0, import_utils.uuid)();
497
522
  console.warn(
498
523
  "Midscene - using random UUID for cache id. Cache may be invalid."
499
524
  );
@@ -512,7 +537,7 @@ function generateCacheId(fileName) {
512
537
  // src/common/task-cache.ts
513
538
  var TaskCache = class {
514
539
  constructor(opts) {
515
- this.midscenePkgInfo = (0, import_fs2.getMidscenePkgInfo)(__dirname);
540
+ this.midscenePkgInfo = (0, import_fs2.getRunningPkgInfo)();
516
541
  this.cacheId = generateCacheId(opts == null ? void 0 : opts.fileName);
517
542
  this.cache = this.readCacheFromFile() || {
518
543
  aiTasks: []
@@ -594,11 +619,17 @@ var TaskCache = class {
594
619
  return this.newCache;
595
620
  }
596
621
  readCacheFromFile() {
597
- const cacheFile = (0, import_node_path2.join)((0, import_utils.getLogDirByType)("cache"), `${this.cacheId}.json`);
622
+ if (import_utils3.ifInBrowser) {
623
+ return void 0;
624
+ }
625
+ const cacheFile = (0, import_node_path2.join)((0, import_utils2.getLogDirByType)("cache"), `${this.cacheId}.json`);
598
626
  if (process.env.MIDSCENE_CACHE === "true" && (0, import_node_fs2.existsSync)(cacheFile)) {
599
627
  try {
600
628
  const data = (0, import_node_fs2.readFileSync)(cacheFile, "utf8");
601
629
  const jsonData = JSON.parse(data);
630
+ if (!this.midscenePkgInfo) {
631
+ return void 0;
632
+ }
602
633
  if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
603
634
  return void 0;
604
635
  }
@@ -610,11 +641,14 @@ var TaskCache = class {
610
641
  return void 0;
611
642
  }
612
643
  writeCacheToFile() {
613
- const midscenePkgInfo = (0, import_fs2.getMidscenePkgInfo)(__dirname);
614
- (0, import_utils.writeLogFile)({
644
+ const midscenePkgInfo = (0, import_fs2.getRunningPkgInfo)();
645
+ if (!midscenePkgInfo) {
646
+ return;
647
+ }
648
+ (0, import_utils2.writeLogFile)({
615
649
  fileName: `${this.cacheId}`,
616
650
  fileExt: "json",
617
- fileContent: (0, import_utils.stringifyDumpData)(
651
+ fileContent: (0, import_utils2.stringifyDumpData)(
618
652
  __spreadValues({
619
653
  pkgName: midscenePkgInfo.name,
620
654
  pkgVersion: midscenePkgInfo.version,
@@ -631,423 +665,435 @@ var TaskCache = class {
631
665
  var PageTaskExecutor = class {
632
666
  constructor(page, opts) {
633
667
  this.page = page;
634
- this.insight = new import_core.Insight(async () => {
635
- return await parseContextFromWebPage(page);
636
- });
668
+ this.insight = new import_core.Insight(() => __async(this, null, function* () {
669
+ return yield parseContextFromWebPage(page);
670
+ }));
637
671
  this.taskCache = new TaskCache({
638
672
  fileName: opts == null ? void 0 : opts.cacheId
639
673
  });
640
674
  }
641
- async recordScreenshot(timing) {
642
- const file = await this.page.screenshot();
643
- const item = {
644
- type: "screenshot",
645
- ts: Date.now(),
646
- screenshot: (0, import_img3.base64Encoded)(file),
647
- timing
648
- };
649
- return item;
675
+ recordScreenshot(timing) {
676
+ return __async(this, null, function* () {
677
+ const base64 = yield this.page.screenshotBase64();
678
+ const item = {
679
+ type: "screenshot",
680
+ ts: Date.now(),
681
+ screenshot: base64,
682
+ timing
683
+ };
684
+ return item;
685
+ });
650
686
  }
651
687
  wrapExecutorWithScreenshot(taskApply) {
652
688
  const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
653
- executor: async (param, context, ...args) => {
689
+ executor: (param, context, ...args) => __async(this, null, function* () {
654
690
  const recorder = [];
655
691
  const { task } = context;
656
692
  task.recorder = recorder;
657
- const shot = await this.recordScreenshot(`before ${task.type}`);
693
+ const shot = yield this.recordScreenshot(`before ${task.type}`);
658
694
  recorder.push(shot);
659
- const result = await taskApply.executor(param, context, ...args);
695
+ const result = yield taskApply.executor(param, context, ...args);
660
696
  if (taskApply.type === "Action") {
661
- await (0, import_utils3.sleep)(1e3);
662
- const shot2 = await this.recordScreenshot("after Action");
697
+ yield (0, import_utils5.sleep)(1e3);
698
+ const shot2 = yield this.recordScreenshot("after Action");
663
699
  recorder.push(shot2);
664
700
  }
665
701
  return result;
666
- }
702
+ })
667
703
  });
668
704
  return taskWithScreenshot;
669
705
  }
670
- async convertPlanToExecutable(plans, cacheGroup) {
671
- const tasks = plans.map((plan2) => {
672
- if (plan2.type === "Locate") {
673
- const taskFind = {
674
- type: "Insight",
675
- subType: "Locate",
676
- param: plan2.param,
677
- executor: async (param, taskContext) => {
678
- const { task } = taskContext;
679
- let insightDump;
680
- const dumpCollector = (dump) => {
681
- insightDump = dump;
682
- };
683
- this.insight.onceDumpUpdatedFn = dumpCollector;
684
- const pageContext = await this.insight.contextRetrieverFn();
685
- const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
686
- pageContext,
687
- "locate",
688
- param.prompt
689
- );
690
- let locateResult;
691
- const callAI = this.insight.aiVendorFn;
692
- const element = await this.insight.locate(param.prompt, {
693
- quickAnswer: plan2.quickAnswer,
694
- callAI: async (...message) => {
695
- if (locateCache) {
696
- locateResult = locateCache;
697
- return Promise.resolve(locateCache);
698
- }
699
- locateResult = await callAI(...message);
700
- (0, import_node_assert2.default)(locateResult);
701
- return locateResult;
702
- }
703
- });
704
- if (locateResult) {
705
- cacheGroup == null ? void 0 : cacheGroup.saveCache({
706
- type: "locate",
707
- pageContext: {
708
- url: pageContext.url,
709
- size: pageContext.size
710
- },
711
- prompt: param.prompt,
712
- response: locateResult
713
- });
714
- }
715
- if (!element) {
716
- task.log = {
717
- dump: insightDump
706
+ convertPlanToExecutable(plans, cacheGroup) {
707
+ return __async(this, null, function* () {
708
+ const tasks = plans.map((plan2) => {
709
+ if (plan2.type === "Locate") {
710
+ const taskFind = {
711
+ type: "Insight",
712
+ subType: "Locate",
713
+ param: plan2.param,
714
+ executor: (param, taskContext) => __async(this, null, function* () {
715
+ const { task } = taskContext;
716
+ let insightDump;
717
+ const dumpCollector = (dump) => {
718
+ insightDump = dump;
718
719
  };
719
- throw new Error(`Element not found: ${param.prompt}`);
720
- }
721
- return {
722
- output: {
723
- element
724
- },
725
- log: {
726
- dump: insightDump
727
- },
728
- cache: {
729
- hit: Boolean(locateCache)
720
+ this.insight.onceDumpUpdatedFn = dumpCollector;
721
+ const pageContext = yield this.insight.contextRetrieverFn();
722
+ const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
723
+ pageContext,
724
+ "locate",
725
+ param.prompt
726
+ );
727
+ let locateResult;
728
+ const callAI = this.insight.aiVendorFn;
729
+ const element = yield this.insight.locate(param.prompt, {
730
+ quickAnswer: plan2.quickAnswer,
731
+ callAI: (...message) => __async(this, null, function* () {
732
+ if (locateCache) {
733
+ locateResult = locateCache;
734
+ return Promise.resolve(locateCache);
735
+ }
736
+ locateResult = yield callAI(...message);
737
+ (0, import_node_assert2.default)(locateResult);
738
+ return locateResult;
739
+ })
740
+ });
741
+ if (locateResult) {
742
+ cacheGroup == null ? void 0 : cacheGroup.saveCache({
743
+ type: "locate",
744
+ pageContext: {
745
+ url: pageContext.url,
746
+ size: pageContext.size
747
+ },
748
+ prompt: param.prompt,
749
+ response: locateResult
750
+ });
730
751
  }
731
- };
732
- }
733
- };
734
- return taskFind;
735
- }
736
- if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
737
- const assertPlan = plan2;
738
- const taskAssert = {
739
- type: "Insight",
740
- subType: "Assert",
741
- param: assertPlan.param,
742
- executor: async (param, taskContext) => {
743
- const { task } = taskContext;
744
- let insightDump;
745
- const dumpCollector = (dump) => {
746
- insightDump = dump;
747
- };
748
- this.insight.onceDumpUpdatedFn = dumpCollector;
749
- const assertion = await this.insight.assert(
750
- assertPlan.param.assertion
751
- );
752
- if (!assertion.pass) {
753
- if (plan2.type === "Assert") {
754
- task.output = assertion;
752
+ if (!element) {
755
753
  task.log = {
756
754
  dump: insightDump
757
755
  };
758
- throw new Error(
759
- assertion.thought || "Assertion failed without reason"
760
- );
756
+ throw new Error(`Element not found: ${param.prompt}`);
761
757
  }
762
- task.error = assertion.thought;
763
- }
764
- return {
765
- output: assertion,
766
- log: {
767
- dump: insightDump
758
+ return {
759
+ output: {
760
+ element
761
+ },
762
+ log: {
763
+ dump: insightDump
764
+ },
765
+ cache: {
766
+ hit: Boolean(locateCache)
767
+ }
768
+ };
769
+ })
770
+ };
771
+ return taskFind;
772
+ }
773
+ if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
774
+ const assertPlan = plan2;
775
+ const taskAssert = {
776
+ type: "Insight",
777
+ subType: "Assert",
778
+ param: assertPlan.param,
779
+ executor: (param, taskContext) => __async(this, null, function* () {
780
+ const { task } = taskContext;
781
+ let insightDump;
782
+ const dumpCollector = (dump) => {
783
+ insightDump = dump;
784
+ };
785
+ this.insight.onceDumpUpdatedFn = dumpCollector;
786
+ const assertion = yield this.insight.assert(
787
+ assertPlan.param.assertion
788
+ );
789
+ if (!assertion.pass) {
790
+ if (plan2.type === "Assert") {
791
+ task.output = assertion;
792
+ task.log = {
793
+ dump: insightDump
794
+ };
795
+ throw new Error(
796
+ assertion.thought || "Assertion failed without reason"
797
+ );
798
+ }
799
+ task.error = assertion.thought;
768
800
  }
769
- };
770
- }
771
- };
772
- return taskAssert;
773
- }
774
- if (plan2.type === "Input") {
775
- const taskActionInput = {
776
- type: "Action",
777
- subType: "Input",
778
- param: plan2.param,
779
- executor: async (taskParam, { element }) => {
780
- if (element) {
781
- await this.page.clearInput(element);
782
- if (taskParam.value === "") {
783
- return;
801
+ return {
802
+ output: assertion,
803
+ log: {
804
+ dump: insightDump
805
+ }
806
+ };
807
+ })
808
+ };
809
+ return taskAssert;
810
+ }
811
+ if (plan2.type === "Input") {
812
+ const taskActionInput = {
813
+ type: "Action",
814
+ subType: "Input",
815
+ param: plan2.param,
816
+ executor: (_0, _1) => __async(this, [_0, _1], function* (taskParam, { element }) {
817
+ if (element) {
818
+ yield this.page.clearInput(element);
819
+ if (taskParam.value === "") {
820
+ return;
821
+ }
822
+ yield this.page.keyboard.type(taskParam.value);
784
823
  }
785
- await this.page.keyboard.type(taskParam.value);
786
- }
787
- }
788
- };
789
- return taskActionInput;
790
- }
791
- if (plan2.type === "KeyboardPress") {
792
- const taskActionKeyboardPress = {
793
- type: "Action",
794
- subType: "KeyboardPress",
795
- param: plan2.param,
796
- executor: async (taskParam) => {
797
- (0, import_node_assert2.default)(taskParam.value, "No key to press");
798
- await this.page.keyboard.press(taskParam.value);
799
- }
800
- };
801
- return taskActionKeyboardPress;
802
- }
803
- if (plan2.type === "Tap") {
804
- const taskActionTap = {
805
- type: "Action",
806
- subType: "Tap",
807
- executor: async (param, { element }) => {
808
- (0, import_node_assert2.default)(element, "Element not found, cannot tap");
809
- await this.page.mouse.click(
810
- element.center[0],
811
- element.center[1]
812
- );
813
- }
814
- };
815
- return taskActionTap;
816
- }
817
- if (plan2.type === "Hover") {
818
- const taskActionHover = {
819
- type: "Action",
820
- subType: "Hover",
821
- executor: async (param, { element }) => {
822
- (0, import_node_assert2.default)(element, "Element not found, cannot hover");
823
- await this.page.mouse.move(
824
- element.center[0],
825
- element.center[1]
826
- );
827
- }
828
- };
829
- return taskActionHover;
830
- }
831
- if (plan2.type === "Scroll") {
832
- const taskActionScroll = {
833
- type: "Action",
834
- subType: "Scroll",
835
- param: plan2.param,
836
- executor: async (taskParam) => {
837
- const scrollToEventName = taskParam.scrollType;
838
- switch (scrollToEventName) {
839
- case "scrollUntilTop":
840
- await this.page.scrollUntilTop();
841
- break;
842
- case "scrollUntilBottom":
843
- await this.page.scrollUntilBottom();
844
- break;
845
- case "scrollUpOneScreen":
846
- await this.page.scrollUpOneScreen();
847
- break;
848
- case "scrollDownOneScreen":
849
- await this.page.scrollDownOneScreen();
850
- break;
851
- default:
852
- console.error(
853
- "Unknown scroll event type:",
854
- scrollToEventName
855
- );
856
- }
857
- }
858
- };
859
- return taskActionScroll;
860
- }
861
- if (plan2.type === "Sleep") {
862
- const taskActionSleep = {
863
- type: "Action",
864
- subType: "Sleep",
865
- param: plan2.param,
866
- executor: async (taskParam) => {
867
- await (0, import_utils3.sleep)(taskParam.timeMs || 3e3);
868
- }
869
- };
870
- return taskActionSleep;
871
- }
872
- if (plan2.type === "Error") {
873
- const taskActionError = {
874
- type: "Action",
875
- subType: "Error",
876
- param: plan2.param,
877
- executor: async (taskParam) => {
878
- (0, import_node_assert2.default)(
879
- taskParam.thought,
880
- "An error occurred, but no thought provided"
881
- );
882
- throw new Error(taskParam.thought);
883
- }
884
- };
885
- return taskActionError;
886
- }
887
- throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
888
- }).map((task) => {
889
- return this.wrapExecutorWithScreenshot(task);
824
+ })
825
+ };
826
+ return taskActionInput;
827
+ }
828
+ if (plan2.type === "KeyboardPress") {
829
+ const taskActionKeyboardPress = {
830
+ type: "Action",
831
+ subType: "KeyboardPress",
832
+ param: plan2.param,
833
+ executor: (taskParam) => __async(this, null, function* () {
834
+ (0, import_node_assert2.default)(taskParam.value, "No key to press");
835
+ yield this.page.keyboard.press(taskParam.value);
836
+ })
837
+ };
838
+ return taskActionKeyboardPress;
839
+ }
840
+ if (plan2.type === "Tap") {
841
+ const taskActionTap = {
842
+ type: "Action",
843
+ subType: "Tap",
844
+ executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
845
+ (0, import_node_assert2.default)(element, "Element not found, cannot tap");
846
+ yield this.page.mouse.click(
847
+ element.center[0],
848
+ element.center[1]
849
+ );
850
+ })
851
+ };
852
+ return taskActionTap;
853
+ }
854
+ if (plan2.type === "Hover") {
855
+ const taskActionHover = {
856
+ type: "Action",
857
+ subType: "Hover",
858
+ executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
859
+ (0, import_node_assert2.default)(element, "Element not found, cannot hover");
860
+ yield this.page.mouse.move(
861
+ element.center[0],
862
+ element.center[1]
863
+ );
864
+ })
865
+ };
866
+ return taskActionHover;
867
+ }
868
+ if (plan2.type === "Scroll") {
869
+ const taskActionScroll = {
870
+ type: "Action",
871
+ subType: "Scroll",
872
+ param: plan2.param,
873
+ executor: (taskParam) => __async(this, null, function* () {
874
+ const scrollToEventName = taskParam.scrollType;
875
+ switch (scrollToEventName) {
876
+ case "scrollUntilTop":
877
+ yield this.page.scrollUntilTop();
878
+ break;
879
+ case "scrollUntilBottom":
880
+ yield this.page.scrollUntilBottom();
881
+ break;
882
+ case "scrollUpOneScreen":
883
+ yield this.page.scrollUpOneScreen();
884
+ break;
885
+ case "scrollDownOneScreen":
886
+ yield this.page.scrollDownOneScreen();
887
+ break;
888
+ default:
889
+ console.error(
890
+ "Unknown scroll event type:",
891
+ scrollToEventName
892
+ );
893
+ }
894
+ })
895
+ };
896
+ return taskActionScroll;
897
+ }
898
+ if (plan2.type === "Sleep") {
899
+ const taskActionSleep = {
900
+ type: "Action",
901
+ subType: "Sleep",
902
+ param: plan2.param,
903
+ executor: (taskParam) => __async(this, null, function* () {
904
+ yield (0, import_utils5.sleep)(taskParam.timeMs || 3e3);
905
+ })
906
+ };
907
+ return taskActionSleep;
908
+ }
909
+ if (plan2.type === "Error") {
910
+ const taskActionError = {
911
+ type: "Action",
912
+ subType: "Error",
913
+ param: plan2.param,
914
+ executor: (taskParam) => __async(this, null, function* () {
915
+ (0, import_node_assert2.default)(
916
+ taskParam.thought,
917
+ "An error occurred, but no thought provided"
918
+ );
919
+ throw new Error(taskParam.thought);
920
+ })
921
+ };
922
+ return taskActionError;
923
+ }
924
+ throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
925
+ }).map((task) => {
926
+ return this.wrapExecutorWithScreenshot(task);
927
+ });
928
+ return tasks;
890
929
  });
891
- return tasks;
892
930
  }
893
- async action(userPrompt) {
894
- const taskExecutor = new import_core.Executor(userPrompt);
895
- const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
896
- let plans = [];
897
- const planningTask = {
898
- type: "Planning",
899
- param: {
900
- userPrompt
901
- },
902
- executor: async (param) => {
903
- const pageContext = await this.insight.contextRetrieverFn();
904
- let planResult;
905
- const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
906
- if (planCache) {
907
- planResult = planCache;
908
- } else {
909
- planResult = await (0, import_core.plan)(param.userPrompt, {
910
- context: pageContext
931
+ action(userPrompt) {
932
+ return __async(this, null, function* () {
933
+ const taskExecutor = new import_core.Executor(userPrompt);
934
+ const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
935
+ let plans = [];
936
+ const planningTask = {
937
+ type: "Planning",
938
+ param: {
939
+ userPrompt
940
+ },
941
+ executor: (param) => __async(this, null, function* () {
942
+ const pageContext = yield this.insight.contextRetrieverFn();
943
+ let planResult;
944
+ const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
945
+ if (planCache) {
946
+ planResult = planCache;
947
+ } else {
948
+ planResult = yield (0, import_core.plan)(param.userPrompt, {
949
+ context: pageContext
950
+ });
951
+ }
952
+ (0, import_node_assert2.default)(planResult.plans.length > 0, "No plans found");
953
+ plans = planResult.plans;
954
+ cacheGroup.saveCache({
955
+ type: "plan",
956
+ pageContext: {
957
+ url: pageContext.url,
958
+ size: pageContext.size
959
+ },
960
+ prompt: userPrompt,
961
+ response: planResult
911
962
  });
912
- }
913
- (0, import_node_assert2.default)(planResult.plans.length > 0, "No plans found");
914
- plans = planResult.plans;
915
- cacheGroup.saveCache({
916
- type: "plan",
917
- pageContext: {
918
- url: pageContext.url,
919
- size: pageContext.size
920
- },
921
- prompt: userPrompt,
922
- response: planResult
923
- });
963
+ return {
964
+ output: planResult,
965
+ cache: {
966
+ hit: Boolean(planCache)
967
+ }
968
+ };
969
+ })
970
+ };
971
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
972
+ let output = yield taskExecutor.flush();
973
+ if (taskExecutor.isInErrorState()) {
924
974
  return {
925
- output: planResult,
926
- cache: {
927
- hit: Boolean(planCache)
928
- }
975
+ output,
976
+ executor: taskExecutor
929
977
  };
930
978
  }
931
- };
932
- await taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
933
- let output = await taskExecutor.flush();
934
- if (taskExecutor.isInErrorState()) {
979
+ const executables = yield this.convertPlanToExecutable(plans, cacheGroup);
980
+ yield taskExecutor.append(executables);
981
+ output = yield taskExecutor.flush();
935
982
  return {
936
983
  output,
937
984
  executor: taskExecutor
938
985
  };
939
- }
940
- const executables = await this.convertPlanToExecutable(plans, cacheGroup);
941
- await taskExecutor.append(executables);
942
- output = await taskExecutor.flush();
943
- return {
944
- output,
945
- executor: taskExecutor
946
- };
947
- }
948
- async query(demand) {
949
- const description = typeof demand === "string" ? demand : JSON.stringify(demand);
950
- const taskExecutor = new import_core.Executor(description);
951
- const queryTask = {
952
- type: "Insight",
953
- subType: "Query",
954
- param: {
955
- dataDemand: demand
956
- },
957
- executor: async (param) => {
958
- let insightDump;
959
- const dumpCollector = (dump) => {
960
- insightDump = dump;
961
- };
962
- this.insight.onceDumpUpdatedFn = dumpCollector;
963
- const data = await this.insight.extract(param.dataDemand);
964
- return {
965
- output: data,
966
- log: { dump: insightDump }
967
- };
968
- }
969
- };
970
- await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
971
- const output = await taskExecutor.flush();
972
- return {
973
- output,
974
- executor: taskExecutor
975
- };
986
+ });
976
987
  }
977
- async assert(assertion) {
978
- const description = `assert: ${assertion}`;
979
- const taskExecutor = new import_core.Executor(description);
980
- const assertionPlan = {
981
- type: "Assert",
982
- param: {
983
- assertion
984
- }
985
- };
986
- const assertTask = await this.convertPlanToExecutable([assertionPlan]);
987
- await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
988
- const output = await taskExecutor.flush();
989
- return {
990
- output,
991
- executor: taskExecutor
992
- };
988
+ query(demand) {
989
+ return __async(this, null, function* () {
990
+ const description = typeof demand === "string" ? demand : JSON.stringify(demand);
991
+ const taskExecutor = new import_core.Executor(description);
992
+ const queryTask = {
993
+ type: "Insight",
994
+ subType: "Query",
995
+ param: {
996
+ dataDemand: demand
997
+ },
998
+ executor: (param) => __async(this, null, function* () {
999
+ let insightDump;
1000
+ const dumpCollector = (dump) => {
1001
+ insightDump = dump;
1002
+ };
1003
+ this.insight.onceDumpUpdatedFn = dumpCollector;
1004
+ const data = yield this.insight.extract(param.dataDemand);
1005
+ return {
1006
+ output: data,
1007
+ log: { dump: insightDump }
1008
+ };
1009
+ })
1010
+ };
1011
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
1012
+ const output = yield taskExecutor.flush();
1013
+ return {
1014
+ output,
1015
+ executor: taskExecutor
1016
+ };
1017
+ });
993
1018
  }
994
- async waitFor(assertion, opt) {
995
- const description = `waitFor: ${assertion}`;
996
- const taskExecutor = new import_core.Executor(description);
997
- const { timeoutMs, checkIntervalMs } = opt;
998
- (0, import_node_assert2.default)(assertion, "No assertion for waitFor");
999
- (0, import_node_assert2.default)(timeoutMs, "No timeoutMs for waitFor");
1000
- (0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
1001
- const overallStartTime = Date.now();
1002
- let startTime = Date.now();
1003
- let errorThought = "";
1004
- while (Date.now() - overallStartTime < timeoutMs) {
1005
- startTime = Date.now();
1006
- const assertPlan = {
1007
- type: "AssertWithoutThrow",
1019
+ assert(assertion) {
1020
+ return __async(this, null, function* () {
1021
+ const description = `assert: ${assertion}`;
1022
+ const taskExecutor = new import_core.Executor(description);
1023
+ const assertionPlan = {
1024
+ type: "Assert",
1008
1025
  param: {
1009
1026
  assertion
1010
1027
  }
1011
1028
  };
1012
- const assertTask = await this.convertPlanToExecutable([assertPlan]);
1013
- await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1014
- const output = await taskExecutor.flush();
1015
- if (output == null ? void 0 : output.pass) {
1016
- return {
1017
- output: void 0,
1018
- executor: taskExecutor
1019
- };
1020
- }
1021
- errorThought = (output == null ? void 0 : output.thought) || "unknown error";
1022
- const now = Date.now();
1023
- if (now - startTime < checkIntervalMs) {
1024
- const timeRemaining = checkIntervalMs - (now - startTime);
1025
- const sleepPlan = {
1026
- type: "Sleep",
1029
+ const assertTask = yield this.convertPlanToExecutable([assertionPlan]);
1030
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1031
+ const output = yield taskExecutor.flush();
1032
+ return {
1033
+ output,
1034
+ executor: taskExecutor
1035
+ };
1036
+ });
1037
+ }
1038
+ waitFor(assertion, opt) {
1039
+ return __async(this, null, function* () {
1040
+ const description = `waitFor: ${assertion}`;
1041
+ const taskExecutor = new import_core.Executor(description);
1042
+ const { timeoutMs, checkIntervalMs } = opt;
1043
+ (0, import_node_assert2.default)(assertion, "No assertion for waitFor");
1044
+ (0, import_node_assert2.default)(timeoutMs, "No timeoutMs for waitFor");
1045
+ (0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
1046
+ const overallStartTime = Date.now();
1047
+ let startTime = Date.now();
1048
+ let errorThought = "";
1049
+ while (Date.now() - overallStartTime < timeoutMs) {
1050
+ startTime = Date.now();
1051
+ const assertPlan = {
1052
+ type: "AssertWithoutThrow",
1027
1053
  param: {
1028
- timeMs: timeRemaining
1054
+ assertion
1029
1055
  }
1030
1056
  };
1031
- const sleepTask = await this.convertPlanToExecutable([sleepPlan]);
1032
- await taskExecutor.append(
1033
- this.wrapExecutorWithScreenshot(sleepTask[0])
1034
- );
1035
- await taskExecutor.flush();
1036
- }
1037
- }
1038
- const errorPlan = {
1039
- type: "Error",
1040
- param: {
1041
- thought: `waitFor timeout: ${errorThought}`
1057
+ const assertTask = yield this.convertPlanToExecutable([assertPlan]);
1058
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1059
+ const output = yield taskExecutor.flush();
1060
+ if (output == null ? void 0 : output.pass) {
1061
+ return {
1062
+ output: void 0,
1063
+ executor: taskExecutor
1064
+ };
1065
+ }
1066
+ errorThought = (output == null ? void 0 : output.thought) || "unknown error";
1067
+ const now = Date.now();
1068
+ if (now - startTime < checkIntervalMs) {
1069
+ const timeRemaining = checkIntervalMs - (now - startTime);
1070
+ const sleepPlan = {
1071
+ type: "Sleep",
1072
+ param: {
1073
+ timeMs: timeRemaining
1074
+ }
1075
+ };
1076
+ const sleepTask = yield this.convertPlanToExecutable([sleepPlan]);
1077
+ yield taskExecutor.append(
1078
+ this.wrapExecutorWithScreenshot(sleepTask[0])
1079
+ );
1080
+ yield taskExecutor.flush();
1081
+ }
1042
1082
  }
1043
- };
1044
- const errorTask = await this.convertPlanToExecutable([errorPlan]);
1045
- await taskExecutor.append(errorTask[0]);
1046
- await taskExecutor.flush();
1047
- return {
1048
- output: void 0,
1049
- executor: taskExecutor
1050
- };
1083
+ const errorPlan = {
1084
+ type: "Error",
1085
+ param: {
1086
+ thought: `waitFor timeout: ${errorThought}`
1087
+ }
1088
+ };
1089
+ const errorTask = yield this.convertPlanToExecutable([errorPlan]);
1090
+ yield taskExecutor.append(errorTask[0]);
1091
+ yield taskExecutor.flush();
1092
+ return {
1093
+ output: void 0,
1094
+ executor: taskExecutor
1095
+ };
1096
+ });
1051
1097
  }
1052
1098
  };
1053
1099
 
@@ -1081,83 +1127,94 @@ var PageAgent = class {
1081
1127
  dumpDataString() {
1082
1128
  this.dump.groupName = this.opts.groupName;
1083
1129
  this.dump.groupDescription = this.opts.groupDescription;
1084
- return (0, import_utils5.stringifyDumpData)(this.dump);
1130
+ return (0, import_utils7.stringifyDumpData)(this.dump);
1085
1131
  }
1086
1132
  writeOutActionDumps() {
1087
1133
  const { generateReport, autoPrintReportMsg } = this.opts;
1088
- this.reportFile = (0, import_utils5.writeLogFile)({
1134
+ this.reportFile = (0, import_utils7.writeLogFile)({
1089
1135
  fileName: this.reportFileName,
1090
- fileExt: import_utils5.groupedActionDumpFileExt,
1136
+ fileExt: import_utils7.groupedActionDumpFileExt,
1091
1137
  fileContent: this.dumpDataString(),
1092
1138
  type: "dump",
1093
1139
  generateReport
1094
1140
  });
1095
- if (generateReport && autoPrintReportMsg) {
1141
+ if (generateReport && autoPrintReportMsg && this.reportFile) {
1096
1142
  printReportMsg(this.reportFile);
1097
1143
  }
1098
1144
  }
1099
- async aiAction(taskPrompt) {
1100
- const { executor } = await this.taskExecutor.action(taskPrompt);
1101
- this.appendExecutionDump(executor.dump());
1102
- this.writeOutActionDumps();
1103
- if (executor.isInErrorState()) {
1104
- const errorTask = executor.latestErrorTask();
1105
- throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1145
+ aiAction(taskPrompt) {
1146
+ return __async(this, null, function* () {
1147
+ const { executor } = yield this.taskExecutor.action(taskPrompt);
1148
+ this.appendExecutionDump(executor.dump());
1149
+ this.writeOutActionDumps();
1150
+ if (executor.isInErrorState()) {
1151
+ const errorTask = executor.latestErrorTask();
1152
+ throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1106
1153
  ${errorTask == null ? void 0 : errorTask.errorStack}`);
1107
- }
1154
+ }
1155
+ });
1108
1156
  }
1109
- async aiQuery(demand) {
1110
- const { output, executor } = await this.taskExecutor.query(demand);
1111
- this.appendExecutionDump(executor.dump());
1112
- this.writeOutActionDumps();
1113
- if (executor.isInErrorState()) {
1114
- const errorTask = executor.latestErrorTask();
1115
- throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1157
+ aiQuery(demand) {
1158
+ return __async(this, null, function* () {
1159
+ const { output, executor } = yield this.taskExecutor.query(demand);
1160
+ this.appendExecutionDump(executor.dump());
1161
+ this.writeOutActionDumps();
1162
+ if (executor.isInErrorState()) {
1163
+ const errorTask = executor.latestErrorTask();
1164
+ throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1116
1165
  ${errorTask == null ? void 0 : errorTask.errorStack}`);
1117
- }
1118
- return output;
1119
- }
1120
- async aiAssert(assertion, msg, opt) {
1121
- const { output, executor } = await this.taskExecutor.assert(assertion);
1122
- this.appendExecutionDump(executor.dump());
1123
- this.writeOutActionDumps();
1124
- if (opt == null ? void 0 : opt.keepRawResponse) {
1166
+ }
1125
1167
  return output;
1126
- }
1127
- if (!(output == null ? void 0 : output.pass)) {
1128
- const errMsg = msg || `Assertion failed: ${assertion}`;
1129
- const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
1130
- throw new Error(`${errMsg}
1131
- ${reasonMsg}`);
1132
- }
1168
+ });
1133
1169
  }
1134
- async aiWaitFor(assertion, opt) {
1135
- const { executor } = await this.taskExecutor.waitFor(assertion, {
1136
- timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
1137
- checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
1138
- assertion
1170
+ aiAssert(assertion, msg, opt) {
1171
+ return __async(this, null, function* () {
1172
+ var _a;
1173
+ const { output, executor } = yield this.taskExecutor.assert(assertion);
1174
+ this.appendExecutionDump(executor.dump());
1175
+ this.writeOutActionDumps();
1176
+ if (opt == null ? void 0 : opt.keepRawResponse) {
1177
+ return output;
1178
+ }
1179
+ if (!(output == null ? void 0 : output.pass)) {
1180
+ const errMsg = msg || `Assertion failed: ${assertion}`;
1181
+ const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || ((_a = executor.latestErrorTask()) == null ? void 0 : _a.error) || "(no_reason)"}`;
1182
+ throw new Error(`${errMsg}
1183
+ ${reasonMsg}`);
1184
+ }
1139
1185
  });
1140
- this.appendExecutionDump(executor.dump());
1141
- this.writeOutActionDumps();
1142
- if (executor.isInErrorState()) {
1143
- const errorTask = executor.latestErrorTask();
1144
- throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1186
+ }
1187
+ aiWaitFor(assertion, opt) {
1188
+ return __async(this, null, function* () {
1189
+ const { executor } = yield this.taskExecutor.waitFor(assertion, {
1190
+ timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
1191
+ checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
1192
+ assertion
1193
+ });
1194
+ this.appendExecutionDump(executor.dump());
1195
+ this.writeOutActionDumps();
1196
+ if (executor.isInErrorState()) {
1197
+ const errorTask = executor.latestErrorTask();
1198
+ throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1145
1199
  ${errorTask == null ? void 0 : errorTask.errorStack}`);
1146
- }
1200
+ }
1201
+ });
1147
1202
  }
1148
- async ai(taskPrompt, type = "action") {
1149
- if (type === "action") {
1150
- return this.aiAction(taskPrompt);
1151
- }
1152
- if (type === "query") {
1153
- return this.aiQuery(taskPrompt);
1154
- }
1155
- if (type === "assert") {
1156
- return this.aiAssert(taskPrompt);
1157
- }
1158
- throw new Error(
1159
- `Unknown type: ${type}, only support 'action', 'query', 'assert'`
1160
- );
1203
+ ai(taskPrompt, type = "action") {
1204
+ return __async(this, null, function* () {
1205
+ if (type === "action") {
1206
+ return this.aiAction(taskPrompt);
1207
+ }
1208
+ if (type === "query") {
1209
+ return this.aiQuery(taskPrompt);
1210
+ }
1211
+ if (type === "assert") {
1212
+ return this.aiAssert(taskPrompt);
1213
+ }
1214
+ throw new Error(
1215
+ `Unknown type: ${type}, only support 'action', 'query', 'assert'`
1216
+ );
1217
+ });
1161
1218
  }
1162
1219
  };
1163
1220
 
@@ -1186,7 +1243,7 @@ var PlaywrightAiFixture = () => {
1186
1243
  const agentForPage = (page, testInfo) => {
1187
1244
  let idForPage = page[midsceneAgentKeyId];
1188
1245
  if (!idForPage) {
1189
- idForPage = (0, import_node_crypto2.randomUUID)();
1246
+ idForPage = (0, import_node_crypto.randomUUID)();
1190
1247
  page[midsceneAgentKeyId] = idForPage;
1191
1248
  const { testId } = testInfo;
1192
1249
  const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
@@ -1215,70 +1272,94 @@ var PlaywrightAiFixture = () => {
1215
1272
  }
1216
1273
  };
1217
1274
  return {
1218
- ai: async ({ page }, use, testInfo) => {
1275
+ ai: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
1219
1276
  const agent = agentForPage(page, testInfo);
1220
- await use(
1221
- async (taskPrompt, opts) => {
1277
+ yield use(
1278
+ (taskPrompt, opts) => __async(void 0, null, function* () {
1222
1279
  return new Promise((resolve, reject) => {
1223
- import_test.test.step(`ai - ${taskPrompt}`, async () => {
1224
- await waitForNetworkIdle(page);
1280
+ import_test.test.step(`ai - ${taskPrompt}`, () => __async(void 0, null, function* () {
1281
+ yield waitForNetworkIdle(page);
1225
1282
  const actionType = (opts == null ? void 0 : opts.type) || "action";
1226
- const result = await agent.ai(taskPrompt, actionType);
1227
- resolve(result);
1228
- });
1283
+ try {
1284
+ const result = yield agent.ai(taskPrompt, actionType);
1285
+ resolve(result);
1286
+ } catch (error) {
1287
+ reject(error);
1288
+ }
1289
+ }));
1229
1290
  });
1230
- }
1291
+ })
1231
1292
  );
1232
1293
  updateDumpAnnotation(testInfo, agent.dumpDataString());
1233
- },
1234
- aiAction: async ({ page }, use, testInfo) => {
1294
+ }),
1295
+ aiAction: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
1235
1296
  const agent = agentForPage(page, testInfo);
1236
- await use(async (taskPrompt) => {
1237
- import_test.test.step(`aiAction - ${taskPrompt}`, async () => {
1238
- await waitForNetworkIdle(page);
1239
- await agent.aiAction(taskPrompt);
1297
+ yield use((taskPrompt) => __async(void 0, null, function* () {
1298
+ return new Promise((resolve, reject) => {
1299
+ import_test.test.step(`aiAction - ${taskPrompt}`, () => __async(void 0, null, function* () {
1300
+ yield waitForNetworkIdle(page);
1301
+ try {
1302
+ const result = yield agent.aiAction(taskPrompt);
1303
+ resolve(result);
1304
+ } catch (error) {
1305
+ reject(error);
1306
+ }
1307
+ }));
1240
1308
  });
1241
- });
1309
+ }));
1242
1310
  updateDumpAnnotation(testInfo, agent.dumpDataString());
1243
- },
1244
- aiQuery: async ({ page }, use, testInfo) => {
1311
+ }),
1312
+ aiQuery: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
1245
1313
  const agent = agentForPage(page, testInfo);
1246
- await use(async (demand) => {
1314
+ yield use((demand) => __async(void 0, null, function* () {
1247
1315
  return new Promise((resolve, reject) => {
1248
- import_test.test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
1249
- await waitForNetworkIdle(page);
1250
- const result = await agent.aiQuery(demand);
1251
- resolve(result);
1252
- });
1316
+ import_test.test.step(`aiQuery - ${JSON.stringify(demand)}`, () => __async(void 0, null, function* () {
1317
+ yield waitForNetworkIdle(page);
1318
+ try {
1319
+ const result = yield agent.aiQuery(demand);
1320
+ resolve(result);
1321
+ } catch (error) {
1322
+ reject(error);
1323
+ }
1324
+ }));
1253
1325
  });
1254
- });
1326
+ }));
1255
1327
  updateDumpAnnotation(testInfo, agent.dumpDataString());
1256
- },
1257
- aiAssert: async ({ page }, use, testInfo) => {
1328
+ }),
1329
+ aiAssert: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
1258
1330
  const agent = agentForPage(page, testInfo);
1259
- await use(async (assertion, errorMsg) => {
1331
+ yield use((assertion, errorMsg) => __async(void 0, null, function* () {
1260
1332
  return new Promise((resolve, reject) => {
1261
- import_test.test.step(`aiAssert - ${assertion}`, async () => {
1262
- await waitForNetworkIdle(page);
1263
- await agent.aiAssert(assertion, errorMsg);
1264
- resolve(null);
1265
- });
1333
+ import_test.test.step(`aiAssert - ${assertion}`, () => __async(void 0, null, function* () {
1334
+ yield waitForNetworkIdle(page);
1335
+ try {
1336
+ yield agent.aiAssert(assertion, errorMsg);
1337
+ resolve(null);
1338
+ } catch (error) {
1339
+ reject(error);
1340
+ }
1341
+ }));
1266
1342
  });
1267
- });
1343
+ }));
1268
1344
  updateDumpAnnotation(testInfo, agent.dumpDataString());
1269
- },
1270
- aiWaitFor: async ({ page }, use, testInfo) => {
1345
+ }),
1346
+ aiWaitFor: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
1271
1347
  const agent = agentForPage(page, testInfo);
1272
- await use(async (assertion, opt) => {
1348
+ yield use((assertion, opt) => __async(void 0, null, function* () {
1273
1349
  return new Promise((resolve, reject) => {
1274
- import_test.test.step(`aiWaitFor - ${assertion}`, async () => {
1275
- await agent.aiWaitFor(assertion, opt);
1276
- resolve(null);
1277
- });
1350
+ import_test.test.step(`aiWaitFor - ${assertion}`, () => __async(void 0, null, function* () {
1351
+ yield waitForNetworkIdle(page);
1352
+ try {
1353
+ yield agent.aiWaitFor(assertion, opt);
1354
+ resolve(null);
1355
+ } catch (error) {
1356
+ reject(error);
1357
+ }
1358
+ }));
1278
1359
  });
1279
- });
1360
+ }));
1280
1361
  updateDumpAnnotation(testInfo, agent.dumpDataString());
1281
- }
1362
+ })
1282
1363
  };
1283
1364
  };
1284
1365
  function waitForNetworkIdle(page) {
@@ -1291,87 +1372,113 @@ function waitForNetworkIdle(page) {
1291
1372
 
1292
1373
  // src/puppeteer/base-page.ts
1293
1374
  var import_node_fs3 = require("fs");
1294
- var import_utils7 = require("@midscene/core/utils");
1295
- var import_img4 = require("@midscene/shared/img");
1375
+ var import_utils9 = require("@midscene/core/utils");
1376
+ var import_img3 = require("@midscene/shared/img");
1296
1377
  var Page = class {
1297
1378
  evaluate(pageFunction, arg) {
1298
1379
  if (this.pageType === "puppeteer") {
1299
- return this.page.evaluate(pageFunction, arg);
1380
+ return this.underlyingPage.evaluate(pageFunction, arg);
1300
1381
  }
1301
- return this.page.evaluate(pageFunction, arg);
1382
+ return this.underlyingPage.evaluate(pageFunction, arg);
1302
1383
  }
1303
- constructor(page, pageType) {
1304
- this.page = page;
1384
+ constructor(underlyingPage, pageType) {
1385
+ this.underlyingPage = underlyingPage;
1305
1386
  this.pageType = pageType;
1306
1387
  }
1307
- async getElementInfos() {
1308
- const scripts = await getExtraReturnLogic();
1309
- const captureElementSnapshot = await this.evaluate(scripts);
1310
- return captureElementSnapshot;
1311
- }
1312
- async screenshot() {
1313
- const viewportSize = await this.evaluate(() => {
1314
- return {
1315
- width: document.documentElement.clientWidth,
1316
- height: document.documentElement.clientHeight,
1317
- deviceScaleFactor: window.devicePixelRatio
1318
- };
1388
+ getElementInfos() {
1389
+ return __async(this, null, function* () {
1390
+ const scripts = yield getExtraReturnLogic();
1391
+ const captureElementSnapshot = yield this.evaluate(scripts);
1392
+ return captureElementSnapshot;
1319
1393
  });
1320
- const path2 = (0, import_utils7.getTmpFile)("png");
1321
- await this.page.screenshot({
1322
- path: path2,
1323
- type: "png"
1324
- });
1325
- let buf;
1326
- if (viewportSize.deviceScaleFactor > 1) {
1327
- buf = await (0, import_img4.resizeImg)((0, import_node_fs3.readFileSync)(path2), {
1328
- width: viewportSize.width,
1329
- height: viewportSize.height
1394
+ }
1395
+ screenshotBase64() {
1396
+ return __async(this, null, function* () {
1397
+ const viewportSize = yield this.evaluate(() => {
1398
+ return {
1399
+ width: document.documentElement.clientWidth,
1400
+ height: document.documentElement.clientHeight,
1401
+ deviceScaleFactor: window.devicePixelRatio
1402
+ };
1330
1403
  });
1331
- (0, import_node_fs3.writeFileSync)(path2, buf);
1332
- }
1333
- return path2;
1404
+ const path2 = (0, import_utils9.getTmpFile)("png");
1405
+ yield this.underlyingPage.screenshot({
1406
+ path: path2,
1407
+ type: "png"
1408
+ });
1409
+ let buf;
1410
+ if (viewportSize.deviceScaleFactor > 1) {
1411
+ buf = yield (0, import_img3.resizeImg)((0, import_node_fs3.readFileSync)(path2), {
1412
+ width: viewportSize.width,
1413
+ height: viewportSize.height
1414
+ });
1415
+ (0, import_node_fs3.writeFileSync)(path2, buf);
1416
+ }
1417
+ return (0, import_img3.base64Encoded)(path2, true);
1418
+ });
1334
1419
  }
1335
1420
  url() {
1336
- return this.page.url();
1421
+ return this.underlyingPage.url();
1337
1422
  }
1338
1423
  get mouse() {
1339
1424
  return {
1340
- click: async (x, y, options) => this.page.mouse.click(x, y, { button: (options == null ? void 0 : options.button) || "left" }),
1341
- wheel: async (deltaX, deltaY) => {
1425
+ click: (x, y, options) => __async(this, null, function* () {
1426
+ return this.underlyingPage.mouse.click(x, y, {
1427
+ button: (options == null ? void 0 : options.button) || "left"
1428
+ });
1429
+ }),
1430
+ wheel: (deltaX, deltaY) => __async(this, null, function* () {
1342
1431
  if (this.pageType === "puppeteer") {
1343
- await this.page.mouse.wheel({ deltaX, deltaY });
1432
+ yield this.underlyingPage.mouse.wheel({
1433
+ deltaX,
1434
+ deltaY
1435
+ });
1344
1436
  } else if (this.pageType === "playwright") {
1345
- await this.page.mouse.wheel(deltaX, deltaY);
1437
+ yield this.underlyingPage.mouse.wheel(
1438
+ deltaX,
1439
+ deltaY
1440
+ );
1346
1441
  }
1347
- },
1348
- move: async (x, y) => this.page.mouse.move(x, y)
1442
+ }),
1443
+ move: (x, y) => __async(this, null, function* () {
1444
+ return this.underlyingPage.mouse.move(x, y);
1445
+ })
1349
1446
  };
1350
1447
  }
1351
1448
  get keyboard() {
1352
1449
  return {
1353
- type: async (text) => this.page.keyboard.type(text),
1354
- press: async (key) => this.page.keyboard.press(key),
1355
- down: async (key) => this.page.keyboard.down(key),
1356
- up: async (key) => this.page.keyboard.up(key)
1450
+ type: (text) => __async(this, null, function* () {
1451
+ return this.underlyingPage.keyboard.type(text);
1452
+ }),
1453
+ press: (key) => __async(this, null, function* () {
1454
+ return this.underlyingPage.keyboard.press(key);
1455
+ }),
1456
+ down: (key) => __async(this, null, function* () {
1457
+ return this.underlyingPage.keyboard.down(key);
1458
+ }),
1459
+ up: (key) => __async(this, null, function* () {
1460
+ return this.underlyingPage.keyboard.up(key);
1461
+ })
1357
1462
  };
1358
1463
  }
1359
- async clearInput(element) {
1360
- if (!element) {
1361
- return;
1362
- }
1363
- await this.mouse.click(element.center[0], element.center[1]);
1364
- const isMac = process.platform === "darwin";
1365
- if (isMac) {
1366
- await this.page.keyboard.down("Meta");
1367
- await this.page.keyboard.press("a");
1368
- await this.page.keyboard.up("Meta");
1369
- } else {
1370
- await this.page.keyboard.down("Control");
1371
- await this.page.keyboard.press("a");
1372
- await this.page.keyboard.up("Control");
1373
- }
1374
- await this.keyboard.press("Backspace");
1464
+ clearInput(element) {
1465
+ return __async(this, null, function* () {
1466
+ if (!element) {
1467
+ return;
1468
+ }
1469
+ yield this.mouse.click(element.center[0], element.center[1]);
1470
+ const isMac = process.platform === "darwin";
1471
+ if (isMac) {
1472
+ yield this.underlyingPage.keyboard.down("Meta");
1473
+ yield this.underlyingPage.keyboard.press("a");
1474
+ yield this.underlyingPage.keyboard.up("Meta");
1475
+ } else {
1476
+ yield this.underlyingPage.keyboard.down("Control");
1477
+ yield this.underlyingPage.keyboard.press("a");
1478
+ yield this.underlyingPage.keyboard.up("Control");
1479
+ }
1480
+ yield this.keyboard.press("Backspace");
1481
+ });
1375
1482
  }
1376
1483
  scrollUntilTop() {
1377
1484
  return this.mouse.wheel(0, -9999999);
@@ -1379,15 +1486,19 @@ var Page = class {
1379
1486
  scrollUntilBottom() {
1380
1487
  return this.mouse.wheel(0, 9999999);
1381
1488
  }
1382
- async scrollUpOneScreen() {
1383
- const innerHeight = await this.evaluate(() => window.innerHeight);
1384
- const distance = innerHeight * 0.7;
1385
- await this.mouse.wheel(0, -distance);
1489
+ scrollUpOneScreen() {
1490
+ return __async(this, null, function* () {
1491
+ const innerHeight = yield this.evaluate(() => window.innerHeight);
1492
+ const distance = innerHeight * 0.7;
1493
+ yield this.mouse.wheel(0, -distance);
1494
+ });
1386
1495
  }
1387
- async scrollDownOneScreen() {
1388
- const innerHeight = await this.evaluate(() => window.innerHeight);
1389
- const distance = innerHeight * 0.7;
1390
- await this.mouse.wheel(0, distance);
1496
+ scrollDownOneScreen() {
1497
+ return __async(this, null, function* () {
1498
+ const innerHeight = yield this.evaluate(() => window.innerHeight);
1499
+ const distance = innerHeight * 0.7;
1500
+ yield this.mouse.wheel(0, distance);
1501
+ });
1391
1502
  }
1392
1503
  };
1393
1504