@midscene/web 0.7.1 → 0.7.2-beta-20241024094141.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -21,14 +21,7 @@ var __spreadValues = (a, b) => {
21
21
  return a;
22
22
  };
23
23
  var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
24
- var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
25
- get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
26
- }) : x)(function(x) {
27
- if (typeof require !== "undefined")
28
- return require.apply(this, arguments);
29
- throw Error('Dynamic require of "' + x + '" is not supported');
30
- });
31
- var __commonJS = (cb, mod) => function __require2() {
24
+ var __commonJS = (cb, mod) => function __require() {
32
25
  return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
33
26
  };
34
27
  var __copyProps = (to, from, except, desc) => {
@@ -47,6 +40,26 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
47
40
  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
48
41
  mod
49
42
  ));
43
+ var __async = (__this, __arguments, generator) => {
44
+ return new Promise((resolve, reject) => {
45
+ var fulfilled = (value) => {
46
+ try {
47
+ step(generator.next(value));
48
+ } catch (e) {
49
+ reject(e);
50
+ }
51
+ };
52
+ var rejected = (value) => {
53
+ try {
54
+ step(generator.throw(value));
55
+ } catch (e) {
56
+ reject(e);
57
+ }
58
+ };
59
+ var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
60
+ step((generator = generator.apply(__this, __arguments)).next());
61
+ });
62
+ };
50
63
 
51
64
  // ../../node_modules/.pnpm/dayjs@1.11.11/node_modules/dayjs/dayjs.min.js
52
65
  var require_dayjs_min = __commonJS({
@@ -343,385 +356,16 @@ var require_dayjs_min = __commonJS({
343
356
  }
344
357
  });
345
358
 
346
- // ../../node_modules/.pnpm/dotenv@16.4.5/node_modules/dotenv/package.json
347
- var require_package = __commonJS({
348
- "../../node_modules/.pnpm/dotenv@16.4.5/node_modules/dotenv/package.json"(exports, module) {
349
- module.exports = {
350
- name: "dotenv",
351
- version: "16.4.5",
352
- description: "Loads environment variables from .env file",
353
- main: "lib/main.js",
354
- types: "lib/main.d.ts",
355
- exports: {
356
- ".": {
357
- types: "./lib/main.d.ts",
358
- require: "./lib/main.js",
359
- default: "./lib/main.js"
360
- },
361
- "./config": "./config.js",
362
- "./config.js": "./config.js",
363
- "./lib/env-options": "./lib/env-options.js",
364
- "./lib/env-options.js": "./lib/env-options.js",
365
- "./lib/cli-options": "./lib/cli-options.js",
366
- "./lib/cli-options.js": "./lib/cli-options.js",
367
- "./package.json": "./package.json"
368
- },
369
- scripts: {
370
- "dts-check": "tsc --project tests/types/tsconfig.json",
371
- lint: "standard",
372
- "lint-readme": "standard-markdown",
373
- pretest: "npm run lint && npm run dts-check",
374
- test: "tap tests/*.js --100 -Rspec",
375
- "test:coverage": "tap --coverage-report=lcov",
376
- prerelease: "npm test",
377
- release: "standard-version"
378
- },
379
- repository: {
380
- type: "git",
381
- url: "git://github.com/motdotla/dotenv.git"
382
- },
383
- funding: "https://dotenvx.com",
384
- keywords: [
385
- "dotenv",
386
- "env",
387
- ".env",
388
- "environment",
389
- "variables",
390
- "config",
391
- "settings"
392
- ],
393
- readmeFilename: "README.md",
394
- license: "BSD-2-Clause",
395
- devDependencies: {
396
- "@definitelytyped/dtslint": "^0.0.133",
397
- "@types/node": "^18.11.3",
398
- decache: "^4.6.1",
399
- sinon: "^14.0.1",
400
- standard: "^17.0.0",
401
- "standard-markdown": "^7.1.0",
402
- "standard-version": "^9.5.0",
403
- tap: "^16.3.0",
404
- tar: "^6.1.11",
405
- typescript: "^4.8.4"
406
- },
407
- engines: {
408
- node: ">=12"
409
- },
410
- browser: {
411
- fs: false
412
- }
413
- };
414
- }
415
- });
416
-
417
- // ../../node_modules/.pnpm/dotenv@16.4.5/node_modules/dotenv/lib/main.js
418
- var require_main = __commonJS({
419
- "../../node_modules/.pnpm/dotenv@16.4.5/node_modules/dotenv/lib/main.js"(exports, module) {
420
- "use strict";
421
- var fs = __require("fs");
422
- var path2 = __require("path");
423
- var os = __require("os");
424
- var crypto = __require("crypto");
425
- var packageJson = require_package();
426
- var version = packageJson.version;
427
- var LINE = /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/mg;
428
- function parse(src) {
429
- const obj = {};
430
- let lines = src.toString();
431
- lines = lines.replace(/\r\n?/mg, "\n");
432
- let match;
433
- while ((match = LINE.exec(lines)) != null) {
434
- const key = match[1];
435
- let value = match[2] || "";
436
- value = value.trim();
437
- const maybeQuote = value[0];
438
- value = value.replace(/^(['"`])([\s\S]*)\1$/mg, "$2");
439
- if (maybeQuote === '"') {
440
- value = value.replace(/\\n/g, "\n");
441
- value = value.replace(/\\r/g, "\r");
442
- }
443
- obj[key] = value;
444
- }
445
- return obj;
446
- }
447
- function _parseVault(options) {
448
- const vaultPath = _vaultPath(options);
449
- const result = DotenvModule.configDotenv({ path: vaultPath });
450
- if (!result.parsed) {
451
- const err = new Error(`MISSING_DATA: Cannot parse ${vaultPath} for an unknown reason`);
452
- err.code = "MISSING_DATA";
453
- throw err;
454
- }
455
- const keys = _dotenvKey(options).split(",");
456
- const length = keys.length;
457
- let decrypted;
458
- for (let i = 0; i < length; i++) {
459
- try {
460
- const key = keys[i].trim();
461
- const attrs = _instructions(result, key);
462
- decrypted = DotenvModule.decrypt(attrs.ciphertext, attrs.key);
463
- break;
464
- } catch (error) {
465
- if (i + 1 >= length) {
466
- throw error;
467
- }
468
- }
469
- }
470
- return DotenvModule.parse(decrypted);
471
- }
472
- function _log(message) {
473
- console.log(`[dotenv@${version}][INFO] ${message}`);
474
- }
475
- function _warn(message) {
476
- console.log(`[dotenv@${version}][WARN] ${message}`);
477
- }
478
- function _debug(message) {
479
- console.log(`[dotenv@${version}][DEBUG] ${message}`);
480
- }
481
- function _dotenvKey(options) {
482
- if (options && options.DOTENV_KEY && options.DOTENV_KEY.length > 0) {
483
- return options.DOTENV_KEY;
484
- }
485
- if (process.env.DOTENV_KEY && process.env.DOTENV_KEY.length > 0) {
486
- return process.env.DOTENV_KEY;
487
- }
488
- return "";
489
- }
490
- function _instructions(result, dotenvKey) {
491
- let uri;
492
- try {
493
- uri = new URL(dotenvKey);
494
- } catch (error) {
495
- if (error.code === "ERR_INVALID_URL") {
496
- const err = new Error("INVALID_DOTENV_KEY: Wrong format. Must be in valid uri format like dotenv://:key_1234@dotenvx.com/vault/.env.vault?environment=development");
497
- err.code = "INVALID_DOTENV_KEY";
498
- throw err;
499
- }
500
- throw error;
501
- }
502
- const key = uri.password;
503
- if (!key) {
504
- const err = new Error("INVALID_DOTENV_KEY: Missing key part");
505
- err.code = "INVALID_DOTENV_KEY";
506
- throw err;
507
- }
508
- const environment = uri.searchParams.get("environment");
509
- if (!environment) {
510
- const err = new Error("INVALID_DOTENV_KEY: Missing environment part");
511
- err.code = "INVALID_DOTENV_KEY";
512
- throw err;
513
- }
514
- const environmentKey = `DOTENV_VAULT_${environment.toUpperCase()}`;
515
- const ciphertext = result.parsed[environmentKey];
516
- if (!ciphertext) {
517
- const err = new Error(`NOT_FOUND_DOTENV_ENVIRONMENT: Cannot locate environment ${environmentKey} in your .env.vault file.`);
518
- err.code = "NOT_FOUND_DOTENV_ENVIRONMENT";
519
- throw err;
520
- }
521
- return { ciphertext, key };
522
- }
523
- function _vaultPath(options) {
524
- let possibleVaultPath = null;
525
- if (options && options.path && options.path.length > 0) {
526
- if (Array.isArray(options.path)) {
527
- for (const filepath of options.path) {
528
- if (fs.existsSync(filepath)) {
529
- possibleVaultPath = filepath.endsWith(".vault") ? filepath : `${filepath}.vault`;
530
- }
531
- }
532
- } else {
533
- possibleVaultPath = options.path.endsWith(".vault") ? options.path : `${options.path}.vault`;
534
- }
535
- } else {
536
- possibleVaultPath = path2.resolve(process.cwd(), ".env.vault");
537
- }
538
- if (fs.existsSync(possibleVaultPath)) {
539
- return possibleVaultPath;
540
- }
541
- return null;
542
- }
543
- function _resolveHome(envPath) {
544
- return envPath[0] === "~" ? path2.join(os.homedir(), envPath.slice(1)) : envPath;
545
- }
546
- function _configVault(options) {
547
- _log("Loading env from encrypted .env.vault");
548
- const parsed = DotenvModule._parseVault(options);
549
- let processEnv = process.env;
550
- if (options && options.processEnv != null) {
551
- processEnv = options.processEnv;
552
- }
553
- DotenvModule.populate(processEnv, parsed, options);
554
- return { parsed };
555
- }
556
- function configDotenv(options) {
557
- const dotenvPath = path2.resolve(process.cwd(), ".env");
558
- let encoding = "utf8";
559
- const debug = Boolean(options && options.debug);
560
- if (options && options.encoding) {
561
- encoding = options.encoding;
562
- } else {
563
- if (debug) {
564
- _debug("No encoding is specified. UTF-8 is used by default");
565
- }
566
- }
567
- let optionPaths = [dotenvPath];
568
- if (options && options.path) {
569
- if (!Array.isArray(options.path)) {
570
- optionPaths = [_resolveHome(options.path)];
571
- } else {
572
- optionPaths = [];
573
- for (const filepath of options.path) {
574
- optionPaths.push(_resolveHome(filepath));
575
- }
576
- }
577
- }
578
- let lastError;
579
- const parsedAll = {};
580
- for (const path3 of optionPaths) {
581
- try {
582
- const parsed = DotenvModule.parse(fs.readFileSync(path3, { encoding }));
583
- DotenvModule.populate(parsedAll, parsed, options);
584
- } catch (e) {
585
- if (debug) {
586
- _debug(`Failed to load ${path3} ${e.message}`);
587
- }
588
- lastError = e;
589
- }
590
- }
591
- let processEnv = process.env;
592
- if (options && options.processEnv != null) {
593
- processEnv = options.processEnv;
594
- }
595
- DotenvModule.populate(processEnv, parsedAll, options);
596
- if (lastError) {
597
- return { parsed: parsedAll, error: lastError };
598
- } else {
599
- return { parsed: parsedAll };
600
- }
601
- }
602
- function config(options) {
603
- if (_dotenvKey(options).length === 0) {
604
- return DotenvModule.configDotenv(options);
605
- }
606
- const vaultPath = _vaultPath(options);
607
- if (!vaultPath) {
608
- _warn(`You set DOTENV_KEY but you are missing a .env.vault file at ${vaultPath}. Did you forget to build it?`);
609
- return DotenvModule.configDotenv(options);
610
- }
611
- return DotenvModule._configVault(options);
612
- }
613
- function decrypt(encrypted, keyStr) {
614
- const key = Buffer.from(keyStr.slice(-64), "hex");
615
- let ciphertext = Buffer.from(encrypted, "base64");
616
- const nonce = ciphertext.subarray(0, 12);
617
- const authTag = ciphertext.subarray(-16);
618
- ciphertext = ciphertext.subarray(12, -16);
619
- try {
620
- const aesgcm = crypto.createDecipheriv("aes-256-gcm", key, nonce);
621
- aesgcm.setAuthTag(authTag);
622
- return `${aesgcm.update(ciphertext)}${aesgcm.final()}`;
623
- } catch (error) {
624
- const isRange = error instanceof RangeError;
625
- const invalidKeyLength = error.message === "Invalid key length";
626
- const decryptionFailed = error.message === "Unsupported state or unable to authenticate data";
627
- if (isRange || invalidKeyLength) {
628
- const err = new Error("INVALID_DOTENV_KEY: It must be 64 characters long (or more)");
629
- err.code = "INVALID_DOTENV_KEY";
630
- throw err;
631
- } else if (decryptionFailed) {
632
- const err = new Error("DECRYPTION_FAILED: Please check your DOTENV_KEY");
633
- err.code = "DECRYPTION_FAILED";
634
- throw err;
635
- } else {
636
- throw error;
637
- }
638
- }
639
- }
640
- function populate(processEnv, parsed, options = {}) {
641
- const debug = Boolean(options && options.debug);
642
- const override = Boolean(options && options.override);
643
- if (typeof parsed !== "object") {
644
- const err = new Error("OBJECT_REQUIRED: Please check the processEnv argument being passed to populate");
645
- err.code = "OBJECT_REQUIRED";
646
- throw err;
647
- }
648
- for (const key of Object.keys(parsed)) {
649
- if (Object.prototype.hasOwnProperty.call(processEnv, key)) {
650
- if (override === true) {
651
- processEnv[key] = parsed[key];
652
- }
653
- if (debug) {
654
- if (override === true) {
655
- _debug(`"${key}" is already defined and WAS overwritten`);
656
- } else {
657
- _debug(`"${key}" is already defined and was NOT overwritten`);
658
- }
659
- }
660
- } else {
661
- processEnv[key] = parsed[key];
662
- }
663
- }
664
- }
665
- var DotenvModule = {
666
- configDotenv,
667
- _configVault,
668
- _parseVault,
669
- config,
670
- decrypt,
671
- parse,
672
- populate
673
- };
674
- module.exports.configDotenv = DotenvModule.configDotenv;
675
- module.exports._configVault = DotenvModule._configVault;
676
- module.exports._parseVault = DotenvModule._parseVault;
677
- module.exports.config = DotenvModule.config;
678
- module.exports.decrypt = DotenvModule.decrypt;
679
- module.exports.parse = DotenvModule.parse;
680
- module.exports.populate = DotenvModule.populate;
681
- module.exports = DotenvModule;
682
- }
683
- });
684
-
685
- // src/common/agent.ts
686
- import {
687
- groupedActionDumpFileExt,
688
- stringifyDumpData as stringifyDumpData2,
689
- writeLogFile as writeLogFile2
690
- } from "@midscene/core/utils";
691
-
692
- // src/common/tasks.ts
693
- import assert2 from "assert";
694
- import {
695
- Executor,
696
- Insight,
697
- plan
698
- } from "@midscene/core";
699
- import { sleep } from "@midscene/core/utils";
700
- import { base64Encoded as base64Encoded2 } from "@midscene/shared/img";
701
-
702
- // src/common/task-cache.ts
703
- import { existsSync, readFileSync as readFileSync2 } from "fs";
704
- import { join } from "path";
705
- import {
706
- getLogDirByType,
707
- stringifyDumpData,
708
- writeLogFile
709
- } from "@midscene/core/utils";
710
- import { getMidscenePkgInfo } from "@midscene/shared/fs";
711
-
712
359
  // src/common/utils.ts
713
360
  var import_dayjs = __toESM(require_dayjs_min());
714
361
  import assert from "assert";
715
- import { randomUUID } from "crypto";
716
362
  import { readFileSync } from "fs";
717
363
  import path from "path";
718
364
  import { NodeType } from "@midscene/shared/constants";
719
365
  import { findNearestPackageJson } from "@midscene/shared/fs";
720
- import {
721
- base64Encoded,
722
- imageInfoOfBase64
723
- } from "@midscene/shared/img";
366
+ import { imageInfoOfBase64 } from "@midscene/shared/img";
724
367
  import { compositeElementInfoImg } from "@midscene/shared/img";
368
+ import { uuid } from "@midscene/shared/utils";
725
369
 
726
370
  // src/web-element.ts
727
371
  var WebElementInfo = class {
@@ -749,56 +393,59 @@ var WebElementInfo = class {
749
393
  };
750
394
 
751
395
  // src/common/utils.ts
752
- async function parseContextFromWebPage(page, _opt) {
753
- assert(page, "page is required");
754
- if (page._forceUsePageContext) {
755
- return await page._forceUsePageContext();
756
- }
757
- const url = page.url();
758
- const file = await page.screenshot();
759
- const screenshotBase64 = base64Encoded(file);
760
- const captureElementSnapshot = await page.getElementInfos();
761
- const elementsInfo = await alignElements(captureElementSnapshot, page);
762
- const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
763
- if (elementInfo.attributes.nodeType === NodeType.TEXT) {
764
- return false;
396
+ function parseContextFromWebPage(page, _opt) {
397
+ return __async(this, null, function* () {
398
+ assert(page, "page is required");
399
+ if (page._forceUsePageContext) {
400
+ return yield page._forceUsePageContext();
765
401
  }
766
- return true;
767
- });
768
- const size = await imageInfoOfBase64(screenshotBase64);
769
- const screenshotBase64WithElementMarker = await compositeElementInfoImg({
770
- inputImgBase64: screenshotBase64.split(";base64,").pop(),
771
- elementsPositionInfo: elementsPositionInfoWithoutText
402
+ const url = page.url();
403
+ const screenshotBase64 = yield page.screenshotBase64();
404
+ const captureElementSnapshot = yield page.getElementInfos();
405
+ const elementsInfo = yield alignElements(captureElementSnapshot, page);
406
+ const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
407
+ if (elementInfo.attributes.nodeType === NodeType.TEXT) {
408
+ return false;
409
+ }
410
+ return true;
411
+ });
412
+ const size = yield imageInfoOfBase64(screenshotBase64);
413
+ const screenshotBase64WithElementMarker = yield compositeElementInfoImg({
414
+ inputImgBase64: screenshotBase64,
415
+ elementsPositionInfo: elementsPositionInfoWithoutText
416
+ });
417
+ return {
418
+ content: elementsInfo,
419
+ size,
420
+ screenshotBase64,
421
+ screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
422
+ url
423
+ };
772
424
  });
773
- return {
774
- content: elementsInfo,
775
- size,
776
- screenshotBase64,
777
- screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
778
- url
779
- };
780
425
  }
781
426
  var sizeThreshold = 3;
782
- async function alignElements(elements, page) {
783
- const validElements = elements.filter((item) => {
784
- return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
427
+ function alignElements(elements, page) {
428
+ return __async(this, null, function* () {
429
+ const validElements = elements.filter((item) => {
430
+ return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
431
+ });
432
+ const textsAligned = [];
433
+ for (const item of validElements) {
434
+ const { rect, id, content, attributes, locator, indexId } = item;
435
+ textsAligned.push(
436
+ new WebElementInfo({
437
+ rect,
438
+ locator,
439
+ id,
440
+ content,
441
+ attributes,
442
+ page,
443
+ indexId
444
+ })
445
+ );
446
+ }
447
+ return textsAligned;
785
448
  });
786
- const textsAligned = [];
787
- for (const item of validElements) {
788
- const { rect, id, content, attributes, locator, indexId } = item;
789
- textsAligned.push(
790
- new WebElementInfo({
791
- rect,
792
- locator,
793
- id,
794
- content,
795
- attributes,
796
- page,
797
- indexId
798
- })
799
- );
800
- }
801
- return textsAligned;
802
449
  }
803
450
  function reportFileName(tag = "web") {
804
451
  const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
@@ -829,7 +476,7 @@ var testFileIndex = /* @__PURE__ */ new Map();
829
476
  function generateCacheId(fileName) {
830
477
  let taskFile = fileName || getCurrentExecutionFile();
831
478
  if (!taskFile) {
832
- taskFile = randomUUID();
479
+ taskFile = uuid();
833
480
  console.warn(
834
481
  "Midscene - using random UUID for cache id. Cache may be invalid."
835
482
  );
@@ -846,10 +493,35 @@ function generateCacheId(fileName) {
846
493
  }
847
494
  var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
848
495
 
496
+ // src/common/agent.ts
497
+ import {
498
+ groupedActionDumpFileExt,
499
+ stringifyDumpData as stringifyDumpData2,
500
+ writeLogFile as writeLogFile2
501
+ } from "@midscene/core/utils";
502
+
503
+ // src/common/tasks.ts
504
+ import assert2 from "assert";
505
+ import {
506
+ Executor,
507
+ Insight,
508
+ plan
509
+ } from "@midscene/core";
510
+ import { sleep } from "@midscene/core/utils";
511
+
849
512
  // src/common/task-cache.ts
513
+ import { existsSync, readFileSync as readFileSync2 } from "fs";
514
+ import { join } from "path";
515
+ import {
516
+ getLogDirByType,
517
+ stringifyDumpData,
518
+ writeLogFile
519
+ } from "@midscene/core/utils";
520
+ import { getRunningPkgInfo } from "@midscene/shared/fs";
521
+ import { ifInBrowser } from "@midscene/shared/utils";
850
522
  var TaskCache = class {
851
523
  constructor(opts) {
852
- this.midscenePkgInfo = getMidscenePkgInfo(__dirname);
524
+ this.midscenePkgInfo = getRunningPkgInfo();
853
525
  this.cacheId = generateCacheId(opts == null ? void 0 : opts.fileName);
854
526
  this.cache = this.readCacheFromFile() || {
855
527
  aiTasks: []
@@ -931,11 +603,17 @@ var TaskCache = class {
931
603
  return this.newCache;
932
604
  }
933
605
  readCacheFromFile() {
606
+ if (ifInBrowser) {
607
+ return void 0;
608
+ }
934
609
  const cacheFile = join(getLogDirByType("cache"), `${this.cacheId}.json`);
935
610
  if (process.env.MIDSCENE_CACHE === "true" && existsSync(cacheFile)) {
936
611
  try {
937
612
  const data = readFileSync2(cacheFile, "utf8");
938
613
  const jsonData = JSON.parse(data);
614
+ if (!this.midscenePkgInfo) {
615
+ return void 0;
616
+ }
939
617
  if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
940
618
  return void 0;
941
619
  }
@@ -947,7 +625,10 @@ var TaskCache = class {
947
625
  return void 0;
948
626
  }
949
627
  writeCacheToFile() {
950
- const midscenePkgInfo = getMidscenePkgInfo(__dirname);
628
+ const midscenePkgInfo = getRunningPkgInfo();
629
+ if (!midscenePkgInfo) {
630
+ return;
631
+ }
951
632
  writeLogFile({
952
633
  fileName: `${this.cacheId}`,
953
634
  fileExt: "json",
@@ -968,423 +649,435 @@ var TaskCache = class {
968
649
  var PageTaskExecutor = class {
969
650
  constructor(page, opts) {
970
651
  this.page = page;
971
- this.insight = new Insight(async () => {
972
- return await parseContextFromWebPage(page);
973
- });
652
+ this.insight = new Insight(() => __async(this, null, function* () {
653
+ return yield parseContextFromWebPage(page);
654
+ }));
974
655
  this.taskCache = new TaskCache({
975
656
  fileName: opts == null ? void 0 : opts.cacheId
976
657
  });
977
658
  }
978
- async recordScreenshot(timing) {
979
- const file = await this.page.screenshot();
980
- const item = {
981
- type: "screenshot",
982
- ts: Date.now(),
983
- screenshot: base64Encoded2(file),
984
- timing
985
- };
986
- return item;
659
+ recordScreenshot(timing) {
660
+ return __async(this, null, function* () {
661
+ const base64 = yield this.page.screenshotBase64();
662
+ const item = {
663
+ type: "screenshot",
664
+ ts: Date.now(),
665
+ screenshot: base64,
666
+ timing
667
+ };
668
+ return item;
669
+ });
987
670
  }
988
671
  wrapExecutorWithScreenshot(taskApply) {
989
672
  const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
990
- executor: async (param, context, ...args) => {
673
+ executor: (param, context, ...args) => __async(this, null, function* () {
991
674
  const recorder = [];
992
675
  const { task } = context;
993
676
  task.recorder = recorder;
994
- const shot = await this.recordScreenshot(`before ${task.type}`);
677
+ const shot = yield this.recordScreenshot(`before ${task.type}`);
995
678
  recorder.push(shot);
996
- const result = await taskApply.executor(param, context, ...args);
679
+ const result = yield taskApply.executor(param, context, ...args);
997
680
  if (taskApply.type === "Action") {
998
- await sleep(1e3);
999
- const shot2 = await this.recordScreenshot("after Action");
681
+ yield sleep(1e3);
682
+ const shot2 = yield this.recordScreenshot("after Action");
1000
683
  recorder.push(shot2);
1001
684
  }
1002
685
  return result;
1003
- }
686
+ })
1004
687
  });
1005
688
  return taskWithScreenshot;
1006
689
  }
1007
- async convertPlanToExecutable(plans, cacheGroup) {
1008
- const tasks = plans.map((plan2) => {
1009
- if (plan2.type === "Locate") {
1010
- const taskFind = {
1011
- type: "Insight",
1012
- subType: "Locate",
1013
- param: plan2.param,
1014
- executor: async (param, taskContext) => {
1015
- const { task } = taskContext;
1016
- let insightDump;
1017
- const dumpCollector = (dump) => {
1018
- insightDump = dump;
1019
- };
1020
- this.insight.onceDumpUpdatedFn = dumpCollector;
1021
- const pageContext = await this.insight.contextRetrieverFn();
1022
- const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
1023
- pageContext,
1024
- "locate",
1025
- param.prompt
1026
- );
1027
- let locateResult;
1028
- const callAI = this.insight.aiVendorFn;
1029
- const element = await this.insight.locate(param.prompt, {
1030
- quickAnswer: plan2.quickAnswer,
1031
- callAI: async (...message) => {
1032
- if (locateCache) {
1033
- locateResult = locateCache;
1034
- return Promise.resolve(locateCache);
1035
- }
1036
- locateResult = await callAI(...message);
1037
- assert2(locateResult);
1038
- return locateResult;
1039
- }
1040
- });
1041
- if (locateResult) {
1042
- cacheGroup == null ? void 0 : cacheGroup.saveCache({
1043
- type: "locate",
1044
- pageContext: {
1045
- url: pageContext.url,
1046
- size: pageContext.size
1047
- },
1048
- prompt: param.prompt,
1049
- response: locateResult
1050
- });
1051
- }
1052
- if (!element) {
1053
- task.log = {
1054
- dump: insightDump
690
+ convertPlanToExecutable(plans, cacheGroup) {
691
+ return __async(this, null, function* () {
692
+ const tasks = plans.map((plan2) => {
693
+ if (plan2.type === "Locate") {
694
+ const taskFind = {
695
+ type: "Insight",
696
+ subType: "Locate",
697
+ param: plan2.param,
698
+ executor: (param, taskContext) => __async(this, null, function* () {
699
+ const { task } = taskContext;
700
+ let insightDump;
701
+ const dumpCollector = (dump) => {
702
+ insightDump = dump;
1055
703
  };
1056
- throw new Error(`Element not found: ${param.prompt}`);
1057
- }
1058
- return {
1059
- output: {
1060
- element
1061
- },
1062
- log: {
1063
- dump: insightDump
1064
- },
1065
- cache: {
1066
- hit: Boolean(locateCache)
704
+ this.insight.onceDumpUpdatedFn = dumpCollector;
705
+ const pageContext = yield this.insight.contextRetrieverFn();
706
+ const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
707
+ pageContext,
708
+ "locate",
709
+ param.prompt
710
+ );
711
+ let locateResult;
712
+ const callAI = this.insight.aiVendorFn;
713
+ const element = yield this.insight.locate(param.prompt, {
714
+ quickAnswer: plan2.quickAnswer,
715
+ callAI: (...message) => __async(this, null, function* () {
716
+ if (locateCache) {
717
+ locateResult = locateCache;
718
+ return Promise.resolve(locateCache);
719
+ }
720
+ locateResult = yield callAI(...message);
721
+ assert2(locateResult);
722
+ return locateResult;
723
+ })
724
+ });
725
+ if (locateResult) {
726
+ cacheGroup == null ? void 0 : cacheGroup.saveCache({
727
+ type: "locate",
728
+ pageContext: {
729
+ url: pageContext.url,
730
+ size: pageContext.size
731
+ },
732
+ prompt: param.prompt,
733
+ response: locateResult
734
+ });
1067
735
  }
1068
- };
1069
- }
1070
- };
1071
- return taskFind;
1072
- }
1073
- if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
1074
- const assertPlan = plan2;
1075
- const taskAssert = {
1076
- type: "Insight",
1077
- subType: "Assert",
1078
- param: assertPlan.param,
1079
- executor: async (param, taskContext) => {
1080
- const { task } = taskContext;
1081
- let insightDump;
1082
- const dumpCollector = (dump) => {
1083
- insightDump = dump;
1084
- };
1085
- this.insight.onceDumpUpdatedFn = dumpCollector;
1086
- const assertion = await this.insight.assert(
1087
- assertPlan.param.assertion
1088
- );
1089
- if (!assertion.pass) {
1090
- if (plan2.type === "Assert") {
1091
- task.output = assertion;
736
+ if (!element) {
1092
737
  task.log = {
1093
738
  dump: insightDump
1094
739
  };
1095
- throw new Error(
1096
- assertion.thought || "Assertion failed without reason"
1097
- );
740
+ throw new Error(`Element not found: ${param.prompt}`);
1098
741
  }
1099
- task.error = assertion.thought;
1100
- }
1101
- return {
1102
- output: assertion,
1103
- log: {
1104
- dump: insightDump
742
+ return {
743
+ output: {
744
+ element
745
+ },
746
+ log: {
747
+ dump: insightDump
748
+ },
749
+ cache: {
750
+ hit: Boolean(locateCache)
751
+ }
752
+ };
753
+ })
754
+ };
755
+ return taskFind;
756
+ }
757
+ if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
758
+ const assertPlan = plan2;
759
+ const taskAssert = {
760
+ type: "Insight",
761
+ subType: "Assert",
762
+ param: assertPlan.param,
763
+ executor: (param, taskContext) => __async(this, null, function* () {
764
+ const { task } = taskContext;
765
+ let insightDump;
766
+ const dumpCollector = (dump) => {
767
+ insightDump = dump;
768
+ };
769
+ this.insight.onceDumpUpdatedFn = dumpCollector;
770
+ const assertion = yield this.insight.assert(
771
+ assertPlan.param.assertion
772
+ );
773
+ if (!assertion.pass) {
774
+ if (plan2.type === "Assert") {
775
+ task.output = assertion;
776
+ task.log = {
777
+ dump: insightDump
778
+ };
779
+ throw new Error(
780
+ assertion.thought || "Assertion failed without reason"
781
+ );
782
+ }
783
+ task.error = assertion.thought;
1105
784
  }
1106
- };
1107
- }
1108
- };
1109
- return taskAssert;
1110
- }
1111
- if (plan2.type === "Input") {
1112
- const taskActionInput = {
1113
- type: "Action",
1114
- subType: "Input",
1115
- param: plan2.param,
1116
- executor: async (taskParam, { element }) => {
1117
- if (element) {
1118
- await this.page.clearInput(element);
1119
- if (taskParam.value === "") {
1120
- return;
785
+ return {
786
+ output: assertion,
787
+ log: {
788
+ dump: insightDump
789
+ }
790
+ };
791
+ })
792
+ };
793
+ return taskAssert;
794
+ }
795
+ if (plan2.type === "Input") {
796
+ const taskActionInput = {
797
+ type: "Action",
798
+ subType: "Input",
799
+ param: plan2.param,
800
+ executor: (_0, _1) => __async(this, [_0, _1], function* (taskParam, { element }) {
801
+ if (element) {
802
+ yield this.page.clearInput(element);
803
+ if (taskParam.value === "") {
804
+ return;
805
+ }
806
+ yield this.page.keyboard.type(taskParam.value);
1121
807
  }
1122
- await this.page.keyboard.type(taskParam.value);
1123
- }
1124
- }
1125
- };
1126
- return taskActionInput;
1127
- }
1128
- if (plan2.type === "KeyboardPress") {
1129
- const taskActionKeyboardPress = {
1130
- type: "Action",
1131
- subType: "KeyboardPress",
1132
- param: plan2.param,
1133
- executor: async (taskParam) => {
1134
- assert2(taskParam.value, "No key to press");
1135
- await this.page.keyboard.press(taskParam.value);
1136
- }
1137
- };
1138
- return taskActionKeyboardPress;
1139
- }
1140
- if (plan2.type === "Tap") {
1141
- const taskActionTap = {
1142
- type: "Action",
1143
- subType: "Tap",
1144
- executor: async (param, { element }) => {
1145
- assert2(element, "Element not found, cannot tap");
1146
- await this.page.mouse.click(
1147
- element.center[0],
1148
- element.center[1]
1149
- );
1150
- }
1151
- };
1152
- return taskActionTap;
1153
- }
1154
- if (plan2.type === "Hover") {
1155
- const taskActionHover = {
1156
- type: "Action",
1157
- subType: "Hover",
1158
- executor: async (param, { element }) => {
1159
- assert2(element, "Element not found, cannot hover");
1160
- await this.page.mouse.move(
1161
- element.center[0],
1162
- element.center[1]
1163
- );
1164
- }
1165
- };
1166
- return taskActionHover;
1167
- }
1168
- if (plan2.type === "Scroll") {
1169
- const taskActionScroll = {
1170
- type: "Action",
1171
- subType: "Scroll",
1172
- param: plan2.param,
1173
- executor: async (taskParam) => {
1174
- const scrollToEventName = taskParam.scrollType;
1175
- switch (scrollToEventName) {
1176
- case "scrollUntilTop":
1177
- await this.page.scrollUntilTop();
1178
- break;
1179
- case "scrollUntilBottom":
1180
- await this.page.scrollUntilBottom();
1181
- break;
1182
- case "scrollUpOneScreen":
1183
- await this.page.scrollUpOneScreen();
1184
- break;
1185
- case "scrollDownOneScreen":
1186
- await this.page.scrollDownOneScreen();
1187
- break;
1188
- default:
1189
- console.error(
1190
- "Unknown scroll event type:",
1191
- scrollToEventName
1192
- );
1193
- }
1194
- }
1195
- };
1196
- return taskActionScroll;
1197
- }
1198
- if (plan2.type === "Sleep") {
1199
- const taskActionSleep = {
1200
- type: "Action",
1201
- subType: "Sleep",
1202
- param: plan2.param,
1203
- executor: async (taskParam) => {
1204
- await sleep(taskParam.timeMs || 3e3);
1205
- }
1206
- };
1207
- return taskActionSleep;
1208
- }
1209
- if (plan2.type === "Error") {
1210
- const taskActionError = {
1211
- type: "Action",
1212
- subType: "Error",
1213
- param: plan2.param,
1214
- executor: async (taskParam) => {
1215
- assert2(
1216
- taskParam.thought,
1217
- "An error occurred, but no thought provided"
1218
- );
1219
- throw new Error(taskParam.thought);
1220
- }
1221
- };
1222
- return taskActionError;
1223
- }
1224
- throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
1225
- }).map((task) => {
1226
- return this.wrapExecutorWithScreenshot(task);
808
+ })
809
+ };
810
+ return taskActionInput;
811
+ }
812
+ if (plan2.type === "KeyboardPress") {
813
+ const taskActionKeyboardPress = {
814
+ type: "Action",
815
+ subType: "KeyboardPress",
816
+ param: plan2.param,
817
+ executor: (taskParam) => __async(this, null, function* () {
818
+ assert2(taskParam.value, "No key to press");
819
+ yield this.page.keyboard.press(taskParam.value);
820
+ })
821
+ };
822
+ return taskActionKeyboardPress;
823
+ }
824
+ if (plan2.type === "Tap") {
825
+ const taskActionTap = {
826
+ type: "Action",
827
+ subType: "Tap",
828
+ executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
829
+ assert2(element, "Element not found, cannot tap");
830
+ yield this.page.mouse.click(
831
+ element.center[0],
832
+ element.center[1]
833
+ );
834
+ })
835
+ };
836
+ return taskActionTap;
837
+ }
838
+ if (plan2.type === "Hover") {
839
+ const taskActionHover = {
840
+ type: "Action",
841
+ subType: "Hover",
842
+ executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
843
+ assert2(element, "Element not found, cannot hover");
844
+ yield this.page.mouse.move(
845
+ element.center[0],
846
+ element.center[1]
847
+ );
848
+ })
849
+ };
850
+ return taskActionHover;
851
+ }
852
+ if (plan2.type === "Scroll") {
853
+ const taskActionScroll = {
854
+ type: "Action",
855
+ subType: "Scroll",
856
+ param: plan2.param,
857
+ executor: (taskParam) => __async(this, null, function* () {
858
+ const scrollToEventName = taskParam.scrollType;
859
+ switch (scrollToEventName) {
860
+ case "scrollUntilTop":
861
+ yield this.page.scrollUntilTop();
862
+ break;
863
+ case "scrollUntilBottom":
864
+ yield this.page.scrollUntilBottom();
865
+ break;
866
+ case "scrollUpOneScreen":
867
+ yield this.page.scrollUpOneScreen();
868
+ break;
869
+ case "scrollDownOneScreen":
870
+ yield this.page.scrollDownOneScreen();
871
+ break;
872
+ default:
873
+ console.error(
874
+ "Unknown scroll event type:",
875
+ scrollToEventName
876
+ );
877
+ }
878
+ })
879
+ };
880
+ return taskActionScroll;
881
+ }
882
+ if (plan2.type === "Sleep") {
883
+ const taskActionSleep = {
884
+ type: "Action",
885
+ subType: "Sleep",
886
+ param: plan2.param,
887
+ executor: (taskParam) => __async(this, null, function* () {
888
+ yield sleep(taskParam.timeMs || 3e3);
889
+ })
890
+ };
891
+ return taskActionSleep;
892
+ }
893
+ if (plan2.type === "Error") {
894
+ const taskActionError = {
895
+ type: "Action",
896
+ subType: "Error",
897
+ param: plan2.param,
898
+ executor: (taskParam) => __async(this, null, function* () {
899
+ assert2(
900
+ taskParam.thought,
901
+ "An error occurred, but no thought provided"
902
+ );
903
+ throw new Error(taskParam.thought);
904
+ })
905
+ };
906
+ return taskActionError;
907
+ }
908
+ throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
909
+ }).map((task) => {
910
+ return this.wrapExecutorWithScreenshot(task);
911
+ });
912
+ return tasks;
1227
913
  });
1228
- return tasks;
1229
914
  }
1230
- async action(userPrompt) {
1231
- const taskExecutor = new Executor(userPrompt);
1232
- const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
1233
- let plans = [];
1234
- const planningTask = {
1235
- type: "Planning",
1236
- param: {
1237
- userPrompt
1238
- },
1239
- executor: async (param) => {
1240
- const pageContext = await this.insight.contextRetrieverFn();
1241
- let planResult;
1242
- const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
1243
- if (planCache) {
1244
- planResult = planCache;
1245
- } else {
1246
- planResult = await plan(param.userPrompt, {
1247
- context: pageContext
915
+ action(userPrompt) {
916
+ return __async(this, null, function* () {
917
+ const taskExecutor = new Executor(userPrompt);
918
+ const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
919
+ let plans = [];
920
+ const planningTask = {
921
+ type: "Planning",
922
+ param: {
923
+ userPrompt
924
+ },
925
+ executor: (param) => __async(this, null, function* () {
926
+ const pageContext = yield this.insight.contextRetrieverFn();
927
+ let planResult;
928
+ const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
929
+ if (planCache) {
930
+ planResult = planCache;
931
+ } else {
932
+ planResult = yield plan(param.userPrompt, {
933
+ context: pageContext
934
+ });
935
+ }
936
+ assert2(planResult.plans.length > 0, "No plans found");
937
+ plans = planResult.plans;
938
+ cacheGroup.saveCache({
939
+ type: "plan",
940
+ pageContext: {
941
+ url: pageContext.url,
942
+ size: pageContext.size
943
+ },
944
+ prompt: userPrompt,
945
+ response: planResult
1248
946
  });
1249
- }
1250
- assert2(planResult.plans.length > 0, "No plans found");
1251
- plans = planResult.plans;
1252
- cacheGroup.saveCache({
1253
- type: "plan",
1254
- pageContext: {
1255
- url: pageContext.url,
1256
- size: pageContext.size
1257
- },
1258
- prompt: userPrompt,
1259
- response: planResult
1260
- });
947
+ return {
948
+ output: planResult,
949
+ cache: {
950
+ hit: Boolean(planCache)
951
+ }
952
+ };
953
+ })
954
+ };
955
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
956
+ let output = yield taskExecutor.flush();
957
+ if (taskExecutor.isInErrorState()) {
1261
958
  return {
1262
- output: planResult,
1263
- cache: {
1264
- hit: Boolean(planCache)
1265
- }
959
+ output,
960
+ executor: taskExecutor
1266
961
  };
1267
962
  }
1268
- };
1269
- await taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
1270
- let output = await taskExecutor.flush();
1271
- if (taskExecutor.isInErrorState()) {
963
+ const executables = yield this.convertPlanToExecutable(plans, cacheGroup);
964
+ yield taskExecutor.append(executables);
965
+ output = yield taskExecutor.flush();
1272
966
  return {
1273
967
  output,
1274
968
  executor: taskExecutor
1275
969
  };
1276
- }
1277
- const executables = await this.convertPlanToExecutable(plans, cacheGroup);
1278
- await taskExecutor.append(executables);
1279
- output = await taskExecutor.flush();
1280
- return {
1281
- output,
1282
- executor: taskExecutor
1283
- };
1284
- }
1285
- async query(demand) {
1286
- const description = typeof demand === "string" ? demand : JSON.stringify(demand);
1287
- const taskExecutor = new Executor(description);
1288
- const queryTask = {
1289
- type: "Insight",
1290
- subType: "Query",
1291
- param: {
1292
- dataDemand: demand
1293
- },
1294
- executor: async (param) => {
1295
- let insightDump;
1296
- const dumpCollector = (dump) => {
1297
- insightDump = dump;
1298
- };
1299
- this.insight.onceDumpUpdatedFn = dumpCollector;
1300
- const data = await this.insight.extract(param.dataDemand);
1301
- return {
1302
- output: data,
1303
- log: { dump: insightDump }
1304
- };
1305
- }
1306
- };
1307
- await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
1308
- const output = await taskExecutor.flush();
1309
- return {
1310
- output,
1311
- executor: taskExecutor
1312
- };
970
+ });
1313
971
  }
1314
- async assert(assertion) {
1315
- const description = `assert: ${assertion}`;
1316
- const taskExecutor = new Executor(description);
1317
- const assertionPlan = {
1318
- type: "Assert",
1319
- param: {
1320
- assertion
1321
- }
1322
- };
1323
- const assertTask = await this.convertPlanToExecutable([assertionPlan]);
1324
- await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1325
- const output = await taskExecutor.flush();
1326
- return {
1327
- output,
1328
- executor: taskExecutor
1329
- };
972
+ query(demand) {
973
+ return __async(this, null, function* () {
974
+ const description = typeof demand === "string" ? demand : JSON.stringify(demand);
975
+ const taskExecutor = new Executor(description);
976
+ const queryTask = {
977
+ type: "Insight",
978
+ subType: "Query",
979
+ param: {
980
+ dataDemand: demand
981
+ },
982
+ executor: (param) => __async(this, null, function* () {
983
+ let insightDump;
984
+ const dumpCollector = (dump) => {
985
+ insightDump = dump;
986
+ };
987
+ this.insight.onceDumpUpdatedFn = dumpCollector;
988
+ const data = yield this.insight.extract(param.dataDemand);
989
+ return {
990
+ output: data,
991
+ log: { dump: insightDump }
992
+ };
993
+ })
994
+ };
995
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
996
+ const output = yield taskExecutor.flush();
997
+ return {
998
+ output,
999
+ executor: taskExecutor
1000
+ };
1001
+ });
1330
1002
  }
1331
- async waitFor(assertion, opt) {
1332
- const description = `waitFor: ${assertion}`;
1333
- const taskExecutor = new Executor(description);
1334
- const { timeoutMs, checkIntervalMs } = opt;
1335
- assert2(assertion, "No assertion for waitFor");
1336
- assert2(timeoutMs, "No timeoutMs for waitFor");
1337
- assert2(checkIntervalMs, "No checkIntervalMs for waitFor");
1338
- const overallStartTime = Date.now();
1339
- let startTime = Date.now();
1340
- let errorThought = "";
1341
- while (Date.now() - overallStartTime < timeoutMs) {
1342
- startTime = Date.now();
1343
- const assertPlan = {
1344
- type: "AssertWithoutThrow",
1003
+ assert(assertion) {
1004
+ return __async(this, null, function* () {
1005
+ const description = `assert: ${assertion}`;
1006
+ const taskExecutor = new Executor(description);
1007
+ const assertionPlan = {
1008
+ type: "Assert",
1345
1009
  param: {
1346
1010
  assertion
1347
1011
  }
1348
1012
  };
1349
- const assertTask = await this.convertPlanToExecutable([assertPlan]);
1350
- await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1351
- const output = await taskExecutor.flush();
1352
- if (output == null ? void 0 : output.pass) {
1353
- return {
1354
- output: void 0,
1355
- executor: taskExecutor
1356
- };
1357
- }
1358
- errorThought = (output == null ? void 0 : output.thought) || "unknown error";
1359
- const now = Date.now();
1360
- if (now - startTime < checkIntervalMs) {
1361
- const timeRemaining = checkIntervalMs - (now - startTime);
1362
- const sleepPlan = {
1363
- type: "Sleep",
1013
+ const assertTask = yield this.convertPlanToExecutable([assertionPlan]);
1014
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1015
+ const output = yield taskExecutor.flush();
1016
+ return {
1017
+ output,
1018
+ executor: taskExecutor
1019
+ };
1020
+ });
1021
+ }
1022
+ waitFor(assertion, opt) {
1023
+ return __async(this, null, function* () {
1024
+ const description = `waitFor: ${assertion}`;
1025
+ const taskExecutor = new Executor(description);
1026
+ const { timeoutMs, checkIntervalMs } = opt;
1027
+ assert2(assertion, "No assertion for waitFor");
1028
+ assert2(timeoutMs, "No timeoutMs for waitFor");
1029
+ assert2(checkIntervalMs, "No checkIntervalMs for waitFor");
1030
+ const overallStartTime = Date.now();
1031
+ let startTime = Date.now();
1032
+ let errorThought = "";
1033
+ while (Date.now() - overallStartTime < timeoutMs) {
1034
+ startTime = Date.now();
1035
+ const assertPlan = {
1036
+ type: "AssertWithoutThrow",
1364
1037
  param: {
1365
- timeMs: timeRemaining
1038
+ assertion
1366
1039
  }
1367
1040
  };
1368
- const sleepTask = await this.convertPlanToExecutable([sleepPlan]);
1369
- await taskExecutor.append(
1370
- this.wrapExecutorWithScreenshot(sleepTask[0])
1371
- );
1372
- await taskExecutor.flush();
1373
- }
1374
- }
1375
- const errorPlan = {
1376
- type: "Error",
1377
- param: {
1378
- thought: `waitFor timeout: ${errorThought}`
1041
+ const assertTask = yield this.convertPlanToExecutable([assertPlan]);
1042
+ yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
1043
+ const output = yield taskExecutor.flush();
1044
+ if (output == null ? void 0 : output.pass) {
1045
+ return {
1046
+ output: void 0,
1047
+ executor: taskExecutor
1048
+ };
1049
+ }
1050
+ errorThought = (output == null ? void 0 : output.thought) || "unknown error";
1051
+ const now = Date.now();
1052
+ if (now - startTime < checkIntervalMs) {
1053
+ const timeRemaining = checkIntervalMs - (now - startTime);
1054
+ const sleepPlan = {
1055
+ type: "Sleep",
1056
+ param: {
1057
+ timeMs: timeRemaining
1058
+ }
1059
+ };
1060
+ const sleepTask = yield this.convertPlanToExecutable([sleepPlan]);
1061
+ yield taskExecutor.append(
1062
+ this.wrapExecutorWithScreenshot(sleepTask[0])
1063
+ );
1064
+ yield taskExecutor.flush();
1065
+ }
1379
1066
  }
1380
- };
1381
- const errorTask = await this.convertPlanToExecutable([errorPlan]);
1382
- await taskExecutor.append(errorTask[0]);
1383
- await taskExecutor.flush();
1384
- return {
1385
- output: void 0,
1386
- executor: taskExecutor
1387
- };
1067
+ const errorPlan = {
1068
+ type: "Error",
1069
+ param: {
1070
+ thought: `waitFor timeout: ${errorThought}`
1071
+ }
1072
+ };
1073
+ const errorTask = yield this.convertPlanToExecutable([errorPlan]);
1074
+ yield taskExecutor.append(errorTask[0]);
1075
+ yield taskExecutor.flush();
1076
+ return {
1077
+ output: void 0,
1078
+ executor: taskExecutor
1079
+ };
1080
+ });
1388
1081
  }
1389
1082
  };
1390
1083
 
@@ -1429,72 +1122,83 @@ var PageAgent = class {
1429
1122
  type: "dump",
1430
1123
  generateReport
1431
1124
  });
1432
- if (generateReport && autoPrintReportMsg) {
1125
+ if (generateReport && autoPrintReportMsg && this.reportFile) {
1433
1126
  printReportMsg(this.reportFile);
1434
1127
  }
1435
1128
  }
1436
- async aiAction(taskPrompt) {
1437
- const { executor } = await this.taskExecutor.action(taskPrompt);
1438
- this.appendExecutionDump(executor.dump());
1439
- this.writeOutActionDumps();
1440
- if (executor.isInErrorState()) {
1441
- const errorTask = executor.latestErrorTask();
1442
- throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1129
+ aiAction(taskPrompt) {
1130
+ return __async(this, null, function* () {
1131
+ const { executor } = yield this.taskExecutor.action(taskPrompt);
1132
+ this.appendExecutionDump(executor.dump());
1133
+ this.writeOutActionDumps();
1134
+ if (executor.isInErrorState()) {
1135
+ const errorTask = executor.latestErrorTask();
1136
+ throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1443
1137
  ${errorTask == null ? void 0 : errorTask.errorStack}`);
1444
- }
1138
+ }
1139
+ });
1445
1140
  }
1446
- async aiQuery(demand) {
1447
- const { output, executor } = await this.taskExecutor.query(demand);
1448
- this.appendExecutionDump(executor.dump());
1449
- this.writeOutActionDumps();
1450
- if (executor.isInErrorState()) {
1451
- const errorTask = executor.latestErrorTask();
1452
- throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1141
+ aiQuery(demand) {
1142
+ return __async(this, null, function* () {
1143
+ const { output, executor } = yield this.taskExecutor.query(demand);
1144
+ this.appendExecutionDump(executor.dump());
1145
+ this.writeOutActionDumps();
1146
+ if (executor.isInErrorState()) {
1147
+ const errorTask = executor.latestErrorTask();
1148
+ throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1453
1149
  ${errorTask == null ? void 0 : errorTask.errorStack}`);
1454
- }
1455
- return output;
1456
- }
1457
- async aiAssert(assertion, msg, opt) {
1458
- const { output, executor } = await this.taskExecutor.assert(assertion);
1459
- this.appendExecutionDump(executor.dump());
1460
- this.writeOutActionDumps();
1461
- if (opt == null ? void 0 : opt.keepRawResponse) {
1150
+ }
1462
1151
  return output;
1463
- }
1464
- if (!(output == null ? void 0 : output.pass)) {
1465
- const errMsg = msg || `Assertion failed: ${assertion}`;
1466
- const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
1467
- throw new Error(`${errMsg}
1468
- ${reasonMsg}`);
1469
- }
1152
+ });
1470
1153
  }
1471
- async aiWaitFor(assertion, opt) {
1472
- const { executor } = await this.taskExecutor.waitFor(assertion, {
1473
- timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
1474
- checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
1475
- assertion
1154
+ aiAssert(assertion, msg, opt) {
1155
+ return __async(this, null, function* () {
1156
+ var _a;
1157
+ const { output, executor } = yield this.taskExecutor.assert(assertion);
1158
+ this.appendExecutionDump(executor.dump());
1159
+ this.writeOutActionDumps();
1160
+ if (opt == null ? void 0 : opt.keepRawResponse) {
1161
+ return output;
1162
+ }
1163
+ if (!(output == null ? void 0 : output.pass)) {
1164
+ const errMsg = msg || `Assertion failed: ${assertion}`;
1165
+ const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || ((_a = executor.latestErrorTask()) == null ? void 0 : _a.error) || "(no_reason)"}`;
1166
+ throw new Error(`${errMsg}
1167
+ ${reasonMsg}`);
1168
+ }
1476
1169
  });
1477
- this.appendExecutionDump(executor.dump());
1478
- this.writeOutActionDumps();
1479
- if (executor.isInErrorState()) {
1480
- const errorTask = executor.latestErrorTask();
1481
- throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1170
+ }
1171
+ aiWaitFor(assertion, opt) {
1172
+ return __async(this, null, function* () {
1173
+ const { executor } = yield this.taskExecutor.waitFor(assertion, {
1174
+ timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
1175
+ checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
1176
+ assertion
1177
+ });
1178
+ this.appendExecutionDump(executor.dump());
1179
+ this.writeOutActionDumps();
1180
+ if (executor.isInErrorState()) {
1181
+ const errorTask = executor.latestErrorTask();
1182
+ throw new Error(`${errorTask == null ? void 0 : errorTask.error}
1482
1183
  ${errorTask == null ? void 0 : errorTask.errorStack}`);
1483
- }
1184
+ }
1185
+ });
1484
1186
  }
1485
- async ai(taskPrompt, type = "action") {
1486
- if (type === "action") {
1487
- return this.aiAction(taskPrompt);
1488
- }
1489
- if (type === "query") {
1490
- return this.aiQuery(taskPrompt);
1491
- }
1492
- if (type === "assert") {
1493
- return this.aiAssert(taskPrompt);
1494
- }
1495
- throw new Error(
1496
- `Unknown type: ${type}, only support 'action', 'query', 'assert'`
1497
- );
1187
+ ai(taskPrompt, type = "action") {
1188
+ return __async(this, null, function* () {
1189
+ if (type === "action") {
1190
+ return this.aiAction(taskPrompt);
1191
+ }
1192
+ if (type === "query") {
1193
+ return this.aiQuery(taskPrompt);
1194
+ }
1195
+ if (type === "assert") {
1196
+ return this.aiAssert(taskPrompt);
1197
+ }
1198
+ throw new Error(
1199
+ `Unknown type: ${type}, only support 'action', 'query', 'assert'`
1200
+ );
1201
+ });
1498
1202
  }
1499
1203
  };
1500
1204
 
@@ -1505,19 +1209,7 @@ var StaticPageAgent = class extends PageAgent {
1505
1209
  }
1506
1210
  };
1507
1211
 
1508
- // src/playground/server.ts
1509
- import assert3 from "assert";
1510
- import { randomUUID as randomUUID2 } from "crypto";
1511
- import { existsSync as existsSync2, readFileSync as readFileSync3, writeFileSync } from "fs";
1512
- import { join as join2 } from "path";
1513
- var import_dotenv = __toESM(require_main());
1514
- import { getTmpDir } from "@midscene/core/utils";
1515
- import cors from "cors";
1516
- import express from "express";
1517
-
1518
1212
  // src/playground/static-page.ts
1519
- import { getTmpFile } from "@midscene/core/utils";
1520
- import { saveBase64Image } from "@midscene/shared/img";
1521
1213
  var ThrowNotImplemented = (methodName) => {
1522
1214
  throw new Error(
1523
1215
  `The method "${methodName}" is not implemented as designed since this is a static UI context. (${ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED})`
@@ -1537,179 +1229,56 @@ var StaticPage = class {
1537
1229
  };
1538
1230
  this.uiContext = uiContext;
1539
1231
  }
1540
- async getElementInfos() {
1541
- return ThrowNotImplemented("getElementInfos");
1232
+ getElementInfos() {
1233
+ return __async(this, null, function* () {
1234
+ return ThrowNotImplemented("getElementInfos");
1235
+ });
1542
1236
  }
1543
- async screenshot() {
1544
- const base64 = this.uiContext.screenshotBase64;
1545
- if (!base64) {
1546
- throw new Error("screenshot base64 is empty");
1547
- }
1548
- const tmpFilePath = getTmpFile("png");
1549
- await saveBase64Image({ base64Data: base64, outputPath: tmpFilePath });
1550
- return tmpFilePath;
1237
+ screenshotBase64() {
1238
+ return __async(this, null, function* () {
1239
+ const base64 = this.uiContext.screenshotBase64;
1240
+ if (!base64) {
1241
+ throw new Error("screenshot base64 is empty");
1242
+ }
1243
+ return base64;
1244
+ });
1551
1245
  }
1552
1246
  url() {
1553
1247
  return this.uiContext.url;
1554
1248
  }
1555
- async scrollUntilTop() {
1556
- return ThrowNotImplemented("scrollUntilTop");
1557
- }
1558
- async scrollUntilBottom() {
1559
- return ThrowNotImplemented("scrollUntilBottom");
1560
- }
1561
- async scrollUpOneScreen() {
1562
- return ThrowNotImplemented("scrollUpOneScreen");
1563
- }
1564
- async scrollDownOneScreen() {
1565
- return ThrowNotImplemented("scrollDownOneScreen");
1566
- }
1567
- async clearInput() {
1568
- return ThrowNotImplemented("clearInput");
1569
- }
1570
- async _forceUsePageContext() {
1571
- return this.uiContext;
1572
- }
1573
- };
1574
-
1575
- // src/playground/server.ts
1576
- var defaultPort = 5800;
1577
- var staticPath = join2(__dirname, "../../static");
1578
- var agentRequestCount = 1;
1579
- var errorHandler = (err, req, res, next) => {
1580
- console.error(err);
1581
- res.status(500).json({
1582
- error: err.message
1583
- });
1584
- };
1585
- var setup = () => {
1586
- import_dotenv.default.config();
1587
- };
1588
- var PlaygroundServer = class {
1589
- constructor() {
1590
- this.app = express();
1591
- this.tmpDir = getTmpDir();
1592
- setup();
1593
- }
1594
- filePathForUuid(uuid) {
1595
- return join2(this.tmpDir, `${uuid}.json`);
1596
- }
1597
- saveContextFile(uuid, context) {
1598
- const tmpFile = this.filePathForUuid(uuid);
1599
- console.log(`save context file: ${tmpFile}`);
1600
- writeFileSync(tmpFile, context);
1601
- return tmpFile;
1602
- }
1603
- async launch() {
1604
- this.app.use(errorHandler);
1605
- this.app.use(
1606
- cors({
1607
- origin: "*",
1608
- credentials: true
1609
- })
1610
- );
1611
- this.app.get("/status", cors(), async (req, res) => {
1612
- res.send({
1613
- status: "ok"
1614
- });
1249
+ scrollUntilTop() {
1250
+ return __async(this, null, function* () {
1251
+ return ThrowNotImplemented("scrollUntilTop");
1615
1252
  });
1616
- this.app.get("/", (req, res) => {
1617
- res.sendFile(join2(staticPath, "index.html"));
1253
+ }
1254
+ scrollUntilBottom() {
1255
+ return __async(this, null, function* () {
1256
+ return ThrowNotImplemented("scrollUntilBottom");
1618
1257
  });
1619
- this.app.get("/playground/:uuid", async (req, res) => {
1620
- res.sendFile(join2(staticPath, "index.html"));
1258
+ }
1259
+ scrollUpOneScreen() {
1260
+ return __async(this, null, function* () {
1261
+ return ThrowNotImplemented("scrollUpOneScreen");
1621
1262
  });
1622
- this.app.get("/context/:uuid", async (req, res) => {
1623
- const { uuid } = req.params;
1624
- const contextFile = this.filePathForUuid(uuid);
1625
- assert3(existsSync2(contextFile), "Context not found");
1626
- const context = readFileSync3(contextFile, "utf8");
1627
- res.json({
1628
- context
1629
- });
1263
+ }
1264
+ scrollDownOneScreen() {
1265
+ return __async(this, null, function* () {
1266
+ return ThrowNotImplemented("scrollDownOneScreen");
1630
1267
  });
1631
- this.app.post(
1632
- "/playground-with-context",
1633
- express.json({ limit: "50mb" }),
1634
- async (req, res) => {
1635
- const context = req.body.context;
1636
- assert3(context, "context is required");
1637
- const uuid = randomUUID2();
1638
- this.saveContextFile(uuid, context);
1639
- return res.json({
1640
- location: `/playground/${uuid}`,
1641
- uuid
1642
- });
1643
- }
1644
- );
1645
- this.app.post(
1646
- "/execute",
1647
- express.json({ limit: "30mb" }),
1648
- async (req, res) => {
1649
- const { context, type, prompt } = req.body;
1650
- assert3(context, "context is required");
1651
- assert3(type, "type is required");
1652
- assert3(prompt, "prompt is required");
1653
- const requestId = agentRequestCount++;
1654
- console.log(`handle request: #${requestId}, ${type}, ${prompt}`);
1655
- const page = new StaticPage(context);
1656
- const agent = new StaticPageAgent(page);
1657
- const response = {
1658
- result: null,
1659
- dump: null,
1660
- error: null
1661
- };
1662
- try {
1663
- if (type === "aiQuery") {
1664
- response.result = await agent.aiQuery(prompt);
1665
- } else if (type === "aiAction") {
1666
- response.result = await agent.aiAction(prompt);
1667
- } else if (type === "aiAssert") {
1668
- response.result = await agent.aiAssert(prompt, void 0, {
1669
- keepRawResponse: true
1670
- });
1671
- } else {
1672
- response.error = `Unknown type: ${type}`;
1673
- }
1674
- } catch (error) {
1675
- if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {
1676
- response.error = error.message;
1677
- }
1678
- }
1679
- try {
1680
- response.dump = JSON.parse(agent.dumpDataString());
1681
- agent.writeOutActionDumps();
1682
- } catch (error) {
1683
- console.error(
1684
- `write out dump failed: #${requestId}, ${error.message}`
1685
- );
1686
- }
1687
- res.send(response);
1688
- if (response.error) {
1689
- console.error(
1690
- `handle request failed: #${requestId}, ${response.error}`
1691
- );
1692
- } else {
1693
- console.log(`handle request done: #${requestId}`);
1694
- }
1695
- }
1696
- );
1697
- return new Promise((resolve, reject) => {
1698
- const port = this.port || defaultPort;
1699
- this.server = this.app.listen(port, () => {
1700
- this.port = port;
1701
- resolve(this);
1702
- });
1268
+ }
1269
+ clearInput() {
1270
+ return __async(this, null, function* () {
1271
+ return ThrowNotImplemented("clearInput");
1703
1272
  });
1704
1273
  }
1705
- close() {
1706
- if (this.server) {
1707
- return this.server.close();
1708
- }
1274
+ _forceUsePageContext() {
1275
+ return __async(this, null, function* () {
1276
+ return this.uiContext;
1277
+ });
1709
1278
  }
1710
1279
  };
1711
1280
  export {
1712
- PlaygroundServer,
1281
+ ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED,
1713
1282
  StaticPage,
1714
1283
  StaticPageAgent
1715
1284
  };