@midscene/web 0.7.1 → 0.7.2-beta-20241024094141.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/playground.js +8438 -0
- package/dist/browser/types/playground.d.ts +313 -0
- package/dist/es/appium.js +680 -604
- package/dist/es/debug.js +95 -73
- package/dist/es/index.js +939 -797
- package/dist/es/midscene-playground.js +678 -609
- package/dist/es/playground.js +593 -1024
- package/dist/es/playwright-report.js +29 -11
- package/dist/es/playwright.js +705 -597
- package/dist/es/puppeteer.js +636 -552
- package/dist/lib/appium.js +688 -609
- package/dist/lib/debug.js +95 -73
- package/dist/lib/index.js +950 -804
- package/dist/lib/midscene-playground.js +687 -615
- package/dist/lib/playground.js +586 -1007
- package/dist/lib/playwright-report.js +30 -9
- package/dist/lib/playwright.js +713 -602
- package/dist/lib/puppeteer.js +644 -557
- package/dist/script/htmlElement.js +11 -10
- package/dist/script/htmlElementDebug.js +11 -10
- package/dist/types/appium.d.ts +2 -3
- package/dist/types/debug.d.ts +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/{page-ad820b3c.d.ts → page-8117b0ad.d.ts} +8 -7
- package/dist/types/playground.d.ts +6 -21
- package/dist/types/playwright.d.ts +3 -4
- package/dist/types/puppeteer.d.ts +2 -3
- package/dist/types/{tasks-82c1054b.d.ts → tasks-cb6bf758.d.ts} +6 -6
- package/package.json +11 -5
- package/static/index.html +1 -1
package/dist/es/playground.js
CHANGED
|
@@ -21,14 +21,7 @@ var __spreadValues = (a, b) => {
|
|
|
21
21
|
return a;
|
|
22
22
|
};
|
|
23
23
|
var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
|
|
24
|
-
var
|
|
25
|
-
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
26
|
-
}) : x)(function(x) {
|
|
27
|
-
if (typeof require !== "undefined")
|
|
28
|
-
return require.apply(this, arguments);
|
|
29
|
-
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
30
|
-
});
|
|
31
|
-
var __commonJS = (cb, mod) => function __require2() {
|
|
24
|
+
var __commonJS = (cb, mod) => function __require() {
|
|
32
25
|
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
33
26
|
};
|
|
34
27
|
var __copyProps = (to, from, except, desc) => {
|
|
@@ -47,6 +40,26 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
47
40
|
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
48
41
|
mod
|
|
49
42
|
));
|
|
43
|
+
var __async = (__this, __arguments, generator) => {
|
|
44
|
+
return new Promise((resolve, reject) => {
|
|
45
|
+
var fulfilled = (value) => {
|
|
46
|
+
try {
|
|
47
|
+
step(generator.next(value));
|
|
48
|
+
} catch (e) {
|
|
49
|
+
reject(e);
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
var rejected = (value) => {
|
|
53
|
+
try {
|
|
54
|
+
step(generator.throw(value));
|
|
55
|
+
} catch (e) {
|
|
56
|
+
reject(e);
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
|
|
60
|
+
step((generator = generator.apply(__this, __arguments)).next());
|
|
61
|
+
});
|
|
62
|
+
};
|
|
50
63
|
|
|
51
64
|
// ../../node_modules/.pnpm/dayjs@1.11.11/node_modules/dayjs/dayjs.min.js
|
|
52
65
|
var require_dayjs_min = __commonJS({
|
|
@@ -343,385 +356,16 @@ var require_dayjs_min = __commonJS({
|
|
|
343
356
|
}
|
|
344
357
|
});
|
|
345
358
|
|
|
346
|
-
// ../../node_modules/.pnpm/dotenv@16.4.5/node_modules/dotenv/package.json
|
|
347
|
-
var require_package = __commonJS({
|
|
348
|
-
"../../node_modules/.pnpm/dotenv@16.4.5/node_modules/dotenv/package.json"(exports, module) {
|
|
349
|
-
module.exports = {
|
|
350
|
-
name: "dotenv",
|
|
351
|
-
version: "16.4.5",
|
|
352
|
-
description: "Loads environment variables from .env file",
|
|
353
|
-
main: "lib/main.js",
|
|
354
|
-
types: "lib/main.d.ts",
|
|
355
|
-
exports: {
|
|
356
|
-
".": {
|
|
357
|
-
types: "./lib/main.d.ts",
|
|
358
|
-
require: "./lib/main.js",
|
|
359
|
-
default: "./lib/main.js"
|
|
360
|
-
},
|
|
361
|
-
"./config": "./config.js",
|
|
362
|
-
"./config.js": "./config.js",
|
|
363
|
-
"./lib/env-options": "./lib/env-options.js",
|
|
364
|
-
"./lib/env-options.js": "./lib/env-options.js",
|
|
365
|
-
"./lib/cli-options": "./lib/cli-options.js",
|
|
366
|
-
"./lib/cli-options.js": "./lib/cli-options.js",
|
|
367
|
-
"./package.json": "./package.json"
|
|
368
|
-
},
|
|
369
|
-
scripts: {
|
|
370
|
-
"dts-check": "tsc --project tests/types/tsconfig.json",
|
|
371
|
-
lint: "standard",
|
|
372
|
-
"lint-readme": "standard-markdown",
|
|
373
|
-
pretest: "npm run lint && npm run dts-check",
|
|
374
|
-
test: "tap tests/*.js --100 -Rspec",
|
|
375
|
-
"test:coverage": "tap --coverage-report=lcov",
|
|
376
|
-
prerelease: "npm test",
|
|
377
|
-
release: "standard-version"
|
|
378
|
-
},
|
|
379
|
-
repository: {
|
|
380
|
-
type: "git",
|
|
381
|
-
url: "git://github.com/motdotla/dotenv.git"
|
|
382
|
-
},
|
|
383
|
-
funding: "https://dotenvx.com",
|
|
384
|
-
keywords: [
|
|
385
|
-
"dotenv",
|
|
386
|
-
"env",
|
|
387
|
-
".env",
|
|
388
|
-
"environment",
|
|
389
|
-
"variables",
|
|
390
|
-
"config",
|
|
391
|
-
"settings"
|
|
392
|
-
],
|
|
393
|
-
readmeFilename: "README.md",
|
|
394
|
-
license: "BSD-2-Clause",
|
|
395
|
-
devDependencies: {
|
|
396
|
-
"@definitelytyped/dtslint": "^0.0.133",
|
|
397
|
-
"@types/node": "^18.11.3",
|
|
398
|
-
decache: "^4.6.1",
|
|
399
|
-
sinon: "^14.0.1",
|
|
400
|
-
standard: "^17.0.0",
|
|
401
|
-
"standard-markdown": "^7.1.0",
|
|
402
|
-
"standard-version": "^9.5.0",
|
|
403
|
-
tap: "^16.3.0",
|
|
404
|
-
tar: "^6.1.11",
|
|
405
|
-
typescript: "^4.8.4"
|
|
406
|
-
},
|
|
407
|
-
engines: {
|
|
408
|
-
node: ">=12"
|
|
409
|
-
},
|
|
410
|
-
browser: {
|
|
411
|
-
fs: false
|
|
412
|
-
}
|
|
413
|
-
};
|
|
414
|
-
}
|
|
415
|
-
});
|
|
416
|
-
|
|
417
|
-
// ../../node_modules/.pnpm/dotenv@16.4.5/node_modules/dotenv/lib/main.js
|
|
418
|
-
var require_main = __commonJS({
|
|
419
|
-
"../../node_modules/.pnpm/dotenv@16.4.5/node_modules/dotenv/lib/main.js"(exports, module) {
|
|
420
|
-
"use strict";
|
|
421
|
-
var fs = __require("fs");
|
|
422
|
-
var path2 = __require("path");
|
|
423
|
-
var os = __require("os");
|
|
424
|
-
var crypto = __require("crypto");
|
|
425
|
-
var packageJson = require_package();
|
|
426
|
-
var version = packageJson.version;
|
|
427
|
-
var LINE = /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/mg;
|
|
428
|
-
function parse(src) {
|
|
429
|
-
const obj = {};
|
|
430
|
-
let lines = src.toString();
|
|
431
|
-
lines = lines.replace(/\r\n?/mg, "\n");
|
|
432
|
-
let match;
|
|
433
|
-
while ((match = LINE.exec(lines)) != null) {
|
|
434
|
-
const key = match[1];
|
|
435
|
-
let value = match[2] || "";
|
|
436
|
-
value = value.trim();
|
|
437
|
-
const maybeQuote = value[0];
|
|
438
|
-
value = value.replace(/^(['"`])([\s\S]*)\1$/mg, "$2");
|
|
439
|
-
if (maybeQuote === '"') {
|
|
440
|
-
value = value.replace(/\\n/g, "\n");
|
|
441
|
-
value = value.replace(/\\r/g, "\r");
|
|
442
|
-
}
|
|
443
|
-
obj[key] = value;
|
|
444
|
-
}
|
|
445
|
-
return obj;
|
|
446
|
-
}
|
|
447
|
-
function _parseVault(options) {
|
|
448
|
-
const vaultPath = _vaultPath(options);
|
|
449
|
-
const result = DotenvModule.configDotenv({ path: vaultPath });
|
|
450
|
-
if (!result.parsed) {
|
|
451
|
-
const err = new Error(`MISSING_DATA: Cannot parse ${vaultPath} for an unknown reason`);
|
|
452
|
-
err.code = "MISSING_DATA";
|
|
453
|
-
throw err;
|
|
454
|
-
}
|
|
455
|
-
const keys = _dotenvKey(options).split(",");
|
|
456
|
-
const length = keys.length;
|
|
457
|
-
let decrypted;
|
|
458
|
-
for (let i = 0; i < length; i++) {
|
|
459
|
-
try {
|
|
460
|
-
const key = keys[i].trim();
|
|
461
|
-
const attrs = _instructions(result, key);
|
|
462
|
-
decrypted = DotenvModule.decrypt(attrs.ciphertext, attrs.key);
|
|
463
|
-
break;
|
|
464
|
-
} catch (error) {
|
|
465
|
-
if (i + 1 >= length) {
|
|
466
|
-
throw error;
|
|
467
|
-
}
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
return DotenvModule.parse(decrypted);
|
|
471
|
-
}
|
|
472
|
-
function _log(message) {
|
|
473
|
-
console.log(`[dotenv@${version}][INFO] ${message}`);
|
|
474
|
-
}
|
|
475
|
-
function _warn(message) {
|
|
476
|
-
console.log(`[dotenv@${version}][WARN] ${message}`);
|
|
477
|
-
}
|
|
478
|
-
function _debug(message) {
|
|
479
|
-
console.log(`[dotenv@${version}][DEBUG] ${message}`);
|
|
480
|
-
}
|
|
481
|
-
function _dotenvKey(options) {
|
|
482
|
-
if (options && options.DOTENV_KEY && options.DOTENV_KEY.length > 0) {
|
|
483
|
-
return options.DOTENV_KEY;
|
|
484
|
-
}
|
|
485
|
-
if (process.env.DOTENV_KEY && process.env.DOTENV_KEY.length > 0) {
|
|
486
|
-
return process.env.DOTENV_KEY;
|
|
487
|
-
}
|
|
488
|
-
return "";
|
|
489
|
-
}
|
|
490
|
-
function _instructions(result, dotenvKey) {
|
|
491
|
-
let uri;
|
|
492
|
-
try {
|
|
493
|
-
uri = new URL(dotenvKey);
|
|
494
|
-
} catch (error) {
|
|
495
|
-
if (error.code === "ERR_INVALID_URL") {
|
|
496
|
-
const err = new Error("INVALID_DOTENV_KEY: Wrong format. Must be in valid uri format like dotenv://:key_1234@dotenvx.com/vault/.env.vault?environment=development");
|
|
497
|
-
err.code = "INVALID_DOTENV_KEY";
|
|
498
|
-
throw err;
|
|
499
|
-
}
|
|
500
|
-
throw error;
|
|
501
|
-
}
|
|
502
|
-
const key = uri.password;
|
|
503
|
-
if (!key) {
|
|
504
|
-
const err = new Error("INVALID_DOTENV_KEY: Missing key part");
|
|
505
|
-
err.code = "INVALID_DOTENV_KEY";
|
|
506
|
-
throw err;
|
|
507
|
-
}
|
|
508
|
-
const environment = uri.searchParams.get("environment");
|
|
509
|
-
if (!environment) {
|
|
510
|
-
const err = new Error("INVALID_DOTENV_KEY: Missing environment part");
|
|
511
|
-
err.code = "INVALID_DOTENV_KEY";
|
|
512
|
-
throw err;
|
|
513
|
-
}
|
|
514
|
-
const environmentKey = `DOTENV_VAULT_${environment.toUpperCase()}`;
|
|
515
|
-
const ciphertext = result.parsed[environmentKey];
|
|
516
|
-
if (!ciphertext) {
|
|
517
|
-
const err = new Error(`NOT_FOUND_DOTENV_ENVIRONMENT: Cannot locate environment ${environmentKey} in your .env.vault file.`);
|
|
518
|
-
err.code = "NOT_FOUND_DOTENV_ENVIRONMENT";
|
|
519
|
-
throw err;
|
|
520
|
-
}
|
|
521
|
-
return { ciphertext, key };
|
|
522
|
-
}
|
|
523
|
-
function _vaultPath(options) {
|
|
524
|
-
let possibleVaultPath = null;
|
|
525
|
-
if (options && options.path && options.path.length > 0) {
|
|
526
|
-
if (Array.isArray(options.path)) {
|
|
527
|
-
for (const filepath of options.path) {
|
|
528
|
-
if (fs.existsSync(filepath)) {
|
|
529
|
-
possibleVaultPath = filepath.endsWith(".vault") ? filepath : `${filepath}.vault`;
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
} else {
|
|
533
|
-
possibleVaultPath = options.path.endsWith(".vault") ? options.path : `${options.path}.vault`;
|
|
534
|
-
}
|
|
535
|
-
} else {
|
|
536
|
-
possibleVaultPath = path2.resolve(process.cwd(), ".env.vault");
|
|
537
|
-
}
|
|
538
|
-
if (fs.existsSync(possibleVaultPath)) {
|
|
539
|
-
return possibleVaultPath;
|
|
540
|
-
}
|
|
541
|
-
return null;
|
|
542
|
-
}
|
|
543
|
-
function _resolveHome(envPath) {
|
|
544
|
-
return envPath[0] === "~" ? path2.join(os.homedir(), envPath.slice(1)) : envPath;
|
|
545
|
-
}
|
|
546
|
-
function _configVault(options) {
|
|
547
|
-
_log("Loading env from encrypted .env.vault");
|
|
548
|
-
const parsed = DotenvModule._parseVault(options);
|
|
549
|
-
let processEnv = process.env;
|
|
550
|
-
if (options && options.processEnv != null) {
|
|
551
|
-
processEnv = options.processEnv;
|
|
552
|
-
}
|
|
553
|
-
DotenvModule.populate(processEnv, parsed, options);
|
|
554
|
-
return { parsed };
|
|
555
|
-
}
|
|
556
|
-
function configDotenv(options) {
|
|
557
|
-
const dotenvPath = path2.resolve(process.cwd(), ".env");
|
|
558
|
-
let encoding = "utf8";
|
|
559
|
-
const debug = Boolean(options && options.debug);
|
|
560
|
-
if (options && options.encoding) {
|
|
561
|
-
encoding = options.encoding;
|
|
562
|
-
} else {
|
|
563
|
-
if (debug) {
|
|
564
|
-
_debug("No encoding is specified. UTF-8 is used by default");
|
|
565
|
-
}
|
|
566
|
-
}
|
|
567
|
-
let optionPaths = [dotenvPath];
|
|
568
|
-
if (options && options.path) {
|
|
569
|
-
if (!Array.isArray(options.path)) {
|
|
570
|
-
optionPaths = [_resolveHome(options.path)];
|
|
571
|
-
} else {
|
|
572
|
-
optionPaths = [];
|
|
573
|
-
for (const filepath of options.path) {
|
|
574
|
-
optionPaths.push(_resolveHome(filepath));
|
|
575
|
-
}
|
|
576
|
-
}
|
|
577
|
-
}
|
|
578
|
-
let lastError;
|
|
579
|
-
const parsedAll = {};
|
|
580
|
-
for (const path3 of optionPaths) {
|
|
581
|
-
try {
|
|
582
|
-
const parsed = DotenvModule.parse(fs.readFileSync(path3, { encoding }));
|
|
583
|
-
DotenvModule.populate(parsedAll, parsed, options);
|
|
584
|
-
} catch (e) {
|
|
585
|
-
if (debug) {
|
|
586
|
-
_debug(`Failed to load ${path3} ${e.message}`);
|
|
587
|
-
}
|
|
588
|
-
lastError = e;
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
let processEnv = process.env;
|
|
592
|
-
if (options && options.processEnv != null) {
|
|
593
|
-
processEnv = options.processEnv;
|
|
594
|
-
}
|
|
595
|
-
DotenvModule.populate(processEnv, parsedAll, options);
|
|
596
|
-
if (lastError) {
|
|
597
|
-
return { parsed: parsedAll, error: lastError };
|
|
598
|
-
} else {
|
|
599
|
-
return { parsed: parsedAll };
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
function config(options) {
|
|
603
|
-
if (_dotenvKey(options).length === 0) {
|
|
604
|
-
return DotenvModule.configDotenv(options);
|
|
605
|
-
}
|
|
606
|
-
const vaultPath = _vaultPath(options);
|
|
607
|
-
if (!vaultPath) {
|
|
608
|
-
_warn(`You set DOTENV_KEY but you are missing a .env.vault file at ${vaultPath}. Did you forget to build it?`);
|
|
609
|
-
return DotenvModule.configDotenv(options);
|
|
610
|
-
}
|
|
611
|
-
return DotenvModule._configVault(options);
|
|
612
|
-
}
|
|
613
|
-
function decrypt(encrypted, keyStr) {
|
|
614
|
-
const key = Buffer.from(keyStr.slice(-64), "hex");
|
|
615
|
-
let ciphertext = Buffer.from(encrypted, "base64");
|
|
616
|
-
const nonce = ciphertext.subarray(0, 12);
|
|
617
|
-
const authTag = ciphertext.subarray(-16);
|
|
618
|
-
ciphertext = ciphertext.subarray(12, -16);
|
|
619
|
-
try {
|
|
620
|
-
const aesgcm = crypto.createDecipheriv("aes-256-gcm", key, nonce);
|
|
621
|
-
aesgcm.setAuthTag(authTag);
|
|
622
|
-
return `${aesgcm.update(ciphertext)}${aesgcm.final()}`;
|
|
623
|
-
} catch (error) {
|
|
624
|
-
const isRange = error instanceof RangeError;
|
|
625
|
-
const invalidKeyLength = error.message === "Invalid key length";
|
|
626
|
-
const decryptionFailed = error.message === "Unsupported state or unable to authenticate data";
|
|
627
|
-
if (isRange || invalidKeyLength) {
|
|
628
|
-
const err = new Error("INVALID_DOTENV_KEY: It must be 64 characters long (or more)");
|
|
629
|
-
err.code = "INVALID_DOTENV_KEY";
|
|
630
|
-
throw err;
|
|
631
|
-
} else if (decryptionFailed) {
|
|
632
|
-
const err = new Error("DECRYPTION_FAILED: Please check your DOTENV_KEY");
|
|
633
|
-
err.code = "DECRYPTION_FAILED";
|
|
634
|
-
throw err;
|
|
635
|
-
} else {
|
|
636
|
-
throw error;
|
|
637
|
-
}
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
function populate(processEnv, parsed, options = {}) {
|
|
641
|
-
const debug = Boolean(options && options.debug);
|
|
642
|
-
const override = Boolean(options && options.override);
|
|
643
|
-
if (typeof parsed !== "object") {
|
|
644
|
-
const err = new Error("OBJECT_REQUIRED: Please check the processEnv argument being passed to populate");
|
|
645
|
-
err.code = "OBJECT_REQUIRED";
|
|
646
|
-
throw err;
|
|
647
|
-
}
|
|
648
|
-
for (const key of Object.keys(parsed)) {
|
|
649
|
-
if (Object.prototype.hasOwnProperty.call(processEnv, key)) {
|
|
650
|
-
if (override === true) {
|
|
651
|
-
processEnv[key] = parsed[key];
|
|
652
|
-
}
|
|
653
|
-
if (debug) {
|
|
654
|
-
if (override === true) {
|
|
655
|
-
_debug(`"${key}" is already defined and WAS overwritten`);
|
|
656
|
-
} else {
|
|
657
|
-
_debug(`"${key}" is already defined and was NOT overwritten`);
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
} else {
|
|
661
|
-
processEnv[key] = parsed[key];
|
|
662
|
-
}
|
|
663
|
-
}
|
|
664
|
-
}
|
|
665
|
-
var DotenvModule = {
|
|
666
|
-
configDotenv,
|
|
667
|
-
_configVault,
|
|
668
|
-
_parseVault,
|
|
669
|
-
config,
|
|
670
|
-
decrypt,
|
|
671
|
-
parse,
|
|
672
|
-
populate
|
|
673
|
-
};
|
|
674
|
-
module.exports.configDotenv = DotenvModule.configDotenv;
|
|
675
|
-
module.exports._configVault = DotenvModule._configVault;
|
|
676
|
-
module.exports._parseVault = DotenvModule._parseVault;
|
|
677
|
-
module.exports.config = DotenvModule.config;
|
|
678
|
-
module.exports.decrypt = DotenvModule.decrypt;
|
|
679
|
-
module.exports.parse = DotenvModule.parse;
|
|
680
|
-
module.exports.populate = DotenvModule.populate;
|
|
681
|
-
module.exports = DotenvModule;
|
|
682
|
-
}
|
|
683
|
-
});
|
|
684
|
-
|
|
685
|
-
// src/common/agent.ts
|
|
686
|
-
import {
|
|
687
|
-
groupedActionDumpFileExt,
|
|
688
|
-
stringifyDumpData as stringifyDumpData2,
|
|
689
|
-
writeLogFile as writeLogFile2
|
|
690
|
-
} from "@midscene/core/utils";
|
|
691
|
-
|
|
692
|
-
// src/common/tasks.ts
|
|
693
|
-
import assert2 from "assert";
|
|
694
|
-
import {
|
|
695
|
-
Executor,
|
|
696
|
-
Insight,
|
|
697
|
-
plan
|
|
698
|
-
} from "@midscene/core";
|
|
699
|
-
import { sleep } from "@midscene/core/utils";
|
|
700
|
-
import { base64Encoded as base64Encoded2 } from "@midscene/shared/img";
|
|
701
|
-
|
|
702
|
-
// src/common/task-cache.ts
|
|
703
|
-
import { existsSync, readFileSync as readFileSync2 } from "fs";
|
|
704
|
-
import { join } from "path";
|
|
705
|
-
import {
|
|
706
|
-
getLogDirByType,
|
|
707
|
-
stringifyDumpData,
|
|
708
|
-
writeLogFile
|
|
709
|
-
} from "@midscene/core/utils";
|
|
710
|
-
import { getMidscenePkgInfo } from "@midscene/shared/fs";
|
|
711
|
-
|
|
712
359
|
// src/common/utils.ts
|
|
713
360
|
var import_dayjs = __toESM(require_dayjs_min());
|
|
714
361
|
import assert from "assert";
|
|
715
|
-
import { randomUUID } from "crypto";
|
|
716
362
|
import { readFileSync } from "fs";
|
|
717
363
|
import path from "path";
|
|
718
364
|
import { NodeType } from "@midscene/shared/constants";
|
|
719
365
|
import { findNearestPackageJson } from "@midscene/shared/fs";
|
|
720
|
-
import {
|
|
721
|
-
base64Encoded,
|
|
722
|
-
imageInfoOfBase64
|
|
723
|
-
} from "@midscene/shared/img";
|
|
366
|
+
import { imageInfoOfBase64 } from "@midscene/shared/img";
|
|
724
367
|
import { compositeElementInfoImg } from "@midscene/shared/img";
|
|
368
|
+
import { uuid } from "@midscene/shared/utils";
|
|
725
369
|
|
|
726
370
|
// src/web-element.ts
|
|
727
371
|
var WebElementInfo = class {
|
|
@@ -749,56 +393,59 @@ var WebElementInfo = class {
|
|
|
749
393
|
};
|
|
750
394
|
|
|
751
395
|
// src/common/utils.ts
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
const url = page.url();
|
|
758
|
-
const file = await page.screenshot();
|
|
759
|
-
const screenshotBase64 = base64Encoded(file);
|
|
760
|
-
const captureElementSnapshot = await page.getElementInfos();
|
|
761
|
-
const elementsInfo = await alignElements(captureElementSnapshot, page);
|
|
762
|
-
const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
|
|
763
|
-
if (elementInfo.attributes.nodeType === NodeType.TEXT) {
|
|
764
|
-
return false;
|
|
396
|
+
function parseContextFromWebPage(page, _opt) {
|
|
397
|
+
return __async(this, null, function* () {
|
|
398
|
+
assert(page, "page is required");
|
|
399
|
+
if (page._forceUsePageContext) {
|
|
400
|
+
return yield page._forceUsePageContext();
|
|
765
401
|
}
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
402
|
+
const url = page.url();
|
|
403
|
+
const screenshotBase64 = yield page.screenshotBase64();
|
|
404
|
+
const captureElementSnapshot = yield page.getElementInfos();
|
|
405
|
+
const elementsInfo = yield alignElements(captureElementSnapshot, page);
|
|
406
|
+
const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
|
|
407
|
+
if (elementInfo.attributes.nodeType === NodeType.TEXT) {
|
|
408
|
+
return false;
|
|
409
|
+
}
|
|
410
|
+
return true;
|
|
411
|
+
});
|
|
412
|
+
const size = yield imageInfoOfBase64(screenshotBase64);
|
|
413
|
+
const screenshotBase64WithElementMarker = yield compositeElementInfoImg({
|
|
414
|
+
inputImgBase64: screenshotBase64,
|
|
415
|
+
elementsPositionInfo: elementsPositionInfoWithoutText
|
|
416
|
+
});
|
|
417
|
+
return {
|
|
418
|
+
content: elementsInfo,
|
|
419
|
+
size,
|
|
420
|
+
screenshotBase64,
|
|
421
|
+
screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
|
|
422
|
+
url
|
|
423
|
+
};
|
|
772
424
|
});
|
|
773
|
-
return {
|
|
774
|
-
content: elementsInfo,
|
|
775
|
-
size,
|
|
776
|
-
screenshotBase64,
|
|
777
|
-
screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
|
|
778
|
-
url
|
|
779
|
-
};
|
|
780
425
|
}
|
|
781
426
|
var sizeThreshold = 3;
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
427
|
+
function alignElements(elements, page) {
|
|
428
|
+
return __async(this, null, function* () {
|
|
429
|
+
const validElements = elements.filter((item) => {
|
|
430
|
+
return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
|
|
431
|
+
});
|
|
432
|
+
const textsAligned = [];
|
|
433
|
+
for (const item of validElements) {
|
|
434
|
+
const { rect, id, content, attributes, locator, indexId } = item;
|
|
435
|
+
textsAligned.push(
|
|
436
|
+
new WebElementInfo({
|
|
437
|
+
rect,
|
|
438
|
+
locator,
|
|
439
|
+
id,
|
|
440
|
+
content,
|
|
441
|
+
attributes,
|
|
442
|
+
page,
|
|
443
|
+
indexId
|
|
444
|
+
})
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
return textsAligned;
|
|
785
448
|
});
|
|
786
|
-
const textsAligned = [];
|
|
787
|
-
for (const item of validElements) {
|
|
788
|
-
const { rect, id, content, attributes, locator, indexId } = item;
|
|
789
|
-
textsAligned.push(
|
|
790
|
-
new WebElementInfo({
|
|
791
|
-
rect,
|
|
792
|
-
locator,
|
|
793
|
-
id,
|
|
794
|
-
content,
|
|
795
|
-
attributes,
|
|
796
|
-
page,
|
|
797
|
-
indexId
|
|
798
|
-
})
|
|
799
|
-
);
|
|
800
|
-
}
|
|
801
|
-
return textsAligned;
|
|
802
449
|
}
|
|
803
450
|
function reportFileName(tag = "web") {
|
|
804
451
|
const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
|
|
@@ -829,7 +476,7 @@ var testFileIndex = /* @__PURE__ */ new Map();
|
|
|
829
476
|
function generateCacheId(fileName) {
|
|
830
477
|
let taskFile = fileName || getCurrentExecutionFile();
|
|
831
478
|
if (!taskFile) {
|
|
832
|
-
taskFile =
|
|
479
|
+
taskFile = uuid();
|
|
833
480
|
console.warn(
|
|
834
481
|
"Midscene - using random UUID for cache id. Cache may be invalid."
|
|
835
482
|
);
|
|
@@ -846,10 +493,35 @@ function generateCacheId(fileName) {
|
|
|
846
493
|
}
|
|
847
494
|
var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
|
|
848
495
|
|
|
496
|
+
// src/common/agent.ts
|
|
497
|
+
import {
|
|
498
|
+
groupedActionDumpFileExt,
|
|
499
|
+
stringifyDumpData as stringifyDumpData2,
|
|
500
|
+
writeLogFile as writeLogFile2
|
|
501
|
+
} from "@midscene/core/utils";
|
|
502
|
+
|
|
503
|
+
// src/common/tasks.ts
|
|
504
|
+
import assert2 from "assert";
|
|
505
|
+
import {
|
|
506
|
+
Executor,
|
|
507
|
+
Insight,
|
|
508
|
+
plan
|
|
509
|
+
} from "@midscene/core";
|
|
510
|
+
import { sleep } from "@midscene/core/utils";
|
|
511
|
+
|
|
849
512
|
// src/common/task-cache.ts
|
|
513
|
+
import { existsSync, readFileSync as readFileSync2 } from "fs";
|
|
514
|
+
import { join } from "path";
|
|
515
|
+
import {
|
|
516
|
+
getLogDirByType,
|
|
517
|
+
stringifyDumpData,
|
|
518
|
+
writeLogFile
|
|
519
|
+
} from "@midscene/core/utils";
|
|
520
|
+
import { getRunningPkgInfo } from "@midscene/shared/fs";
|
|
521
|
+
import { ifInBrowser } from "@midscene/shared/utils";
|
|
850
522
|
var TaskCache = class {
|
|
851
523
|
constructor(opts) {
|
|
852
|
-
this.midscenePkgInfo =
|
|
524
|
+
this.midscenePkgInfo = getRunningPkgInfo();
|
|
853
525
|
this.cacheId = generateCacheId(opts == null ? void 0 : opts.fileName);
|
|
854
526
|
this.cache = this.readCacheFromFile() || {
|
|
855
527
|
aiTasks: []
|
|
@@ -931,11 +603,17 @@ var TaskCache = class {
|
|
|
931
603
|
return this.newCache;
|
|
932
604
|
}
|
|
933
605
|
readCacheFromFile() {
|
|
606
|
+
if (ifInBrowser) {
|
|
607
|
+
return void 0;
|
|
608
|
+
}
|
|
934
609
|
const cacheFile = join(getLogDirByType("cache"), `${this.cacheId}.json`);
|
|
935
610
|
if (process.env.MIDSCENE_CACHE === "true" && existsSync(cacheFile)) {
|
|
936
611
|
try {
|
|
937
612
|
const data = readFileSync2(cacheFile, "utf8");
|
|
938
613
|
const jsonData = JSON.parse(data);
|
|
614
|
+
if (!this.midscenePkgInfo) {
|
|
615
|
+
return void 0;
|
|
616
|
+
}
|
|
939
617
|
if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
|
|
940
618
|
return void 0;
|
|
941
619
|
}
|
|
@@ -947,7 +625,10 @@ var TaskCache = class {
|
|
|
947
625
|
return void 0;
|
|
948
626
|
}
|
|
949
627
|
writeCacheToFile() {
|
|
950
|
-
const midscenePkgInfo =
|
|
628
|
+
const midscenePkgInfo = getRunningPkgInfo();
|
|
629
|
+
if (!midscenePkgInfo) {
|
|
630
|
+
return;
|
|
631
|
+
}
|
|
951
632
|
writeLogFile({
|
|
952
633
|
fileName: `${this.cacheId}`,
|
|
953
634
|
fileExt: "json",
|
|
@@ -968,423 +649,435 @@ var TaskCache = class {
|
|
|
968
649
|
var PageTaskExecutor = class {
|
|
969
650
|
constructor(page, opts) {
|
|
970
651
|
this.page = page;
|
|
971
|
-
this.insight = new Insight(
|
|
972
|
-
return
|
|
973
|
-
});
|
|
652
|
+
this.insight = new Insight(() => __async(this, null, function* () {
|
|
653
|
+
return yield parseContextFromWebPage(page);
|
|
654
|
+
}));
|
|
974
655
|
this.taskCache = new TaskCache({
|
|
975
656
|
fileName: opts == null ? void 0 : opts.cacheId
|
|
976
657
|
});
|
|
977
658
|
}
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
659
|
+
recordScreenshot(timing) {
|
|
660
|
+
return __async(this, null, function* () {
|
|
661
|
+
const base64 = yield this.page.screenshotBase64();
|
|
662
|
+
const item = {
|
|
663
|
+
type: "screenshot",
|
|
664
|
+
ts: Date.now(),
|
|
665
|
+
screenshot: base64,
|
|
666
|
+
timing
|
|
667
|
+
};
|
|
668
|
+
return item;
|
|
669
|
+
});
|
|
987
670
|
}
|
|
988
671
|
wrapExecutorWithScreenshot(taskApply) {
|
|
989
672
|
const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
|
|
990
|
-
executor:
|
|
673
|
+
executor: (param, context, ...args) => __async(this, null, function* () {
|
|
991
674
|
const recorder = [];
|
|
992
675
|
const { task } = context;
|
|
993
676
|
task.recorder = recorder;
|
|
994
|
-
const shot =
|
|
677
|
+
const shot = yield this.recordScreenshot(`before ${task.type}`);
|
|
995
678
|
recorder.push(shot);
|
|
996
|
-
const result =
|
|
679
|
+
const result = yield taskApply.executor(param, context, ...args);
|
|
997
680
|
if (taskApply.type === "Action") {
|
|
998
|
-
|
|
999
|
-
const shot2 =
|
|
681
|
+
yield sleep(1e3);
|
|
682
|
+
const shot2 = yield this.recordScreenshot("after Action");
|
|
1000
683
|
recorder.push(shot2);
|
|
1001
684
|
}
|
|
1002
685
|
return result;
|
|
1003
|
-
}
|
|
686
|
+
})
|
|
1004
687
|
});
|
|
1005
688
|
return taskWithScreenshot;
|
|
1006
689
|
}
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1021
|
-
const pageContext = await this.insight.contextRetrieverFn();
|
|
1022
|
-
const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
|
|
1023
|
-
pageContext,
|
|
1024
|
-
"locate",
|
|
1025
|
-
param.prompt
|
|
1026
|
-
);
|
|
1027
|
-
let locateResult;
|
|
1028
|
-
const callAI = this.insight.aiVendorFn;
|
|
1029
|
-
const element = await this.insight.locate(param.prompt, {
|
|
1030
|
-
quickAnswer: plan2.quickAnswer,
|
|
1031
|
-
callAI: async (...message) => {
|
|
1032
|
-
if (locateCache) {
|
|
1033
|
-
locateResult = locateCache;
|
|
1034
|
-
return Promise.resolve(locateCache);
|
|
1035
|
-
}
|
|
1036
|
-
locateResult = await callAI(...message);
|
|
1037
|
-
assert2(locateResult);
|
|
1038
|
-
return locateResult;
|
|
1039
|
-
}
|
|
1040
|
-
});
|
|
1041
|
-
if (locateResult) {
|
|
1042
|
-
cacheGroup == null ? void 0 : cacheGroup.saveCache({
|
|
1043
|
-
type: "locate",
|
|
1044
|
-
pageContext: {
|
|
1045
|
-
url: pageContext.url,
|
|
1046
|
-
size: pageContext.size
|
|
1047
|
-
},
|
|
1048
|
-
prompt: param.prompt,
|
|
1049
|
-
response: locateResult
|
|
1050
|
-
});
|
|
1051
|
-
}
|
|
1052
|
-
if (!element) {
|
|
1053
|
-
task.log = {
|
|
1054
|
-
dump: insightDump
|
|
690
|
+
convertPlanToExecutable(plans, cacheGroup) {
|
|
691
|
+
return __async(this, null, function* () {
|
|
692
|
+
const tasks = plans.map((plan2) => {
|
|
693
|
+
if (plan2.type === "Locate") {
|
|
694
|
+
const taskFind = {
|
|
695
|
+
type: "Insight",
|
|
696
|
+
subType: "Locate",
|
|
697
|
+
param: plan2.param,
|
|
698
|
+
executor: (param, taskContext) => __async(this, null, function* () {
|
|
699
|
+
const { task } = taskContext;
|
|
700
|
+
let insightDump;
|
|
701
|
+
const dumpCollector = (dump) => {
|
|
702
|
+
insightDump = dump;
|
|
1055
703
|
};
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
704
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
705
|
+
const pageContext = yield this.insight.contextRetrieverFn();
|
|
706
|
+
const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
|
|
707
|
+
pageContext,
|
|
708
|
+
"locate",
|
|
709
|
+
param.prompt
|
|
710
|
+
);
|
|
711
|
+
let locateResult;
|
|
712
|
+
const callAI = this.insight.aiVendorFn;
|
|
713
|
+
const element = yield this.insight.locate(param.prompt, {
|
|
714
|
+
quickAnswer: plan2.quickAnswer,
|
|
715
|
+
callAI: (...message) => __async(this, null, function* () {
|
|
716
|
+
if (locateCache) {
|
|
717
|
+
locateResult = locateCache;
|
|
718
|
+
return Promise.resolve(locateCache);
|
|
719
|
+
}
|
|
720
|
+
locateResult = yield callAI(...message);
|
|
721
|
+
assert2(locateResult);
|
|
722
|
+
return locateResult;
|
|
723
|
+
})
|
|
724
|
+
});
|
|
725
|
+
if (locateResult) {
|
|
726
|
+
cacheGroup == null ? void 0 : cacheGroup.saveCache({
|
|
727
|
+
type: "locate",
|
|
728
|
+
pageContext: {
|
|
729
|
+
url: pageContext.url,
|
|
730
|
+
size: pageContext.size
|
|
731
|
+
},
|
|
732
|
+
prompt: param.prompt,
|
|
733
|
+
response: locateResult
|
|
734
|
+
});
|
|
1067
735
|
}
|
|
1068
|
-
|
|
1069
|
-
}
|
|
1070
|
-
};
|
|
1071
|
-
return taskFind;
|
|
1072
|
-
}
|
|
1073
|
-
if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
|
|
1074
|
-
const assertPlan = plan2;
|
|
1075
|
-
const taskAssert = {
|
|
1076
|
-
type: "Insight",
|
|
1077
|
-
subType: "Assert",
|
|
1078
|
-
param: assertPlan.param,
|
|
1079
|
-
executor: async (param, taskContext) => {
|
|
1080
|
-
const { task } = taskContext;
|
|
1081
|
-
let insightDump;
|
|
1082
|
-
const dumpCollector = (dump) => {
|
|
1083
|
-
insightDump = dump;
|
|
1084
|
-
};
|
|
1085
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1086
|
-
const assertion = await this.insight.assert(
|
|
1087
|
-
assertPlan.param.assertion
|
|
1088
|
-
);
|
|
1089
|
-
if (!assertion.pass) {
|
|
1090
|
-
if (plan2.type === "Assert") {
|
|
1091
|
-
task.output = assertion;
|
|
736
|
+
if (!element) {
|
|
1092
737
|
task.log = {
|
|
1093
738
|
dump: insightDump
|
|
1094
739
|
};
|
|
1095
|
-
throw new Error(
|
|
1096
|
-
assertion.thought || "Assertion failed without reason"
|
|
1097
|
-
);
|
|
740
|
+
throw new Error(`Element not found: ${param.prompt}`);
|
|
1098
741
|
}
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
742
|
+
return {
|
|
743
|
+
output: {
|
|
744
|
+
element
|
|
745
|
+
},
|
|
746
|
+
log: {
|
|
747
|
+
dump: insightDump
|
|
748
|
+
},
|
|
749
|
+
cache: {
|
|
750
|
+
hit: Boolean(locateCache)
|
|
751
|
+
}
|
|
752
|
+
};
|
|
753
|
+
})
|
|
754
|
+
};
|
|
755
|
+
return taskFind;
|
|
756
|
+
}
|
|
757
|
+
if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
|
|
758
|
+
const assertPlan = plan2;
|
|
759
|
+
const taskAssert = {
|
|
760
|
+
type: "Insight",
|
|
761
|
+
subType: "Assert",
|
|
762
|
+
param: assertPlan.param,
|
|
763
|
+
executor: (param, taskContext) => __async(this, null, function* () {
|
|
764
|
+
const { task } = taskContext;
|
|
765
|
+
let insightDump;
|
|
766
|
+
const dumpCollector = (dump) => {
|
|
767
|
+
insightDump = dump;
|
|
768
|
+
};
|
|
769
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
770
|
+
const assertion = yield this.insight.assert(
|
|
771
|
+
assertPlan.param.assertion
|
|
772
|
+
);
|
|
773
|
+
if (!assertion.pass) {
|
|
774
|
+
if (plan2.type === "Assert") {
|
|
775
|
+
task.output = assertion;
|
|
776
|
+
task.log = {
|
|
777
|
+
dump: insightDump
|
|
778
|
+
};
|
|
779
|
+
throw new Error(
|
|
780
|
+
assertion.thought || "Assertion failed without reason"
|
|
781
|
+
);
|
|
782
|
+
}
|
|
783
|
+
task.error = assertion.thought;
|
|
1105
784
|
}
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
785
|
+
return {
|
|
786
|
+
output: assertion,
|
|
787
|
+
log: {
|
|
788
|
+
dump: insightDump
|
|
789
|
+
}
|
|
790
|
+
};
|
|
791
|
+
})
|
|
792
|
+
};
|
|
793
|
+
return taskAssert;
|
|
794
|
+
}
|
|
795
|
+
if (plan2.type === "Input") {
|
|
796
|
+
const taskActionInput = {
|
|
797
|
+
type: "Action",
|
|
798
|
+
subType: "Input",
|
|
799
|
+
param: plan2.param,
|
|
800
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (taskParam, { element }) {
|
|
801
|
+
if (element) {
|
|
802
|
+
yield this.page.clearInput(element);
|
|
803
|
+
if (taskParam.value === "") {
|
|
804
|
+
return;
|
|
805
|
+
}
|
|
806
|
+
yield this.page.keyboard.type(taskParam.value);
|
|
1121
807
|
}
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
}
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
}
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
}
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
}
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
}
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
}
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
)
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
}
|
|
1222
|
-
|
|
1223
|
-
}
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
return
|
|
808
|
+
})
|
|
809
|
+
};
|
|
810
|
+
return taskActionInput;
|
|
811
|
+
}
|
|
812
|
+
if (plan2.type === "KeyboardPress") {
|
|
813
|
+
const taskActionKeyboardPress = {
|
|
814
|
+
type: "Action",
|
|
815
|
+
subType: "KeyboardPress",
|
|
816
|
+
param: plan2.param,
|
|
817
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
818
|
+
assert2(taskParam.value, "No key to press");
|
|
819
|
+
yield this.page.keyboard.press(taskParam.value);
|
|
820
|
+
})
|
|
821
|
+
};
|
|
822
|
+
return taskActionKeyboardPress;
|
|
823
|
+
}
|
|
824
|
+
if (plan2.type === "Tap") {
|
|
825
|
+
const taskActionTap = {
|
|
826
|
+
type: "Action",
|
|
827
|
+
subType: "Tap",
|
|
828
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
|
|
829
|
+
assert2(element, "Element not found, cannot tap");
|
|
830
|
+
yield this.page.mouse.click(
|
|
831
|
+
element.center[0],
|
|
832
|
+
element.center[1]
|
|
833
|
+
);
|
|
834
|
+
})
|
|
835
|
+
};
|
|
836
|
+
return taskActionTap;
|
|
837
|
+
}
|
|
838
|
+
if (plan2.type === "Hover") {
|
|
839
|
+
const taskActionHover = {
|
|
840
|
+
type: "Action",
|
|
841
|
+
subType: "Hover",
|
|
842
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
|
|
843
|
+
assert2(element, "Element not found, cannot hover");
|
|
844
|
+
yield this.page.mouse.move(
|
|
845
|
+
element.center[0],
|
|
846
|
+
element.center[1]
|
|
847
|
+
);
|
|
848
|
+
})
|
|
849
|
+
};
|
|
850
|
+
return taskActionHover;
|
|
851
|
+
}
|
|
852
|
+
if (plan2.type === "Scroll") {
|
|
853
|
+
const taskActionScroll = {
|
|
854
|
+
type: "Action",
|
|
855
|
+
subType: "Scroll",
|
|
856
|
+
param: plan2.param,
|
|
857
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
858
|
+
const scrollToEventName = taskParam.scrollType;
|
|
859
|
+
switch (scrollToEventName) {
|
|
860
|
+
case "scrollUntilTop":
|
|
861
|
+
yield this.page.scrollUntilTop();
|
|
862
|
+
break;
|
|
863
|
+
case "scrollUntilBottom":
|
|
864
|
+
yield this.page.scrollUntilBottom();
|
|
865
|
+
break;
|
|
866
|
+
case "scrollUpOneScreen":
|
|
867
|
+
yield this.page.scrollUpOneScreen();
|
|
868
|
+
break;
|
|
869
|
+
case "scrollDownOneScreen":
|
|
870
|
+
yield this.page.scrollDownOneScreen();
|
|
871
|
+
break;
|
|
872
|
+
default:
|
|
873
|
+
console.error(
|
|
874
|
+
"Unknown scroll event type:",
|
|
875
|
+
scrollToEventName
|
|
876
|
+
);
|
|
877
|
+
}
|
|
878
|
+
})
|
|
879
|
+
};
|
|
880
|
+
return taskActionScroll;
|
|
881
|
+
}
|
|
882
|
+
if (plan2.type === "Sleep") {
|
|
883
|
+
const taskActionSleep = {
|
|
884
|
+
type: "Action",
|
|
885
|
+
subType: "Sleep",
|
|
886
|
+
param: plan2.param,
|
|
887
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
888
|
+
yield sleep(taskParam.timeMs || 3e3);
|
|
889
|
+
})
|
|
890
|
+
};
|
|
891
|
+
return taskActionSleep;
|
|
892
|
+
}
|
|
893
|
+
if (plan2.type === "Error") {
|
|
894
|
+
const taskActionError = {
|
|
895
|
+
type: "Action",
|
|
896
|
+
subType: "Error",
|
|
897
|
+
param: plan2.param,
|
|
898
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
899
|
+
assert2(
|
|
900
|
+
taskParam.thought,
|
|
901
|
+
"An error occurred, but no thought provided"
|
|
902
|
+
);
|
|
903
|
+
throw new Error(taskParam.thought);
|
|
904
|
+
})
|
|
905
|
+
};
|
|
906
|
+
return taskActionError;
|
|
907
|
+
}
|
|
908
|
+
throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
|
|
909
|
+
}).map((task) => {
|
|
910
|
+
return this.wrapExecutorWithScreenshot(task);
|
|
911
|
+
});
|
|
912
|
+
return tasks;
|
|
1227
913
|
});
|
|
1228
|
-
return tasks;
|
|
1229
914
|
}
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
915
|
+
action(userPrompt) {
|
|
916
|
+
return __async(this, null, function* () {
|
|
917
|
+
const taskExecutor = new Executor(userPrompt);
|
|
918
|
+
const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
|
|
919
|
+
let plans = [];
|
|
920
|
+
const planningTask = {
|
|
921
|
+
type: "Planning",
|
|
922
|
+
param: {
|
|
923
|
+
userPrompt
|
|
924
|
+
},
|
|
925
|
+
executor: (param) => __async(this, null, function* () {
|
|
926
|
+
const pageContext = yield this.insight.contextRetrieverFn();
|
|
927
|
+
let planResult;
|
|
928
|
+
const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
|
|
929
|
+
if (planCache) {
|
|
930
|
+
planResult = planCache;
|
|
931
|
+
} else {
|
|
932
|
+
planResult = yield plan(param.userPrompt, {
|
|
933
|
+
context: pageContext
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
assert2(planResult.plans.length > 0, "No plans found");
|
|
937
|
+
plans = planResult.plans;
|
|
938
|
+
cacheGroup.saveCache({
|
|
939
|
+
type: "plan",
|
|
940
|
+
pageContext: {
|
|
941
|
+
url: pageContext.url,
|
|
942
|
+
size: pageContext.size
|
|
943
|
+
},
|
|
944
|
+
prompt: userPrompt,
|
|
945
|
+
response: planResult
|
|
1248
946
|
});
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
});
|
|
947
|
+
return {
|
|
948
|
+
output: planResult,
|
|
949
|
+
cache: {
|
|
950
|
+
hit: Boolean(planCache)
|
|
951
|
+
}
|
|
952
|
+
};
|
|
953
|
+
})
|
|
954
|
+
};
|
|
955
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
|
|
956
|
+
let output = yield taskExecutor.flush();
|
|
957
|
+
if (taskExecutor.isInErrorState()) {
|
|
1261
958
|
return {
|
|
1262
|
-
output
|
|
1263
|
-
|
|
1264
|
-
hit: Boolean(planCache)
|
|
1265
|
-
}
|
|
959
|
+
output,
|
|
960
|
+
executor: taskExecutor
|
|
1266
961
|
};
|
|
1267
962
|
}
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
if (taskExecutor.isInErrorState()) {
|
|
963
|
+
const executables = yield this.convertPlanToExecutable(plans, cacheGroup);
|
|
964
|
+
yield taskExecutor.append(executables);
|
|
965
|
+
output = yield taskExecutor.flush();
|
|
1272
966
|
return {
|
|
1273
967
|
output,
|
|
1274
968
|
executor: taskExecutor
|
|
1275
969
|
};
|
|
1276
|
-
}
|
|
1277
|
-
const executables = await this.convertPlanToExecutable(plans, cacheGroup);
|
|
1278
|
-
await taskExecutor.append(executables);
|
|
1279
|
-
output = await taskExecutor.flush();
|
|
1280
|
-
return {
|
|
1281
|
-
output,
|
|
1282
|
-
executor: taskExecutor
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
async query(demand) {
|
|
1286
|
-
const description = typeof demand === "string" ? demand : JSON.stringify(demand);
|
|
1287
|
-
const taskExecutor = new Executor(description);
|
|
1288
|
-
const queryTask = {
|
|
1289
|
-
type: "Insight",
|
|
1290
|
-
subType: "Query",
|
|
1291
|
-
param: {
|
|
1292
|
-
dataDemand: demand
|
|
1293
|
-
},
|
|
1294
|
-
executor: async (param) => {
|
|
1295
|
-
let insightDump;
|
|
1296
|
-
const dumpCollector = (dump) => {
|
|
1297
|
-
insightDump = dump;
|
|
1298
|
-
};
|
|
1299
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1300
|
-
const data = await this.insight.extract(param.dataDemand);
|
|
1301
|
-
return {
|
|
1302
|
-
output: data,
|
|
1303
|
-
log: { dump: insightDump }
|
|
1304
|
-
};
|
|
1305
|
-
}
|
|
1306
|
-
};
|
|
1307
|
-
await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
|
|
1308
|
-
const output = await taskExecutor.flush();
|
|
1309
|
-
return {
|
|
1310
|
-
output,
|
|
1311
|
-
executor: taskExecutor
|
|
1312
|
-
};
|
|
970
|
+
});
|
|
1313
971
|
}
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
972
|
+
query(demand) {
|
|
973
|
+
return __async(this, null, function* () {
|
|
974
|
+
const description = typeof demand === "string" ? demand : JSON.stringify(demand);
|
|
975
|
+
const taskExecutor = new Executor(description);
|
|
976
|
+
const queryTask = {
|
|
977
|
+
type: "Insight",
|
|
978
|
+
subType: "Query",
|
|
979
|
+
param: {
|
|
980
|
+
dataDemand: demand
|
|
981
|
+
},
|
|
982
|
+
executor: (param) => __async(this, null, function* () {
|
|
983
|
+
let insightDump;
|
|
984
|
+
const dumpCollector = (dump) => {
|
|
985
|
+
insightDump = dump;
|
|
986
|
+
};
|
|
987
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
988
|
+
const data = yield this.insight.extract(param.dataDemand);
|
|
989
|
+
return {
|
|
990
|
+
output: data,
|
|
991
|
+
log: { dump: insightDump }
|
|
992
|
+
};
|
|
993
|
+
})
|
|
994
|
+
};
|
|
995
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
|
|
996
|
+
const output = yield taskExecutor.flush();
|
|
997
|
+
return {
|
|
998
|
+
output,
|
|
999
|
+
executor: taskExecutor
|
|
1000
|
+
};
|
|
1001
|
+
});
|
|
1330
1002
|
}
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
assert2(checkIntervalMs, "No checkIntervalMs for waitFor");
|
|
1338
|
-
const overallStartTime = Date.now();
|
|
1339
|
-
let startTime = Date.now();
|
|
1340
|
-
let errorThought = "";
|
|
1341
|
-
while (Date.now() - overallStartTime < timeoutMs) {
|
|
1342
|
-
startTime = Date.now();
|
|
1343
|
-
const assertPlan = {
|
|
1344
|
-
type: "AssertWithoutThrow",
|
|
1003
|
+
assert(assertion) {
|
|
1004
|
+
return __async(this, null, function* () {
|
|
1005
|
+
const description = `assert: ${assertion}`;
|
|
1006
|
+
const taskExecutor = new Executor(description);
|
|
1007
|
+
const assertionPlan = {
|
|
1008
|
+
type: "Assert",
|
|
1345
1009
|
param: {
|
|
1346
1010
|
assertion
|
|
1347
1011
|
}
|
|
1348
1012
|
};
|
|
1349
|
-
const assertTask =
|
|
1350
|
-
|
|
1351
|
-
const output =
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1013
|
+
const assertTask = yield this.convertPlanToExecutable([assertionPlan]);
|
|
1014
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
|
|
1015
|
+
const output = yield taskExecutor.flush();
|
|
1016
|
+
return {
|
|
1017
|
+
output,
|
|
1018
|
+
executor: taskExecutor
|
|
1019
|
+
};
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
waitFor(assertion, opt) {
|
|
1023
|
+
return __async(this, null, function* () {
|
|
1024
|
+
const description = `waitFor: ${assertion}`;
|
|
1025
|
+
const taskExecutor = new Executor(description);
|
|
1026
|
+
const { timeoutMs, checkIntervalMs } = opt;
|
|
1027
|
+
assert2(assertion, "No assertion for waitFor");
|
|
1028
|
+
assert2(timeoutMs, "No timeoutMs for waitFor");
|
|
1029
|
+
assert2(checkIntervalMs, "No checkIntervalMs for waitFor");
|
|
1030
|
+
const overallStartTime = Date.now();
|
|
1031
|
+
let startTime = Date.now();
|
|
1032
|
+
let errorThought = "";
|
|
1033
|
+
while (Date.now() - overallStartTime < timeoutMs) {
|
|
1034
|
+
startTime = Date.now();
|
|
1035
|
+
const assertPlan = {
|
|
1036
|
+
type: "AssertWithoutThrow",
|
|
1364
1037
|
param: {
|
|
1365
|
-
|
|
1038
|
+
assertion
|
|
1366
1039
|
}
|
|
1367
1040
|
};
|
|
1368
|
-
const
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
)
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1041
|
+
const assertTask = yield this.convertPlanToExecutable([assertPlan]);
|
|
1042
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
|
|
1043
|
+
const output = yield taskExecutor.flush();
|
|
1044
|
+
if (output == null ? void 0 : output.pass) {
|
|
1045
|
+
return {
|
|
1046
|
+
output: void 0,
|
|
1047
|
+
executor: taskExecutor
|
|
1048
|
+
};
|
|
1049
|
+
}
|
|
1050
|
+
errorThought = (output == null ? void 0 : output.thought) || "unknown error";
|
|
1051
|
+
const now = Date.now();
|
|
1052
|
+
if (now - startTime < checkIntervalMs) {
|
|
1053
|
+
const timeRemaining = checkIntervalMs - (now - startTime);
|
|
1054
|
+
const sleepPlan = {
|
|
1055
|
+
type: "Sleep",
|
|
1056
|
+
param: {
|
|
1057
|
+
timeMs: timeRemaining
|
|
1058
|
+
}
|
|
1059
|
+
};
|
|
1060
|
+
const sleepTask = yield this.convertPlanToExecutable([sleepPlan]);
|
|
1061
|
+
yield taskExecutor.append(
|
|
1062
|
+
this.wrapExecutorWithScreenshot(sleepTask[0])
|
|
1063
|
+
);
|
|
1064
|
+
yield taskExecutor.flush();
|
|
1065
|
+
}
|
|
1379
1066
|
}
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1067
|
+
const errorPlan = {
|
|
1068
|
+
type: "Error",
|
|
1069
|
+
param: {
|
|
1070
|
+
thought: `waitFor timeout: ${errorThought}`
|
|
1071
|
+
}
|
|
1072
|
+
};
|
|
1073
|
+
const errorTask = yield this.convertPlanToExecutable([errorPlan]);
|
|
1074
|
+
yield taskExecutor.append(errorTask[0]);
|
|
1075
|
+
yield taskExecutor.flush();
|
|
1076
|
+
return {
|
|
1077
|
+
output: void 0,
|
|
1078
|
+
executor: taskExecutor
|
|
1079
|
+
};
|
|
1080
|
+
});
|
|
1388
1081
|
}
|
|
1389
1082
|
};
|
|
1390
1083
|
|
|
@@ -1429,72 +1122,83 @@ var PageAgent = class {
|
|
|
1429
1122
|
type: "dump",
|
|
1430
1123
|
generateReport
|
|
1431
1124
|
});
|
|
1432
|
-
if (generateReport && autoPrintReportMsg) {
|
|
1125
|
+
if (generateReport && autoPrintReportMsg && this.reportFile) {
|
|
1433
1126
|
printReportMsg(this.reportFile);
|
|
1434
1127
|
}
|
|
1435
1128
|
}
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1129
|
+
aiAction(taskPrompt) {
|
|
1130
|
+
return __async(this, null, function* () {
|
|
1131
|
+
const { executor } = yield this.taskExecutor.action(taskPrompt);
|
|
1132
|
+
this.appendExecutionDump(executor.dump());
|
|
1133
|
+
this.writeOutActionDumps();
|
|
1134
|
+
if (executor.isInErrorState()) {
|
|
1135
|
+
const errorTask = executor.latestErrorTask();
|
|
1136
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1443
1137
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1444
|
-
|
|
1138
|
+
}
|
|
1139
|
+
});
|
|
1445
1140
|
}
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1141
|
+
aiQuery(demand) {
|
|
1142
|
+
return __async(this, null, function* () {
|
|
1143
|
+
const { output, executor } = yield this.taskExecutor.query(demand);
|
|
1144
|
+
this.appendExecutionDump(executor.dump());
|
|
1145
|
+
this.writeOutActionDumps();
|
|
1146
|
+
if (executor.isInErrorState()) {
|
|
1147
|
+
const errorTask = executor.latestErrorTask();
|
|
1148
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1453
1149
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1454
|
-
|
|
1455
|
-
return output;
|
|
1456
|
-
}
|
|
1457
|
-
async aiAssert(assertion, msg, opt) {
|
|
1458
|
-
const { output, executor } = await this.taskExecutor.assert(assertion);
|
|
1459
|
-
this.appendExecutionDump(executor.dump());
|
|
1460
|
-
this.writeOutActionDumps();
|
|
1461
|
-
if (opt == null ? void 0 : opt.keepRawResponse) {
|
|
1150
|
+
}
|
|
1462
1151
|
return output;
|
|
1463
|
-
}
|
|
1464
|
-
if (!(output == null ? void 0 : output.pass)) {
|
|
1465
|
-
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1466
|
-
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
|
|
1467
|
-
throw new Error(`${errMsg}
|
|
1468
|
-
${reasonMsg}`);
|
|
1469
|
-
}
|
|
1152
|
+
});
|
|
1470
1153
|
}
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1154
|
+
aiAssert(assertion, msg, opt) {
|
|
1155
|
+
return __async(this, null, function* () {
|
|
1156
|
+
var _a;
|
|
1157
|
+
const { output, executor } = yield this.taskExecutor.assert(assertion);
|
|
1158
|
+
this.appendExecutionDump(executor.dump());
|
|
1159
|
+
this.writeOutActionDumps();
|
|
1160
|
+
if (opt == null ? void 0 : opt.keepRawResponse) {
|
|
1161
|
+
return output;
|
|
1162
|
+
}
|
|
1163
|
+
if (!(output == null ? void 0 : output.pass)) {
|
|
1164
|
+
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1165
|
+
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || ((_a = executor.latestErrorTask()) == null ? void 0 : _a.error) || "(no_reason)"}`;
|
|
1166
|
+
throw new Error(`${errMsg}
|
|
1167
|
+
${reasonMsg}`);
|
|
1168
|
+
}
|
|
1476
1169
|
});
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
const
|
|
1481
|
-
|
|
1170
|
+
}
|
|
1171
|
+
aiWaitFor(assertion, opt) {
|
|
1172
|
+
return __async(this, null, function* () {
|
|
1173
|
+
const { executor } = yield this.taskExecutor.waitFor(assertion, {
|
|
1174
|
+
timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
|
|
1175
|
+
checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
|
|
1176
|
+
assertion
|
|
1177
|
+
});
|
|
1178
|
+
this.appendExecutionDump(executor.dump());
|
|
1179
|
+
this.writeOutActionDumps();
|
|
1180
|
+
if (executor.isInErrorState()) {
|
|
1181
|
+
const errorTask = executor.latestErrorTask();
|
|
1182
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1482
1183
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1483
|
-
|
|
1184
|
+
}
|
|
1185
|
+
});
|
|
1484
1186
|
}
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1187
|
+
ai(taskPrompt, type = "action") {
|
|
1188
|
+
return __async(this, null, function* () {
|
|
1189
|
+
if (type === "action") {
|
|
1190
|
+
return this.aiAction(taskPrompt);
|
|
1191
|
+
}
|
|
1192
|
+
if (type === "query") {
|
|
1193
|
+
return this.aiQuery(taskPrompt);
|
|
1194
|
+
}
|
|
1195
|
+
if (type === "assert") {
|
|
1196
|
+
return this.aiAssert(taskPrompt);
|
|
1197
|
+
}
|
|
1198
|
+
throw new Error(
|
|
1199
|
+
`Unknown type: ${type}, only support 'action', 'query', 'assert'`
|
|
1200
|
+
);
|
|
1201
|
+
});
|
|
1498
1202
|
}
|
|
1499
1203
|
};
|
|
1500
1204
|
|
|
@@ -1505,19 +1209,7 @@ var StaticPageAgent = class extends PageAgent {
|
|
|
1505
1209
|
}
|
|
1506
1210
|
};
|
|
1507
1211
|
|
|
1508
|
-
// src/playground/server.ts
|
|
1509
|
-
import assert3 from "assert";
|
|
1510
|
-
import { randomUUID as randomUUID2 } from "crypto";
|
|
1511
|
-
import { existsSync as existsSync2, readFileSync as readFileSync3, writeFileSync } from "fs";
|
|
1512
|
-
import { join as join2 } from "path";
|
|
1513
|
-
var import_dotenv = __toESM(require_main());
|
|
1514
|
-
import { getTmpDir } from "@midscene/core/utils";
|
|
1515
|
-
import cors from "cors";
|
|
1516
|
-
import express from "express";
|
|
1517
|
-
|
|
1518
1212
|
// src/playground/static-page.ts
|
|
1519
|
-
import { getTmpFile } from "@midscene/core/utils";
|
|
1520
|
-
import { saveBase64Image } from "@midscene/shared/img";
|
|
1521
1213
|
var ThrowNotImplemented = (methodName) => {
|
|
1522
1214
|
throw new Error(
|
|
1523
1215
|
`The method "${methodName}" is not implemented as designed since this is a static UI context. (${ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED})`
|
|
@@ -1537,179 +1229,56 @@ var StaticPage = class {
|
|
|
1537
1229
|
};
|
|
1538
1230
|
this.uiContext = uiContext;
|
|
1539
1231
|
}
|
|
1540
|
-
|
|
1541
|
-
return
|
|
1232
|
+
getElementInfos() {
|
|
1233
|
+
return __async(this, null, function* () {
|
|
1234
|
+
return ThrowNotImplemented("getElementInfos");
|
|
1235
|
+
});
|
|
1542
1236
|
}
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1237
|
+
screenshotBase64() {
|
|
1238
|
+
return __async(this, null, function* () {
|
|
1239
|
+
const base64 = this.uiContext.screenshotBase64;
|
|
1240
|
+
if (!base64) {
|
|
1241
|
+
throw new Error("screenshot base64 is empty");
|
|
1242
|
+
}
|
|
1243
|
+
return base64;
|
|
1244
|
+
});
|
|
1551
1245
|
}
|
|
1552
1246
|
url() {
|
|
1553
1247
|
return this.uiContext.url;
|
|
1554
1248
|
}
|
|
1555
|
-
|
|
1556
|
-
return
|
|
1557
|
-
|
|
1558
|
-
async scrollUntilBottom() {
|
|
1559
|
-
return ThrowNotImplemented("scrollUntilBottom");
|
|
1560
|
-
}
|
|
1561
|
-
async scrollUpOneScreen() {
|
|
1562
|
-
return ThrowNotImplemented("scrollUpOneScreen");
|
|
1563
|
-
}
|
|
1564
|
-
async scrollDownOneScreen() {
|
|
1565
|
-
return ThrowNotImplemented("scrollDownOneScreen");
|
|
1566
|
-
}
|
|
1567
|
-
async clearInput() {
|
|
1568
|
-
return ThrowNotImplemented("clearInput");
|
|
1569
|
-
}
|
|
1570
|
-
async _forceUsePageContext() {
|
|
1571
|
-
return this.uiContext;
|
|
1572
|
-
}
|
|
1573
|
-
};
|
|
1574
|
-
|
|
1575
|
-
// src/playground/server.ts
|
|
1576
|
-
var defaultPort = 5800;
|
|
1577
|
-
var staticPath = join2(__dirname, "../../static");
|
|
1578
|
-
var agentRequestCount = 1;
|
|
1579
|
-
var errorHandler = (err, req, res, next) => {
|
|
1580
|
-
console.error(err);
|
|
1581
|
-
res.status(500).json({
|
|
1582
|
-
error: err.message
|
|
1583
|
-
});
|
|
1584
|
-
};
|
|
1585
|
-
var setup = () => {
|
|
1586
|
-
import_dotenv.default.config();
|
|
1587
|
-
};
|
|
1588
|
-
var PlaygroundServer = class {
|
|
1589
|
-
constructor() {
|
|
1590
|
-
this.app = express();
|
|
1591
|
-
this.tmpDir = getTmpDir();
|
|
1592
|
-
setup();
|
|
1593
|
-
}
|
|
1594
|
-
filePathForUuid(uuid) {
|
|
1595
|
-
return join2(this.tmpDir, `${uuid}.json`);
|
|
1596
|
-
}
|
|
1597
|
-
saveContextFile(uuid, context) {
|
|
1598
|
-
const tmpFile = this.filePathForUuid(uuid);
|
|
1599
|
-
console.log(`save context file: ${tmpFile}`);
|
|
1600
|
-
writeFileSync(tmpFile, context);
|
|
1601
|
-
return tmpFile;
|
|
1602
|
-
}
|
|
1603
|
-
async launch() {
|
|
1604
|
-
this.app.use(errorHandler);
|
|
1605
|
-
this.app.use(
|
|
1606
|
-
cors({
|
|
1607
|
-
origin: "*",
|
|
1608
|
-
credentials: true
|
|
1609
|
-
})
|
|
1610
|
-
);
|
|
1611
|
-
this.app.get("/status", cors(), async (req, res) => {
|
|
1612
|
-
res.send({
|
|
1613
|
-
status: "ok"
|
|
1614
|
-
});
|
|
1249
|
+
scrollUntilTop() {
|
|
1250
|
+
return __async(this, null, function* () {
|
|
1251
|
+
return ThrowNotImplemented("scrollUntilTop");
|
|
1615
1252
|
});
|
|
1616
|
-
|
|
1617
|
-
|
|
1253
|
+
}
|
|
1254
|
+
scrollUntilBottom() {
|
|
1255
|
+
return __async(this, null, function* () {
|
|
1256
|
+
return ThrowNotImplemented("scrollUntilBottom");
|
|
1618
1257
|
});
|
|
1619
|
-
|
|
1620
|
-
|
|
1258
|
+
}
|
|
1259
|
+
scrollUpOneScreen() {
|
|
1260
|
+
return __async(this, null, function* () {
|
|
1261
|
+
return ThrowNotImplemented("scrollUpOneScreen");
|
|
1621
1262
|
});
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
const context = readFileSync3(contextFile, "utf8");
|
|
1627
|
-
res.json({
|
|
1628
|
-
context
|
|
1629
|
-
});
|
|
1263
|
+
}
|
|
1264
|
+
scrollDownOneScreen() {
|
|
1265
|
+
return __async(this, null, function* () {
|
|
1266
|
+
return ThrowNotImplemented("scrollDownOneScreen");
|
|
1630
1267
|
});
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
const context = req.body.context;
|
|
1636
|
-
assert3(context, "context is required");
|
|
1637
|
-
const uuid = randomUUID2();
|
|
1638
|
-
this.saveContextFile(uuid, context);
|
|
1639
|
-
return res.json({
|
|
1640
|
-
location: `/playground/${uuid}`,
|
|
1641
|
-
uuid
|
|
1642
|
-
});
|
|
1643
|
-
}
|
|
1644
|
-
);
|
|
1645
|
-
this.app.post(
|
|
1646
|
-
"/execute",
|
|
1647
|
-
express.json({ limit: "30mb" }),
|
|
1648
|
-
async (req, res) => {
|
|
1649
|
-
const { context, type, prompt } = req.body;
|
|
1650
|
-
assert3(context, "context is required");
|
|
1651
|
-
assert3(type, "type is required");
|
|
1652
|
-
assert3(prompt, "prompt is required");
|
|
1653
|
-
const requestId = agentRequestCount++;
|
|
1654
|
-
console.log(`handle request: #${requestId}, ${type}, ${prompt}`);
|
|
1655
|
-
const page = new StaticPage(context);
|
|
1656
|
-
const agent = new StaticPageAgent(page);
|
|
1657
|
-
const response = {
|
|
1658
|
-
result: null,
|
|
1659
|
-
dump: null,
|
|
1660
|
-
error: null
|
|
1661
|
-
};
|
|
1662
|
-
try {
|
|
1663
|
-
if (type === "aiQuery") {
|
|
1664
|
-
response.result = await agent.aiQuery(prompt);
|
|
1665
|
-
} else if (type === "aiAction") {
|
|
1666
|
-
response.result = await agent.aiAction(prompt);
|
|
1667
|
-
} else if (type === "aiAssert") {
|
|
1668
|
-
response.result = await agent.aiAssert(prompt, void 0, {
|
|
1669
|
-
keepRawResponse: true
|
|
1670
|
-
});
|
|
1671
|
-
} else {
|
|
1672
|
-
response.error = `Unknown type: ${type}`;
|
|
1673
|
-
}
|
|
1674
|
-
} catch (error) {
|
|
1675
|
-
if (!error.message.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {
|
|
1676
|
-
response.error = error.message;
|
|
1677
|
-
}
|
|
1678
|
-
}
|
|
1679
|
-
try {
|
|
1680
|
-
response.dump = JSON.parse(agent.dumpDataString());
|
|
1681
|
-
agent.writeOutActionDumps();
|
|
1682
|
-
} catch (error) {
|
|
1683
|
-
console.error(
|
|
1684
|
-
`write out dump failed: #${requestId}, ${error.message}`
|
|
1685
|
-
);
|
|
1686
|
-
}
|
|
1687
|
-
res.send(response);
|
|
1688
|
-
if (response.error) {
|
|
1689
|
-
console.error(
|
|
1690
|
-
`handle request failed: #${requestId}, ${response.error}`
|
|
1691
|
-
);
|
|
1692
|
-
} else {
|
|
1693
|
-
console.log(`handle request done: #${requestId}`);
|
|
1694
|
-
}
|
|
1695
|
-
}
|
|
1696
|
-
);
|
|
1697
|
-
return new Promise((resolve, reject) => {
|
|
1698
|
-
const port = this.port || defaultPort;
|
|
1699
|
-
this.server = this.app.listen(port, () => {
|
|
1700
|
-
this.port = port;
|
|
1701
|
-
resolve(this);
|
|
1702
|
-
});
|
|
1268
|
+
}
|
|
1269
|
+
clearInput() {
|
|
1270
|
+
return __async(this, null, function* () {
|
|
1271
|
+
return ThrowNotImplemented("clearInput");
|
|
1703
1272
|
});
|
|
1704
1273
|
}
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
return this.
|
|
1708
|
-
}
|
|
1274
|
+
_forceUsePageContext() {
|
|
1275
|
+
return __async(this, null, function* () {
|
|
1276
|
+
return this.uiContext;
|
|
1277
|
+
});
|
|
1709
1278
|
}
|
|
1710
1279
|
};
|
|
1711
1280
|
export {
|
|
1712
|
-
|
|
1281
|
+
ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED,
|
|
1713
1282
|
StaticPage,
|
|
1714
1283
|
StaticPageAgent
|
|
1715
1284
|
};
|