@midscene/web 0.7.1 → 0.7.2-beta-20241024094141.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/playground.js +8438 -0
- package/dist/browser/types/playground.d.ts +313 -0
- package/dist/es/appium.js +680 -604
- package/dist/es/debug.js +95 -73
- package/dist/es/index.js +939 -797
- package/dist/es/midscene-playground.js +678 -609
- package/dist/es/playground.js +593 -1024
- package/dist/es/playwright-report.js +29 -11
- package/dist/es/playwright.js +705 -597
- package/dist/es/puppeteer.js +636 -552
- package/dist/lib/appium.js +688 -609
- package/dist/lib/debug.js +95 -73
- package/dist/lib/index.js +950 -804
- package/dist/lib/midscene-playground.js +687 -615
- package/dist/lib/playground.js +586 -1007
- package/dist/lib/playwright-report.js +30 -9
- package/dist/lib/playwright.js +713 -602
- package/dist/lib/puppeteer.js +644 -557
- package/dist/script/htmlElement.js +11 -10
- package/dist/script/htmlElementDebug.js +11 -10
- package/dist/types/appium.d.ts +2 -3
- package/dist/types/debug.d.ts +1 -1
- package/dist/types/index.d.ts +3 -3
- package/dist/types/{page-ad820b3c.d.ts → page-8117b0ad.d.ts} +8 -7
- package/dist/types/playground.d.ts +6 -21
- package/dist/types/playwright.d.ts +3 -4
- package/dist/types/puppeteer.d.ts +2 -3
- package/dist/types/{tasks-82c1054b.d.ts → tasks-cb6bf758.d.ts} +6 -6
- package/package.json +11 -5
- package/static/index.html +1 -1
package/dist/es/playwright.js
CHANGED
|
@@ -40,6 +40,26 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
40
40
|
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
41
41
|
mod
|
|
42
42
|
));
|
|
43
|
+
var __async = (__this, __arguments, generator) => {
|
|
44
|
+
return new Promise((resolve, reject) => {
|
|
45
|
+
var fulfilled = (value) => {
|
|
46
|
+
try {
|
|
47
|
+
step(generator.next(value));
|
|
48
|
+
} catch (e) {
|
|
49
|
+
reject(e);
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
var rejected = (value) => {
|
|
53
|
+
try {
|
|
54
|
+
step(generator.throw(value));
|
|
55
|
+
} catch (e) {
|
|
56
|
+
reject(e);
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
|
|
60
|
+
step((generator = generator.apply(__this, __arguments)).next());
|
|
61
|
+
});
|
|
62
|
+
};
|
|
43
63
|
|
|
44
64
|
// ../../node_modules/.pnpm/dayjs@1.11.11/node_modules/dayjs/dayjs.min.js
|
|
45
65
|
var require_dayjs_min = __commonJS({
|
|
@@ -337,7 +357,7 @@ var require_dayjs_min = __commonJS({
|
|
|
337
357
|
});
|
|
338
358
|
|
|
339
359
|
// src/playwright/ai-fixture.ts
|
|
340
|
-
import { randomUUID
|
|
360
|
+
import { randomUUID } from "crypto";
|
|
341
361
|
|
|
342
362
|
// src/common/agent.ts
|
|
343
363
|
import {
|
|
@@ -354,7 +374,6 @@ import {
|
|
|
354
374
|
plan
|
|
355
375
|
} from "@midscene/core";
|
|
356
376
|
import { sleep } from "@midscene/core/utils";
|
|
357
|
-
import { base64Encoded as base64Encoded2 } from "@midscene/shared/img";
|
|
358
377
|
|
|
359
378
|
// src/common/task-cache.ts
|
|
360
379
|
import { existsSync, readFileSync as readFileSync2 } from "fs";
|
|
@@ -364,21 +383,19 @@ import {
|
|
|
364
383
|
stringifyDumpData,
|
|
365
384
|
writeLogFile
|
|
366
385
|
} from "@midscene/core/utils";
|
|
367
|
-
import {
|
|
386
|
+
import { getRunningPkgInfo } from "@midscene/shared/fs";
|
|
387
|
+
import { ifInBrowser } from "@midscene/shared/utils";
|
|
368
388
|
|
|
369
389
|
// src/common/utils.ts
|
|
370
390
|
var import_dayjs = __toESM(require_dayjs_min());
|
|
371
391
|
import assert from "assert";
|
|
372
|
-
import { randomUUID } from "crypto";
|
|
373
392
|
import { readFileSync } from "fs";
|
|
374
393
|
import path from "path";
|
|
375
394
|
import { NodeType } from "@midscene/shared/constants";
|
|
376
395
|
import { findNearestPackageJson } from "@midscene/shared/fs";
|
|
377
|
-
import {
|
|
378
|
-
base64Encoded,
|
|
379
|
-
imageInfoOfBase64
|
|
380
|
-
} from "@midscene/shared/img";
|
|
396
|
+
import { imageInfoOfBase64 } from "@midscene/shared/img";
|
|
381
397
|
import { compositeElementInfoImg } from "@midscene/shared/img";
|
|
398
|
+
import { uuid } from "@midscene/shared/utils";
|
|
382
399
|
|
|
383
400
|
// src/web-element.ts
|
|
384
401
|
var WebElementInfo = class {
|
|
@@ -406,63 +423,68 @@ var WebElementInfo = class {
|
|
|
406
423
|
};
|
|
407
424
|
|
|
408
425
|
// src/common/utils.ts
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
const url = page.url();
|
|
415
|
-
const file = await page.screenshot();
|
|
416
|
-
const screenshotBase64 = base64Encoded(file);
|
|
417
|
-
const captureElementSnapshot = await page.getElementInfos();
|
|
418
|
-
const elementsInfo = await alignElements(captureElementSnapshot, page);
|
|
419
|
-
const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
|
|
420
|
-
if (elementInfo.attributes.nodeType === NodeType.TEXT) {
|
|
421
|
-
return false;
|
|
426
|
+
function parseContextFromWebPage(page, _opt) {
|
|
427
|
+
return __async(this, null, function* () {
|
|
428
|
+
assert(page, "page is required");
|
|
429
|
+
if (page._forceUsePageContext) {
|
|
430
|
+
return yield page._forceUsePageContext();
|
|
422
431
|
}
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
432
|
+
const url = page.url();
|
|
433
|
+
const screenshotBase64 = yield page.screenshotBase64();
|
|
434
|
+
const captureElementSnapshot = yield page.getElementInfos();
|
|
435
|
+
const elementsInfo = yield alignElements(captureElementSnapshot, page);
|
|
436
|
+
const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
|
|
437
|
+
if (elementInfo.attributes.nodeType === NodeType.TEXT) {
|
|
438
|
+
return false;
|
|
439
|
+
}
|
|
440
|
+
return true;
|
|
441
|
+
});
|
|
442
|
+
const size = yield imageInfoOfBase64(screenshotBase64);
|
|
443
|
+
const screenshotBase64WithElementMarker = yield compositeElementInfoImg({
|
|
444
|
+
inputImgBase64: screenshotBase64,
|
|
445
|
+
elementsPositionInfo: elementsPositionInfoWithoutText
|
|
446
|
+
});
|
|
447
|
+
return {
|
|
448
|
+
content: elementsInfo,
|
|
449
|
+
size,
|
|
450
|
+
screenshotBase64,
|
|
451
|
+
screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
|
|
452
|
+
url
|
|
453
|
+
};
|
|
429
454
|
});
|
|
430
|
-
return {
|
|
431
|
-
content: elementsInfo,
|
|
432
|
-
size,
|
|
433
|
-
screenshotBase64,
|
|
434
|
-
screenshotBase64WithElementMarker: `data:image/png;base64,${screenshotBase64WithElementMarker}`,
|
|
435
|
-
url
|
|
436
|
-
};
|
|
437
455
|
}
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
456
|
+
function getExtraReturnLogic() {
|
|
457
|
+
return __async(this, null, function* () {
|
|
458
|
+
const pathDir = findNearestPackageJson(__dirname);
|
|
459
|
+
assert(pathDir, `can't find pathDir, with ${__dirname}`);
|
|
460
|
+
const scriptPath = path.join(pathDir, "./dist/script/htmlElement.js");
|
|
461
|
+
const elementInfosScriptContent = readFileSync(scriptPath, "utf-8");
|
|
462
|
+
return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
|
|
463
|
+
});
|
|
444
464
|
}
|
|
445
465
|
var sizeThreshold = 3;
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
466
|
+
function alignElements(elements, page) {
|
|
467
|
+
return __async(this, null, function* () {
|
|
468
|
+
const validElements = elements.filter((item) => {
|
|
469
|
+
return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
|
|
470
|
+
});
|
|
471
|
+
const textsAligned = [];
|
|
472
|
+
for (const item of validElements) {
|
|
473
|
+
const { rect, id, content, attributes, locator, indexId } = item;
|
|
474
|
+
textsAligned.push(
|
|
475
|
+
new WebElementInfo({
|
|
476
|
+
rect,
|
|
477
|
+
locator,
|
|
478
|
+
id,
|
|
479
|
+
content,
|
|
480
|
+
attributes,
|
|
481
|
+
page,
|
|
482
|
+
indexId
|
|
483
|
+
})
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
return textsAligned;
|
|
449
487
|
});
|
|
450
|
-
const textsAligned = [];
|
|
451
|
-
for (const item of validElements) {
|
|
452
|
-
const { rect, id, content, attributes, locator, indexId } = item;
|
|
453
|
-
textsAligned.push(
|
|
454
|
-
new WebElementInfo({
|
|
455
|
-
rect,
|
|
456
|
-
locator,
|
|
457
|
-
id,
|
|
458
|
-
content,
|
|
459
|
-
attributes,
|
|
460
|
-
page,
|
|
461
|
-
indexId
|
|
462
|
-
})
|
|
463
|
-
);
|
|
464
|
-
}
|
|
465
|
-
return textsAligned;
|
|
466
488
|
}
|
|
467
489
|
function reportFileName(tag = "web") {
|
|
468
490
|
const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
|
|
@@ -493,7 +515,7 @@ var testFileIndex = /* @__PURE__ */ new Map();
|
|
|
493
515
|
function generateCacheId(fileName) {
|
|
494
516
|
let taskFile = fileName || getCurrentExecutionFile();
|
|
495
517
|
if (!taskFile) {
|
|
496
|
-
taskFile =
|
|
518
|
+
taskFile = uuid();
|
|
497
519
|
console.warn(
|
|
498
520
|
"Midscene - using random UUID for cache id. Cache may be invalid."
|
|
499
521
|
);
|
|
@@ -512,7 +534,7 @@ function generateCacheId(fileName) {
|
|
|
512
534
|
// src/common/task-cache.ts
|
|
513
535
|
var TaskCache = class {
|
|
514
536
|
constructor(opts) {
|
|
515
|
-
this.midscenePkgInfo =
|
|
537
|
+
this.midscenePkgInfo = getRunningPkgInfo();
|
|
516
538
|
this.cacheId = generateCacheId(opts == null ? void 0 : opts.fileName);
|
|
517
539
|
this.cache = this.readCacheFromFile() || {
|
|
518
540
|
aiTasks: []
|
|
@@ -594,11 +616,17 @@ var TaskCache = class {
|
|
|
594
616
|
return this.newCache;
|
|
595
617
|
}
|
|
596
618
|
readCacheFromFile() {
|
|
619
|
+
if (ifInBrowser) {
|
|
620
|
+
return void 0;
|
|
621
|
+
}
|
|
597
622
|
const cacheFile = join(getLogDirByType("cache"), `${this.cacheId}.json`);
|
|
598
623
|
if (process.env.MIDSCENE_CACHE === "true" && existsSync(cacheFile)) {
|
|
599
624
|
try {
|
|
600
625
|
const data = readFileSync2(cacheFile, "utf8");
|
|
601
626
|
const jsonData = JSON.parse(data);
|
|
627
|
+
if (!this.midscenePkgInfo) {
|
|
628
|
+
return void 0;
|
|
629
|
+
}
|
|
602
630
|
if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
|
|
603
631
|
return void 0;
|
|
604
632
|
}
|
|
@@ -610,7 +638,10 @@ var TaskCache = class {
|
|
|
610
638
|
return void 0;
|
|
611
639
|
}
|
|
612
640
|
writeCacheToFile() {
|
|
613
|
-
const midscenePkgInfo =
|
|
641
|
+
const midscenePkgInfo = getRunningPkgInfo();
|
|
642
|
+
if (!midscenePkgInfo) {
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
614
645
|
writeLogFile({
|
|
615
646
|
fileName: `${this.cacheId}`,
|
|
616
647
|
fileExt: "json",
|
|
@@ -631,423 +662,435 @@ var TaskCache = class {
|
|
|
631
662
|
var PageTaskExecutor = class {
|
|
632
663
|
constructor(page, opts) {
|
|
633
664
|
this.page = page;
|
|
634
|
-
this.insight = new Insight(
|
|
635
|
-
return
|
|
636
|
-
});
|
|
665
|
+
this.insight = new Insight(() => __async(this, null, function* () {
|
|
666
|
+
return yield parseContextFromWebPage(page);
|
|
667
|
+
}));
|
|
637
668
|
this.taskCache = new TaskCache({
|
|
638
669
|
fileName: opts == null ? void 0 : opts.cacheId
|
|
639
670
|
});
|
|
640
671
|
}
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
672
|
+
recordScreenshot(timing) {
|
|
673
|
+
return __async(this, null, function* () {
|
|
674
|
+
const base64 = yield this.page.screenshotBase64();
|
|
675
|
+
const item = {
|
|
676
|
+
type: "screenshot",
|
|
677
|
+
ts: Date.now(),
|
|
678
|
+
screenshot: base64,
|
|
679
|
+
timing
|
|
680
|
+
};
|
|
681
|
+
return item;
|
|
682
|
+
});
|
|
650
683
|
}
|
|
651
684
|
wrapExecutorWithScreenshot(taskApply) {
|
|
652
685
|
const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
|
|
653
|
-
executor:
|
|
686
|
+
executor: (param, context, ...args) => __async(this, null, function* () {
|
|
654
687
|
const recorder = [];
|
|
655
688
|
const { task } = context;
|
|
656
689
|
task.recorder = recorder;
|
|
657
|
-
const shot =
|
|
690
|
+
const shot = yield this.recordScreenshot(`before ${task.type}`);
|
|
658
691
|
recorder.push(shot);
|
|
659
|
-
const result =
|
|
692
|
+
const result = yield taskApply.executor(param, context, ...args);
|
|
660
693
|
if (taskApply.type === "Action") {
|
|
661
|
-
|
|
662
|
-
const shot2 =
|
|
694
|
+
yield sleep(1e3);
|
|
695
|
+
const shot2 = yield this.recordScreenshot("after Action");
|
|
663
696
|
recorder.push(shot2);
|
|
664
697
|
}
|
|
665
698
|
return result;
|
|
666
|
-
}
|
|
699
|
+
})
|
|
667
700
|
});
|
|
668
701
|
return taskWithScreenshot;
|
|
669
702
|
}
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
684
|
-
const pageContext = await this.insight.contextRetrieverFn();
|
|
685
|
-
const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
|
|
686
|
-
pageContext,
|
|
687
|
-
"locate",
|
|
688
|
-
param.prompt
|
|
689
|
-
);
|
|
690
|
-
let locateResult;
|
|
691
|
-
const callAI = this.insight.aiVendorFn;
|
|
692
|
-
const element = await this.insight.locate(param.prompt, {
|
|
693
|
-
quickAnswer: plan2.quickAnswer,
|
|
694
|
-
callAI: async (...message) => {
|
|
695
|
-
if (locateCache) {
|
|
696
|
-
locateResult = locateCache;
|
|
697
|
-
return Promise.resolve(locateCache);
|
|
698
|
-
}
|
|
699
|
-
locateResult = await callAI(...message);
|
|
700
|
-
assert2(locateResult);
|
|
701
|
-
return locateResult;
|
|
702
|
-
}
|
|
703
|
-
});
|
|
704
|
-
if (locateResult) {
|
|
705
|
-
cacheGroup == null ? void 0 : cacheGroup.saveCache({
|
|
706
|
-
type: "locate",
|
|
707
|
-
pageContext: {
|
|
708
|
-
url: pageContext.url,
|
|
709
|
-
size: pageContext.size
|
|
710
|
-
},
|
|
711
|
-
prompt: param.prompt,
|
|
712
|
-
response: locateResult
|
|
713
|
-
});
|
|
714
|
-
}
|
|
715
|
-
if (!element) {
|
|
716
|
-
task.log = {
|
|
717
|
-
dump: insightDump
|
|
703
|
+
convertPlanToExecutable(plans, cacheGroup) {
|
|
704
|
+
return __async(this, null, function* () {
|
|
705
|
+
const tasks = plans.map((plan2) => {
|
|
706
|
+
if (plan2.type === "Locate") {
|
|
707
|
+
const taskFind = {
|
|
708
|
+
type: "Insight",
|
|
709
|
+
subType: "Locate",
|
|
710
|
+
param: plan2.param,
|
|
711
|
+
executor: (param, taskContext) => __async(this, null, function* () {
|
|
712
|
+
const { task } = taskContext;
|
|
713
|
+
let insightDump;
|
|
714
|
+
const dumpCollector = (dump) => {
|
|
715
|
+
insightDump = dump;
|
|
718
716
|
};
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
717
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
718
|
+
const pageContext = yield this.insight.contextRetrieverFn();
|
|
719
|
+
const locateCache = cacheGroup == null ? void 0 : cacheGroup.readCache(
|
|
720
|
+
pageContext,
|
|
721
|
+
"locate",
|
|
722
|
+
param.prompt
|
|
723
|
+
);
|
|
724
|
+
let locateResult;
|
|
725
|
+
const callAI = this.insight.aiVendorFn;
|
|
726
|
+
const element = yield this.insight.locate(param.prompt, {
|
|
727
|
+
quickAnswer: plan2.quickAnswer,
|
|
728
|
+
callAI: (...message) => __async(this, null, function* () {
|
|
729
|
+
if (locateCache) {
|
|
730
|
+
locateResult = locateCache;
|
|
731
|
+
return Promise.resolve(locateCache);
|
|
732
|
+
}
|
|
733
|
+
locateResult = yield callAI(...message);
|
|
734
|
+
assert2(locateResult);
|
|
735
|
+
return locateResult;
|
|
736
|
+
})
|
|
737
|
+
});
|
|
738
|
+
if (locateResult) {
|
|
739
|
+
cacheGroup == null ? void 0 : cacheGroup.saveCache({
|
|
740
|
+
type: "locate",
|
|
741
|
+
pageContext: {
|
|
742
|
+
url: pageContext.url,
|
|
743
|
+
size: pageContext.size
|
|
744
|
+
},
|
|
745
|
+
prompt: param.prompt,
|
|
746
|
+
response: locateResult
|
|
747
|
+
});
|
|
730
748
|
}
|
|
731
|
-
|
|
732
|
-
}
|
|
733
|
-
};
|
|
734
|
-
return taskFind;
|
|
735
|
-
}
|
|
736
|
-
if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
|
|
737
|
-
const assertPlan = plan2;
|
|
738
|
-
const taskAssert = {
|
|
739
|
-
type: "Insight",
|
|
740
|
-
subType: "Assert",
|
|
741
|
-
param: assertPlan.param,
|
|
742
|
-
executor: async (param, taskContext) => {
|
|
743
|
-
const { task } = taskContext;
|
|
744
|
-
let insightDump;
|
|
745
|
-
const dumpCollector = (dump) => {
|
|
746
|
-
insightDump = dump;
|
|
747
|
-
};
|
|
748
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
749
|
-
const assertion = await this.insight.assert(
|
|
750
|
-
assertPlan.param.assertion
|
|
751
|
-
);
|
|
752
|
-
if (!assertion.pass) {
|
|
753
|
-
if (plan2.type === "Assert") {
|
|
754
|
-
task.output = assertion;
|
|
749
|
+
if (!element) {
|
|
755
750
|
task.log = {
|
|
756
751
|
dump: insightDump
|
|
757
752
|
};
|
|
758
|
-
throw new Error(
|
|
759
|
-
assertion.thought || "Assertion failed without reason"
|
|
760
|
-
);
|
|
753
|
+
throw new Error(`Element not found: ${param.prompt}`);
|
|
761
754
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
755
|
+
return {
|
|
756
|
+
output: {
|
|
757
|
+
element
|
|
758
|
+
},
|
|
759
|
+
log: {
|
|
760
|
+
dump: insightDump
|
|
761
|
+
},
|
|
762
|
+
cache: {
|
|
763
|
+
hit: Boolean(locateCache)
|
|
764
|
+
}
|
|
765
|
+
};
|
|
766
|
+
})
|
|
767
|
+
};
|
|
768
|
+
return taskFind;
|
|
769
|
+
}
|
|
770
|
+
if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
|
|
771
|
+
const assertPlan = plan2;
|
|
772
|
+
const taskAssert = {
|
|
773
|
+
type: "Insight",
|
|
774
|
+
subType: "Assert",
|
|
775
|
+
param: assertPlan.param,
|
|
776
|
+
executor: (param, taskContext) => __async(this, null, function* () {
|
|
777
|
+
const { task } = taskContext;
|
|
778
|
+
let insightDump;
|
|
779
|
+
const dumpCollector = (dump) => {
|
|
780
|
+
insightDump = dump;
|
|
781
|
+
};
|
|
782
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
783
|
+
const assertion = yield this.insight.assert(
|
|
784
|
+
assertPlan.param.assertion
|
|
785
|
+
);
|
|
786
|
+
if (!assertion.pass) {
|
|
787
|
+
if (plan2.type === "Assert") {
|
|
788
|
+
task.output = assertion;
|
|
789
|
+
task.log = {
|
|
790
|
+
dump: insightDump
|
|
791
|
+
};
|
|
792
|
+
throw new Error(
|
|
793
|
+
assertion.thought || "Assertion failed without reason"
|
|
794
|
+
);
|
|
795
|
+
}
|
|
796
|
+
task.error = assertion.thought;
|
|
768
797
|
}
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
798
|
+
return {
|
|
799
|
+
output: assertion,
|
|
800
|
+
log: {
|
|
801
|
+
dump: insightDump
|
|
802
|
+
}
|
|
803
|
+
};
|
|
804
|
+
})
|
|
805
|
+
};
|
|
806
|
+
return taskAssert;
|
|
807
|
+
}
|
|
808
|
+
if (plan2.type === "Input") {
|
|
809
|
+
const taskActionInput = {
|
|
810
|
+
type: "Action",
|
|
811
|
+
subType: "Input",
|
|
812
|
+
param: plan2.param,
|
|
813
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (taskParam, { element }) {
|
|
814
|
+
if (element) {
|
|
815
|
+
yield this.page.clearInput(element);
|
|
816
|
+
if (taskParam.value === "") {
|
|
817
|
+
return;
|
|
818
|
+
}
|
|
819
|
+
yield this.page.keyboard.type(taskParam.value);
|
|
784
820
|
}
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
}
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
}
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
)
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
}
|
|
885
|
-
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
return
|
|
821
|
+
})
|
|
822
|
+
};
|
|
823
|
+
return taskActionInput;
|
|
824
|
+
}
|
|
825
|
+
if (plan2.type === "KeyboardPress") {
|
|
826
|
+
const taskActionKeyboardPress = {
|
|
827
|
+
type: "Action",
|
|
828
|
+
subType: "KeyboardPress",
|
|
829
|
+
param: plan2.param,
|
|
830
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
831
|
+
assert2(taskParam.value, "No key to press");
|
|
832
|
+
yield this.page.keyboard.press(taskParam.value);
|
|
833
|
+
})
|
|
834
|
+
};
|
|
835
|
+
return taskActionKeyboardPress;
|
|
836
|
+
}
|
|
837
|
+
if (plan2.type === "Tap") {
|
|
838
|
+
const taskActionTap = {
|
|
839
|
+
type: "Action",
|
|
840
|
+
subType: "Tap",
|
|
841
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
|
|
842
|
+
assert2(element, "Element not found, cannot tap");
|
|
843
|
+
yield this.page.mouse.click(
|
|
844
|
+
element.center[0],
|
|
845
|
+
element.center[1]
|
|
846
|
+
);
|
|
847
|
+
})
|
|
848
|
+
};
|
|
849
|
+
return taskActionTap;
|
|
850
|
+
}
|
|
851
|
+
if (plan2.type === "Hover") {
|
|
852
|
+
const taskActionHover = {
|
|
853
|
+
type: "Action",
|
|
854
|
+
subType: "Hover",
|
|
855
|
+
executor: (_0, _1) => __async(this, [_0, _1], function* (param, { element }) {
|
|
856
|
+
assert2(element, "Element not found, cannot hover");
|
|
857
|
+
yield this.page.mouse.move(
|
|
858
|
+
element.center[0],
|
|
859
|
+
element.center[1]
|
|
860
|
+
);
|
|
861
|
+
})
|
|
862
|
+
};
|
|
863
|
+
return taskActionHover;
|
|
864
|
+
}
|
|
865
|
+
if (plan2.type === "Scroll") {
|
|
866
|
+
const taskActionScroll = {
|
|
867
|
+
type: "Action",
|
|
868
|
+
subType: "Scroll",
|
|
869
|
+
param: plan2.param,
|
|
870
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
871
|
+
const scrollToEventName = taskParam.scrollType;
|
|
872
|
+
switch (scrollToEventName) {
|
|
873
|
+
case "scrollUntilTop":
|
|
874
|
+
yield this.page.scrollUntilTop();
|
|
875
|
+
break;
|
|
876
|
+
case "scrollUntilBottom":
|
|
877
|
+
yield this.page.scrollUntilBottom();
|
|
878
|
+
break;
|
|
879
|
+
case "scrollUpOneScreen":
|
|
880
|
+
yield this.page.scrollUpOneScreen();
|
|
881
|
+
break;
|
|
882
|
+
case "scrollDownOneScreen":
|
|
883
|
+
yield this.page.scrollDownOneScreen();
|
|
884
|
+
break;
|
|
885
|
+
default:
|
|
886
|
+
console.error(
|
|
887
|
+
"Unknown scroll event type:",
|
|
888
|
+
scrollToEventName
|
|
889
|
+
);
|
|
890
|
+
}
|
|
891
|
+
})
|
|
892
|
+
};
|
|
893
|
+
return taskActionScroll;
|
|
894
|
+
}
|
|
895
|
+
if (plan2.type === "Sleep") {
|
|
896
|
+
const taskActionSleep = {
|
|
897
|
+
type: "Action",
|
|
898
|
+
subType: "Sleep",
|
|
899
|
+
param: plan2.param,
|
|
900
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
901
|
+
yield sleep(taskParam.timeMs || 3e3);
|
|
902
|
+
})
|
|
903
|
+
};
|
|
904
|
+
return taskActionSleep;
|
|
905
|
+
}
|
|
906
|
+
if (plan2.type === "Error") {
|
|
907
|
+
const taskActionError = {
|
|
908
|
+
type: "Action",
|
|
909
|
+
subType: "Error",
|
|
910
|
+
param: plan2.param,
|
|
911
|
+
executor: (taskParam) => __async(this, null, function* () {
|
|
912
|
+
assert2(
|
|
913
|
+
taskParam.thought,
|
|
914
|
+
"An error occurred, but no thought provided"
|
|
915
|
+
);
|
|
916
|
+
throw new Error(taskParam.thought);
|
|
917
|
+
})
|
|
918
|
+
};
|
|
919
|
+
return taskActionError;
|
|
920
|
+
}
|
|
921
|
+
throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
|
|
922
|
+
}).map((task) => {
|
|
923
|
+
return this.wrapExecutorWithScreenshot(task);
|
|
924
|
+
});
|
|
925
|
+
return tasks;
|
|
890
926
|
});
|
|
891
|
-
return tasks;
|
|
892
927
|
}
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
928
|
+
action(userPrompt) {
|
|
929
|
+
return __async(this, null, function* () {
|
|
930
|
+
const taskExecutor = new Executor(userPrompt);
|
|
931
|
+
const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
|
|
932
|
+
let plans = [];
|
|
933
|
+
const planningTask = {
|
|
934
|
+
type: "Planning",
|
|
935
|
+
param: {
|
|
936
|
+
userPrompt
|
|
937
|
+
},
|
|
938
|
+
executor: (param) => __async(this, null, function* () {
|
|
939
|
+
const pageContext = yield this.insight.contextRetrieverFn();
|
|
940
|
+
let planResult;
|
|
941
|
+
const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
|
|
942
|
+
if (planCache) {
|
|
943
|
+
planResult = planCache;
|
|
944
|
+
} else {
|
|
945
|
+
planResult = yield plan(param.userPrompt, {
|
|
946
|
+
context: pageContext
|
|
947
|
+
});
|
|
948
|
+
}
|
|
949
|
+
assert2(planResult.plans.length > 0, "No plans found");
|
|
950
|
+
plans = planResult.plans;
|
|
951
|
+
cacheGroup.saveCache({
|
|
952
|
+
type: "plan",
|
|
953
|
+
pageContext: {
|
|
954
|
+
url: pageContext.url,
|
|
955
|
+
size: pageContext.size
|
|
956
|
+
},
|
|
957
|
+
prompt: userPrompt,
|
|
958
|
+
response: planResult
|
|
911
959
|
});
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
});
|
|
960
|
+
return {
|
|
961
|
+
output: planResult,
|
|
962
|
+
cache: {
|
|
963
|
+
hit: Boolean(planCache)
|
|
964
|
+
}
|
|
965
|
+
};
|
|
966
|
+
})
|
|
967
|
+
};
|
|
968
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
|
|
969
|
+
let output = yield taskExecutor.flush();
|
|
970
|
+
if (taskExecutor.isInErrorState()) {
|
|
924
971
|
return {
|
|
925
|
-
output
|
|
926
|
-
|
|
927
|
-
hit: Boolean(planCache)
|
|
928
|
-
}
|
|
972
|
+
output,
|
|
973
|
+
executor: taskExecutor
|
|
929
974
|
};
|
|
930
975
|
}
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
if (taskExecutor.isInErrorState()) {
|
|
976
|
+
const executables = yield this.convertPlanToExecutable(plans, cacheGroup);
|
|
977
|
+
yield taskExecutor.append(executables);
|
|
978
|
+
output = yield taskExecutor.flush();
|
|
935
979
|
return {
|
|
936
980
|
output,
|
|
937
981
|
executor: taskExecutor
|
|
938
982
|
};
|
|
939
|
-
}
|
|
940
|
-
const executables = await this.convertPlanToExecutable(plans, cacheGroup);
|
|
941
|
-
await taskExecutor.append(executables);
|
|
942
|
-
output = await taskExecutor.flush();
|
|
943
|
-
return {
|
|
944
|
-
output,
|
|
945
|
-
executor: taskExecutor
|
|
946
|
-
};
|
|
947
|
-
}
|
|
948
|
-
async query(demand) {
|
|
949
|
-
const description = typeof demand === "string" ? demand : JSON.stringify(demand);
|
|
950
|
-
const taskExecutor = new Executor(description);
|
|
951
|
-
const queryTask = {
|
|
952
|
-
type: "Insight",
|
|
953
|
-
subType: "Query",
|
|
954
|
-
param: {
|
|
955
|
-
dataDemand: demand
|
|
956
|
-
},
|
|
957
|
-
executor: async (param) => {
|
|
958
|
-
let insightDump;
|
|
959
|
-
const dumpCollector = (dump) => {
|
|
960
|
-
insightDump = dump;
|
|
961
|
-
};
|
|
962
|
-
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
963
|
-
const data = await this.insight.extract(param.dataDemand);
|
|
964
|
-
return {
|
|
965
|
-
output: data,
|
|
966
|
-
log: { dump: insightDump }
|
|
967
|
-
};
|
|
968
|
-
}
|
|
969
|
-
};
|
|
970
|
-
await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
|
|
971
|
-
const output = await taskExecutor.flush();
|
|
972
|
-
return {
|
|
973
|
-
output,
|
|
974
|
-
executor: taskExecutor
|
|
975
|
-
};
|
|
983
|
+
});
|
|
976
984
|
}
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
985
|
+
query(demand) {
|
|
986
|
+
return __async(this, null, function* () {
|
|
987
|
+
const description = typeof demand === "string" ? demand : JSON.stringify(demand);
|
|
988
|
+
const taskExecutor = new Executor(description);
|
|
989
|
+
const queryTask = {
|
|
990
|
+
type: "Insight",
|
|
991
|
+
subType: "Query",
|
|
992
|
+
param: {
|
|
993
|
+
dataDemand: demand
|
|
994
|
+
},
|
|
995
|
+
executor: (param) => __async(this, null, function* () {
|
|
996
|
+
let insightDump;
|
|
997
|
+
const dumpCollector = (dump) => {
|
|
998
|
+
insightDump = dump;
|
|
999
|
+
};
|
|
1000
|
+
this.insight.onceDumpUpdatedFn = dumpCollector;
|
|
1001
|
+
const data = yield this.insight.extract(param.dataDemand);
|
|
1002
|
+
return {
|
|
1003
|
+
output: data,
|
|
1004
|
+
log: { dump: insightDump }
|
|
1005
|
+
};
|
|
1006
|
+
})
|
|
1007
|
+
};
|
|
1008
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
|
|
1009
|
+
const output = yield taskExecutor.flush();
|
|
1010
|
+
return {
|
|
1011
|
+
output,
|
|
1012
|
+
executor: taskExecutor
|
|
1013
|
+
};
|
|
1014
|
+
});
|
|
993
1015
|
}
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
assert2(checkIntervalMs, "No checkIntervalMs for waitFor");
|
|
1001
|
-
const overallStartTime = Date.now();
|
|
1002
|
-
let startTime = Date.now();
|
|
1003
|
-
let errorThought = "";
|
|
1004
|
-
while (Date.now() - overallStartTime < timeoutMs) {
|
|
1005
|
-
startTime = Date.now();
|
|
1006
|
-
const assertPlan = {
|
|
1007
|
-
type: "AssertWithoutThrow",
|
|
1016
|
+
assert(assertion) {
|
|
1017
|
+
return __async(this, null, function* () {
|
|
1018
|
+
const description = `assert: ${assertion}`;
|
|
1019
|
+
const taskExecutor = new Executor(description);
|
|
1020
|
+
const assertionPlan = {
|
|
1021
|
+
type: "Assert",
|
|
1008
1022
|
param: {
|
|
1009
1023
|
assertion
|
|
1010
1024
|
}
|
|
1011
1025
|
};
|
|
1012
|
-
const assertTask =
|
|
1013
|
-
|
|
1014
|
-
const output =
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1026
|
+
const assertTask = yield this.convertPlanToExecutable([assertionPlan]);
|
|
1027
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
|
|
1028
|
+
const output = yield taskExecutor.flush();
|
|
1029
|
+
return {
|
|
1030
|
+
output,
|
|
1031
|
+
executor: taskExecutor
|
|
1032
|
+
};
|
|
1033
|
+
});
|
|
1034
|
+
}
|
|
1035
|
+
waitFor(assertion, opt) {
|
|
1036
|
+
return __async(this, null, function* () {
|
|
1037
|
+
const description = `waitFor: ${assertion}`;
|
|
1038
|
+
const taskExecutor = new Executor(description);
|
|
1039
|
+
const { timeoutMs, checkIntervalMs } = opt;
|
|
1040
|
+
assert2(assertion, "No assertion for waitFor");
|
|
1041
|
+
assert2(timeoutMs, "No timeoutMs for waitFor");
|
|
1042
|
+
assert2(checkIntervalMs, "No checkIntervalMs for waitFor");
|
|
1043
|
+
const overallStartTime = Date.now();
|
|
1044
|
+
let startTime = Date.now();
|
|
1045
|
+
let errorThought = "";
|
|
1046
|
+
while (Date.now() - overallStartTime < timeoutMs) {
|
|
1047
|
+
startTime = Date.now();
|
|
1048
|
+
const assertPlan = {
|
|
1049
|
+
type: "AssertWithoutThrow",
|
|
1027
1050
|
param: {
|
|
1028
|
-
|
|
1051
|
+
assertion
|
|
1029
1052
|
}
|
|
1030
1053
|
};
|
|
1031
|
-
const
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
)
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1054
|
+
const assertTask = yield this.convertPlanToExecutable([assertPlan]);
|
|
1055
|
+
yield taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
|
|
1056
|
+
const output = yield taskExecutor.flush();
|
|
1057
|
+
if (output == null ? void 0 : output.pass) {
|
|
1058
|
+
return {
|
|
1059
|
+
output: void 0,
|
|
1060
|
+
executor: taskExecutor
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
errorThought = (output == null ? void 0 : output.thought) || "unknown error";
|
|
1064
|
+
const now = Date.now();
|
|
1065
|
+
if (now - startTime < checkIntervalMs) {
|
|
1066
|
+
const timeRemaining = checkIntervalMs - (now - startTime);
|
|
1067
|
+
const sleepPlan = {
|
|
1068
|
+
type: "Sleep",
|
|
1069
|
+
param: {
|
|
1070
|
+
timeMs: timeRemaining
|
|
1071
|
+
}
|
|
1072
|
+
};
|
|
1073
|
+
const sleepTask = yield this.convertPlanToExecutable([sleepPlan]);
|
|
1074
|
+
yield taskExecutor.append(
|
|
1075
|
+
this.wrapExecutorWithScreenshot(sleepTask[0])
|
|
1076
|
+
);
|
|
1077
|
+
yield taskExecutor.flush();
|
|
1078
|
+
}
|
|
1042
1079
|
}
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1080
|
+
const errorPlan = {
|
|
1081
|
+
type: "Error",
|
|
1082
|
+
param: {
|
|
1083
|
+
thought: `waitFor timeout: ${errorThought}`
|
|
1084
|
+
}
|
|
1085
|
+
};
|
|
1086
|
+
const errorTask = yield this.convertPlanToExecutable([errorPlan]);
|
|
1087
|
+
yield taskExecutor.append(errorTask[0]);
|
|
1088
|
+
yield taskExecutor.flush();
|
|
1089
|
+
return {
|
|
1090
|
+
output: void 0,
|
|
1091
|
+
executor: taskExecutor
|
|
1092
|
+
};
|
|
1093
|
+
});
|
|
1051
1094
|
}
|
|
1052
1095
|
};
|
|
1053
1096
|
|
|
@@ -1092,72 +1135,83 @@ var PageAgent = class {
|
|
|
1092
1135
|
type: "dump",
|
|
1093
1136
|
generateReport
|
|
1094
1137
|
});
|
|
1095
|
-
if (generateReport && autoPrintReportMsg) {
|
|
1138
|
+
if (generateReport && autoPrintReportMsg && this.reportFile) {
|
|
1096
1139
|
printReportMsg(this.reportFile);
|
|
1097
1140
|
}
|
|
1098
1141
|
}
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1142
|
+
aiAction(taskPrompt) {
|
|
1143
|
+
return __async(this, null, function* () {
|
|
1144
|
+
const { executor } = yield this.taskExecutor.action(taskPrompt);
|
|
1145
|
+
this.appendExecutionDump(executor.dump());
|
|
1146
|
+
this.writeOutActionDumps();
|
|
1147
|
+
if (executor.isInErrorState()) {
|
|
1148
|
+
const errorTask = executor.latestErrorTask();
|
|
1149
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1106
1150
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1107
|
-
|
|
1151
|
+
}
|
|
1152
|
+
});
|
|
1108
1153
|
}
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1154
|
+
aiQuery(demand) {
|
|
1155
|
+
return __async(this, null, function* () {
|
|
1156
|
+
const { output, executor } = yield this.taskExecutor.query(demand);
|
|
1157
|
+
this.appendExecutionDump(executor.dump());
|
|
1158
|
+
this.writeOutActionDumps();
|
|
1159
|
+
if (executor.isInErrorState()) {
|
|
1160
|
+
const errorTask = executor.latestErrorTask();
|
|
1161
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1116
1162
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1117
|
-
|
|
1118
|
-
return output;
|
|
1119
|
-
}
|
|
1120
|
-
async aiAssert(assertion, msg, opt) {
|
|
1121
|
-
const { output, executor } = await this.taskExecutor.assert(assertion);
|
|
1122
|
-
this.appendExecutionDump(executor.dump());
|
|
1123
|
-
this.writeOutActionDumps();
|
|
1124
|
-
if (opt == null ? void 0 : opt.keepRawResponse) {
|
|
1163
|
+
}
|
|
1125
1164
|
return output;
|
|
1126
|
-
}
|
|
1127
|
-
if (!(output == null ? void 0 : output.pass)) {
|
|
1128
|
-
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1129
|
-
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
|
|
1130
|
-
throw new Error(`${errMsg}
|
|
1131
|
-
${reasonMsg}`);
|
|
1132
|
-
}
|
|
1165
|
+
});
|
|
1133
1166
|
}
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1167
|
+
aiAssert(assertion, msg, opt) {
|
|
1168
|
+
return __async(this, null, function* () {
|
|
1169
|
+
var _a;
|
|
1170
|
+
const { output, executor } = yield this.taskExecutor.assert(assertion);
|
|
1171
|
+
this.appendExecutionDump(executor.dump());
|
|
1172
|
+
this.writeOutActionDumps();
|
|
1173
|
+
if (opt == null ? void 0 : opt.keepRawResponse) {
|
|
1174
|
+
return output;
|
|
1175
|
+
}
|
|
1176
|
+
if (!(output == null ? void 0 : output.pass)) {
|
|
1177
|
+
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1178
|
+
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || ((_a = executor.latestErrorTask()) == null ? void 0 : _a.error) || "(no_reason)"}`;
|
|
1179
|
+
throw new Error(`${errMsg}
|
|
1180
|
+
${reasonMsg}`);
|
|
1181
|
+
}
|
|
1139
1182
|
});
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
const
|
|
1144
|
-
|
|
1183
|
+
}
|
|
1184
|
+
aiWaitFor(assertion, opt) {
|
|
1185
|
+
return __async(this, null, function* () {
|
|
1186
|
+
const { executor } = yield this.taskExecutor.waitFor(assertion, {
|
|
1187
|
+
timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
|
|
1188
|
+
checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
|
|
1189
|
+
assertion
|
|
1190
|
+
});
|
|
1191
|
+
this.appendExecutionDump(executor.dump());
|
|
1192
|
+
this.writeOutActionDumps();
|
|
1193
|
+
if (executor.isInErrorState()) {
|
|
1194
|
+
const errorTask = executor.latestErrorTask();
|
|
1195
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1145
1196
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1146
|
-
|
|
1197
|
+
}
|
|
1198
|
+
});
|
|
1147
1199
|
}
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1200
|
+
ai(taskPrompt, type = "action") {
|
|
1201
|
+
return __async(this, null, function* () {
|
|
1202
|
+
if (type === "action") {
|
|
1203
|
+
return this.aiAction(taskPrompt);
|
|
1204
|
+
}
|
|
1205
|
+
if (type === "query") {
|
|
1206
|
+
return this.aiQuery(taskPrompt);
|
|
1207
|
+
}
|
|
1208
|
+
if (type === "assert") {
|
|
1209
|
+
return this.aiAssert(taskPrompt);
|
|
1210
|
+
}
|
|
1211
|
+
throw new Error(
|
|
1212
|
+
`Unknown type: ${type}, only support 'action', 'query', 'assert'`
|
|
1213
|
+
);
|
|
1214
|
+
});
|
|
1161
1215
|
}
|
|
1162
1216
|
};
|
|
1163
1217
|
|
|
@@ -1186,7 +1240,7 @@ var PlaywrightAiFixture = () => {
|
|
|
1186
1240
|
const agentForPage = (page, testInfo) => {
|
|
1187
1241
|
let idForPage = page[midsceneAgentKeyId];
|
|
1188
1242
|
if (!idForPage) {
|
|
1189
|
-
idForPage =
|
|
1243
|
+
idForPage = randomUUID();
|
|
1190
1244
|
page[midsceneAgentKeyId] = idForPage;
|
|
1191
1245
|
const { testId } = testInfo;
|
|
1192
1246
|
const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
|
|
@@ -1215,70 +1269,94 @@ var PlaywrightAiFixture = () => {
|
|
|
1215
1269
|
}
|
|
1216
1270
|
};
|
|
1217
1271
|
return {
|
|
1218
|
-
ai:
|
|
1272
|
+
ai: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
|
|
1219
1273
|
const agent = agentForPage(page, testInfo);
|
|
1220
|
-
|
|
1221
|
-
|
|
1274
|
+
yield use(
|
|
1275
|
+
(taskPrompt, opts) => __async(void 0, null, function* () {
|
|
1222
1276
|
return new Promise((resolve, reject) => {
|
|
1223
|
-
test.step(`ai - ${taskPrompt}`,
|
|
1224
|
-
|
|
1277
|
+
test.step(`ai - ${taskPrompt}`, () => __async(void 0, null, function* () {
|
|
1278
|
+
yield waitForNetworkIdle(page);
|
|
1225
1279
|
const actionType = (opts == null ? void 0 : opts.type) || "action";
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1280
|
+
try {
|
|
1281
|
+
const result = yield agent.ai(taskPrompt, actionType);
|
|
1282
|
+
resolve(result);
|
|
1283
|
+
} catch (error) {
|
|
1284
|
+
reject(error);
|
|
1285
|
+
}
|
|
1286
|
+
}));
|
|
1229
1287
|
});
|
|
1230
|
-
}
|
|
1288
|
+
})
|
|
1231
1289
|
);
|
|
1232
1290
|
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
|
1233
|
-
},
|
|
1234
|
-
aiAction:
|
|
1291
|
+
}),
|
|
1292
|
+
aiAction: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
|
|
1235
1293
|
const agent = agentForPage(page, testInfo);
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1294
|
+
yield use((taskPrompt) => __async(void 0, null, function* () {
|
|
1295
|
+
return new Promise((resolve, reject) => {
|
|
1296
|
+
test.step(`aiAction - ${taskPrompt}`, () => __async(void 0, null, function* () {
|
|
1297
|
+
yield waitForNetworkIdle(page);
|
|
1298
|
+
try {
|
|
1299
|
+
const result = yield agent.aiAction(taskPrompt);
|
|
1300
|
+
resolve(result);
|
|
1301
|
+
} catch (error) {
|
|
1302
|
+
reject(error);
|
|
1303
|
+
}
|
|
1304
|
+
}));
|
|
1240
1305
|
});
|
|
1241
|
-
});
|
|
1306
|
+
}));
|
|
1242
1307
|
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
|
1243
|
-
},
|
|
1244
|
-
aiQuery:
|
|
1308
|
+
}),
|
|
1309
|
+
aiQuery: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
|
|
1245
1310
|
const agent = agentForPage(page, testInfo);
|
|
1246
|
-
|
|
1311
|
+
yield use((demand) => __async(void 0, null, function* () {
|
|
1247
1312
|
return new Promise((resolve, reject) => {
|
|
1248
|
-
test.step(`aiQuery - ${JSON.stringify(demand)}`,
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1313
|
+
test.step(`aiQuery - ${JSON.stringify(demand)}`, () => __async(void 0, null, function* () {
|
|
1314
|
+
yield waitForNetworkIdle(page);
|
|
1315
|
+
try {
|
|
1316
|
+
const result = yield agent.aiQuery(demand);
|
|
1317
|
+
resolve(result);
|
|
1318
|
+
} catch (error) {
|
|
1319
|
+
reject(error);
|
|
1320
|
+
}
|
|
1321
|
+
}));
|
|
1253
1322
|
});
|
|
1254
|
-
});
|
|
1323
|
+
}));
|
|
1255
1324
|
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
|
1256
|
-
},
|
|
1257
|
-
aiAssert:
|
|
1325
|
+
}),
|
|
1326
|
+
aiAssert: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
|
|
1258
1327
|
const agent = agentForPage(page, testInfo);
|
|
1259
|
-
|
|
1328
|
+
yield use((assertion, errorMsg) => __async(void 0, null, function* () {
|
|
1260
1329
|
return new Promise((resolve, reject) => {
|
|
1261
|
-
test.step(`aiAssert - ${assertion}`,
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1330
|
+
test.step(`aiAssert - ${assertion}`, () => __async(void 0, null, function* () {
|
|
1331
|
+
yield waitForNetworkIdle(page);
|
|
1332
|
+
try {
|
|
1333
|
+
yield agent.aiAssert(assertion, errorMsg);
|
|
1334
|
+
resolve(null);
|
|
1335
|
+
} catch (error) {
|
|
1336
|
+
reject(error);
|
|
1337
|
+
}
|
|
1338
|
+
}));
|
|
1266
1339
|
});
|
|
1267
|
-
});
|
|
1340
|
+
}));
|
|
1268
1341
|
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
|
1269
|
-
},
|
|
1270
|
-
aiWaitFor:
|
|
1342
|
+
}),
|
|
1343
|
+
aiWaitFor: (_0, _1, _2) => __async(void 0, [_0, _1, _2], function* ({ page }, use, testInfo) {
|
|
1271
1344
|
const agent = agentForPage(page, testInfo);
|
|
1272
|
-
|
|
1345
|
+
yield use((assertion, opt) => __async(void 0, null, function* () {
|
|
1273
1346
|
return new Promise((resolve, reject) => {
|
|
1274
|
-
test.step(`aiWaitFor - ${assertion}`,
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1347
|
+
test.step(`aiWaitFor - ${assertion}`, () => __async(void 0, null, function* () {
|
|
1348
|
+
yield waitForNetworkIdle(page);
|
|
1349
|
+
try {
|
|
1350
|
+
yield agent.aiWaitFor(assertion, opt);
|
|
1351
|
+
resolve(null);
|
|
1352
|
+
} catch (error) {
|
|
1353
|
+
reject(error);
|
|
1354
|
+
}
|
|
1355
|
+
}));
|
|
1278
1356
|
});
|
|
1279
|
-
});
|
|
1357
|
+
}));
|
|
1280
1358
|
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
|
1281
|
-
}
|
|
1359
|
+
})
|
|
1282
1360
|
};
|
|
1283
1361
|
};
|
|
1284
1362
|
function waitForNetworkIdle(page) {
|
|
@@ -1292,86 +1370,112 @@ function waitForNetworkIdle(page) {
|
|
|
1292
1370
|
// src/puppeteer/base-page.ts
|
|
1293
1371
|
import { readFileSync as readFileSync3, writeFileSync } from "fs";
|
|
1294
1372
|
import { getTmpFile } from "@midscene/core/utils";
|
|
1295
|
-
import { resizeImg } from "@midscene/shared/img";
|
|
1373
|
+
import { base64Encoded, resizeImg } from "@midscene/shared/img";
|
|
1296
1374
|
var Page = class {
|
|
1297
1375
|
evaluate(pageFunction, arg) {
|
|
1298
1376
|
if (this.pageType === "puppeteer") {
|
|
1299
|
-
return this.
|
|
1377
|
+
return this.underlyingPage.evaluate(pageFunction, arg);
|
|
1300
1378
|
}
|
|
1301
|
-
return this.
|
|
1379
|
+
return this.underlyingPage.evaluate(pageFunction, arg);
|
|
1302
1380
|
}
|
|
1303
|
-
constructor(
|
|
1304
|
-
this.
|
|
1381
|
+
constructor(underlyingPage, pageType) {
|
|
1382
|
+
this.underlyingPage = underlyingPage;
|
|
1305
1383
|
this.pageType = pageType;
|
|
1306
1384
|
}
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
async screenshot() {
|
|
1313
|
-
const viewportSize = await this.evaluate(() => {
|
|
1314
|
-
return {
|
|
1315
|
-
width: document.documentElement.clientWidth,
|
|
1316
|
-
height: document.documentElement.clientHeight,
|
|
1317
|
-
deviceScaleFactor: window.devicePixelRatio
|
|
1318
|
-
};
|
|
1319
|
-
});
|
|
1320
|
-
const path2 = getTmpFile("png");
|
|
1321
|
-
await this.page.screenshot({
|
|
1322
|
-
path: path2,
|
|
1323
|
-
type: "png"
|
|
1385
|
+
getElementInfos() {
|
|
1386
|
+
return __async(this, null, function* () {
|
|
1387
|
+
const scripts = yield getExtraReturnLogic();
|
|
1388
|
+
const captureElementSnapshot = yield this.evaluate(scripts);
|
|
1389
|
+
return captureElementSnapshot;
|
|
1324
1390
|
});
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1391
|
+
}
|
|
1392
|
+
screenshotBase64() {
|
|
1393
|
+
return __async(this, null, function* () {
|
|
1394
|
+
const viewportSize = yield this.evaluate(() => {
|
|
1395
|
+
return {
|
|
1396
|
+
width: document.documentElement.clientWidth,
|
|
1397
|
+
height: document.documentElement.clientHeight,
|
|
1398
|
+
deviceScaleFactor: window.devicePixelRatio
|
|
1399
|
+
};
|
|
1330
1400
|
});
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1401
|
+
const path2 = getTmpFile("png");
|
|
1402
|
+
yield this.underlyingPage.screenshot({
|
|
1403
|
+
path: path2,
|
|
1404
|
+
type: "png"
|
|
1405
|
+
});
|
|
1406
|
+
let buf;
|
|
1407
|
+
if (viewportSize.deviceScaleFactor > 1) {
|
|
1408
|
+
buf = yield resizeImg(readFileSync3(path2), {
|
|
1409
|
+
width: viewportSize.width,
|
|
1410
|
+
height: viewportSize.height
|
|
1411
|
+
});
|
|
1412
|
+
writeFileSync(path2, buf);
|
|
1413
|
+
}
|
|
1414
|
+
return base64Encoded(path2, true);
|
|
1415
|
+
});
|
|
1334
1416
|
}
|
|
1335
1417
|
url() {
|
|
1336
|
-
return this.
|
|
1418
|
+
return this.underlyingPage.url();
|
|
1337
1419
|
}
|
|
1338
1420
|
get mouse() {
|
|
1339
1421
|
return {
|
|
1340
|
-
click:
|
|
1341
|
-
|
|
1422
|
+
click: (x, y, options) => __async(this, null, function* () {
|
|
1423
|
+
return this.underlyingPage.mouse.click(x, y, {
|
|
1424
|
+
button: (options == null ? void 0 : options.button) || "left"
|
|
1425
|
+
});
|
|
1426
|
+
}),
|
|
1427
|
+
wheel: (deltaX, deltaY) => __async(this, null, function* () {
|
|
1342
1428
|
if (this.pageType === "puppeteer") {
|
|
1343
|
-
|
|
1429
|
+
yield this.underlyingPage.mouse.wheel({
|
|
1430
|
+
deltaX,
|
|
1431
|
+
deltaY
|
|
1432
|
+
});
|
|
1344
1433
|
} else if (this.pageType === "playwright") {
|
|
1345
|
-
|
|
1434
|
+
yield this.underlyingPage.mouse.wheel(
|
|
1435
|
+
deltaX,
|
|
1436
|
+
deltaY
|
|
1437
|
+
);
|
|
1346
1438
|
}
|
|
1347
|
-
},
|
|
1348
|
-
move:
|
|
1439
|
+
}),
|
|
1440
|
+
move: (x, y) => __async(this, null, function* () {
|
|
1441
|
+
return this.underlyingPage.mouse.move(x, y);
|
|
1442
|
+
})
|
|
1349
1443
|
};
|
|
1350
1444
|
}
|
|
1351
1445
|
get keyboard() {
|
|
1352
1446
|
return {
|
|
1353
|
-
type:
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1447
|
+
type: (text) => __async(this, null, function* () {
|
|
1448
|
+
return this.underlyingPage.keyboard.type(text);
|
|
1449
|
+
}),
|
|
1450
|
+
press: (key) => __async(this, null, function* () {
|
|
1451
|
+
return this.underlyingPage.keyboard.press(key);
|
|
1452
|
+
}),
|
|
1453
|
+
down: (key) => __async(this, null, function* () {
|
|
1454
|
+
return this.underlyingPage.keyboard.down(key);
|
|
1455
|
+
}),
|
|
1456
|
+
up: (key) => __async(this, null, function* () {
|
|
1457
|
+
return this.underlyingPage.keyboard.up(key);
|
|
1458
|
+
})
|
|
1357
1459
|
};
|
|
1358
1460
|
}
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1461
|
+
clearInput(element) {
|
|
1462
|
+
return __async(this, null, function* () {
|
|
1463
|
+
if (!element) {
|
|
1464
|
+
return;
|
|
1465
|
+
}
|
|
1466
|
+
yield this.mouse.click(element.center[0], element.center[1]);
|
|
1467
|
+
const isMac = process.platform === "darwin";
|
|
1468
|
+
if (isMac) {
|
|
1469
|
+
yield this.underlyingPage.keyboard.down("Meta");
|
|
1470
|
+
yield this.underlyingPage.keyboard.press("a");
|
|
1471
|
+
yield this.underlyingPage.keyboard.up("Meta");
|
|
1472
|
+
} else {
|
|
1473
|
+
yield this.underlyingPage.keyboard.down("Control");
|
|
1474
|
+
yield this.underlyingPage.keyboard.press("a");
|
|
1475
|
+
yield this.underlyingPage.keyboard.up("Control");
|
|
1476
|
+
}
|
|
1477
|
+
yield this.keyboard.press("Backspace");
|
|
1478
|
+
});
|
|
1375
1479
|
}
|
|
1376
1480
|
scrollUntilTop() {
|
|
1377
1481
|
return this.mouse.wheel(0, -9999999);
|
|
@@ -1379,15 +1483,19 @@ var Page = class {
|
|
|
1379
1483
|
scrollUntilBottom() {
|
|
1380
1484
|
return this.mouse.wheel(0, 9999999);
|
|
1381
1485
|
}
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1486
|
+
scrollUpOneScreen() {
|
|
1487
|
+
return __async(this, null, function* () {
|
|
1488
|
+
const innerHeight = yield this.evaluate(() => window.innerHeight);
|
|
1489
|
+
const distance = innerHeight * 0.7;
|
|
1490
|
+
yield this.mouse.wheel(0, -distance);
|
|
1491
|
+
});
|
|
1386
1492
|
}
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1493
|
+
scrollDownOneScreen() {
|
|
1494
|
+
return __async(this, null, function* () {
|
|
1495
|
+
const innerHeight = yield this.evaluate(() => window.innerHeight);
|
|
1496
|
+
const distance = innerHeight * 0.7;
|
|
1497
|
+
yield this.mouse.wheel(0, distance);
|
|
1498
|
+
});
|
|
1391
1499
|
}
|
|
1392
1500
|
};
|
|
1393
1501
|
|