misoai-web 1.0.6 → 1.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -349
- package/dist/es/agent.js +165 -428
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +10 -9
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +167 -430
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +173 -435
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +185 -432
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +165 -428
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +165 -428
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +182 -429
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +169 -432
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +169 -432
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js +7 -4
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js +29 -3
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +163 -426
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +10 -9
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +165 -428
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +171 -433
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +183 -430
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +163 -426
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +163 -426
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +180 -427
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +167 -430
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +167 -430
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js +7 -4
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js +29 -3
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +13 -51
- package/dist/types/bridge-mode-browser.d.ts +2 -3
- package/dist/types/bridge-mode.d.ts +2 -3
- package/dist/types/{browser-aec1055d.d.ts → browser-9b472ffb.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -3
- package/dist/types/index.d.ts +1 -2
- package/dist/types/midscene-server.d.ts +1 -2
- package/dist/types/{page-86ab0fe1.d.ts → page-ed0ecb44.d.ts} +19 -9
- package/dist/types/playground.d.ts +2 -3
- package/dist/types/playwright.d.ts +9 -2
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -2
- package/dist/types/puppeteer.d.ts +6 -5
- package/dist/types/ui-utils.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -2
- package/dist/types/yaml.d.ts +1 -2
- package/iife-script/htmlElement.js +53 -75
- package/iife-script/htmlElementDebug.js +35 -56
- package/package.json +24 -24
- package/LICENSE +0 -21
package/dist/lib/puppeteer.js
CHANGED
@@ -55,10 +55,11 @@ var ScriptPlayer = class {
|
|
55
55
|
this.unnamedResultIndex = 0;
|
56
56
|
this.pageAgent = null;
|
57
57
|
this.result = {};
|
58
|
+
const target = script.target || script.web || script.android;
|
58
59
|
if (import_utils.ifInBrowser) {
|
59
60
|
this.output = void 0;
|
60
|
-
} else if (
|
61
|
-
this.output = (0, import_node_path.resolve)(process.cwd(),
|
61
|
+
} else if (target?.output) {
|
62
|
+
this.output = (0, import_node_path.resolve)(process.cwd(), target.output);
|
62
63
|
} else {
|
63
64
|
this.output = (0, import_node_path.join)((0, import_common.getMidsceneRunSubDir)("output"), `${process.pid}.json`);
|
64
65
|
}
|
@@ -132,15 +133,20 @@ var ScriptPlayer = class {
|
|
132
133
|
} else if ("aiAssert" in flowItem) {
|
133
134
|
const assertTask = flowItem;
|
134
135
|
const prompt = assertTask.aiAssert;
|
136
|
+
const msg = assertTask.errorMessage;
|
135
137
|
(0, import_utils.assert)(prompt, "missing prompt for aiAssert");
|
136
138
|
(0, import_utils.assert)(
|
137
139
|
typeof prompt === "string",
|
138
140
|
"prompt for aiAssert must be a string"
|
139
141
|
);
|
140
|
-
await agent.aiAssert(prompt);
|
142
|
+
await agent.aiAssert(prompt, msg);
|
141
143
|
} else if ("aiQuery" in flowItem) {
|
142
144
|
const queryTask = flowItem;
|
143
145
|
const prompt = queryTask.aiQuery;
|
146
|
+
const options = {
|
147
|
+
domIncluded: queryTask.domIncluded,
|
148
|
+
screenshotIncluded: queryTask.screenshotIncluded
|
149
|
+
};
|
144
150
|
(0, import_utils.assert)(prompt, "missing prompt for aiQuery");
|
145
151
|
(0, import_utils.assert)(
|
146
152
|
typeof prompt === "string",
|
@@ -151,6 +157,10 @@ var ScriptPlayer = class {
|
|
151
157
|
} else if ("aiNumber" in flowItem) {
|
152
158
|
const numberTask = flowItem;
|
153
159
|
const prompt = numberTask.aiNumber;
|
160
|
+
const options = {
|
161
|
+
domIncluded: numberTask.domIncluded,
|
162
|
+
screenshotIncluded: numberTask.screenshotIncluded
|
163
|
+
};
|
154
164
|
(0, import_utils.assert)(prompt, "missing prompt for number");
|
155
165
|
(0, import_utils.assert)(
|
156
166
|
typeof prompt === "string",
|
@@ -161,6 +171,10 @@ var ScriptPlayer = class {
|
|
161
171
|
} else if ("aiString" in flowItem) {
|
162
172
|
const stringTask = flowItem;
|
163
173
|
const prompt = stringTask.aiString;
|
174
|
+
const options = {
|
175
|
+
domIncluded: stringTask.domIncluded,
|
176
|
+
screenshotIncluded: stringTask.screenshotIncluded
|
177
|
+
};
|
164
178
|
(0, import_utils.assert)(prompt, "missing prompt for string");
|
165
179
|
(0, import_utils.assert)(
|
166
180
|
typeof prompt === "string",
|
@@ -171,6 +185,10 @@ var ScriptPlayer = class {
|
|
171
185
|
} else if ("aiBoolean" in flowItem) {
|
172
186
|
const booleanTask = flowItem;
|
173
187
|
const prompt = booleanTask.aiBoolean;
|
188
|
+
const options = {
|
189
|
+
domIncluded: booleanTask.domIncluded,
|
190
|
+
screenshotIncluded: booleanTask.screenshotIncluded
|
191
|
+
};
|
174
192
|
(0, import_utils.assert)(prompt, "missing prompt for boolean");
|
175
193
|
(0, import_utils.assert)(
|
176
194
|
typeof prompt === "string",
|
@@ -213,6 +231,9 @@ var ScriptPlayer = class {
|
|
213
231
|
} else if ("aiTap" in flowItem) {
|
214
232
|
const tapTask = flowItem;
|
215
233
|
await agent.aiTap(tapTask.aiTap, tapTask);
|
234
|
+
} else if ("aiRightClick" in flowItem) {
|
235
|
+
const rightClickTask = flowItem;
|
236
|
+
await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
|
216
237
|
} else if ("aiHover" in flowItem) {
|
217
238
|
const hoverTask = flowItem;
|
218
239
|
await agent.aiHover(hoverTask.aiHover, hoverTask);
|
@@ -235,6 +256,11 @@ var ScriptPlayer = class {
|
|
235
256
|
evaluateJavaScriptTask.javascript
|
236
257
|
);
|
237
258
|
this.setResult(evaluateJavaScriptTask.name, result);
|
259
|
+
} else if ("logScreenshot" in flowItem) {
|
260
|
+
const logScreenshotTask = flowItem;
|
261
|
+
await agent.logScreenshot(logScreenshotTask.logScreenshot, {
|
262
|
+
content: logScreenshotTask.content || ""
|
263
|
+
});
|
238
264
|
} else {
|
239
265
|
throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
|
240
266
|
}
|
@@ -483,7 +509,8 @@ var WebElementInfo = class {
|
|
483
509
|
id,
|
484
510
|
attributes,
|
485
511
|
indexId,
|
486
|
-
xpaths
|
512
|
+
xpaths,
|
513
|
+
isVisible
|
487
514
|
}) {
|
488
515
|
this.content = content;
|
489
516
|
this.rect = rect;
|
@@ -496,6 +523,7 @@ var WebElementInfo = class {
|
|
496
523
|
this.attributes = attributes;
|
497
524
|
this.indexId = indexId;
|
498
525
|
this.xpaths = xpaths;
|
526
|
+
this.isVisible = isVisible;
|
499
527
|
}
|
500
528
|
};
|
501
529
|
|
@@ -518,14 +546,15 @@ async function parseContextFromWebPage(page, _opt) {
|
|
518
546
|
})
|
519
547
|
]);
|
520
548
|
const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
|
521
|
-
const { rect, id, content, attributes, locator, indexId } = elementInfo;
|
549
|
+
const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
|
522
550
|
return new WebElementInfo({
|
523
551
|
rect,
|
524
552
|
locator,
|
525
553
|
id,
|
526
554
|
content,
|
527
555
|
attributes,
|
528
|
-
indexId
|
556
|
+
indexId,
|
557
|
+
isVisible
|
529
558
|
});
|
530
559
|
});
|
531
560
|
(0, import_utils4.assert)(screenshotBase64, "screenshotBase64 is required");
|
@@ -555,7 +584,7 @@ function printReportMsg(filepath) {
|
|
555
584
|
(0, import_utils4.logMsg)(`Midscene - report file updated: ${filepath}`);
|
556
585
|
}
|
557
586
|
function replaceIllegalPathCharsAndSpace(str) {
|
558
|
-
return str.replace(/[
|
587
|
+
return str.replace(/[:*?"<>| ]/g, "-");
|
559
588
|
}
|
560
589
|
function forceClosePopup(page, debug6) {
|
561
590
|
page.on("popup", async (popup) => {
|
@@ -869,10 +898,10 @@ var PageTaskExecutor = class {
|
|
869
898
|
if (!taskParam || !taskParam.value) {
|
870
899
|
return;
|
871
900
|
}
|
872
|
-
await this.page.keyboard.type(taskParam.value);
|
873
|
-
} else {
|
874
|
-
await this.page.keyboard.type(taskParam.value);
|
875
901
|
}
|
902
|
+
await this.page.keyboard.type(taskParam.value, {
|
903
|
+
autoDismissKeyboard: taskParam.autoDismissKeyboard
|
904
|
+
});
|
876
905
|
}
|
877
906
|
};
|
878
907
|
tasks.push(taskActionInput);
|
@@ -901,6 +930,22 @@ var PageTaskExecutor = class {
|
|
901
930
|
}
|
902
931
|
};
|
903
932
|
tasks.push(taskActionTap);
|
933
|
+
} else if (plan2.type === "RightClick") {
|
934
|
+
const taskActionRightClick = {
|
935
|
+
type: "Action",
|
936
|
+
subType: "RightClick",
|
937
|
+
thought: plan2.thought,
|
938
|
+
locate: plan2.locate,
|
939
|
+
executor: async (param, { element }) => {
|
940
|
+
(0, import_utils6.assert)(element, "Element not found, cannot right click");
|
941
|
+
await this.page.mouse.click(
|
942
|
+
element.center[0],
|
943
|
+
element.center[1],
|
944
|
+
{ button: "right" }
|
945
|
+
);
|
946
|
+
}
|
947
|
+
};
|
948
|
+
tasks.push(taskActionRightClick);
|
904
949
|
} else if (plan2.type === "Drag") {
|
905
950
|
const taskActionDrag = {
|
906
951
|
type: "Action",
|
@@ -1429,7 +1474,7 @@ var PageTaskExecutor = class {
|
|
1429
1474
|
executor: taskExecutor
|
1430
1475
|
};
|
1431
1476
|
}
|
1432
|
-
async createTypeQueryTask(type, demand) {
|
1477
|
+
async createTypeQueryTask(type, demand, opt) {
|
1433
1478
|
const taskExecutor = new import_misoai_core.Executor(
|
1434
1479
|
taskTitleStr(
|
1435
1480
|
type,
|
@@ -1460,7 +1505,10 @@ var PageTaskExecutor = class {
|
|
1460
1505
|
result: `${type}, ${demand}`
|
1461
1506
|
};
|
1462
1507
|
}
|
1463
|
-
const { data, usage } = await this.insight.extract(
|
1508
|
+
const { data, usage } = await this.insight.extract(
|
1509
|
+
demandInput,
|
1510
|
+
opt
|
1511
|
+
);
|
1464
1512
|
let outputResult = data;
|
1465
1513
|
if (ifTypeRestricted) {
|
1466
1514
|
(0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
|
@@ -1480,17 +1528,17 @@ var PageTaskExecutor = class {
|
|
1480
1528
|
executor: taskExecutor
|
1481
1529
|
};
|
1482
1530
|
}
|
1483
|
-
async query(demand) {
|
1484
|
-
return this.createTypeQueryTask("Query", demand);
|
1531
|
+
async query(demand, opt) {
|
1532
|
+
return this.createTypeQueryTask("Query", demand, opt);
|
1485
1533
|
}
|
1486
|
-
async boolean(prompt) {
|
1487
|
-
return this.createTypeQueryTask("Boolean", prompt);
|
1534
|
+
async boolean(prompt, opt) {
|
1535
|
+
return this.createTypeQueryTask("Boolean", prompt, opt);
|
1488
1536
|
}
|
1489
|
-
async number(prompt) {
|
1490
|
-
return this.createTypeQueryTask("Number", prompt);
|
1537
|
+
async number(prompt, opt) {
|
1538
|
+
return this.createTypeQueryTask("Number", prompt, opt);
|
1491
1539
|
}
|
1492
|
-
async string(prompt) {
|
1493
|
-
return this.createTypeQueryTask("String", prompt);
|
1540
|
+
async string(prompt, opt) {
|
1541
|
+
return this.createTypeQueryTask("String", prompt, opt);
|
1494
1542
|
}
|
1495
1543
|
async assert(assertion) {
|
1496
1544
|
const description = `assert: ${assertion}`;
|
@@ -1626,7 +1674,7 @@ function buildPlans(type, locateParam, param) {
|
|
1626
1674
|
param: locateParam,
|
1627
1675
|
thought: ""
|
1628
1676
|
} : null;
|
1629
|
-
if (type === "Tap" || type === "Hover") {
|
1677
|
+
if (type === "Tap" || type === "Hover" || type === "RightClick") {
|
1630
1678
|
(0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
|
1631
1679
|
(0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
|
1632
1680
|
const tapPlan = {
|
@@ -1706,7 +1754,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
|
|
1706
1754
|
var import_semver = __toESM(require("semver"));
|
1707
1755
|
|
1708
1756
|
// package.json
|
1709
|
-
var version = "1.0.
|
1757
|
+
var version = "1.0.3";
|
1710
1758
|
|
1711
1759
|
// src/common/task-cache.ts
|
1712
1760
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -1734,70 +1782,44 @@ var TaskCache = class {
|
|
1734
1782
|
this.cache = cacheContent;
|
1735
1783
|
this.cacheOriginalLength = this.cache.caches.length;
|
1736
1784
|
}
|
1737
|
-
matchCache(prompt, type
|
1738
|
-
const contextHash = contextData ? this.generateContextHash(contextData) : void 0;
|
1785
|
+
matchCache(prompt, type) {
|
1739
1786
|
for (let i = 0; i < this.cacheOriginalLength; i++) {
|
1740
1787
|
const item = this.cache.caches[i];
|
1741
1788
|
const key = `${type}:${prompt}:${i}`;
|
1742
|
-
if (item.type
|
1743
|
-
|
1744
|
-
|
1745
|
-
|
1746
|
-
|
1747
|
-
|
1748
|
-
|
1749
|
-
|
1750
|
-
|
1789
|
+
if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
|
1790
|
+
this.matchedCacheIndices.add(key);
|
1791
|
+
debug3(
|
1792
|
+
"cache found and marked as used, type: %s, prompt: %s, index: %d",
|
1793
|
+
type,
|
1794
|
+
prompt,
|
1795
|
+
i
|
1796
|
+
);
|
1797
|
+
return {
|
1798
|
+
cacheContent: item,
|
1799
|
+
updateFn: (cb) => {
|
1800
|
+
debug3(
|
1801
|
+
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1802
|
+
type,
|
1803
|
+
prompt,
|
1804
|
+
i
|
1805
|
+
);
|
1806
|
+
cb(item);
|
1807
|
+
debug3(
|
1808
|
+
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1809
|
+
type,
|
1810
|
+
prompt,
|
1811
|
+
i
|
1812
|
+
);
|
1813
|
+
this.flushCacheToFile();
|
1751
1814
|
}
|
1752
|
-
}
|
1753
|
-
debug3("cache context availability mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
|
1754
|
-
continue;
|
1755
|
-
}
|
1815
|
+
};
|
1756
1816
|
}
|
1757
|
-
this.matchedCacheIndices.add(key);
|
1758
|
-
debug3(
|
1759
|
-
"cache found and marked as used, type: %s, prompt: %s, index: %d, contextMatch: %s",
|
1760
|
-
type,
|
1761
|
-
prompt,
|
1762
|
-
i,
|
1763
|
-
contextHash ? "yes" : "no-context"
|
1764
|
-
);
|
1765
|
-
return {
|
1766
|
-
cacheContent: item,
|
1767
|
-
updateFn: (cb) => {
|
1768
|
-
debug3(
|
1769
|
-
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1770
|
-
type,
|
1771
|
-
prompt,
|
1772
|
-
i
|
1773
|
-
);
|
1774
|
-
cb(item);
|
1775
|
-
debug3(
|
1776
|
-
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1777
|
-
type,
|
1778
|
-
prompt,
|
1779
|
-
i
|
1780
|
-
);
|
1781
|
-
this.flushCacheToFile();
|
1782
|
-
}
|
1783
|
-
};
|
1784
1817
|
}
|
1785
|
-
debug3("no unused cache found, type: %s, prompt: %s
|
1818
|
+
debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
|
1786
1819
|
return void 0;
|
1787
1820
|
}
|
1788
|
-
|
1789
|
-
|
1790
|
-
const stableString = sortedKeys.map((key) => `${key}:${JSON.stringify(contextData[key])}`).join("|");
|
1791
|
-
let hash = 0;
|
1792
|
-
for (let i = 0; i < stableString.length; i++) {
|
1793
|
-
const char = stableString.charCodeAt(i);
|
1794
|
-
hash = (hash << 5) - hash + char;
|
1795
|
-
hash = hash & hash;
|
1796
|
-
}
|
1797
|
-
return hash.toString(36);
|
1798
|
-
}
|
1799
|
-
matchPlanCache(prompt, contextData) {
|
1800
|
-
return this.matchCache(prompt, "plan", contextData);
|
1821
|
+
matchPlanCache(prompt) {
|
1822
|
+
return this.matchCache(prompt, "plan");
|
1801
1823
|
}
|
1802
1824
|
matchLocateCache(prompt) {
|
1803
1825
|
return this.matchCache(prompt, "locate");
|
@@ -1863,8 +1885,14 @@ cache file: ${cacheFile}`
|
|
1863
1885
|
return;
|
1864
1886
|
}
|
1865
1887
|
try {
|
1888
|
+
const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
|
1889
|
+
if (!(0, import_node_fs2.existsSync)(dir)) {
|
1890
|
+
(0, import_node_fs2.mkdirSync)(dir, { recursive: true });
|
1891
|
+
debug3("created cache directory: %s", dir);
|
1892
|
+
}
|
1866
1893
|
const yamlData = import_js_yaml3.default.dump(this.cache);
|
1867
1894
|
(0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
|
1895
|
+
debug3("cache flushed to file: %s", this.cacheFilePath);
|
1868
1896
|
} catch (err) {
|
1869
1897
|
debug3(
|
1870
1898
|
"write cache to file failed, path: %s, error: %s",
|
@@ -1873,16 +1901,11 @@ cache file: ${cacheFile}`
|
|
1873
1901
|
);
|
1874
1902
|
}
|
1875
1903
|
}
|
1876
|
-
updateOrAppendCacheRecord(newRecord, cachedRecord
|
1904
|
+
updateOrAppendCacheRecord(newRecord, cachedRecord) {
|
1877
1905
|
if (cachedRecord) {
|
1878
1906
|
if (newRecord.type === "plan") {
|
1879
1907
|
cachedRecord.updateFn((cache) => {
|
1880
|
-
|
1881
|
-
planCache.yamlWorkflow = newRecord.yamlWorkflow;
|
1882
|
-
if (contextData) {
|
1883
|
-
planCache.contextHash = this.generateContextHash(contextData);
|
1884
|
-
planCache.contextData = { ...contextData };
|
1885
|
-
}
|
1908
|
+
cache.yamlWorkflow = newRecord.yamlWorkflow;
|
1886
1909
|
});
|
1887
1910
|
} else {
|
1888
1911
|
cachedRecord.updateFn((cache) => {
|
@@ -1890,11 +1913,6 @@ cache file: ${cacheFile}`
|
|
1890
1913
|
});
|
1891
1914
|
}
|
1892
1915
|
} else {
|
1893
|
-
if (newRecord.type === "plan" && contextData) {
|
1894
|
-
const planRecord = newRecord;
|
1895
|
-
planRecord.contextHash = this.generateContextHash(contextData);
|
1896
|
-
planRecord.contextData = { ...contextData };
|
1897
|
-
}
|
1898
1916
|
this.appendCache(newRecord);
|
1899
1917
|
}
|
1900
1918
|
}
|
@@ -1924,13 +1942,10 @@ var PageAgent = class {
|
|
1924
1942
|
generateReport: true,
|
1925
1943
|
autoPrintReportMsg: true,
|
1926
1944
|
groupName: "Midscene Report",
|
1927
|
-
groupDescription: ""
|
1928
|
-
enableCumulativeContext: true,
|
1929
|
-
autoClearContext: false
|
1945
|
+
groupDescription: ""
|
1930
1946
|
},
|
1931
1947
|
opts || {}
|
1932
1948
|
);
|
1933
|
-
this.initializeContextStore();
|
1934
1949
|
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1935
1950
|
this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || import_constants2.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
|
1936
1951
|
this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || import_constants2.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
|
@@ -1957,69 +1972,6 @@ var PageAgent = class {
|
|
1957
1972
|
opts?.testId || this.page.pageType || "web"
|
1958
1973
|
);
|
1959
1974
|
}
|
1960
|
-
/**
|
1961
|
-
* Initialize context store for cumulative context functionality
|
1962
|
-
*/
|
1963
|
-
async initializeContextStore() {
|
1964
|
-
if (!this.opts.enableCumulativeContext) {
|
1965
|
-
debug4("Cumulative context disabled via options");
|
1966
|
-
return;
|
1967
|
-
}
|
1968
|
-
try {
|
1969
|
-
const aiModel = await import("misoai-core/ai-model");
|
1970
|
-
this.contextStore = aiModel.getContextStore();
|
1971
|
-
debug4("Context store initialized successfully", {
|
1972
|
-
autoClearContext: this.opts.autoClearContext,
|
1973
|
-
testId: this.opts.testId
|
1974
|
-
});
|
1975
|
-
if (this.opts.autoClearContext) {
|
1976
|
-
this.contextStore.clear();
|
1977
|
-
debug4("Context store cleared due to autoClearContext option");
|
1978
|
-
} else {
|
1979
|
-
const existingData = this.contextStore.getAllData();
|
1980
|
-
const existingSteps = this.contextStore.getRecentSteps(100).length;
|
1981
|
-
debug4("Context store preserving existing data", {
|
1982
|
-
existingDataKeys: Object.keys(existingData),
|
1983
|
-
existingStepsCount: existingSteps
|
1984
|
-
});
|
1985
|
-
}
|
1986
|
-
} catch (error) {
|
1987
|
-
debug4("Failed to initialize context store:", error);
|
1988
|
-
console.warn("⚠️ Could not initialize context store:", error);
|
1989
|
-
}
|
1990
|
-
}
|
1991
|
-
/**
|
1992
|
-
* Get the context store instance
|
1993
|
-
*/
|
1994
|
-
getContextStore() {
|
1995
|
-
return this.contextStore;
|
1996
|
-
}
|
1997
|
-
/**
|
1998
|
-
* Clear the context store
|
1999
|
-
*/
|
2000
|
-
clearContext() {
|
2001
|
-
if (this.contextStore) {
|
2002
|
-
this.contextStore.clear();
|
2003
|
-
}
|
2004
|
-
}
|
2005
|
-
/**
|
2006
|
-
* Get all stored data from context store
|
2007
|
-
*/
|
2008
|
-
getStoredData() {
|
2009
|
-
if (this.contextStore) {
|
2010
|
-
return this.contextStore.getAllData();
|
2011
|
-
}
|
2012
|
-
return {};
|
2013
|
-
}
|
2014
|
-
/**
|
2015
|
-
* Get step summary from context store
|
2016
|
-
*/
|
2017
|
-
getStepSummary() {
|
2018
|
-
if (this.contextStore) {
|
2019
|
-
return this.contextStore.getStepSummary();
|
2020
|
-
}
|
2021
|
-
return "";
|
2022
|
-
}
|
2023
1975
|
async getUIContext(action) {
|
2024
1976
|
if (action && (action === "extract" || action === "assert" || action === "captcha")) {
|
2025
1977
|
return await parseContextFromWebPage(this.page, {
|
@@ -2198,6 +2150,23 @@ var PageAgent = class {
|
|
2198
2150
|
metadata
|
2199
2151
|
};
|
2200
2152
|
}
|
2153
|
+
async aiRightClick(locatePrompt, opt) {
|
2154
|
+
const detailedLocateParam = this.buildDetailedLocateParam(
|
2155
|
+
locatePrompt,
|
2156
|
+
opt
|
2157
|
+
);
|
2158
|
+
const plans = buildPlans("RightClick", detailedLocateParam);
|
2159
|
+
const { executor, output } = await this.taskExecutor.runPlans(
|
2160
|
+
taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
|
2161
|
+
plans,
|
2162
|
+
{ cacheable: opt?.cacheable }
|
2163
|
+
);
|
2164
|
+
const metadata = this.afterTaskRunning(executor);
|
2165
|
+
return {
|
2166
|
+
result: output,
|
2167
|
+
metadata
|
2168
|
+
};
|
2169
|
+
}
|
2201
2170
|
async aiInput(value, locatePrompt, opt) {
|
2202
2171
|
(0, import_utils12.assert)(
|
2203
2172
|
typeof value === "string",
|
@@ -2255,35 +2224,9 @@ var PageAgent = class {
|
|
2255
2224
|
};
|
2256
2225
|
}
|
2257
2226
|
async aiAction(taskPrompt, opt) {
|
2258
|
-
const originalPrompt = taskPrompt;
|
2259
|
-
let processedPrompt = taskPrompt;
|
2260
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2261
|
-
try {
|
2262
|
-
const storedData = this.contextStore.getAllData();
|
2263
|
-
if (Object.keys(storedData).length > 0) {
|
2264
|
-
debug4("Available data for aiAction:", {
|
2265
|
-
prompt: taskPrompt,
|
2266
|
-
availableData: storedData
|
2267
|
-
});
|
2268
|
-
}
|
2269
|
-
} catch (error) {
|
2270
|
-
debug4("Context store operation failed:", error);
|
2271
|
-
}
|
2272
|
-
}
|
2273
2227
|
const cacheable = opt?.cacheable;
|
2274
2228
|
const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
|
2275
|
-
|
2276
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2277
|
-
try {
|
2278
|
-
contextData = this.contextStore.getAllData();
|
2279
|
-
if (contextData && Object.keys(contextData).length === 0) {
|
2280
|
-
contextData = void 0;
|
2281
|
-
}
|
2282
|
-
} catch (error) {
|
2283
|
-
debug4("Failed to get context data for cache:", error);
|
2284
|
-
}
|
2285
|
-
}
|
2286
|
-
const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt, contextData);
|
2229
|
+
const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
|
2287
2230
|
if (matchedCache && this.taskCache?.isCacheResultUsed) {
|
2288
2231
|
const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
|
2289
2232
|
taskPrompt,
|
@@ -2293,28 +2236,6 @@ var PageAgent = class {
|
|
2293
2236
|
debug4("matched cache, will call .runYaml to run the action");
|
2294
2237
|
const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
|
2295
2238
|
const result = await this.runYaml(yaml5);
|
2296
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2297
|
-
try {
|
2298
|
-
const executionResult = {
|
2299
|
-
success: true,
|
2300
|
-
actionType: "cached",
|
2301
|
-
description: `Executed cached action: ${processedPrompt}`,
|
2302
|
-
timing: result.metadata?.totalTime
|
2303
|
-
};
|
2304
|
-
this.contextStore.addStep({
|
2305
|
-
type: "action",
|
2306
|
-
summary: `Action: ${processedPrompt} (cached)`,
|
2307
|
-
prompt: processedPrompt,
|
2308
|
-
executionResult
|
2309
|
-
});
|
2310
|
-
debug4("Added cached action step to context store:", {
|
2311
|
-
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2312
|
-
totalSteps: this.contextStore.getRecentSteps(100).length
|
2313
|
-
});
|
2314
|
-
} catch (error) {
|
2315
|
-
debug4("Failed to add cached action step:", error);
|
2316
|
-
}
|
2317
|
-
}
|
2318
2239
|
return {
|
2319
2240
|
result: result.result,
|
2320
2241
|
metadata: metadata2
|
@@ -2339,114 +2260,17 @@ var PageAgent = class {
|
|
2339
2260
|
prompt: taskPrompt,
|
2340
2261
|
yamlWorkflow: yamlFlowStr
|
2341
2262
|
},
|
2342
|
-
matchedCache
|
2343
|
-
contextData
|
2344
|
-
// Pass context data for cache creation
|
2263
|
+
matchedCache
|
2345
2264
|
);
|
2346
2265
|
}
|
2347
2266
|
const metadata = this.afterTaskRunning(executor);
|
2348
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2349
|
-
try {
|
2350
|
-
const executionResult = this.analyzeExecutionResults(executor, originalPrompt);
|
2351
|
-
this.contextStore.addStep({
|
2352
|
-
type: "action",
|
2353
|
-
summary: `Action: ${processedPrompt}`,
|
2354
|
-
prompt: processedPrompt,
|
2355
|
-
executionResult
|
2356
|
-
});
|
2357
|
-
debug4("Added action step with execution result to context store:", {
|
2358
|
-
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2359
|
-
totalSteps: this.contextStore.getRecentSteps(100).length,
|
2360
|
-
executionResult
|
2361
|
-
});
|
2362
|
-
} catch (error) {
|
2363
|
-
debug4("Failed to analyze execution results, adding step without execution result:", error);
|
2364
|
-
try {
|
2365
|
-
this.contextStore.addStep({
|
2366
|
-
type: "action",
|
2367
|
-
summary: `Action: ${processedPrompt}`,
|
2368
|
-
prompt: processedPrompt
|
2369
|
-
});
|
2370
|
-
} catch (stepError) {
|
2371
|
-
debug4("Failed to add action step:", stepError);
|
2372
|
-
}
|
2373
|
-
}
|
2374
|
-
}
|
2375
2267
|
return {
|
2376
2268
|
result: output,
|
2377
2269
|
metadata
|
2378
2270
|
};
|
2379
2271
|
}
|
2380
2272
|
async aiQuery(demand) {
|
2381
|
-
|
2382
|
-
let storageKey;
|
2383
|
-
try {
|
2384
|
-
const aiModel = await import("misoai-core/ai-model");
|
2385
|
-
const contextStore = aiModel.getContextStore();
|
2386
|
-
if (typeof demand === "string") {
|
2387
|
-
const storageInstruction = contextStore.parseStorageInstruction(demand);
|
2388
|
-
if (storageInstruction) {
|
2389
|
-
storageKey = storageInstruction.key;
|
2390
|
-
processedDemand = storageInstruction.cleanText;
|
2391
|
-
contextStore._pendingAliases = storageInstruction.aliases;
|
2392
|
-
} else {
|
2393
|
-
const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
|
2394
|
-
if (storageMatch) {
|
2395
|
-
storageKey = storageMatch[1];
|
2396
|
-
processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
|
2397
|
-
}
|
2398
|
-
}
|
2399
|
-
}
|
2400
|
-
} catch (error) {
|
2401
|
-
debug4("Context store not available:", error);
|
2402
|
-
}
|
2403
|
-
const { output, executor } = await this.taskExecutor.query(processedDemand);
|
2404
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2405
|
-
if (storageKey && output) {
|
2406
|
-
try {
|
2407
|
-
const pendingAliases = this.contextStore._pendingAliases;
|
2408
|
-
if (pendingAliases) {
|
2409
|
-
this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
|
2410
|
-
delete this.contextStore._pendingAliases;
|
2411
|
-
debug4("Stored query result with aliases:", {
|
2412
|
-
key: storageKey,
|
2413
|
-
value: output,
|
2414
|
-
aliases: pendingAliases
|
2415
|
-
});
|
2416
|
-
} else {
|
2417
|
-
this.contextStore.storeData(storageKey, output);
|
2418
|
-
debug4("Stored query result:", {
|
2419
|
-
key: storageKey,
|
2420
|
-
value: output
|
2421
|
-
});
|
2422
|
-
}
|
2423
|
-
this.contextStore.addStep({
|
2424
|
-
type: "query",
|
2425
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
|
2426
|
-
data: output,
|
2427
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2428
|
-
});
|
2429
|
-
debug4("Added query step to context store:", {
|
2430
|
-
storageKey,
|
2431
|
-
totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
|
2432
|
-
totalSteps: this.contextStore.getRecentSteps(100).length
|
2433
|
-
});
|
2434
|
-
} catch (error) {
|
2435
|
-
debug4("Failed to store query result:", error);
|
2436
|
-
}
|
2437
|
-
} else {
|
2438
|
-
try {
|
2439
|
-
this.contextStore.addStep({
|
2440
|
-
type: "query",
|
2441
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
|
2442
|
-
data: output,
|
2443
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2444
|
-
});
|
2445
|
-
} catch (error) {
|
2446
|
-
debug4("Failed to add query step:", error);
|
2447
|
-
}
|
2448
|
-
}
|
2449
|
-
}
|
2273
|
+
const { output, executor } = await this.taskExecutor.query(demand);
|
2450
2274
|
const metadata = this.afterTaskRunning(executor);
|
2451
2275
|
return {
|
2452
2276
|
result: output,
|
@@ -2556,48 +2380,6 @@ var PageAgent = class {
|
|
2556
2380
|
};
|
2557
2381
|
}
|
2558
2382
|
async aiAssert(assertion, msg, opt) {
|
2559
|
-
let executionContext = "";
|
2560
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2561
|
-
try {
|
2562
|
-
const recentSteps = this.contextStore.getRecentSteps(3);
|
2563
|
-
const stepsWithExecutionResults = recentSteps.filter((step) => step.executionResult);
|
2564
|
-
const storedData = this.contextStore.getAllData();
|
2565
|
-
if (stepsWithExecutionResults.length > 0) {
|
2566
|
-
const recentActions = stepsWithExecutionResults.map((step) => {
|
2567
|
-
const result = step.executionResult;
|
2568
|
-
return `- ${result.description}${result.success ? "" : " (FAILED)"}`;
|
2569
|
-
}).join("\n");
|
2570
|
-
executionContext = `
|
2571
|
-
|
2572
|
-
Recent actions performed:
|
2573
|
-
${recentActions}
|
2574
|
-
|
2575
|
-
This context may help verify the assertion.`;
|
2576
|
-
}
|
2577
|
-
if (storedData && Object.keys(storedData).length > 0) {
|
2578
|
-
executionContext += `
|
2579
|
-
|
2580
|
-
Available data for reference:
|
2581
|
-
${JSON.stringify(storedData, null, 2)}
|
2582
|
-
|
2583
|
-
Note: If the assertion references any data keys or natural language equivalents, consider the stored values when verifying.`;
|
2584
|
-
debug4("Available data for aiAssert:", {
|
2585
|
-
assertion,
|
2586
|
-
availableData: storedData
|
2587
|
-
});
|
2588
|
-
}
|
2589
|
-
this.contextStore.addStep({
|
2590
|
-
type: "assertion",
|
2591
|
-
summary: `Assertion: ${assertion}`,
|
2592
|
-
prompt: assertion
|
2593
|
-
});
|
2594
|
-
debug4("Added assertion step to context store:", {
|
2595
|
-
totalSteps: this.contextStore.getRecentSteps(100).length
|
2596
|
-
});
|
2597
|
-
} catch (error) {
|
2598
|
-
debug4("Context store operation failed:", error);
|
2599
|
-
}
|
2600
|
-
}
|
2601
2383
|
let currentUrl = "";
|
2602
2384
|
if (this.page.url) {
|
2603
2385
|
try {
|
@@ -2605,13 +2387,7 @@ Note: If the assertion references any data keys or natural language equivalents,
|
|
2605
2387
|
} catch (e) {
|
2606
2388
|
}
|
2607
2389
|
}
|
2608
|
-
|
2609
|
-
if (currentUrl) {
|
2610
|
-
assertionWithContext = `For the page at URL "${currentUrl}", ${assertion}`;
|
2611
|
-
}
|
2612
|
-
if (executionContext) {
|
2613
|
-
assertionWithContext += executionContext;
|
2614
|
-
}
|
2390
|
+
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2615
2391
|
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2616
2392
|
const metadata = this.afterTaskRunning(executor, true);
|
2617
2393
|
if (output && opt?.keepRawResponse) {
|
@@ -2822,81 +2598,42 @@ ${errors}`);
|
|
2822
2598
|
}
|
2823
2599
|
throw new Error("evaluateJavaScript is not supported in current agent");
|
2824
2600
|
}
|
2825
|
-
async
|
2826
|
-
|
2827
|
-
|
2828
|
-
|
2829
|
-
|
2830
|
-
|
2831
|
-
|
2832
|
-
|
2833
|
-
|
2834
|
-
|
2835
|
-
|
2836
|
-
|
2837
|
-
|
2838
|
-
|
2839
|
-
|
2840
|
-
|
2841
|
-
|
2842
|
-
|
2843
|
-
|
2844
|
-
|
2845
|
-
|
2846
|
-
|
2847
|
-
|
2848
|
-
|
2849
|
-
|
2850
|
-
|
2851
|
-
|
2601
|
+
async logScreenshot(title, options) {
|
2602
|
+
const screenshotTitle = title || "untitled";
|
2603
|
+
const content = options?.content || "";
|
2604
|
+
const screenshot = await this.page.screenshotBase64?.();
|
2605
|
+
if (screenshot) {
|
2606
|
+
const executionDump = {
|
2607
|
+
name: screenshotTitle,
|
2608
|
+
description: content,
|
2609
|
+
tasks: [{
|
2610
|
+
type: "Screenshot",
|
2611
|
+
subType: "log",
|
2612
|
+
status: "finished",
|
2613
|
+
executor: null,
|
2614
|
+
param: {
|
2615
|
+
title: screenshotTitle,
|
2616
|
+
content
|
2617
|
+
},
|
2618
|
+
output: {
|
2619
|
+
screenshot
|
2620
|
+
},
|
2621
|
+
thought: `Logged screenshot: ${screenshotTitle}`,
|
2622
|
+
timing: {
|
2623
|
+
start: Date.now(),
|
2624
|
+
end: Date.now(),
|
2625
|
+
cost: 0
|
2626
|
+
}
|
2627
|
+
}],
|
2628
|
+
sdkVersion: "1.0.0",
|
2629
|
+
logTime: Date.now(),
|
2630
|
+
model_name: "screenshot"
|
2852
2631
|
};
|
2632
|
+
this.appendExecutionDump(executionDump);
|
2853
2633
|
}
|
2854
|
-
const actionType = lastAction.subType || "unknown";
|
2855
|
-
const elementInfo = this.extractElementInfo(lastLocate, lastAction);
|
2856
|
-
const description = this.generateActionDescription(actionType, lastAction.param, elementInfo);
|
2857
|
-
return {
|
2858
|
-
success: true,
|
2859
|
-
actionType,
|
2860
|
-
description,
|
2861
|
-
elementInfo,
|
2862
|
-
timing: lastAction.timing?.cost
|
2863
|
-
};
|
2864
|
-
}
|
2865
|
-
/**
|
2866
|
-
* Extract element information from locate task
|
2867
|
-
*/
|
2868
|
-
extractElementInfo(locateTask, _actionTask) {
|
2869
|
-
if (!locateTask?.output?.element)
|
2870
|
-
return void 0;
|
2871
|
-
const element = locateTask.output.element;
|
2872
|
-
return {
|
2873
|
-
type: element.attributes?.nodeType || "unknown",
|
2874
|
-
text: element.content || element.attributes?.placeholder || element.attributes?.title || "",
|
2875
|
-
location: `(${element.center[0]}, ${element.center[1]})`
|
2876
|
-
};
|
2877
2634
|
}
|
2878
|
-
|
2879
|
-
|
2880
|
-
*/
|
2881
|
-
generateActionDescription(actionType, param, elementInfo) {
|
2882
|
-
const elementDesc = elementInfo ? `'${elementInfo.text || elementInfo.type}' element` : "element";
|
2883
|
-
switch (actionType) {
|
2884
|
-
case "Tap":
|
2885
|
-
return `Clicked on ${elementDesc}`;
|
2886
|
-
case "Input":
|
2887
|
-
const inputValue = param?.value || "";
|
2888
|
-
return `Entered "${inputValue}" into ${elementDesc}`;
|
2889
|
-
case "KeyboardPress":
|
2890
|
-
return `Pressed ${param?.value || "key"}`;
|
2891
|
-
case "Scroll":
|
2892
|
-
return `Scrolled ${param?.direction || "on page"}`;
|
2893
|
-
case "Hover":
|
2894
|
-
return `Hovered over ${elementDesc}`;
|
2895
|
-
case "Drag":
|
2896
|
-
return `Dragged ${elementDesc}`;
|
2897
|
-
default:
|
2898
|
-
return `Performed ${actionType} action on ${elementDesc}`;
|
2899
|
-
}
|
2635
|
+
async destroy() {
|
2636
|
+
await this.page.destroy();
|
2900
2637
|
}
|
2901
2638
|
};
|
2902
2639
|
|
@@ -2919,7 +2656,7 @@ var Page = class {
|
|
2919
2656
|
this.everMoved = false;
|
2920
2657
|
this.underlyingPage = underlyingPage;
|
2921
2658
|
this.pageType = pageType;
|
2922
|
-
this.waitForNavigationTimeout = opts?.waitForNavigationTimeout
|
2659
|
+
this.waitForNavigationTimeout = opts?.waitForNavigationTimeout ?? import_constants3.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
|
2923
2660
|
}
|
2924
2661
|
async evaluate(pageFunction, arg) {
|
2925
2662
|
let result;
|
@@ -3199,9 +2936,9 @@ var WebPage = class extends Page {
|
|
3199
2936
|
}
|
3200
2937
|
async waitUntilNetworkIdle(options) {
|
3201
2938
|
await this.underlyingPage.waitForNetworkIdle({
|
3202
|
-
idleTime: options?.idleTime
|
3203
|
-
concurrency: options?.concurrency
|
3204
|
-
timeout: options?.timeout
|
2939
|
+
idleTime: options?.idleTime ?? import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME,
|
2940
|
+
concurrency: options?.concurrency ?? import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY,
|
2941
|
+
timeout: options?.timeout ?? this.waitForNetworkIdleTimeout
|
3205
2942
|
});
|
3206
2943
|
}
|
3207
2944
|
};
|