misoai-web 1.0.6 → 1.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +5 -349
  2. package/dist/es/agent.js +165 -428
  3. package/dist/es/agent.js.map +1 -1
  4. package/dist/es/bridge-mode-browser.js +10 -9
  5. package/dist/es/bridge-mode-browser.js.map +1 -1
  6. package/dist/es/bridge-mode.js +167 -430
  7. package/dist/es/bridge-mode.js.map +1 -1
  8. package/dist/es/chrome-extension.js +173 -435
  9. package/dist/es/chrome-extension.js.map +1 -1
  10. package/dist/es/index.js +185 -432
  11. package/dist/es/index.js.map +1 -1
  12. package/dist/es/midscene-playground.js +165 -428
  13. package/dist/es/midscene-playground.js.map +1 -1
  14. package/dist/es/midscene-server.js.map +1 -1
  15. package/dist/es/playground.js +165 -428
  16. package/dist/es/playground.js.map +1 -1
  17. package/dist/es/playwright-report.js +1 -1
  18. package/dist/es/playwright-report.js.map +1 -1
  19. package/dist/es/playwright.js +182 -429
  20. package/dist/es/playwright.js.map +1 -1
  21. package/dist/es/puppeteer-agent-launcher.js +169 -432
  22. package/dist/es/puppeteer-agent-launcher.js.map +1 -1
  23. package/dist/es/puppeteer.js +169 -432
  24. package/dist/es/puppeteer.js.map +1 -1
  25. package/dist/es/ui-utils.js.map +1 -1
  26. package/dist/es/utils.js +7 -4
  27. package/dist/es/utils.js.map +1 -1
  28. package/dist/es/yaml.js +29 -3
  29. package/dist/es/yaml.js.map +1 -1
  30. package/dist/lib/agent.js +163 -426
  31. package/dist/lib/agent.js.map +1 -1
  32. package/dist/lib/bridge-mode-browser.js +10 -9
  33. package/dist/lib/bridge-mode-browser.js.map +1 -1
  34. package/dist/lib/bridge-mode.js +165 -428
  35. package/dist/lib/bridge-mode.js.map +1 -1
  36. package/dist/lib/chrome-extension.js +171 -433
  37. package/dist/lib/chrome-extension.js.map +1 -1
  38. package/dist/lib/index.js +183 -430
  39. package/dist/lib/index.js.map +1 -1
  40. package/dist/lib/midscene-playground.js +163 -426
  41. package/dist/lib/midscene-playground.js.map +1 -1
  42. package/dist/lib/midscene-server.js.map +1 -1
  43. package/dist/lib/playground.js +163 -426
  44. package/dist/lib/playground.js.map +1 -1
  45. package/dist/lib/playwright-report.js +1 -1
  46. package/dist/lib/playwright-report.js.map +1 -1
  47. package/dist/lib/playwright.js +180 -427
  48. package/dist/lib/playwright.js.map +1 -1
  49. package/dist/lib/puppeteer-agent-launcher.js +167 -430
  50. package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
  51. package/dist/lib/puppeteer.js +167 -430
  52. package/dist/lib/puppeteer.js.map +1 -1
  53. package/dist/lib/ui-utils.js.map +1 -1
  54. package/dist/lib/utils.js +7 -4
  55. package/dist/lib/utils.js.map +1 -1
  56. package/dist/lib/yaml.js +29 -3
  57. package/dist/lib/yaml.js.map +1 -1
  58. package/dist/types/agent.d.ts +13 -51
  59. package/dist/types/bridge-mode-browser.d.ts +2 -3
  60. package/dist/types/bridge-mode.d.ts +2 -3
  61. package/dist/types/{browser-aec1055d.d.ts → browser-9b472ffb.d.ts} +1 -1
  62. package/dist/types/chrome-extension.d.ts +2 -3
  63. package/dist/types/index.d.ts +1 -2
  64. package/dist/types/midscene-server.d.ts +1 -2
  65. package/dist/types/{page-86ab0fe1.d.ts → page-ed0ecb44.d.ts} +19 -9
  66. package/dist/types/playground.d.ts +2 -3
  67. package/dist/types/playwright.d.ts +9 -2
  68. package/dist/types/puppeteer-agent-launcher.d.ts +1 -2
  69. package/dist/types/puppeteer.d.ts +6 -5
  70. package/dist/types/ui-utils.d.ts +1 -1
  71. package/dist/types/utils.d.ts +1 -2
  72. package/dist/types/yaml.d.ts +1 -2
  73. package/iife-script/htmlElement.js +53 -75
  74. package/iife-script/htmlElementDebug.js +35 -56
  75. package/package.json +24 -24
  76. package/LICENSE +0 -21
@@ -55,10 +55,11 @@ var ScriptPlayer = class {
55
55
  this.unnamedResultIndex = 0;
56
56
  this.pageAgent = null;
57
57
  this.result = {};
58
+ const target = script.target || script.web || script.android;
58
59
  if (import_utils.ifInBrowser) {
59
60
  this.output = void 0;
60
- } else if (script.target?.output) {
61
- this.output = (0, import_node_path.resolve)(process.cwd(), script.target.output);
61
+ } else if (target?.output) {
62
+ this.output = (0, import_node_path.resolve)(process.cwd(), target.output);
62
63
  } else {
63
64
  this.output = (0, import_node_path.join)((0, import_common.getMidsceneRunSubDir)("output"), `${process.pid}.json`);
64
65
  }
@@ -132,15 +133,20 @@ var ScriptPlayer = class {
132
133
  } else if ("aiAssert" in flowItem) {
133
134
  const assertTask = flowItem;
134
135
  const prompt = assertTask.aiAssert;
136
+ const msg = assertTask.errorMessage;
135
137
  (0, import_utils.assert)(prompt, "missing prompt for aiAssert");
136
138
  (0, import_utils.assert)(
137
139
  typeof prompt === "string",
138
140
  "prompt for aiAssert must be a string"
139
141
  );
140
- await agent.aiAssert(prompt);
142
+ await agent.aiAssert(prompt, msg);
141
143
  } else if ("aiQuery" in flowItem) {
142
144
  const queryTask = flowItem;
143
145
  const prompt = queryTask.aiQuery;
146
+ const options = {
147
+ domIncluded: queryTask.domIncluded,
148
+ screenshotIncluded: queryTask.screenshotIncluded
149
+ };
144
150
  (0, import_utils.assert)(prompt, "missing prompt for aiQuery");
145
151
  (0, import_utils.assert)(
146
152
  typeof prompt === "string",
@@ -151,6 +157,10 @@ var ScriptPlayer = class {
151
157
  } else if ("aiNumber" in flowItem) {
152
158
  const numberTask = flowItem;
153
159
  const prompt = numberTask.aiNumber;
160
+ const options = {
161
+ domIncluded: numberTask.domIncluded,
162
+ screenshotIncluded: numberTask.screenshotIncluded
163
+ };
154
164
  (0, import_utils.assert)(prompt, "missing prompt for number");
155
165
  (0, import_utils.assert)(
156
166
  typeof prompt === "string",
@@ -161,6 +171,10 @@ var ScriptPlayer = class {
161
171
  } else if ("aiString" in flowItem) {
162
172
  const stringTask = flowItem;
163
173
  const prompt = stringTask.aiString;
174
+ const options = {
175
+ domIncluded: stringTask.domIncluded,
176
+ screenshotIncluded: stringTask.screenshotIncluded
177
+ };
164
178
  (0, import_utils.assert)(prompt, "missing prompt for string");
165
179
  (0, import_utils.assert)(
166
180
  typeof prompt === "string",
@@ -171,6 +185,10 @@ var ScriptPlayer = class {
171
185
  } else if ("aiBoolean" in flowItem) {
172
186
  const booleanTask = flowItem;
173
187
  const prompt = booleanTask.aiBoolean;
188
+ const options = {
189
+ domIncluded: booleanTask.domIncluded,
190
+ screenshotIncluded: booleanTask.screenshotIncluded
191
+ };
174
192
  (0, import_utils.assert)(prompt, "missing prompt for boolean");
175
193
  (0, import_utils.assert)(
176
194
  typeof prompt === "string",
@@ -213,6 +231,9 @@ var ScriptPlayer = class {
213
231
  } else if ("aiTap" in flowItem) {
214
232
  const tapTask = flowItem;
215
233
  await agent.aiTap(tapTask.aiTap, tapTask);
234
+ } else if ("aiRightClick" in flowItem) {
235
+ const rightClickTask = flowItem;
236
+ await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
216
237
  } else if ("aiHover" in flowItem) {
217
238
  const hoverTask = flowItem;
218
239
  await agent.aiHover(hoverTask.aiHover, hoverTask);
@@ -235,6 +256,11 @@ var ScriptPlayer = class {
235
256
  evaluateJavaScriptTask.javascript
236
257
  );
237
258
  this.setResult(evaluateJavaScriptTask.name, result);
259
+ } else if ("logScreenshot" in flowItem) {
260
+ const logScreenshotTask = flowItem;
261
+ await agent.logScreenshot(logScreenshotTask.logScreenshot, {
262
+ content: logScreenshotTask.content || ""
263
+ });
238
264
  } else {
239
265
  throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
240
266
  }
@@ -483,7 +509,8 @@ var WebElementInfo = class {
483
509
  id,
484
510
  attributes,
485
511
  indexId,
486
- xpaths
512
+ xpaths,
513
+ isVisible
487
514
  }) {
488
515
  this.content = content;
489
516
  this.rect = rect;
@@ -496,6 +523,7 @@ var WebElementInfo = class {
496
523
  this.attributes = attributes;
497
524
  this.indexId = indexId;
498
525
  this.xpaths = xpaths;
526
+ this.isVisible = isVisible;
499
527
  }
500
528
  };
501
529
 
@@ -518,14 +546,15 @@ async function parseContextFromWebPage(page, _opt) {
518
546
  })
519
547
  ]);
520
548
  const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
521
- const { rect, id, content, attributes, locator, indexId } = elementInfo;
549
+ const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
522
550
  return new WebElementInfo({
523
551
  rect,
524
552
  locator,
525
553
  id,
526
554
  content,
527
555
  attributes,
528
- indexId
556
+ indexId,
557
+ isVisible
529
558
  });
530
559
  });
531
560
  (0, import_utils4.assert)(screenshotBase64, "screenshotBase64 is required");
@@ -555,7 +584,7 @@ function printReportMsg(filepath) {
555
584
  (0, import_utils4.logMsg)(`Midscene - report file updated: ${filepath}`);
556
585
  }
557
586
  function replaceIllegalPathCharsAndSpace(str) {
558
- return str.replace(/[/\\:*?"<>| ]/g, "-");
587
+ return str.replace(/[:*?"<>| ]/g, "-");
559
588
  }
560
589
  function forceClosePopup(page, debug6) {
561
590
  page.on("popup", async (popup) => {
@@ -869,10 +898,10 @@ var PageTaskExecutor = class {
869
898
  if (!taskParam || !taskParam.value) {
870
899
  return;
871
900
  }
872
- await this.page.keyboard.type(taskParam.value);
873
- } else {
874
- await this.page.keyboard.type(taskParam.value);
875
901
  }
902
+ await this.page.keyboard.type(taskParam.value, {
903
+ autoDismissKeyboard: taskParam.autoDismissKeyboard
904
+ });
876
905
  }
877
906
  };
878
907
  tasks.push(taskActionInput);
@@ -901,6 +930,22 @@ var PageTaskExecutor = class {
901
930
  }
902
931
  };
903
932
  tasks.push(taskActionTap);
933
+ } else if (plan2.type === "RightClick") {
934
+ const taskActionRightClick = {
935
+ type: "Action",
936
+ subType: "RightClick",
937
+ thought: plan2.thought,
938
+ locate: plan2.locate,
939
+ executor: async (param, { element }) => {
940
+ (0, import_utils6.assert)(element, "Element not found, cannot right click");
941
+ await this.page.mouse.click(
942
+ element.center[0],
943
+ element.center[1],
944
+ { button: "right" }
945
+ );
946
+ }
947
+ };
948
+ tasks.push(taskActionRightClick);
904
949
  } else if (plan2.type === "Drag") {
905
950
  const taskActionDrag = {
906
951
  type: "Action",
@@ -1429,7 +1474,7 @@ var PageTaskExecutor = class {
1429
1474
  executor: taskExecutor
1430
1475
  };
1431
1476
  }
1432
- async createTypeQueryTask(type, demand) {
1477
+ async createTypeQueryTask(type, demand, opt) {
1433
1478
  const taskExecutor = new import_misoai_core.Executor(
1434
1479
  taskTitleStr(
1435
1480
  type,
@@ -1460,7 +1505,10 @@ var PageTaskExecutor = class {
1460
1505
  result: `${type}, ${demand}`
1461
1506
  };
1462
1507
  }
1463
- const { data, usage } = await this.insight.extract(demandInput);
1508
+ const { data, usage } = await this.insight.extract(
1509
+ demandInput,
1510
+ opt
1511
+ );
1464
1512
  let outputResult = data;
1465
1513
  if (ifTypeRestricted) {
1466
1514
  (0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
@@ -1480,17 +1528,17 @@ var PageTaskExecutor = class {
1480
1528
  executor: taskExecutor
1481
1529
  };
1482
1530
  }
1483
- async query(demand) {
1484
- return this.createTypeQueryTask("Query", demand);
1531
+ async query(demand, opt) {
1532
+ return this.createTypeQueryTask("Query", demand, opt);
1485
1533
  }
1486
- async boolean(prompt) {
1487
- return this.createTypeQueryTask("Boolean", prompt);
1534
+ async boolean(prompt, opt) {
1535
+ return this.createTypeQueryTask("Boolean", prompt, opt);
1488
1536
  }
1489
- async number(prompt) {
1490
- return this.createTypeQueryTask("Number", prompt);
1537
+ async number(prompt, opt) {
1538
+ return this.createTypeQueryTask("Number", prompt, opt);
1491
1539
  }
1492
- async string(prompt) {
1493
- return this.createTypeQueryTask("String", prompt);
1540
+ async string(prompt, opt) {
1541
+ return this.createTypeQueryTask("String", prompt, opt);
1494
1542
  }
1495
1543
  async assert(assertion) {
1496
1544
  const description = `assert: ${assertion}`;
@@ -1626,7 +1674,7 @@ function buildPlans(type, locateParam, param) {
1626
1674
  param: locateParam,
1627
1675
  thought: ""
1628
1676
  } : null;
1629
- if (type === "Tap" || type === "Hover") {
1677
+ if (type === "Tap" || type === "Hover" || type === "RightClick") {
1630
1678
  (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
1631
1679
  (0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
1632
1680
  const tapPlan = {
@@ -1706,7 +1754,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
1706
1754
  var import_semver = __toESM(require("semver"));
1707
1755
 
1708
1756
  // package.json
1709
- var version = "1.0.5";
1757
+ var version = "1.0.3";
1710
1758
 
1711
1759
  // src/common/task-cache.ts
1712
1760
  var debug3 = (0, import_logger3.getDebug)("cache");
@@ -1734,70 +1782,44 @@ var TaskCache = class {
1734
1782
  this.cache = cacheContent;
1735
1783
  this.cacheOriginalLength = this.cache.caches.length;
1736
1784
  }
1737
- matchCache(prompt, type, contextData) {
1738
- const contextHash = contextData ? this.generateContextHash(contextData) : void 0;
1785
+ matchCache(prompt, type) {
1739
1786
  for (let i = 0; i < this.cacheOriginalLength; i++) {
1740
1787
  const item = this.cache.caches[i];
1741
1788
  const key = `${type}:${prompt}:${i}`;
1742
- if (item.type !== type || item.prompt !== prompt || this.matchedCacheIndices.has(key)) {
1743
- continue;
1744
- }
1745
- if (type === "plan" && item.type === "plan") {
1746
- const planItem = item;
1747
- if (contextHash && planItem.contextHash) {
1748
- if (contextHash !== planItem.contextHash) {
1749
- debug3("cache context mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
1750
- continue;
1789
+ if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
1790
+ this.matchedCacheIndices.add(key);
1791
+ debug3(
1792
+ "cache found and marked as used, type: %s, prompt: %s, index: %d",
1793
+ type,
1794
+ prompt,
1795
+ i
1796
+ );
1797
+ return {
1798
+ cacheContent: item,
1799
+ updateFn: (cb) => {
1800
+ debug3(
1801
+ "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
1802
+ type,
1803
+ prompt,
1804
+ i
1805
+ );
1806
+ cb(item);
1807
+ debug3(
1808
+ "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
1809
+ type,
1810
+ prompt,
1811
+ i
1812
+ );
1813
+ this.flushCacheToFile();
1751
1814
  }
1752
- } else if (contextHash || planItem.contextHash) {
1753
- debug3("cache context availability mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
1754
- continue;
1755
- }
1815
+ };
1756
1816
  }
1757
- this.matchedCacheIndices.add(key);
1758
- debug3(
1759
- "cache found and marked as used, type: %s, prompt: %s, index: %d, contextMatch: %s",
1760
- type,
1761
- prompt,
1762
- i,
1763
- contextHash ? "yes" : "no-context"
1764
- );
1765
- return {
1766
- cacheContent: item,
1767
- updateFn: (cb) => {
1768
- debug3(
1769
- "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
1770
- type,
1771
- prompt,
1772
- i
1773
- );
1774
- cb(item);
1775
- debug3(
1776
- "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
1777
- type,
1778
- prompt,
1779
- i
1780
- );
1781
- this.flushCacheToFile();
1782
- }
1783
- };
1784
1817
  }
1785
- debug3("no unused cache found, type: %s, prompt: %s, contextHash: %s", type, prompt, contextHash);
1818
+ debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
1786
1819
  return void 0;
1787
1820
  }
1788
- generateContextHash(contextData) {
1789
- const sortedKeys = Object.keys(contextData).sort();
1790
- const stableString = sortedKeys.map((key) => `${key}:${JSON.stringify(contextData[key])}`).join("|");
1791
- let hash = 0;
1792
- for (let i = 0; i < stableString.length; i++) {
1793
- const char = stableString.charCodeAt(i);
1794
- hash = (hash << 5) - hash + char;
1795
- hash = hash & hash;
1796
- }
1797
- return hash.toString(36);
1798
- }
1799
- matchPlanCache(prompt, contextData) {
1800
- return this.matchCache(prompt, "plan", contextData);
1821
+ matchPlanCache(prompt) {
1822
+ return this.matchCache(prompt, "plan");
1801
1823
  }
1802
1824
  matchLocateCache(prompt) {
1803
1825
  return this.matchCache(prompt, "locate");
@@ -1863,8 +1885,14 @@ cache file: ${cacheFile}`
1863
1885
  return;
1864
1886
  }
1865
1887
  try {
1888
+ const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
1889
+ if (!(0, import_node_fs2.existsSync)(dir)) {
1890
+ (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
1891
+ debug3("created cache directory: %s", dir);
1892
+ }
1866
1893
  const yamlData = import_js_yaml3.default.dump(this.cache);
1867
1894
  (0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
1895
+ debug3("cache flushed to file: %s", this.cacheFilePath);
1868
1896
  } catch (err) {
1869
1897
  debug3(
1870
1898
  "write cache to file failed, path: %s, error: %s",
@@ -1873,16 +1901,11 @@ cache file: ${cacheFile}`
1873
1901
  );
1874
1902
  }
1875
1903
  }
1876
- updateOrAppendCacheRecord(newRecord, cachedRecord, contextData) {
1904
+ updateOrAppendCacheRecord(newRecord, cachedRecord) {
1877
1905
  if (cachedRecord) {
1878
1906
  if (newRecord.type === "plan") {
1879
1907
  cachedRecord.updateFn((cache) => {
1880
- const planCache = cache;
1881
- planCache.yamlWorkflow = newRecord.yamlWorkflow;
1882
- if (contextData) {
1883
- planCache.contextHash = this.generateContextHash(contextData);
1884
- planCache.contextData = { ...contextData };
1885
- }
1908
+ cache.yamlWorkflow = newRecord.yamlWorkflow;
1886
1909
  });
1887
1910
  } else {
1888
1911
  cachedRecord.updateFn((cache) => {
@@ -1890,11 +1913,6 @@ cache file: ${cacheFile}`
1890
1913
  });
1891
1914
  }
1892
1915
  } else {
1893
- if (newRecord.type === "plan" && contextData) {
1894
- const planRecord = newRecord;
1895
- planRecord.contextHash = this.generateContextHash(contextData);
1896
- planRecord.contextData = { ...contextData };
1897
- }
1898
1916
  this.appendCache(newRecord);
1899
1917
  }
1900
1918
  }
@@ -1924,13 +1942,10 @@ var PageAgent = class {
1924
1942
  generateReport: true,
1925
1943
  autoPrintReportMsg: true,
1926
1944
  groupName: "Midscene Report",
1927
- groupDescription: "",
1928
- enableCumulativeContext: true,
1929
- autoClearContext: false
1945
+ groupDescription: ""
1930
1946
  },
1931
1947
  opts || {}
1932
1948
  );
1933
- this.initializeContextStore();
1934
1949
  if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
1935
1950
  this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || import_constants2.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
1936
1951
  this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || import_constants2.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
@@ -1957,69 +1972,6 @@ var PageAgent = class {
1957
1972
  opts?.testId || this.page.pageType || "web"
1958
1973
  );
1959
1974
  }
1960
- /**
1961
- * Initialize context store for cumulative context functionality
1962
- */
1963
- async initializeContextStore() {
1964
- if (!this.opts.enableCumulativeContext) {
1965
- debug4("Cumulative context disabled via options");
1966
- return;
1967
- }
1968
- try {
1969
- const aiModel = await import("misoai-core/ai-model");
1970
- this.contextStore = aiModel.getContextStore();
1971
- debug4("Context store initialized successfully", {
1972
- autoClearContext: this.opts.autoClearContext,
1973
- testId: this.opts.testId
1974
- });
1975
- if (this.opts.autoClearContext) {
1976
- this.contextStore.clear();
1977
- debug4("Context store cleared due to autoClearContext option");
1978
- } else {
1979
- const existingData = this.contextStore.getAllData();
1980
- const existingSteps = this.contextStore.getRecentSteps(100).length;
1981
- debug4("Context store preserving existing data", {
1982
- existingDataKeys: Object.keys(existingData),
1983
- existingStepsCount: existingSteps
1984
- });
1985
- }
1986
- } catch (error) {
1987
- debug4("Failed to initialize context store:", error);
1988
- console.warn("⚠️ Could not initialize context store:", error);
1989
- }
1990
- }
1991
- /**
1992
- * Get the context store instance
1993
- */
1994
- getContextStore() {
1995
- return this.contextStore;
1996
- }
1997
- /**
1998
- * Clear the context store
1999
- */
2000
- clearContext() {
2001
- if (this.contextStore) {
2002
- this.contextStore.clear();
2003
- }
2004
- }
2005
- /**
2006
- * Get all stored data from context store
2007
- */
2008
- getStoredData() {
2009
- if (this.contextStore) {
2010
- return this.contextStore.getAllData();
2011
- }
2012
- return {};
2013
- }
2014
- /**
2015
- * Get step summary from context store
2016
- */
2017
- getStepSummary() {
2018
- if (this.contextStore) {
2019
- return this.contextStore.getStepSummary();
2020
- }
2021
- return "";
2022
- }
2023
1975
  async getUIContext(action) {
2024
1976
  if (action && (action === "extract" || action === "assert" || action === "captcha")) {
2025
1977
  return await parseContextFromWebPage(this.page, {
@@ -2198,6 +2150,23 @@ var PageAgent = class {
2198
2150
  metadata
2199
2151
  };
2200
2152
  }
2153
+ async aiRightClick(locatePrompt, opt) {
2154
+ const detailedLocateParam = this.buildDetailedLocateParam(
2155
+ locatePrompt,
2156
+ opt
2157
+ );
2158
+ const plans = buildPlans("RightClick", detailedLocateParam);
2159
+ const { executor, output } = await this.taskExecutor.runPlans(
2160
+ taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
2161
+ plans,
2162
+ { cacheable: opt?.cacheable }
2163
+ );
2164
+ const metadata = this.afterTaskRunning(executor);
2165
+ return {
2166
+ result: output,
2167
+ metadata
2168
+ };
2169
+ }
2201
2170
  async aiInput(value, locatePrompt, opt) {
2202
2171
  (0, import_utils12.assert)(
2203
2172
  typeof value === "string",
@@ -2255,35 +2224,9 @@ var PageAgent = class {
2255
2224
  };
2256
2225
  }
2257
2226
  async aiAction(taskPrompt, opt) {
2258
- const originalPrompt = taskPrompt;
2259
- let processedPrompt = taskPrompt;
2260
- if (this.opts.enableCumulativeContext && this.contextStore) {
2261
- try {
2262
- const storedData = this.contextStore.getAllData();
2263
- if (Object.keys(storedData).length > 0) {
2264
- debug4("Available data for aiAction:", {
2265
- prompt: taskPrompt,
2266
- availableData: storedData
2267
- });
2268
- }
2269
- } catch (error) {
2270
- debug4("Context store operation failed:", error);
2271
- }
2272
- }
2273
2227
  const cacheable = opt?.cacheable;
2274
2228
  const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
2275
- let contextData;
2276
- if (this.opts.enableCumulativeContext && this.contextStore) {
2277
- try {
2278
- contextData = this.contextStore.getAllData();
2279
- if (contextData && Object.keys(contextData).length === 0) {
2280
- contextData = void 0;
2281
- }
2282
- } catch (error) {
2283
- debug4("Failed to get context data for cache:", error);
2284
- }
2285
- }
2286
- const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt, contextData);
2229
+ const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
2287
2230
  if (matchedCache && this.taskCache?.isCacheResultUsed) {
2288
2231
  const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
2289
2232
  taskPrompt,
@@ -2293,28 +2236,6 @@ var PageAgent = class {
2293
2236
  debug4("matched cache, will call .runYaml to run the action");
2294
2237
  const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
2295
2238
  const result = await this.runYaml(yaml5);
2296
- if (this.opts.enableCumulativeContext && this.contextStore) {
2297
- try {
2298
- const executionResult = {
2299
- success: true,
2300
- actionType: "cached",
2301
- description: `Executed cached action: ${processedPrompt}`,
2302
- timing: result.metadata?.totalTime
2303
- };
2304
- this.contextStore.addStep({
2305
- type: "action",
2306
- summary: `Action: ${processedPrompt} (cached)`,
2307
- prompt: processedPrompt,
2308
- executionResult
2309
- });
2310
- debug4("Added cached action step to context store:", {
2311
- stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
2312
- totalSteps: this.contextStore.getRecentSteps(100).length
2313
- });
2314
- } catch (error) {
2315
- debug4("Failed to add cached action step:", error);
2316
- }
2317
- }
2318
2239
  return {
2319
2240
  result: result.result,
2320
2241
  metadata: metadata2
@@ -2339,114 +2260,17 @@ var PageAgent = class {
2339
2260
  prompt: taskPrompt,
2340
2261
  yamlWorkflow: yamlFlowStr
2341
2262
  },
2342
- matchedCache,
2343
- contextData
2344
- // Pass context data for cache creation
2263
+ matchedCache
2345
2264
  );
2346
2265
  }
2347
2266
  const metadata = this.afterTaskRunning(executor);
2348
- if (this.opts.enableCumulativeContext && this.contextStore) {
2349
- try {
2350
- const executionResult = this.analyzeExecutionResults(executor, originalPrompt);
2351
- this.contextStore.addStep({
2352
- type: "action",
2353
- summary: `Action: ${processedPrompt}`,
2354
- prompt: processedPrompt,
2355
- executionResult
2356
- });
2357
- debug4("Added action step with execution result to context store:", {
2358
- stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
2359
- totalSteps: this.contextStore.getRecentSteps(100).length,
2360
- executionResult
2361
- });
2362
- } catch (error) {
2363
- debug4("Failed to analyze execution results, adding step without execution result:", error);
2364
- try {
2365
- this.contextStore.addStep({
2366
- type: "action",
2367
- summary: `Action: ${processedPrompt}`,
2368
- prompt: processedPrompt
2369
- });
2370
- } catch (stepError) {
2371
- debug4("Failed to add action step:", stepError);
2372
- }
2373
- }
2374
- }
2375
2267
  return {
2376
2268
  result: output,
2377
2269
  metadata
2378
2270
  };
2379
2271
  }
2380
2272
  async aiQuery(demand) {
2381
- let processedDemand = demand;
2382
- let storageKey;
2383
- try {
2384
- const aiModel = await import("misoai-core/ai-model");
2385
- const contextStore = aiModel.getContextStore();
2386
- if (typeof demand === "string") {
2387
- const storageInstruction = contextStore.parseStorageInstruction(demand);
2388
- if (storageInstruction) {
2389
- storageKey = storageInstruction.key;
2390
- processedDemand = storageInstruction.cleanText;
2391
- contextStore._pendingAliases = storageInstruction.aliases;
2392
- } else {
2393
- const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
2394
- if (storageMatch) {
2395
- storageKey = storageMatch[1];
2396
- processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
2397
- }
2398
- }
2399
- }
2400
- } catch (error) {
2401
- debug4("Context store not available:", error);
2402
- }
2403
- const { output, executor } = await this.taskExecutor.query(processedDemand);
2404
- if (this.opts.enableCumulativeContext && this.contextStore) {
2405
- if (storageKey && output) {
2406
- try {
2407
- const pendingAliases = this.contextStore._pendingAliases;
2408
- if (pendingAliases) {
2409
- this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
2410
- delete this.contextStore._pendingAliases;
2411
- debug4("Stored query result with aliases:", {
2412
- key: storageKey,
2413
- value: output,
2414
- aliases: pendingAliases
2415
- });
2416
- } else {
2417
- this.contextStore.storeData(storageKey, output);
2418
- debug4("Stored query result:", {
2419
- key: storageKey,
2420
- value: output
2421
- });
2422
- }
2423
- this.contextStore.addStep({
2424
- type: "query",
2425
- summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
2426
- data: output,
2427
- prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2428
- });
2429
- debug4("Added query step to context store:", {
2430
- storageKey,
2431
- totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
2432
- totalSteps: this.contextStore.getRecentSteps(100).length
2433
- });
2434
- } catch (error) {
2435
- debug4("Failed to store query result:", error);
2436
- }
2437
- } else {
2438
- try {
2439
- this.contextStore.addStep({
2440
- type: "query",
2441
- summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
2442
- data: output,
2443
- prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2444
- });
2445
- } catch (error) {
2446
- debug4("Failed to add query step:", error);
2447
- }
2448
- }
2449
- }
2273
+ const { output, executor } = await this.taskExecutor.query(demand);
2450
2274
  const metadata = this.afterTaskRunning(executor);
2451
2275
  return {
2452
2276
  result: output,
@@ -2556,48 +2380,6 @@ var PageAgent = class {
2556
2380
  };
2557
2381
  }
2558
2382
  async aiAssert(assertion, msg, opt) {
2559
- let executionContext = "";
2560
- if (this.opts.enableCumulativeContext && this.contextStore) {
2561
- try {
2562
- const recentSteps = this.contextStore.getRecentSteps(3);
2563
- const stepsWithExecutionResults = recentSteps.filter((step) => step.executionResult);
2564
- const storedData = this.contextStore.getAllData();
2565
- if (stepsWithExecutionResults.length > 0) {
2566
- const recentActions = stepsWithExecutionResults.map((step) => {
2567
- const result = step.executionResult;
2568
- return `- ${result.description}${result.success ? "" : " (FAILED)"}`;
2569
- }).join("\n");
2570
- executionContext = `
2571
-
2572
- Recent actions performed:
2573
- ${recentActions}
2574
-
2575
- This context may help verify the assertion.`;
2576
- }
2577
- if (storedData && Object.keys(storedData).length > 0) {
2578
- executionContext += `
2579
-
2580
- Available data for reference:
2581
- ${JSON.stringify(storedData, null, 2)}
2582
-
2583
- Note: If the assertion references any data keys or natural language equivalents, consider the stored values when verifying.`;
2584
- debug4("Available data for aiAssert:", {
2585
- assertion,
2586
- availableData: storedData
2587
- });
2588
- }
2589
- this.contextStore.addStep({
2590
- type: "assertion",
2591
- summary: `Assertion: ${assertion}`,
2592
- prompt: assertion
2593
- });
2594
- debug4("Added assertion step to context store:", {
2595
- totalSteps: this.contextStore.getRecentSteps(100).length
2596
- });
2597
- } catch (error) {
2598
- debug4("Context store operation failed:", error);
2599
- }
2600
- }
2601
2383
  let currentUrl = "";
2602
2384
  if (this.page.url) {
2603
2385
  try {
@@ -2605,13 +2387,7 @@ Note: If the assertion references any data keys or natural language equivalents,
2605
2387
  } catch (e) {
2606
2388
  }
2607
2389
  }
2608
- let assertionWithContext = assertion;
2609
- if (currentUrl) {
2610
- assertionWithContext = `For the page at URL "${currentUrl}", ${assertion}`;
2611
- }
2612
- if (executionContext) {
2613
- assertionWithContext += executionContext;
2614
- }
2390
+ const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
2615
2391
  const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
2616
2392
  const metadata = this.afterTaskRunning(executor, true);
2617
2393
  if (output && opt?.keepRawResponse) {
@@ -2822,81 +2598,42 @@ ${errors}`);
2822
2598
  }
2823
2599
  throw new Error("evaluateJavaScript is not supported in current agent");
2824
2600
  }
2825
- async destroy() {
2826
- await this.page.destroy();
2827
- }
2828
- /**
2829
- * Analyze execution results from executor to generate meaningful descriptions
2830
- */
2831
- analyzeExecutionResults(executor, originalPrompt) {
2832
- const tasks = executor.tasks;
2833
- const success = !executor.isInErrorState();
2834
- if (!success) {
2835
- const errorTask = executor.latestErrorTask();
2836
- return {
2837
- success: false,
2838
- actionType: "error",
2839
- description: `Failed to execute: ${originalPrompt}`,
2840
- error: errorTask?.error
2841
- };
2842
- }
2843
- const actionTasks = tasks.filter((t) => t.type === "Action" && t.status === "finished");
2844
- const locateTasks = tasks.filter((t) => t.type === "Insight" && t.subType === "Locate");
2845
- const lastAction = actionTasks[actionTasks.length - 1];
2846
- const lastLocate = locateTasks[locateTasks.length - 1];
2847
- if (!lastAction) {
2848
- return {
2849
- success: true,
2850
- actionType: "unknown",
2851
- description: `Completed: ${originalPrompt}`
2601
+ async logScreenshot(title, options) {
2602
+ const screenshotTitle = title || "untitled";
2603
+ const content = options?.content || "";
2604
+ const screenshot = await this.page.screenshotBase64?.();
2605
+ if (screenshot) {
2606
+ const executionDump = {
2607
+ name: screenshotTitle,
2608
+ description: content,
2609
+ tasks: [{
2610
+ type: "Screenshot",
2611
+ subType: "log",
2612
+ status: "finished",
2613
+ executor: null,
2614
+ param: {
2615
+ title: screenshotTitle,
2616
+ content
2617
+ },
2618
+ output: {
2619
+ screenshot
2620
+ },
2621
+ thought: `Logged screenshot: ${screenshotTitle}`,
2622
+ timing: {
2623
+ start: Date.now(),
2624
+ end: Date.now(),
2625
+ cost: 0
2626
+ }
2627
+ }],
2628
+ sdkVersion: "1.0.0",
2629
+ logTime: Date.now(),
2630
+ model_name: "screenshot"
2852
2631
  };
2632
+ this.appendExecutionDump(executionDump);
2853
2633
  }
2854
- const actionType = lastAction.subType || "unknown";
2855
- const elementInfo = this.extractElementInfo(lastLocate, lastAction);
2856
- const description = this.generateActionDescription(actionType, lastAction.param, elementInfo);
2857
- return {
2858
- success: true,
2859
- actionType,
2860
- description,
2861
- elementInfo,
2862
- timing: lastAction.timing?.cost
2863
- };
2864
- }
2865
- /**
2866
- * Extract element information from locate task
2867
- */
2868
- extractElementInfo(locateTask, _actionTask) {
2869
- if (!locateTask?.output?.element)
2870
- return void 0;
2871
- const element = locateTask.output.element;
2872
- return {
2873
- type: element.attributes?.nodeType || "unknown",
2874
- text: element.content || element.attributes?.placeholder || element.attributes?.title || "",
2875
- location: `(${element.center[0]}, ${element.center[1]})`
2876
- };
2877
2634
  }
2878
- /**
2879
- * Generate natural language description for actions
2880
- */
2881
- generateActionDescription(actionType, param, elementInfo) {
2882
- const elementDesc = elementInfo ? `'${elementInfo.text || elementInfo.type}' element` : "element";
2883
- switch (actionType) {
2884
- case "Tap":
2885
- return `Clicked on ${elementDesc}`;
2886
- case "Input":
2887
- const inputValue = param?.value || "";
2888
- return `Entered "${inputValue}" into ${elementDesc}`;
2889
- case "KeyboardPress":
2890
- return `Pressed ${param?.value || "key"}`;
2891
- case "Scroll":
2892
- return `Scrolled ${param?.direction || "on page"}`;
2893
- case "Hover":
2894
- return `Hovered over ${elementDesc}`;
2895
- case "Drag":
2896
- return `Dragged ${elementDesc}`;
2897
- default:
2898
- return `Performed ${actionType} action on ${elementDesc}`;
2899
- }
2635
+ async destroy() {
2636
+ await this.page.destroy();
2900
2637
  }
2901
2638
  };
2902
2639
 
@@ -2919,7 +2656,7 @@ var Page = class {
2919
2656
  this.everMoved = false;
2920
2657
  this.underlyingPage = underlyingPage;
2921
2658
  this.pageType = pageType;
2922
- this.waitForNavigationTimeout = opts?.waitForNavigationTimeout || import_constants3.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
2659
+ this.waitForNavigationTimeout = opts?.waitForNavigationTimeout ?? import_constants3.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
2923
2660
  }
2924
2661
  async evaluate(pageFunction, arg) {
2925
2662
  let result;
@@ -3199,9 +2936,9 @@ var WebPage = class extends Page {
3199
2936
  }
3200
2937
  async waitUntilNetworkIdle(options) {
3201
2938
  await this.underlyingPage.waitForNetworkIdle({
3202
- idleTime: options?.idleTime || import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME,
3203
- concurrency: options?.concurrency || import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY,
3204
- timeout: options?.timeout || this.waitForNetworkIdleTimeout
2939
+ idleTime: options?.idleTime ?? import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME,
2940
+ concurrency: options?.concurrency ?? import_constants4.DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY,
2941
+ timeout: options?.timeout ?? this.waitForNetworkIdleTimeout
3205
2942
  });
3206
2943
  }
3207
2944
  };