misoai-web 1.0.4 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +353 -9
  3. package/bin/midscene-playground +2 -2
  4. package/dist/es/agent.js +378 -92
  5. package/dist/es/agent.js.map +1 -1
  6. package/dist/es/bridge-mode-browser.js +3 -3
  7. package/dist/es/bridge-mode-browser.js.map +1 -1
  8. package/dist/es/bridge-mode.js +380 -94
  9. package/dist/es/bridge-mode.js.map +1 -1
  10. package/dist/es/chrome-extension.js +379 -93
  11. package/dist/es/chrome-extension.js.map +1 -1
  12. package/dist/es/index.js +378 -92
  13. package/dist/es/index.js.map +1 -1
  14. package/dist/es/midscene-playground.js +378 -92
  15. package/dist/es/midscene-playground.js.map +1 -1
  16. package/dist/es/midscene-server.js.map +1 -1
  17. package/dist/es/playground.js +378 -92
  18. package/dist/es/playground.js.map +1 -1
  19. package/dist/es/playwright-report.js.map +1 -1
  20. package/dist/es/playwright.js +378 -92
  21. package/dist/es/playwright.js.map +1 -1
  22. package/dist/es/puppeteer-agent-launcher.js +378 -92
  23. package/dist/es/puppeteer-agent-launcher.js.map +1 -1
  24. package/dist/es/puppeteer.js +378 -92
  25. package/dist/es/puppeteer.js.map +1 -1
  26. package/dist/es/ui-utils.js.map +1 -1
  27. package/dist/es/utils.js.map +1 -1
  28. package/dist/es/yaml.js.map +1 -1
  29. package/dist/lib/agent.js +378 -92
  30. package/dist/lib/agent.js.map +1 -1
  31. package/dist/lib/bridge-mode-browser.js +3 -3
  32. package/dist/lib/bridge-mode-browser.js.map +1 -1
  33. package/dist/lib/bridge-mode.js +380 -94
  34. package/dist/lib/bridge-mode.js.map +1 -1
  35. package/dist/lib/chrome-extension.js +379 -93
  36. package/dist/lib/chrome-extension.js.map +1 -1
  37. package/dist/lib/index.js +378 -92
  38. package/dist/lib/index.js.map +1 -1
  39. package/dist/lib/midscene-playground.js +378 -92
  40. package/dist/lib/midscene-playground.js.map +1 -1
  41. package/dist/lib/midscene-server.js.map +1 -1
  42. package/dist/lib/playground.js +378 -92
  43. package/dist/lib/playground.js.map +1 -1
  44. package/dist/lib/playwright-report.js.map +1 -1
  45. package/dist/lib/playwright.js +378 -92
  46. package/dist/lib/playwright.js.map +1 -1
  47. package/dist/lib/puppeteer-agent-launcher.js +378 -92
  48. package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
  49. package/dist/lib/puppeteer.js +378 -92
  50. package/dist/lib/puppeteer.js.map +1 -1
  51. package/dist/lib/ui-utils.js.map +1 -1
  52. package/dist/lib/utils.js.map +1 -1
  53. package/dist/lib/yaml.js.map +1 -1
  54. package/dist/types/agent.d.ts +45 -4
  55. package/dist/types/bridge-mode-browser.d.ts +2 -2
  56. package/dist/types/bridge-mode.d.ts +2 -2
  57. package/dist/types/{browser-a1877d18.d.ts → browser-aec1055d.d.ts} +1 -1
  58. package/dist/types/chrome-extension.d.ts +2 -2
  59. package/dist/types/index.d.ts +1 -1
  60. package/dist/types/midscene-server.d.ts +1 -1
  61. package/dist/types/{page-663ece08.d.ts → page-86ab0fe1.d.ts} +34 -34
  62. package/dist/types/playground.d.ts +2 -2
  63. package/dist/types/playwright.d.ts +1 -1
  64. package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
  65. package/dist/types/puppeteer.d.ts +1 -1
  66. package/dist/types/utils.d.ts +1 -1
  67. package/dist/types/yaml.d.ts +1 -1
  68. package/package.json +3 -3
@@ -1713,7 +1713,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
1713
1713
  var import_semver = __toESM(require("semver"));
1714
1714
 
1715
1715
  // package.json
1716
- var version = "1.0.4";
1716
+ var version = "1.0.5";
1717
1717
 
1718
1718
  // src/common/task-cache.ts
1719
1719
  var debug3 = (0, import_logger3.getDebug)("cache");
@@ -1741,44 +1741,70 @@ var TaskCache = class {
1741
1741
  this.cache = cacheContent;
1742
1742
  this.cacheOriginalLength = this.cache.caches.length;
1743
1743
  }
1744
- matchCache(prompt, type) {
1744
+ matchCache(prompt, type, contextData) {
1745
+ const contextHash = contextData ? this.generateContextHash(contextData) : void 0;
1745
1746
  for (let i = 0; i < this.cacheOriginalLength; i++) {
1746
1747
  const item = this.cache.caches[i];
1747
1748
  const key = `${type}:${prompt}:${i}`;
1748
- if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
1749
- this.matchedCacheIndices.add(key);
1750
- debug3(
1751
- "cache found and marked as used, type: %s, prompt: %s, index: %d",
1752
- type,
1753
- prompt,
1754
- i
1755
- );
1756
- return {
1757
- cacheContent: item,
1758
- updateFn: (cb) => {
1759
- debug3(
1760
- "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
1761
- type,
1762
- prompt,
1763
- i
1764
- );
1765
- cb(item);
1766
- debug3(
1767
- "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
1768
- type,
1769
- prompt,
1770
- i
1771
- );
1772
- this.flushCacheToFile();
1749
+ if (item.type !== type || item.prompt !== prompt || this.matchedCacheIndices.has(key)) {
1750
+ continue;
1751
+ }
1752
+ if (type === "plan" && item.type === "plan") {
1753
+ const planItem = item;
1754
+ if (contextHash && planItem.contextHash) {
1755
+ if (contextHash !== planItem.contextHash) {
1756
+ debug3("cache context mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
1757
+ continue;
1773
1758
  }
1774
- };
1759
+ } else if (contextHash || planItem.contextHash) {
1760
+ debug3("cache context availability mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
1761
+ continue;
1762
+ }
1775
1763
  }
1764
+ this.matchedCacheIndices.add(key);
1765
+ debug3(
1766
+ "cache found and marked as used, type: %s, prompt: %s, index: %d, contextMatch: %s",
1767
+ type,
1768
+ prompt,
1769
+ i,
1770
+ contextHash ? "yes" : "no-context"
1771
+ );
1772
+ return {
1773
+ cacheContent: item,
1774
+ updateFn: (cb) => {
1775
+ debug3(
1776
+ "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
1777
+ type,
1778
+ prompt,
1779
+ i
1780
+ );
1781
+ cb(item);
1782
+ debug3(
1783
+ "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
1784
+ type,
1785
+ prompt,
1786
+ i
1787
+ );
1788
+ this.flushCacheToFile();
1789
+ }
1790
+ };
1776
1791
  }
1777
- debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
1792
+ debug3("no unused cache found, type: %s, prompt: %s, contextHash: %s", type, prompt, contextHash);
1778
1793
  return void 0;
1779
1794
  }
1780
- matchPlanCache(prompt) {
1781
- return this.matchCache(prompt, "plan");
1795
+ generateContextHash(contextData) {
1796
+ const sortedKeys = Object.keys(contextData).sort();
1797
+ const stableString = sortedKeys.map((key) => `${key}:${JSON.stringify(contextData[key])}`).join("|");
1798
+ let hash = 0;
1799
+ for (let i = 0; i < stableString.length; i++) {
1800
+ const char = stableString.charCodeAt(i);
1801
+ hash = (hash << 5) - hash + char;
1802
+ hash = hash & hash;
1803
+ }
1804
+ return hash.toString(36);
1805
+ }
1806
+ matchPlanCache(prompt, contextData) {
1807
+ return this.matchCache(prompt, "plan", contextData);
1782
1808
  }
1783
1809
  matchLocateCache(prompt) {
1784
1810
  return this.matchCache(prompt, "locate");
@@ -1854,11 +1880,16 @@ cache file: ${cacheFile}`
1854
1880
  );
1855
1881
  }
1856
1882
  }
1857
- updateOrAppendCacheRecord(newRecord, cachedRecord) {
1883
+ updateOrAppendCacheRecord(newRecord, cachedRecord, contextData) {
1858
1884
  if (cachedRecord) {
1859
1885
  if (newRecord.type === "plan") {
1860
1886
  cachedRecord.updateFn((cache) => {
1861
- cache.yamlWorkflow = newRecord.yamlWorkflow;
1887
+ const planCache = cache;
1888
+ planCache.yamlWorkflow = newRecord.yamlWorkflow;
1889
+ if (contextData) {
1890
+ planCache.contextHash = this.generateContextHash(contextData);
1891
+ planCache.contextData = { ...contextData };
1892
+ }
1862
1893
  });
1863
1894
  } else {
1864
1895
  cachedRecord.updateFn((cache) => {
@@ -1866,6 +1897,11 @@ cache file: ${cacheFile}`
1866
1897
  });
1867
1898
  }
1868
1899
  } else {
1900
+ if (newRecord.type === "plan" && contextData) {
1901
+ const planRecord = newRecord;
1902
+ planRecord.contextHash = this.generateContextHash(contextData);
1903
+ planRecord.contextData = { ...contextData };
1904
+ }
1869
1905
  this.appendCache(newRecord);
1870
1906
  }
1871
1907
  }
@@ -1895,10 +1931,13 @@ var PageAgent = class {
1895
1931
  generateReport: true,
1896
1932
  autoPrintReportMsg: true,
1897
1933
  groupName: "Midscene Report",
1898
- groupDescription: ""
1934
+ groupDescription: "",
1935
+ enableCumulativeContext: true,
1936
+ autoClearContext: false
1899
1937
  },
1900
1938
  opts || {}
1901
1939
  );
1940
+ this.initializeContextStore();
1902
1941
  if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
1903
1942
  this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || import_constants2.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
1904
1943
  this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || import_constants2.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
@@ -1925,6 +1964,69 @@ var PageAgent = class {
1925
1964
  opts?.testId || this.page.pageType || "web"
1926
1965
  );
1927
1966
  }
1967
+ /**
1968
+ * Initialize context store for cumulative context functionality
1969
+ */
1970
+ async initializeContextStore() {
1971
+ if (!this.opts.enableCumulativeContext) {
1972
+ debug4("Cumulative context disabled via options");
1973
+ return;
1974
+ }
1975
+ try {
1976
+ const aiModel = await import("misoai-core/ai-model");
1977
+ this.contextStore = aiModel.getContextStore();
1978
+ debug4("Context store initialized successfully", {
1979
+ autoClearContext: this.opts.autoClearContext,
1980
+ testId: this.opts.testId
1981
+ });
1982
+ if (this.opts.autoClearContext) {
1983
+ this.contextStore.clear();
1984
+ debug4("Context store cleared due to autoClearContext option");
1985
+ } else {
1986
+ const existingData = this.contextStore.getAllData();
1987
+ const existingSteps = this.contextStore.getRecentSteps(100).length;
1988
+ debug4("Context store preserving existing data", {
1989
+ existingDataKeys: Object.keys(existingData),
1990
+ existingStepsCount: existingSteps
1991
+ });
1992
+ }
1993
+ } catch (error) {
1994
+ debug4("Failed to initialize context store:", error);
1995
+ console.warn("⚠️ Could not initialize context store:", error);
1996
+ }
1997
+ }
1998
+ /**
1999
+ * Get the context store instance
2000
+ */
2001
+ getContextStore() {
2002
+ return this.contextStore;
2003
+ }
2004
+ /**
2005
+ * Clear the context store
2006
+ */
2007
+ clearContext() {
2008
+ if (this.contextStore) {
2009
+ this.contextStore.clear();
2010
+ }
2011
+ }
2012
+ /**
2013
+ * Get all stored data from context store
2014
+ */
2015
+ getStoredData() {
2016
+ if (this.contextStore) {
2017
+ return this.contextStore.getAllData();
2018
+ }
2019
+ return {};
2020
+ }
2021
+ /**
2022
+ * Get step summary from context store
2023
+ */
2024
+ getStepSummary() {
2025
+ if (this.contextStore) {
2026
+ return this.contextStore.getStepSummary();
2027
+ }
2028
+ return "";
2029
+ }
1928
2030
  async getUIContext(action) {
1929
2031
  if (action && (action === "extract" || action === "assert" || action === "captcha")) {
1930
2032
  return await parseContextFromWebPage(this.page, {
@@ -2160,22 +2262,35 @@ var PageAgent = class {
2160
2262
  };
2161
2263
  }
2162
2264
  async aiAction(taskPrompt, opt) {
2163
- try {
2164
- const aiModel = await import("misoai-core/ai-model");
2165
- const contextStore = aiModel.getContextStore();
2166
- const processedPrompt = contextStore.replaceAllReferences(taskPrompt, "action");
2167
- contextStore.addStep({
2168
- type: "action",
2169
- summary: `Action: ${processedPrompt}`,
2170
- prompt: processedPrompt
2171
- });
2172
- taskPrompt = processedPrompt;
2173
- } catch (error) {
2174
- debug4("Context store not available:", error);
2265
+ const originalPrompt = taskPrompt;
2266
+ let processedPrompt = taskPrompt;
2267
+ if (this.opts.enableCumulativeContext && this.contextStore) {
2268
+ try {
2269
+ const storedData = this.contextStore.getAllData();
2270
+ if (Object.keys(storedData).length > 0) {
2271
+ debug4("Available data for aiAction:", {
2272
+ prompt: taskPrompt,
2273
+ availableData: storedData
2274
+ });
2275
+ }
2276
+ } catch (error) {
2277
+ debug4("Context store operation failed:", error);
2278
+ }
2175
2279
  }
2176
2280
  const cacheable = opt?.cacheable;
2177
2281
  const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
2178
- const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
2282
+ let contextData;
2283
+ if (this.opts.enableCumulativeContext && this.contextStore) {
2284
+ try {
2285
+ contextData = this.contextStore.getAllData();
2286
+ if (contextData && Object.keys(contextData).length === 0) {
2287
+ contextData = void 0;
2288
+ }
2289
+ } catch (error) {
2290
+ debug4("Failed to get context data for cache:", error);
2291
+ }
2292
+ }
2293
+ const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt, contextData);
2179
2294
  if (matchedCache && this.taskCache?.isCacheResultUsed) {
2180
2295
  const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
2181
2296
  taskPrompt,
@@ -2185,6 +2300,28 @@ var PageAgent = class {
2185
2300
  debug4("matched cache, will call .runYaml to run the action");
2186
2301
  const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
2187
2302
  const result = await this.runYaml(yaml5);
2303
+ if (this.opts.enableCumulativeContext && this.contextStore) {
2304
+ try {
2305
+ const executionResult = {
2306
+ success: true,
2307
+ actionType: "cached",
2308
+ description: `Executed cached action: ${processedPrompt}`,
2309
+ timing: result.metadata?.totalTime
2310
+ };
2311
+ this.contextStore.addStep({
2312
+ type: "action",
2313
+ summary: `Action: ${processedPrompt} (cached)`,
2314
+ prompt: processedPrompt,
2315
+ executionResult
2316
+ });
2317
+ debug4("Added cached action step to context store:", {
2318
+ stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
2319
+ totalSteps: this.contextStore.getRecentSteps(100).length
2320
+ });
2321
+ } catch (error) {
2322
+ debug4("Failed to add cached action step:", error);
2323
+ }
2324
+ }
2188
2325
  return {
2189
2326
  result: result.result,
2190
2327
  metadata: metadata2
@@ -2209,10 +2346,39 @@ var PageAgent = class {
2209
2346
  prompt: taskPrompt,
2210
2347
  yamlWorkflow: yamlFlowStr
2211
2348
  },
2212
- matchedCache
2349
+ matchedCache,
2350
+ contextData
2351
+ // Pass context data for cache creation
2213
2352
  );
2214
2353
  }
2215
2354
  const metadata = this.afterTaskRunning(executor);
2355
+ if (this.opts.enableCumulativeContext && this.contextStore) {
2356
+ try {
2357
+ const executionResult = this.analyzeExecutionResults(executor, originalPrompt);
2358
+ this.contextStore.addStep({
2359
+ type: "action",
2360
+ summary: `Action: ${processedPrompt}`,
2361
+ prompt: processedPrompt,
2362
+ executionResult
2363
+ });
2364
+ debug4("Added action step with execution result to context store:", {
2365
+ stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
2366
+ totalSteps: this.contextStore.getRecentSteps(100).length,
2367
+ executionResult
2368
+ });
2369
+ } catch (error) {
2370
+ debug4("Failed to analyze execution results, adding step without execution result:", error);
2371
+ try {
2372
+ this.contextStore.addStep({
2373
+ type: "action",
2374
+ summary: `Action: ${processedPrompt}`,
2375
+ prompt: processedPrompt
2376
+ });
2377
+ } catch (stepError) {
2378
+ debug4("Failed to add action step:", stepError);
2379
+ }
2380
+ }
2381
+ }
2216
2382
  return {
2217
2383
  result: output,
2218
2384
  metadata
@@ -2242,38 +2408,50 @@ var PageAgent = class {
2242
2408
  debug4("Context store not available:", error);
2243
2409
  }
2244
2410
  const { output, executor } = await this.taskExecutor.query(processedDemand);
2245
- if (storageKey && output) {
2246
- try {
2247
- const aiModel = await import("misoai-core/ai-model");
2248
- const contextStore = aiModel.getContextStore();
2249
- const pendingAliases = contextStore._pendingAliases;
2250
- if (pendingAliases) {
2251
- contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
2252
- delete contextStore._pendingAliases;
2253
- } else {
2254
- contextStore.storeData(storageKey, output);
2411
+ if (this.opts.enableCumulativeContext && this.contextStore) {
2412
+ if (storageKey && output) {
2413
+ try {
2414
+ const pendingAliases = this.contextStore._pendingAliases;
2415
+ if (pendingAliases) {
2416
+ this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
2417
+ delete this.contextStore._pendingAliases;
2418
+ debug4("Stored query result with aliases:", {
2419
+ key: storageKey,
2420
+ value: output,
2421
+ aliases: pendingAliases
2422
+ });
2423
+ } else {
2424
+ this.contextStore.storeData(storageKey, output);
2425
+ debug4("Stored query result:", {
2426
+ key: storageKey,
2427
+ value: output
2428
+ });
2429
+ }
2430
+ this.contextStore.addStep({
2431
+ type: "query",
2432
+ summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
2433
+ data: output,
2434
+ prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2435
+ });
2436
+ debug4("Added query step to context store:", {
2437
+ storageKey,
2438
+ totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
2439
+ totalSteps: this.contextStore.getRecentSteps(100).length
2440
+ });
2441
+ } catch (error) {
2442
+ debug4("Failed to store query result:", error);
2443
+ }
2444
+ } else {
2445
+ try {
2446
+ this.contextStore.addStep({
2447
+ type: "query",
2448
+ summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
2449
+ data: output,
2450
+ prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2451
+ });
2452
+ } catch (error) {
2453
+ debug4("Failed to add query step:", error);
2255
2454
  }
2256
- contextStore.addStep({
2257
- type: "query",
2258
- summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
2259
- data: output,
2260
- prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2261
- });
2262
- } catch (error) {
2263
- debug4("Failed to store query result:", error);
2264
- }
2265
- } else {
2266
- try {
2267
- const aiModel = await import("misoai-core/ai-model");
2268
- const contextStore = aiModel.getContextStore();
2269
- contextStore.addStep({
2270
- type: "query",
2271
- summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
2272
- data: output,
2273
- prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
2274
- });
2275
- } catch (error) {
2276
- debug4("Failed to add query step:", error);
2277
2455
  }
2278
2456
  }
2279
2457
  const metadata = this.afterTaskRunning(executor);
@@ -2385,18 +2563,47 @@ var PageAgent = class {
2385
2563
  };
2386
2564
  }
2387
2565
  async aiAssert(assertion, msg, opt) {
2388
- let processedAssertion = assertion;
2389
- try {
2390
- const aiModel = await import("misoai-core/ai-model");
2391
- const contextStore = aiModel.getContextStore();
2392
- processedAssertion = contextStore.replaceAllReferences(assertion, "assertion");
2393
- contextStore.addStep({
2394
- type: "assertion",
2395
- summary: `Assertion: ${processedAssertion}`,
2396
- prompt: processedAssertion
2397
- });
2398
- } catch (error) {
2399
- debug4("Context store not available:", error);
2566
+ let executionContext = "";
2567
+ if (this.opts.enableCumulativeContext && this.contextStore) {
2568
+ try {
2569
+ const recentSteps = this.contextStore.getRecentSteps(3);
2570
+ const stepsWithExecutionResults = recentSteps.filter((step) => step.executionResult);
2571
+ const storedData = this.contextStore.getAllData();
2572
+ if (stepsWithExecutionResults.length > 0) {
2573
+ const recentActions = stepsWithExecutionResults.map((step) => {
2574
+ const result = step.executionResult;
2575
+ return `- ${result.description}${result.success ? "" : " (FAILED)"}`;
2576
+ }).join("\n");
2577
+ executionContext = `
2578
+
2579
+ Recent actions performed:
2580
+ ${recentActions}
2581
+
2582
+ This context may help verify the assertion.`;
2583
+ }
2584
+ if (storedData && Object.keys(storedData).length > 0) {
2585
+ executionContext += `
2586
+
2587
+ Available data for reference:
2588
+ ${JSON.stringify(storedData, null, 2)}
2589
+
2590
+ Note: If the assertion references any data keys or natural language equivalents, consider the stored values when verifying.`;
2591
+ debug4("Available data for aiAssert:", {
2592
+ assertion,
2593
+ availableData: storedData
2594
+ });
2595
+ }
2596
+ this.contextStore.addStep({
2597
+ type: "assertion",
2598
+ summary: `Assertion: ${assertion}`,
2599
+ prompt: assertion
2600
+ });
2601
+ debug4("Added assertion step to context store:", {
2602
+ totalSteps: this.contextStore.getRecentSteps(100).length
2603
+ });
2604
+ } catch (error) {
2605
+ debug4("Context store operation failed:", error);
2606
+ }
2400
2607
  }
2401
2608
  let currentUrl = "";
2402
2609
  if (this.page.url) {
@@ -2405,7 +2612,13 @@ var PageAgent = class {
2405
2612
  } catch (e) {
2406
2613
  }
2407
2614
  }
2408
- const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
2615
+ let assertionWithContext = assertion;
2616
+ if (currentUrl) {
2617
+ assertionWithContext = `For the page at URL "${currentUrl}", ${assertion}`;
2618
+ }
2619
+ if (executionContext) {
2620
+ assertionWithContext += executionContext;
2621
+ }
2409
2622
  const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
2410
2623
  const metadata = this.afterTaskRunning(executor, true);
2411
2624
  if (output && opt?.keepRawResponse) {
@@ -2619,6 +2832,79 @@ ${errors}`);
2619
2832
  async destroy() {
2620
2833
  await this.page.destroy();
2621
2834
  }
2835
+ /**
2836
+ * Analyze execution results from executor to generate meaningful descriptions
2837
+ */
2838
+ analyzeExecutionResults(executor, originalPrompt) {
2839
+ const tasks = executor.tasks;
2840
+ const success = !executor.isInErrorState();
2841
+ if (!success) {
2842
+ const errorTask = executor.latestErrorTask();
2843
+ return {
2844
+ success: false,
2845
+ actionType: "error",
2846
+ description: `Failed to execute: ${originalPrompt}`,
2847
+ error: errorTask?.error
2848
+ };
2849
+ }
2850
+ const actionTasks = tasks.filter((t) => t.type === "Action" && t.status === "finished");
2851
+ const locateTasks = tasks.filter((t) => t.type === "Insight" && t.subType === "Locate");
2852
+ const lastAction = actionTasks[actionTasks.length - 1];
2853
+ const lastLocate = locateTasks[locateTasks.length - 1];
2854
+ if (!lastAction) {
2855
+ return {
2856
+ success: true,
2857
+ actionType: "unknown",
2858
+ description: `Completed: ${originalPrompt}`
2859
+ };
2860
+ }
2861
+ const actionType = lastAction.subType || "unknown";
2862
+ const elementInfo = this.extractElementInfo(lastLocate, lastAction);
2863
+ const description = this.generateActionDescription(actionType, lastAction.param, elementInfo);
2864
+ return {
2865
+ success: true,
2866
+ actionType,
2867
+ description,
2868
+ elementInfo,
2869
+ timing: lastAction.timing?.cost
2870
+ };
2871
+ }
2872
+ /**
2873
+ * Extract element information from locate task
2874
+ */
2875
+ extractElementInfo(locateTask, _actionTask) {
2876
+ if (!locateTask?.output?.element)
2877
+ return void 0;
2878
+ const element = locateTask.output.element;
2879
+ return {
2880
+ type: element.attributes?.nodeType || "unknown",
2881
+ text: element.content || element.attributes?.placeholder || element.attributes?.title || "",
2882
+ location: `(${element.center[0]}, ${element.center[1]})`
2883
+ };
2884
+ }
2885
+ /**
2886
+ * Generate natural language description for actions
2887
+ */
2888
+ generateActionDescription(actionType, param, elementInfo) {
2889
+ const elementDesc = elementInfo ? `'${elementInfo.text || elementInfo.type}' element` : "element";
2890
+ switch (actionType) {
2891
+ case "Tap":
2892
+ return `Clicked on ${elementDesc}`;
2893
+ case "Input":
2894
+ const inputValue = param?.value || "";
2895
+ return `Entered "${inputValue}" into ${elementDesc}`;
2896
+ case "KeyboardPress":
2897
+ return `Pressed ${param?.value || "key"}`;
2898
+ case "Scroll":
2899
+ return `Scrolled ${param?.direction || "on page"}`;
2900
+ case "Hover":
2901
+ return `Hovered over ${elementDesc}`;
2902
+ case "Drag":
2903
+ return `Dragged ${elementDesc}`;
2904
+ default:
2905
+ return `Performed ${actionType} action on ${elementDesc}`;
2906
+ }
2907
+ }
2622
2908
  };
2623
2909
 
2624
2910
  // src/puppeteer/index.ts