misoai-web 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +353 -9
- package/bin/midscene-playground +2 -2
- package/dist/es/agent.js +378 -92
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +3 -3
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +380 -94
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +379 -93
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +378 -92
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +378 -92
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +378 -92
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +378 -92
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +378 -92
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +378 -92
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +378 -92
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +3 -3
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +380 -94
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +379 -93
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +378 -92
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +378 -92
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +378 -92
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +378 -92
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +378 -92
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +378 -92
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +45 -4
- package/dist/types/bridge-mode-browser.d.ts +2 -2
- package/dist/types/bridge-mode.d.ts +2 -2
- package/dist/types/{browser-a1877d18.d.ts → browser-aec1055d.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -2
- package/dist/types/index.d.ts +1 -1
- package/dist/types/midscene-server.d.ts +1 -1
- package/dist/types/{page-663ece08.d.ts → page-86ab0fe1.d.ts} +34 -34
- package/dist/types/playground.d.ts +2 -2
- package/dist/types/playwright.d.ts +1 -1
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
- package/dist/types/puppeteer.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +1 -1
- package/package.json +3 -3
@@ -1665,7 +1665,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
|
|
1665
1665
|
var import_semver = __toESM(require("semver"));
|
1666
1666
|
|
1667
1667
|
// package.json
|
1668
|
-
var version = "1.0.
|
1668
|
+
var version = "1.0.5";
|
1669
1669
|
|
1670
1670
|
// src/common/task-cache.ts
|
1671
1671
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -1693,44 +1693,70 @@ var TaskCache = class {
|
|
1693
1693
|
this.cache = cacheContent;
|
1694
1694
|
this.cacheOriginalLength = this.cache.caches.length;
|
1695
1695
|
}
|
1696
|
-
matchCache(prompt, type) {
|
1696
|
+
matchCache(prompt, type, contextData) {
|
1697
|
+
const contextHash = contextData ? this.generateContextHash(contextData) : void 0;
|
1697
1698
|
for (let i = 0; i < this.cacheOriginalLength; i++) {
|
1698
1699
|
const item = this.cache.caches[i];
|
1699
1700
|
const key = `${type}:${prompt}:${i}`;
|
1700
|
-
if (item.type
|
1701
|
-
|
1702
|
-
|
1703
|
-
|
1704
|
-
|
1705
|
-
|
1706
|
-
|
1707
|
-
|
1708
|
-
|
1709
|
-
cacheContent: item,
|
1710
|
-
updateFn: (cb) => {
|
1711
|
-
debug3(
|
1712
|
-
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1713
|
-
type,
|
1714
|
-
prompt,
|
1715
|
-
i
|
1716
|
-
);
|
1717
|
-
cb(item);
|
1718
|
-
debug3(
|
1719
|
-
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1720
|
-
type,
|
1721
|
-
prompt,
|
1722
|
-
i
|
1723
|
-
);
|
1724
|
-
this.flushCacheToFile();
|
1701
|
+
if (item.type !== type || item.prompt !== prompt || this.matchedCacheIndices.has(key)) {
|
1702
|
+
continue;
|
1703
|
+
}
|
1704
|
+
if (type === "plan" && item.type === "plan") {
|
1705
|
+
const planItem = item;
|
1706
|
+
if (contextHash && planItem.contextHash) {
|
1707
|
+
if (contextHash !== planItem.contextHash) {
|
1708
|
+
debug3("cache context mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
|
1709
|
+
continue;
|
1725
1710
|
}
|
1726
|
-
}
|
1711
|
+
} else if (contextHash || planItem.contextHash) {
|
1712
|
+
debug3("cache context availability mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
|
1713
|
+
continue;
|
1714
|
+
}
|
1727
1715
|
}
|
1716
|
+
this.matchedCacheIndices.add(key);
|
1717
|
+
debug3(
|
1718
|
+
"cache found and marked as used, type: %s, prompt: %s, index: %d, contextMatch: %s",
|
1719
|
+
type,
|
1720
|
+
prompt,
|
1721
|
+
i,
|
1722
|
+
contextHash ? "yes" : "no-context"
|
1723
|
+
);
|
1724
|
+
return {
|
1725
|
+
cacheContent: item,
|
1726
|
+
updateFn: (cb) => {
|
1727
|
+
debug3(
|
1728
|
+
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1729
|
+
type,
|
1730
|
+
prompt,
|
1731
|
+
i
|
1732
|
+
);
|
1733
|
+
cb(item);
|
1734
|
+
debug3(
|
1735
|
+
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1736
|
+
type,
|
1737
|
+
prompt,
|
1738
|
+
i
|
1739
|
+
);
|
1740
|
+
this.flushCacheToFile();
|
1741
|
+
}
|
1742
|
+
};
|
1728
1743
|
}
|
1729
|
-
debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
|
1744
|
+
debug3("no unused cache found, type: %s, prompt: %s, contextHash: %s", type, prompt, contextHash);
|
1730
1745
|
return void 0;
|
1731
1746
|
}
|
1732
|
-
|
1733
|
-
|
1747
|
+
generateContextHash(contextData) {
|
1748
|
+
const sortedKeys = Object.keys(contextData).sort();
|
1749
|
+
const stableString = sortedKeys.map((key) => `${key}:${JSON.stringify(contextData[key])}`).join("|");
|
1750
|
+
let hash = 0;
|
1751
|
+
for (let i = 0; i < stableString.length; i++) {
|
1752
|
+
const char = stableString.charCodeAt(i);
|
1753
|
+
hash = (hash << 5) - hash + char;
|
1754
|
+
hash = hash & hash;
|
1755
|
+
}
|
1756
|
+
return hash.toString(36);
|
1757
|
+
}
|
1758
|
+
matchPlanCache(prompt, contextData) {
|
1759
|
+
return this.matchCache(prompt, "plan", contextData);
|
1734
1760
|
}
|
1735
1761
|
matchLocateCache(prompt) {
|
1736
1762
|
return this.matchCache(prompt, "locate");
|
@@ -1806,11 +1832,16 @@ cache file: ${cacheFile}`
|
|
1806
1832
|
);
|
1807
1833
|
}
|
1808
1834
|
}
|
1809
|
-
updateOrAppendCacheRecord(newRecord, cachedRecord) {
|
1835
|
+
updateOrAppendCacheRecord(newRecord, cachedRecord, contextData) {
|
1810
1836
|
if (cachedRecord) {
|
1811
1837
|
if (newRecord.type === "plan") {
|
1812
1838
|
cachedRecord.updateFn((cache) => {
|
1813
|
-
|
1839
|
+
const planCache = cache;
|
1840
|
+
planCache.yamlWorkflow = newRecord.yamlWorkflow;
|
1841
|
+
if (contextData) {
|
1842
|
+
planCache.contextHash = this.generateContextHash(contextData);
|
1843
|
+
planCache.contextData = { ...contextData };
|
1844
|
+
}
|
1814
1845
|
});
|
1815
1846
|
} else {
|
1816
1847
|
cachedRecord.updateFn((cache) => {
|
@@ -1818,6 +1849,11 @@ cache file: ${cacheFile}`
|
|
1818
1849
|
});
|
1819
1850
|
}
|
1820
1851
|
} else {
|
1852
|
+
if (newRecord.type === "plan" && contextData) {
|
1853
|
+
const planRecord = newRecord;
|
1854
|
+
planRecord.contextHash = this.generateContextHash(contextData);
|
1855
|
+
planRecord.contextData = { ...contextData };
|
1856
|
+
}
|
1821
1857
|
this.appendCache(newRecord);
|
1822
1858
|
}
|
1823
1859
|
}
|
@@ -1847,10 +1883,13 @@ var PageAgent = class {
|
|
1847
1883
|
generateReport: true,
|
1848
1884
|
autoPrintReportMsg: true,
|
1849
1885
|
groupName: "Midscene Report",
|
1850
|
-
groupDescription: ""
|
1886
|
+
groupDescription: "",
|
1887
|
+
enableCumulativeContext: true,
|
1888
|
+
autoClearContext: false
|
1851
1889
|
},
|
1852
1890
|
opts || {}
|
1853
1891
|
);
|
1892
|
+
this.initializeContextStore();
|
1854
1893
|
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1855
1894
|
this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || import_constants2.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
|
1856
1895
|
this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || import_constants2.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
|
@@ -1877,6 +1916,69 @@ var PageAgent = class {
|
|
1877
1916
|
opts?.testId || this.page.pageType || "web"
|
1878
1917
|
);
|
1879
1918
|
}
|
1919
|
+
/**
|
1920
|
+
* Initialize context store for cumulative context functionality
|
1921
|
+
*/
|
1922
|
+
async initializeContextStore() {
|
1923
|
+
if (!this.opts.enableCumulativeContext) {
|
1924
|
+
debug4("Cumulative context disabled via options");
|
1925
|
+
return;
|
1926
|
+
}
|
1927
|
+
try {
|
1928
|
+
const aiModel = await import("misoai-core/ai-model");
|
1929
|
+
this.contextStore = aiModel.getContextStore();
|
1930
|
+
debug4("Context store initialized successfully", {
|
1931
|
+
autoClearContext: this.opts.autoClearContext,
|
1932
|
+
testId: this.opts.testId
|
1933
|
+
});
|
1934
|
+
if (this.opts.autoClearContext) {
|
1935
|
+
this.contextStore.clear();
|
1936
|
+
debug4("Context store cleared due to autoClearContext option");
|
1937
|
+
} else {
|
1938
|
+
const existingData = this.contextStore.getAllData();
|
1939
|
+
const existingSteps = this.contextStore.getRecentSteps(100).length;
|
1940
|
+
debug4("Context store preserving existing data", {
|
1941
|
+
existingDataKeys: Object.keys(existingData),
|
1942
|
+
existingStepsCount: existingSteps
|
1943
|
+
});
|
1944
|
+
}
|
1945
|
+
} catch (error) {
|
1946
|
+
debug4("Failed to initialize context store:", error);
|
1947
|
+
console.warn("⚠️ Could not initialize context store:", error);
|
1948
|
+
}
|
1949
|
+
}
|
1950
|
+
/**
|
1951
|
+
* Get the context store instance
|
1952
|
+
*/
|
1953
|
+
getContextStore() {
|
1954
|
+
return this.contextStore;
|
1955
|
+
}
|
1956
|
+
/**
|
1957
|
+
* Clear the context store
|
1958
|
+
*/
|
1959
|
+
clearContext() {
|
1960
|
+
if (this.contextStore) {
|
1961
|
+
this.contextStore.clear();
|
1962
|
+
}
|
1963
|
+
}
|
1964
|
+
/**
|
1965
|
+
* Get all stored data from context store
|
1966
|
+
*/
|
1967
|
+
getStoredData() {
|
1968
|
+
if (this.contextStore) {
|
1969
|
+
return this.contextStore.getAllData();
|
1970
|
+
}
|
1971
|
+
return {};
|
1972
|
+
}
|
1973
|
+
/**
|
1974
|
+
* Get step summary from context store
|
1975
|
+
*/
|
1976
|
+
getStepSummary() {
|
1977
|
+
if (this.contextStore) {
|
1978
|
+
return this.contextStore.getStepSummary();
|
1979
|
+
}
|
1980
|
+
return "";
|
1981
|
+
}
|
1880
1982
|
async getUIContext(action) {
|
1881
1983
|
if (action && (action === "extract" || action === "assert" || action === "captcha")) {
|
1882
1984
|
return await parseContextFromWebPage(this.page, {
|
@@ -2112,22 +2214,35 @@ var PageAgent = class {
|
|
2112
2214
|
};
|
2113
2215
|
}
|
2114
2216
|
async aiAction(taskPrompt, opt) {
|
2115
|
-
|
2116
|
-
|
2117
|
-
|
2118
|
-
|
2119
|
-
|
2120
|
-
|
2121
|
-
|
2122
|
-
|
2123
|
-
|
2124
|
-
|
2125
|
-
|
2126
|
-
|
2217
|
+
const originalPrompt = taskPrompt;
|
2218
|
+
let processedPrompt = taskPrompt;
|
2219
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2220
|
+
try {
|
2221
|
+
const storedData = this.contextStore.getAllData();
|
2222
|
+
if (Object.keys(storedData).length > 0) {
|
2223
|
+
debug4("Available data for aiAction:", {
|
2224
|
+
prompt: taskPrompt,
|
2225
|
+
availableData: storedData
|
2226
|
+
});
|
2227
|
+
}
|
2228
|
+
} catch (error) {
|
2229
|
+
debug4("Context store operation failed:", error);
|
2230
|
+
}
|
2127
2231
|
}
|
2128
2232
|
const cacheable = opt?.cacheable;
|
2129
2233
|
const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
|
2130
|
-
|
2234
|
+
let contextData;
|
2235
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2236
|
+
try {
|
2237
|
+
contextData = this.contextStore.getAllData();
|
2238
|
+
if (contextData && Object.keys(contextData).length === 0) {
|
2239
|
+
contextData = void 0;
|
2240
|
+
}
|
2241
|
+
} catch (error) {
|
2242
|
+
debug4("Failed to get context data for cache:", error);
|
2243
|
+
}
|
2244
|
+
}
|
2245
|
+
const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt, contextData);
|
2131
2246
|
if (matchedCache && this.taskCache?.isCacheResultUsed) {
|
2132
2247
|
const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
|
2133
2248
|
taskPrompt,
|
@@ -2137,6 +2252,28 @@ var PageAgent = class {
|
|
2137
2252
|
debug4("matched cache, will call .runYaml to run the action");
|
2138
2253
|
const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
|
2139
2254
|
const result = await this.runYaml(yaml5);
|
2255
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2256
|
+
try {
|
2257
|
+
const executionResult = {
|
2258
|
+
success: true,
|
2259
|
+
actionType: "cached",
|
2260
|
+
description: `Executed cached action: ${processedPrompt}`,
|
2261
|
+
timing: result.metadata?.totalTime
|
2262
|
+
};
|
2263
|
+
this.contextStore.addStep({
|
2264
|
+
type: "action",
|
2265
|
+
summary: `Action: ${processedPrompt} (cached)`,
|
2266
|
+
prompt: processedPrompt,
|
2267
|
+
executionResult
|
2268
|
+
});
|
2269
|
+
debug4("Added cached action step to context store:", {
|
2270
|
+
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2271
|
+
totalSteps: this.contextStore.getRecentSteps(100).length
|
2272
|
+
});
|
2273
|
+
} catch (error) {
|
2274
|
+
debug4("Failed to add cached action step:", error);
|
2275
|
+
}
|
2276
|
+
}
|
2140
2277
|
return {
|
2141
2278
|
result: result.result,
|
2142
2279
|
metadata: metadata2
|
@@ -2161,10 +2298,39 @@ var PageAgent = class {
|
|
2161
2298
|
prompt: taskPrompt,
|
2162
2299
|
yamlWorkflow: yamlFlowStr
|
2163
2300
|
},
|
2164
|
-
matchedCache
|
2301
|
+
matchedCache,
|
2302
|
+
contextData
|
2303
|
+
// Pass context data for cache creation
|
2165
2304
|
);
|
2166
2305
|
}
|
2167
2306
|
const metadata = this.afterTaskRunning(executor);
|
2307
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2308
|
+
try {
|
2309
|
+
const executionResult = this.analyzeExecutionResults(executor, originalPrompt);
|
2310
|
+
this.contextStore.addStep({
|
2311
|
+
type: "action",
|
2312
|
+
summary: `Action: ${processedPrompt}`,
|
2313
|
+
prompt: processedPrompt,
|
2314
|
+
executionResult
|
2315
|
+
});
|
2316
|
+
debug4("Added action step with execution result to context store:", {
|
2317
|
+
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2318
|
+
totalSteps: this.contextStore.getRecentSteps(100).length,
|
2319
|
+
executionResult
|
2320
|
+
});
|
2321
|
+
} catch (error) {
|
2322
|
+
debug4("Failed to analyze execution results, adding step without execution result:", error);
|
2323
|
+
try {
|
2324
|
+
this.contextStore.addStep({
|
2325
|
+
type: "action",
|
2326
|
+
summary: `Action: ${processedPrompt}`,
|
2327
|
+
prompt: processedPrompt
|
2328
|
+
});
|
2329
|
+
} catch (stepError) {
|
2330
|
+
debug4("Failed to add action step:", stepError);
|
2331
|
+
}
|
2332
|
+
}
|
2333
|
+
}
|
2168
2334
|
return {
|
2169
2335
|
result: output,
|
2170
2336
|
metadata
|
@@ -2194,38 +2360,50 @@ var PageAgent = class {
|
|
2194
2360
|
debug4("Context store not available:", error);
|
2195
2361
|
}
|
2196
2362
|
const { output, executor } = await this.taskExecutor.query(processedDemand);
|
2197
|
-
if (
|
2198
|
-
|
2199
|
-
|
2200
|
-
|
2201
|
-
|
2202
|
-
|
2203
|
-
|
2204
|
-
|
2205
|
-
|
2206
|
-
|
2363
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2364
|
+
if (storageKey && output) {
|
2365
|
+
try {
|
2366
|
+
const pendingAliases = this.contextStore._pendingAliases;
|
2367
|
+
if (pendingAliases) {
|
2368
|
+
this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
|
2369
|
+
delete this.contextStore._pendingAliases;
|
2370
|
+
debug4("Stored query result with aliases:", {
|
2371
|
+
key: storageKey,
|
2372
|
+
value: output,
|
2373
|
+
aliases: pendingAliases
|
2374
|
+
});
|
2375
|
+
} else {
|
2376
|
+
this.contextStore.storeData(storageKey, output);
|
2377
|
+
debug4("Stored query result:", {
|
2378
|
+
key: storageKey,
|
2379
|
+
value: output
|
2380
|
+
});
|
2381
|
+
}
|
2382
|
+
this.contextStore.addStep({
|
2383
|
+
type: "query",
|
2384
|
+
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
|
2385
|
+
data: output,
|
2386
|
+
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2387
|
+
});
|
2388
|
+
debug4("Added query step to context store:", {
|
2389
|
+
storageKey,
|
2390
|
+
totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
|
2391
|
+
totalSteps: this.contextStore.getRecentSteps(100).length
|
2392
|
+
});
|
2393
|
+
} catch (error) {
|
2394
|
+
debug4("Failed to store query result:", error);
|
2395
|
+
}
|
2396
|
+
} else {
|
2397
|
+
try {
|
2398
|
+
this.contextStore.addStep({
|
2399
|
+
type: "query",
|
2400
|
+
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
|
2401
|
+
data: output,
|
2402
|
+
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2403
|
+
});
|
2404
|
+
} catch (error) {
|
2405
|
+
debug4("Failed to add query step:", error);
|
2207
2406
|
}
|
2208
|
-
contextStore.addStep({
|
2209
|
-
type: "query",
|
2210
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
|
2211
|
-
data: output,
|
2212
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2213
|
-
});
|
2214
|
-
} catch (error) {
|
2215
|
-
debug4("Failed to store query result:", error);
|
2216
|
-
}
|
2217
|
-
} else {
|
2218
|
-
try {
|
2219
|
-
const aiModel = await import("misoai-core/ai-model");
|
2220
|
-
const contextStore = aiModel.getContextStore();
|
2221
|
-
contextStore.addStep({
|
2222
|
-
type: "query",
|
2223
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
|
2224
|
-
data: output,
|
2225
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2226
|
-
});
|
2227
|
-
} catch (error) {
|
2228
|
-
debug4("Failed to add query step:", error);
|
2229
2407
|
}
|
2230
2408
|
}
|
2231
2409
|
const metadata = this.afterTaskRunning(executor);
|
@@ -2337,18 +2515,47 @@ var PageAgent = class {
|
|
2337
2515
|
};
|
2338
2516
|
}
|
2339
2517
|
async aiAssert(assertion, msg, opt) {
|
2340
|
-
let
|
2341
|
-
|
2342
|
-
|
2343
|
-
|
2344
|
-
|
2345
|
-
|
2346
|
-
|
2347
|
-
|
2348
|
-
|
2349
|
-
|
2350
|
-
|
2351
|
-
|
2518
|
+
let executionContext = "";
|
2519
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2520
|
+
try {
|
2521
|
+
const recentSteps = this.contextStore.getRecentSteps(3);
|
2522
|
+
const stepsWithExecutionResults = recentSteps.filter((step) => step.executionResult);
|
2523
|
+
const storedData = this.contextStore.getAllData();
|
2524
|
+
if (stepsWithExecutionResults.length > 0) {
|
2525
|
+
const recentActions = stepsWithExecutionResults.map((step) => {
|
2526
|
+
const result = step.executionResult;
|
2527
|
+
return `- ${result.description}${result.success ? "" : " (FAILED)"}`;
|
2528
|
+
}).join("\n");
|
2529
|
+
executionContext = `
|
2530
|
+
|
2531
|
+
Recent actions performed:
|
2532
|
+
${recentActions}
|
2533
|
+
|
2534
|
+
This context may help verify the assertion.`;
|
2535
|
+
}
|
2536
|
+
if (storedData && Object.keys(storedData).length > 0) {
|
2537
|
+
executionContext += `
|
2538
|
+
|
2539
|
+
Available data for reference:
|
2540
|
+
${JSON.stringify(storedData, null, 2)}
|
2541
|
+
|
2542
|
+
Note: If the assertion references any data keys or natural language equivalents, consider the stored values when verifying.`;
|
2543
|
+
debug4("Available data for aiAssert:", {
|
2544
|
+
assertion,
|
2545
|
+
availableData: storedData
|
2546
|
+
});
|
2547
|
+
}
|
2548
|
+
this.contextStore.addStep({
|
2549
|
+
type: "assertion",
|
2550
|
+
summary: `Assertion: ${assertion}`,
|
2551
|
+
prompt: assertion
|
2552
|
+
});
|
2553
|
+
debug4("Added assertion step to context store:", {
|
2554
|
+
totalSteps: this.contextStore.getRecentSteps(100).length
|
2555
|
+
});
|
2556
|
+
} catch (error) {
|
2557
|
+
debug4("Context store operation failed:", error);
|
2558
|
+
}
|
2352
2559
|
}
|
2353
2560
|
let currentUrl = "";
|
2354
2561
|
if (this.page.url) {
|
@@ -2357,7 +2564,13 @@ var PageAgent = class {
|
|
2357
2564
|
} catch (e) {
|
2358
2565
|
}
|
2359
2566
|
}
|
2360
|
-
|
2567
|
+
let assertionWithContext = assertion;
|
2568
|
+
if (currentUrl) {
|
2569
|
+
assertionWithContext = `For the page at URL "${currentUrl}", ${assertion}`;
|
2570
|
+
}
|
2571
|
+
if (executionContext) {
|
2572
|
+
assertionWithContext += executionContext;
|
2573
|
+
}
|
2361
2574
|
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2362
2575
|
const metadata = this.afterTaskRunning(executor, true);
|
2363
2576
|
if (output && opt?.keepRawResponse) {
|
@@ -2571,6 +2784,79 @@ ${errors}`);
|
|
2571
2784
|
async destroy() {
|
2572
2785
|
await this.page.destroy();
|
2573
2786
|
}
|
2787
|
+
/**
|
2788
|
+
* Analyze execution results from executor to generate meaningful descriptions
|
2789
|
+
*/
|
2790
|
+
analyzeExecutionResults(executor, originalPrompt) {
|
2791
|
+
const tasks = executor.tasks;
|
2792
|
+
const success = !executor.isInErrorState();
|
2793
|
+
if (!success) {
|
2794
|
+
const errorTask = executor.latestErrorTask();
|
2795
|
+
return {
|
2796
|
+
success: false,
|
2797
|
+
actionType: "error",
|
2798
|
+
description: `Failed to execute: ${originalPrompt}`,
|
2799
|
+
error: errorTask?.error
|
2800
|
+
};
|
2801
|
+
}
|
2802
|
+
const actionTasks = tasks.filter((t) => t.type === "Action" && t.status === "finished");
|
2803
|
+
const locateTasks = tasks.filter((t) => t.type === "Insight" && t.subType === "Locate");
|
2804
|
+
const lastAction = actionTasks[actionTasks.length - 1];
|
2805
|
+
const lastLocate = locateTasks[locateTasks.length - 1];
|
2806
|
+
if (!lastAction) {
|
2807
|
+
return {
|
2808
|
+
success: true,
|
2809
|
+
actionType: "unknown",
|
2810
|
+
description: `Completed: ${originalPrompt}`
|
2811
|
+
};
|
2812
|
+
}
|
2813
|
+
const actionType = lastAction.subType || "unknown";
|
2814
|
+
const elementInfo = this.extractElementInfo(lastLocate, lastAction);
|
2815
|
+
const description = this.generateActionDescription(actionType, lastAction.param, elementInfo);
|
2816
|
+
return {
|
2817
|
+
success: true,
|
2818
|
+
actionType,
|
2819
|
+
description,
|
2820
|
+
elementInfo,
|
2821
|
+
timing: lastAction.timing?.cost
|
2822
|
+
};
|
2823
|
+
}
|
2824
|
+
/**
|
2825
|
+
* Extract element information from locate task
|
2826
|
+
*/
|
2827
|
+
extractElementInfo(locateTask, _actionTask) {
|
2828
|
+
if (!locateTask?.output?.element)
|
2829
|
+
return void 0;
|
2830
|
+
const element = locateTask.output.element;
|
2831
|
+
return {
|
2832
|
+
type: element.attributes?.nodeType || "unknown",
|
2833
|
+
text: element.content || element.attributes?.placeholder || element.attributes?.title || "",
|
2834
|
+
location: `(${element.center[0]}, ${element.center[1]})`
|
2835
|
+
};
|
2836
|
+
}
|
2837
|
+
/**
|
2838
|
+
* Generate natural language description for actions
|
2839
|
+
*/
|
2840
|
+
generateActionDescription(actionType, param, elementInfo) {
|
2841
|
+
const elementDesc = elementInfo ? `'${elementInfo.text || elementInfo.type}' element` : "element";
|
2842
|
+
switch (actionType) {
|
2843
|
+
case "Tap":
|
2844
|
+
return `Clicked on ${elementDesc}`;
|
2845
|
+
case "Input":
|
2846
|
+
const inputValue = param?.value || "";
|
2847
|
+
return `Entered "${inputValue}" into ${elementDesc}`;
|
2848
|
+
case "KeyboardPress":
|
2849
|
+
return `Pressed ${param?.value || "key"}`;
|
2850
|
+
case "Scroll":
|
2851
|
+
return `Scrolled ${param?.direction || "on page"}`;
|
2852
|
+
case "Hover":
|
2853
|
+
return `Hovered over ${elementDesc}`;
|
2854
|
+
case "Drag":
|
2855
|
+
return `Dragged ${elementDesc}`;
|
2856
|
+
default:
|
2857
|
+
return `Performed ${actionType} action on ${elementDesc}`;
|
2858
|
+
}
|
2859
|
+
}
|
2574
2860
|
};
|
2575
2861
|
|
2576
2862
|
// src/playground/agent.ts
|