misoai-web 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +353 -9
- package/bin/midscene-playground +2 -2
- package/dist/es/agent.js +378 -92
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +3 -3
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +380 -94
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +379 -93
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +378 -92
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +378 -92
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +378 -92
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +378 -92
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +378 -92
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +378 -92
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +378 -92
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +3 -3
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +380 -94
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +379 -93
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +378 -92
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +378 -92
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +378 -92
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +378 -92
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +378 -92
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +378 -92
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +45 -4
- package/dist/types/bridge-mode-browser.d.ts +2 -2
- package/dist/types/bridge-mode.d.ts +2 -2
- package/dist/types/{browser-a1877d18.d.ts → browser-aec1055d.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -2
- package/dist/types/index.d.ts +1 -1
- package/dist/types/midscene-server.d.ts +1 -1
- package/dist/types/{page-663ece08.d.ts → page-86ab0fe1.d.ts} +34 -34
- package/dist/types/playground.d.ts +2 -2
- package/dist/types/playwright.d.ts +1 -1
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
- package/dist/types/puppeteer.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +1 -1
- package/package.json +3 -3
@@ -1713,7 +1713,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
|
|
1713
1713
|
var import_semver = __toESM(require("semver"));
|
1714
1714
|
|
1715
1715
|
// package.json
|
1716
|
-
var version = "1.0.
|
1716
|
+
var version = "1.0.5";
|
1717
1717
|
|
1718
1718
|
// src/common/task-cache.ts
|
1719
1719
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -1741,44 +1741,70 @@ var TaskCache = class {
|
|
1741
1741
|
this.cache = cacheContent;
|
1742
1742
|
this.cacheOriginalLength = this.cache.caches.length;
|
1743
1743
|
}
|
1744
|
-
matchCache(prompt, type) {
|
1744
|
+
matchCache(prompt, type, contextData) {
|
1745
|
+
const contextHash = contextData ? this.generateContextHash(contextData) : void 0;
|
1745
1746
|
for (let i = 0; i < this.cacheOriginalLength; i++) {
|
1746
1747
|
const item = this.cache.caches[i];
|
1747
1748
|
const key = `${type}:${prompt}:${i}`;
|
1748
|
-
if (item.type
|
1749
|
-
|
1750
|
-
|
1751
|
-
|
1752
|
-
|
1753
|
-
|
1754
|
-
|
1755
|
-
|
1756
|
-
|
1757
|
-
cacheContent: item,
|
1758
|
-
updateFn: (cb) => {
|
1759
|
-
debug3(
|
1760
|
-
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1761
|
-
type,
|
1762
|
-
prompt,
|
1763
|
-
i
|
1764
|
-
);
|
1765
|
-
cb(item);
|
1766
|
-
debug3(
|
1767
|
-
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1768
|
-
type,
|
1769
|
-
prompt,
|
1770
|
-
i
|
1771
|
-
);
|
1772
|
-
this.flushCacheToFile();
|
1749
|
+
if (item.type !== type || item.prompt !== prompt || this.matchedCacheIndices.has(key)) {
|
1750
|
+
continue;
|
1751
|
+
}
|
1752
|
+
if (type === "plan" && item.type === "plan") {
|
1753
|
+
const planItem = item;
|
1754
|
+
if (contextHash && planItem.contextHash) {
|
1755
|
+
if (contextHash !== planItem.contextHash) {
|
1756
|
+
debug3("cache context mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
|
1757
|
+
continue;
|
1773
1758
|
}
|
1774
|
-
}
|
1759
|
+
} else if (contextHash || planItem.contextHash) {
|
1760
|
+
debug3("cache context availability mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
|
1761
|
+
continue;
|
1762
|
+
}
|
1775
1763
|
}
|
1764
|
+
this.matchedCacheIndices.add(key);
|
1765
|
+
debug3(
|
1766
|
+
"cache found and marked as used, type: %s, prompt: %s, index: %d, contextMatch: %s",
|
1767
|
+
type,
|
1768
|
+
prompt,
|
1769
|
+
i,
|
1770
|
+
contextHash ? "yes" : "no-context"
|
1771
|
+
);
|
1772
|
+
return {
|
1773
|
+
cacheContent: item,
|
1774
|
+
updateFn: (cb) => {
|
1775
|
+
debug3(
|
1776
|
+
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1777
|
+
type,
|
1778
|
+
prompt,
|
1779
|
+
i
|
1780
|
+
);
|
1781
|
+
cb(item);
|
1782
|
+
debug3(
|
1783
|
+
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1784
|
+
type,
|
1785
|
+
prompt,
|
1786
|
+
i
|
1787
|
+
);
|
1788
|
+
this.flushCacheToFile();
|
1789
|
+
}
|
1790
|
+
};
|
1776
1791
|
}
|
1777
|
-
debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
|
1792
|
+
debug3("no unused cache found, type: %s, prompt: %s, contextHash: %s", type, prompt, contextHash);
|
1778
1793
|
return void 0;
|
1779
1794
|
}
|
1780
|
-
|
1781
|
-
|
1795
|
+
generateContextHash(contextData) {
|
1796
|
+
const sortedKeys = Object.keys(contextData).sort();
|
1797
|
+
const stableString = sortedKeys.map((key) => `${key}:${JSON.stringify(contextData[key])}`).join("|");
|
1798
|
+
let hash = 0;
|
1799
|
+
for (let i = 0; i < stableString.length; i++) {
|
1800
|
+
const char = stableString.charCodeAt(i);
|
1801
|
+
hash = (hash << 5) - hash + char;
|
1802
|
+
hash = hash & hash;
|
1803
|
+
}
|
1804
|
+
return hash.toString(36);
|
1805
|
+
}
|
1806
|
+
matchPlanCache(prompt, contextData) {
|
1807
|
+
return this.matchCache(prompt, "plan", contextData);
|
1782
1808
|
}
|
1783
1809
|
matchLocateCache(prompt) {
|
1784
1810
|
return this.matchCache(prompt, "locate");
|
@@ -1854,11 +1880,16 @@ cache file: ${cacheFile}`
|
|
1854
1880
|
);
|
1855
1881
|
}
|
1856
1882
|
}
|
1857
|
-
updateOrAppendCacheRecord(newRecord, cachedRecord) {
|
1883
|
+
updateOrAppendCacheRecord(newRecord, cachedRecord, contextData) {
|
1858
1884
|
if (cachedRecord) {
|
1859
1885
|
if (newRecord.type === "plan") {
|
1860
1886
|
cachedRecord.updateFn((cache) => {
|
1861
|
-
|
1887
|
+
const planCache = cache;
|
1888
|
+
planCache.yamlWorkflow = newRecord.yamlWorkflow;
|
1889
|
+
if (contextData) {
|
1890
|
+
planCache.contextHash = this.generateContextHash(contextData);
|
1891
|
+
planCache.contextData = { ...contextData };
|
1892
|
+
}
|
1862
1893
|
});
|
1863
1894
|
} else {
|
1864
1895
|
cachedRecord.updateFn((cache) => {
|
@@ -1866,6 +1897,11 @@ cache file: ${cacheFile}`
|
|
1866
1897
|
});
|
1867
1898
|
}
|
1868
1899
|
} else {
|
1900
|
+
if (newRecord.type === "plan" && contextData) {
|
1901
|
+
const planRecord = newRecord;
|
1902
|
+
planRecord.contextHash = this.generateContextHash(contextData);
|
1903
|
+
planRecord.contextData = { ...contextData };
|
1904
|
+
}
|
1869
1905
|
this.appendCache(newRecord);
|
1870
1906
|
}
|
1871
1907
|
}
|
@@ -1895,10 +1931,13 @@ var PageAgent = class {
|
|
1895
1931
|
generateReport: true,
|
1896
1932
|
autoPrintReportMsg: true,
|
1897
1933
|
groupName: "Midscene Report",
|
1898
|
-
groupDescription: ""
|
1934
|
+
groupDescription: "",
|
1935
|
+
enableCumulativeContext: true,
|
1936
|
+
autoClearContext: false
|
1899
1937
|
},
|
1900
1938
|
opts || {}
|
1901
1939
|
);
|
1940
|
+
this.initializeContextStore();
|
1902
1941
|
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1903
1942
|
this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || import_constants2.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
|
1904
1943
|
this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || import_constants2.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
|
@@ -1925,6 +1964,69 @@ var PageAgent = class {
|
|
1925
1964
|
opts?.testId || this.page.pageType || "web"
|
1926
1965
|
);
|
1927
1966
|
}
|
1967
|
+
/**
|
1968
|
+
* Initialize context store for cumulative context functionality
|
1969
|
+
*/
|
1970
|
+
async initializeContextStore() {
|
1971
|
+
if (!this.opts.enableCumulativeContext) {
|
1972
|
+
debug4("Cumulative context disabled via options");
|
1973
|
+
return;
|
1974
|
+
}
|
1975
|
+
try {
|
1976
|
+
const aiModel = await import("misoai-core/ai-model");
|
1977
|
+
this.contextStore = aiModel.getContextStore();
|
1978
|
+
debug4("Context store initialized successfully", {
|
1979
|
+
autoClearContext: this.opts.autoClearContext,
|
1980
|
+
testId: this.opts.testId
|
1981
|
+
});
|
1982
|
+
if (this.opts.autoClearContext) {
|
1983
|
+
this.contextStore.clear();
|
1984
|
+
debug4("Context store cleared due to autoClearContext option");
|
1985
|
+
} else {
|
1986
|
+
const existingData = this.contextStore.getAllData();
|
1987
|
+
const existingSteps = this.contextStore.getRecentSteps(100).length;
|
1988
|
+
debug4("Context store preserving existing data", {
|
1989
|
+
existingDataKeys: Object.keys(existingData),
|
1990
|
+
existingStepsCount: existingSteps
|
1991
|
+
});
|
1992
|
+
}
|
1993
|
+
} catch (error) {
|
1994
|
+
debug4("Failed to initialize context store:", error);
|
1995
|
+
console.warn("⚠️ Could not initialize context store:", error);
|
1996
|
+
}
|
1997
|
+
}
|
1998
|
+
/**
|
1999
|
+
* Get the context store instance
|
2000
|
+
*/
|
2001
|
+
getContextStore() {
|
2002
|
+
return this.contextStore;
|
2003
|
+
}
|
2004
|
+
/**
|
2005
|
+
* Clear the context store
|
2006
|
+
*/
|
2007
|
+
clearContext() {
|
2008
|
+
if (this.contextStore) {
|
2009
|
+
this.contextStore.clear();
|
2010
|
+
}
|
2011
|
+
}
|
2012
|
+
/**
|
2013
|
+
* Get all stored data from context store
|
2014
|
+
*/
|
2015
|
+
getStoredData() {
|
2016
|
+
if (this.contextStore) {
|
2017
|
+
return this.contextStore.getAllData();
|
2018
|
+
}
|
2019
|
+
return {};
|
2020
|
+
}
|
2021
|
+
/**
|
2022
|
+
* Get step summary from context store
|
2023
|
+
*/
|
2024
|
+
getStepSummary() {
|
2025
|
+
if (this.contextStore) {
|
2026
|
+
return this.contextStore.getStepSummary();
|
2027
|
+
}
|
2028
|
+
return "";
|
2029
|
+
}
|
1928
2030
|
async getUIContext(action) {
|
1929
2031
|
if (action && (action === "extract" || action === "assert" || action === "captcha")) {
|
1930
2032
|
return await parseContextFromWebPage(this.page, {
|
@@ -2160,22 +2262,35 @@ var PageAgent = class {
|
|
2160
2262
|
};
|
2161
2263
|
}
|
2162
2264
|
async aiAction(taskPrompt, opt) {
|
2163
|
-
|
2164
|
-
|
2165
|
-
|
2166
|
-
|
2167
|
-
|
2168
|
-
|
2169
|
-
|
2170
|
-
|
2171
|
-
|
2172
|
-
|
2173
|
-
|
2174
|
-
|
2265
|
+
const originalPrompt = taskPrompt;
|
2266
|
+
let processedPrompt = taskPrompt;
|
2267
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2268
|
+
try {
|
2269
|
+
const storedData = this.contextStore.getAllData();
|
2270
|
+
if (Object.keys(storedData).length > 0) {
|
2271
|
+
debug4("Available data for aiAction:", {
|
2272
|
+
prompt: taskPrompt,
|
2273
|
+
availableData: storedData
|
2274
|
+
});
|
2275
|
+
}
|
2276
|
+
} catch (error) {
|
2277
|
+
debug4("Context store operation failed:", error);
|
2278
|
+
}
|
2175
2279
|
}
|
2176
2280
|
const cacheable = opt?.cacheable;
|
2177
2281
|
const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
|
2178
|
-
|
2282
|
+
let contextData;
|
2283
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2284
|
+
try {
|
2285
|
+
contextData = this.contextStore.getAllData();
|
2286
|
+
if (contextData && Object.keys(contextData).length === 0) {
|
2287
|
+
contextData = void 0;
|
2288
|
+
}
|
2289
|
+
} catch (error) {
|
2290
|
+
debug4("Failed to get context data for cache:", error);
|
2291
|
+
}
|
2292
|
+
}
|
2293
|
+
const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt, contextData);
|
2179
2294
|
if (matchedCache && this.taskCache?.isCacheResultUsed) {
|
2180
2295
|
const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
|
2181
2296
|
taskPrompt,
|
@@ -2185,6 +2300,28 @@ var PageAgent = class {
|
|
2185
2300
|
debug4("matched cache, will call .runYaml to run the action");
|
2186
2301
|
const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
|
2187
2302
|
const result = await this.runYaml(yaml5);
|
2303
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2304
|
+
try {
|
2305
|
+
const executionResult = {
|
2306
|
+
success: true,
|
2307
|
+
actionType: "cached",
|
2308
|
+
description: `Executed cached action: ${processedPrompt}`,
|
2309
|
+
timing: result.metadata?.totalTime
|
2310
|
+
};
|
2311
|
+
this.contextStore.addStep({
|
2312
|
+
type: "action",
|
2313
|
+
summary: `Action: ${processedPrompt} (cached)`,
|
2314
|
+
prompt: processedPrompt,
|
2315
|
+
executionResult
|
2316
|
+
});
|
2317
|
+
debug4("Added cached action step to context store:", {
|
2318
|
+
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2319
|
+
totalSteps: this.contextStore.getRecentSteps(100).length
|
2320
|
+
});
|
2321
|
+
} catch (error) {
|
2322
|
+
debug4("Failed to add cached action step:", error);
|
2323
|
+
}
|
2324
|
+
}
|
2188
2325
|
return {
|
2189
2326
|
result: result.result,
|
2190
2327
|
metadata: metadata2
|
@@ -2209,10 +2346,39 @@ var PageAgent = class {
|
|
2209
2346
|
prompt: taskPrompt,
|
2210
2347
|
yamlWorkflow: yamlFlowStr
|
2211
2348
|
},
|
2212
|
-
matchedCache
|
2349
|
+
matchedCache,
|
2350
|
+
contextData
|
2351
|
+
// Pass context data for cache creation
|
2213
2352
|
);
|
2214
2353
|
}
|
2215
2354
|
const metadata = this.afterTaskRunning(executor);
|
2355
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2356
|
+
try {
|
2357
|
+
const executionResult = this.analyzeExecutionResults(executor, originalPrompt);
|
2358
|
+
this.contextStore.addStep({
|
2359
|
+
type: "action",
|
2360
|
+
summary: `Action: ${processedPrompt}`,
|
2361
|
+
prompt: processedPrompt,
|
2362
|
+
executionResult
|
2363
|
+
});
|
2364
|
+
debug4("Added action step with execution result to context store:", {
|
2365
|
+
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2366
|
+
totalSteps: this.contextStore.getRecentSteps(100).length,
|
2367
|
+
executionResult
|
2368
|
+
});
|
2369
|
+
} catch (error) {
|
2370
|
+
debug4("Failed to analyze execution results, adding step without execution result:", error);
|
2371
|
+
try {
|
2372
|
+
this.contextStore.addStep({
|
2373
|
+
type: "action",
|
2374
|
+
summary: `Action: ${processedPrompt}`,
|
2375
|
+
prompt: processedPrompt
|
2376
|
+
});
|
2377
|
+
} catch (stepError) {
|
2378
|
+
debug4("Failed to add action step:", stepError);
|
2379
|
+
}
|
2380
|
+
}
|
2381
|
+
}
|
2216
2382
|
return {
|
2217
2383
|
result: output,
|
2218
2384
|
metadata
|
@@ -2242,38 +2408,50 @@ var PageAgent = class {
|
|
2242
2408
|
debug4("Context store not available:", error);
|
2243
2409
|
}
|
2244
2410
|
const { output, executor } = await this.taskExecutor.query(processedDemand);
|
2245
|
-
if (
|
2246
|
-
|
2247
|
-
|
2248
|
-
|
2249
|
-
|
2250
|
-
|
2251
|
-
|
2252
|
-
|
2253
|
-
|
2254
|
-
|
2411
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2412
|
+
if (storageKey && output) {
|
2413
|
+
try {
|
2414
|
+
const pendingAliases = this.contextStore._pendingAliases;
|
2415
|
+
if (pendingAliases) {
|
2416
|
+
this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
|
2417
|
+
delete this.contextStore._pendingAliases;
|
2418
|
+
debug4("Stored query result with aliases:", {
|
2419
|
+
key: storageKey,
|
2420
|
+
value: output,
|
2421
|
+
aliases: pendingAliases
|
2422
|
+
});
|
2423
|
+
} else {
|
2424
|
+
this.contextStore.storeData(storageKey, output);
|
2425
|
+
debug4("Stored query result:", {
|
2426
|
+
key: storageKey,
|
2427
|
+
value: output
|
2428
|
+
});
|
2429
|
+
}
|
2430
|
+
this.contextStore.addStep({
|
2431
|
+
type: "query",
|
2432
|
+
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
|
2433
|
+
data: output,
|
2434
|
+
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2435
|
+
});
|
2436
|
+
debug4("Added query step to context store:", {
|
2437
|
+
storageKey,
|
2438
|
+
totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
|
2439
|
+
totalSteps: this.contextStore.getRecentSteps(100).length
|
2440
|
+
});
|
2441
|
+
} catch (error) {
|
2442
|
+
debug4("Failed to store query result:", error);
|
2443
|
+
}
|
2444
|
+
} else {
|
2445
|
+
try {
|
2446
|
+
this.contextStore.addStep({
|
2447
|
+
type: "query",
|
2448
|
+
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
|
2449
|
+
data: output,
|
2450
|
+
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2451
|
+
});
|
2452
|
+
} catch (error) {
|
2453
|
+
debug4("Failed to add query step:", error);
|
2255
2454
|
}
|
2256
|
-
contextStore.addStep({
|
2257
|
-
type: "query",
|
2258
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
|
2259
|
-
data: output,
|
2260
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2261
|
-
});
|
2262
|
-
} catch (error) {
|
2263
|
-
debug4("Failed to store query result:", error);
|
2264
|
-
}
|
2265
|
-
} else {
|
2266
|
-
try {
|
2267
|
-
const aiModel = await import("misoai-core/ai-model");
|
2268
|
-
const contextStore = aiModel.getContextStore();
|
2269
|
-
contextStore.addStep({
|
2270
|
-
type: "query",
|
2271
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
|
2272
|
-
data: output,
|
2273
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2274
|
-
});
|
2275
|
-
} catch (error) {
|
2276
|
-
debug4("Failed to add query step:", error);
|
2277
2455
|
}
|
2278
2456
|
}
|
2279
2457
|
const metadata = this.afterTaskRunning(executor);
|
@@ -2385,18 +2563,47 @@ var PageAgent = class {
|
|
2385
2563
|
};
|
2386
2564
|
}
|
2387
2565
|
async aiAssert(assertion, msg, opt) {
|
2388
|
-
let
|
2389
|
-
|
2390
|
-
|
2391
|
-
|
2392
|
-
|
2393
|
-
|
2394
|
-
|
2395
|
-
|
2396
|
-
|
2397
|
-
|
2398
|
-
|
2399
|
-
|
2566
|
+
let executionContext = "";
|
2567
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2568
|
+
try {
|
2569
|
+
const recentSteps = this.contextStore.getRecentSteps(3);
|
2570
|
+
const stepsWithExecutionResults = recentSteps.filter((step) => step.executionResult);
|
2571
|
+
const storedData = this.contextStore.getAllData();
|
2572
|
+
if (stepsWithExecutionResults.length > 0) {
|
2573
|
+
const recentActions = stepsWithExecutionResults.map((step) => {
|
2574
|
+
const result = step.executionResult;
|
2575
|
+
return `- ${result.description}${result.success ? "" : " (FAILED)"}`;
|
2576
|
+
}).join("\n");
|
2577
|
+
executionContext = `
|
2578
|
+
|
2579
|
+
Recent actions performed:
|
2580
|
+
${recentActions}
|
2581
|
+
|
2582
|
+
This context may help verify the assertion.`;
|
2583
|
+
}
|
2584
|
+
if (storedData && Object.keys(storedData).length > 0) {
|
2585
|
+
executionContext += `
|
2586
|
+
|
2587
|
+
Available data for reference:
|
2588
|
+
${JSON.stringify(storedData, null, 2)}
|
2589
|
+
|
2590
|
+
Note: If the assertion references any data keys or natural language equivalents, consider the stored values when verifying.`;
|
2591
|
+
debug4("Available data for aiAssert:", {
|
2592
|
+
assertion,
|
2593
|
+
availableData: storedData
|
2594
|
+
});
|
2595
|
+
}
|
2596
|
+
this.contextStore.addStep({
|
2597
|
+
type: "assertion",
|
2598
|
+
summary: `Assertion: ${assertion}`,
|
2599
|
+
prompt: assertion
|
2600
|
+
});
|
2601
|
+
debug4("Added assertion step to context store:", {
|
2602
|
+
totalSteps: this.contextStore.getRecentSteps(100).length
|
2603
|
+
});
|
2604
|
+
} catch (error) {
|
2605
|
+
debug4("Context store operation failed:", error);
|
2606
|
+
}
|
2400
2607
|
}
|
2401
2608
|
let currentUrl = "";
|
2402
2609
|
if (this.page.url) {
|
@@ -2405,7 +2612,13 @@ var PageAgent = class {
|
|
2405
2612
|
} catch (e) {
|
2406
2613
|
}
|
2407
2614
|
}
|
2408
|
-
|
2615
|
+
let assertionWithContext = assertion;
|
2616
|
+
if (currentUrl) {
|
2617
|
+
assertionWithContext = `For the page at URL "${currentUrl}", ${assertion}`;
|
2618
|
+
}
|
2619
|
+
if (executionContext) {
|
2620
|
+
assertionWithContext += executionContext;
|
2621
|
+
}
|
2409
2622
|
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2410
2623
|
const metadata = this.afterTaskRunning(executor, true);
|
2411
2624
|
if (output && opt?.keepRawResponse) {
|
@@ -2619,6 +2832,79 @@ ${errors}`);
|
|
2619
2832
|
async destroy() {
|
2620
2833
|
await this.page.destroy();
|
2621
2834
|
}
|
2835
|
+
/**
|
2836
|
+
* Analyze execution results from executor to generate meaningful descriptions
|
2837
|
+
*/
|
2838
|
+
analyzeExecutionResults(executor, originalPrompt) {
|
2839
|
+
const tasks = executor.tasks;
|
2840
|
+
const success = !executor.isInErrorState();
|
2841
|
+
if (!success) {
|
2842
|
+
const errorTask = executor.latestErrorTask();
|
2843
|
+
return {
|
2844
|
+
success: false,
|
2845
|
+
actionType: "error",
|
2846
|
+
description: `Failed to execute: ${originalPrompt}`,
|
2847
|
+
error: errorTask?.error
|
2848
|
+
};
|
2849
|
+
}
|
2850
|
+
const actionTasks = tasks.filter((t) => t.type === "Action" && t.status === "finished");
|
2851
|
+
const locateTasks = tasks.filter((t) => t.type === "Insight" && t.subType === "Locate");
|
2852
|
+
const lastAction = actionTasks[actionTasks.length - 1];
|
2853
|
+
const lastLocate = locateTasks[locateTasks.length - 1];
|
2854
|
+
if (!lastAction) {
|
2855
|
+
return {
|
2856
|
+
success: true,
|
2857
|
+
actionType: "unknown",
|
2858
|
+
description: `Completed: ${originalPrompt}`
|
2859
|
+
};
|
2860
|
+
}
|
2861
|
+
const actionType = lastAction.subType || "unknown";
|
2862
|
+
const elementInfo = this.extractElementInfo(lastLocate, lastAction);
|
2863
|
+
const description = this.generateActionDescription(actionType, lastAction.param, elementInfo);
|
2864
|
+
return {
|
2865
|
+
success: true,
|
2866
|
+
actionType,
|
2867
|
+
description,
|
2868
|
+
elementInfo,
|
2869
|
+
timing: lastAction.timing?.cost
|
2870
|
+
};
|
2871
|
+
}
|
2872
|
+
/**
|
2873
|
+
* Extract element information from locate task
|
2874
|
+
*/
|
2875
|
+
extractElementInfo(locateTask, _actionTask) {
|
2876
|
+
if (!locateTask?.output?.element)
|
2877
|
+
return void 0;
|
2878
|
+
const element = locateTask.output.element;
|
2879
|
+
return {
|
2880
|
+
type: element.attributes?.nodeType || "unknown",
|
2881
|
+
text: element.content || element.attributes?.placeholder || element.attributes?.title || "",
|
2882
|
+
location: `(${element.center[0]}, ${element.center[1]})`
|
2883
|
+
};
|
2884
|
+
}
|
2885
|
+
/**
|
2886
|
+
* Generate natural language description for actions
|
2887
|
+
*/
|
2888
|
+
generateActionDescription(actionType, param, elementInfo) {
|
2889
|
+
const elementDesc = elementInfo ? `'${elementInfo.text || elementInfo.type}' element` : "element";
|
2890
|
+
switch (actionType) {
|
2891
|
+
case "Tap":
|
2892
|
+
return `Clicked on ${elementDesc}`;
|
2893
|
+
case "Input":
|
2894
|
+
const inputValue = param?.value || "";
|
2895
|
+
return `Entered "${inputValue}" into ${elementDesc}`;
|
2896
|
+
case "KeyboardPress":
|
2897
|
+
return `Pressed ${param?.value || "key"}`;
|
2898
|
+
case "Scroll":
|
2899
|
+
return `Scrolled ${param?.direction || "on page"}`;
|
2900
|
+
case "Hover":
|
2901
|
+
return `Hovered over ${elementDesc}`;
|
2902
|
+
case "Drag":
|
2903
|
+
return `Dragged ${elementDesc}`;
|
2904
|
+
default:
|
2905
|
+
return `Performed ${actionType} action on ${elementDesc}`;
|
2906
|
+
}
|
2907
|
+
}
|
2622
2908
|
};
|
2623
2909
|
|
2624
2910
|
// src/puppeteer/index.ts
|