misoai-web 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +353 -9
- package/bin/midscene-playground +2 -2
- package/dist/es/agent.js +378 -92
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +3 -3
- package/dist/es/bridge-mode-browser.js.map +1 -1
- package/dist/es/bridge-mode.js +380 -94
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +379 -93
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +378 -92
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +378 -92
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/midscene-server.js.map +1 -1
- package/dist/es/playground.js +378 -92
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright-report.js.map +1 -1
- package/dist/es/playwright.js +378 -92
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +378 -92
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +378 -92
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/ui-utils.js.map +1 -1
- package/dist/es/utils.js.map +1 -1
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +378 -92
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +3 -3
- package/dist/lib/bridge-mode-browser.js.map +1 -1
- package/dist/lib/bridge-mode.js +380 -94
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +379 -93
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +378 -92
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +378 -92
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/midscene-server.js.map +1 -1
- package/dist/lib/playground.js +378 -92
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright-report.js.map +1 -1
- package/dist/lib/playwright.js +378 -92
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +378 -92
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +378 -92
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/ui-utils.js.map +1 -1
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +45 -4
- package/dist/types/bridge-mode-browser.d.ts +2 -2
- package/dist/types/bridge-mode.d.ts +2 -2
- package/dist/types/{browser-a1877d18.d.ts → browser-aec1055d.d.ts} +1 -1
- package/dist/types/chrome-extension.d.ts +2 -2
- package/dist/types/index.d.ts +1 -1
- package/dist/types/midscene-server.d.ts +1 -1
- package/dist/types/{page-663ece08.d.ts → page-86ab0fe1.d.ts} +34 -34
- package/dist/types/playground.d.ts +2 -2
- package/dist/types/playwright.d.ts +1 -1
- package/dist/types/puppeteer-agent-launcher.d.ts +1 -1
- package/dist/types/puppeteer.d.ts +1 -1
- package/dist/types/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +1 -1
- package/package.json +3 -3
package/dist/lib/index.js
CHANGED
@@ -1711,7 +1711,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
|
|
1711
1711
|
var import_semver = __toESM(require("semver"));
|
1712
1712
|
|
1713
1713
|
// package.json
|
1714
|
-
var version = "1.0.
|
1714
|
+
var version = "1.0.5";
|
1715
1715
|
|
1716
1716
|
// src/common/task-cache.ts
|
1717
1717
|
var debug3 = (0, import_logger3.getDebug)("cache");
|
@@ -1739,44 +1739,70 @@ var TaskCache = class {
|
|
1739
1739
|
this.cache = cacheContent;
|
1740
1740
|
this.cacheOriginalLength = this.cache.caches.length;
|
1741
1741
|
}
|
1742
|
-
matchCache(prompt, type) {
|
1742
|
+
matchCache(prompt, type, contextData) {
|
1743
|
+
const contextHash = contextData ? this.generateContextHash(contextData) : void 0;
|
1743
1744
|
for (let i = 0; i < this.cacheOriginalLength; i++) {
|
1744
1745
|
const item = this.cache.caches[i];
|
1745
1746
|
const key = `${type}:${prompt}:${i}`;
|
1746
|
-
if (item.type
|
1747
|
-
|
1748
|
-
|
1749
|
-
|
1750
|
-
|
1751
|
-
|
1752
|
-
|
1753
|
-
|
1754
|
-
|
1755
|
-
cacheContent: item,
|
1756
|
-
updateFn: (cb) => {
|
1757
|
-
debug3(
|
1758
|
-
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1759
|
-
type,
|
1760
|
-
prompt,
|
1761
|
-
i
|
1762
|
-
);
|
1763
|
-
cb(item);
|
1764
|
-
debug3(
|
1765
|
-
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1766
|
-
type,
|
1767
|
-
prompt,
|
1768
|
-
i
|
1769
|
-
);
|
1770
|
-
this.flushCacheToFile();
|
1747
|
+
if (item.type !== type || item.prompt !== prompt || this.matchedCacheIndices.has(key)) {
|
1748
|
+
continue;
|
1749
|
+
}
|
1750
|
+
if (type === "plan" && item.type === "plan") {
|
1751
|
+
const planItem = item;
|
1752
|
+
if (contextHash && planItem.contextHash) {
|
1753
|
+
if (contextHash !== planItem.contextHash) {
|
1754
|
+
debug3("cache context mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
|
1755
|
+
continue;
|
1771
1756
|
}
|
1772
|
-
}
|
1757
|
+
} else if (contextHash || planItem.contextHash) {
|
1758
|
+
debug3("cache context availability mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
|
1759
|
+
continue;
|
1760
|
+
}
|
1773
1761
|
}
|
1762
|
+
this.matchedCacheIndices.add(key);
|
1763
|
+
debug3(
|
1764
|
+
"cache found and marked as used, type: %s, prompt: %s, index: %d, contextMatch: %s",
|
1765
|
+
type,
|
1766
|
+
prompt,
|
1767
|
+
i,
|
1768
|
+
contextHash ? "yes" : "no-context"
|
1769
|
+
);
|
1770
|
+
return {
|
1771
|
+
cacheContent: item,
|
1772
|
+
updateFn: (cb) => {
|
1773
|
+
debug3(
|
1774
|
+
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1775
|
+
type,
|
1776
|
+
prompt,
|
1777
|
+
i
|
1778
|
+
);
|
1779
|
+
cb(item);
|
1780
|
+
debug3(
|
1781
|
+
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1782
|
+
type,
|
1783
|
+
prompt,
|
1784
|
+
i
|
1785
|
+
);
|
1786
|
+
this.flushCacheToFile();
|
1787
|
+
}
|
1788
|
+
};
|
1774
1789
|
}
|
1775
|
-
debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
|
1790
|
+
debug3("no unused cache found, type: %s, prompt: %s, contextHash: %s", type, prompt, contextHash);
|
1776
1791
|
return void 0;
|
1777
1792
|
}
|
1778
|
-
|
1779
|
-
|
1793
|
+
generateContextHash(contextData) {
|
1794
|
+
const sortedKeys = Object.keys(contextData).sort();
|
1795
|
+
const stableString = sortedKeys.map((key) => `${key}:${JSON.stringify(contextData[key])}`).join("|");
|
1796
|
+
let hash = 0;
|
1797
|
+
for (let i = 0; i < stableString.length; i++) {
|
1798
|
+
const char = stableString.charCodeAt(i);
|
1799
|
+
hash = (hash << 5) - hash + char;
|
1800
|
+
hash = hash & hash;
|
1801
|
+
}
|
1802
|
+
return hash.toString(36);
|
1803
|
+
}
|
1804
|
+
matchPlanCache(prompt, contextData) {
|
1805
|
+
return this.matchCache(prompt, "plan", contextData);
|
1780
1806
|
}
|
1781
1807
|
matchLocateCache(prompt) {
|
1782
1808
|
return this.matchCache(prompt, "locate");
|
@@ -1852,11 +1878,16 @@ cache file: ${cacheFile}`
|
|
1852
1878
|
);
|
1853
1879
|
}
|
1854
1880
|
}
|
1855
|
-
updateOrAppendCacheRecord(newRecord, cachedRecord) {
|
1881
|
+
updateOrAppendCacheRecord(newRecord, cachedRecord, contextData) {
|
1856
1882
|
if (cachedRecord) {
|
1857
1883
|
if (newRecord.type === "plan") {
|
1858
1884
|
cachedRecord.updateFn((cache) => {
|
1859
|
-
|
1885
|
+
const planCache = cache;
|
1886
|
+
planCache.yamlWorkflow = newRecord.yamlWorkflow;
|
1887
|
+
if (contextData) {
|
1888
|
+
planCache.contextHash = this.generateContextHash(contextData);
|
1889
|
+
planCache.contextData = { ...contextData };
|
1890
|
+
}
|
1860
1891
|
});
|
1861
1892
|
} else {
|
1862
1893
|
cachedRecord.updateFn((cache) => {
|
@@ -1864,6 +1895,11 @@ cache file: ${cacheFile}`
|
|
1864
1895
|
});
|
1865
1896
|
}
|
1866
1897
|
} else {
|
1898
|
+
if (newRecord.type === "plan" && contextData) {
|
1899
|
+
const planRecord = newRecord;
|
1900
|
+
planRecord.contextHash = this.generateContextHash(contextData);
|
1901
|
+
planRecord.contextData = { ...contextData };
|
1902
|
+
}
|
1867
1903
|
this.appendCache(newRecord);
|
1868
1904
|
}
|
1869
1905
|
}
|
@@ -1893,10 +1929,13 @@ var PageAgent = class {
|
|
1893
1929
|
generateReport: true,
|
1894
1930
|
autoPrintReportMsg: true,
|
1895
1931
|
groupName: "Midscene Report",
|
1896
|
-
groupDescription: ""
|
1932
|
+
groupDescription: "",
|
1933
|
+
enableCumulativeContext: true,
|
1934
|
+
autoClearContext: false
|
1897
1935
|
},
|
1898
1936
|
opts || {}
|
1899
1937
|
);
|
1938
|
+
this.initializeContextStore();
|
1900
1939
|
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1901
1940
|
this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || import_constants2.DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
|
1902
1941
|
this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || import_constants2.DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
|
@@ -1923,6 +1962,69 @@ var PageAgent = class {
|
|
1923
1962
|
opts?.testId || this.page.pageType || "web"
|
1924
1963
|
);
|
1925
1964
|
}
|
1965
|
+
/**
|
1966
|
+
* Initialize context store for cumulative context functionality
|
1967
|
+
*/
|
1968
|
+
async initializeContextStore() {
|
1969
|
+
if (!this.opts.enableCumulativeContext) {
|
1970
|
+
debug4("Cumulative context disabled via options");
|
1971
|
+
return;
|
1972
|
+
}
|
1973
|
+
try {
|
1974
|
+
const aiModel = await import("misoai-core/ai-model");
|
1975
|
+
this.contextStore = aiModel.getContextStore();
|
1976
|
+
debug4("Context store initialized successfully", {
|
1977
|
+
autoClearContext: this.opts.autoClearContext,
|
1978
|
+
testId: this.opts.testId
|
1979
|
+
});
|
1980
|
+
if (this.opts.autoClearContext) {
|
1981
|
+
this.contextStore.clear();
|
1982
|
+
debug4("Context store cleared due to autoClearContext option");
|
1983
|
+
} else {
|
1984
|
+
const existingData = this.contextStore.getAllData();
|
1985
|
+
const existingSteps = this.contextStore.getRecentSteps(100).length;
|
1986
|
+
debug4("Context store preserving existing data", {
|
1987
|
+
existingDataKeys: Object.keys(existingData),
|
1988
|
+
existingStepsCount: existingSteps
|
1989
|
+
});
|
1990
|
+
}
|
1991
|
+
} catch (error) {
|
1992
|
+
debug4("Failed to initialize context store:", error);
|
1993
|
+
console.warn("⚠️ Could not initialize context store:", error);
|
1994
|
+
}
|
1995
|
+
}
|
1996
|
+
/**
|
1997
|
+
* Get the context store instance
|
1998
|
+
*/
|
1999
|
+
getContextStore() {
|
2000
|
+
return this.contextStore;
|
2001
|
+
}
|
2002
|
+
/**
|
2003
|
+
* Clear the context store
|
2004
|
+
*/
|
2005
|
+
clearContext() {
|
2006
|
+
if (this.contextStore) {
|
2007
|
+
this.contextStore.clear();
|
2008
|
+
}
|
2009
|
+
}
|
2010
|
+
/**
|
2011
|
+
* Get all stored data from context store
|
2012
|
+
*/
|
2013
|
+
getStoredData() {
|
2014
|
+
if (this.contextStore) {
|
2015
|
+
return this.contextStore.getAllData();
|
2016
|
+
}
|
2017
|
+
return {};
|
2018
|
+
}
|
2019
|
+
/**
|
2020
|
+
* Get step summary from context store
|
2021
|
+
*/
|
2022
|
+
getStepSummary() {
|
2023
|
+
if (this.contextStore) {
|
2024
|
+
return this.contextStore.getStepSummary();
|
2025
|
+
}
|
2026
|
+
return "";
|
2027
|
+
}
|
1926
2028
|
async getUIContext(action) {
|
1927
2029
|
if (action && (action === "extract" || action === "assert" || action === "captcha")) {
|
1928
2030
|
return await parseContextFromWebPage(this.page, {
|
@@ -2158,22 +2260,35 @@ var PageAgent = class {
|
|
2158
2260
|
};
|
2159
2261
|
}
|
2160
2262
|
async aiAction(taskPrompt, opt) {
|
2161
|
-
|
2162
|
-
|
2163
|
-
|
2164
|
-
|
2165
|
-
|
2166
|
-
|
2167
|
-
|
2168
|
-
|
2169
|
-
|
2170
|
-
|
2171
|
-
|
2172
|
-
|
2263
|
+
const originalPrompt = taskPrompt;
|
2264
|
+
let processedPrompt = taskPrompt;
|
2265
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2266
|
+
try {
|
2267
|
+
const storedData = this.contextStore.getAllData();
|
2268
|
+
if (Object.keys(storedData).length > 0) {
|
2269
|
+
debug4("Available data for aiAction:", {
|
2270
|
+
prompt: taskPrompt,
|
2271
|
+
availableData: storedData
|
2272
|
+
});
|
2273
|
+
}
|
2274
|
+
} catch (error) {
|
2275
|
+
debug4("Context store operation failed:", error);
|
2276
|
+
}
|
2173
2277
|
}
|
2174
2278
|
const cacheable = opt?.cacheable;
|
2175
2279
|
const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
|
2176
|
-
|
2280
|
+
let contextData;
|
2281
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2282
|
+
try {
|
2283
|
+
contextData = this.contextStore.getAllData();
|
2284
|
+
if (contextData && Object.keys(contextData).length === 0) {
|
2285
|
+
contextData = void 0;
|
2286
|
+
}
|
2287
|
+
} catch (error) {
|
2288
|
+
debug4("Failed to get context data for cache:", error);
|
2289
|
+
}
|
2290
|
+
}
|
2291
|
+
const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt, contextData);
|
2177
2292
|
if (matchedCache && this.taskCache?.isCacheResultUsed) {
|
2178
2293
|
const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
|
2179
2294
|
taskPrompt,
|
@@ -2183,6 +2298,28 @@ var PageAgent = class {
|
|
2183
2298
|
debug4("matched cache, will call .runYaml to run the action");
|
2184
2299
|
const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
|
2185
2300
|
const result = await this.runYaml(yaml5);
|
2301
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2302
|
+
try {
|
2303
|
+
const executionResult = {
|
2304
|
+
success: true,
|
2305
|
+
actionType: "cached",
|
2306
|
+
description: `Executed cached action: ${processedPrompt}`,
|
2307
|
+
timing: result.metadata?.totalTime
|
2308
|
+
};
|
2309
|
+
this.contextStore.addStep({
|
2310
|
+
type: "action",
|
2311
|
+
summary: `Action: ${processedPrompt} (cached)`,
|
2312
|
+
prompt: processedPrompt,
|
2313
|
+
executionResult
|
2314
|
+
});
|
2315
|
+
debug4("Added cached action step to context store:", {
|
2316
|
+
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2317
|
+
totalSteps: this.contextStore.getRecentSteps(100).length
|
2318
|
+
});
|
2319
|
+
} catch (error) {
|
2320
|
+
debug4("Failed to add cached action step:", error);
|
2321
|
+
}
|
2322
|
+
}
|
2186
2323
|
return {
|
2187
2324
|
result: result.result,
|
2188
2325
|
metadata: metadata2
|
@@ -2207,10 +2344,39 @@ var PageAgent = class {
|
|
2207
2344
|
prompt: taskPrompt,
|
2208
2345
|
yamlWorkflow: yamlFlowStr
|
2209
2346
|
},
|
2210
|
-
matchedCache
|
2347
|
+
matchedCache,
|
2348
|
+
contextData
|
2349
|
+
// Pass context data for cache creation
|
2211
2350
|
);
|
2212
2351
|
}
|
2213
2352
|
const metadata = this.afterTaskRunning(executor);
|
2353
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2354
|
+
try {
|
2355
|
+
const executionResult = this.analyzeExecutionResults(executor, originalPrompt);
|
2356
|
+
this.contextStore.addStep({
|
2357
|
+
type: "action",
|
2358
|
+
summary: `Action: ${processedPrompt}`,
|
2359
|
+
prompt: processedPrompt,
|
2360
|
+
executionResult
|
2361
|
+
});
|
2362
|
+
debug4("Added action step with execution result to context store:", {
|
2363
|
+
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2364
|
+
totalSteps: this.contextStore.getRecentSteps(100).length,
|
2365
|
+
executionResult
|
2366
|
+
});
|
2367
|
+
} catch (error) {
|
2368
|
+
debug4("Failed to analyze execution results, adding step without execution result:", error);
|
2369
|
+
try {
|
2370
|
+
this.contextStore.addStep({
|
2371
|
+
type: "action",
|
2372
|
+
summary: `Action: ${processedPrompt}`,
|
2373
|
+
prompt: processedPrompt
|
2374
|
+
});
|
2375
|
+
} catch (stepError) {
|
2376
|
+
debug4("Failed to add action step:", stepError);
|
2377
|
+
}
|
2378
|
+
}
|
2379
|
+
}
|
2214
2380
|
return {
|
2215
2381
|
result: output,
|
2216
2382
|
metadata
|
@@ -2240,38 +2406,50 @@ var PageAgent = class {
|
|
2240
2406
|
debug4("Context store not available:", error);
|
2241
2407
|
}
|
2242
2408
|
const { output, executor } = await this.taskExecutor.query(processedDemand);
|
2243
|
-
if (
|
2244
|
-
|
2245
|
-
|
2246
|
-
|
2247
|
-
|
2248
|
-
|
2249
|
-
|
2250
|
-
|
2251
|
-
|
2252
|
-
|
2409
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2410
|
+
if (storageKey && output) {
|
2411
|
+
try {
|
2412
|
+
const pendingAliases = this.contextStore._pendingAliases;
|
2413
|
+
if (pendingAliases) {
|
2414
|
+
this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
|
2415
|
+
delete this.contextStore._pendingAliases;
|
2416
|
+
debug4("Stored query result with aliases:", {
|
2417
|
+
key: storageKey,
|
2418
|
+
value: output,
|
2419
|
+
aliases: pendingAliases
|
2420
|
+
});
|
2421
|
+
} else {
|
2422
|
+
this.contextStore.storeData(storageKey, output);
|
2423
|
+
debug4("Stored query result:", {
|
2424
|
+
key: storageKey,
|
2425
|
+
value: output
|
2426
|
+
});
|
2427
|
+
}
|
2428
|
+
this.contextStore.addStep({
|
2429
|
+
type: "query",
|
2430
|
+
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
|
2431
|
+
data: output,
|
2432
|
+
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2433
|
+
});
|
2434
|
+
debug4("Added query step to context store:", {
|
2435
|
+
storageKey,
|
2436
|
+
totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
|
2437
|
+
totalSteps: this.contextStore.getRecentSteps(100).length
|
2438
|
+
});
|
2439
|
+
} catch (error) {
|
2440
|
+
debug4("Failed to store query result:", error);
|
2441
|
+
}
|
2442
|
+
} else {
|
2443
|
+
try {
|
2444
|
+
this.contextStore.addStep({
|
2445
|
+
type: "query",
|
2446
|
+
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
|
2447
|
+
data: output,
|
2448
|
+
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2449
|
+
});
|
2450
|
+
} catch (error) {
|
2451
|
+
debug4("Failed to add query step:", error);
|
2253
2452
|
}
|
2254
|
-
contextStore.addStep({
|
2255
|
-
type: "query",
|
2256
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
|
2257
|
-
data: output,
|
2258
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2259
|
-
});
|
2260
|
-
} catch (error) {
|
2261
|
-
debug4("Failed to store query result:", error);
|
2262
|
-
}
|
2263
|
-
} else {
|
2264
|
-
try {
|
2265
|
-
const aiModel = await import("misoai-core/ai-model");
|
2266
|
-
const contextStore = aiModel.getContextStore();
|
2267
|
-
contextStore.addStep({
|
2268
|
-
type: "query",
|
2269
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
|
2270
|
-
data: output,
|
2271
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2272
|
-
});
|
2273
|
-
} catch (error) {
|
2274
|
-
debug4("Failed to add query step:", error);
|
2275
2453
|
}
|
2276
2454
|
}
|
2277
2455
|
const metadata = this.afterTaskRunning(executor);
|
@@ -2383,18 +2561,47 @@ var PageAgent = class {
|
|
2383
2561
|
};
|
2384
2562
|
}
|
2385
2563
|
async aiAssert(assertion, msg, opt) {
|
2386
|
-
let
|
2387
|
-
|
2388
|
-
|
2389
|
-
|
2390
|
-
|
2391
|
-
|
2392
|
-
|
2393
|
-
|
2394
|
-
|
2395
|
-
|
2396
|
-
|
2397
|
-
|
2564
|
+
let executionContext = "";
|
2565
|
+
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2566
|
+
try {
|
2567
|
+
const recentSteps = this.contextStore.getRecentSteps(3);
|
2568
|
+
const stepsWithExecutionResults = recentSteps.filter((step) => step.executionResult);
|
2569
|
+
const storedData = this.contextStore.getAllData();
|
2570
|
+
if (stepsWithExecutionResults.length > 0) {
|
2571
|
+
const recentActions = stepsWithExecutionResults.map((step) => {
|
2572
|
+
const result = step.executionResult;
|
2573
|
+
return `- ${result.description}${result.success ? "" : " (FAILED)"}`;
|
2574
|
+
}).join("\n");
|
2575
|
+
executionContext = `
|
2576
|
+
|
2577
|
+
Recent actions performed:
|
2578
|
+
${recentActions}
|
2579
|
+
|
2580
|
+
This context may help verify the assertion.`;
|
2581
|
+
}
|
2582
|
+
if (storedData && Object.keys(storedData).length > 0) {
|
2583
|
+
executionContext += `
|
2584
|
+
|
2585
|
+
Available data for reference:
|
2586
|
+
${JSON.stringify(storedData, null, 2)}
|
2587
|
+
|
2588
|
+
Note: If the assertion references any data keys or natural language equivalents, consider the stored values when verifying.`;
|
2589
|
+
debug4("Available data for aiAssert:", {
|
2590
|
+
assertion,
|
2591
|
+
availableData: storedData
|
2592
|
+
});
|
2593
|
+
}
|
2594
|
+
this.contextStore.addStep({
|
2595
|
+
type: "assertion",
|
2596
|
+
summary: `Assertion: ${assertion}`,
|
2597
|
+
prompt: assertion
|
2598
|
+
});
|
2599
|
+
debug4("Added assertion step to context store:", {
|
2600
|
+
totalSteps: this.contextStore.getRecentSteps(100).length
|
2601
|
+
});
|
2602
|
+
} catch (error) {
|
2603
|
+
debug4("Context store operation failed:", error);
|
2604
|
+
}
|
2398
2605
|
}
|
2399
2606
|
let currentUrl = "";
|
2400
2607
|
if (this.page.url) {
|
@@ -2403,7 +2610,13 @@ var PageAgent = class {
|
|
2403
2610
|
} catch (e) {
|
2404
2611
|
}
|
2405
2612
|
}
|
2406
|
-
|
2613
|
+
let assertionWithContext = assertion;
|
2614
|
+
if (currentUrl) {
|
2615
|
+
assertionWithContext = `For the page at URL "${currentUrl}", ${assertion}`;
|
2616
|
+
}
|
2617
|
+
if (executionContext) {
|
2618
|
+
assertionWithContext += executionContext;
|
2619
|
+
}
|
2407
2620
|
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2408
2621
|
const metadata = this.afterTaskRunning(executor, true);
|
2409
2622
|
if (output && opt?.keepRawResponse) {
|
@@ -2617,6 +2830,79 @@ ${errors}`);
|
|
2617
2830
|
async destroy() {
|
2618
2831
|
await this.page.destroy();
|
2619
2832
|
}
|
2833
|
+
/**
|
2834
|
+
* Analyze execution results from executor to generate meaningful descriptions
|
2835
|
+
*/
|
2836
|
+
analyzeExecutionResults(executor, originalPrompt) {
|
2837
|
+
const tasks = executor.tasks;
|
2838
|
+
const success = !executor.isInErrorState();
|
2839
|
+
if (!success) {
|
2840
|
+
const errorTask = executor.latestErrorTask();
|
2841
|
+
return {
|
2842
|
+
success: false,
|
2843
|
+
actionType: "error",
|
2844
|
+
description: `Failed to execute: ${originalPrompt}`,
|
2845
|
+
error: errorTask?.error
|
2846
|
+
};
|
2847
|
+
}
|
2848
|
+
const actionTasks = tasks.filter((t) => t.type === "Action" && t.status === "finished");
|
2849
|
+
const locateTasks = tasks.filter((t) => t.type === "Insight" && t.subType === "Locate");
|
2850
|
+
const lastAction = actionTasks[actionTasks.length - 1];
|
2851
|
+
const lastLocate = locateTasks[locateTasks.length - 1];
|
2852
|
+
if (!lastAction) {
|
2853
|
+
return {
|
2854
|
+
success: true,
|
2855
|
+
actionType: "unknown",
|
2856
|
+
description: `Completed: ${originalPrompt}`
|
2857
|
+
};
|
2858
|
+
}
|
2859
|
+
const actionType = lastAction.subType || "unknown";
|
2860
|
+
const elementInfo = this.extractElementInfo(lastLocate, lastAction);
|
2861
|
+
const description = this.generateActionDescription(actionType, lastAction.param, elementInfo);
|
2862
|
+
return {
|
2863
|
+
success: true,
|
2864
|
+
actionType,
|
2865
|
+
description,
|
2866
|
+
elementInfo,
|
2867
|
+
timing: lastAction.timing?.cost
|
2868
|
+
};
|
2869
|
+
}
|
2870
|
+
/**
|
2871
|
+
* Extract element information from locate task
|
2872
|
+
*/
|
2873
|
+
extractElementInfo(locateTask, _actionTask) {
|
2874
|
+
if (!locateTask?.output?.element)
|
2875
|
+
return void 0;
|
2876
|
+
const element = locateTask.output.element;
|
2877
|
+
return {
|
2878
|
+
type: element.attributes?.nodeType || "unknown",
|
2879
|
+
text: element.content || element.attributes?.placeholder || element.attributes?.title || "",
|
2880
|
+
location: `(${element.center[0]}, ${element.center[1]})`
|
2881
|
+
};
|
2882
|
+
}
|
2883
|
+
/**
|
2884
|
+
* Generate natural language description for actions
|
2885
|
+
*/
|
2886
|
+
generateActionDescription(actionType, param, elementInfo) {
|
2887
|
+
const elementDesc = elementInfo ? `'${elementInfo.text || elementInfo.type}' element` : "element";
|
2888
|
+
switch (actionType) {
|
2889
|
+
case "Tap":
|
2890
|
+
return `Clicked on ${elementDesc}`;
|
2891
|
+
case "Input":
|
2892
|
+
const inputValue = param?.value || "";
|
2893
|
+
return `Entered "${inputValue}" into ${elementDesc}`;
|
2894
|
+
case "KeyboardPress":
|
2895
|
+
return `Pressed ${param?.value || "key"}`;
|
2896
|
+
case "Scroll":
|
2897
|
+
return `Scrolled ${param?.direction || "on page"}`;
|
2898
|
+
case "Hover":
|
2899
|
+
return `Hovered over ${elementDesc}`;
|
2900
|
+
case "Drag":
|
2901
|
+
return `Dragged ${elementDesc}`;
|
2902
|
+
default:
|
2903
|
+
return `Performed ${actionType} action on ${elementDesc}`;
|
2904
|
+
}
|
2905
|
+
}
|
2620
2906
|
};
|
2621
2907
|
|
2622
2908
|
// src/puppeteer/base-page.ts
|