misoai-web 1.0.6 → 1.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -349
- package/dist/es/agent.js +43 -409
- package/dist/es/agent.js.map +1 -1
- package/dist/es/bridge-mode-browser.js +3 -3
- package/dist/es/bridge-mode.js +45 -411
- package/dist/es/bridge-mode.js.map +1 -1
- package/dist/es/chrome-extension.js +44 -410
- package/dist/es/chrome-extension.js.map +1 -1
- package/dist/es/index.js +47 -413
- package/dist/es/index.js.map +1 -1
- package/dist/es/midscene-playground.js +43 -409
- package/dist/es/midscene-playground.js.map +1 -1
- package/dist/es/playground.js +43 -409
- package/dist/es/playground.js.map +1 -1
- package/dist/es/playwright.js +44 -410
- package/dist/es/playwright.js.map +1 -1
- package/dist/es/puppeteer-agent-launcher.js +47 -413
- package/dist/es/puppeteer-agent-launcher.js.map +1 -1
- package/dist/es/puppeteer.js +47 -413
- package/dist/es/puppeteer.js.map +1 -1
- package/dist/es/yaml.js +5 -3
- package/dist/es/yaml.js.map +1 -1
- package/dist/lib/agent.js +43 -409
- package/dist/lib/agent.js.map +1 -1
- package/dist/lib/bridge-mode-browser.js +3 -3
- package/dist/lib/bridge-mode.js +45 -411
- package/dist/lib/bridge-mode.js.map +1 -1
- package/dist/lib/chrome-extension.js +44 -410
- package/dist/lib/chrome-extension.js.map +1 -1
- package/dist/lib/index.js +47 -413
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/midscene-playground.js +43 -409
- package/dist/lib/midscene-playground.js.map +1 -1
- package/dist/lib/playground.js +43 -409
- package/dist/lib/playground.js.map +1 -1
- package/dist/lib/playwright.js +44 -410
- package/dist/lib/playwright.js.map +1 -1
- package/dist/lib/puppeteer-agent-launcher.js +47 -413
- package/dist/lib/puppeteer-agent-launcher.js.map +1 -1
- package/dist/lib/puppeteer.js +47 -413
- package/dist/lib/puppeteer.js.map +1 -1
- package/dist/lib/yaml.js +5 -3
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent.d.ts +3 -44
- package/iife-script/htmlElement.js +2 -2
- package/iife-script/htmlElementDebug.js +2 -2
- package/package.json +4 -4
package/dist/es/agent.js
CHANGED
@@ -17,10 +17,11 @@ var ScriptPlayer = class {
|
|
17
17
|
this.unnamedResultIndex = 0;
|
18
18
|
this.pageAgent = null;
|
19
19
|
this.result = {};
|
20
|
+
const target = script.target || script.web || script.android;
|
20
21
|
if (ifInBrowser) {
|
21
22
|
this.output = void 0;
|
22
|
-
} else if (
|
23
|
-
this.output = resolve(process.cwd(),
|
23
|
+
} else if (target?.output) {
|
24
|
+
this.output = resolve(process.cwd(), target.output);
|
24
25
|
} else {
|
25
26
|
this.output = join(getMidsceneRunSubDir("output"), `${process.pid}.json`);
|
26
27
|
}
|
@@ -94,12 +95,13 @@ var ScriptPlayer = class {
|
|
94
95
|
} else if ("aiAssert" in flowItem) {
|
95
96
|
const assertTask = flowItem;
|
96
97
|
const prompt = assertTask.aiAssert;
|
98
|
+
const msg = assertTask.errorMessage;
|
97
99
|
assert(prompt, "missing prompt for aiAssert");
|
98
100
|
assert(
|
99
101
|
typeof prompt === "string",
|
100
102
|
"prompt for aiAssert must be a string"
|
101
103
|
);
|
102
|
-
await agent.aiAssert(prompt);
|
104
|
+
await agent.aiAssert(prompt, msg);
|
103
105
|
} else if ("aiQuery" in flowItem) {
|
104
106
|
const queryTask = flowItem;
|
105
107
|
const prompt = queryTask.aiQuery;
|
@@ -1660,7 +1662,7 @@ import yaml3 from "js-yaml";
|
|
1660
1662
|
import semver from "semver";
|
1661
1663
|
|
1662
1664
|
// package.json
|
1663
|
-
var version = "1.
|
1665
|
+
var version = "1.5.6";
|
1664
1666
|
|
1665
1667
|
// src/common/task-cache.ts
|
1666
1668
|
var debug3 = getDebug3("cache");
|
@@ -1688,70 +1690,44 @@ var TaskCache = class {
|
|
1688
1690
|
this.cache = cacheContent;
|
1689
1691
|
this.cacheOriginalLength = this.cache.caches.length;
|
1690
1692
|
}
|
1691
|
-
matchCache(prompt, type
|
1692
|
-
const contextHash = contextData ? this.generateContextHash(contextData) : void 0;
|
1693
|
+
matchCache(prompt, type) {
|
1693
1694
|
for (let i = 0; i < this.cacheOriginalLength; i++) {
|
1694
1695
|
const item = this.cache.caches[i];
|
1695
1696
|
const key = `${type}:${prompt}:${i}`;
|
1696
|
-
if (item.type
|
1697
|
-
|
1698
|
-
|
1699
|
-
|
1700
|
-
|
1701
|
-
|
1702
|
-
|
1703
|
-
|
1704
|
-
|
1697
|
+
if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
|
1698
|
+
this.matchedCacheIndices.add(key);
|
1699
|
+
debug3(
|
1700
|
+
"cache found and marked as used, type: %s, prompt: %s, index: %d",
|
1701
|
+
type,
|
1702
|
+
prompt,
|
1703
|
+
i
|
1704
|
+
);
|
1705
|
+
return {
|
1706
|
+
cacheContent: item,
|
1707
|
+
updateFn: (cb) => {
|
1708
|
+
debug3(
|
1709
|
+
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1710
|
+
type,
|
1711
|
+
prompt,
|
1712
|
+
i
|
1713
|
+
);
|
1714
|
+
cb(item);
|
1715
|
+
debug3(
|
1716
|
+
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1717
|
+
type,
|
1718
|
+
prompt,
|
1719
|
+
i
|
1720
|
+
);
|
1721
|
+
this.flushCacheToFile();
|
1705
1722
|
}
|
1706
|
-
}
|
1707
|
-
debug3("cache context availability mismatch, type: %s, prompt: %s, index: %d", type, prompt, i);
|
1708
|
-
continue;
|
1709
|
-
}
|
1723
|
+
};
|
1710
1724
|
}
|
1711
|
-
this.matchedCacheIndices.add(key);
|
1712
|
-
debug3(
|
1713
|
-
"cache found and marked as used, type: %s, prompt: %s, index: %d, contextMatch: %s",
|
1714
|
-
type,
|
1715
|
-
prompt,
|
1716
|
-
i,
|
1717
|
-
contextHash ? "yes" : "no-context"
|
1718
|
-
);
|
1719
|
-
return {
|
1720
|
-
cacheContent: item,
|
1721
|
-
updateFn: (cb) => {
|
1722
|
-
debug3(
|
1723
|
-
"will call updateFn to update cache, type: %s, prompt: %s, index: %d",
|
1724
|
-
type,
|
1725
|
-
prompt,
|
1726
|
-
i
|
1727
|
-
);
|
1728
|
-
cb(item);
|
1729
|
-
debug3(
|
1730
|
-
"cache updated, will flush to file, type: %s, prompt: %s, index: %d",
|
1731
|
-
type,
|
1732
|
-
prompt,
|
1733
|
-
i
|
1734
|
-
);
|
1735
|
-
this.flushCacheToFile();
|
1736
|
-
}
|
1737
|
-
};
|
1738
1725
|
}
|
1739
|
-
debug3("no unused cache found, type: %s, prompt: %s
|
1726
|
+
debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
|
1740
1727
|
return void 0;
|
1741
1728
|
}
|
1742
|
-
|
1743
|
-
|
1744
|
-
const stableString = sortedKeys.map((key) => `${key}:${JSON.stringify(contextData[key])}`).join("|");
|
1745
|
-
let hash = 0;
|
1746
|
-
for (let i = 0; i < stableString.length; i++) {
|
1747
|
-
const char = stableString.charCodeAt(i);
|
1748
|
-
hash = (hash << 5) - hash + char;
|
1749
|
-
hash = hash & hash;
|
1750
|
-
}
|
1751
|
-
return hash.toString(36);
|
1752
|
-
}
|
1753
|
-
matchPlanCache(prompt, contextData) {
|
1754
|
-
return this.matchCache(prompt, "plan", contextData);
|
1729
|
+
matchPlanCache(prompt) {
|
1730
|
+
return this.matchCache(prompt, "plan");
|
1755
1731
|
}
|
1756
1732
|
matchLocateCache(prompt) {
|
1757
1733
|
return this.matchCache(prompt, "locate");
|
@@ -1827,16 +1803,11 @@ cache file: ${cacheFile}`
|
|
1827
1803
|
);
|
1828
1804
|
}
|
1829
1805
|
}
|
1830
|
-
updateOrAppendCacheRecord(newRecord, cachedRecord
|
1806
|
+
updateOrAppendCacheRecord(newRecord, cachedRecord) {
|
1831
1807
|
if (cachedRecord) {
|
1832
1808
|
if (newRecord.type === "plan") {
|
1833
1809
|
cachedRecord.updateFn((cache) => {
|
1834
|
-
|
1835
|
-
planCache.yamlWorkflow = newRecord.yamlWorkflow;
|
1836
|
-
if (contextData) {
|
1837
|
-
planCache.contextHash = this.generateContextHash(contextData);
|
1838
|
-
planCache.contextData = { ...contextData };
|
1839
|
-
}
|
1810
|
+
cache.yamlWorkflow = newRecord.yamlWorkflow;
|
1840
1811
|
});
|
1841
1812
|
} else {
|
1842
1813
|
cachedRecord.updateFn((cache) => {
|
@@ -1844,11 +1815,6 @@ cache file: ${cacheFile}`
|
|
1844
1815
|
});
|
1845
1816
|
}
|
1846
1817
|
} else {
|
1847
|
-
if (newRecord.type === "plan" && contextData) {
|
1848
|
-
const planRecord = newRecord;
|
1849
|
-
planRecord.contextHash = this.generateContextHash(contextData);
|
1850
|
-
planRecord.contextData = { ...contextData };
|
1851
|
-
}
|
1852
1818
|
this.appendCache(newRecord);
|
1853
1819
|
}
|
1854
1820
|
}
|
@@ -1878,13 +1844,10 @@ var PageAgent = class {
|
|
1878
1844
|
generateReport: true,
|
1879
1845
|
autoPrintReportMsg: true,
|
1880
1846
|
groupName: "Midscene Report",
|
1881
|
-
groupDescription: ""
|
1882
|
-
enableCumulativeContext: true,
|
1883
|
-
autoClearContext: false
|
1847
|
+
groupDescription: ""
|
1884
1848
|
},
|
1885
1849
|
opts || {}
|
1886
1850
|
);
|
1887
|
-
this.initializeContextStore();
|
1888
1851
|
if (this.page.pageType === "puppeteer" || this.page.pageType === "playwright") {
|
1889
1852
|
this.page.waitForNavigationTimeout = this.opts.waitForNavigationTimeout || DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
|
1890
1853
|
this.page.waitForNetworkIdleTimeout = this.opts.waitForNetworkIdleTimeout || DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
|
@@ -1911,69 +1874,6 @@ var PageAgent = class {
|
|
1911
1874
|
opts?.testId || this.page.pageType || "web"
|
1912
1875
|
);
|
1913
1876
|
}
|
1914
|
-
/**
|
1915
|
-
* Initialize context store for cumulative context functionality
|
1916
|
-
*/
|
1917
|
-
async initializeContextStore() {
|
1918
|
-
if (!this.opts.enableCumulativeContext) {
|
1919
|
-
debug4("Cumulative context disabled via options");
|
1920
|
-
return;
|
1921
|
-
}
|
1922
|
-
try {
|
1923
|
-
const aiModel = await import("misoai-core/ai-model");
|
1924
|
-
this.contextStore = aiModel.getContextStore();
|
1925
|
-
debug4("Context store initialized successfully", {
|
1926
|
-
autoClearContext: this.opts.autoClearContext,
|
1927
|
-
testId: this.opts.testId
|
1928
|
-
});
|
1929
|
-
if (this.opts.autoClearContext) {
|
1930
|
-
this.contextStore.clear();
|
1931
|
-
debug4("Context store cleared due to autoClearContext option");
|
1932
|
-
} else {
|
1933
|
-
const existingData = this.contextStore.getAllData();
|
1934
|
-
const existingSteps = this.contextStore.getRecentSteps(100).length;
|
1935
|
-
debug4("Context store preserving existing data", {
|
1936
|
-
existingDataKeys: Object.keys(existingData),
|
1937
|
-
existingStepsCount: existingSteps
|
1938
|
-
});
|
1939
|
-
}
|
1940
|
-
} catch (error) {
|
1941
|
-
debug4("Failed to initialize context store:", error);
|
1942
|
-
console.warn("⚠️ Could not initialize context store:", error);
|
1943
|
-
}
|
1944
|
-
}
|
1945
|
-
/**
|
1946
|
-
* Get the context store instance
|
1947
|
-
*/
|
1948
|
-
getContextStore() {
|
1949
|
-
return this.contextStore;
|
1950
|
-
}
|
1951
|
-
/**
|
1952
|
-
* Clear the context store
|
1953
|
-
*/
|
1954
|
-
clearContext() {
|
1955
|
-
if (this.contextStore) {
|
1956
|
-
this.contextStore.clear();
|
1957
|
-
}
|
1958
|
-
}
|
1959
|
-
/**
|
1960
|
-
* Get all stored data from context store
|
1961
|
-
*/
|
1962
|
-
getStoredData() {
|
1963
|
-
if (this.contextStore) {
|
1964
|
-
return this.contextStore.getAllData();
|
1965
|
-
}
|
1966
|
-
return {};
|
1967
|
-
}
|
1968
|
-
/**
|
1969
|
-
* Get step summary from context store
|
1970
|
-
*/
|
1971
|
-
getStepSummary() {
|
1972
|
-
if (this.contextStore) {
|
1973
|
-
return this.contextStore.getStepSummary();
|
1974
|
-
}
|
1975
|
-
return "";
|
1976
|
-
}
|
1977
1877
|
async getUIContext(action) {
|
1978
1878
|
if (action && (action === "extract" || action === "assert" || action === "captcha")) {
|
1979
1879
|
return await parseContextFromWebPage(this.page, {
|
@@ -2209,35 +2109,9 @@ var PageAgent = class {
|
|
2209
2109
|
};
|
2210
2110
|
}
|
2211
2111
|
async aiAction(taskPrompt, opt) {
|
2212
|
-
const originalPrompt = taskPrompt;
|
2213
|
-
let processedPrompt = taskPrompt;
|
2214
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2215
|
-
try {
|
2216
|
-
const storedData = this.contextStore.getAllData();
|
2217
|
-
if (Object.keys(storedData).length > 0) {
|
2218
|
-
debug4("Available data for aiAction:", {
|
2219
|
-
prompt: taskPrompt,
|
2220
|
-
availableData: storedData
|
2221
|
-
});
|
2222
|
-
}
|
2223
|
-
} catch (error) {
|
2224
|
-
debug4("Context store operation failed:", error);
|
2225
|
-
}
|
2226
|
-
}
|
2227
2112
|
const cacheable = opt?.cacheable;
|
2228
2113
|
const isVlmUiTars = vlLocateMode() === "vlm-ui-tars";
|
2229
|
-
|
2230
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2231
|
-
try {
|
2232
|
-
contextData = this.contextStore.getAllData();
|
2233
|
-
if (contextData && Object.keys(contextData).length === 0) {
|
2234
|
-
contextData = void 0;
|
2235
|
-
}
|
2236
|
-
} catch (error) {
|
2237
|
-
debug4("Failed to get context data for cache:", error);
|
2238
|
-
}
|
2239
|
-
}
|
2240
|
-
const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt, contextData);
|
2114
|
+
const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
|
2241
2115
|
if (matchedCache && this.taskCache?.isCacheResultUsed) {
|
2242
2116
|
const { executor: executor2 } = await this.taskExecutor.loadYamlFlowAsPlanning(
|
2243
2117
|
taskPrompt,
|
@@ -2247,28 +2121,6 @@ var PageAgent = class {
|
|
2247
2121
|
debug4("matched cache, will call .runYaml to run the action");
|
2248
2122
|
const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
|
2249
2123
|
const result = await this.runYaml(yaml5);
|
2250
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2251
|
-
try {
|
2252
|
-
const executionResult = {
|
2253
|
-
success: true,
|
2254
|
-
actionType: "cached",
|
2255
|
-
description: `Executed cached action: ${processedPrompt}`,
|
2256
|
-
timing: result.metadata?.totalTime
|
2257
|
-
};
|
2258
|
-
this.contextStore.addStep({
|
2259
|
-
type: "action",
|
2260
|
-
summary: `Action: ${processedPrompt} (cached)`,
|
2261
|
-
prompt: processedPrompt,
|
2262
|
-
executionResult
|
2263
|
-
});
|
2264
|
-
debug4("Added cached action step to context store:", {
|
2265
|
-
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2266
|
-
totalSteps: this.contextStore.getRecentSteps(100).length
|
2267
|
-
});
|
2268
|
-
} catch (error) {
|
2269
|
-
debug4("Failed to add cached action step:", error);
|
2270
|
-
}
|
2271
|
-
}
|
2272
2124
|
return {
|
2273
2125
|
result: result.result,
|
2274
2126
|
metadata: metadata2
|
@@ -2293,114 +2145,17 @@ var PageAgent = class {
|
|
2293
2145
|
prompt: taskPrompt,
|
2294
2146
|
yamlWorkflow: yamlFlowStr
|
2295
2147
|
},
|
2296
|
-
matchedCache
|
2297
|
-
contextData
|
2298
|
-
// Pass context data for cache creation
|
2148
|
+
matchedCache
|
2299
2149
|
);
|
2300
2150
|
}
|
2301
2151
|
const metadata = this.afterTaskRunning(executor);
|
2302
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2303
|
-
try {
|
2304
|
-
const executionResult = this.analyzeExecutionResults(executor, originalPrompt);
|
2305
|
-
this.contextStore.addStep({
|
2306
|
-
type: "action",
|
2307
|
-
summary: `Action: ${processedPrompt}`,
|
2308
|
-
prompt: processedPrompt,
|
2309
|
-
executionResult
|
2310
|
-
});
|
2311
|
-
debug4("Added action step with execution result to context store:", {
|
2312
|
-
stepNumber: this.contextStore.getRecentSteps(1)[0]?.stepNumber,
|
2313
|
-
totalSteps: this.contextStore.getRecentSteps(100).length,
|
2314
|
-
executionResult
|
2315
|
-
});
|
2316
|
-
} catch (error) {
|
2317
|
-
debug4("Failed to analyze execution results, adding step without execution result:", error);
|
2318
|
-
try {
|
2319
|
-
this.contextStore.addStep({
|
2320
|
-
type: "action",
|
2321
|
-
summary: `Action: ${processedPrompt}`,
|
2322
|
-
prompt: processedPrompt
|
2323
|
-
});
|
2324
|
-
} catch (stepError) {
|
2325
|
-
debug4("Failed to add action step:", stepError);
|
2326
|
-
}
|
2327
|
-
}
|
2328
|
-
}
|
2329
2152
|
return {
|
2330
2153
|
result: output,
|
2331
2154
|
metadata
|
2332
2155
|
};
|
2333
2156
|
}
|
2334
2157
|
async aiQuery(demand) {
|
2335
|
-
|
2336
|
-
let storageKey;
|
2337
|
-
try {
|
2338
|
-
const aiModel = await import("misoai-core/ai-model");
|
2339
|
-
const contextStore = aiModel.getContextStore();
|
2340
|
-
if (typeof demand === "string") {
|
2341
|
-
const storageInstruction = contextStore.parseStorageInstruction(demand);
|
2342
|
-
if (storageInstruction) {
|
2343
|
-
storageKey = storageInstruction.key;
|
2344
|
-
processedDemand = storageInstruction.cleanText;
|
2345
|
-
contextStore._pendingAliases = storageInstruction.aliases;
|
2346
|
-
} else {
|
2347
|
-
const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
|
2348
|
-
if (storageMatch) {
|
2349
|
-
storageKey = storageMatch[1];
|
2350
|
-
processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
|
2351
|
-
}
|
2352
|
-
}
|
2353
|
-
}
|
2354
|
-
} catch (error) {
|
2355
|
-
debug4("Context store not available:", error);
|
2356
|
-
}
|
2357
|
-
const { output, executor } = await this.taskExecutor.query(processedDemand);
|
2358
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2359
|
-
if (storageKey && output) {
|
2360
|
-
try {
|
2361
|
-
const pendingAliases = this.contextStore._pendingAliases;
|
2362
|
-
if (pendingAliases) {
|
2363
|
-
this.contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
|
2364
|
-
delete this.contextStore._pendingAliases;
|
2365
|
-
debug4("Stored query result with aliases:", {
|
2366
|
-
key: storageKey,
|
2367
|
-
value: output,
|
2368
|
-
aliases: pendingAliases
|
2369
|
-
});
|
2370
|
-
} else {
|
2371
|
-
this.contextStore.storeData(storageKey, output);
|
2372
|
-
debug4("Stored query result:", {
|
2373
|
-
key: storageKey,
|
2374
|
-
value: output
|
2375
|
-
});
|
2376
|
-
}
|
2377
|
-
this.contextStore.addStep({
|
2378
|
-
type: "query",
|
2379
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
|
2380
|
-
data: output,
|
2381
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2382
|
-
});
|
2383
|
-
debug4("Added query step to context store:", {
|
2384
|
-
storageKey,
|
2385
|
-
totalStoredItems: Object.keys(this.contextStore.getAllData()).length,
|
2386
|
-
totalSteps: this.contextStore.getRecentSteps(100).length
|
2387
|
-
});
|
2388
|
-
} catch (error) {
|
2389
|
-
debug4("Failed to store query result:", error);
|
2390
|
-
}
|
2391
|
-
} else {
|
2392
|
-
try {
|
2393
|
-
this.contextStore.addStep({
|
2394
|
-
type: "query",
|
2395
|
-
summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
|
2396
|
-
data: output,
|
2397
|
-
prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
|
2398
|
-
});
|
2399
|
-
} catch (error) {
|
2400
|
-
debug4("Failed to add query step:", error);
|
2401
|
-
}
|
2402
|
-
}
|
2403
|
-
}
|
2158
|
+
const { output, executor } = await this.taskExecutor.query(demand);
|
2404
2159
|
const metadata = this.afterTaskRunning(executor);
|
2405
2160
|
return {
|
2406
2161
|
result: output,
|
@@ -2510,48 +2265,6 @@ var PageAgent = class {
|
|
2510
2265
|
};
|
2511
2266
|
}
|
2512
2267
|
async aiAssert(assertion, msg, opt) {
|
2513
|
-
let executionContext = "";
|
2514
|
-
if (this.opts.enableCumulativeContext && this.contextStore) {
|
2515
|
-
try {
|
2516
|
-
const recentSteps = this.contextStore.getRecentSteps(3);
|
2517
|
-
const stepsWithExecutionResults = recentSteps.filter((step) => step.executionResult);
|
2518
|
-
const storedData = this.contextStore.getAllData();
|
2519
|
-
if (stepsWithExecutionResults.length > 0) {
|
2520
|
-
const recentActions = stepsWithExecutionResults.map((step) => {
|
2521
|
-
const result = step.executionResult;
|
2522
|
-
return `- ${result.description}${result.success ? "" : " (FAILED)"}`;
|
2523
|
-
}).join("\n");
|
2524
|
-
executionContext = `
|
2525
|
-
|
2526
|
-
Recent actions performed:
|
2527
|
-
${recentActions}
|
2528
|
-
|
2529
|
-
This context may help verify the assertion.`;
|
2530
|
-
}
|
2531
|
-
if (storedData && Object.keys(storedData).length > 0) {
|
2532
|
-
executionContext += `
|
2533
|
-
|
2534
|
-
Available data for reference:
|
2535
|
-
${JSON.stringify(storedData, null, 2)}
|
2536
|
-
|
2537
|
-
Note: If the assertion references any data keys or natural language equivalents, consider the stored values when verifying.`;
|
2538
|
-
debug4("Available data for aiAssert:", {
|
2539
|
-
assertion,
|
2540
|
-
availableData: storedData
|
2541
|
-
});
|
2542
|
-
}
|
2543
|
-
this.contextStore.addStep({
|
2544
|
-
type: "assertion",
|
2545
|
-
summary: `Assertion: ${assertion}`,
|
2546
|
-
prompt: assertion
|
2547
|
-
});
|
2548
|
-
debug4("Added assertion step to context store:", {
|
2549
|
-
totalSteps: this.contextStore.getRecentSteps(100).length
|
2550
|
-
});
|
2551
|
-
} catch (error) {
|
2552
|
-
debug4("Context store operation failed:", error);
|
2553
|
-
}
|
2554
|
-
}
|
2555
2268
|
let currentUrl = "";
|
2556
2269
|
if (this.page.url) {
|
2557
2270
|
try {
|
@@ -2559,13 +2272,7 @@ Note: If the assertion references any data keys or natural language equivalents,
|
|
2559
2272
|
} catch (e) {
|
2560
2273
|
}
|
2561
2274
|
}
|
2562
|
-
|
2563
|
-
if (currentUrl) {
|
2564
|
-
assertionWithContext = `For the page at URL "${currentUrl}", ${assertion}`;
|
2565
|
-
}
|
2566
|
-
if (executionContext) {
|
2567
|
-
assertionWithContext += executionContext;
|
2568
|
-
}
|
2275
|
+
const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
|
2569
2276
|
const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
|
2570
2277
|
const metadata = this.afterTaskRunning(executor, true);
|
2571
2278
|
if (output && opt?.keepRawResponse) {
|
@@ -2779,79 +2486,6 @@ ${errors}`);
|
|
2779
2486
|
async destroy() {
|
2780
2487
|
await this.page.destroy();
|
2781
2488
|
}
|
2782
|
-
/**
|
2783
|
-
* Analyze execution results from executor to generate meaningful descriptions
|
2784
|
-
*/
|
2785
|
-
analyzeExecutionResults(executor, originalPrompt) {
|
2786
|
-
const tasks = executor.tasks;
|
2787
|
-
const success = !executor.isInErrorState();
|
2788
|
-
if (!success) {
|
2789
|
-
const errorTask = executor.latestErrorTask();
|
2790
|
-
return {
|
2791
|
-
success: false,
|
2792
|
-
actionType: "error",
|
2793
|
-
description: `Failed to execute: ${originalPrompt}`,
|
2794
|
-
error: errorTask?.error
|
2795
|
-
};
|
2796
|
-
}
|
2797
|
-
const actionTasks = tasks.filter((t) => t.type === "Action" && t.status === "finished");
|
2798
|
-
const locateTasks = tasks.filter((t) => t.type === "Insight" && t.subType === "Locate");
|
2799
|
-
const lastAction = actionTasks[actionTasks.length - 1];
|
2800
|
-
const lastLocate = locateTasks[locateTasks.length - 1];
|
2801
|
-
if (!lastAction) {
|
2802
|
-
return {
|
2803
|
-
success: true,
|
2804
|
-
actionType: "unknown",
|
2805
|
-
description: `Completed: ${originalPrompt}`
|
2806
|
-
};
|
2807
|
-
}
|
2808
|
-
const actionType = lastAction.subType || "unknown";
|
2809
|
-
const elementInfo = this.extractElementInfo(lastLocate, lastAction);
|
2810
|
-
const description = this.generateActionDescription(actionType, lastAction.param, elementInfo);
|
2811
|
-
return {
|
2812
|
-
success: true,
|
2813
|
-
actionType,
|
2814
|
-
description,
|
2815
|
-
elementInfo,
|
2816
|
-
timing: lastAction.timing?.cost
|
2817
|
-
};
|
2818
|
-
}
|
2819
|
-
/**
|
2820
|
-
* Extract element information from locate task
|
2821
|
-
*/
|
2822
|
-
extractElementInfo(locateTask, _actionTask) {
|
2823
|
-
if (!locateTask?.output?.element)
|
2824
|
-
return void 0;
|
2825
|
-
const element = locateTask.output.element;
|
2826
|
-
return {
|
2827
|
-
type: element.attributes?.nodeType || "unknown",
|
2828
|
-
text: element.content || element.attributes?.placeholder || element.attributes?.title || "",
|
2829
|
-
location: `(${element.center[0]}, ${element.center[1]})`
|
2830
|
-
};
|
2831
|
-
}
|
2832
|
-
/**
|
2833
|
-
* Generate natural language description for actions
|
2834
|
-
*/
|
2835
|
-
generateActionDescription(actionType, param, elementInfo) {
|
2836
|
-
const elementDesc = elementInfo ? `'${elementInfo.text || elementInfo.type}' element` : "element";
|
2837
|
-
switch (actionType) {
|
2838
|
-
case "Tap":
|
2839
|
-
return `Clicked on ${elementDesc}`;
|
2840
|
-
case "Input":
|
2841
|
-
const inputValue = param?.value || "";
|
2842
|
-
return `Entered "${inputValue}" into ${elementDesc}`;
|
2843
|
-
case "KeyboardPress":
|
2844
|
-
return `Pressed ${param?.value || "key"}`;
|
2845
|
-
case "Scroll":
|
2846
|
-
return `Scrolled ${param?.direction || "on page"}`;
|
2847
|
-
case "Hover":
|
2848
|
-
return `Hovered over ${elementDesc}`;
|
2849
|
-
case "Drag":
|
2850
|
-
return `Dragged ${elementDesc}`;
|
2851
|
-
default:
|
2852
|
-
return `Performed ${actionType} action on ${elementDesc}`;
|
2853
|
-
}
|
2854
|
-
}
|
2855
2489
|
};
|
2856
2490
|
export {
|
2857
2491
|
PageAgent
|