@kkeve/suite 0.0.15 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/keve.js +2 -1
- package/dist/commands/clean.js +2 -2
- package/dist/commands/config.js +7 -7
- package/dist/commands/init.js +2 -2
- package/dist/commands/report.js +7 -7
- package/dist/commands/run.js +12 -11
- package/dist/commands/test.js +7 -7
- package/dist/config.js +7 -7
- package/dist/core/action-log-writer.js +4 -1
- package/dist/core/decorator/keve-decorators.d.ts +128 -0
- package/dist/core/decorator/keve-decorators.js +254 -0
- package/dist/core/decorator/keve-host.d.ts +40 -0
- package/dist/core/decorator/keve-host.js +33 -0
- package/dist/core/{keve-registry.d.ts → decorator/keve-registry.d.ts} +19 -0
- package/dist/core/{keve-registry.js → decorator/keve-registry.js} +13 -1
- package/dist/core/generateReport/report.d.ts +1 -1
- package/dist/core/generateReport/report.js +2 -2
- package/dist/core/generateReport/reportData.d.ts +3 -0
- package/dist/core/generateReport/reportData.js +49 -15
- package/dist/core/judge/confidence.d.ts +119 -0
- package/dist/core/judge/confidence.js +295 -0
- package/dist/core/judge/goal-heuristics.d.ts +32 -0
- package/dist/core/judge/goal-heuristics.js +128 -0
- package/dist/core/judge/pngCodec.d.ts +23 -0
- package/dist/core/judge/pngCodec.js +215 -0
- package/dist/core/judge/screenshotDiff.d.ts +112 -0
- package/dist/core/judge/screenshotDiff.js +420 -0
- package/dist/core/page-object/definePage.d.ts +95 -0
- package/dist/core/page-object/definePage.js +62 -0
- package/dist/core/page-object/elementResolver.d.ts +32 -0
- package/dist/core/page-object/elementResolver.js +236 -0
- package/dist/core/page-object/engine-types.d.ts +140 -0
- package/dist/core/page-object/engine-types.js +34 -0
- package/dist/core/sso-cookie.d.ts +18 -0
- package/dist/core/sso-cookie.js +10 -0
- package/dist/dsl/index.d.ts +15 -0
- package/dist/dsl/index.js +29 -0
- package/dist/dsl/locatorSource.d.ts +25 -0
- package/dist/dsl/locatorSource.js +323 -0
- package/dist/dsl/ops.d.ts +46 -0
- package/dist/dsl/ops.js +95 -0
- package/dist/dsl/toDecorator.d.ts +98 -0
- package/dist/dsl/toDecorator.js +747 -0
- package/dist/engine-cypress/adapter.d.ts +108 -0
- package/dist/engine-cypress/adapter.js +514 -0
- package/dist/engine-cypress/agent-page.d.ts +159 -0
- package/dist/engine-cypress/agent-page.js +566 -0
- package/dist/engine-cypress/agentDom.d.ts +22 -0
- package/dist/engine-cypress/agentDom.js +590 -0
- package/dist/engine-cypress/bridge.d.ts +39 -0
- package/dist/engine-cypress/bridge.js +598 -0
- package/dist/engine-cypress/cdp.d.ts +78 -0
- package/dist/engine-cypress/cdp.js +339 -0
- package/dist/engine-cypress/host.d.ts +22 -0
- package/dist/engine-cypress/host.js +480 -0
- package/dist/engine-cypress/index.d.ts +26 -0
- package/dist/engine-cypress/index.js +58 -0
- package/dist/engine-cypress/keveGoal.d.ts +43 -0
- package/dist/engine-cypress/keveGoal.js +265 -0
- package/dist/engine-cypress/pageOps.d.ts +41 -0
- package/dist/engine-cypress/pageOps.js +439 -0
- package/dist/engine-cypress/reporter.d.ts +20 -0
- package/dist/engine-cypress/reporter.js +128 -0
- package/dist/engine-cypress/setup.d.ts +21 -0
- package/dist/engine-cypress/setup.js +27 -0
- package/dist/engine-cypress/shotBuffer.d.ts +40 -0
- package/dist/engine-cypress/shotBuffer.js +41 -0
- package/dist/engine-cypress/state.d.ts +56 -0
- package/dist/engine-cypress/state.js +81 -0
- package/dist/engine-playwright/engineAdapter.d.ts +78 -0
- package/dist/engine-playwright/engineAdapter.js +371 -0
- package/dist/engine-playwright/global-setup.d.ts +28 -0
- package/dist/engine-playwright/global-setup.js +142 -0
- package/dist/{core → engine-playwright}/keve-report.d.ts +0 -1
- package/dist/engine-playwright/keve-report.js +137 -0
- package/dist/{core → engine-playwright}/keve-test.d.ts +10 -8
- package/dist/{core → engine-playwright}/keve-test.js +204 -200
- package/dist/{core → engine-playwright}/pw-config.js +8 -7
- package/dist/index.d.ts +13 -8
- package/dist/index.js +43 -28
- package/dist/page-agent/agent.d.ts +82 -7
- package/dist/page-agent/agent.js +252 -88
- package/dist/page-agent/diagnostics.d.ts +184 -0
- package/dist/page-agent/diagnostics.js +377 -0
- package/dist/page-agent/hooks.d.ts +3 -2
- package/dist/page-agent/hooks.js +4 -3
- package/dist/page-agent/page-like.d.ts +116 -0
- package/dist/page-agent/page-like.js +14 -0
- package/dist/page-agent/playwright-page.d.ts +20 -0
- package/dist/page-agent/playwright-page.js +98 -0
- package/dist/page-agent/system-prompt.d.ts +7 -0
- package/dist/page-agent/system-prompt.js +10 -0
- package/dist/page-agent/tools.d.ts +1 -1
- package/dist/page-agent/tools.js +144 -8
- package/package.json +24 -11
- package/scripts/gen-system-prompt.mjs +36 -0
- package/scripts/run-dsl-task.mjs +619 -0
- package/scripts/runtime/cypress-preprocessor.mjs +44 -0
- package/scripts/runtime/cypress.config.mjs +35 -0
- package/scripts/runtime/playwright.config.mjs +41 -0
- package/scripts/verify-shot-diff.mjs +243 -0
- package/dist/commands/auth-update.d.ts +0 -16
- package/dist/commands/auth-update.js +0 -88
- package/dist/core/auth.d.ts +0 -17
- package/dist/core/auth.js +0 -224
- package/dist/core/cdp-resolver.d.ts +0 -5
- package/dist/core/cdp-resolver.js +0 -93
- package/dist/core/global-setup.d.ts +0 -6
- package/dist/core/global-setup.js +0 -123
- package/dist/core/keve-decorators.d.ts +0 -51
- package/dist/core/keve-decorators.js +0 -95
- package/dist/core/keve-report.js +0 -376
- /package/dist/core/{keve-aspect.d.ts → decorator/keve-aspect.d.ts} +0 -0
- /package/dist/core/{keve-aspect.js → decorator/keve-aspect.js} +0 -0
- /package/dist/{core → engine-playwright}/pw-config.d.ts +0 -0
package/dist/page-agent/agent.js
CHANGED
|
@@ -16,51 +16,16 @@
|
|
|
16
16
|
* const agent = new KevePageAgent(page, { maxSteps: 5 });
|
|
17
17
|
* const result = await agent.execute(step, expected, { learnedActionsHint, fnResult });
|
|
18
18
|
*/
|
|
19
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
20
|
-
if (k2 === undefined) k2 = k;
|
|
21
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
22
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
23
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
24
|
-
}
|
|
25
|
-
Object.defineProperty(o, k2, desc);
|
|
26
|
-
}) : (function(o, m, k, k2) {
|
|
27
|
-
if (k2 === undefined) k2 = k;
|
|
28
|
-
o[k2] = m[k];
|
|
29
|
-
}));
|
|
30
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
31
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
32
|
-
}) : function(o, v) {
|
|
33
|
-
o["default"] = v;
|
|
34
|
-
});
|
|
35
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
36
|
-
var ownKeys = function(o) {
|
|
37
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
38
|
-
var ar = [];
|
|
39
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
40
|
-
return ar;
|
|
41
|
-
};
|
|
42
|
-
return ownKeys(o);
|
|
43
|
-
};
|
|
44
|
-
return function (mod) {
|
|
45
|
-
if (mod && mod.__esModule) return mod;
|
|
46
|
-
var result = {};
|
|
47
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
48
|
-
__setModuleDefault(result, mod);
|
|
49
|
-
return result;
|
|
50
|
-
};
|
|
51
|
-
})();
|
|
52
19
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
53
20
|
exports.KevePageAgent = void 0;
|
|
54
21
|
exports.reactLoop = reactLoop;
|
|
55
22
|
const llm_1 = require("@kkeve/core/llm");
|
|
56
23
|
const zod_1 = require("zod");
|
|
57
|
-
const
|
|
58
|
-
const
|
|
59
|
-
const
|
|
60
|
-
const
|
|
61
|
-
const
|
|
62
|
-
// ─── System Prompt ────────────────────────────────────────────────────
|
|
63
|
-
const SYSTEM_PROMPT = fs.readFileSync(path.join(__dirname, "system_prompt.md"), "utf-8");
|
|
24
|
+
const hooks_js_1 = require("./hooks.js");
|
|
25
|
+
const tools_js_1 = require("./tools.js");
|
|
26
|
+
const system_prompt_js_1 = require("./system-prompt.js");
|
|
27
|
+
const playwright_page_js_1 = require("./playwright-page.js");
|
|
28
|
+
const diagnostics_js_1 = require("./diagnostics.js");
|
|
64
29
|
// ─── Snapshot diff helpers (used by assembleUserPrompt to add Page Change to history) ───
|
|
65
30
|
/** Find the snapshot string of the previous step event before index `i`. */
|
|
66
31
|
function findPrevStepSnapshot(events, currentIdx) {
|
|
@@ -154,6 +119,11 @@ class KevePageAgent {
|
|
|
154
119
|
events = [];
|
|
155
120
|
abortController = new AbortController();
|
|
156
121
|
systemPrompt;
|
|
122
|
+
/** 目标应用 URL(原 process.env.KEVE_TARGET_URL) */
|
|
123
|
+
targetUrl;
|
|
124
|
+
getEnvFn;
|
|
125
|
+
saveScreenshotFn;
|
|
126
|
+
loadInitialImagesFn;
|
|
157
127
|
/** Current execute options (set at start of execute(), accessible from hooks) */
|
|
158
128
|
options;
|
|
159
129
|
hooks;
|
|
@@ -163,21 +133,46 @@ class KevePageAgent {
|
|
|
163
133
|
_pendingFileChooser = null;
|
|
164
134
|
_consoleMessages = [];
|
|
165
135
|
_networkRequests = [];
|
|
136
|
+
_browserDiagnostics = [];
|
|
137
|
+
_networkDiagnostics = [];
|
|
138
|
+
/** 本次 execute 内 LLM 推理累计耗时(ms) */
|
|
139
|
+
_aiReasoningMs = 0;
|
|
140
|
+
/** 本次 execute 内 LLM token 累计用量 */
|
|
141
|
+
_aiUsage = { llmCalls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
142
|
+
/** 本次 execute 内工具操作累计耗时(ms) */
|
|
143
|
+
_actionMs = 0;
|
|
144
|
+
/** 本次 execute 起始时间,用于总耗时诊断 */
|
|
145
|
+
_executeStartedAt = 0;
|
|
166
146
|
_pageListenersSetup = false;
|
|
147
|
+
/** 页面级性能指标(execute 结束时一次性读取;读取失败则留空) */
|
|
148
|
+
_lastPageMetrics;
|
|
167
149
|
constructor(page, options) {
|
|
168
150
|
this.page = page;
|
|
169
151
|
this.maxSteps = options?.maxSteps ?? 8;
|
|
170
|
-
this.systemPrompt = options?.customSystemPrompt || SYSTEM_PROMPT;
|
|
152
|
+
this.systemPrompt = options?.customSystemPrompt || system_prompt_js_1.SYSTEM_PROMPT;
|
|
171
153
|
this.hooks = options?.hooks;
|
|
172
|
-
|
|
154
|
+
this.getEnvFn = options?.getEnv || (() => '');
|
|
155
|
+
this.targetUrl = options?.targetUrl || this.getEnvFn('KEVE_TARGET_URL') || '';
|
|
156
|
+
this.saveScreenshotFn = options?.saveScreenshot;
|
|
157
|
+
this.loadInitialImagesFn = options?.loadInitialImages;
|
|
158
|
+
const llmConfig = options?.llm || {
|
|
159
|
+
baseURL: this.getEnvFn('KEVE_LLM_BASE_URL'),
|
|
160
|
+
model: this.getEnvFn('KEVE_LLM_MODEL_NAME'),
|
|
161
|
+
apiKey: this.getEnvFn('KEVE_LLM_API_KEY'),
|
|
162
|
+
};
|
|
173
163
|
this.llm = new llm_1.LLM({
|
|
174
|
-
baseURL:
|
|
175
|
-
model:
|
|
176
|
-
apiKey:
|
|
164
|
+
baseURL: llmConfig.baseURL,
|
|
165
|
+
model: llmConfig.model,
|
|
166
|
+
apiKey: llmConfig.apiKey || '',
|
|
177
167
|
temperature: 0.1,
|
|
178
168
|
maxRetries: 3,
|
|
169
|
+
customFetch: llmConfig.customFetch,
|
|
179
170
|
});
|
|
180
171
|
}
|
|
172
|
+
/** 读取宿主注入的环境变量(Cypress 侧由 cy.env 转发,Playwright 侧即 process.env) */
|
|
173
|
+
getEnv(name) {
|
|
174
|
+
return this.getEnvFn(name) || '';
|
|
175
|
+
}
|
|
181
176
|
/** Stop the current execution */
|
|
182
177
|
stop() {
|
|
183
178
|
this.abortController.abort();
|
|
@@ -221,23 +216,33 @@ class KevePageAgent {
|
|
|
221
216
|
});
|
|
222
217
|
// ── Console messages ──
|
|
223
218
|
this.page.on('console', (msg) => {
|
|
224
|
-
|
|
219
|
+
const level = msg.type();
|
|
220
|
+
const text = msg.text();
|
|
221
|
+
this._consoleMessages.push({ level, text });
|
|
225
222
|
if (this._consoleMessages.length > 200)
|
|
226
223
|
this._consoleMessages.shift();
|
|
224
|
+
this._browserDiagnostics.push({ level, text, at: Date.now(), kind: 'console' });
|
|
225
|
+
if (this._browserDiagnostics.length > 500)
|
|
226
|
+
this._browserDiagnostics.shift();
|
|
227
227
|
});
|
|
228
228
|
this.page.on('pageerror', (err) => {
|
|
229
|
-
|
|
229
|
+
const text = `Uncaught: ${err?.message || String(err)}`;
|
|
230
|
+
this._consoleMessages.push({ level: 'error', text });
|
|
231
|
+
this._browserDiagnostics.push({ level: 'error', text, at: Date.now(), kind: 'pageerror' });
|
|
232
|
+
if (this._browserDiagnostics.length > 500)
|
|
233
|
+
this._browserDiagnostics.shift();
|
|
230
234
|
});
|
|
231
235
|
// ── Network requests ──
|
|
232
236
|
this.page.on('request', (req) => {
|
|
233
237
|
const url = req.url();
|
|
234
|
-
const isStatic =
|
|
238
|
+
const isStatic = (0, diagnostics_js_1.isStaticUrl)(url);
|
|
235
239
|
this._networkRequests.push({
|
|
236
240
|
_req: req,
|
|
237
241
|
method: req.method(),
|
|
238
242
|
url,
|
|
239
243
|
isStatic,
|
|
240
244
|
status: null,
|
|
245
|
+
startedAt: Date.now(),
|
|
241
246
|
});
|
|
242
247
|
if (this._networkRequests.length > 100)
|
|
243
248
|
this._networkRequests.shift();
|
|
@@ -247,11 +252,56 @@ class KevePageAgent {
|
|
|
247
252
|
// Match by reference identity — Playwright guarantees same Request object
|
|
248
253
|
for (let i = this._networkRequests.length - 1; i >= 0; i--) {
|
|
249
254
|
if (this._networkRequests[i]._req === req) {
|
|
250
|
-
this._networkRequests[i]
|
|
255
|
+
const item = this._networkRequests[i];
|
|
256
|
+
const status = res.status();
|
|
257
|
+
const durationMs = item.startedAt ? Date.now() - item.startedAt : undefined;
|
|
258
|
+
item.status = status;
|
|
259
|
+
item.durationMs = durationMs;
|
|
260
|
+
this._networkDiagnostics.push({
|
|
261
|
+
url: item.url,
|
|
262
|
+
method: item.method,
|
|
263
|
+
status,
|
|
264
|
+
durationMs,
|
|
265
|
+
isStatic: item.isStatic,
|
|
266
|
+
at: item.startedAt,
|
|
267
|
+
});
|
|
268
|
+
if (this._networkDiagnostics.length > 500)
|
|
269
|
+
this._networkDiagnostics.shift();
|
|
251
270
|
break;
|
|
252
271
|
}
|
|
253
272
|
}
|
|
254
273
|
});
|
|
274
|
+
// 请求失败(DNS/超时/连接重置等)没有 response 事件,必须在 requestfailed 记录。
|
|
275
|
+
const onRequestFailed = (req) => {
|
|
276
|
+
const failure = req.failure?.();
|
|
277
|
+
const error = failure?.errorText || 'request failed';
|
|
278
|
+
let item;
|
|
279
|
+
for (let i = this._networkRequests.length - 1; i >= 0; i--) {
|
|
280
|
+
if (this._networkRequests[i]._req === req) {
|
|
281
|
+
item = this._networkRequests[i];
|
|
282
|
+
break;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
const url = item?.url || req.url();
|
|
286
|
+
const durationMs = item?.startedAt ? Date.now() - item.startedAt : undefined;
|
|
287
|
+
this._networkDiagnostics.push({
|
|
288
|
+
url,
|
|
289
|
+
method: item?.method || req.method(),
|
|
290
|
+
status: 0,
|
|
291
|
+
durationMs,
|
|
292
|
+
error: (0, diagnostics_js_1.clipDiagnosticText)(error),
|
|
293
|
+
isStatic: item?.isStatic ?? (0, diagnostics_js_1.isStaticUrl)(url),
|
|
294
|
+
at: item?.startedAt,
|
|
295
|
+
});
|
|
296
|
+
if (this._networkDiagnostics.length > 500)
|
|
297
|
+
this._networkDiagnostics.shift();
|
|
298
|
+
};
|
|
299
|
+
// Playwright 的 AgentPage 契约只保证 on/emit,requestfailed 是原生事件。
|
|
300
|
+
// 用可选探测,避免 Cypress 占位实现或其他 AgentPage 实现因未知事件报错。
|
|
301
|
+
try {
|
|
302
|
+
this.page.on('requestfailed', onRequestFailed);
|
|
303
|
+
}
|
|
304
|
+
catch { /* 非 Playwright 实现可忽略 */ }
|
|
255
305
|
// ── File chooser ──
|
|
256
306
|
this.page.on('filechooser', (fileChooser) => {
|
|
257
307
|
this._pendingFileChooser = fileChooser;
|
|
@@ -269,6 +319,13 @@ class KevePageAgent {
|
|
|
269
319
|
this._pendingFileChooser = null;
|
|
270
320
|
this._consoleMessages = [];
|
|
271
321
|
this._networkRequests = [];
|
|
322
|
+
this._browserDiagnostics = [];
|
|
323
|
+
this._networkDiagnostics = [];
|
|
324
|
+
this._aiReasoningMs = 0;
|
|
325
|
+
this._aiUsage = { llmCalls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
326
|
+
this._actionMs = 0;
|
|
327
|
+
this._executeStartedAt = Date.now();
|
|
328
|
+
this._lastPageMetrics = undefined;
|
|
272
329
|
// Set up page event listeners (idempotent — only runs once per page)
|
|
273
330
|
this.setupPageListeners();
|
|
274
331
|
const hooks = options?.hooks ?? this.hooks;
|
|
@@ -285,7 +342,7 @@ class KevePageAgent {
|
|
|
285
342
|
const maxSteps = options?.maxSteps ?? this.maxSteps;
|
|
286
343
|
const systemPrompt = options?.customSystemPrompt || this.systemPrompt;
|
|
287
344
|
// Inject context/learned hints as initial observations
|
|
288
|
-
const targetUrl =
|
|
345
|
+
const targetUrl = this.targetUrl;
|
|
289
346
|
if (targetUrl) {
|
|
290
347
|
this.pushObservation(`Target application URL: ${targetUrl}. Use this URL for navigation. Do NOT construct URLs yourself — use the navigate tool with this URL or read process.env.KEVE_TARGET_URL.`);
|
|
291
348
|
}
|
|
@@ -313,13 +370,13 @@ class KevePageAgent {
|
|
|
313
370
|
// ── Observe: get browser state (ariaSnapshot only — no per-step screenshot) ──
|
|
314
371
|
console.log('\x1b[34m\x1b[1m👀 Observing...\x1b[0m');
|
|
315
372
|
const snapshot = await this.page.ariaSnapshot({ mode: 'ai' });
|
|
316
|
-
const url = this.page.url();
|
|
373
|
+
const url = await this.page.url();
|
|
317
374
|
console.log(` url: ${url.slice(0, 120)} | step ${stepCount + 1}/${maxSteps}`);
|
|
318
375
|
// ── Assemble messages (text-only per step; initial screenshots on step 0) ──
|
|
319
376
|
const userText = this.assembleUserPrompt(step, expected, snapshot, url, stepCount, maxSteps);
|
|
320
377
|
let userContent;
|
|
321
378
|
if (stepCount === 0) {
|
|
322
|
-
const initialImages = this.loadInitialScreenshots(options);
|
|
379
|
+
const initialImages = await this.loadInitialScreenshots(options);
|
|
323
380
|
userContent = initialImages.length > 0
|
|
324
381
|
? [{ type: 'text', text: userText }, ...initialImages]
|
|
325
382
|
: userText;
|
|
@@ -334,7 +391,15 @@ class KevePageAgent {
|
|
|
334
391
|
// ── Think + Act: LLM decides ──
|
|
335
392
|
console.log('\x1b[34m\x1b[1m🧠 Thinking...\x1b[0m');
|
|
336
393
|
const macroTool = this.buildMacroTool();
|
|
337
|
-
const
|
|
394
|
+
const llmStartedAt = Date.now();
|
|
395
|
+
let llmResult;
|
|
396
|
+
try {
|
|
397
|
+
llmResult = await this.llm.invoke(messages, macroTool, this.abortController.signal, { toolChoiceName: 'AgentOutput' });
|
|
398
|
+
this.recordAiUsage(llmResult.usage);
|
|
399
|
+
}
|
|
400
|
+
finally {
|
|
401
|
+
this._aiReasoningMs += Date.now() - llmStartedAt;
|
|
402
|
+
}
|
|
338
403
|
const macroInput = llmResult.toolCall.args;
|
|
339
404
|
const execResult = llmResult.toolResult;
|
|
340
405
|
const reflection = {
|
|
@@ -373,7 +438,10 @@ class KevePageAgent {
|
|
|
373
438
|
toolError: execResult.error,
|
|
374
439
|
snapshot,
|
|
375
440
|
screenshotPath: undefined,
|
|
441
|
+
duration: execResult.duration,
|
|
376
442
|
};
|
|
443
|
+
if (execResult.duration !== undefined)
|
|
444
|
+
this._actionMs += execResult.duration;
|
|
377
445
|
this.events.push(stepEvent);
|
|
378
446
|
// If execution had error, add observation
|
|
379
447
|
if (execResult.error) {
|
|
@@ -397,8 +465,8 @@ class KevePageAgent {
|
|
|
397
465
|
if (actionName === 'done') {
|
|
398
466
|
// Capture done-time screenshot as goal-after evidence
|
|
399
467
|
try {
|
|
400
|
-
const
|
|
401
|
-
stepEvent.screenshotPath = this.saveScreenshotBuffer(
|
|
468
|
+
const doneShot = await this.page.screenshot({ type: 'png', timeout: 15000 });
|
|
469
|
+
stepEvent.screenshotPath = await this.saveScreenshotBuffer(doneShot.base64, stepCount, step);
|
|
402
470
|
}
|
|
403
471
|
catch { /* non-critical */ }
|
|
404
472
|
// Extract conclusion: 4-level backoff
|
|
@@ -531,6 +599,12 @@ class KevePageAgent {
|
|
|
531
599
|
}
|
|
532
600
|
}
|
|
533
601
|
finally {
|
|
602
|
+
// 页面级性能只能在用例收敛后读一次(LCP/longtask 需要缓冲已完成)
|
|
603
|
+
await this.capturePageMetrics();
|
|
604
|
+
if (taskResult) {
|
|
605
|
+
// 用完整诊断(含页面性能)覆盖此前构建的结果
|
|
606
|
+
taskResult.diagnostics = this.buildDiagnostics();
|
|
607
|
+
}
|
|
534
608
|
// ── onAfterTask hook — may return partial overrides ──
|
|
535
609
|
const hookResult = await hooks?.onAfterTask?.(this, taskResult);
|
|
536
610
|
if (hookResult && taskResult) {
|
|
@@ -543,6 +617,75 @@ class KevePageAgent {
|
|
|
543
617
|
pushObservation(content) {
|
|
544
618
|
this.events.push({ type: 'observation', content });
|
|
545
619
|
}
|
|
620
|
+
/**
|
|
621
|
+
* 读取页面级性能指标。
|
|
622
|
+
*
|
|
623
|
+
* Playwright 与 Cypress 都走同一段探针源码:Playwright 在真实页面上下文,
|
|
624
|
+
* Cypress 在 AUT 的 isolated world(navigation timing / performance entries
|
|
625
|
+
* 与主世界共享同一份 Performance 数据)。
|
|
626
|
+
*/
|
|
627
|
+
async capturePageMetrics() {
|
|
628
|
+
try {
|
|
629
|
+
const raw = await this.page.evaluate((0, diagnostics_js_1.pagePerformanceProbeSource)());
|
|
630
|
+
if (raw && typeof raw === 'object') {
|
|
631
|
+
this._lastPageMetrics = raw;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
catch {
|
|
635
|
+
// 性能指标是增强信息,读取失败不影响用例结论
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
/**
|
|
639
|
+
* 汇总当前执行期的运行诊断(含页面性能与质量信号)。
|
|
640
|
+
* 公开给 reactLoop 的异常兜底分支复用,避免中断时丢失已采集数据。
|
|
641
|
+
*/
|
|
642
|
+
buildDiagnostics() {
|
|
643
|
+
const stepEvents = this.events.filter((e) => e.type === 'step');
|
|
644
|
+
const actionRecords = stepEvents.map(e => ({
|
|
645
|
+
name: e.toolName || 'unknown',
|
|
646
|
+
durationMs: e.duration,
|
|
647
|
+
error: !!e.toolError,
|
|
648
|
+
}));
|
|
649
|
+
// 重复动作:连续两个以上步骤产生相同工具输出时,累计重复次数。
|
|
650
|
+
let repeatedActionCount = 0;
|
|
651
|
+
let repeatRun = 0;
|
|
652
|
+
let prevOutput = '';
|
|
653
|
+
for (const event of stepEvents) {
|
|
654
|
+
const output = String(event.toolOutput || '').slice(0, 200);
|
|
655
|
+
if (output && output === prevOutput)
|
|
656
|
+
repeatRun += 1;
|
|
657
|
+
else
|
|
658
|
+
repeatRun = 0;
|
|
659
|
+
repeatedActionCount += repeatRun;
|
|
660
|
+
prevOutput = output;
|
|
661
|
+
}
|
|
662
|
+
return (0, diagnostics_js_1.buildRuntimeDiagnostics)({
|
|
663
|
+
totalMs: this._executeStartedAt ? Date.now() - this._executeStartedAt : 0,
|
|
664
|
+
aiReasoningMs: this._aiReasoningMs,
|
|
665
|
+
actionMs: this._actionMs,
|
|
666
|
+
actions: actionRecords,
|
|
667
|
+
consoleRecords: this._browserDiagnostics,
|
|
668
|
+
networkRecords: this._networkDiagnostics,
|
|
669
|
+
repeatedActionCount,
|
|
670
|
+
pageMetrics: this._lastPageMetrics,
|
|
671
|
+
aiUsage: this._aiUsage,
|
|
672
|
+
});
|
|
673
|
+
}
|
|
674
|
+
/** 累加单次 LLM 调用的 token 用量(重试中的失败请求不在返回结果里,无法计量) */
|
|
675
|
+
recordAiUsage(usage) {
|
|
676
|
+
if (!usage)
|
|
677
|
+
return;
|
|
678
|
+
this._aiUsage.llmCalls = (this._aiUsage.llmCalls || 0) + 1;
|
|
679
|
+
this._aiUsage.promptTokens = (this._aiUsage.promptTokens || 0) + (usage.promptTokens || 0);
|
|
680
|
+
this._aiUsage.completionTokens = (this._aiUsage.completionTokens || 0) + (usage.completionTokens || 0);
|
|
681
|
+
this._aiUsage.totalTokens = (this._aiUsage.totalTokens || 0) + (usage.totalTokens || 0);
|
|
682
|
+
if (usage.cachedTokens !== undefined) {
|
|
683
|
+
this._aiUsage.cachedTokens = (this._aiUsage.cachedTokens || 0) + usage.cachedTokens;
|
|
684
|
+
}
|
|
685
|
+
if (usage.reasoningTokens !== undefined) {
|
|
686
|
+
this._aiUsage.reasoningTokens = (this._aiUsage.reasoningTokens || 0) + usage.reasoningTokens;
|
|
687
|
+
}
|
|
688
|
+
}
|
|
546
689
|
assembleUserPrompt(step, expected, snapshot, url, stepIndex, maxSteps) {
|
|
547
690
|
let prompt = '';
|
|
548
691
|
// <agent_state>
|
|
@@ -612,7 +755,7 @@ class KevePageAgent {
|
|
|
612
755
|
return prompt;
|
|
613
756
|
}
|
|
614
757
|
buildMacroTool() {
|
|
615
|
-
const schema = (0,
|
|
758
|
+
const schema = (0, tools_js_1.packMacroToolSchema)();
|
|
616
759
|
const signal = this.abortController.signal;
|
|
617
760
|
return {
|
|
618
761
|
AgentOutput: {
|
|
@@ -621,12 +764,12 @@ class KevePageAgent {
|
|
|
621
764
|
execute: async (input) => {
|
|
622
765
|
signal.throwIfAborted();
|
|
623
766
|
const toolName = String(input.tool || 'done');
|
|
624
|
-
const toolDef =
|
|
767
|
+
const toolDef = tools_js_1.tools.get(toolName);
|
|
625
768
|
if (!toolDef) {
|
|
626
769
|
return { toolName, output: '', error: `Unknown tool: ${toolName}` };
|
|
627
770
|
}
|
|
628
771
|
// Extract only the fields this tool needs from the flat input
|
|
629
|
-
const shape = (0,
|
|
772
|
+
const shape = (0, tools_js_1.getZodShape)(toolDef.inputSchema);
|
|
630
773
|
const toolInput = {};
|
|
631
774
|
for (const key of Object.keys(shape)) {
|
|
632
775
|
if (input[key] !== undefined) {
|
|
@@ -649,35 +792,34 @@ class KevePageAgent {
|
|
|
649
792
|
},
|
|
650
793
|
};
|
|
651
794
|
}
|
|
652
|
-
/**
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
795
|
+
/**
|
|
796
|
+
* Save screenshot base64 to test artifacts.
|
|
797
|
+
*
|
|
798
|
+
* 落盘能力由宿主注入:Playwright 侧写本地文件系统,Cypress 侧经 HTTP 桥落盘。
|
|
799
|
+
*/
|
|
800
|
+
async saveScreenshotBuffer(pngBase64, stepIndex, stepName) {
|
|
801
|
+
if (!this.saveScreenshotFn)
|
|
802
|
+
return undefined;
|
|
803
|
+
try {
|
|
804
|
+
return await this.saveScreenshotFn(pngBase64, stepIndex, stepName);
|
|
805
|
+
}
|
|
806
|
+
catch {
|
|
807
|
+
return undefined;
|
|
808
|
+
}
|
|
662
809
|
}
|
|
663
810
|
/** Load initial screenshots (goal-before + fn-after) as ContentItem[] for step 0 multimodal input */
|
|
664
|
-
loadInitialScreenshots(options) {
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
image_url: { url: `data:image/png;base64,${buf.toString('base64')}`, detail: 'high' },
|
|
676
|
-
});
|
|
677
|
-
}
|
|
678
|
-
catch { /* non-critical */ }
|
|
811
|
+
async loadInitialScreenshots(options) {
|
|
812
|
+
if (!this.loadInitialImagesFn)
|
|
813
|
+
return [];
|
|
814
|
+
try {
|
|
815
|
+
return (await this.loadInitialImagesFn({
|
|
816
|
+
goalScreenshotBefore: options?.goalScreenshotBefore,
|
|
817
|
+
fnAfterScreenshot: options?.fnAfterScreenshot,
|
|
818
|
+
}));
|
|
819
|
+
}
|
|
820
|
+
catch {
|
|
821
|
+
return [];
|
|
679
822
|
}
|
|
680
|
-
return items;
|
|
681
823
|
}
|
|
682
824
|
/**
|
|
683
825
|
* Visual element location: uses LLM multimodal to identify element bbox in screenshot.
|
|
@@ -717,6 +859,7 @@ IMPORTANT: Respond in Chinese.`,
|
|
|
717
859
|
execute: async (args) => args,
|
|
718
860
|
},
|
|
719
861
|
}, this.abortController.signal, { toolChoiceName: 'LocateResult' });
|
|
862
|
+
this.recordAiUsage(result.usage);
|
|
720
863
|
const parsed = result.toolCall.args;
|
|
721
864
|
const bbox = parsed.bbox || [];
|
|
722
865
|
if (bbox.length === 4 && bbox.every(v => typeof v === 'number' && !isNaN(v) && v >= 0 && v <= 1000) && bbox[2] > bbox[0] && bbox[3] > bbox[1]) {
|
|
@@ -764,6 +907,7 @@ IMPORTANT:
|
|
|
764
907
|
execute: async (args) => args,
|
|
765
908
|
},
|
|
766
909
|
}, this.abortController.signal, { toolChoiceName: 'AssertResult' });
|
|
910
|
+
this.recordAiUsage(result.usage);
|
|
767
911
|
const parsed = result.toolCall.args;
|
|
768
912
|
return {
|
|
769
913
|
passed: typeof parsed.passed === 'boolean' ? parsed.passed : false,
|
|
@@ -785,7 +929,16 @@ IMPORTANT:
|
|
|
785
929
|
.filter((e) => e.type === 'step')
|
|
786
930
|
.filter(e => e.screenshotPath)
|
|
787
931
|
.map(e => e.screenshotPath);
|
|
788
|
-
|
|
932
|
+
const diagnostics = this.buildDiagnostics();
|
|
933
|
+
return {
|
|
934
|
+
success,
|
|
935
|
+
data,
|
|
936
|
+
events: this.events,
|
|
937
|
+
finalSnapshot: snapshot,
|
|
938
|
+
diagnostics,
|
|
939
|
+
/* refinePatch, */
|
|
940
|
+
agentScreenshots: screenshotPaths.length ? screenshotPaths : undefined,
|
|
941
|
+
};
|
|
789
942
|
}
|
|
790
943
|
}
|
|
791
944
|
exports.KevePageAgent = KevePageAgent;
|
|
@@ -794,16 +947,25 @@ exports.KevePageAgent = KevePageAgent;
|
|
|
794
947
|
* Drop-in replacement for the old reactLoop function.
|
|
795
948
|
* Creates a KevePageAgent, runs it, and maps the result.
|
|
796
949
|
*/
|
|
797
|
-
async function reactLoop(
|
|
798
|
-
|
|
950
|
+
async function reactLoop(
|
|
951
|
+
// 接受原生 Playwright Page 或任意 AgentPage 实现(用 toAgentPage 的参数类型
|
|
952
|
+
// 表达,避免在浏览器安全模块里引入 @playwright/test 的类型依赖)
|
|
953
|
+
page, step, expected, options = {}) {
|
|
954
|
+
const agent = new KevePageAgent((0, playwright_page_js_1.toAgentPage)(page), {
|
|
799
955
|
maxSteps: options.maxSteps ?? 20,
|
|
956
|
+
systemPrompt: options.systemPrompt,
|
|
957
|
+
targetUrl: options.targetUrl,
|
|
958
|
+
getEnv: options.getEnv,
|
|
959
|
+
saveScreenshot: options.saveScreenshot,
|
|
960
|
+
loadInitialImages: options.loadInitialImages,
|
|
961
|
+
llm: options.llm,
|
|
800
962
|
hooks: {
|
|
801
963
|
onAfterTask: async (_agent, result) => {
|
|
802
964
|
const partial = {};
|
|
803
965
|
// 1. Diagnostic hints for failures
|
|
804
966
|
if (!result.success) {
|
|
805
967
|
try {
|
|
806
|
-
partial.diagnosticHints = await (0,
|
|
968
|
+
partial.diagnosticHints = await (0, hooks_js_1.extractDiagnosticHints)(_agent.page, result);
|
|
807
969
|
}
|
|
808
970
|
catch { /* non-critical */ }
|
|
809
971
|
}
|
|
@@ -855,6 +1017,7 @@ async function reactLoop(page, step, expected, options = {}) {
|
|
|
855
1017
|
stepCount: partialEvents.length,
|
|
856
1018
|
finalSnapshot: '',
|
|
857
1019
|
events: agent.events,
|
|
1020
|
+
diagnostics: agent.buildDiagnostics(),
|
|
858
1021
|
agentScreenshots: agent.events
|
|
859
1022
|
.filter((e) => e.type === 'step')
|
|
860
1023
|
.filter(e => e.screenshotPath)
|
|
@@ -881,6 +1044,7 @@ async function reactLoop(page, step, expected, options = {}) {
|
|
|
881
1044
|
expectedMet: result.success,
|
|
882
1045
|
conclusion: result.conclusion,
|
|
883
1046
|
finalSnapshot: result.finalSnapshot,
|
|
1047
|
+
diagnostics: result.diagnostics,
|
|
884
1048
|
// refinePatch: result.refinePatch, // 已注释:scriptRefine 已禁用
|
|
885
1049
|
agentScreenshots: result.agentScreenshots,
|
|
886
1050
|
};
|