@kkeve/suite 0.0.13 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/keve.js +2 -1
- package/dist/commands/clean.js +2 -2
- package/dist/commands/config.js +7 -7
- package/dist/commands/init.js +2 -2
- package/dist/commands/report.js +7 -7
- package/dist/commands/run.js +16 -12
- package/dist/commands/test.js +7 -7
- package/dist/config.js +7 -7
- package/dist/core/action-log-writer.js +4 -1
- package/dist/core/decorator/keve-decorators.d.ts +128 -0
- package/dist/core/decorator/keve-decorators.js +254 -0
- package/dist/core/decorator/keve-host.d.ts +40 -0
- package/dist/core/decorator/keve-host.js +33 -0
- package/dist/core/{keve-registry.d.ts → decorator/keve-registry.d.ts} +19 -0
- package/dist/core/{keve-registry.js → decorator/keve-registry.js} +13 -1
- package/dist/core/generateReport/report.d.ts +1 -1
- package/dist/core/generateReport/report.js +2 -2
- package/dist/core/generateReport/reportData.d.ts +3 -0
- package/dist/core/generateReport/reportData.js +55 -18
- package/dist/core/judge/confidence.d.ts +119 -0
- package/dist/core/judge/confidence.js +295 -0
- package/dist/core/judge/goal-heuristics.d.ts +32 -0
- package/dist/core/judge/goal-heuristics.js +128 -0
- package/dist/core/judge/pngCodec.d.ts +23 -0
- package/dist/core/judge/pngCodec.js +215 -0
- package/dist/core/judge/screenshotDiff.d.ts +112 -0
- package/dist/core/judge/screenshotDiff.js +420 -0
- package/dist/core/page-object/definePage.d.ts +95 -0
- package/dist/core/page-object/definePage.js +62 -0
- package/dist/core/page-object/elementResolver.d.ts +32 -0
- package/dist/core/page-object/elementResolver.js +236 -0
- package/dist/core/page-object/engine-types.d.ts +140 -0
- package/dist/core/page-object/engine-types.js +34 -0
- package/dist/core/sso-cookie.d.ts +18 -0
- package/dist/core/sso-cookie.js +10 -0
- package/dist/dsl/index.d.ts +15 -0
- package/dist/dsl/index.js +29 -0
- package/dist/dsl/locatorSource.d.ts +25 -0
- package/dist/dsl/locatorSource.js +323 -0
- package/dist/dsl/ops.d.ts +46 -0
- package/dist/dsl/ops.js +95 -0
- package/dist/dsl/toDecorator.d.ts +98 -0
- package/dist/dsl/toDecorator.js +747 -0
- package/dist/engine-cypress/adapter.d.ts +108 -0
- package/dist/engine-cypress/adapter.js +514 -0
- package/dist/engine-cypress/agent-page.d.ts +159 -0
- package/dist/engine-cypress/agent-page.js +566 -0
- package/dist/engine-cypress/agentDom.d.ts +22 -0
- package/dist/engine-cypress/agentDom.js +590 -0
- package/dist/engine-cypress/bridge.d.ts +39 -0
- package/dist/engine-cypress/bridge.js +598 -0
- package/dist/engine-cypress/cdp.d.ts +78 -0
- package/dist/engine-cypress/cdp.js +339 -0
- package/dist/engine-cypress/host.d.ts +22 -0
- package/dist/engine-cypress/host.js +480 -0
- package/dist/engine-cypress/index.d.ts +26 -0
- package/dist/engine-cypress/index.js +58 -0
- package/dist/engine-cypress/keveGoal.d.ts +43 -0
- package/dist/engine-cypress/keveGoal.js +265 -0
- package/dist/engine-cypress/pageOps.d.ts +41 -0
- package/dist/engine-cypress/pageOps.js +439 -0
- package/dist/engine-cypress/reporter.d.ts +20 -0
- package/dist/engine-cypress/reporter.js +128 -0
- package/dist/engine-cypress/setup.d.ts +21 -0
- package/dist/engine-cypress/setup.js +27 -0
- package/dist/engine-cypress/shotBuffer.d.ts +40 -0
- package/dist/engine-cypress/shotBuffer.js +41 -0
- package/dist/engine-cypress/state.d.ts +56 -0
- package/dist/engine-cypress/state.js +81 -0
- package/dist/engine-playwright/engineAdapter.d.ts +78 -0
- package/dist/engine-playwright/engineAdapter.js +371 -0
- package/dist/engine-playwright/global-setup.d.ts +28 -0
- package/dist/engine-playwright/global-setup.js +142 -0
- package/dist/{core → engine-playwright}/keve-report.d.ts +0 -1
- package/dist/engine-playwright/keve-report.js +137 -0
- package/dist/{core → engine-playwright}/keve-test.d.ts +10 -8
- package/dist/{core → engine-playwright}/keve-test.js +257 -207
- package/dist/{core → engine-playwright}/pw-config.js +8 -7
- package/dist/index.d.ts +13 -8
- package/dist/index.js +43 -28
- package/dist/page-agent/agent.d.ts +98 -7
- package/dist/page-agent/agent.js +338 -84
- package/dist/page-agent/diagnostics.d.ts +184 -0
- package/dist/page-agent/diagnostics.js +377 -0
- package/dist/page-agent/hooks.d.ts +3 -2
- package/dist/page-agent/hooks.js +4 -3
- package/dist/page-agent/page-like.d.ts +116 -0
- package/dist/page-agent/page-like.js +14 -0
- package/dist/page-agent/playwright-page.d.ts +20 -0
- package/dist/page-agent/playwright-page.js +98 -0
- package/dist/page-agent/system-prompt.d.ts +7 -0
- package/dist/page-agent/system-prompt.js +10 -0
- package/dist/page-agent/system_prompt.md +1 -1
- package/dist/page-agent/tools.d.ts +1 -1
- package/dist/page-agent/tools.js +441 -9
- package/doc/skills/keve-e2e-gen/references/script-generation-guide.md +11 -0
- package/package.json +24 -11
- package/scripts/gen-system-prompt.mjs +36 -0
- package/scripts/run-dsl-task.mjs +619 -0
- package/scripts/runtime/cypress-preprocessor.mjs +44 -0
- package/scripts/runtime/cypress.config.mjs +35 -0
- package/scripts/runtime/playwright.config.mjs +41 -0
- package/scripts/verify-shot-diff.mjs +243 -0
- package/dist/commands/auth-update.d.ts +0 -16
- package/dist/commands/auth-update.js +0 -88
- package/dist/core/auth.d.ts +0 -17
- package/dist/core/auth.js +0 -224
- package/dist/core/cdp-resolver.d.ts +0 -5
- package/dist/core/cdp-resolver.js +0 -93
- package/dist/core/global-setup.d.ts +0 -6
- package/dist/core/global-setup.js +0 -113
- package/dist/core/keve-decorators.d.ts +0 -51
- package/dist/core/keve-decorators.js +0 -95
- package/dist/core/keve-report.js +0 -376
- /package/dist/core/{keve-aspect.d.ts → decorator/keve-aspect.d.ts} +0 -0
- /package/dist/core/{keve-aspect.js → decorator/keve-aspect.js} +0 -0
- /package/dist/{core → engine-playwright}/pw-config.d.ts +0 -0
package/dist/page-agent/agent.js
CHANGED
|
@@ -16,51 +16,16 @@
|
|
|
16
16
|
* const agent = new KevePageAgent(page, { maxSteps: 5 });
|
|
17
17
|
* const result = await agent.execute(step, expected, { learnedActionsHint, fnResult });
|
|
18
18
|
*/
|
|
19
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
20
|
-
if (k2 === undefined) k2 = k;
|
|
21
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
22
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
23
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
24
|
-
}
|
|
25
|
-
Object.defineProperty(o, k2, desc);
|
|
26
|
-
}) : (function(o, m, k, k2) {
|
|
27
|
-
if (k2 === undefined) k2 = k;
|
|
28
|
-
o[k2] = m[k];
|
|
29
|
-
}));
|
|
30
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
31
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
32
|
-
}) : function(o, v) {
|
|
33
|
-
o["default"] = v;
|
|
34
|
-
});
|
|
35
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
36
|
-
var ownKeys = function(o) {
|
|
37
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
38
|
-
var ar = [];
|
|
39
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
40
|
-
return ar;
|
|
41
|
-
};
|
|
42
|
-
return ownKeys(o);
|
|
43
|
-
};
|
|
44
|
-
return function (mod) {
|
|
45
|
-
if (mod && mod.__esModule) return mod;
|
|
46
|
-
var result = {};
|
|
47
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
48
|
-
__setModuleDefault(result, mod);
|
|
49
|
-
return result;
|
|
50
|
-
};
|
|
51
|
-
})();
|
|
52
19
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
53
20
|
exports.KevePageAgent = void 0;
|
|
54
21
|
exports.reactLoop = reactLoop;
|
|
55
22
|
const llm_1 = require("@kkeve/core/llm");
|
|
56
23
|
const zod_1 = require("zod");
|
|
57
|
-
const
|
|
58
|
-
const
|
|
59
|
-
const
|
|
60
|
-
const
|
|
61
|
-
const
|
|
62
|
-
// ─── System Prompt ────────────────────────────────────────────────────
|
|
63
|
-
const SYSTEM_PROMPT = fs.readFileSync(path.join(__dirname, "system_prompt.md"), "utf-8");
|
|
24
|
+
const hooks_js_1 = require("./hooks.js");
|
|
25
|
+
const tools_js_1 = require("./tools.js");
|
|
26
|
+
const system_prompt_js_1 = require("./system-prompt.js");
|
|
27
|
+
const playwright_page_js_1 = require("./playwright-page.js");
|
|
28
|
+
const diagnostics_js_1 = require("./diagnostics.js");
|
|
64
29
|
// ─── Snapshot diff helpers (used by assembleUserPrompt to add Page Change to history) ───
|
|
65
30
|
/** Find the snapshot string of the previous step event before index `i`. */
|
|
66
31
|
function findPrevStepSnapshot(events, currentIdx) {
|
|
@@ -154,33 +119,215 @@ class KevePageAgent {
|
|
|
154
119
|
events = [];
|
|
155
120
|
abortController = new AbortController();
|
|
156
121
|
systemPrompt;
|
|
122
|
+
/** 目标应用 URL(原 process.env.KEVE_TARGET_URL) */
|
|
123
|
+
targetUrl;
|
|
124
|
+
getEnvFn;
|
|
125
|
+
saveScreenshotFn;
|
|
126
|
+
loadInitialImagesFn;
|
|
157
127
|
/** Current execute options (set at start of execute(), accessible from hooks) */
|
|
158
128
|
options;
|
|
159
129
|
hooks;
|
|
130
|
+
// Page event listener state (used by handle_dialog, console_messages, file_upload, network_requests)
|
|
131
|
+
_dialogHandler = null;
|
|
132
|
+
_lastDialogInfo = null;
|
|
133
|
+
_pendingFileChooser = null;
|
|
134
|
+
_consoleMessages = [];
|
|
135
|
+
_networkRequests = [];
|
|
136
|
+
_browserDiagnostics = [];
|
|
137
|
+
_networkDiagnostics = [];
|
|
138
|
+
/** 本次 execute 内 LLM 推理累计耗时(ms) */
|
|
139
|
+
_aiReasoningMs = 0;
|
|
140
|
+
/** 本次 execute 内 LLM token 累计用量 */
|
|
141
|
+
_aiUsage = { llmCalls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
142
|
+
/** 本次 execute 内工具操作累计耗时(ms) */
|
|
143
|
+
_actionMs = 0;
|
|
144
|
+
/** 本次 execute 起始时间,用于总耗时诊断 */
|
|
145
|
+
_executeStartedAt = 0;
|
|
146
|
+
_pageListenersSetup = false;
|
|
147
|
+
/** 页面级性能指标(execute 结束时一次性读取;读取失败则留空) */
|
|
148
|
+
_lastPageMetrics;
|
|
160
149
|
constructor(page, options) {
|
|
161
150
|
this.page = page;
|
|
162
151
|
this.maxSteps = options?.maxSteps ?? 8;
|
|
163
|
-
this.systemPrompt = options?.customSystemPrompt || SYSTEM_PROMPT;
|
|
152
|
+
this.systemPrompt = options?.customSystemPrompt || system_prompt_js_1.SYSTEM_PROMPT;
|
|
164
153
|
this.hooks = options?.hooks;
|
|
165
|
-
|
|
154
|
+
this.getEnvFn = options?.getEnv || (() => '');
|
|
155
|
+
this.targetUrl = options?.targetUrl || this.getEnvFn('KEVE_TARGET_URL') || '';
|
|
156
|
+
this.saveScreenshotFn = options?.saveScreenshot;
|
|
157
|
+
this.loadInitialImagesFn = options?.loadInitialImages;
|
|
158
|
+
const llmConfig = options?.llm || {
|
|
159
|
+
baseURL: this.getEnvFn('KEVE_LLM_BASE_URL'),
|
|
160
|
+
model: this.getEnvFn('KEVE_LLM_MODEL_NAME'),
|
|
161
|
+
apiKey: this.getEnvFn('KEVE_LLM_API_KEY'),
|
|
162
|
+
};
|
|
166
163
|
this.llm = new llm_1.LLM({
|
|
167
|
-
baseURL:
|
|
168
|
-
model:
|
|
169
|
-
apiKey:
|
|
164
|
+
baseURL: llmConfig.baseURL,
|
|
165
|
+
model: llmConfig.model,
|
|
166
|
+
apiKey: llmConfig.apiKey || '',
|
|
170
167
|
temperature: 0.1,
|
|
171
168
|
maxRetries: 3,
|
|
169
|
+
customFetch: llmConfig.customFetch,
|
|
172
170
|
});
|
|
173
171
|
}
|
|
172
|
+
/** 读取宿主注入的环境变量(Cypress 侧由 cy.env 转发,Playwright 侧即 process.env) */
|
|
173
|
+
getEnv(name) {
|
|
174
|
+
return this.getEnvFn(name) || '';
|
|
175
|
+
}
|
|
174
176
|
/** Stop the current execution */
|
|
175
177
|
stop() {
|
|
176
178
|
this.abortController.abort();
|
|
177
179
|
}
|
|
180
|
+
/** Set up page event listeners for dialog, console, network, and file chooser capture.
|
|
181
|
+
* Called once per page; subsequent calls are no-ops. */
|
|
182
|
+
setupPageListeners() {
|
|
183
|
+
if (this._pageListenersSetup)
|
|
184
|
+
return;
|
|
185
|
+
this._pageListenersSetup = true;
|
|
186
|
+
// ── Dialog handler ──
|
|
187
|
+
// Playwright auto-dismisses dialogs unless a handler is registered.
|
|
188
|
+
// Our handler auto-accepts by default, but _dialogHandler can override.
|
|
189
|
+
this.page.on('dialog', async (dialog) => {
|
|
190
|
+
const handler = this._dialogHandler;
|
|
191
|
+
const accept = handler?.accept ?? true;
|
|
192
|
+
const promptText = handler?.promptText;
|
|
193
|
+
try {
|
|
194
|
+
if (accept) {
|
|
195
|
+
await dialog.accept(promptText || '');
|
|
196
|
+
}
|
|
197
|
+
else {
|
|
198
|
+
await dialog.dismiss();
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
// Fallback: always accept to unblock the page
|
|
203
|
+
try {
|
|
204
|
+
await dialog.accept();
|
|
205
|
+
}
|
|
206
|
+
catch { /* give up */ }
|
|
207
|
+
}
|
|
208
|
+
this._lastDialogInfo = {
|
|
209
|
+
type: dialog.type(),
|
|
210
|
+
message: dialog.message(),
|
|
211
|
+
accepted: accept,
|
|
212
|
+
};
|
|
213
|
+
this._dialogHandler = null;
|
|
214
|
+
// Notify the Agent so it knows a dialog appeared
|
|
215
|
+
this.pushObservation(`🔔 Native dialog [${dialog.type()}] appeared: "${dialog.message().slice(0, 100)}" — was ${accept ? 'accepted' : 'dismissed'}.`);
|
|
216
|
+
});
|
|
217
|
+
// ── Console messages ──
|
|
218
|
+
this.page.on('console', (msg) => {
|
|
219
|
+
const level = msg.type();
|
|
220
|
+
const text = msg.text();
|
|
221
|
+
this._consoleMessages.push({ level, text });
|
|
222
|
+
if (this._consoleMessages.length > 200)
|
|
223
|
+
this._consoleMessages.shift();
|
|
224
|
+
this._browserDiagnostics.push({ level, text, at: Date.now(), kind: 'console' });
|
|
225
|
+
if (this._browserDiagnostics.length > 500)
|
|
226
|
+
this._browserDiagnostics.shift();
|
|
227
|
+
});
|
|
228
|
+
this.page.on('pageerror', (err) => {
|
|
229
|
+
const text = `Uncaught: ${err?.message || String(err)}`;
|
|
230
|
+
this._consoleMessages.push({ level: 'error', text });
|
|
231
|
+
this._browserDiagnostics.push({ level: 'error', text, at: Date.now(), kind: 'pageerror' });
|
|
232
|
+
if (this._browserDiagnostics.length > 500)
|
|
233
|
+
this._browserDiagnostics.shift();
|
|
234
|
+
});
|
|
235
|
+
// ── Network requests ──
|
|
236
|
+
this.page.on('request', (req) => {
|
|
237
|
+
const url = req.url();
|
|
238
|
+
const isStatic = (0, diagnostics_js_1.isStaticUrl)(url);
|
|
239
|
+
this._networkRequests.push({
|
|
240
|
+
_req: req,
|
|
241
|
+
method: req.method(),
|
|
242
|
+
url,
|
|
243
|
+
isStatic,
|
|
244
|
+
status: null,
|
|
245
|
+
startedAt: Date.now(),
|
|
246
|
+
});
|
|
247
|
+
if (this._networkRequests.length > 100)
|
|
248
|
+
this._networkRequests.shift();
|
|
249
|
+
});
|
|
250
|
+
this.page.on('response', (res) => {
|
|
251
|
+
const req = res.request();
|
|
252
|
+
// Match by reference identity — Playwright guarantees same Request object
|
|
253
|
+
for (let i = this._networkRequests.length - 1; i >= 0; i--) {
|
|
254
|
+
if (this._networkRequests[i]._req === req) {
|
|
255
|
+
const item = this._networkRequests[i];
|
|
256
|
+
const status = res.status();
|
|
257
|
+
const durationMs = item.startedAt ? Date.now() - item.startedAt : undefined;
|
|
258
|
+
item.status = status;
|
|
259
|
+
item.durationMs = durationMs;
|
|
260
|
+
this._networkDiagnostics.push({
|
|
261
|
+
url: item.url,
|
|
262
|
+
method: item.method,
|
|
263
|
+
status,
|
|
264
|
+
durationMs,
|
|
265
|
+
isStatic: item.isStatic,
|
|
266
|
+
at: item.startedAt,
|
|
267
|
+
});
|
|
268
|
+
if (this._networkDiagnostics.length > 500)
|
|
269
|
+
this._networkDiagnostics.shift();
|
|
270
|
+
break;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
});
|
|
274
|
+
// 请求失败(DNS/超时/连接重置等)没有 response 事件,必须在 requestfailed 记录。
|
|
275
|
+
const onRequestFailed = (req) => {
|
|
276
|
+
const failure = req.failure?.();
|
|
277
|
+
const error = failure?.errorText || 'request failed';
|
|
278
|
+
let item;
|
|
279
|
+
for (let i = this._networkRequests.length - 1; i >= 0; i--) {
|
|
280
|
+
if (this._networkRequests[i]._req === req) {
|
|
281
|
+
item = this._networkRequests[i];
|
|
282
|
+
break;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
const url = item?.url || req.url();
|
|
286
|
+
const durationMs = item?.startedAt ? Date.now() - item.startedAt : undefined;
|
|
287
|
+
this._networkDiagnostics.push({
|
|
288
|
+
url,
|
|
289
|
+
method: item?.method || req.method(),
|
|
290
|
+
status: 0,
|
|
291
|
+
durationMs,
|
|
292
|
+
error: (0, diagnostics_js_1.clipDiagnosticText)(error),
|
|
293
|
+
isStatic: item?.isStatic ?? (0, diagnostics_js_1.isStaticUrl)(url),
|
|
294
|
+
at: item?.startedAt,
|
|
295
|
+
});
|
|
296
|
+
if (this._networkDiagnostics.length > 500)
|
|
297
|
+
this._networkDiagnostics.shift();
|
|
298
|
+
};
|
|
299
|
+
// Playwright 的 AgentPage 契约只保证 on/emit,requestfailed 是原生事件。
|
|
300
|
+
// 用可选探测,避免 Cypress 占位实现或其他 AgentPage 实现因未知事件报错。
|
|
301
|
+
try {
|
|
302
|
+
this.page.on('requestfailed', onRequestFailed);
|
|
303
|
+
}
|
|
304
|
+
catch { /* 非 Playwright 实现可忽略 */ }
|
|
305
|
+
// ── File chooser ──
|
|
306
|
+
this.page.on('filechooser', (fileChooser) => {
|
|
307
|
+
this._pendingFileChooser = fileChooser;
|
|
308
|
+
});
|
|
309
|
+
}
|
|
178
310
|
/** Main Re-Act execute loop */
|
|
179
311
|
async execute(step, expected, options) {
|
|
180
312
|
// Reset state
|
|
181
313
|
this.events = [];
|
|
182
314
|
this.abortController = new AbortController();
|
|
183
315
|
this.options = options;
|
|
316
|
+
// Reset page event listener state
|
|
317
|
+
this._dialogHandler = null;
|
|
318
|
+
this._lastDialogInfo = null;
|
|
319
|
+
this._pendingFileChooser = null;
|
|
320
|
+
this._consoleMessages = [];
|
|
321
|
+
this._networkRequests = [];
|
|
322
|
+
this._browserDiagnostics = [];
|
|
323
|
+
this._networkDiagnostics = [];
|
|
324
|
+
this._aiReasoningMs = 0;
|
|
325
|
+
this._aiUsage = { llmCalls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
326
|
+
this._actionMs = 0;
|
|
327
|
+
this._executeStartedAt = Date.now();
|
|
328
|
+
this._lastPageMetrics = undefined;
|
|
329
|
+
// Set up page event listeners (idempotent — only runs once per page)
|
|
330
|
+
this.setupPageListeners();
|
|
184
331
|
const hooks = options?.hooks ?? this.hooks;
|
|
185
332
|
// Link external signal (e.g. per-goal timeout) to internal abort controller
|
|
186
333
|
const externalSignal = options?.signal;
|
|
@@ -195,7 +342,7 @@ class KevePageAgent {
|
|
|
195
342
|
const maxSteps = options?.maxSteps ?? this.maxSteps;
|
|
196
343
|
const systemPrompt = options?.customSystemPrompt || this.systemPrompt;
|
|
197
344
|
// Inject context/learned hints as initial observations
|
|
198
|
-
const targetUrl =
|
|
345
|
+
const targetUrl = this.targetUrl;
|
|
199
346
|
if (targetUrl) {
|
|
200
347
|
this.pushObservation(`Target application URL: ${targetUrl}. Use this URL for navigation. Do NOT construct URLs yourself — use the navigate tool with this URL or read process.env.KEVE_TARGET_URL.`);
|
|
201
348
|
}
|
|
@@ -223,13 +370,13 @@ class KevePageAgent {
|
|
|
223
370
|
// ── Observe: get browser state (ariaSnapshot only — no per-step screenshot) ──
|
|
224
371
|
console.log('\x1b[34m\x1b[1m👀 Observing...\x1b[0m');
|
|
225
372
|
const snapshot = await this.page.ariaSnapshot({ mode: 'ai' });
|
|
226
|
-
const url = this.page.url();
|
|
373
|
+
const url = await this.page.url();
|
|
227
374
|
console.log(` url: ${url.slice(0, 120)} | step ${stepCount + 1}/${maxSteps}`);
|
|
228
375
|
// ── Assemble messages (text-only per step; initial screenshots on step 0) ──
|
|
229
376
|
const userText = this.assembleUserPrompt(step, expected, snapshot, url, stepCount, maxSteps);
|
|
230
377
|
let userContent;
|
|
231
378
|
if (stepCount === 0) {
|
|
232
|
-
const initialImages = this.loadInitialScreenshots(options);
|
|
379
|
+
const initialImages = await this.loadInitialScreenshots(options);
|
|
233
380
|
userContent = initialImages.length > 0
|
|
234
381
|
? [{ type: 'text', text: userText }, ...initialImages]
|
|
235
382
|
: userText;
|
|
@@ -244,7 +391,15 @@ class KevePageAgent {
|
|
|
244
391
|
// ── Think + Act: LLM decides ──
|
|
245
392
|
console.log('\x1b[34m\x1b[1m🧠 Thinking...\x1b[0m');
|
|
246
393
|
const macroTool = this.buildMacroTool();
|
|
247
|
-
const
|
|
394
|
+
const llmStartedAt = Date.now();
|
|
395
|
+
let llmResult;
|
|
396
|
+
try {
|
|
397
|
+
llmResult = await this.llm.invoke(messages, macroTool, this.abortController.signal, { toolChoiceName: 'AgentOutput' });
|
|
398
|
+
this.recordAiUsage(llmResult.usage);
|
|
399
|
+
}
|
|
400
|
+
finally {
|
|
401
|
+
this._aiReasoningMs += Date.now() - llmStartedAt;
|
|
402
|
+
}
|
|
248
403
|
const macroInput = llmResult.toolCall.args;
|
|
249
404
|
const execResult = llmResult.toolResult;
|
|
250
405
|
const reflection = {
|
|
@@ -283,7 +438,10 @@ class KevePageAgent {
|
|
|
283
438
|
toolError: execResult.error,
|
|
284
439
|
snapshot,
|
|
285
440
|
screenshotPath: undefined,
|
|
441
|
+
duration: execResult.duration,
|
|
286
442
|
};
|
|
443
|
+
if (execResult.duration !== undefined)
|
|
444
|
+
this._actionMs += execResult.duration;
|
|
287
445
|
this.events.push(stepEvent);
|
|
288
446
|
// If execution had error, add observation
|
|
289
447
|
if (execResult.error) {
|
|
@@ -307,8 +465,8 @@ class KevePageAgent {
|
|
|
307
465
|
if (actionName === 'done') {
|
|
308
466
|
// Capture done-time screenshot as goal-after evidence
|
|
309
467
|
try {
|
|
310
|
-
const
|
|
311
|
-
stepEvent.screenshotPath = this.saveScreenshotBuffer(
|
|
468
|
+
const doneShot = await this.page.screenshot({ type: 'png', timeout: 15000 });
|
|
469
|
+
stepEvent.screenshotPath = await this.saveScreenshotBuffer(doneShot.base64, stepCount, step);
|
|
312
470
|
}
|
|
313
471
|
catch { /* non-critical */ }
|
|
314
472
|
// Extract conclusion: 4-level backoff
|
|
@@ -441,6 +599,12 @@ class KevePageAgent {
|
|
|
441
599
|
}
|
|
442
600
|
}
|
|
443
601
|
finally {
|
|
602
|
+
// 页面级性能只能在用例收敛后读一次(LCP/longtask 需要缓冲已完成)
|
|
603
|
+
await this.capturePageMetrics();
|
|
604
|
+
if (taskResult) {
|
|
605
|
+
// 用完整诊断(含页面性能)覆盖此前构建的结果
|
|
606
|
+
taskResult.diagnostics = this.buildDiagnostics();
|
|
607
|
+
}
|
|
444
608
|
// ── onAfterTask hook — may return partial overrides ──
|
|
445
609
|
const hookResult = await hooks?.onAfterTask?.(this, taskResult);
|
|
446
610
|
if (hookResult && taskResult) {
|
|
@@ -453,6 +617,75 @@ class KevePageAgent {
|
|
|
453
617
|
pushObservation(content) {
|
|
454
618
|
this.events.push({ type: 'observation', content });
|
|
455
619
|
}
|
|
620
|
+
/**
|
|
621
|
+
* 读取页面级性能指标。
|
|
622
|
+
*
|
|
623
|
+
* Playwright 与 Cypress 都走同一段探针源码:Playwright 在真实页面上下文,
|
|
624
|
+
* Cypress 在 AUT 的 isolated world(navigation timing / performance entries
|
|
625
|
+
* 与主世界共享同一份 Performance 数据)。
|
|
626
|
+
*/
|
|
627
|
+
async capturePageMetrics() {
|
|
628
|
+
try {
|
|
629
|
+
const raw = await this.page.evaluate((0, diagnostics_js_1.pagePerformanceProbeSource)());
|
|
630
|
+
if (raw && typeof raw === 'object') {
|
|
631
|
+
this._lastPageMetrics = raw;
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
catch {
|
|
635
|
+
// 性能指标是增强信息,读取失败不影响用例结论
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
/**
|
|
639
|
+
* 汇总当前执行期的运行诊断(含页面性能与质量信号)。
|
|
640
|
+
* 公开给 reactLoop 的异常兜底分支复用,避免中断时丢失已采集数据。
|
|
641
|
+
*/
|
|
642
|
+
buildDiagnostics() {
|
|
643
|
+
const stepEvents = this.events.filter((e) => e.type === 'step');
|
|
644
|
+
const actionRecords = stepEvents.map(e => ({
|
|
645
|
+
name: e.toolName || 'unknown',
|
|
646
|
+
durationMs: e.duration,
|
|
647
|
+
error: !!e.toolError,
|
|
648
|
+
}));
|
|
649
|
+
// 重复动作:连续两个以上步骤产生相同工具输出时,累计重复次数。
|
|
650
|
+
let repeatedActionCount = 0;
|
|
651
|
+
let repeatRun = 0;
|
|
652
|
+
let prevOutput = '';
|
|
653
|
+
for (const event of stepEvents) {
|
|
654
|
+
const output = String(event.toolOutput || '').slice(0, 200);
|
|
655
|
+
if (output && output === prevOutput)
|
|
656
|
+
repeatRun += 1;
|
|
657
|
+
else
|
|
658
|
+
repeatRun = 0;
|
|
659
|
+
repeatedActionCount += repeatRun;
|
|
660
|
+
prevOutput = output;
|
|
661
|
+
}
|
|
662
|
+
return (0, diagnostics_js_1.buildRuntimeDiagnostics)({
|
|
663
|
+
totalMs: this._executeStartedAt ? Date.now() - this._executeStartedAt : 0,
|
|
664
|
+
aiReasoningMs: this._aiReasoningMs,
|
|
665
|
+
actionMs: this._actionMs,
|
|
666
|
+
actions: actionRecords,
|
|
667
|
+
consoleRecords: this._browserDiagnostics,
|
|
668
|
+
networkRecords: this._networkDiagnostics,
|
|
669
|
+
repeatedActionCount,
|
|
670
|
+
pageMetrics: this._lastPageMetrics,
|
|
671
|
+
aiUsage: this._aiUsage,
|
|
672
|
+
});
|
|
673
|
+
}
|
|
674
|
+
/** 累加单次 LLM 调用的 token 用量(重试中的失败请求不在返回结果里,无法计量) */
|
|
675
|
+
recordAiUsage(usage) {
|
|
676
|
+
if (!usage)
|
|
677
|
+
return;
|
|
678
|
+
this._aiUsage.llmCalls = (this._aiUsage.llmCalls || 0) + 1;
|
|
679
|
+
this._aiUsage.promptTokens = (this._aiUsage.promptTokens || 0) + (usage.promptTokens || 0);
|
|
680
|
+
this._aiUsage.completionTokens = (this._aiUsage.completionTokens || 0) + (usage.completionTokens || 0);
|
|
681
|
+
this._aiUsage.totalTokens = (this._aiUsage.totalTokens || 0) + (usage.totalTokens || 0);
|
|
682
|
+
if (usage.cachedTokens !== undefined) {
|
|
683
|
+
this._aiUsage.cachedTokens = (this._aiUsage.cachedTokens || 0) + usage.cachedTokens;
|
|
684
|
+
}
|
|
685
|
+
if (usage.reasoningTokens !== undefined) {
|
|
686
|
+
this._aiUsage.reasoningTokens = (this._aiUsage.reasoningTokens || 0) + usage.reasoningTokens;
|
|
687
|
+
}
|
|
688
|
+
}
|
|
456
689
|
assembleUserPrompt(step, expected, snapshot, url, stepIndex, maxSteps) {
|
|
457
690
|
let prompt = '';
|
|
458
691
|
// <agent_state>
|
|
@@ -522,7 +755,7 @@ class KevePageAgent {
|
|
|
522
755
|
return prompt;
|
|
523
756
|
}
|
|
524
757
|
buildMacroTool() {
|
|
525
|
-
const schema = (0,
|
|
758
|
+
const schema = (0, tools_js_1.packMacroToolSchema)();
|
|
526
759
|
const signal = this.abortController.signal;
|
|
527
760
|
return {
|
|
528
761
|
AgentOutput: {
|
|
@@ -531,12 +764,12 @@ class KevePageAgent {
|
|
|
531
764
|
execute: async (input) => {
|
|
532
765
|
signal.throwIfAborted();
|
|
533
766
|
const toolName = String(input.tool || 'done');
|
|
534
|
-
const toolDef =
|
|
767
|
+
const toolDef = tools_js_1.tools.get(toolName);
|
|
535
768
|
if (!toolDef) {
|
|
536
769
|
return { toolName, output: '', error: `Unknown tool: ${toolName}` };
|
|
537
770
|
}
|
|
538
771
|
// Extract only the fields this tool needs from the flat input
|
|
539
|
-
const shape = (0,
|
|
772
|
+
const shape = (0, tools_js_1.getZodShape)(toolDef.inputSchema);
|
|
540
773
|
const toolInput = {};
|
|
541
774
|
for (const key of Object.keys(shape)) {
|
|
542
775
|
if (input[key] !== undefined) {
|
|
@@ -559,35 +792,34 @@ class KevePageAgent {
|
|
|
559
792
|
},
|
|
560
793
|
};
|
|
561
794
|
}
|
|
562
|
-
/**
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
795
|
+
/**
|
|
796
|
+
* Save screenshot base64 to test artifacts.
|
|
797
|
+
*
|
|
798
|
+
* 落盘能力由宿主注入:Playwright 侧写本地文件系统,Cypress 侧经 HTTP 桥落盘。
|
|
799
|
+
*/
|
|
800
|
+
async saveScreenshotBuffer(pngBase64, stepIndex, stepName) {
|
|
801
|
+
if (!this.saveScreenshotFn)
|
|
802
|
+
return undefined;
|
|
803
|
+
try {
|
|
804
|
+
return await this.saveScreenshotFn(pngBase64, stepIndex, stepName);
|
|
805
|
+
}
|
|
806
|
+
catch {
|
|
807
|
+
return undefined;
|
|
808
|
+
}
|
|
572
809
|
}
|
|
573
810
|
/** Load initial screenshots (goal-before + fn-after) as ContentItem[] for step 0 multimodal input */
|
|
574
|
-
loadInitialScreenshots(options) {
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
image_url: { url: `data:image/png;base64,${buf.toString('base64')}`, detail: 'high' },
|
|
586
|
-
});
|
|
587
|
-
}
|
|
588
|
-
catch { /* non-critical */ }
|
|
811
|
+
async loadInitialScreenshots(options) {
|
|
812
|
+
if (!this.loadInitialImagesFn)
|
|
813
|
+
return [];
|
|
814
|
+
try {
|
|
815
|
+
return (await this.loadInitialImagesFn({
|
|
816
|
+
goalScreenshotBefore: options?.goalScreenshotBefore,
|
|
817
|
+
fnAfterScreenshot: options?.fnAfterScreenshot,
|
|
818
|
+
}));
|
|
819
|
+
}
|
|
820
|
+
catch {
|
|
821
|
+
return [];
|
|
589
822
|
}
|
|
590
|
-
return items;
|
|
591
823
|
}
|
|
592
824
|
/**
|
|
593
825
|
* Visual element location: uses LLM multimodal to identify element bbox in screenshot.
|
|
@@ -627,6 +859,7 @@ IMPORTANT: Respond in Chinese.`,
|
|
|
627
859
|
execute: async (args) => args,
|
|
628
860
|
},
|
|
629
861
|
}, this.abortController.signal, { toolChoiceName: 'LocateResult' });
|
|
862
|
+
this.recordAiUsage(result.usage);
|
|
630
863
|
const parsed = result.toolCall.args;
|
|
631
864
|
const bbox = parsed.bbox || [];
|
|
632
865
|
if (bbox.length === 4 && bbox.every(v => typeof v === 'number' && !isNaN(v) && v >= 0 && v <= 1000) && bbox[2] > bbox[0] && bbox[3] > bbox[1]) {
|
|
@@ -674,6 +907,7 @@ IMPORTANT:
|
|
|
674
907
|
execute: async (args) => args,
|
|
675
908
|
},
|
|
676
909
|
}, this.abortController.signal, { toolChoiceName: 'AssertResult' });
|
|
910
|
+
this.recordAiUsage(result.usage);
|
|
677
911
|
const parsed = result.toolCall.args;
|
|
678
912
|
return {
|
|
679
913
|
passed: typeof parsed.passed === 'boolean' ? parsed.passed : false,
|
|
@@ -695,7 +929,16 @@ IMPORTANT:
|
|
|
695
929
|
.filter((e) => e.type === 'step')
|
|
696
930
|
.filter(e => e.screenshotPath)
|
|
697
931
|
.map(e => e.screenshotPath);
|
|
698
|
-
|
|
932
|
+
const diagnostics = this.buildDiagnostics();
|
|
933
|
+
return {
|
|
934
|
+
success,
|
|
935
|
+
data,
|
|
936
|
+
events: this.events,
|
|
937
|
+
finalSnapshot: snapshot,
|
|
938
|
+
diagnostics,
|
|
939
|
+
/* refinePatch, */
|
|
940
|
+
agentScreenshots: screenshotPaths.length ? screenshotPaths : undefined,
|
|
941
|
+
};
|
|
699
942
|
}
|
|
700
943
|
}
|
|
701
944
|
exports.KevePageAgent = KevePageAgent;
|
|
@@ -704,16 +947,25 @@ exports.KevePageAgent = KevePageAgent;
|
|
|
704
947
|
* Drop-in replacement for the old reactLoop function.
|
|
705
948
|
* Creates a KevePageAgent, runs it, and maps the result.
|
|
706
949
|
*/
|
|
707
|
-
async function reactLoop(
|
|
708
|
-
|
|
950
|
+
async function reactLoop(
|
|
951
|
+
// 接受原生 Playwright Page 或任意 AgentPage 实现(用 toAgentPage 的参数类型
|
|
952
|
+
// 表达,避免在浏览器安全模块里引入 @playwright/test 的类型依赖)
|
|
953
|
+
page, step, expected, options = {}) {
|
|
954
|
+
const agent = new KevePageAgent((0, playwright_page_js_1.toAgentPage)(page), {
|
|
709
955
|
maxSteps: options.maxSteps ?? 20,
|
|
956
|
+
systemPrompt: options.systemPrompt,
|
|
957
|
+
targetUrl: options.targetUrl,
|
|
958
|
+
getEnv: options.getEnv,
|
|
959
|
+
saveScreenshot: options.saveScreenshot,
|
|
960
|
+
loadInitialImages: options.loadInitialImages,
|
|
961
|
+
llm: options.llm,
|
|
710
962
|
hooks: {
|
|
711
963
|
onAfterTask: async (_agent, result) => {
|
|
712
964
|
const partial = {};
|
|
713
965
|
// 1. Diagnostic hints for failures
|
|
714
966
|
if (!result.success) {
|
|
715
967
|
try {
|
|
716
|
-
partial.diagnosticHints = await (0,
|
|
968
|
+
partial.diagnosticHints = await (0, hooks_js_1.extractDiagnosticHints)(_agent.page, result);
|
|
717
969
|
}
|
|
718
970
|
catch { /* non-critical */ }
|
|
719
971
|
}
|
|
@@ -765,6 +1017,7 @@ async function reactLoop(page, step, expected, options = {}) {
|
|
|
765
1017
|
stepCount: partialEvents.length,
|
|
766
1018
|
finalSnapshot: '',
|
|
767
1019
|
events: agent.events,
|
|
1020
|
+
diagnostics: agent.buildDiagnostics(),
|
|
768
1021
|
agentScreenshots: agent.events
|
|
769
1022
|
.filter((e) => e.type === 'step')
|
|
770
1023
|
.filter(e => e.screenshotPath)
|
|
@@ -791,6 +1044,7 @@ async function reactLoop(page, step, expected, options = {}) {
|
|
|
791
1044
|
expectedMet: result.success,
|
|
792
1045
|
conclusion: result.conclusion,
|
|
793
1046
|
finalSnapshot: result.finalSnapshot,
|
|
1047
|
+
diagnostics: result.diagnostics,
|
|
794
1048
|
// refinePatch: result.refinePatch, // 已注释:scriptRefine 已禁用
|
|
795
1049
|
agentScreenshots: result.agentScreenshots,
|
|
796
1050
|
};
|