@kkeve/suite 0.0.6 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +1 -1
- package/dist/core/generateReport/reportData.d.ts +2 -0
- package/dist/core/generateReport/reportData.js +1 -0
- package/dist/core/keve-aspect.d.ts +5 -7
- package/dist/core/keve-aspect.js +5 -7
- package/dist/core/keve-test.js +44 -28
- package/dist/core/pw-config.js +2 -2
- package/dist/page-agent/agent.d.ts +12 -2
- package/dist/page-agent/agent.js +55 -43
- package/dist/page-agent/system_prompt.md +9 -3
- package/dist/page-agent/tools.js +2 -2
- package/package.json +1 -1
package/dist/config.js
CHANGED
|
@@ -97,6 +97,8 @@ export interface StepReportItem {
|
|
|
97
97
|
precondition?: string;
|
|
98
98
|
order?: number;
|
|
99
99
|
success?: boolean;
|
|
100
|
+
/** fn 执行前截图路径(goal-before,页面原始状态) */
|
|
101
|
+
goalScreenshotBefore?: string;
|
|
100
102
|
/** 目标结果截图路径(keveGoal 执行后截图,来自 goalScreenshotPath / goalScreenshotAfter) */
|
|
101
103
|
goalScreenshotPath?: string;
|
|
102
104
|
/** 验证截图路径列表(agent 每步操作后截图,来自 verifyScreenshots / agentScreenshots) */
|
|
@@ -564,6 +564,7 @@ async function generateReportData(opts) {
|
|
|
564
564
|
order: s.order,
|
|
565
565
|
success: s.success,
|
|
566
566
|
conclusion: s.conclusion,
|
|
567
|
+
goalScreenshotBefore: s.goalScreenshotBefore,
|
|
567
568
|
goalScreenshotPath: s.goalScreenshotPath || s.goalScreenshotAfter,
|
|
568
569
|
verifyScreenshots: s.verifyScreenshots || s.agentScreenshots,
|
|
569
570
|
actions: (s.actions || []).map((a) => ({
|
|
@@ -4,13 +4,11 @@
|
|
|
4
4
|
* Three phases: before → (goal body or rescue) → after
|
|
5
5
|
* User-invisible: aspects are registered by the framework, not by test authors.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
* -
|
|
9
|
-
*
|
|
10
|
-
* -
|
|
11
|
-
*
|
|
12
|
-
* - learnedActions-update (after, order=20) — update in-memory cache
|
|
13
|
-
* - screenshot-on-fail (after, order=30) — capture screenshot on failure
|
|
7
|
+
* Registered aspects:
|
|
8
|
+
* - learnedActions-update (after, order=20) — update in-memory cache on success
|
|
9
|
+
*
|
|
10
|
+
* 截图由 keveGoal 内部直接管理(goal-before / goal-after / agent 每步截图),
|
|
11
|
+
* 不通过 aspect 注册,避免截图时机与 agent 执行步骤脱节。
|
|
14
12
|
*/
|
|
15
13
|
export type AspectPhase = 'before' | 'rescue' | 'after';
|
|
16
14
|
export interface GoalContext {
|
package/dist/core/keve-aspect.js
CHANGED
|
@@ -5,13 +5,11 @@
|
|
|
5
5
|
* Three phases: before → (goal body or rescue) → after
|
|
6
6
|
* User-invisible: aspects are registered by the framework, not by test authors.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
11
|
-
* -
|
|
12
|
-
*
|
|
13
|
-
* - learnedActions-update (after, order=20) — update in-memory cache
|
|
14
|
-
* - screenshot-on-fail (after, order=30) — capture screenshot on failure
|
|
8
|
+
* Registered aspects:
|
|
9
|
+
* - learnedActions-update (after, order=20) — update in-memory cache on success
|
|
10
|
+
*
|
|
11
|
+
* 截图由 keveGoal 内部直接管理(goal-before / goal-after / agent 每步截图),
|
|
12
|
+
* 不通过 aspect 注册,避免截图时机与 agent 执行步骤脱节。
|
|
15
13
|
*/
|
|
16
14
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
15
|
exports.keveAspect = exports.KeveAspectRegistry = void 0;
|
package/dist/core/keve-test.js
CHANGED
|
@@ -160,6 +160,13 @@ exports.test = test_1.test.extend({
|
|
|
160
160
|
};
|
|
161
161
|
// Run before aspects
|
|
162
162
|
await keve_aspect_1.keveAspect.runPhase('before', ctx);
|
|
163
|
+
// ── Capture goal-before screenshot (fn 之前的页面原始状态) ──
|
|
164
|
+
// 语义:记录"测试动作发生前"的页面,供报告 before/after 对照
|
|
165
|
+
let goalScreenshotBefore = '';
|
|
166
|
+
try {
|
|
167
|
+
goalScreenshotBefore = await captureScreenshot(page, options.step, goalMeta.order, 'before');
|
|
168
|
+
}
|
|
169
|
+
catch { /* non-critical */ }
|
|
163
170
|
let result;
|
|
164
171
|
const fnSource = fn?.toString() || undefined;
|
|
165
172
|
let fnError;
|
|
@@ -173,27 +180,21 @@ exports.test = test_1.test.extend({
|
|
|
173
180
|
console.log(`[keveGoal] "${options.step}" fn error: ${fnError}`);
|
|
174
181
|
}
|
|
175
182
|
}
|
|
176
|
-
// ── Capture
|
|
177
|
-
let
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
fs.mkdirSync(screenshotsDir, { recursive: true });
|
|
184
|
-
const safeName = options.step.replace(/[^a-zA-Z0-9\u4e00-\u9fff]/g, '_').slice(0, 30);
|
|
185
|
-
const file = path.join(screenshotsDir, `goal-before-${goalMeta.order}-${safeName}-${Date.now()}.png`);
|
|
186
|
-
fs.writeFileSync(file, buf);
|
|
187
|
-
goalScreenshotBefore = path.relative(taskDir, file);
|
|
183
|
+
// ── Capture fn-after screenshot (fn 执行后的页面状态,传给 agent 说明执行前后) ──
|
|
184
|
+
let fnAfterScreenshot = '';
|
|
185
|
+
if (fn) {
|
|
186
|
+
try {
|
|
187
|
+
fnAfterScreenshot = await captureScreenshot(page, options.step, goalMeta.order, 'after-fn');
|
|
188
|
+
}
|
|
189
|
+
catch { /* non-critical */ }
|
|
188
190
|
}
|
|
189
|
-
catch { /* non-critical */ }
|
|
190
191
|
// ── Hand off to agent: agent does Re-Act ──
|
|
191
192
|
const learnedHint = learned_actions_1.learnedActions.getHint(options.step);
|
|
192
193
|
let reactResult;
|
|
193
194
|
let reactTimedOut = false;
|
|
194
195
|
// ── Per-goal timeout: prevent one slow goal from exhausting the test timeout ──
|
|
195
|
-
// Default
|
|
196
|
-
const GOAL_TIMEOUT_MS =
|
|
196
|
+
// Default 300s per goal; total test timeout should be ≥ (goals × 300s + overhead)
|
|
197
|
+
const GOAL_TIMEOUT_MS = 300_000;
|
|
197
198
|
const goalTimeoutController = new AbortController();
|
|
198
199
|
const goalTimeout = setTimeout(() => {
|
|
199
200
|
goalTimeoutController.abort();
|
|
@@ -206,6 +207,8 @@ exports.test = test_1.test.extend({
|
|
|
206
207
|
fnSource,
|
|
207
208
|
fnResult: fnError ? { error: fnError } : { success: true },
|
|
208
209
|
signal: goalTimeoutController.signal,
|
|
210
|
+
goalScreenshotBefore: goalScreenshotBefore || undefined,
|
|
211
|
+
fnAfterScreenshot: fnAfterScreenshot || undefined,
|
|
209
212
|
});
|
|
210
213
|
}
|
|
211
214
|
catch (reactErr) {
|
|
@@ -250,22 +253,19 @@ exports.test = test_1.test.extend({
|
|
|
250
253
|
// 通过 Playwright attachment 流到 KeveReporter.onTestEnd
|
|
251
254
|
// 注意:screenshotBase64 不再写入 attachment(截图已保存到文件系统,路径见 goalScreenshotBefore/goalScreenshotAfter)
|
|
252
255
|
const diagnosticHints = reactResult.diagnosticHints || [];
|
|
256
|
+
// ── goalScreenshotAfter = agent done 时的截图(done-time screenshot) ──
|
|
257
|
+
// agent.ts 在 done 工具触发时截图并写入 stepEvent.screenshotPath,
|
|
258
|
+
// 这里复用该路径作为 goal-after 证据(与 agent 判定时刻一致,避免状态漂移)。
|
|
253
259
|
let goalScreenshotAfter = '';
|
|
254
260
|
try {
|
|
255
|
-
const
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
const file = path.join(screenshotsDir, `goal-after-${goalMeta.order}-${safeName}-${Date.now()}.png`);
|
|
262
|
-
fs.writeFileSync(file, buf);
|
|
263
|
-
goalScreenshotAfter = path.relative(taskDir, file);
|
|
264
|
-
console.log(`[keveGoal] screenshot captured: ${buf.length} bytes (path: ${goalScreenshotAfter})`);
|
|
265
|
-
}
|
|
266
|
-
catch (e) {
|
|
267
|
-
console.log(`[keveGoal] screenshot failed: ${e.message}`);
|
|
261
|
+
const lastActionWithShot = [...(reactResult.actions || [])]
|
|
262
|
+
.reverse()
|
|
263
|
+
.find((a) => a.screenshotPath);
|
|
264
|
+
if (lastActionWithShot?.screenshotPath) {
|
|
265
|
+
goalScreenshotAfter = lastActionWithShot.screenshotPath;
|
|
266
|
+
}
|
|
268
267
|
}
|
|
268
|
+
catch { /* non-critical */ }
|
|
269
269
|
await testInfo.attach('keveGoalResult', {
|
|
270
270
|
contentType: 'application/json',
|
|
271
271
|
body: Buffer.from(JSON.stringify({
|
|
@@ -362,3 +362,19 @@ exports.test = test_1.test.extend({
|
|
|
362
362
|
// Register extended test with keve-decorators so @keveModel uses the correct test
|
|
363
363
|
const keve_decorators_2 = require("./keve-decorators");
|
|
364
364
|
(0, keve_decorators_2.setKeveTest)(exports.test);
|
|
365
|
+
// ── 辅助函数 ──
|
|
366
|
+
/**
|
|
367
|
+
* 截图并保存到 test-artifacts,返回相对 taskDir 的路径
|
|
368
|
+
* @param phase 'before' | 'after-fn' — 用于文件名前缀
|
|
369
|
+
*/
|
|
370
|
+
async function captureScreenshot(page, stepName, order, phase) {
|
|
371
|
+
const buf = await page.screenshot({ type: 'png', timeout: 15000 });
|
|
372
|
+
const taskDir = process.env.KEVE_TASK_DIR || '.keve';
|
|
373
|
+
const round = process.env.KEVE_ROUND || 'latest';
|
|
374
|
+
const screenshotsDir = path.join(taskDir, 'test-artifacts', `round-${round}`, 'screenshots');
|
|
375
|
+
fs.mkdirSync(screenshotsDir, { recursive: true });
|
|
376
|
+
const safeName = stepName.replace(/[^a-zA-Z0-9一-鿿]/g, '_').slice(0, 30);
|
|
377
|
+
const file = path.join(screenshotsDir, `goal-${phase}-${order}-${safeName}-${Date.now()}.png`);
|
|
378
|
+
fs.writeFileSync(file, buf);
|
|
379
|
+
return path.relative(taskDir, file);
|
|
380
|
+
}
|
package/dist/core/pw-config.js
CHANGED
|
@@ -88,7 +88,7 @@ export default defineConfig({
|
|
|
88
88
|
use: {
|
|
89
89
|
baseURL: ${baseURLLine},${cdp ? `\n // CDP mode: login state preserved via connected Chrome` : `\n storageState: path.resolve(projectRoot, '.auth/storage-state.json'),`}
|
|
90
90
|
trace: 'on-first-retry',
|
|
91
|
-
screenshot: '
|
|
91
|
+
screenshot: 'off',
|
|
92
92
|
video: 'retain-on-failure',
|
|
93
93
|
actionTimeout: 5000,${cdpBlock}
|
|
94
94
|
},
|
|
@@ -136,7 +136,7 @@ export default defineConfig({
|
|
|
136
136
|
use: {
|
|
137
137
|
baseURL: ${baseURLLine},${cdp ? `\n // CDP mode: login state preserved via connected Chrome` : `\n storageState: resolve(projectRoot, '.auth', 'storage-state.json'),`}
|
|
138
138
|
trace: 'on-first-retry',
|
|
139
|
-
screenshot: '
|
|
139
|
+
screenshot: 'off',
|
|
140
140
|
video: 'retain-on-failure',
|
|
141
141
|
actionTimeout: 5000,${cdpBlock}
|
|
142
142
|
},
|
|
@@ -6,8 +6,10 @@
|
|
|
6
6
|
* - MacroTool: merge all tool schemas into one LLM output
|
|
7
7
|
* - History events: step/observation/error stream
|
|
8
8
|
* - Reflection-before-action: evaluation + memory + next_goal + action
|
|
9
|
-
* -
|
|
10
|
-
* -
|
|
9
|
+
* - Per-step: ariaSnapshot text only (no screenshot) — Option C
|
|
10
|
+
* - Initial screenshots (goal-before + fn-after) injected on step 0 as multimodal context
|
|
11
|
+
* - Done-time screenshot captured as goal-after evidence
|
|
12
|
+
* - On-demand screenshots via visual_locate tool (when element needs visual identification)
|
|
11
13
|
*
|
|
12
14
|
* Usage:
|
|
13
15
|
* const agent = new KevePageAgent(page, { maxSteps: 5 });
|
|
@@ -68,6 +70,10 @@ export interface AgentOptions {
|
|
|
68
70
|
hooks?: AgentHooks;
|
|
69
71
|
/** External abort signal (e.g. per-goal timeout). When aborted, internal abortController is also triggered. */
|
|
70
72
|
signal?: AbortSignal;
|
|
73
|
+
/** goal-before 截图路径(相对 taskDir)— fn 执行前的页面原始状态 */
|
|
74
|
+
goalScreenshotBefore?: string;
|
|
75
|
+
/** fn-after 截图路径(相对 taskDir)— fn 执行后的页面状态 */
|
|
76
|
+
fnAfterScreenshot?: string;
|
|
71
77
|
}
|
|
72
78
|
export declare class KevePageAgent {
|
|
73
79
|
readonly page: Page;
|
|
@@ -93,6 +99,8 @@ export declare class KevePageAgent {
|
|
|
93
99
|
private buildMacroTool;
|
|
94
100
|
/** Save screenshot buffer to test artifacts (reuses already-captured buffer) */
|
|
95
101
|
private saveScreenshotBuffer;
|
|
102
|
+
/** Load initial screenshots (goal-before + fn-after) as ContentItem[] for step 0 multimodal input */
|
|
103
|
+
private loadInitialScreenshots;
|
|
96
104
|
/**
|
|
97
105
|
* Visual element location: uses LLM multimodal to identify element bbox in screenshot.
|
|
98
106
|
* Returns normalized bbox (0-1000 scale) for coordinate-based clicking.
|
|
@@ -120,6 +128,8 @@ export declare function reactLoop(page: Page, step: string, expected: string, op
|
|
|
120
128
|
success?: boolean;
|
|
121
129
|
};
|
|
122
130
|
signal?: AbortSignal;
|
|
131
|
+
goalScreenshotBefore?: string;
|
|
132
|
+
fnAfterScreenshot?: string;
|
|
123
133
|
}): Promise<{
|
|
124
134
|
actions: any[];
|
|
125
135
|
expectedMet: boolean;
|
package/dist/page-agent/agent.js
CHANGED
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
* - MacroTool: merge all tool schemas into one LLM output
|
|
8
8
|
* - History events: step/observation/error stream
|
|
9
9
|
* - Reflection-before-action: evaluation + memory + next_goal + action
|
|
10
|
-
* -
|
|
11
|
-
* -
|
|
10
|
+
* - Per-step: ariaSnapshot text only (no screenshot) — Option C
|
|
11
|
+
* - Initial screenshots (goal-before + fn-after) injected on step 0 as multimodal context
|
|
12
|
+
* - Done-time screenshot captured as goal-after evidence
|
|
13
|
+
* - On-demand screenshots via visual_locate tool (when element needs visual identification)
|
|
12
14
|
*
|
|
13
15
|
* Usage:
|
|
14
16
|
* const agent = new KevePageAgent(page, { maxSteps: 5 });
|
|
@@ -133,37 +135,19 @@ class KevePageAgent {
|
|
|
133
135
|
await hooks?.onBeforeStep?.(this, stepCount);
|
|
134
136
|
try {
|
|
135
137
|
console.group(`step: ${stepCount}`);
|
|
136
|
-
// ── Observe: get browser state ──
|
|
138
|
+
// ── Observe: get browser state (ariaSnapshot only — no per-step screenshot) ──
|
|
137
139
|
console.log('\x1b[34m\x1b[1m👀 Observing...\x1b[0m');
|
|
138
140
|
const snapshot = await this.page.ariaSnapshot({ mode: 'ai' });
|
|
139
141
|
const url = this.page.url();
|
|
140
142
|
console.log(` url: ${url.slice(0, 120)} | step ${stepCount + 1}/${maxSteps}`);
|
|
141
|
-
// ──
|
|
142
|
-
|
|
143
|
-
let screenshotBase64;
|
|
144
|
-
try {
|
|
145
|
-
screenshotBuf = await this.page.screenshot({ type: 'png', timeout: 10000 });
|
|
146
|
-
screenshotBase64 = screenshotBuf.toString('base64');
|
|
147
|
-
}
|
|
148
|
-
catch (e) {
|
|
149
|
-
// Retry once with longer timeout (font loading can be slow on some pages)
|
|
150
|
-
try {
|
|
151
|
-
await new Promise(r => setTimeout(r, 500));
|
|
152
|
-
screenshotBuf = await this.page.screenshot({ type: 'png', timeout: 15000 });
|
|
153
|
-
screenshotBase64 = screenshotBuf.toString('base64');
|
|
154
|
-
}
|
|
155
|
-
catch (e2) {
|
|
156
|
-
console.log(`[agent] screenshot capture failed (non-critical): ${e2.message}`);
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
// ── Assemble messages (multimodal: text + screenshot) ──
|
|
160
|
-
const userText = this.assembleUserPrompt(step, expected, snapshot, url, stepCount, maxSteps, !!screenshotBase64);
|
|
143
|
+
// ── Assemble messages (text-only per step; initial screenshots on step 0) ──
|
|
144
|
+
const userText = this.assembleUserPrompt(step, expected, snapshot, url, stepCount, maxSteps);
|
|
161
145
|
let userContent;
|
|
162
|
-
if (
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
{ type: '
|
|
166
|
-
|
|
146
|
+
if (stepCount === 0) {
|
|
147
|
+
const initialImages = this.loadInitialScreenshots(options);
|
|
148
|
+
userContent = initialImages.length > 0
|
|
149
|
+
? [{ type: 'text', text: userText }, ...initialImages]
|
|
150
|
+
: userText;
|
|
167
151
|
}
|
|
168
152
|
else {
|
|
169
153
|
userContent = userText;
|
|
@@ -192,15 +176,7 @@ class KevePageAgent {
|
|
|
192
176
|
console.log(`🎯: ${reflection.nextGoal.slice(0, 150)}`);
|
|
193
177
|
const actionName = execResult.toolName;
|
|
194
178
|
const actionInput = macroInput;
|
|
195
|
-
//
|
|
196
|
-
let screenshotPath;
|
|
197
|
-
if (screenshotBuf) {
|
|
198
|
-
try {
|
|
199
|
-
screenshotPath = this.saveScreenshotBuffer(screenshotBuf, stepCount, step);
|
|
200
|
-
}
|
|
201
|
-
catch { /* non-critical */ }
|
|
202
|
-
}
|
|
203
|
-
// Record step event
|
|
179
|
+
// Record step event (screenshotPath set later for done actions)
|
|
204
180
|
if (execResult.error) {
|
|
205
181
|
console.log(`\x1b[31m\x1b[1m${actionName} (error: ${execResult.error.slice(0, 80)})\x1b[0m`);
|
|
206
182
|
}
|
|
@@ -221,15 +197,21 @@ class KevePageAgent {
|
|
|
221
197
|
toolOutput: execResult.output,
|
|
222
198
|
toolError: execResult.error,
|
|
223
199
|
snapshot,
|
|
224
|
-
screenshotPath,
|
|
200
|
+
screenshotPath: undefined,
|
|
225
201
|
};
|
|
226
202
|
this.events.push(stepEvent);
|
|
227
203
|
// If execution had error, add observation
|
|
228
204
|
if (execResult.error) {
|
|
229
205
|
this.pushObservation(`Action "${actionName}" failed: ${execResult.error}. Try a different approach.`);
|
|
230
206
|
}
|
|
231
|
-
// ── Check: if "done", return
|
|
207
|
+
// ── Check: if "done", capture done-time screenshot + return with test conclusion ──
|
|
232
208
|
if (actionName === 'done') {
|
|
209
|
+
// Capture done-time screenshot as goal-after evidence
|
|
210
|
+
try {
|
|
211
|
+
const doneBuf = await this.page.screenshot({ type: 'png', timeout: 15000 });
|
|
212
|
+
stepEvent.screenshotPath = this.saveScreenshotBuffer(doneBuf, stepCount, step);
|
|
213
|
+
}
|
|
214
|
+
catch { /* non-critical */ }
|
|
233
215
|
// Extract conclusion: 4-level backoff
|
|
234
216
|
let conclusion;
|
|
235
217
|
const verdictField = actionInput?.verdict;
|
|
@@ -325,7 +307,7 @@ class KevePageAgent {
|
|
|
325
307
|
pushObservation(content) {
|
|
326
308
|
this.events.push({ type: 'observation', content });
|
|
327
309
|
}
|
|
328
|
-
assembleUserPrompt(step, expected, snapshot, url, stepIndex, maxSteps
|
|
310
|
+
assembleUserPrompt(step, expected, snapshot, url, stepIndex, maxSteps) {
|
|
329
311
|
let prompt = '';
|
|
330
312
|
// <agent_state>
|
|
331
313
|
prompt += '<agent_state>\n';
|
|
@@ -343,6 +325,18 @@ class KevePageAgent {
|
|
|
343
325
|
: '';
|
|
344
326
|
prompt += `<prior_execution>\nA deterministic script was executed BEFORE your Re-Act loop.\n${resultLine}${sourceLine}\n</prior_execution>\n`;
|
|
345
327
|
}
|
|
328
|
+
// <initial_screenshots>: only on first step, describe the before/after fn screenshots
|
|
329
|
+
if (stepIndex === 0 && (this.options?.goalScreenshotBefore || this.options?.fnAfterScreenshot)) {
|
|
330
|
+
prompt += '<initial_screenshots>\n';
|
|
331
|
+
if (this.options?.goalScreenshotBefore) {
|
|
332
|
+
prompt += 'goal-before: Screenshot of the page state BEFORE any test action (fn执行前的页面原始状态).\n';
|
|
333
|
+
}
|
|
334
|
+
if (this.options?.fnAfterScreenshot) {
|
|
335
|
+
prompt += 'fn-after: Screenshot of the page state AFTER the deterministic fn script executed (fn执行后的页面状态). Compare with goal-before to understand what fn changed.\n';
|
|
336
|
+
}
|
|
337
|
+
prompt += 'These screenshots are attached as images to give you visual context of the pre/post fn execution state. Use them to understand what fn did, then rely on the accessibility tree (ariaSnapshot) for per-step decisions.\n';
|
|
338
|
+
prompt += '</initial_screenshots>\n';
|
|
339
|
+
}
|
|
346
340
|
prompt += '<step_info>\n';
|
|
347
341
|
prompt += `Step ${stepIndex + 1} of ${maxSteps} max steps\n`;
|
|
348
342
|
prompt += `Current URL: ${url}\n`;
|
|
@@ -370,9 +364,6 @@ class KevePageAgent {
|
|
|
370
364
|
prompt += '<browser_state>\n';
|
|
371
365
|
prompt += `Current URL: ${url}\n\n`;
|
|
372
366
|
prompt += `Accessibility Tree (YAML):\n\`\`\`yaml\n${snapshot}\n\`\`\`\n`;
|
|
373
|
-
if (hasScreenshot) {
|
|
374
|
-
prompt += '\n<page_screenshot>\nA screenshot of the current page is provided as an image attached to this message. Use it as the primary visual source of truth for what is actually visible on the page. Cross-reference the screenshot with the accessibility tree to make accurate judgments — especially before calling done(success=true) to verify the expected state is truly achieved.\n</page_screenshot>\n';
|
|
375
|
-
}
|
|
376
367
|
prompt += '</browser_state>\n\n';
|
|
377
368
|
return prompt;
|
|
378
369
|
}
|
|
@@ -425,6 +416,25 @@ class KevePageAgent {
|
|
|
425
416
|
fs.writeFileSync(file, buf);
|
|
426
417
|
return path.relative(taskDir, file);
|
|
427
418
|
}
|
|
419
|
+
/** Load initial screenshots (goal-before + fn-after) as ContentItem[] for step 0 multimodal input */
|
|
420
|
+
loadInitialScreenshots(options) {
|
|
421
|
+
const items = [];
|
|
422
|
+
const taskDir = process.env.KEVE_TASK_DIR || '.keve';
|
|
423
|
+
for (const relPath of [options?.goalScreenshotBefore, options?.fnAfterScreenshot]) {
|
|
424
|
+
if (!relPath)
|
|
425
|
+
continue;
|
|
426
|
+
try {
|
|
427
|
+
const absPath = path.resolve(taskDir, relPath);
|
|
428
|
+
const buf = fs.readFileSync(absPath);
|
|
429
|
+
items.push({
|
|
430
|
+
type: 'image_url',
|
|
431
|
+
image_url: { url: `data:image/png;base64,${buf.toString('base64')}`, detail: 'high' },
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
catch { /* non-critical */ }
|
|
435
|
+
}
|
|
436
|
+
return items;
|
|
437
|
+
}
|
|
428
438
|
/**
|
|
429
439
|
* Visual element location: uses LLM multimodal to identify element bbox in screenshot.
|
|
430
440
|
* Returns normalized bbox (0-1000 scale) for coordinate-based clicking.
|
|
@@ -541,6 +551,8 @@ async function reactLoop(page, step, expected, options = {}) {
|
|
|
541
551
|
fnSource: options.fnSource,
|
|
542
552
|
fnResult: options.fnResult,
|
|
543
553
|
signal: options.signal,
|
|
554
|
+
goalScreenshotBefore: options.goalScreenshotBefore,
|
|
555
|
+
fnAfterScreenshot: options.fnAfterScreenshot,
|
|
544
556
|
});
|
|
545
557
|
}
|
|
546
558
|
catch (execErr) {
|
|
@@ -126,14 +126,20 @@ You must call the `done` action in one of these cases:
|
|
|
126
126
|
The `done` action requires two fields:
|
|
127
127
|
- `verdict`: Your test verdict, must be one of:
|
|
128
128
|
- `"pass"`: The expected state described in <user_request> is fully achieved. Examine the screenshot carefully before deciding.
|
|
129
|
-
- `"fail"`: The page is accessible but the expected state is NOT achieved (e.g. text mismatch, wrong element state, visual difference).
|
|
130
|
-
- `"blocked"`: Cannot continue testing (e.g. SSO login redirect, connection refused, page not loading).
|
|
129
|
+
- `"fail"`: The page is accessible but the expected state is NOT achieved (e.g. text mismatch, wrong element state, visual difference). This indicates a **code defect**.
|
|
130
|
+
- `"blocked"`: Cannot continue testing. This covers BOTH environment issues (e.g. SSO login redirect, connection refused, page not loading) **AND permission-denied / no-access states** (e.g. page shows "您没有...编辑权限", "无访问权限", "暂无数据", "申请体验权限") where the **test account lacks the required access**. A permission-denied page is a **precondition issue**, not a code defect — use `"blocked"`, NOT `"fail"`.
|
|
131
|
+
|
|
132
|
+
**Permission-denied → blocked (critical rule):**
|
|
133
|
+
When the screenshot or accessibility tree shows a permission-denied / no-access message (e.g. "您没有数据知识库[...]的编辑权限", "无访问权限", "暂无数据", "申请体验权限"), this means the **test account does not have the required permission for the target resource**. This is a precondition issue, NOT an application bug.
|
|
134
|
+
- Call `done(verdict="blocked")` immediately with the permission message in `text`.
|
|
135
|
+
- Do NOT call `done(verdict="fail")` — the application is correctly enforcing access control, the code is NOT defective.
|
|
136
|
+
- Do NOT try to navigate away to find another resource or "work around" the permission issue — the test must verify the **target resource specified in the precondition**, not a substitute.
|
|
131
137
|
- `text`: Describe what you observed and why you reached this conclusion.
|
|
132
138
|
|
|
133
139
|
Important:
|
|
134
140
|
- Set `verdict` to `"pass"` ONLY if the expected state is fully achieved. Be honest and rigorous.
|
|
135
141
|
- Set `verdict` to `"fail"` when the page works but does not match the expected state. Describe the mismatch in `text`.
|
|
136
|
-
- Set `verdict` to `"blocked"` when the environment prevents testing (
|
|
142
|
+
- Set `verdict` to `"blocked"` when the environment prevents testing OR when the page shows a permission-denied / no-access state (test account lacks required access). Not the application's fault.
|
|
137
143
|
- Do NOT call done with verdict="pass" if the screenshot shows the expected state is NOT achieved.
|
|
138
144
|
- **When to stop immediately**: If `execute_javascript` or another deterministic tool returns a result that definitively proves the expected state CANNOT be achieved (e.g. computed style value mismatch), call `done` with the appropriate verdict right away. Do NOT continue exploring, scrolling, or navigating — more actions will not change a CSS computed value or a DOM property. Wasting steps on already-failed assertions reduces test reliability.
|
|
139
145
|
- **When prior_execution shows the conclusion**: If the `<prior_execution>` block shows an assertion failure with `Expected:` and `Received:` values, the deterministic script has already captured the precise value. You may use `execute_javascript` once to confirm the current value, then immediately call `done(verdict="fail")` with the actual vs expected values in `text`. Do NOT explore further — the assertion already proved the mismatch.
|
package/dist/page-agent/tools.js
CHANGED
|
@@ -22,9 +22,9 @@ function tool(options) {
|
|
|
22
22
|
exports.tools = new Map();
|
|
23
23
|
// --- done ---
|
|
24
24
|
exports.tools.set('done', tool({
|
|
25
|
-
description: 'Complete the task with a test conclusion. Use this when the expected state is achieved, not achieved, or you are blocked.',
|
|
25
|
+
description: 'Complete the task with a test conclusion. Use this when the expected state is achieved, not achieved, or you are blocked. When the page shows a permission-denied / no-access message (e.g. "您没有...编辑权限", "无访问权限", "暂无数据") that is NOT caused by an application bug but by the test account lacking the required permission, use verdict="blocked" — this is a precondition issue, not a code defect. Only use verdict="fail" when the page IS accessible/functional but the expected UI state is not met.',
|
|
26
26
|
inputSchema: zod_1.z.object({
|
|
27
|
-
verdict: zod_1.z.enum(['pass', 'fail', 'blocked']).describe('Test verdict: "pass"=expected state achieved; "fail"=page accessible but expected not met; "blocked"=cannot continue
|
|
27
|
+
verdict: zod_1.z.enum(['pass', 'fail', 'blocked']).describe('Test verdict: "pass"=expected state achieved; "fail"=page accessible but expected not met (code defect); "blocked"=cannot continue — includes SSO redirect, connection refused, AND permission-denied/no-access pages where the test account lacks the required access (precondition not met, NOT a code defect)'),
|
|
28
28
|
text: zod_1.z.string().describe('Summary of what was achieved, why it failed, or why blocked'),
|
|
29
29
|
}),
|
|
30
30
|
execute: async function (input) {
|
package/package.json
CHANGED