@kkeve/suite 0.0.6 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/config.js CHANGED
@@ -71,7 +71,7 @@ const DEFAULT_CONFIG = {
71
71
  timeout_ms: 1200000,
72
72
  retries: 0,
73
73
  reporter: 'json',
74
- screenshot: 'only-on-failure',
74
+ screenshot: 'off',
75
75
  },
76
76
  };
77
77
  /**
@@ -97,6 +97,8 @@ export interface StepReportItem {
97
97
  precondition?: string;
98
98
  order?: number;
99
99
  success?: boolean;
100
+ /** fn 执行前截图路径(goal-before,页面原始状态) */
101
+ goalScreenshotBefore?: string;
100
102
  /** 目标结果截图路径(keveGoal 执行后截图,来自 goalScreenshotPath / goalScreenshotAfter) */
101
103
  goalScreenshotPath?: string;
102
104
  /** 验证截图路径列表(agent 每步操作后截图,来自 verifyScreenshots / agentScreenshots) */
@@ -564,6 +564,7 @@ async function generateReportData(opts) {
564
564
  order: s.order,
565
565
  success: s.success,
566
566
  conclusion: s.conclusion,
567
+ goalScreenshotBefore: s.goalScreenshotBefore,
567
568
  goalScreenshotPath: s.goalScreenshotPath || s.goalScreenshotAfter,
568
569
  verifyScreenshots: s.verifyScreenshots || s.agentScreenshots,
569
570
  actions: (s.actions || []).map((a) => ({
@@ -4,13 +4,11 @@
4
4
  * Three phases: before → (goal body or rescue) → after
5
5
  * User-invisible: aspects are registered by the framework, not by test authors.
6
6
  *
7
- * Built-in aspects:
8
- * - screenshot-on-enter (before, order=10) — capture screenshot before goal
9
- * - react-rescue (rescue, order=0) — Re-Act loop when fn() throws
10
- * - expected-check (after, order=0) — verify expected state via aiAssert
11
- * - action-log-write (after, order=10) — append to action-log.jsonl
12
- * - learnedActions-update (after, order=20) — update in-memory cache
13
- * - screenshot-on-fail (after, order=30) — capture screenshot on failure
7
+ * Registered aspects:
8
+ * - learnedActions-update (after, order=20) — update in-memory cache on success
9
+ *
10
+ * 截图由 keveGoal 内部直接管理(goal-before / goal-after / agent 每步截图),
11
+ * 不通过 aspect 注册,避免截图时机与 agent 执行步骤脱节。
14
12
  */
15
13
  export type AspectPhase = 'before' | 'rescue' | 'after';
16
14
  export interface GoalContext {
@@ -5,13 +5,11 @@
5
5
  * Three phases: before → (goal body or rescue) → after
6
6
  * User-invisible: aspects are registered by the framework, not by test authors.
7
7
  *
8
- * Built-in aspects:
9
- * - screenshot-on-enter (before, order=10) — capture screenshot before goal
10
- * - react-rescue (rescue, order=0) — Re-Act loop when fn() throws
11
- * - expected-check (after, order=0) — verify expected state via aiAssert
12
- * - action-log-write (after, order=10) — append to action-log.jsonl
13
- * - learnedActions-update (after, order=20) — update in-memory cache
14
- * - screenshot-on-fail (after, order=30) — capture screenshot on failure
8
+ * Registered aspects:
9
+ * - learnedActions-update (after, order=20) — update in-memory cache on success
10
+ *
11
+ * 截图由 keveGoal 内部直接管理(goal-before / goal-after / agent 每步截图),
12
+ * 不通过 aspect 注册,避免截图时机与 agent 执行步骤脱节。
15
13
  */
16
14
  Object.defineProperty(exports, "__esModule", { value: true });
17
15
  exports.keveAspect = exports.KeveAspectRegistry = void 0;
@@ -160,6 +160,13 @@ exports.test = test_1.test.extend({
160
160
  };
161
161
  // Run before aspects
162
162
  await keve_aspect_1.keveAspect.runPhase('before', ctx);
163
+ // ── Capture goal-before screenshot (fn 之前的页面原始状态) ──
164
+ // 语义:记录"测试动作发生前"的页面,供报告 before/after 对照
165
+ let goalScreenshotBefore = '';
166
+ try {
167
+ goalScreenshotBefore = await captureScreenshot(page, options.step, goalMeta.order, 'before');
168
+ }
169
+ catch { /* non-critical */ }
163
170
  let result;
164
171
  const fnSource = fn?.toString() || undefined;
165
172
  let fnError;
@@ -173,27 +180,21 @@ exports.test = test_1.test.extend({
173
180
  console.log(`[keveGoal] "${options.step}" fn error: ${fnError}`);
174
181
  }
175
182
  }
176
- // ── Capture "before" screenshot (page state before agent starts) ──
177
- let goalScreenshotBefore = '';
178
- try {
179
- const buf = await page.screenshot({ type: 'png', timeout: 15000 });
180
- const taskDir = process.env.KEVE_TASK_DIR || '.keve';
181
- const round = process.env.KEVE_ROUND || 'latest';
182
- const screenshotsDir = path.join(taskDir, 'test-artifacts', `round-${round}`, 'screenshots');
183
- fs.mkdirSync(screenshotsDir, { recursive: true });
184
- const safeName = options.step.replace(/[^a-zA-Z0-9\u4e00-\u9fff]/g, '_').slice(0, 30);
185
- const file = path.join(screenshotsDir, `goal-before-${goalMeta.order}-${safeName}-${Date.now()}.png`);
186
- fs.writeFileSync(file, buf);
187
- goalScreenshotBefore = path.relative(taskDir, file);
183
+ // ── Capture fn-after screenshot (fn 执行后的页面状态,传给 agent 说明执行前后) ──
184
+ let fnAfterScreenshot = '';
185
+ if (fn) {
186
+ try {
187
+ fnAfterScreenshot = await captureScreenshot(page, options.step, goalMeta.order, 'after-fn');
188
+ }
189
+ catch { /* non-critical */ }
188
190
  }
189
- catch { /* non-critical */ }
190
191
  // ── Hand off to agent: agent does Re-Act ──
191
192
  const learnedHint = learned_actions_1.learnedActions.getHint(options.step);
192
193
  let reactResult;
193
194
  let reactTimedOut = false;
194
195
  // ── Per-goal timeout: prevent one slow goal from exhausting the test timeout ──
195
- // Default 120s per goal; total test timeout should be ≥ (goals × 120s + overhead)
196
- const GOAL_TIMEOUT_MS = 120_000;
196
+ // Default 300s per goal; total test timeout should be ≥ (goals × 300s + overhead)
197
+ const GOAL_TIMEOUT_MS = 300_000;
197
198
  const goalTimeoutController = new AbortController();
198
199
  const goalTimeout = setTimeout(() => {
199
200
  goalTimeoutController.abort();
@@ -206,6 +207,8 @@ exports.test = test_1.test.extend({
206
207
  fnSource,
207
208
  fnResult: fnError ? { error: fnError } : { success: true },
208
209
  signal: goalTimeoutController.signal,
210
+ goalScreenshotBefore: goalScreenshotBefore || undefined,
211
+ fnAfterScreenshot: fnAfterScreenshot || undefined,
209
212
  });
210
213
  }
211
214
  catch (reactErr) {
@@ -250,22 +253,19 @@ exports.test = test_1.test.extend({
250
253
  // 通过 Playwright attachment 流到 KeveReporter.onTestEnd
251
254
  // 注意:screenshotBase64 不再写入 attachment(截图已保存到文件系统,路径见 goalScreenshotBefore/goalScreenshotAfter)
252
255
  const diagnosticHints = reactResult.diagnosticHints || [];
256
+ // ── goalScreenshotAfter = agent done 时的截图(done-time screenshot) ──
257
+ // agent.ts 在 done 工具触发时截图并写入 stepEvent.screenshotPath,
258
+ // 这里复用该路径作为 goal-after 证据(与 agent 判定时刻一致,避免状态漂移)。
253
259
  let goalScreenshotAfter = '';
254
260
  try {
255
- const buf = await page.screenshot({ type: 'png', timeout: 15000 });
256
- const taskDir = process.env.KEVE_TASK_DIR || '.keve';
257
- const round = process.env.KEVE_ROUND || 'latest';
258
- const screenshotsDir = path.join(taskDir, 'test-artifacts', `round-${round}`, 'screenshots');
259
- fs.mkdirSync(screenshotsDir, { recursive: true });
260
- const safeName = options.step.replace(/[^a-zA-Z0-9\u4e00-\u9fff]/g, '_').slice(0, 30);
261
- const file = path.join(screenshotsDir, `goal-after-${goalMeta.order}-${safeName}-${Date.now()}.png`);
262
- fs.writeFileSync(file, buf);
263
- goalScreenshotAfter = path.relative(taskDir, file);
264
- console.log(`[keveGoal] screenshot captured: ${buf.length} bytes (path: ${goalScreenshotAfter})`);
265
- }
266
- catch (e) {
267
- console.log(`[keveGoal] screenshot failed: ${e.message}`);
261
+ const lastActionWithShot = [...(reactResult.actions || [])]
262
+ .reverse()
263
+ .find((a) => a.screenshotPath);
264
+ if (lastActionWithShot?.screenshotPath) {
265
+ goalScreenshotAfter = lastActionWithShot.screenshotPath;
266
+ }
268
267
  }
268
+ catch { /* non-critical */ }
269
269
  await testInfo.attach('keveGoalResult', {
270
270
  contentType: 'application/json',
271
271
  body: Buffer.from(JSON.stringify({
@@ -362,3 +362,19 @@ exports.test = test_1.test.extend({
362
362
  // Register extended test with keve-decorators so @keveModel uses the correct test
363
363
  const keve_decorators_2 = require("./keve-decorators");
364
364
  (0, keve_decorators_2.setKeveTest)(exports.test);
365
+ // ── 辅助函数 ──
366
+ /**
367
+ * 截图并保存到 test-artifacts,返回相对 taskDir 的路径
368
+ * @param phase 'before' | 'after-fn' — 用于文件名前缀
369
+ */
370
+ async function captureScreenshot(page, stepName, order, phase) {
371
+ const buf = await page.screenshot({ type: 'png', timeout: 15000 });
372
+ const taskDir = process.env.KEVE_TASK_DIR || '.keve';
373
+ const round = process.env.KEVE_ROUND || 'latest';
374
+ const screenshotsDir = path.join(taskDir, 'test-artifacts', `round-${round}`, 'screenshots');
375
+ fs.mkdirSync(screenshotsDir, { recursive: true });
376
+ const safeName = stepName.replace(/[^a-zA-Z0-9一-鿿]/g, '_').slice(0, 30);
377
+ const file = path.join(screenshotsDir, `goal-${phase}-${order}-${safeName}-${Date.now()}.png`);
378
+ fs.writeFileSync(file, buf);
379
+ return path.relative(taskDir, file);
380
+ }
@@ -88,7 +88,7 @@ export default defineConfig({
88
88
  use: {
89
89
  baseURL: ${baseURLLine},${cdp ? `\n // CDP mode: login state preserved via connected Chrome` : `\n storageState: path.resolve(projectRoot, '.auth/storage-state.json'),`}
90
90
  trace: 'on-first-retry',
91
- screenshot: 'only-on-failure',
91
+ screenshot: 'off',
92
92
  video: 'retain-on-failure',
93
93
  actionTimeout: 5000,${cdpBlock}
94
94
  },
@@ -136,7 +136,7 @@ export default defineConfig({
136
136
  use: {
137
137
  baseURL: ${baseURLLine},${cdp ? `\n // CDP mode: login state preserved via connected Chrome` : `\n storageState: resolve(projectRoot, '.auth', 'storage-state.json'),`}
138
138
  trace: 'on-first-retry',
139
- screenshot: 'only-on-failure',
139
+ screenshot: 'off',
140
140
  video: 'retain-on-failure',
141
141
  actionTimeout: 5000,${cdpBlock}
142
142
  },
@@ -6,8 +6,10 @@
6
6
  * - MacroTool: merge all tool schemas into one LLM output
7
7
  * - History events: step/observation/error stream
8
8
  * - Reflection-before-action: evaluation + memory + next_goal + action
9
- * - Screenshot injected into LLM prompt as multimodal input — LLM self-evaluates visually
10
- * - No external verification — LLM sees screenshot each step, judges goal state itself
9
+ * - Per-step: ariaSnapshot text only (no screenshot) — Option C
10
+ * - Initial screenshots (goal-before + fn-after) injected on step 0 as multimodal context
11
+ * - Done-time screenshot captured as goal-after evidence
12
+ * - On-demand screenshots via visual_locate tool (when element needs visual identification)
11
13
  *
12
14
  * Usage:
13
15
  * const agent = new KevePageAgent(page, { maxSteps: 5 });
@@ -68,6 +70,10 @@ export interface AgentOptions {
68
70
  hooks?: AgentHooks;
69
71
  /** External abort signal (e.g. per-goal timeout). When aborted, internal abortController is also triggered. */
70
72
  signal?: AbortSignal;
73
+ /** goal-before 截图路径(相对 taskDir)— fn 执行前的页面原始状态 */
74
+ goalScreenshotBefore?: string;
75
+ /** fn-after 截图路径(相对 taskDir)— fn 执行后的页面状态 */
76
+ fnAfterScreenshot?: string;
71
77
  }
72
78
  export declare class KevePageAgent {
73
79
  readonly page: Page;
@@ -93,6 +99,8 @@ export declare class KevePageAgent {
93
99
  private buildMacroTool;
94
100
  /** Save screenshot buffer to test artifacts (reuses already-captured buffer) */
95
101
  private saveScreenshotBuffer;
102
+ /** Load initial screenshots (goal-before + fn-after) as ContentItem[] for step 0 multimodal input */
103
+ private loadInitialScreenshots;
96
104
  /**
97
105
  * Visual element location: uses LLM multimodal to identify element bbox in screenshot.
98
106
  * Returns normalized bbox (0-1000 scale) for coordinate-based clicking.
@@ -120,6 +128,8 @@ export declare function reactLoop(page: Page, step: string, expected: string, op
120
128
  success?: boolean;
121
129
  };
122
130
  signal?: AbortSignal;
131
+ goalScreenshotBefore?: string;
132
+ fnAfterScreenshot?: string;
123
133
  }): Promise<{
124
134
  actions: any[];
125
135
  expectedMet: boolean;
@@ -7,8 +7,10 @@
7
7
  * - MacroTool: merge all tool schemas into one LLM output
8
8
  * - History events: step/observation/error stream
9
9
  * - Reflection-before-action: evaluation + memory + next_goal + action
10
- * - Screenshot injected into LLM prompt as multimodal input — LLM self-evaluates visually
11
- * - No external verification — LLM sees screenshot each step, judges goal state itself
10
+ * - Per-step: ariaSnapshot text only (no screenshot) — Option C
11
+ * - Initial screenshots (goal-before + fn-after) injected on step 0 as multimodal context
12
+ * - Done-time screenshot captured as goal-after evidence
13
+ * - On-demand screenshots via visual_locate tool (when element needs visual identification)
12
14
  *
13
15
  * Usage:
14
16
  * const agent = new KevePageAgent(page, { maxSteps: 5 });
@@ -133,37 +135,19 @@ class KevePageAgent {
133
135
  await hooks?.onBeforeStep?.(this, stepCount);
134
136
  try {
135
137
  console.group(`step: ${stepCount}`);
136
- // ── Observe: get browser state ──
138
+ // ── Observe: get browser state (ariaSnapshot only — no per-step screenshot) ──
137
139
  console.log('\x1b[34m\x1b[1m👀 Observing...\x1b[0m');
138
140
  const snapshot = await this.page.ariaSnapshot({ mode: 'ai' });
139
141
  const url = this.page.url();
140
142
  console.log(` url: ${url.slice(0, 120)} | step ${stepCount + 1}/${maxSteps}`);
141
- // ── Capture screenshot (used for both LLM multimodal input and artifact save) ──
142
- let screenshotBuf;
143
- let screenshotBase64;
144
- try {
145
- screenshotBuf = await this.page.screenshot({ type: 'png', timeout: 10000 });
146
- screenshotBase64 = screenshotBuf.toString('base64');
147
- }
148
- catch (e) {
149
- // Retry once with longer timeout (font loading can be slow on some pages)
150
- try {
151
- await new Promise(r => setTimeout(r, 500));
152
- screenshotBuf = await this.page.screenshot({ type: 'png', timeout: 15000 });
153
- screenshotBase64 = screenshotBuf.toString('base64');
154
- }
155
- catch (e2) {
156
- console.log(`[agent] screenshot capture failed (non-critical): ${e2.message}`);
157
- }
158
- }
159
- // ── Assemble messages (multimodal: text + screenshot) ──
160
- const userText = this.assembleUserPrompt(step, expected, snapshot, url, stepCount, maxSteps, !!screenshotBase64);
143
+ // ── Assemble messages (text-only per step; initial screenshots on step 0) ──
144
+ const userText = this.assembleUserPrompt(step, expected, snapshot, url, stepCount, maxSteps);
161
145
  let userContent;
162
- if (screenshotBase64) {
163
- userContent = [
164
- { type: 'text', text: userText },
165
- { type: 'image_url', image_url: { url: `data:image/png;base64,${screenshotBase64}`, detail: 'high' } },
166
- ];
146
+ if (stepCount === 0) {
147
+ const initialImages = this.loadInitialScreenshots(options);
148
+ userContent = initialImages.length > 0
149
+ ? [{ type: 'text', text: userText }, ...initialImages]
150
+ : userText;
167
151
  }
168
152
  else {
169
153
  userContent = userText;
@@ -192,15 +176,7 @@ class KevePageAgent {
192
176
  console.log(`🎯: ${reflection.nextGoal.slice(0, 150)}`);
193
177
  const actionName = execResult.toolName;
194
178
  const actionInput = macroInput;
195
- // Save screenshot (reuse buffer already captured for LLM)
196
- let screenshotPath;
197
- if (screenshotBuf) {
198
- try {
199
- screenshotPath = this.saveScreenshotBuffer(screenshotBuf, stepCount, step);
200
- }
201
- catch { /* non-critical */ }
202
- }
203
- // Record step event
179
+ // Record step event (screenshotPath set later for done actions)
204
180
  if (execResult.error) {
205
181
  console.log(`\x1b[31m\x1b[1m${actionName} (error: ${execResult.error.slice(0, 80)})\x1b[0m`);
206
182
  }
@@ -221,15 +197,21 @@ class KevePageAgent {
221
197
  toolOutput: execResult.output,
222
198
  toolError: execResult.error,
223
199
  snapshot,
224
- screenshotPath,
200
+ screenshotPath: undefined,
225
201
  };
226
202
  this.events.push(stepEvent);
227
203
  // If execution had error, add observation
228
204
  if (execResult.error) {
229
205
  this.pushObservation(`Action "${actionName}" failed: ${execResult.error}. Try a different approach.`);
230
206
  }
231
- // ── Check: if "done", return directly with test conclusion ──
207
+ // ── Check: if "done", capture done-time screenshot + return with test conclusion ──
232
208
  if (actionName === 'done') {
209
+ // Capture done-time screenshot as goal-after evidence
210
+ try {
211
+ const doneBuf = await this.page.screenshot({ type: 'png', timeout: 15000 });
212
+ stepEvent.screenshotPath = this.saveScreenshotBuffer(doneBuf, stepCount, step);
213
+ }
214
+ catch { /* non-critical */ }
233
215
  // Extract conclusion: 4-level backoff
234
216
  let conclusion;
235
217
  const verdictField = actionInput?.verdict;
@@ -325,7 +307,7 @@ class KevePageAgent {
325
307
  pushObservation(content) {
326
308
  this.events.push({ type: 'observation', content });
327
309
  }
328
- assembleUserPrompt(step, expected, snapshot, url, stepIndex, maxSteps, hasScreenshot) {
310
+ assembleUserPrompt(step, expected, snapshot, url, stepIndex, maxSteps) {
329
311
  let prompt = '';
330
312
  // <agent_state>
331
313
  prompt += '<agent_state>\n';
@@ -343,6 +325,18 @@ class KevePageAgent {
343
325
  : '';
344
326
  prompt += `<prior_execution>\nA deterministic script was executed BEFORE your Re-Act loop.\n${resultLine}${sourceLine}\n</prior_execution>\n`;
345
327
  }
328
+ // <initial_screenshots>: only on first step, describe the before/after fn screenshots
329
+ if (stepIndex === 0 && (this.options?.goalScreenshotBefore || this.options?.fnAfterScreenshot)) {
330
+ prompt += '<initial_screenshots>\n';
331
+ if (this.options?.goalScreenshotBefore) {
332
+ prompt += 'goal-before: Screenshot of the page state BEFORE any test action (fn执行前的页面原始状态).\n';
333
+ }
334
+ if (this.options?.fnAfterScreenshot) {
335
+ prompt += 'fn-after: Screenshot of the page state AFTER the deterministic fn script executed (fn执行后的页面状态). Compare with goal-before to understand what fn changed.\n';
336
+ }
337
+ prompt += 'These screenshots are attached as images to give you visual context of the pre/post fn execution state. Use them to understand what fn did, then rely on the accessibility tree (ariaSnapshot) for per-step decisions.\n';
338
+ prompt += '</initial_screenshots>\n';
339
+ }
346
340
  prompt += '<step_info>\n';
347
341
  prompt += `Step ${stepIndex + 1} of ${maxSteps} max steps\n`;
348
342
  prompt += `Current URL: ${url}\n`;
@@ -370,9 +364,6 @@ class KevePageAgent {
370
364
  prompt += '<browser_state>\n';
371
365
  prompt += `Current URL: ${url}\n\n`;
372
366
  prompt += `Accessibility Tree (YAML):\n\`\`\`yaml\n${snapshot}\n\`\`\`\n`;
373
- if (hasScreenshot) {
374
- prompt += '\n<page_screenshot>\nA screenshot of the current page is provided as an image attached to this message. Use it as the primary visual source of truth for what is actually visible on the page. Cross-reference the screenshot with the accessibility tree to make accurate judgments — especially before calling done(success=true) to verify the expected state is truly achieved.\n</page_screenshot>\n';
375
- }
376
367
  prompt += '</browser_state>\n\n';
377
368
  return prompt;
378
369
  }
@@ -425,6 +416,25 @@ class KevePageAgent {
425
416
  fs.writeFileSync(file, buf);
426
417
  return path.relative(taskDir, file);
427
418
  }
419
+ /** Load initial screenshots (goal-before + fn-after) as ContentItem[] for step 0 multimodal input */
420
+ loadInitialScreenshots(options) {
421
+ const items = [];
422
+ const taskDir = process.env.KEVE_TASK_DIR || '.keve';
423
+ for (const relPath of [options?.goalScreenshotBefore, options?.fnAfterScreenshot]) {
424
+ if (!relPath)
425
+ continue;
426
+ try {
427
+ const absPath = path.resolve(taskDir, relPath);
428
+ const buf = fs.readFileSync(absPath);
429
+ items.push({
430
+ type: 'image_url',
431
+ image_url: { url: `data:image/png;base64,${buf.toString('base64')}`, detail: 'high' },
432
+ });
433
+ }
434
+ catch { /* non-critical */ }
435
+ }
436
+ return items;
437
+ }
428
438
  /**
429
439
  * Visual element location: uses LLM multimodal to identify element bbox in screenshot.
430
440
  * Returns normalized bbox (0-1000 scale) for coordinate-based clicking.
@@ -541,6 +551,8 @@ async function reactLoop(page, step, expected, options = {}) {
541
551
  fnSource: options.fnSource,
542
552
  fnResult: options.fnResult,
543
553
  signal: options.signal,
554
+ goalScreenshotBefore: options.goalScreenshotBefore,
555
+ fnAfterScreenshot: options.fnAfterScreenshot,
544
556
  });
545
557
  }
546
558
  catch (execErr) {
@@ -126,14 +126,20 @@ You must call the `done` action in one of these cases:
126
126
  The `done` action requires two fields:
127
127
  - `verdict`: Your test verdict, must be one of:
128
128
  - `"pass"`: The expected state described in <user_request> is fully achieved. Examine the screenshot carefully before deciding.
129
- - `"fail"`: The page is accessible but the expected state is NOT achieved (e.g. text mismatch, wrong element state, visual difference).
130
- - `"blocked"`: Cannot continue testing (e.g. SSO login redirect, connection refused, page not loading).
129
+ - `"fail"`: The page is accessible but the expected state is NOT achieved (e.g. text mismatch, wrong element state, visual difference). This indicates a **code defect**.
130
+ - `"blocked"`: Cannot continue testing. This covers BOTH environment issues (e.g. SSO login redirect, connection refused, page not loading) **AND permission-denied / no-access states** (e.g. page shows "您没有...编辑权限", "无访问权限", "暂无数据", "申请体验权限") where the **test account lacks the required access**. A permission-denied page is a **precondition issue**, not a code defect — use `"blocked"`, NOT `"fail"`.
131
+
132
+ **Permission-denied → blocked (critical rule):**
133
+ When the screenshot or accessibility tree shows a permission-denied / no-access message (e.g. "您没有数据知识库[...]的编辑权限", "无访问权限", "暂无数据", "申请体验权限"), this means the **test account does not have the required permission for the target resource**. This is a precondition issue, NOT an application bug.
134
+ - Call `done(verdict="blocked")` immediately with the permission message in `text`.
135
+ - Do NOT call `done(verdict="fail")` — the application is correctly enforcing access control, the code is NOT defective.
136
+ - Do NOT try to navigate away to find another resource or "work around" the permission issue — the test must verify the **target resource specified in the precondition**, not a substitute.
131
137
  - `text`: Describe what you observed and why you reached this conclusion.
132
138
 
133
139
  Important:
134
140
  - Set `verdict` to `"pass"` ONLY if the expected state is fully achieved. Be honest and rigorous.
135
141
  - Set `verdict` to `"fail"` when the page works but does not match the expected state. Describe the mismatch in `text`.
136
- - Set `verdict` to `"blocked"` when the environment prevents testing (not the application's fault).
142
+ - Set `verdict` to `"blocked"` when the environment prevents testing OR when the page shows a permission-denied / no-access state (test account lacks required access). Not the application's fault.
137
143
  - Do NOT call done with verdict="pass" if the screenshot shows the expected state is NOT achieved.
138
144
  - **When to stop immediately**: If `execute_javascript` or another deterministic tool returns a result that definitively proves the expected state CANNOT be achieved (e.g. computed style value mismatch), call `done` with the appropriate verdict right away. Do NOT continue exploring, scrolling, or navigating — more actions will not change a CSS computed value or a DOM property. Wasting steps on already-failed assertions reduces test reliability.
139
145
  - **When prior_execution shows the conclusion**: If the `<prior_execution>` block shows an assertion failure with `Expected:` and `Received:` values, the deterministic script has already captured the precise value. You may use `execute_javascript` once to confirm the current value, then immediately call `done(verdict="fail")` with the actual vs expected values in `text`. Do NOT explore further — the assertion already proved the mismatch.
@@ -22,9 +22,9 @@ function tool(options) {
22
22
  exports.tools = new Map();
23
23
  // --- done ---
24
24
  exports.tools.set('done', tool({
25
- description: 'Complete the task with a test conclusion. Use this when the expected state is achieved, not achieved, or you are blocked.',
25
+ description: 'Complete the task with a test conclusion. Use this when the expected state is achieved, not achieved, or you are blocked. When the page shows a permission-denied / no-access message (e.g. "您没有...编辑权限", "无访问权限", "暂无数据") that is NOT caused by an application bug but by the test account lacking the required permission, use verdict="blocked" — this is a precondition issue, not a code defect. Only use verdict="fail" when the page IS accessible/functional but the expected UI state is not met.',
26
26
  inputSchema: zod_1.z.object({
27
- verdict: zod_1.z.enum(['pass', 'fail', 'blocked']).describe('Test verdict: "pass"=expected state achieved; "fail"=page accessible but expected not met; "blocked"=cannot continue (SSO, connection refused, etc.)'),
27
+ verdict: zod_1.z.enum(['pass', 'fail', 'blocked']).describe('Test verdict: "pass"=expected state achieved; "fail"=page accessible but expected not met (code defect); "blocked"=cannot continue — includes SSO redirect, connection refused, AND permission-denied/no-access pages where the test account lacks the required access (precondition not met, NOT a code defect)'),
28
28
  text: zod_1.z.string().describe('Summary of what was achieved, why it failed, or why blocked'),
29
29
  }),
30
30
  execute: async function (input) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@kkeve/suite",
3
- "version": "0.0.6",
3
+ "version": "0.0.9",
4
4
  "description": "AI-powered full-stack test agent CLI - static analysis, unit test, component test, E2E test, execute, evaluate, and report",
5
5
  "license": "MIT",
6
6
  "bin": {