@kkeve/suite 0.0.11 → 0.0.12

package/dist/core/generateReport/reportData.d.ts

@@ -165,11 +165,13 @@ export interface CaseReportItem {
     stdout: string;
     /** Playwright 步骤 */
     pwSteps: any[];
-    /** AI评估结论: "通过" | "不通过" | "待确认" */
+    /** AI评估结论: "通过" | "不通过" | "阻塞" | "跳过" */
     data: string;
     /** AI评估置信度 0-100 */
     confidence: number;
-    /** AI评估推理过程 */
+    /** AI评估 thought（原始推理/原因文本，用于根因展示） */
+    thought: string;
+    /** AI评估推理过程（同 thought，保持向后兼容） */
     confidenceReason: string;
     /** 步骤截图路径列表（前端按需加载，不再内嵌 base64） */
     stepScreenshots: {

package/dist/core/generateReport/reportData.js

@@ -437,7 +437,7 @@ async function generateReportData(opts) {
     // Derive taskDir from resultsPath: .../test-artifacts/round-N/test-results.json → taskDir is 2 levels up from test-artifacts/
     const taskDir = path.resolve(path.dirname(TEST_RESULTS_JSON), '..', '..');
     const caseResultMap = buildCaseResultMap(specs, taskDir);
-    let total = 0, passed = 0, failed = 0, skipped = 0;
+    let total = 0;
     const caseResults = [];
     const confidenceDist = { '95-100': 0, '61-90': 0, '41-60': 0, '0-40': 0 };
     // Read confidence-data.jsonl (AI evaluation results from KeveReporter)
@@ -516,12 +516,6 @@ async function generateReportData(opts) {
         total++;
         // Playwright returns "timedOut" for test timeout — treat as failed
         const effectiveStatus = result.status === 'timedOut' ? 'failed' : result.status;
-        if (effectiveStatus === 'passed')
-            passed++;
-        else if (effectiveStatus === 'skipped')
-            skipped++;
-        else
-            failed++;
         // AI evaluation: match by Playwright title (scene.title, the canonical key)
         const aiEntry = aiConfidenceMap[result.title] || aiConfidenceMap[caseId];
         const data = effectiveStatus === 'skipped' ? '待确认' : (aiEntry?.data || '待确认');
@@ -652,6 +646,7 @@ async function generateReportData(opts) {
             stdout: result.stdout,
             data,
             confidence,
+            thought: aiEntry?.thought || (effectiveStatus === 'skipped' ? '用例被跳过（非自动执行）' : ''),
             confidenceReason,
             stepScreenshots: stepScreenshotsPaths,
             screenshotPath: result.screenshotPath,
@@ -668,7 +663,6 @@ async function generateReportData(opts) {
             if (caseResultMap[caseDef.id])
                 continue; // Already included above
             total++;
-            failed++;
             // Missing cases: test was not executed by Playwright, so AI evaluation is skipped
             const data = '跳过';
             const confidence = 0;
@@ -692,6 +686,7 @@ async function generateReportData(opts) {
                 stdout: '',
                 data,
                 confidence,
+                thought: 'YAML中定义但Playwright未执行',
                 confidenceReason: 'YAML中定义但Playwright未执行，无AI评估数据',
                 stepScreenshots: [],
                 screenshotPath: '',
@@ -702,15 +697,16 @@ async function generateReportData(opts) {
             });
         }
     }
-    const passRate = total > 0 ? ((passed / total) * 100).toFixed(1) : '0';
     const summaryJson = {
-        status: failed > 0 ? 'partial' : 'success',
+        status: 'partial',
         mr_id: testCasesData.mr_id, description: testCasesData.description,
         docs_url: testCasesData.docs_url,
         generated_at: new Date().toISOString().split('T')[0],
         round: currentRound,
         phases: { phase1: 'completed', phase2: 'completed', phase3: 'completed', phase4: 'completed' },
-        summary: { total, passed, failed, skipped, pending: 0, passRate: `${passRate}%` },
+        // passed/failed/skipped/passRate 已移至前端 computedSummary 实时计算
+        // 保留 total 供兜底展示
+        summary: { total, round: currentRound },
         confidenceDist: {
             '95-100%可信': confidenceDist['95-100'],
             '61-90%需分析': confidenceDist['61-90'],

package/dist/core/keve-registry.d.ts

@@ -12,7 +12,7 @@ export interface KeveGoalMeta {
     order: number;
 }
 /** 错误分类类型 */
-export type ErrorCategory = 'script' | 'env' | 'assert' | 'visual' | 'text-mismatch' | 'react-fail' | 'pass' | 'unknown';
+export type ErrorCategory = 'script' | 'env' | 'assert' | 'visual' | 'text-mismatch' | 'incomplete' | 'pass' | 'unknown';
 /** 场景执行后的评估元数据，由 @keveScene 装饰器写入，Reporter 读取 */
 export interface KeveEvalMeta {
     /** 错误分类 */

package/dist/core/keve-report.js

@@ -76,7 +76,7 @@ class KeveReporter {
         visual: 80, // 视觉断言已评估，置信度较高
         'text-mismatch': 70, // UI文案格式不匹配（功能逻辑正确，文案措辞变了）
         assert: 50, // Playwright 断言失败，需人工确认
-        'react-fail': 40, // AI 探索未达预期，需人工确认
+        incomplete: 0, // 执行被中断（超时/action 耗尽），无法得出结论
         env: 0, // 环境问题，测试结果不可信
         unknown: 0, // 未知异常，无参考价值
     };
@@ -89,8 +89,18 @@ class KeveReporter {
         let skipAI = false;
         let classifiedMessage = '';
         if (result.status === 'skipped') {
-            skipAI = true;
-            effectiveCategory = 'unknown';
+            // 主动跳过（testInfo.skip，如"非自动执行"）— 独立处理，data='跳过'
+            this.appendRecord({
+                title: test.title,
+                data: '跳过',
+                confidence: 0,
+                thought: '用例被跳过（非自动执行）',
+                errorCategory: 'unknown',
+                steps,
+                diagnosticHint: undefined,
+            });
+            console.log(`[keve-reporter] ${test.title}: SKIPPED`);
+            return;
         }
         else if (result.status === 'passed') {
             effectiveCategory = 'pass';
@@ -107,6 +117,11 @@ class KeveReporter {
                 effectiveCategory = 'env';
                 skipAI = true;
             }
+            else if (agentConclusion.result === 'incomplete') {
+                // Agent 执行被中断（超时/action 耗尽），无法得出结论
+                effectiveCategory = 'incomplete';
+                skipAI = true;
+            }
             else {
                 // Agent said fail → assert/text-mismatch depending on diagnostic hints
                 const hint = buildDiagnosticHint(result.attachments);
@@ -145,10 +160,10 @@ class KeveReporter {
                 effectiveCategory = 'env';
                 skipAI = true;
             }
-            // 5. Re-Act loop 失败：AI 探索未达到预期（关键分类）
+            // 5. Re-Act loop 失败：执行被中断（action 耗尽 / 未达预期）→ incomplete
             else if (rawError.includes('Re-Act loop did not achieve') || rawError.includes('Expected not achieved after agent explore')) {
-                effectiveCategory = 'react-fail';
-                skipAI = false;
+                effectiveCategory = 'incomplete';
+                skipAI = true;
             }
             // 5.5 文案不匹配
             else if (rawError.includes('Expected not achieved')) {
@@ -158,8 +173,8 @@ class KeveReporter {
                     skipAI = false;
                 }
                 else {
-                    effectiveCategory = 'react-fail';
-                    skipAI = false;
+                    effectiveCategory = 'incomplete';
+                    skipAI = true;
                 }
             }
             // 6. 兜底
@@ -180,9 +195,13 @@ class KeveReporter {
                             ? `无法分类的异常(${classifiedMessage})，跳过AI评估`
                             : effectiveCategory === 'pass'
                                 ? `Agent判定通过但fn脚本执行失败(${classifiedMessage})，脚本需修复`
-                                : '用例被跳过';
-            // AI评估列显示"跳过"（表示AI未参与评估），而非"不通过"
-            const aiData = (effectiveCategory !== 'assert' && effectiveCategory !== 'react-fail' && effectiveCategory !== 'text-mismatch') ? '跳过' : '不通过';
+                                : effectiveCategory === 'incomplete'
+                                    ? `执行被中断(${classifiedMessage})，Agent未完成目标，建议重跑`
+                                    : `无法分类的异常(${classifiedMessage})`;
+            // AI评估列输出：
+            // - assert/text-mismatch → "不通过"（有明确失败结论，skipAI=false）
+            // - 其余所有无法得出结论的 → "阻塞"（包括 incomplete/env/script/visual/pass/unknown）
+            const aiData = (effectiveCategory === 'assert' || effectiveCategory === 'text-mismatch') ? '不通过' : '阻塞';
             const keveScreenshots = (0, reportData_1.parseKeveAsserts)(result.attachments).map(ka => ka.label);
             this.appendRecord({
                 title: test.title,
@@ -307,26 +326,26 @@ function parseSteps(attachments) {
 }
 /** Extract Agent conclusion (result + text) from keveGoalResult steps */
 function extractAgentConclusion(steps) {
-    // Walk steps in reverse to find the most recent conclusion
-    for (let i = steps.length - 1; i >= 0; i--) {
-        const step = steps[i];
-        // Step-level conclusion (from agent.ts buildResult → keveGoalResult attachment)
-        if (step.conclusion) {
-            // Find the done action text for context
-            const doneAction = step.actions?.find(a => a.tool === 'done');
-            return { result: step.conclusion, text: doneAction?.text || '' };
-        }
-        // Action-level conclusion (from done action's 'result' field)
-        if (step.actions) {
-            for (let j = step.actions.length - 1; j >= 0; j--) {
-                const action = step.actions[j];
-                if (action?.tool === 'done' && action?.conclusion) {
-                    return { result: action.conclusion, text: action.text || '' };
-                }
+    if (steps.length === 0)
+        return undefined;
+    // keveGoal 串行执行：取最后一步的结论作为用例整体结论
+    const lastStep = steps[steps.length - 1];
+    // 最后一步有明确结论（Agent 调用了 done）→ 直接使用
+    if (lastStep.conclusion) {
+        const doneAction = lastStep.actions?.find(a => a.tool === 'done');
+        return { result: lastStep.conclusion, text: doneAction?.text || '' };
+    }
+    // Action 级别结论兜底（来自 done action 的 result 字段）
+    if (lastStep.actions) {
+        for (let j = lastStep.actions.length - 1; j >= 0; j--) {
+            const action = lastStep.actions[j];
+            if (action?.tool === 'done' && action?.conclusion) {
+                return { result: action.conclusion, text: action.text || '' };
             }
         }
     }
-    return undefined;
+    // 最后一步无任何结论 → 执行被中断（超时 / action 耗尽）
+    return { result: 'incomplete', text: '执行被中断，Agent未完成目标' };
 }
 /** Build diagnosticHint from keveGoalResult attachments (primary) + fallback to keveDiagnosticHint */
 function buildDiagnosticHint(attachments) {

package/dist/core/keve-test.d.ts

@@ -39,5 +39,7 @@ type KeveFixture = {
         noImages?: boolean;
         outputPath?: string;
     }) => Promise<any>;
+    /** 将内置 testInfo 包装为可解构 fixture，允许脚本直接写 { page, keveGoal, testInfo } */
+    testInfo: import('@playwright/test').TestInfo;
 };
 export declare const test: import("@playwright/test").TestType<import("@playwright/test").PlaywrightTestArgs & import("@playwright/test").PlaywrightTestOptions & KeveFixture, import("@playwright/test").PlaywrightWorkerArgs & import("@playwright/test").PlaywrightWorkerOptions>;

package/dist/core/keve-test.js

@@ -95,6 +95,11 @@ keve_aspect_1.keveAspect.register({
 // ─── (diagnostic extraction moved to page-agent/diagnostic.ts, called via onAfterTask hook) ──
 // ─── Fixture ────────────────────────────────────────────────────────
 exports.test = test_1.test.extend({
+    // ── 将内置 testInfo 包装为可解构 fixture ─────────────────────────
+    // 测试方法可直接写 { page, keveGoal, testInfo }，不再报 "unknown parameter" 错误
+    testInfo: async ({}, use, testInfo) => {
+        await use(testInfo);
+    },
     browser: async ({}, use, testInfo) => {
         if ((0, global_setup_1.isCdpMode)()) {
             const browser = await (0, global_setup_1.getCdpBrowser)(); // 复用全局共享的 CDP browser
@@ -170,6 +175,7 @@ exports.test = test_1.test.extend({
             let result;
             const fnSource = fn?.toString() || undefined;
             let fnError;
+            let fnBlocked = false; // true when fn error is environment-blocked (skip agent Re-Act)
             if (fn) {
                 try {
                     await fn();
@@ -178,6 +184,13 @@ exports.test = test_1.test.extend({
                 catch (err) {
                     fnError = err?.message || String(err);
                     console.log(`[keveGoal] "${options.step}" fn error: ${fnError}`);
+                    // ── Detect environment-blocked errors: skip Agent Re-Act, directly BLOCKED ──
+                    // When fn fails due to empty URL, connection refused, timeout, or navigation to invalid URL,
+                    // Agent exploration cannot fix environment issues — waste of time and tokens.
+                    fnBlocked = isEnvironmentBlockedError(err);
+                    if (fnBlocked) {
+                        console.log(`[keveGoal] 🚫 "${options.step}" BLOCKED — environment error detected, skipping agent Re-Act`);
+                    }
                 }
             }
             // ── Capture fn-after screenshot (fn 执行后的页面状态，传给 agent 说明执行前后) ──
@@ -188,46 +201,76 @@ exports.test = test_1.test.extend({
                 }
                 catch { /* non-critical */ }
             }
-            // ── Hand off to agent: agent does Re-Act ──
+            // ── Hand off to agent: agent does Re-Act (OR shortcut to blocked if fnBlocked) ──
             const learnedHint = learned_actions_1.learnedActions.getHint(options.step);
             let reactResult;
             let reactTimedOut = false;
-            // ── Per-goal timeout: prevent one slow goal from exhausting the test timeout ──
-            // Default 300s per goal; total test timeout should be ≥ (goals × 300s + overhead)
-            const GOAL_TIMEOUT_MS = 300_000;
-            const goalTimeoutController = new AbortController();
-            const goalTimeout = setTimeout(() => {
-                goalTimeoutController.abort();
-                console.log(`[keveGoal] ⏰ "${options.step}" timed out after ${GOAL_TIMEOUT_MS}ms — stopping agent`);
-            }, GOAL_TIMEOUT_MS);
-            try {
-                reactResult = await (0, agent_1.reactLoop)(page, options.step, options.expected, {
-                    learnedActionsHint: learnedHint,
-                    specFilePath: ctx.specFilePath,
-                    fnSource,
-                    fnResult: fnError ? { error: fnError } : { success: true },
-                    signal: goalTimeoutController.signal,
-                    goalScreenshotBefore: goalScreenshotBefore || undefined,
-                    fnAfterScreenshot: fnAfterScreenshot || undefined,
-                });
-            }
-            catch (reactErr) {
-                reactTimedOut = true;
-                const isGoalTimeout = goalTimeoutController.signal.aborted;
-                const msg = isGoalTimeout
-                    ? `Goal timed out (${GOAL_TIMEOUT_MS}ms)`
-                    : (reactErr?.message || String(reactErr));
-                console.log(`[keveGoal] reactLoop interrupted: ${msg.slice(0, 200)}`);
+            if (fnBlocked) {
+                // ── Shortcut: environment error detected in fn, skip Agent Re-Act entirely ──
+                // Agent exploration cannot fix empty URLs, connection refused, or navigation failures.
+                // Directly produce a blocked result with the fn error as explanation.
                 reactResult = {
                     expectedMet: false,
-                    actions: [],
+                    actions: [{
+                            action: { tool: 'done', verdict: 'blocked', text: `Environment blocked: ${fnError}` },
+                            toolOutput: '',
+                            result: 'ok',
+                        }],
                     finalSnapshot: '',
                     agentScreenshots: [],
-                    conclusion: isGoalTimeout ? 'blocked' : 'fail',
+                    conclusion: 'blocked',
                 };
             }
-            finally {
-                clearTimeout(goalTimeout);
+            else {
+                // ── Per-goal timeout: prevent one slow goal from exhausting the test timeout ──
+                // Default 300s per goal; total test timeout should be ≥ (goals × 300s + overhead)
+                const GOAL_TIMEOUT_MS = 300_000;
+                const goalTimeoutController = new AbortController();
+                const goalTimeout = setTimeout(() => {
+                    goalTimeoutController.abort();
+                    console.log(`[keveGoal] ⏰ "${options.step}" timed out after ${GOAL_TIMEOUT_MS}ms — stopping agent`);
+                }, GOAL_TIMEOUT_MS);
+                try {
+                    reactResult = await (0, agent_1.reactLoop)(page, options.step, options.expected, {
+                        learnedActionsHint: learnedHint,
+                        specFilePath: ctx.specFilePath,
+                        fnSource,
+                        fnResult: fnError ? { error: fnError } : { success: true },
+                        signal: goalTimeoutController.signal,
+                        goalScreenshotBefore: goalScreenshotBefore || undefined,
+                        fnAfterScreenshot: fnAfterScreenshot || undefined,
+                    });
+                }
+                catch (reactErr) {
+                    reactTimedOut = true;
+                    const isGoalTimeout = goalTimeoutController.signal.aborted;
+                    const msg = isGoalTimeout
+                        ? `Goal timed out (${GOAL_TIMEOUT_MS}ms)`
+                        : (reactErr?.message || String(reactErr));
+                    console.log(`[keveGoal] reactLoop interrupted: ${msg.slice(0, 200)}`);
+                    reactResult = {
+                        expectedMet: false,
+                        actions: [],
+                        finalSnapshot: '',
+                        agentScreenshots: [],
+                        conclusion: isGoalTimeout ? 'blocked' : 'fail',
+                    };
+                }
+                finally {
+                    clearTimeout(goalTimeout);
+                }
+            } // end of else (non-blocked path)
+            // ── Deterministic override: fn assertion failure cannot be overridden by Agent "pass" ──
+            // When fn has a hard assertion failure (Expected X, Received Y), the value mismatch
+            // is a fact, not a judgment. The Agent may rationalize "consistency" (e.g. "both pages
+            // show 15px, so consistent") and override to pass, but 15px ≠ 16px is a definitive failure.
+            // This is the per-goal equivalent of pipeline afterHook (case-gen-validate) —
+            // deterministic check > LLM judgment.
+            if (fnError && !fnBlocked && reactResult.expectedMet && isAssertionFailure(fnError)) {
+                console.log(`[keveGoal] ⚠️ "${options.step}" Agent said PASS but fn had assertion failure — overriding to FAIL`);
+                console.log(`[keveGoal]    fn assertion: ${fnError}`);
+                reactResult.expectedMet = false;
+                reactResult.conclusion = 'fail';
             }
             result = {
                 success: reactResult.expectedMet,
@@ -236,8 +279,10 @@ exports.test = test_1.test.extend({
                 error: reactResult.expectedMet
                     ? undefined
                     : reactResult.conclusion === 'blocked'
-                        ? new Error(`Blocked: ${reactTimedOut ? `Agent 超时 (${GOAL_TIMEOUT_MS}ms)` : (reactResult.actions?.filter((a) => a.action?.tool === 'done').pop()?.action?.text || 'Agent blocked')}`)
-                        : new Error(`Expected not achieved: ${options.expected}`),
+                        ? new Error(`Blocked: ${fnBlocked ? fnError : (reactTimedOut ? `Agent 超时 (300000ms)` : (reactResult.actions?.filter((a) => a.action?.tool === 'done').pop()?.action?.text || 'Agent blocked'))}`)
+                        : (fnError && isAssertionFailure(fnError) && !fnBlocked)
+                            ? new Error(`Assertion failed: ${fnError}`)
+                            : new Error(`Expected not achieved: ${options.expected}`),
             };
             // Attach Agent conclusion for downstream consumers
             if (reactResult.conclusion)
@@ -363,6 +408,109 @@ exports.test = test_1.test.extend({
 const keve_decorators_2 = require("./keve-decorators");
 (0, keve_decorators_2.setKeveTest)(exports.test);
 // ── 辅助函数 ──
+/**
+ * Detect environment-blocked errors from fn execution.
+ *
+ * When fn (deterministic script) fails with these patterns, Agent Re-Act cannot
+ * fix the issue — the error is environmental, not application-level:
+ *
+ * 1. Empty/undefined URL → page.goto(''), page.goto(undefined)
+ * 2. Connection refused / net::ERR_CONNECTION_REFUSED
+ * 3. Navigation timeout (page not reachable)
+ * 4. Invalid URL format
+ * 5. SSO redirect / auth failure (not the app's fault)
+ *
+ * Returns true → keveGoal skips Agent Re-Act and directly reports BLOCKED
+ * with the fn error as the blocking reason.
+ */
+function isEnvironmentBlockedError(err) {
+    const msg = (err?.message || String(err)).toLowerCase();
+    // Empty/undefined URL: page.goto('') or page.goto(undefined)
+    if (msg.includes('url must not be empty') || msg.includes('url is empty')
+        || msg.includes('invalid url') || msg.includes('url is undefined')
+        || msg.includes('navigation to ""') || msg.includes("navigation to ''")) {
+        return true;
+    }
+    // page.goto(undefined): Playwright type error "url: expected string, got undefined"
+    // This happens when process.env.PAGE_* is not set — environment config issue, not app issue
+    if ((msg.includes('expected string') || msg.includes('expected a string'))
+        && (msg.includes('got undefined') || msg.includes('got null') || msg.includes('received undefined'))) {
+        return true;
+    }
+    // Connection refused / network unreachable
+    if (msg.includes('err_connection_refused') || msg.includes('connection refused')
+        || msg.includes('net::err_connection') || msg.includes('err_name_not_resolved')
+        || msg.includes('err_address_unreachable') || msg.includes('err_internet_disconnected')
+        || msg.includes('err_connection_timed_out') || msg.includes('err_connection_reset')) {
+        return true;
+    }
+    // Navigation timeout (page not reachable in time)
+    if (msg.includes('navigation timeout of') || msg.includes('timeout of') && msg.includes('exceeded')
+        || msg.includes('page.goto: timeout') || msg.includes('navigating to') && msg.includes('timed out')) {
+        return true;
+    }
+    // Playwright locator/waitForSelector timeout — selector not found on page (fn script issue, not app crash)
+    // e.g. "Timeout 15000ms exceeded while waiting for locator('.selector') to be visible"
+    // This is a script-quality issue (wrong selector), not an env blocked error.
+    // Return false here — let Agent take over rather than treating as BLOCKED.
+    // Note: we intentionally do NOT return true; these should fall through to Agent Re-Act.
+    // IDC network segment restriction
+    if (msg.includes('unable to access on the idc network segment')
+        || msg.includes('idc network segment') || msg.includes('40314')) {
+        return true;
+    }
+    // SSO / auth redirect (not the app's fault)
+    if (msg.includes('sso redirect') || msg.includes('login redirect detected')
+        || msg.includes('err_too_many_redirects')) {
+        return true;
+    }
+    return false;
+}
+/**
+ * Detect assertion failures from fn execution (Expected/Received patterns).
+ *
+ * When fn (deterministic script) throws a hard assertion failure with concrete
+ * Expected vs Received values, the mismatch is a *fact*, not a judgment.
+ * The Agent may rationalize "consistency" (e.g. "both pages show 15px → consistent → pass")
+ * and override to pass, but 15px ≠ 16px is a definitive failure.
+ *
+ * This function identifies such assertion failures so keveGoal can:
+ *   - Override Agent "pass" → "fail" when the fn proved a value mismatch
+ *   - Use the precise assertion message in the error (instead of generic "expected not achieved")
+ *
+ * Returns true for assertion errors, false for environment/runtime errors.
+ */
+function isAssertionFailure(error) {
+    const msg = (error || '').toLowerCase();
+    // Exclude Playwright API type errors (parameter type mismatch, not value assertion)
+    // e.g. "page.goto: url: expected string, got undefined" — this is an env config error,
+    // not a functional assertion failure. isEnvironmentBlockedError() handles it instead.
+    if (msg.includes('expected string') || msg.includes('expected a string')
+        || msg.includes('expected number') || msg.includes('expected boolean')) {
+        // Only exclude if it looks like an API parameter error (not an assertion)
+        if (msg.includes('got undefined') || msg.includes('got null')
+            || msg.includes('received undefined') || msg.includes('received null')) {
+            return false;
+        }
+    }
+    // Jest/Vitest/playwright assertion: "Expected: X, Received: Y"
+    if (msg.includes('expected') && msg.includes('received'))
+        return true;
+    // AssertionError / assert keyword
+    if (msg.includes('assertionerror') || msg.includes('assertion failed')
+        || msg.includes('assertion error'))
+        return true;
+    // expect(x).toBe(y) / expect(x).toEqual(y) pattern
+    // Note: must NOT match Playwright timeout messages like "Timeout 15000ms exceeded waiting for locator(...) to be visible"
+    // Those are env/timeout errors, not value assertion failures.
+    if ((msg.includes(' to be ') && !msg.includes('timeout') && !msg.includes('waiting for') && !msg.includes('exceeded'))
+        || msg.includes(' to equal ') || msg.includes(' to deeply equal '))
+        return true;
+    // Playwright expect: "expected string, received number" etc.
+    if (msg.includes('expected ') && (msg.includes(' but ') || msg.includes(' got ') || msg.includes(' received ')))
+        return true;
+    return false;
+}
 /**
  * 截图并保存到 test-artifacts，返回相对 taskDir 的路径
  * @param phase 'before' | 'after-fn' — 用于文件名前缀

package/dist/page-agent/agent.js

@@ -204,6 +204,18 @@ class KevePageAgent {
                     if (execResult.error) {
                         this.pushObservation(`Action "${actionName}" failed: ${execResult.error}. Try a different approach.`);
                     }
+                    // If execute_javascript returned a business-level rejection (❌ prefix),
+                    // re-inject the core rules so the LLM corrects its next attempt.
+                    if (actionName === 'execute_javascript' && execResult.output && execResult.output.startsWith('❌')) {
+                        const jsRulesHint = [
+                            'execute_javascript rules — fix your script:',
+                            '1. MUST start with `return` to capture the result.',
+                            '2. Use `var` for declarations — `const`/`let` cause SyntaxError.',
+                            '3. NO top-level `await` or `async () => {}` wrappers — they return undefined.',
+                            '4. NO Playwright APIs (`page`, `locator`) — only browser DOM APIs.',
+                        ].join('\n');
+                        this.pushObservation(jsRulesHint);
+                    }
                     // ── Check: if "done", capture done-time screenshot + return with test conclusion ──
                     if (actionName === 'done') {
                         // Capture done-time screenshot as goal-after evidence

package/dist/page-agent/system_prompt.md

@@ -108,6 +108,7 @@ Common pitfalls the screenshot helps catch:
 
 <browser_rules>
 Strictly follow these rules while using the browser:
+- **You are an observer, not an editor**: Your role is to VERIFY the page state, not to CHANGE it. If the actual state differs from expected, report the mismatch — do NOT modify the page to make it match.
 - Use `ref` to target elements. Find the element by its role and name in the tree, then use its ref value.
 - If the page changes after an action, re-read the browser state before acting again.
 - If expected elements are missing from the visible area, use scroll to find them.
@@ -139,12 +140,30 @@ When the screenshot or accessibility tree shows a permission-denied / no-access
 Important:
 - Set `verdict` to `"pass"` ONLY if the expected state is fully achieved. Be honest and rigorous.
 - Set `verdict` to `"fail"` when the page works but does not match the expected state. Describe the mismatch in `text`.
-- Set `verdict` to `"blocked"` when the environment prevents testing OR when the page shows a permission-denied / no-access state (test account lacks required access). Not the application's fault.
+- `verdict` to `"blocked"` when the environment prevents testing OR when the page shows a permission-denied / no-access state (test account lacks required access). Not the application's fault.
+- **Environment-blocked → blocked (critical rule):**
+When the `navigate` tool returns an error indicating a missing/empty environment variable (e.g. "❌ Cannot navigate: environment variable PAGE_BI is empty/undefined"), this means the test data is incomplete — the URL for the target page was never provided. Agent exploration CANNOT fix this.
+- Call `done(verdict="blocked")` immediately with the missing variable name in `text`.
+- Do NOT try to navigate to the main page or any alternative URL as a workaround — the test requires the specific page defined by the missing env var.
+- Do NOT continue exploring — the missing URL is a configuration issue, not an application defect.
 - Do NOT call done with verdict="pass" if the screenshot shows the expected state is NOT achieved.
 - **When to stop immediately**: If `execute_javascript` or another deterministic tool returns a result that definitively proves the expected state CANNOT be achieved (e.g. computed style value mismatch), call `done` with the appropriate verdict right away. Do NOT continue exploring, scrolling, or navigating — more actions will not change a CSS computed value or a DOM property. Wasting steps on already-failed assertions reduces test reliability.
 - **When prior_execution shows the conclusion**: If the `<prior_execution>` block shows an assertion failure with `Expected:` and `Received:` values, the deterministic script has already captured the precise value. You may use `execute_javascript` once to confirm the current value, then immediately call `done(verdict="fail")` with the actual vs expected values in `text`. Do NOT explore further — the assertion already proved the mismatch.
 - **execute_javascript is ONLY for reading precise values**: Use it to query computed styles, DOM properties, or measurements that screenshots/accessibility tree cannot provide. Do NOT use it for clicking, typing, navigating, form filling, or any page interaction — use the dedicated tools (click, type, navigate) instead. Do NOT use it to repeat what the accessibility tree already shows. Do NOT use it for logic/assertions — just return the raw value and judge the result yourself.
-- **execute_javascript runs in BROWSER DOM context — no Playwright APIs**: The script executes inside the browser page via `page.evaluate`. There is NO `page` object, NO `locator`, NO top-level `await`. Use ONLY browser DOM APIs: `document.querySelector`, `window.getComputedStyle`, etc. Use `var` for variable declarations — `const`/`let` cause SyntaxError. Do NOT wrap in `async () => {}` — it returns undefined.
+- **NEVER modify the page under test (CRITICAL RULE)**: You are a TESTER — your job is to OBSERVE and REPORT the actual state of the page, never to ALTER it to match expectations. If the actual value differs from the expected value, that is a TEST FAILURE — report it honestly as `done(verdict="fail")`. Do NOT use `execute_javascript`, `click`, `type`, or any other tool to "fix" the page so the test passes. Specifically:
+  - Do NOT inject CSS (style.setProperty, style overrides, CSS variable changes)
+  - Do NOT modify DOM structure (adding/removing elements)
+  - Do NOT change element attributes or classes
+  - Do NOT override computed styles to match expected values
+  - If `execute_javascript` returns a value that differs from expected, call `done(verdict="fail")` with the actual vs expected — do NOT attempt to change the value
+- **execute_javascript <rule>**:
+  1. Script MUST start with `return` — no auto-wrapping. Omitting `return` causes execution rejection.
+  2. Use `var` for variable declarations. `const`/`let` cause SyntaxError in page.evaluate context.
+  3. NO top-level `await` — the script runs in a synchronous IIFE. There is no async context.
+  4. NO `async () => {}` wrappers — they return undefined. Write flat statements ending with `return`.
+  5. NO Playwright APIs (`page`, `locator`, `expect`) — only browser DOM APIs exist inside the page.
+  6. Read-only: do NOT modify DOM, styles, cookies, or localStorage. Only query/return values. This includes style.setProperty() — it is BLOCKED. If the actual value differs from expected, report it as fail, do NOT override it.
+  If your script returns `❌`, the rules will be re-injected in the next step — fix the script immediately instead of trying other approaches.
 
 Example — correct use of execute_javascript + immediate done:
 ```

package/dist/page-agent/tools.js

@@ -131,8 +131,20 @@ exports.tools.set('navigate', tool({
     }),
     execute: async function (input) {
         let url = input.url;
+        // ── Resolve environment variable references (e.g., "process.env.PAGE_BI") ──
+        const envRefMatch = url.match(/^process\.env\.(\w+)$/);
+        if (envRefMatch) {
+            const envValue = process.env[envRefMatch[1]] || '';
+            if (!envValue) {
+                return `❌ Cannot navigate: environment variable ${envRefMatch[1]} is empty/undefined. This is a test data configuration issue — call done(verdict="blocked") immediately.`;
+            }
+            url = envValue;
+        }
         if (!url.startsWith('http://') && !url.startsWith('https://')) {
             const base = process.env.KEVE_TARGET_URL || process.env.BASE_URL || '';
+            if (!base) {
+                return `❌ Cannot navigate: no base URL available (KEVE_TARGET_URL and BASE_URL are both empty). Call done(verdict="blocked") immediately.`;
+            }
             url = new URL(url, base).href;
         }
         await this.page.goto(url, { waitUntil: 'networkidle', timeout: 15000 });
@@ -141,9 +153,9 @@ exports.tools.set('navigate', tool({
 }));
 // --- execute_javascript ---
 exports.tools.set('execute_javascript', tool({
-    description: 'READ-ONLY query tool: execute JavaScript in the BROWSER DOM context to retrieve precise values that screenshots and accessibility trees CANNOT provide — such as computed CSS styles (getComputedStyle), DOM measurements (offsetWidth, getBoundingClientRect), or element properties (checked, disabled, value).\n\n⚠️ EXECUTION CONTEXT: This runs inside the browser page via page.evaluate — it is a plain browser JavaScript sandbox. There is NO `page` object, NO Playwright API, NO `locator`, NO top-level `await`. Using `page.locator(...)`, `page.evaluate(...)`, `await`, or `async () => {}` wrappers will cause SyntaxError or return undefined.\n\nDo NOT use for: clicking, typing, navigating, form filling, or any page interaction — use click/type/navigate tools instead. Do NOT use for: assertions or logical checks — just return the raw value, you judge the result yourself. Do NOT use for: modifying page state (setting styles, cookies, localStorage, DOM mutations).',
+    description: 'READ-ONLY query tool: execute JavaScript in the BROWSER DOM context to retrieve precise values that screenshots and accessibility trees CANNOT provide — such as computed CSS styles (getComputedStyle), DOM measurements (offsetWidth, getBoundingClientRect), or element properties (checked, disabled, value).\n\nThis tool is STRICTLY READ-ONLY. You MUST NOT modify the page under test — you are a TESTER who observes and reports, never an editor who alters. If the actual value differs from expected, report it as fail; do NOT override styles/DOM to make it match.\n\n<rule>\n1. Script MUST start with `return` to capture the result. No auto-wrapping — if you omit `return`, execution is rejected.\n2. Use `var` for variable declarations. `const`/`let` cause SyntaxError in page.evaluate context.\n3. NO top-level `await` — the script runs in a synchronous IIFE inside page.evaluate. There is no async context.\n4. NO `async () => {}` wrappers — they return undefined. Write flat statements ending with `return`.\n5. NO Playwright APIs (`page`, `locator`, `expect`) — only browser DOM APIs exist inside the page.\n6. STRICTLY READ-ONLY: do NOT modify DOM, styles, cookies, or localStorage. This includes style.setProperty(), style.removeProperty(), className changes, setAttribute, and any DOM mutation. All write APIs are BLOCKED at execution level. Only query/return values.\n</rule>\n\nDo NOT use for: clicking, typing, navigating, form filling, or any page interaction — use click/type/navigate tools instead. Do NOT use for: assertions or logical checks — just return the raw value, you judge the result yourself.',
     inputSchema: zod_1.z.object({
-        script: zod_1.z.string().describe("JavaScript code to execute inside the browser DOM. MUST start with `return` to capture the result. Use `var` for variable declarations (NOT `const`/`let` — they cause SyntaxError in some evaluate contexts). Only browser DOM APIs: document.querySelector, window.getComputedStyle, etc.\n\n✅ Correct: 'return document.querySelector(\".x\").innerText'\n✅ Correct: 'var el = document.querySelector(\".x\"); return window.getComputedStyle(el).backgroundColor'\n❌ Wrong: 'page.locator(\".x\")' — no Playwright `page` object in browser context\n❌ Wrong: 'await page.locator(...)' — no `await` or `async`\n❌ Wrong: 'const el = ...' — use `var` instead of `const`/`let`\n❌ Wrong: 'async () => { ... }' — do NOT wrap in async function, returns undefined\n\nRead-only queries only."),
+        script: zod_1.z.string().describe("JavaScript to execute in browser DOM. <rule> MUST start with `return`. Use `var` (not `const`/`let`). No top-level `await` or `async` wrappers. No Playwright APIs. STRICTLY READ-ONLY — no style.setProperty, no DOM mutation. </rule>\n\n✅ 'return document.querySelector(\".x\").innerText'\n✅ 'var el = document.querySelector(\".x\"); return window.getComputedStyle(el).backgroundColor'\n❌ 'page.locator(\".x\")' — no page object\n❌ 'await page.locator(...)' — no await\n❌ 'const el = ...' — use var\n❌ 'async () => { ... }' — returns undefined\n❌ 'document.documentElement.style.setProperty(...)' — WRITE, use getComputedStyle instead"),
     }),
     execute: async function (input) {
         try {
@@ -164,11 +176,15 @@ exports.tools.set('execute_javascript', tool({
                 script = iifeMatch[1].trim();
             }
             // Read-only guard: block DOM mutation APIs to prevent Agent from altering page state.
+            // CRITICAL: E2E testing must OBSERVE and REPORT the actual page state.
+            // Never modify the page under test to make assertions pass.
             const writePatterns = [
                 /\b(?:document|element|el|node)\.\s*(?:remove|appendChild|insertBefore|replaceChild|removeChild|appendChild)\s*\(/,
                 /\b(?:document|element|el|node)\.\s*(?:innerHTML|outerHTML|textContent|innerText)\s*=/,
                 /\b(?:document|element|el|node)\.\s*(?:setAttribute|removeAttribute|classList)\s*\./,
                 /\b\.style\s*\.\s*\w+\s*=/,
+                /\b\.style\s*\.\s*(?:setProperty|removeProperty)\s*\(/,
+                /\b(?:document|element|el|node)\.\s*className\s*=/,
                 /\blocation\s*(?:\.href\s*=|\.assign\s*\(|\.replace\s*\()/,
                 /\bwindow\s*\.\s*(?:close|stop|open)\s*\(/,
                 /\bdocument\.\s*(?:write|writeln|execCommand)\s*\(/,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kkeve/suite",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "description": "AI-powered full-stack test agent CLI - static analysis, unit test, component test, E2E test, execute, evaluate, and report",
   "license": "MIT",
   "bin": {