@kkeve/suite 0.0.1 → 0.0.2

package/dist/commands/run.js

@@ -0,0 +1,417 @@
+"use strict";
+/**
+ * keve run - Execute Playwright tests
+ *
+ * Wraps `npx playwright test` with round management and output organization.
+ * Does NOT hardcode reporter or resultDir — the Playwright config handles those.
+ * Only adds: round number management, retry-from/cases filtering, and post-run report generation.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KeveRunError = void 0;
+exports.run = run;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const os = __importStar(require("node:os"));
+const node_child_process_1 = require("node:child_process");
+const chalk_1 = __importDefault(require("chalk"));
+const config_1 = require("../config");
+const report_1 = require("../core/generateReport/report");
+const reportData_1 = require("../core/generateReport/reportData");
+function isPlaywrightBrowserInstalled() {
+    const cacheDir = os.platform() === 'darwin'
+        ? path.join(os.homedir(), 'Library/Caches/ms-playwright')
+        : os.platform() === 'win32'
+            ? path.join(process.env.LOCALAPPDATA || '', 'ms-playwright')
+            : path.join(os.homedir(), '.cache/ms-playwright');
+    if (!fs.existsSync(cacheDir))
+        return false;
+    try {
+        const entries = fs.readdirSync(cacheDir);
+        return entries.some(e => e.startsWith('chromium-'));
+    }
+    catch {
+        return false;
+    }
+}
+async function ensurePlaywrightBrowsers() {
+    if (isPlaywrightBrowserInstalled())
+        return;
+    console.log(chalk_1.default.yellow('  ⚠ Playwright browsers not found. Installing Chromium...'));
+    console.log(chalk_1.default.gray('    This is a one-time setup (~200MB download).'));
+    try {
+        (0, node_child_process_1.execSync)('npx playwright install chromium', {
+            stdio: 'inherit',
+            timeout: 300_000,
+        });
+        console.log(chalk_1.default.green('  ✓ Playwright browsers installed.'));
+    }
+    catch (err) {
+        console.error(chalk_1.default.red('  ✗ Failed to install Playwright browsers.'));
+        console.error(chalk_1.default.gray('    Please run manually: npx playwright install chromium'));
+        process.exit(1);
+    }
+}
+function findPlaywrightConfig(keveDir, optionsConfig) {
+    if (optionsConfig) {
+        const resolved = path.resolve(optionsConfig);
+        if (fs.existsSync(resolved))
+            return resolved;
+        console.error(chalk_1.default.red(`  ✗ Config not found: ${optionsConfig}`));
+        return null;
+    }
+    const taskDir = (0, config_1.resolveTaskDir)(keveDir);
+    const searchPaths = [
+        taskDir ? path.join(taskDir, 'keve-test.config.mjs') : null,
+        taskDir ? path.join(taskDir, 'keve-test.config.ts') : null,
+        keveDir ? path.join(keveDir, 'keve-test.config.mjs') : null,
+        keveDir ? path.join(keveDir, 'keve-test.config.ts') : null,
+        path.join(process.cwd(), 'keve_test_spec', 'keve-test.config.mjs'),
+        path.join(process.cwd(), 'keve_test_spec', 'keve-test.config.ts'),
+    ].filter(Boolean);
+    for (const p of searchPaths) {
+        if (fs.existsSync(p))
+            return p;
+    }
+    console.error(chalk_1.default.red('  ✗ Playwright config not found'));
+    console.error(chalk_1.default.gray('    Searched:'));
+    for (const p of searchPaths) {
+        console.error(chalk_1.default.gray(`      ${p}`));
+    }
+    console.error(chalk_1.default.gray('    Run: keve config  to create one'));
+    return null;
+}
+function collectFailedTitles(suite, titles) {
+    for (const spec of suite.specs || []) {
+        for (const test of spec.tests || []) {
+            if (test.status === 'failed' || test.status === 'timedOut') {
+                titles.push(spec.title || spec.name || '');
+            }
+        }
+    }
+    for (const child of suite.suites || []) {
+        collectFailedTitles(child, titles);
+    }
+}
+function escapeGrepPattern(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, String.raw `\$&`);
+}
+class KeveRunError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'KeveRunError';
+    }
+}
+exports.KeveRunError = KeveRunError;
+async function run(options) {
+    const keveDir = (0, config_1.findKeveDir)();
+    const cwd = process.cwd();
+    const configPath = findPlaywrightConfig(keveDir || '', options.config);
+    if (!configPath) {
+        throw new KeveRunError('Playwright config not found');
+    }
+    let pwConfig;
+    try {
+        pwConfig = await new Function('return import(' + JSON.stringify(configPath) + ')')();
+    }
+    catch (e) {
+        throw new KeveRunError(`Failed to import Playwright config: ${e.message}`);
+    }
+    const configDefault = pwConfig?.default || pwConfig;
+    const taskDir = process.env.KEVE_TASK_DIR
+        ? path.resolve(process.env.KEVE_TASK_DIR)
+        : (0, config_1.resolveTaskDir)(keveDir) || cwd;
+    let resultDir = '';
+    let resultsFile = '';
+    if (configDefault?.reporter) {
+        for (const r of configDefault.reporter) {
+            if (Array.isArray(r) && r[0] === 'json' && r[1]?.outputFile) {
+                resultsFile = path.resolve(r[1].outputFile);
+                resultDir = path.dirname(resultsFile);
+                break;
+            }
+        }
+    }
+    console.log(chalk_1.default.cyan('\n🚀 keve run\n'));
+    let round;
+    if (options.round) {
+        round = parseInt(options.round, 10);
+    }
+    else {
+        round = (0, config_1.getNextRoundFromState)(taskDir);
+    }
+    resultDir = path.join(taskDir, 'test-artifacts', `round-${round}`);
+    resultsFile = path.join(resultDir, 'test-results.json');
+    const pwArgs = ['playwright', 'test', '--config', configPath];
+    let effectiveGrep = options.grep || '';
+    if (options.retryFrom) {
+        const sourceRound = parseInt(options.retryFrom, 10);
+        const sourceResultDir = path.join(taskDir, 'test-artifacts', `round-${sourceRound}`);
+        const sourceResultsFile = path.join(sourceResultDir, 'test-results.json');
+        if (!fs.existsSync(sourceResultsFile)) {
+            throw new KeveRunError(`Round ${sourceRound} results not found: ${sourceResultsFile}`);
+        }
+        const sourceResults = JSON.parse(fs.readFileSync(sourceResultsFile, 'utf-8'));
+        const failedTitles = [];
+        for (const suite of sourceResults.suites || []) {
+            collectFailedTitles(suite, failedTitles);
+        }
+        if (failedTitles.length === 0) {
+            console.log(chalk_1.default.green(`  ✓ Round ${sourceRound} had no failures — nothing to retry`));
+            return {
+                exitCode: 0, resultsFile: sourceResultsFile, resultDir: sourceResultDir,
+                taskDir, round, passed: 0, failed: 0, skipped: 0,
+            };
+        }
+        const retryGrep = failedTitles.map(t => escapeGrepPattern(t)).join('|');
+        effectiveGrep = effectiveGrep ? `${effectiveGrep}|${retryGrep}` : retryGrep;
+        console.log(chalk_1.default.gray(`  Retrying ${failedTitles.length} failed cases from round ${sourceRound}`));
+    }
+    if (options.cases) {
+        const caseNames = options.cases.split(',').map(c => c.trim());
+        const casesGrep = caseNames.map(c => escapeGrepPattern(c)).join('|');
+        effectiveGrep = effectiveGrep ? `${effectiveGrep}|${casesGrep}` : casesGrep;
+        console.log(chalk_1.default.gray(`  Running specific cases: ${caseNames.join(', ')}`));
+    }
+    if (effectiveGrep) {
+        pwArgs.push('--grep', effectiveGrep);
+    }
+    const env = {
+        ...process.env,
+        KEVE_TASK_DIR: taskDir,
+        KEVE_ROUND: String(round),
+    };
+    if (!env.BASE_URL && !env.PWGEN_TARGET_URL) {
+        console.log(chalk_1.default.gray('  Note: BASE_URL not set, Playwright config baseURL will be used for relative paths'));
+    }
+    fs.mkdirSync(resultDir, { recursive: true });
+    if (options.verbose) {
+        console.log(chalk_1.default.gray(`  Round:     ${round}`));
+        console.log(chalk_1.default.gray(`  Config:    ${configPath}`));
+        console.log(chalk_1.default.gray(`  ResultDir: ${resultDir}`));
+        console.log(chalk_1.default.gray(`  Grep:      ${effectiveGrep || '(all)'}`));
+        console.log();
+    }
+    console.log(chalk_1.default.blue(`  ▶ Round ${round} — Executing Playwright tests...\n`));
+    await ensurePlaywrightBrowsers();
+    if (!env.KEVE_CDP_ENDPOINT && !env.KEVE_CDP_WS_ENDPOINT) {
+        env.KEVE_CDP_ENDPOINT = 'http://127.0.0.1:9222';
+    }
+    const exitCode = await new Promise((resolve) => {
+        const child = (0, node_child_process_1.spawn)('npx', pwArgs, {
+            cwd,
+            env,
+            stdio: ['inherit', 'pipe', 'pipe'],
+        });
+        child.stdout.on('data', (data) => process.stdout.write(data));
+        child.stderr.on('data', (data) => process.stderr.write(data));
+        child.on('close', (code) => resolve(code ?? 1));
+        child.on('error', () => resolve(1));
+    });
+    if (exitCode !== 0 && options.verbose) {
+        console.log(chalk_1.default.gray('  Playwright exited with errors (tests may have failed)'));
+    }
+    if (!fs.existsSync(resultsFile)) {
+        console.error(chalk_1.default.red('  ✗ test-results.json not found after execution'));
+        console.error(chalk_1.default.gray('    Expected at: ' + resultsFile));
+        console.error(chalk_1.default.gray('    Check that config has json reporter with outputFile pointing to resultDir'));
+        return {
+            exitCode: 1, resultsFile, resultDir, taskDir, round, passed: 0, failed: 0, skipped: 0,
+        };
+    }
+    const results = JSON.parse(fs.readFileSync(resultsFile, 'utf-8'));
+    const stats = results.stats || {};
+    const passed = stats.expected || (stats.passed || 0);
+    const failed = stats.unexpected || (stats.failed || 0);
+    const skipped = stats.skipped || 0;
+    console.log(chalk_1.default.green(`\n  ✓ Round ${round} complete`));
+    console.log(chalk_1.default.gray(`    Tests: ${stats.expected || 0} expected, ${stats.actual || 0} actual`));
+    console.log(chalk_1.default.gray(`    Passed: ${passed}, Failed: ${failed}, Skipped: ${skipped}`));
+    (0, config_1.completeRound)(taskDir, round, { passed, failed, skipped });
+    stripBase64FromTestResults(resultsFile, resultDir);
+    // ── 兜底：确保 report-data.json 始终存在 ──
+    const reportDataPath = path.join(resultDir, 'report-data.json');
+    if (!fs.existsSync(reportDataPath)) {
+        console.log(chalk_1.default.yellow('  ⚠ report-data.json missing — generating fallback report'));
+        try {
+            const confidencePath = path.join(resultDir, 'confidence-data.jsonl');
+            if (fs.existsSync(resultsFile) && fs.existsSync(confidencePath)) {
+                // 正常兜底：双文件都存在，调用 generateReportData 生成
+                const reportData = await (0, reportData_1.generateReportData)({
+                    projectRoot: taskDir,
+                    resultsPath: resultsFile,
+                    confidencePath,
+                });
+                fs.writeFileSync(reportDataPath, JSON.stringify(reportData, null, 2));
+                console.log(chalk_1.default.green(`  ✓ Fallback report-data.json generated at: ${reportDataPath}`));
+            }
+            else {
+                // 极端兜底：缺少 confidence-data.jsonl（Playwright crash 场景）
+                const fallbackReport = {
+                    round,
+                    taskDir,
+                    generatedBy: 'fallback',
+                    cases: [],
+                    warning: 'report-data.json generated by fallback — some test data may be incomplete',
+                };
+                // 尝试从 test-results.json 提取基础统计
+                if (fs.existsSync(resultsFile)) {
+                    try {
+                        const results = JSON.parse(fs.readFileSync(resultsFile, 'utf-8'));
+                        const stats = results.stats || {};
+                        fallbackReport.summary = {
+                            total: (stats.expected || 0) + (stats.unexpected || 0) + (stats.skipped || 0),
+                            passed: stats.expected || 0,
+                            failed: stats.unexpected || 0,
+                            skipped: stats.skipped || 0,
+                        };
+                    }
+                    catch { /* ignore parse error in fallback */ }
+                }
+                fs.writeFileSync(reportDataPath, JSON.stringify(fallbackReport, null, 2));
+                console.log(chalk_1.default.yellow(`  ⚠ Minimal fallback report-data.json generated (missing confidence data)`));
+            }
+            // 同步到 latest 目录 → symlink 指向最新 round（避免重复拷贝 34MB+）
+            const latestDir = path.join(taskDir, 'test-artifacts', 'latest');
+            if (latestDir !== resultDir) {
+                try {
+                    if (fs.existsSync(latestDir)) {
+                        const stat = fs.lstatSync(latestDir);
+                        if (stat.isSymbolicLink()) {
+                            fs.unlinkSync(latestDir);
+                        }
+                        else {
+                            fs.rmSync(latestDir, { recursive: true, force: true });
+                        }
+                    }
+                    fs.symlinkSync(path.basename(resultDir), latestDir, 'junction');
+                }
+                catch { /* ignore latest symlink error */ }
+            }
+        }
+        catch (err) {
+            console.error(chalk_1.default.red(`  ✗ Fallback report generation failed: ${err.message}`));
+        }
+    }
+    return { exitCode, resultsFile, resultDir, taskDir, round, passed, failed, skipped };
+}
+function stripBase64FromTestResults(resultsFile, resultDir) {
+    if (!fs.existsSync(resultsFile))
+        return;
+    try {
+        const data = JSON.parse(fs.readFileSync(resultsFile, 'utf-8'));
+        const screenshotsDir = path.join(resultDir, 'screenshots');
+        let extractedCount = 0;
+        function stripSuite(suite) {
+            for (const spec of suite.specs || []) {
+                for (const test of spec.tests || []) {
+                    for (const result of test.results || []) {
+                        for (const att of result.attachments || []) {
+                            if (att.body && typeof att.body === 'string' && att.contentType !== 'application/json') {
+                                const fileName = `${att.name || 'screenshot'}.png`;
+                                const filePath = path.join(screenshotsDir, fileName);
+                                fs.mkdirSync(path.dirname(filePath), { recursive: true });
+                                fs.writeFileSync(filePath, Buffer.from(att.body, 'base64'));
+                                delete att.body;
+                                att.path = path.relative(resultDir, filePath);
+                                extractedCount++;
+                            }
+                        }
+                    }
+                }
+            }
+            for (const child of suite.suites || []) {
+                stripSuite(child);
+            }
+        }
+        for (const suite of data.suites || []) {
+            stripSuite(suite);
+        }
+        fs.writeFileSync(resultsFile, JSON.stringify(data, null, 2));
+        if (extractedCount > 0) {
+            console.log(chalk_1.default.gray(`    Stripped ${extractedCount} base64 attachments from test-results.json`));
+        }
+    }
+    catch (err) {
+        console.log(chalk_1.default.gray(`    Note: Could not strip base64 from test-results.json: ${err.message}`));
+    }
+}
+async function runGenerateReport(resultDir, resultsFile, keveDir, taskDir) {
+    let casesPath = '';
+    const casesCandidates = [
+        path.join(process.cwd(), '.keve', 'test-cases.yaml'),
+        keveDir ? path.join(keveDir, 'test-cases.yaml') : '',
+        path.join(process.cwd(), 'keve_test_spec', 'test-cases.yaml'),
+    ];
+    for (const candidate of casesCandidates) {
+        if (candidate && fs.existsSync(candidate)) {
+            casesPath = candidate;
+            break;
+        }
+    }
+    if (!casesPath && taskDir) {
+        const taskCase = path.join(taskDir, 'test-cases.yaml');
+        if (fs.existsSync(taskCase))
+            casesPath = taskCase;
+    }
+    const projectRoot = process.cwd();
+    const confidenceFile = path.join(resultDir, 'confidence-data.jsonl');
+    console.log(chalk_1.default.gray('  Step 1: Generating report data...'));
+    try {
+        const { reportData } = await (0, report_1.generateReport)({
+            projectRoot,
+            resultsPath: resultsFile,
+            confidencePath: fs.existsSync(confidenceFile) ? confidenceFile : undefined,
+            casesPath: casesPath || undefined,
+        });
+        const roundDir = reportData.roundDir;
+        const latestDir = reportData.latestDir;
+        try {
+            if (fs.existsSync(latestDir))
+                fs.unlinkSync(latestDir);
+            fs.symlinkSync(path.basename(roundDir), latestDir, 'junction');
+        }
+        catch { /* non-critical */ }
+        console.log(chalk_1.default.green(`  ✓ Report: ${path.relative(process.cwd(), path.join(roundDir, 'report-data.json'))}`));
+    }
+    catch (err) {
+        console.log(chalk_1.default.yellow(`  ⚠ Report generation failed: ${err.message}`));
+        if (process.env.KEVE_VERBOSE)
+            console.error(err);
+    }
+}

package/dist/commands/test.d.ts

@@ -0,0 +1,12 @@
+/**
+ * keve test - Full pipeline: gen → run → evaluate → report
+ * Orchestrates the complete KEVE test loop with fix cycles.
+ */
+export interface TestOptions {
+    config?: string;
+    maxRounds?: string;
+    grep?: string;
+    verbose?: boolean;
+    inlineImages?: boolean;
+}
+export declare function test(options: TestOptions): Promise<void>;

package/dist/commands/test.js

@@ -0,0 +1,102 @@
+"use strict";
+/**
+ * keve test - Full pipeline: gen → run → evaluate → report
+ * Orchestrates the complete KEVE test loop with fix cycles.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.test = test;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const chalk_1 = __importDefault(require("chalk"));
+const config_1 = require("../config");
+const run_1 = require("./run");
+const report_1 = require("./report");
+async function test(options) {
+    const maxRounds = parseInt(options.maxRounds || '3', 10);
+    const keveDir = (0, config_1.findKeveDir)();
+    console.log(chalk_1.default.cyan('\n🔄 keve test — Full pipeline\n'));
+    console.log(chalk_1.default.gray(`  Max rounds: ${maxRounds}`));
+    for (let round = 1; round <= maxRounds; round++) {
+        console.log(chalk_1.default.blue(`\n━━━ Round ${round}/${maxRounds} ━━━\n`));
+        // Step 1: Run tests
+        console.log(chalk_1.default.yellow('  [1/2] Running tests...'));
+        try {
+            await (0, run_1.run)({
+                config: options.config,
+                grep: options.grep,
+                verbose: options.verbose,
+            });
+        }
+        catch (e) {
+            if (e instanceof run_1.KeveRunError) {
+                console.error(chalk_1.default.red(`  ✗ Run failed: ${e.message}`));
+                break;
+            }
+            throw e;
+        }
+        // Step 2: Generate report
+        console.log(chalk_1.default.yellow('  [2/2] Generating report...'));
+        await (0, report_1.report)({});
+        // Check if all passed — if so, we're done
+        if (keveDir) {
+            const summaryPath = path.join(keveDir, 'reports', 'latest', 'execution-summary.json');
+            if (fs.existsSync(summaryPath)) {
+                try {
+                    const summary = JSON.parse(fs.readFileSync(summaryPath, 'utf-8'));
+                    if (summary.summary?.failed === 0) {
+                        console.log(chalk_1.default.green(`\n  ✓ All tests passed in round ${round}!`));
+                        break;
+                    }
+                    console.log(chalk_1.default.yellow(`  ⚠ ${summary.summary.failed} failed, ${summary.summary.passed} passed`));
+                    // Check for D-class failures that could be auto-fixed
+                    const dClassFailures = (summary.failedCases || []).filter((c) => c.type?.startsWith('D'));
+                    if (dClassFailures.length === 0 || round >= maxRounds) {
+                        if (round >= maxRounds) {
+                            console.log(chalk_1.default.yellow(`\n  ⚠ Reached max rounds (${maxRounds})`));
+                        }
+                        break;
+                    }
+                    console.log(chalk_1.default.gray(`  → ${dClassFailures.length} D-class failures, will retry...`));
+                }
+                catch { /* ignore parse errors */ }
+            }
+        }
+    }
+    console.log(chalk_1.default.green('\n  ✓ Pipeline complete\n'));
+}

package/dist/commands/vitest.d.ts

@@ -0,0 +1,14 @@
+/**
+ * keve vitest - Execute Vitest unit/component tests
+ *
+ * Wraps `npx vitest` with keve-specific config and coverage support.
+ * Uses vitest.config.ts from keve_test_spec/ by default.
+ */
+export interface VitestOptions {
+    config?: string;
+    watch?: boolean;
+    coverage?: boolean;
+    filter?: string;
+    verbose?: boolean;
+}
+export declare function vitest(options: VitestOptions): Promise<void>;