@nhonh/qabot 0.6.0 → 1.0.0

package/src/cli/commands/test.js

@@ -2,8 +2,9 @@ import chalk from "chalk";
 import ora from "ora";
 import path from "node:path";
 import { writeFile, readFile } from "node:fs/promises";
-import { execSync } from "node:child_process";
-import { logger } from "../../core/logger.js";
+import { execSync, spawn } from "node:child_process";
+import Enquirer from "enquirer";
+import { logger, formatMs } from "../../core/logger.js";
 import { loadConfig } from "../../core/config.js";
 import { analyzeProject } from "../../analyzers/project-analyzer.js";
 import { AIEngine } from "../../ai/ai-engine.js";
@@ -23,6 +24,14 @@ import {
 } from "../../utils/file-utils.js";
 import { ReportGenerator } from "../../reporter/report-generator.js";
 
+const V = chalk.hex("#A78BFA");
+const V2 = chalk.hex("#7C3AED");
+const DIM = chalk.hex("#6B7280");
+const W = chalk.hex("#F3F4F6");
+const G = chalk.hex("#34D399");
+const R = chalk.hex("#F87171");
+const Y = chalk.hex("#FBBF24");
+
 const MAX_FIX_ATTEMPTS = 3;
 
 export function registerTestCommand(program) {
@@ -33,6 +42,7 @@ export function registerTestCommand(program) {
     .option("--headed", "Run browser in headed mode (visible)")
     .option("--no-fix", "Skip auto-fix on failure")
     .option("--skip-gen", "Skip test generation, run existing specs only")
+    .option("-u, --url <url>", "Target URL (overrides environment config)")
     .option("--use-cases <dir>", "Directory containing use case documents")
     .option("--model <model>", "AI model to use")
     .option("-d, --dir <path>", "Project directory", process.cwd())
@@ -51,28 +61,27 @@ async function runTest(feature, options) {
   const profile = await analyzeProject(projectDir);
   const envConfig =
     config.environments?.[options.env] || config.environments?.default;
-  const baseUrl = envConfig?.url || "http://localhost:3000";
+  const baseUrl = options.url || envConfig?.url || "http://localhost:3000";
 
-  logger.header("QABot \u2014 E2E Automation Test");
+  logger.header("E2E Automation Test");
   logger.blank();
-  logger.info(`Feature:     ${chalk.bold(feature || "all")}`);
-  logger.info(`Target:      ${chalk.bold.cyan(baseUrl)}`);
-  logger.info(`Environment: ${chalk.bold(options.env)}`);
-  logger.info(
-    `Browser:     ${options.headed ? "Headed (visible)" : "Headless"}`,
-  );
+  console.log(DIM("  Feature: ") + W(feature || "all"));
+  console.log(DIM("  Target:  ") + chalk.hex("#22D3EE")(baseUrl));
+  console.log(DIM("  Browser: ") + W(options.headed ? "Headed" : "Headless"));
   logger.blank();
-  console.log(chalk.dim(`  ${"\u2500".repeat(50)}`));
 
-  const spinner = ora("Setting up Playwright...").start();
+  const spinner = ora({
+    text: V(" Setting up Playwright..."),
+    spinner: "dots",
+  }).start();
   try {
     await ensurePlaywright(projectDir);
     await ensureE2EStructure(projectDir);
     await writePlaywrightConfig(projectDir, config);
     await writeAuthHelper(projectDir, config);
-    spinner.succeed("Playwright ready");
+    spinner.succeed(G(" Playwright ready"));
   } catch (err) {
-    spinner.fail(`Playwright setup failed: ${err.message}`);
+    spinner.fail(R(` Setup failed: ${err.message}`));
     return;
   }
 
@@ -83,22 +92,15 @@ async function runTest(feature, options) {
   if (options.skipGen) {
     const existing = await findFiles(specDir, "*.spec.js");
     if (existing.length === 0) {
-      logger.error(
-        "No existing E2E specs found. Run without --skip-gen to generate.",
-      );
+      logger.error("No existing E2E specs found. Run without --skip-gen.");
       return;
     }
     specFile = feature
       ? existing.find((f) => f.toLowerCase().includes(feature.toLowerCase()))
       : null;
-    logger.info(
-      `Running ${specFile ? path.basename(specFile) : "all"} existing specs`,
-    );
   } else {
     if (!feature) {
-      logger.error(
-        "Feature name required for test generation. Usage: qabot test <feature>",
-      );
+      logger.error("Feature name required. Usage: qabot test <feature>");
       return;
     }
 
@@ -111,9 +113,10 @@ async function runTest(feature, options) {
       return;
     }
 
-    const spinner2 = ora(
-      "AI is analyzing feature and generating E2E tests...",
-    ).start();
+    const spinner2 = ora({
+      text: V(" AI analyzing feature..."),
+      spinner: "dots",
+    }).start();
 
     let sourceCode = "";
     if (featureConfig?.src) {
@@ -158,183 +161,225 @@ async function runTest(feature, options) {
         authProvider: config.auth?.provider || "none",
         useCases,
       });
-
       specFile = path.join(specDir, `${feature}.spec.js`);
       await writeFile(specFile, spec, "utf-8");
       spinner2.succeed(
-        `E2E spec generated: ${chalk.underline(path.relative(projectDir, specFile))}`,
+        G(` E2E spec: ${chalk.underline(path.relative(projectDir, specFile))}`),
       );
     } catch (err) {
-      spinner2.fail(`E2E generation failed: ${err.message}`);
+      spinner2.fail(R(` Generation failed: ${err.message}`));
       return;
     }
 
     if (options.fix !== false) {
-      const aiForFix = ai;
-
       for (let attempt = 1; attempt <= MAX_FIX_ATTEMPTS; attempt++) {
         logger.blank();
-        logger.step(
-          attempt,
-          MAX_FIX_ATTEMPTS,
-          `Running E2E tests (attempt ${attempt})...`,
-        );
+        logger.step(attempt, MAX_FIX_ATTEMPTS, `Attempt ${attempt}`);
+        logger.blank();
 
-        const {
-          exitCode,
-          error: testError,
-          stdout,
-        } = runPlaywright(projectDir, specFile, options);
+        const result = await runPlaywrightStreaming(
+          projectDir,
+          specFile,
+          options,
+          baseUrl,
+        );
 
-        if (exitCode === 0) {
-          printPlaywrightSummary(stdout);
-          logger.blank();
-          logger.success(`E2E tests passed on attempt ${attempt}!`);
+        if (result.exitCode === 0) {
+          logger.summary(
+            result.passed,
+            result.failed,
+            result.skipped,
+            result.duration,
+          );
+          if (result.passed > 0)
+            logger.success(`E2E passed on attempt ${attempt}`);
           await generateE2EReport(
             projectDir,
             config,
             feature,
             options.env,
             baseUrl,
+            result,
           );
           return;
         }
 
         if (attempt >= MAX_FIX_ATTEMPTS) {
-          printPlaywrightSummary(stdout || testError);
-          logger.blank();
-          logger.warn(
-            `E2E tests still failing after ${MAX_FIX_ATTEMPTS} attempts.`,
+          logger.summary(
+            result.passed,
+            result.failed,
+            result.skipped,
+            result.duration,
           );
-          logger.dim(`  Review: cat ${path.relative(projectDir, specFile)}`);
-          logger.dim(`  Debug:  qabot test ${feature} --headed`);
+          logger.warn(`Still failing after ${MAX_FIX_ATTEMPTS} attempts`);
+          logger.dim(`Review: cat ${path.relative(projectDir, specFile)}`);
+          logger.dim(`Debug:  qabot test ${feature} --headed`);
           await generateE2EReport(
             projectDir,
             config,
             feature,
             options.env,
             baseUrl,
+            result,
           );
           return;
         }
 
-        logger.dim(`  ${extractFirstError(testError || stdout)}`);
-        logger.step(attempt, MAX_FIX_ATTEMPTS, "AI is fixing E2E test...");
-
+        const fixSpinner = ora({
+          text: V(" AI fixing errors..."),
+          spinner: "dots",
+        }).start();
         try {
           const currentCode = await readFile(specFile, "utf-8");
-          const fixedCode = await fixE2ESpec(
-            aiForFix,
-            currentCode,
-            testError || stdout,
-            {
-              baseUrl,
-              authProvider: config.auth?.provider,
-            },
-          );
+          const fixedCode = await fixE2ESpec(ai, currentCode, result.rawError, {
+            baseUrl,
+            authProvider: config.auth?.provider,
+          });
           await writeFile(specFile, fixedCode, "utf-8");
-          logger.dim("  Fix applied. Re-running...");
+          fixSpinner.succeed(G(" Fix applied"));
         } catch (err) {
-          logger.warn(`  AI fix failed: ${err.message}`);
+          fixSpinner.fail(R(` Fix failed: ${err.message}`));
           break;
         }
       }
+      return;
     }
   }
 
   if (!specFile && !options.skipGen) return;
 
-  logger.blank();
-  logger.info("Running E2E tests...");
-
-  const {
-    exitCode,
-    stdout,
-    error: testError,
-  } = runPlaywright(projectDir, specFile, options);
-  printPlaywrightSummary(stdout || testError);
+  logger.section("Running E2E");
+  const result = await runPlaywrightStreaming(
+    projectDir,
+    specFile,
+    options,
+    baseUrl,
+  );
+  logger.summary(result.passed, result.failed, result.skipped, result.duration);
   await generateE2EReport(
     projectDir,
     config,
     feature || "all",
     options.env,
     baseUrl,
+    result,
   );
-
-  if (exitCode !== 0) process.exit(1);
+  if (result.failed > 0) process.exit(1);
 }
 
-function runPlaywright(projectDir, specFile, options) {
-  const configPath = path.join(projectDir, "e2e", "playwright.config.js");
-  const parts = ["npx", "playwright", "test"];
-  if (specFile)
-    parts.push(path.relative(path.join(projectDir, "e2e"), specFile));
-  parts.push(`--config="${configPath}"`);
-  parts.push("--project=chromium");
-  if (options.headed) parts.push("--headed");
-
-  const env = {
-    ...process.env,
-    E2E_ENV: options.env || "default",
-    FORCE_COLOR: "0",
-  };
+function runPlaywrightStreaming(projectDir, specFile, options, baseUrl) {
+  return new Promise((resolve) => {
+    const configPath = path.join(projectDir, "e2e", "playwright.config.js");
+    const args = ["playwright", "test"];
+    if (specFile)
+      args.push(path.relative(path.join(projectDir, "e2e"), specFile));
+    args.push(
+      `--config=${configPath}`,
+      "--project=chromium",
+      "--reporter=line",
+    );
+    if (options.headed) args.push("--headed");
 
-  try {
-    const stdout = execSync(parts.join(" "), {
+    const env = {
+      ...process.env,
+      E2E_ENV: options.env || "default",
+      BASE_URL: baseUrl,
+      FORCE_COLOR: "1",
+    };
+
+    const startTime = Date.now();
+    let stdout = "";
+    let stderr = "";
+    let passed = 0;
+    let failed = 0;
+    let skipped = 0;
+    let testIndex = 0;
+
+    const child = spawn("npx", args, {
       cwd: projectDir,
-      stdio: "pipe",
-      timeout: 120000,
       env,
-    }).toString();
-    return { exitCode: 0, stdout, error: "" };
-  } catch (err) {
-    return {
-      exitCode: err.status || 1,
-      stdout: err.stdout?.toString() || "",
-      error: err.stderr?.toString() || err.stdout?.toString() || "",
-    };
-  }
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+
+    child.stdout.on("data", (data) => {
+      const text = data.toString();
+      stdout += text;
+
+      for (const line of text.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed) continue;
+
+        if (
+          trimmed.includes("\u2713") ||
+          trimmed.includes("passed") ||
+          trimmed.match(/\[.*\].*ok/i)
+        ) {
+          testIndex++;
+          passed++;
+          const name = extractTestName(trimmed);
+          logger.testResult(name, "passed", extractDuration(trimmed));
+        } else if (
+          trimmed.includes("\u2717") ||
+          trimmed.includes("failed") ||
+          trimmed.includes("FAIL")
+        ) {
+          testIndex++;
+          failed++;
+          const name = extractTestName(trimmed);
+          logger.testResult(name, "failed", extractDuration(trimmed));
+        } else if (trimmed.includes("skipped")) {
+          testIndex++;
+          skipped++;
+          const name = extractTestName(trimmed);
+          logger.testResult(name, "skipped", 0);
+        } else if (trimmed.match(/Running \d+ test/)) {
+          logger.dim(trimmed);
+        } else if (trimmed.includes("Error") || trimmed.includes("expect")) {
+          logger.dim(R(`     ${trimmed.slice(0, 120)}`));
+        }
+      }
+    });
+
+    child.stderr.on("data", (data) => {
+      stderr += data.toString();
+    });
+
+    child.on("close", (exitCode) => {
+      const duration = Date.now() - startTime;
+      resolve({
+        exitCode: exitCode ?? 1,
+        passed,
+        failed,
+        skipped,
+        duration,
+        stdout,
+        stderr,
+        rawError: (stderr || stdout).slice(0, 4000),
+        tests: [],
+      });
+    });
+  });
 }
 
-function printPlaywrightSummary(output) {
-  if (!output) return;
-  const lines = output.split("\n");
-  for (const line of lines) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    if (trimmed.includes("\u2713") || trimmed.includes("passed")) {
-      logger.success(trimmed);
-    } else if (
-      trimmed.includes("\u2717") ||
-      trimmed.includes("failed") ||
-      trimmed.includes("Error")
-    ) {
-      logger.error(trimmed);
-    } else if (
-      trimmed.includes("─") ||
-      trimmed.includes("Running") ||
-      trimmed.includes("test")
-    ) {
-      logger.dim(trimmed);
-    }
-  }
+function extractTestName(line) {
+  const cleaned = line
+    .replace(/\u2713|\u2717|\u25CB/g, "")
+    .replace(/\[.*?\]/g, "")
+    .replace(/\(\d+(\.\d+)?[ms]+\)/g, "")
+    .replace(/›/g, " > ")
+    .replace(/\s+/g, " ")
+    .trim();
+  const parts = cleaned.split(" > ");
+  return parts[parts.length - 1] || cleaned;
 }
 
-function extractFirstError(output) {
-  if (!output) return "Unknown error";
-  const lines = output.split("\n").filter((l) => l.trim());
-  const errorLine = lines.find(
-    (l) =>
-      l.includes("Error") ||
-      l.includes("expect") ||
-      l.includes("Timeout") ||
-      l.includes("locator"),
-  );
-  return (
-    errorLine?.trim().slice(0, 150) ||
-    lines[0]?.trim().slice(0, 150) ||
-    "Test failed"
-  );
+function extractDuration(line) {
+  const match = line.match(/\((\d+(?:\.\d+)?)(ms|s|m)\)/);
+  if (!match) return 0;
+  const val = parseFloat(match[1]);
+  if (match[2] === "s") return val * 1000;
+  if (match[2] === "m") return val * 60000;
+  return val;
 }
 
 function guessRoute(featureName) {
@@ -355,112 +400,67 @@ function guessRoute(featureName) {
   return routes[featureName] || `/${featureName}`;
 }
 
-async function generateE2EReport(projectDir, config, feature, env, baseUrl) {
+async function generateE2EReport(
+  projectDir,
+  config,
+  feature,
+  env,
+  baseUrl,
+  result,
+) {
   try {
-    const screenshotsDir = path.join(projectDir, "e2e", "screenshots");
-    const screenshots = await findFiles(screenshotsDir, "*.png").catch(
-      () => [],
-    );
-
     const reporter = new ReportGenerator(config);
+    const tests = result.tests || [];
     const results = {
       summary: {
-        totalTests: 0,
-        totalPassed: 0,
-        totalFailed: 0,
-        totalSkipped: 0,
-        overallPassRate: 0,
-        byLayer: {},
+        totalTests: result.passed + result.failed + result.skipped,
+        totalPassed: result.passed,
+        totalFailed: result.failed,
+        totalSkipped: result.skipped,
+        overallPassRate:
+          result.passed + result.failed + result.skipped > 0
+            ? Math.round(
+                (result.passed /
+                  (result.passed + result.failed + result.skipped)) *
+                  100,
+              )
+            : 0,
+        totalDuration: result.duration,
+        byLayer: {
+          e2e: {
+            total: result.passed + result.failed + result.skipped,
+            passed: result.passed,
+            failed: result.failed,
+            skipped: result.skipped,
+          },
+        },
       },
-      results: [],
+      results: [
+        {
+          runner: "playwright",
+          layer: "e2e",
+          feature,
+          tests,
+          summary: {
+            total: result.passed + result.failed + result.skipped,
+            passed: result.passed,
+            failed: result.failed,
+            skipped: result.skipped,
+          },
+        },
+      ],
     };
 
-    const pwResultsPath = path.join(
-      projectDir,
-      "qabot-reports",
-      "playwright",
-      "results.json",
-    );
-    if (await fileExists(pwResultsPath)) {
-      try {
-        const pwResults = JSON.parse(await readFile(pwResultsPath, "utf-8"));
-        if (pwResults.suites) {
-          const tests = [];
-          flattenSuites(pwResults.suites, tests);
-          results.results = [
-            {
-              runner: "playwright",
-              layer: "e2e",
-              feature,
-              tests,
-              summary: {
-                total: tests.length,
-                passed: tests.filter((t) => t.status === "passed").length,
-                failed: tests.filter((t) => t.status === "failed").length,
-                skipped: tests.filter((t) => t.status === "skipped").length,
-              },
-            },
-          ];
-          results.summary.totalTests = tests.length;
-          results.summary.totalPassed = tests.filter(
-            (t) => t.status === "passed",
-          ).length;
-          results.summary.totalFailed = tests.filter(
-            (t) => t.status === "failed",
-          ).length;
-          results.summary.overallPassRate =
-            tests.length > 0
-              ? Math.round((results.summary.totalPassed / tests.length) * 100)
-              : 0;
-        }
-      } catch {
-        /* skip malformed results */
-      }
-    }
-
     const reportPaths = await reporter.generate(results, {
       feature,
       environment: env,
       projectName: config.project?.name || "unknown",
       timestamp: new Date().toISOString(),
-      duration: 0,
+      duration: result.duration,
     });
 
-    logger.blank();
     logger.info(`Report: ${chalk.underline(reportPaths.htmlPath)}`);
   } catch {
-    /* report generation is best-effort */
-  }
-}
-
-function flattenSuites(suites, tests) {
-  for (const suite of suites || []) {
-    for (const spec of suite.specs || []) {
-      for (const test of spec.tests || []) {
-        const result = test.results?.[test.results.length - 1];
-        tests.push({
-          name: spec.title,
-          suite: suite.title,
-          file: suite.file || "",
-          status:
-            test.status === "expected"
-              ? "passed"
-              : test.status === "skipped"
-                ? "skipped"
-                : "failed",
-          duration: result?.duration || 0,
-          error: result?.error
-            ? {
-                message: result.error.message || "",
-                stack: result.error.stack || "",
-              }
-            : null,
-          screenshots: (result?.attachments || [])
-            .filter((a) => a.contentType?.startsWith("image/"))
-            .map((a) => a.path),
-        });
-      }
-    }
-    if (suite.suites) flattenSuites(suite.suites, tests);
+    /* best-effort */
   }
 }