ccqa 0.1.1 → 0.1.2

package/package.json

@@ -1,14 +1,11 @@
 {
   "name": "ccqa",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "type": "module",
   "description": "Browser test recorder powered by Claude Code and agent-browser",
   "bin": {
     "ccqa": "./bin/ccqa.ts"
   },
-  "exports": {
-    "./test-helpers": "./src/runtime/test-helpers.ts"
-  },
   "files": [
     "bin/",
     "src/"

package/src/cli/index.ts

@@ -1,16 +1,20 @@
 import { Command } from "commander";
+import { readFileSync } from "node:fs";
 import { traceCommand } from "./trace.ts";
 import { generateCommand } from "./generate.ts";
 import { runCommand } from "./run.ts";
 import { traceSetupCommand } from "./trace-setup.ts";
 import { generateSetupCommand } from "./generate-setup.ts";
 
+const packageJsonPath = new URL("../../package.json", import.meta.url);
+const { version } = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { version: string };
+
 const program = new Command();
 
 program
   .name("ccqa")
   .description("E2E test CLI using Claude Code + agent-browser")
-  .version("0.1.0");
+  .version(version);
 
 program.addCommand(traceCommand);
 program.addCommand(generateCommand);

package/src/cli/run.ts

@@ -1,4 +1,7 @@
 import { Command } from "commander";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import {
   parseSpecPath,
   getTestScript,
@@ -7,6 +10,38 @@ import {
 } from "../store/index.ts";
 import * as log from "./logger.ts";
 
+type VitestAssertionResult = {
+  status: "passed" | "failed" | "skipped" | "pending" | "todo";
+  title: string;
+  fullName: string;
+  duration?: number;
+  failureMessages?: string[];
+};
+
+type VitestTestResult = {
+  name: string;
+  status: "passed" | "failed";
+  assertionResults: VitestAssertionResult[];
+};
+
+type VitestJsonReport = {
+  numTotalTests: number;
+  numPassedTests: number;
+  numFailedTests: number;
+  numPendingTests: number;
+  startTime: number;
+  success: boolean;
+  testResults: VitestTestResult[];
+};
+
+type SpecRunSummary = {
+  featureName: string;
+  specName: string;
+  scriptFile: string;
+  report: VitestJsonReport | null;
+  exitCode: number;
+};
+
 export const runCommand = new Command("run")
   .argument("[target]", "Spec to run: '<feature>/<spec>', '<feature>', or omit for all")
   .description("Run generated agent-browser test scripts")
@@ -25,29 +60,167 @@ async function runTests(target?: string): Promise<void> {
     process.exit(1);
   }
 
+  const tmpDir = await mkdtemp(join(tmpdir(), "ccqa-run-"));
+  const summaries: SpecRunSummary[] = [];
   let overallExitCode = 0;
 
-  for (const { featureName, specName } of specs) {
-    const scriptFile = await getTestScript(featureName, specName);
-    if (!scriptFile) {
-      log.warn(`${featureName}/${specName}: no test.spec.ts found`);
+  try {
+    for (let i = 0; i < specs.length; i++) {
+      const { featureName, specName } = specs[i]!;
+      const scriptFile = await getTestScript(featureName, specName);
+      if (!scriptFile) {
+        log.warn(`${featureName}/${specName}: no test.spec.ts found`);
+        continue;
+      }
+
+      log.info(`▶ ${featureName}/${specName}`);
+      log.meta("test", scriptFile);
+      log.blank();
+
+      const reportFile = join(tmpDir, `report-${i}.json`);
+      const proc = Bun.spawn(
+        [
+          "bunx",
+          "vitest",
+          "run",
+          scriptFile,
+          "--reporter=default",
+          "--reporter=json",
+          `--outputFile.json=${reportFile}`,
+        ],
+        { stdout: "pipe", stderr: "pipe" },
+      );
+
+      await Promise.all([
+        streamFiltered(proc.stdout, Bun.stdout),
+        streamFiltered(proc.stderr, Bun.stderr),
+      ]);
+      const exitCode = await proc.exited;
+      if (exitCode !== 0) overallExitCode = exitCode;
+
+      const report = await readReport(reportFile);
+      summaries.push({ featureName, specName, scriptFile, report, exitCode });
+      log.blank();
+    }
+
+    printSummary(summaries);
+  } finally {
+    await rm(tmpDir, { recursive: true, force: true });
+  }
+
+  process.exit(overallExitCode);
+}
+
+async function readReport(path: string): Promise<VitestJsonReport | null> {
+  try {
+    const raw = await readFile(path, "utf8");
+    return JSON.parse(raw) as VitestJsonReport;
+  } catch {
+    return null;
+  }
+}
+
+function printSummary(summaries: SpecRunSummary[]): void {
+  process.stdout.write("\n────── ccqa summary ──────\n\n");
+
+  let totalTests = 0;
+  let totalPassed = 0;
+  let totalFailed = 0;
+  let totalSkipped = 0;
+
+  for (const s of summaries) {
+    const header = `${s.featureName}/${s.specName}`;
+    if (!s.report) {
+      const icon = s.exitCode === 0 ? "✓" : "✗";
+      process.stdout.write(`${icon} ${header} (no report)\n`);
       continue;
     }
 
-    log.info(`${featureName}/${specName}`);
-    log.meta("test", scriptFile);
-    log.blank();
+    totalTests += s.report.numTotalTests;
+    totalPassed += s.report.numPassedTests;
+    totalFailed += s.report.numFailedTests;
+    totalSkipped += s.report.numPendingTests;
 
-    const proc = Bun.spawn(["bunx", "vitest", "run", scriptFile], {
-      stdout: "inherit",
-      stderr: "inherit",
-    });
+    const icon = s.report.success ? "✓" : "✗";
+    process.stdout.write(
+      `${icon} ${header}  (${s.report.numPassedTests}/${s.report.numTotalTests} passed)\n`,
+    );
 
-    const exitCode = await proc.exited;
-    if (exitCode !== 0) overallExitCode = exitCode;
+    for (const file of s.report.testResults) {
+      for (const a of file.assertionResults) {
+        const aIcon = assertionIcon(a.status);
+        const dur = a.duration != null ? `  ${formatDuration(a.duration)}` : "";
+        process.stdout.write(`    ${aIcon} ${a.fullName}${dur}\n`);
+        if (a.status === "failed" && a.failureMessages?.length) {
+          for (const msg of a.failureMessages) {
+            const firstLine = msg.split("\n")[0] ?? msg;
+            process.stdout.write(`        ${firstLine}\n`);
+          }
+        }
+      }
+    }
   }
 
-  process.exit(overallExitCode);
+  process.stdout.write("\n");
+  process.stdout.write(
+    `  Specs      ${summaries.length} (${summaries.filter((s) => s.exitCode === 0).length} passed, ${summaries.filter((s) => s.exitCode !== 0).length} failed)\n`,
+  );
+  process.stdout.write(
+    `  Tests      ${totalTests} (${totalPassed} passed, ${totalFailed} failed, ${totalSkipped} skipped)\n`,
+  );
+  process.stdout.write("\n");
+}
+
+function assertionIcon(status: VitestAssertionResult["status"]): string {
+  switch (status) {
+    case "passed":
+      return "✓";
+    case "failed":
+      return "✗";
+    case "skipped":
+    case "pending":
+    case "todo":
+      return "⊘";
+  }
+}
+
+function formatDuration(ms: number): string {
+  if (ms < 1000) return `${Math.round(ms)}ms`;
+  return `${(ms / 1000).toFixed(2)}s`;
+}
+
+const NOISE_LINE_PATTERNS = [/^JSON report written to /];
+
+async function streamFiltered(
+  source: ReadableStream<Uint8Array>,
+  sink: { write: (chunk: Uint8Array) => unknown },
+): Promise<void> {
+  const decoder = new TextDecoder();
+  const encoder = new TextEncoder();
+  let buffer = "";
+  const reader = source.getReader();
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      let nl = buffer.indexOf("\n");
+      while (nl !== -1) {
+        const line = buffer.slice(0, nl);
+        buffer = buffer.slice(nl + 1);
+        if (!NOISE_LINE_PATTERNS.some((p) => p.test(line))) {
+          sink.write(encoder.encode(line + "\n"));
+        }
+        nl = buffer.indexOf("\n");
+      }
+    }
+    buffer += decoder.decode();
+    if (buffer.length > 0 && !NOISE_LINE_PATTERNS.some((p) => p.test(buffer))) {
+      sink.write(encoder.encode(buffer));
+    }
+  } finally {
+    reader.releaseLock();
+  }
 }
 
 async function resolveSpecs(target?: string): Promise<Array<{ featureName: string; specName: string }>> {

package/src/codegen/actions-to-script.ts

@@ -11,12 +11,13 @@ export interface SetupScript {
 }
 
 export function actionsToScript(actions: TraceAction[], title: string, setupScripts?: SetupScript[]): string {
-  const helpersImport = "ccqa/test-helpers";
+  // Resolve the helpers path relative to this file so it works from any cwd
+  const helpersPath = new URL("../runtime/test-helpers.ts", import.meta.url).pathname;
 
   const imports = [
     `import { test } from "vitest";`,
     `import { spawnSync } from "node:child_process";`,
-    `import { ab, abWait, abAssertTextVisible, abAssertVisible, abAssertNotVisible, abAssertUrl, abAssertEnabled, abAssertDisabled, abAssertChecked, abAssertUnchecked } from ${JSON.stringify(helpersImport)};`,
+    `import { ab, abWait, abAssertTextVisible, abAssertVisible, abAssertNotVisible, abAssertUrl, abAssertEnabled, abAssertDisabled, abAssertChecked, abAssertUnchecked } from ${JSON.stringify(helpersPath)};`,
     "",
     `// Single session shared across all tests — reset per run via cookies clear in first test`,
     `process.env.AGENT_BROWSER_SESSION = \`ccqa-run-\${Date.now()}\`;`,