ccqa 0.1.5 → 0.3.2

package/src/cli/run.ts

@@ -1,258 +0,0 @@
-import { Command } from "commander";
-import { mkdtemp, readFile, rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import {
-  parseSpecPath,
-  getTestScript,
-  listAllSpecs,
-  listSpecsForFeature,
-} from "../store/index.ts";
-import * as log from "./logger.ts";
-
-type VitestAssertionResult = {
-  status: "passed" | "failed" | "skipped" | "pending" | "todo";
-  title: string;
-  fullName: string;
-  duration?: number;
-  failureMessages?: string[];
-};
-
-type VitestTestResult = {
-  name: string;
-  status: "passed" | "failed";
-  assertionResults: VitestAssertionResult[];
-};
-
-type VitestJsonReport = {
-  numTotalTests: number;
-  numPassedTests: number;
-  numFailedTests: number;
-  numPendingTests: number;
-  startTime: number;
-  success: boolean;
-  testResults: VitestTestResult[];
-};
-
-type SpecRunSummary = {
-  featureName: string;
-  specName: string;
-  scriptFile: string;
-  report: VitestJsonReport | null;
-  exitCode: number;
-};
-
-export const runCommand = new Command("run")
-  .argument("[target]", "Spec to run: '<feature>/<spec>', '<feature>', or omit for all")
-  .description("Run generated agent-browser test scripts")
-  .action(async (target?: string) => {
-    await runTests(target);
-  });
-
-async function runTests(target?: string): Promise<void> {
-  log.header("run", target);
-
-  const specs = await resolveSpecs(target);
-
-  if (specs.length === 0) {
-    log.error("no test scripts found");
-    log.hint("run 'ccqa generate <feature>/<spec>' first to generate tests");
-    process.exit(1);
-  }
-
-  const tmpDir = await mkdtemp(join(tmpdir(), "ccqa-run-"));
-  const summaries: SpecRunSummary[] = [];
-  let overallExitCode = 0;
-
-  try {
-    for (let i = 0; i < specs.length; i++) {
-      const { featureName, specName } = specs[i]!;
-      const scriptFile = await getTestScript(featureName, specName);
-      if (!scriptFile) {
-        log.warn(`${featureName}/${specName}: no test.spec.ts found`);
-        continue;
-      }
-
-      log.info(`▶ ${featureName}/${specName}`);
-      log.meta("test", scriptFile);
-      log.blank();
-
-      const reportFile = join(tmpDir, `report-${i}.json`);
-      const proc = Bun.spawn(
-        [
-          "bunx",
-          "vitest",
-          "run",
-          scriptFile,
-          "--reporter=json",
-          `--outputFile.json=${reportFile}`,
-        ],
-        { stdout: "pipe", stderr: "pipe" },
-      );
-
-      await Promise.all([
-        streamFiltered(proc.stdout, Bun.stdout),
-        streamFiltered(proc.stderr, Bun.stderr),
-      ]);
-      const exitCode = await proc.exited;
-      if (exitCode !== 0) overallExitCode = exitCode;
-
-      const report = await readReport(reportFile);
-      summaries.push({ featureName, specName, scriptFile, report, exitCode });
-      log.blank();
-    }
-
-    printSummary(summaries);
-  } finally {
-    await rm(tmpDir, { recursive: true, force: true });
-  }
-
-  process.exit(overallExitCode);
-}
-
-async function readReport(path: string): Promise<VitestJsonReport | null> {
-  try {
-    const raw = await readFile(path, "utf8");
-    return JSON.parse(raw) as VitestJsonReport;
-  } catch {
-    return null;
-  }
-}
-
-const useColor = process.stdout.isTTY && process.env.NO_COLOR == null;
-const C = {
-  reset: useColor ? "\x1b[0m" : "",
-  bold: useColor ? "\x1b[1m" : "",
-  dim: useColor ? "\x1b[2m" : "",
-  green: useColor ? "\x1b[32m" : "",
-  red: useColor ? "\x1b[31m" : "",
-  yellow: useColor ? "\x1b[33m" : "",
-  cyan: useColor ? "\x1b[36m" : "",
-  gray: useColor ? "\x1b[90m" : "",
-};
-
-function printSummary(summaries: SpecRunSummary[]): void {
-  process.stdout.write(
-    `\n${C.cyan}${C.bold}──────── ccqa summary ────────${C.reset}\n\n`,
-  );
-
-  let totalTests = 0;
-  let totalPassed = 0;
-  let totalFailed = 0;
-  let totalSkipped = 0;
-
-  for (const s of summaries) {
-    const header = `${C.bold}${s.featureName}/${s.specName}${C.reset}`;
-    if (!s.report) {
-      const ok = s.exitCode === 0;
-      const icon = ok ? `${C.green}✔${C.reset}` : `${C.red}✖${C.reset}`;
-      process.stdout.write(`${icon} ${header} ${C.dim}(no report)${C.reset}\n`);
-      continue;
-    }
-
-    totalTests += s.report.numTotalTests;
-    totalPassed += s.report.numPassedTests;
-    totalFailed += s.report.numFailedTests;
-    totalSkipped += s.report.numPendingTests;
-
-    const ok = s.report.success;
-    const icon = ok ? `${C.green}✔${C.reset}` : `${C.red}✖${C.reset}`;
-    const countColor = ok ? C.green : C.red;
-    process.stdout.write(
-      `${icon} ${header}  ${countColor}${s.report.numPassedTests}/${s.report.numTotalTests}${C.reset} ${C.dim}passed${C.reset}\n`,
-    );
-
-    for (const file of s.report.testResults) {
-      for (const a of file.assertionResults) {
-        const aIcon = assertionIcon(a.status);
-        const dur = a.duration != null ? ` ${C.gray}${formatDuration(a.duration)}${C.reset}` : "";
-        process.stdout.write(`    ${aIcon} ${a.fullName}${dur}\n`);
-        if (a.status === "failed" && a.failureMessages?.length) {
-          for (const msg of a.failureMessages) {
-            const firstLine = msg.split("\n")[0] ?? msg;
-            process.stdout.write(`        ${C.red}${firstLine}${C.reset}\n`);
-          }
-        }
-      }
-    }
-  }
-
-  const specsPassed = summaries.filter((s) => s.exitCode === 0).length;
-  const specsFailed = summaries.filter((s) => s.exitCode !== 0).length;
-  process.stdout.write("\n");
-  process.stdout.write(
-    `  ${C.bold}Specs${C.reset}   ${summaries.length}  ` +
-      `(${C.green}${specsPassed} passed${C.reset}, ${specsFailed > 0 ? C.red : C.dim}${specsFailed} failed${C.reset})\n`,
-  );
-  process.stdout.write(
-    `  ${C.bold}Tests${C.reset}   ${totalTests}  ` +
-      `(${C.green}${totalPassed} passed${C.reset}, ${totalFailed > 0 ? C.red : C.dim}${totalFailed} failed${C.reset}, ${C.yellow}${totalSkipped} skipped${C.reset})\n`,
-  );
-  process.stdout.write("\n");
-}
-
-function assertionIcon(status: VitestAssertionResult["status"]): string {
-  switch (status) {
-    case "passed":
-      return `${C.green}✔${C.reset}`;
-    case "failed":
-      return `${C.red}✖${C.reset}`;
-    case "skipped":
-    case "pending":
-    case "todo":
-      return `${C.yellow}◌${C.reset}`;
-  }
-}
-
-function formatDuration(ms: number): string {
-  if (ms < 1000) return `${Math.round(ms)}ms`;
-  return `${(ms / 1000).toFixed(2)}s`;
-}
-
-const NOISE_LINE_PATTERNS = [/^JSON report written to /];
-
-async function streamFiltered(
-  source: ReadableStream<Uint8Array>,
-  sink: { write: (chunk: Uint8Array) => unknown },
-): Promise<void> {
-  const decoder = new TextDecoder();
-  const encoder = new TextEncoder();
-  let buffer = "";
-  const reader = source.getReader();
-  try {
-    while (true) {
-      const { value, done } = await reader.read();
-      if (done) break;
-      buffer += decoder.decode(value, { stream: true });
-      let nl = buffer.indexOf("\n");
-      while (nl !== -1) {
-        const line = buffer.slice(0, nl);
-        buffer = buffer.slice(nl + 1);
-        if (!NOISE_LINE_PATTERNS.some((p) => p.test(line))) {
-          sink.write(encoder.encode(line + "\n"));
-        }
-        nl = buffer.indexOf("\n");
-      }
-    }
-    buffer += decoder.decode();
-    if (buffer.length > 0 && !NOISE_LINE_PATTERNS.some((p) => p.test(buffer))) {
-      sink.write(encoder.encode(buffer));
-    }
-  } finally {
-    reader.releaseLock();
-  }
-}
-
-async function resolveSpecs(target?: string): Promise<Array<{ featureName: string; specName: string }>> {
-  if (!target) {
-    return listAllSpecs();
-  }
-
-  if (target.includes("/")) {
-    const { featureName, specName } = parseSpecPath(target);
-    return [{ featureName, specName }];
-  }
-
-  const specNames = await listSpecsForFeature(target);
-  return specNames.map((specName) => ({ featureName: target, specName }));
-}

package/src/cli/trace-setup.ts

@@ -1,124 +0,0 @@
-import { Command } from "commander";
-import { buildSetupTraceSystemPrompt, buildSetupTracePrompt } from "../prompts/trace.ts";
-import { invokeClaudeStreaming } from "../claude/invoke.ts";
-import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk";
-import { ensureCcqaDir, readSetupSpecFile, saveSetupActions, saveSetupRoute } from "../store/index.ts";
-import { parseSetupSpec } from "../spec/parser.ts";
-import { parseAbAction, parseStatusLine, parseRouteStep } from "./trace.ts";
-import type { Route, RouteStep, TraceAction } from "../types.ts";
-import * as log from "./logger.ts";
-
-export const traceSetupCommand = new Command("trace-setup")
-  .argument("<name>", "Setup name to trace (e.g. login)")
-  .description("Trace a setup procedure using dummy placeholder values")
-  .action(async (name: string) => {
-    await runTraceSetup(name);
-  });
-
-async function runTraceSetup(name: string): Promise<void> {
-  log.header("trace-setup", name);
-
-  await ensureCcqaDir();
-
-  const specContent = await readSetupSpecFile(name);
-  const spec = parseSetupSpec(specContent);
-
-  // Replace {{key}} with dummy values for actual browser operation
-  const resolvedSpec = replacePlaceholdersWithDummies(spec);
-
-  log.meta("setup", spec.title);
-  log.meta("steps", spec.steps.length);
-  if (spec.placeholders) {
-    log.meta("placeholders", Object.keys(spec.placeholders).join(", "));
-  }
-  log.blank();
-
-  const systemPrompt = buildSetupTraceSystemPrompt(resolvedSpec);
-  const prompt = buildSetupTracePrompt(resolvedSpec);
-
-  log.info("Running agent-browser session...");
-  log.blank();
-
-  const routeSteps: RouteStep[] = [];
-  let overallStatus: "passed" | "failed" = "passed";
-  const traceActions: TraceAction[] = [];
-
-  const { isError } = await invokeClaudeStreaming(
-    {
-      prompt,
-      systemPrompt,
-      allowedTools: ["Bash(*)", "Read", "Grep", "Glob"],
-      onAbAction: (abAction: string) => {
-        const action = parseAbAction(abAction);
-        if (action) traceActions.push(action);
-      },
-      onAbActionFailed: () => {
-        traceActions.pop();
-      },
-    },
-    (msg: SDKMessage) => {
-      if (msg.type !== "assistant") return;
-
-      for (const block of msg.message.content ?? []) {
-        if (block.type !== "text" || !block.text) continue;
-        const text = block.text;
-
-        const statusLine = parseStatusLine(text);
-        if (statusLine) log.step(statusLine.type, statusLine.stepId, statusLine.detail);
-
-        for (const line of text.split("\n")) {
-          const trimmed = line.trim();
-          if (trimmed.startsWith("ROUTE_STEP|")) {
-            const routeStep = parseRouteStep(trimmed);
-            if (routeStep) {
-              routeSteps.push(routeStep);
-              if (routeStep.status === "FAILED") overallStatus = "failed";
-            }
-          } else if (trimmed.startsWith("AB_ACTION|snapshot|") || trimmed.startsWith("AB_ACTION|assert|")) {
-            const action = parseAbAction(trimmed);
-            if (action) traceActions.push(action);
-          }
-        }
-      }
-    },
-  );
-
-  if (isError) overallStatus = "failed";
-
-  const timestamp = new Date().toISOString();
-  const route: Route = { specName: name, timestamp, status: overallStatus, steps: routeSteps };
-
-  const [routePath, actionsPath] = await Promise.all([
-    saveSetupRoute(name, route),
-    saveSetupActions(name, traceActions),
-  ]);
-
-  log.blank();
-  log.meta("route", routePath);
-  log.meta("saved", actionsPath);
-  log.meta("actions", traceActions.length);
-  log.meta("status", overallStatus.toUpperCase());
-  log.hint(`run 'ccqa generate-setup ${name}' to generate and validate the setup`);
-}
-
-function replacePlaceholdersWithDummies(spec: ReturnType<typeof parseSetupSpec>): typeof spec {
-  if (!spec.placeholders) return spec;
-
-  const dummies = spec.placeholders as Record<string, { dummy: string; description?: string }>;
-  const resolve = (text: string): string => {
-    let result = text;
-    for (const [key, def] of Object.entries(dummies)) {
-      result = result.replaceAll(`{{${key}}}`, def.dummy);
-    }
-    return result;
-  };
-
-  return {
-    ...spec,
-    steps: spec.steps.map((step) => ({
-      ...step,
-      instruction: resolve(step.instruction),
-      expected: resolve(step.expected),
-    })),
-  };
-}

package/src/cli/trace.test.ts

@@ -1,233 +0,0 @@
-import { describe, test, expect } from "bun:test";
-import { parseAbAction, parseStatusLine, parseRouteStep } from "./trace.ts";
-
-describe("parseAbAction", () => {
-  test("returns null for non-AB_ACTION lines", () => {
-    expect(parseAbAction("some text")).toBeNull();
-    expect(parseAbAction("")).toBeNull();
-    expect(parseAbAction("ROUTE_STEP|step-01|title")).toBeNull();
-  });
-
-  test("returns null for unknown commands", () => {
-    expect(parseAbAction("AB_ACTION|unknown|arg")).toBeNull();
-    expect(parseAbAction("AB_ACTION|navigate|http://example.com")).toBeNull();
-  });
-
-  test("parses open", () => {
-    expect(parseAbAction("AB_ACTION|open|http://localhost:3000")).toEqual({
-      command: "open",
-      value: "http://localhost:3000",
-    });
-  });
-
-  test("parses press", () => {
-    expect(parseAbAction("AB_ACTION|press|Enter")).toEqual({
-      command: "press",
-      value: "Enter",
-    });
-  });
-
-  test("parses scroll", () => {
-    expect(parseAbAction("AB_ACTION|scroll|down|300")).toEqual({
-      command: "scroll",
-      direction: "down",
-      pixels: "300",
-    });
-  });
-
-  test("parses snapshot", () => {
-    expect(parseAbAction("AB_ACTION|snapshot|Login page loaded")).toEqual({
-      command: "snapshot",
-      observation: "Login page loaded",
-    });
-  });
-
-  test("parses click", () => {
-    expect(parseAbAction("AB_ACTION|click|[aria-label='Login']|Login")).toEqual({
-      command: "click",
-      selector: "[aria-label='Login']",
-      label: "Login",
-    });
-  });
-
-  test("parses dblclick", () => {
-    expect(parseAbAction("AB_ACTION|dblclick|[aria-label='Item']|Item")).toEqual({
-      command: "dblclick",
-      selector: "[aria-label='Item']",
-      label: "Item",
-    });
-  });
-
-  test("parses check", () => {
-    expect(parseAbAction("AB_ACTION|check|[aria-label='Agree']|Agree")).toEqual({
-      command: "check",
-      selector: "[aria-label='Agree']",
-      label: "Agree",
-    });
-  });
-
-  test("parses uncheck", () => {
-    expect(parseAbAction("AB_ACTION|uncheck|[aria-label='Agree']|Agree")).toEqual({
-      command: "uncheck",
-      selector: "[aria-label='Agree']",
-      label: "Agree",
-    });
-  });
-
-  test("parses hover", () => {
-    expect(parseAbAction("AB_ACTION|hover|[aria-label='Menu']|Menu")).toEqual({
-      command: "hover",
-      selector: "[aria-label='Menu']",
-      label: "Menu",
-    });
-  });
-
-  test("parses wait with selector", () => {
-    expect(parseAbAction("AB_ACTION|wait|[aria-label='Loading']|Loading")).toEqual({
-      command: "wait",
-      selector: "[aria-label='Loading']",
-      label: "Loading",
-    });
-  });
-
-  test("parses wait --text as text= selector", () => {
-    expect(parseAbAction("AB_ACTION|wait|--text|Done")).toEqual({
-      command: "wait",
-      selector: "text=Done",
-      label: undefined,
-    });
-  });
-
-  test("parses fill", () => {
-    expect(parseAbAction("AB_ACTION|fill|[aria-label='Email']|user@example.com|Email")).toEqual({
-      command: "fill",
-      selector: "[aria-label='Email']",
-      value: "user@example.com",
-      label: "Email",
-    });
-  });
-
-  test("parses type", () => {
-    expect(parseAbAction("AB_ACTION|type|[aria-label='Search']|query text|Search")).toEqual({
-      command: "type",
-      selector: "[aria-label='Search']",
-      value: "query text",
-      label: "Search",
-    });
-  });
-
-  test("parses select", () => {
-    expect(parseAbAction("AB_ACTION|select|[aria-label='Color']|red|Color")).toEqual({
-      command: "select",
-      selector: "[aria-label='Color']",
-      value: "red",
-      label: "Color",
-    });
-  });
-
-  test("parses drag", () => {
-    expect(parseAbAction("AB_ACTION|drag|[aria-label='Source']|[aria-label='Target']|Source")).toEqual({
-      command: "drag",
-      selector: "[aria-label='Source']",
-      target: "[aria-label='Target']",
-      label: "Source",
-    });
-  });
-});
-
-describe("parseStatusLine", () => {
-  test("parses STEP_START", () => {
-    expect(parseStatusLine("STEP_START|step-01|Login")).toEqual({
-      type: "STEP_START",
-      stepId: "step-01",
-      detail: "Login",
-    });
-  });
-
-  test("parses STEP_DONE", () => {
-    expect(parseStatusLine("STEP_DONE|step-01|Verified redirect")).toEqual({
-      type: "STEP_DONE",
-      stepId: "step-01",
-      detail: "Verified redirect",
-    });
-  });
-
-  test("parses ASSERTION_FAILED", () => {
-    expect(parseStatusLine("ASSERTION_FAILED|step-03|app-bug: button not disabled")).toEqual({
-      type: "ASSERTION_FAILED",
-      stepId: "step-03",
-      detail: "app-bug: button not disabled",
-    });
-  });
-
-  test("parses STEP_SKIPPED", () => {
-    expect(parseStatusLine("STEP_SKIPPED|step-02|previous step failed")).toEqual({
-      type: "STEP_SKIPPED",
-      stepId: "step-02",
-      detail: "previous step failed",
-    });
-  });
-
-  test("parses RUN_COMPLETED passed", () => {
-    expect(parseStatusLine("RUN_COMPLETED|passed|All steps done")).toEqual({
-      type: "RUN_COMPLETED",
-      stepId: "passed",
-      detail: "All steps done",
-    });
-  });
-
-  test("returns null for non-matching lines", () => {
-    expect(parseStatusLine("some random text")).toBeNull();
-    expect(parseStatusLine("")).toBeNull();
-    expect(parseStatusLine("ROUTE_STEP|step-01|title|ACTION:did|OBS:saw|STATUS:PASSED")).toBeNull();
-  });
-
-  test("returns first matching line from multi-line text", () => {
-    const text = "some preamble\nSTEP_START|step-01|Title\nmore text";
-    expect(parseStatusLine(text)).toEqual({
-      type: "STEP_START",
-      stepId: "step-01",
-      detail: "Title",
-    });
-  });
-});
-
-describe("parseRouteStep", () => {
-  test("parses a complete ROUTE_STEP line", () => {
-    const line = "ROUTE_STEP|step-01|Login|ACTION:filled form and pressed Enter|OBSERVATION:redirected to /dashboard|STATUS:PASSED";
-    expect(parseRouteStep(line)).toEqual({
-      title: "Login",
-      action: "filled form and pressed Enter",
-      observation: "redirected to /dashboard",
-      status: "PASSED",
-    });
-  });
-
-  test("parses FAILED status", () => {
-    const line = "ROUTE_STEP|step-02|Check|ACTION:clicked button|OBSERVATION:nothing happened|STATUS:FAILED";
-    expect(parseRouteStep(line)?.status).toBe("FAILED");
-  });
-
-  test("parses SKIPPED status", () => {
-    const line = "ROUTE_STEP|step-03|Skip|ACTION:skipped|OBSERVATION:n/a|STATUS:SKIPPED";
-    expect(parseRouteStep(line)?.status).toBe("SKIPPED");
-  });
-
-  test("returns null when fewer than 6 parts", () => {
-    expect(parseRouteStep("ROUTE_STEP|step-01|title|ACTION:did")).toBeNull();
-    expect(parseRouteStep("")).toBeNull();
-  });
-
-  test("defaults to FAILED for unrecognized status", () => {
-    const line = "ROUTE_STEP|step-01|Title|ACTION:did|OBSERVATION:saw|STATUS:UNKNOWN";
-    expect(parseRouteStep(line)?.status).toBe("FAILED");
-  });
-
-  test("strips ACTION: and OBSERVATION: prefixes", () => {
-    const line = "ROUTE_STEP|step-01|Title|ACTION:my action|OBSERVATION:my observation|STATUS:PASSED";
-    const result = parseRouteStep(line);
-    expect(result?.action).toBe("my action");
-    expect(result?.observation).toBe("my observation");
-  });
-});
-