ccqa 0.1.0

package/src/cli/logger.ts

@@ -0,0 +1,45 @@
+import type { StepStatus } from "../types.ts";
+
+const STEP_ICONS: Record<StepStatus, string> = {
+  STEP_START: "▶",
+  STEP_DONE: "✓",
+  ASSERTION_FAILED: "✗",
+  STEP_SKIPPED: "⊘",
+  RUN_COMPLETED: "■",
+};
+
+export function header(command: string, target?: string): void {
+  process.stdout.write(`\nccqa ${command}${target ? ` ${target}` : ""}\n\n`);
+}
+
+export function meta(key: string, value: string | number): void {
+  process.stdout.write(`  ${key}: ${value}\n`);
+}
+
+export function blank(): void {
+  process.stdout.write("\n");
+}
+
+export function info(message: string): void {
+  process.stdout.write(`${message}\n`);
+}
+
+export function step(type: StepStatus, stepId: string, detail: string): void {
+  process.stdout.write(`  ${STEP_ICONS[type]} [${stepId}] ${detail}\n`);
+}
+
+export function bash(command: string): void {
+  process.stdout.write(`  $ ${command.slice(0, 120)}\n`);
+}
+
+export function error(message: string): void {
+  process.stderr.write(`error: ${message}\n`);
+}
+
+export function warn(message: string): void {
+  process.stderr.write(`warn: ${message}\n`);
+}
+
+export function hint(message: string): void {
+  process.stdout.write(`\nhint: ${message}\n`);
+}

package/src/cli/run.ts

@@ -0,0 +1,65 @@
+import { Command } from "commander";
+import {
+  parseSpecPath,
+  getTestScript,
+  listAllSpecs,
+  listSpecsForFeature,
+} from "../store/index.ts";
+import * as log from "./logger.ts";
+
+export const runCommand = new Command("run")
+  .argument("[target]", "Spec to run: '<feature>/<spec>', '<feature>', or omit for all")
+  .description("Run generated agent-browser test scripts")
+  .action(async (target?: string) => {
+    await runTests(target);
+  });
+
+async function runTests(target?: string): Promise<void> {
+  log.header("run", target);
+
+  const specs = await resolveSpecs(target);
+
+  if (specs.length === 0) {
+    log.error("no test scripts found");
+    log.hint("run 'ccqa generate <feature>/<spec>' first to generate tests");
+    process.exit(1);
+  }
+
+  let overallExitCode = 0;
+
+  for (const { featureName, specName } of specs) {
+    const scriptFile = await getTestScript(featureName, specName);
+    if (!scriptFile) {
+      log.warn(`${featureName}/${specName}: no test.spec.ts found`);
+      continue;
+    }
+
+    log.info(`${featureName}/${specName}`);
+    log.meta("test", scriptFile);
+    log.blank();
+
+    const proc = Bun.spawn(["bunx", "vitest", "run", scriptFile], {
+      stdout: "inherit",
+      stderr: "inherit",
+    });
+
+    const exitCode = await proc.exited;
+    if (exitCode !== 0) overallExitCode = exitCode;
+  }
+
+  process.exit(overallExitCode);
+}
+
+async function resolveSpecs(target?: string): Promise<Array<{ featureName: string; specName: string }>> {
+  if (!target) {
+    return listAllSpecs();
+  }
+
+  if (target.includes("/")) {
+    const { featureName, specName } = parseSpecPath(target);
+    return [{ featureName, specName }];
+  }
+
+  const specNames = await listSpecsForFeature(target);
+  return specNames.map((specName) => ({ featureName: target, specName }));
+}

package/src/cli/trace-setup.ts

@@ -0,0 +1,124 @@
+import { Command } from "commander";
+import { buildSetupTraceSystemPrompt, buildSetupTracePrompt } from "../prompts/trace.ts";
+import { invokeClaudeStreaming } from "../claude/invoke.ts";
+import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk";
+import { ensureCcqaDir, readSetupSpecFile, saveSetupActions, saveSetupRoute } from "../store/index.ts";
+import { parseSetupSpec } from "../spec/parser.ts";
+import { parseAbAction, parseStatusLine, parseRouteStep } from "./trace.ts";
+import type { Route, RouteStep, TraceAction } from "../types.ts";
+import * as log from "./logger.ts";
+
+export const traceSetupCommand = new Command("trace-setup")
+  .argument("<name>", "Setup name to trace (e.g. login)")
+  .description("Trace a setup procedure using dummy placeholder values")
+  .action(async (name: string) => {
+    await runTraceSetup(name);
+  });
+
+async function runTraceSetup(name: string): Promise<void> {
+  log.header("trace-setup", name);
+
+  await ensureCcqaDir();
+
+  const specContent = await readSetupSpecFile(name);
+  const spec = parseSetupSpec(specContent);
+
+  // Replace {{key}} with dummy values for actual browser operation
+  const resolvedSpec = replacePlaceholdersWithDummies(spec);
+
+  log.meta("setup", spec.title);
+  log.meta("steps", spec.steps.length);
+  if (spec.placeholders) {
+    log.meta("placeholders", Object.keys(spec.placeholders).join(", "));
+  }
+  log.blank();
+
+  const systemPrompt = buildSetupTraceSystemPrompt(resolvedSpec);
+  const prompt = buildSetupTracePrompt(resolvedSpec);
+
+  log.info("Running agent-browser session...");
+  log.blank();
+
+  const routeSteps: RouteStep[] = [];
+  let overallStatus: "passed" | "failed" = "passed";
+  const traceActions: TraceAction[] = [];
+
+  const { isError } = await invokeClaudeStreaming(
+    {
+      prompt,
+      systemPrompt,
+      allowedTools: ["Bash(*)", "Read", "Grep", "Glob"],
+      onAbAction: (abAction: string) => {
+        const action = parseAbAction(abAction);
+        if (action) traceActions.push(action);
+      },
+      onAbActionFailed: () => {
+        traceActions.pop();
+      },
+    },
+    (msg: SDKMessage) => {
+      if (msg.type !== "assistant") return;
+
+      for (const block of msg.message.content ?? []) {
+        if (block.type !== "text" || !block.text) continue;
+        const text = block.text;
+
+        const statusLine = parseStatusLine(text);
+        if (statusLine) log.step(statusLine.type, statusLine.stepId, statusLine.detail);
+
+        for (const line of text.split("\n")) {
+          const trimmed = line.trim();
+          if (trimmed.startsWith("ROUTE_STEP|")) {
+            const routeStep = parseRouteStep(trimmed);
+            if (routeStep) {
+              routeSteps.push(routeStep);
+              if (routeStep.status === "FAILED") overallStatus = "failed";
+            }
+          } else if (trimmed.startsWith("AB_ACTION|snapshot|") || trimmed.startsWith("AB_ACTION|assert|")) {
+            const action = parseAbAction(trimmed);
+            if (action) traceActions.push(action);
+          }
+        }
+      }
+    },
+  );
+
+  if (isError) overallStatus = "failed";
+
+  const timestamp = new Date().toISOString();
+  const route: Route = { specName: name, timestamp, status: overallStatus, steps: routeSteps };
+
+  const [routePath, actionsPath] = await Promise.all([
+    saveSetupRoute(name, route),
+    saveSetupActions(name, traceActions),
+  ]);
+
+  log.blank();
+  log.meta("route", routePath);
+  log.meta("saved", actionsPath);
+  log.meta("actions", traceActions.length);
+  log.meta("status", overallStatus.toUpperCase());
+  log.hint(`run 'ccqa generate-setup ${name}' to generate and validate the setup`);
+}
+
+function replacePlaceholdersWithDummies(spec: ReturnType<typeof parseSetupSpec>): typeof spec {
+  if (!spec.placeholders) return spec;
+
+  const dummies = spec.placeholders as Record<string, { dummy: string; description?: string }>;
+  const resolve = (text: string): string => {
+    let result = text;
+    for (const [key, def] of Object.entries(dummies)) {
+      result = result.replaceAll(`{{${key}}}`, def.dummy);
+    }
+    return result;
+  };
+
+  return {
+    ...spec,
+    steps: spec.steps.map((step) => ({
+      ...step,
+      instruction: resolve(step.instruction),
+      expected: resolve(step.expected),
+    })),
+  };
+}

package/src/cli/trace.test.ts

@@ -0,0 +1,233 @@
+import { describe, test, expect } from "bun:test";
+import { parseAbAction, parseStatusLine, parseRouteStep } from "./trace.ts";
+
+describe("parseAbAction", () => {
+  test("returns null for non-AB_ACTION lines", () => {
+    expect(parseAbAction("some text")).toBeNull();
+    expect(parseAbAction("")).toBeNull();
+    expect(parseAbAction("ROUTE_STEP|step-01|title")).toBeNull();
+  });
+
+  test("returns null for unknown commands", () => {
+    expect(parseAbAction("AB_ACTION|unknown|arg")).toBeNull();
+    expect(parseAbAction("AB_ACTION|navigate|http://example.com")).toBeNull();
+  });
+
+  test("parses open", () => {
+    expect(parseAbAction("AB_ACTION|open|http://localhost:3000")).toEqual({
+      command: "open",
+      value: "http://localhost:3000",
+    });
+  });
+
+  test("parses press", () => {
+    expect(parseAbAction("AB_ACTION|press|Enter")).toEqual({
+      command: "press",
+      value: "Enter",
+    });
+  });
+
+  test("parses scroll", () => {
+    expect(parseAbAction("AB_ACTION|scroll|down|300")).toEqual({
+      command: "scroll",
+      direction: "down",
+      pixels: "300",
+    });
+  });
+
+  test("parses snapshot", () => {
+    expect(parseAbAction("AB_ACTION|snapshot|Login page loaded")).toEqual({
+      command: "snapshot",
+      observation: "Login page loaded",
+    });
+  });
+
+  test("parses click", () => {
+    expect(parseAbAction("AB_ACTION|click|[aria-label='Login']|Login")).toEqual({
+      command: "click",
+      selector: "[aria-label='Login']",
+      label: "Login",
+    });
+  });
+
+  test("parses dblclick", () => {
+    expect(parseAbAction("AB_ACTION|dblclick|[aria-label='Item']|Item")).toEqual({
+      command: "dblclick",
+      selector: "[aria-label='Item']",
+      label: "Item",
+    });
+  });
+
+  test("parses check", () => {
+    expect(parseAbAction("AB_ACTION|check|[aria-label='Agree']|Agree")).toEqual({
+      command: "check",
+      selector: "[aria-label='Agree']",
+      label: "Agree",
+    });
+  });
+
+  test("parses uncheck", () => {
+    expect(parseAbAction("AB_ACTION|uncheck|[aria-label='Agree']|Agree")).toEqual({
+      command: "uncheck",
+      selector: "[aria-label='Agree']",
+      label: "Agree",
+    });
+  });
+
+  test("parses hover", () => {
+    expect(parseAbAction("AB_ACTION|hover|[aria-label='Menu']|Menu")).toEqual({
+      command: "hover",
+      selector: "[aria-label='Menu']",
+      label: "Menu",
+    });
+  });
+
+  test("parses wait with selector", () => {
+    expect(parseAbAction("AB_ACTION|wait|[aria-label='Loading']|Loading")).toEqual({
+      command: "wait",
+      selector: "[aria-label='Loading']",
+      label: "Loading",
+    });
+  });
+
+  test("parses wait --text as text= selector", () => {
+    expect(parseAbAction("AB_ACTION|wait|--text|Done")).toEqual({
+      command: "wait",
+      selector: "text=Done",
+      label: undefined,
+    });
+  });
+
+  test("parses fill", () => {
+    expect(parseAbAction("AB_ACTION|fill|[aria-label='Email']|user@example.com|Email")).toEqual({
+      command: "fill",
+      selector: "[aria-label='Email']",
+      value: "user@example.com",
+      label: "Email",
+    });
+  });
+
+  test("parses type", () => {
+    expect(parseAbAction("AB_ACTION|type|[aria-label='Search']|query text|Search")).toEqual({
+      command: "type",
+      selector: "[aria-label='Search']",
+      value: "query text",
+      label: "Search",
+    });
+  });
+
+  test("parses select", () => {
+    expect(parseAbAction("AB_ACTION|select|[aria-label='Color']|red|Color")).toEqual({
+      command: "select",
+      selector: "[aria-label='Color']",
+      value: "red",
+      label: "Color",
+    });
+  });
+
+  test("parses drag", () => {
+    expect(parseAbAction("AB_ACTION|drag|[aria-label='Source']|[aria-label='Target']|Source")).toEqual({
+      command: "drag",
+      selector: "[aria-label='Source']",
+      target: "[aria-label='Target']",
+      label: "Source",
+    });
+  });
+});
+
+describe("parseStatusLine", () => {
+  test("parses STEP_START", () => {
+    expect(parseStatusLine("STEP_START|step-01|Login")).toEqual({
+      type: "STEP_START",
+      stepId: "step-01",
+      detail: "Login",
+    });
+  });
+
+  test("parses STEP_DONE", () => {
+    expect(parseStatusLine("STEP_DONE|step-01|Verified redirect")).toEqual({
+      type: "STEP_DONE",
+      stepId: "step-01",
+      detail: "Verified redirect",
+    });
+  });
+
+  test("parses ASSERTION_FAILED", () => {
+    expect(parseStatusLine("ASSERTION_FAILED|step-03|app-bug: button not disabled")).toEqual({
+      type: "ASSERTION_FAILED",
+      stepId: "step-03",
+      detail: "app-bug: button not disabled",
+    });
+  });
+
+  test("parses STEP_SKIPPED", () => {
+    expect(parseStatusLine("STEP_SKIPPED|step-02|previous step failed")).toEqual({
+      type: "STEP_SKIPPED",
+      stepId: "step-02",
+      detail: "previous step failed",
+    });
+  });
+
+  test("parses RUN_COMPLETED passed", () => {
+    expect(parseStatusLine("RUN_COMPLETED|passed|All steps done")).toEqual({
+      type: "RUN_COMPLETED",
+      stepId: "passed",
+      detail: "All steps done",
+    });
+  });
+
+  test("returns null for non-matching lines", () => {
+    expect(parseStatusLine("some random text")).toBeNull();
+    expect(parseStatusLine("")).toBeNull();
+    expect(parseStatusLine("ROUTE_STEP|step-01|title|ACTION:did|OBS:saw|STATUS:PASSED")).toBeNull();
+  });
+
+  test("returns first matching line from multi-line text", () => {
+    const text = "some preamble\nSTEP_START|step-01|Title\nmore text";
+    expect(parseStatusLine(text)).toEqual({
+      type: "STEP_START",
+      stepId: "step-01",
+      detail: "Title",
+    });
+  });
+});
+
+describe("parseRouteStep", () => {
+  test("parses a complete ROUTE_STEP line", () => {
+    const line = "ROUTE_STEP|step-01|Login|ACTION:filled form and pressed Enter|OBSERVATION:redirected to /dashboard|STATUS:PASSED";
+    expect(parseRouteStep(line)).toEqual({
+      title: "Login",
+      action: "filled form and pressed Enter",
+      observation: "redirected to /dashboard",
+      status: "PASSED",
+    });
+  });
+
+  test("parses FAILED status", () => {
+    const line = "ROUTE_STEP|step-02|Check|ACTION:clicked button|OBSERVATION:nothing happened|STATUS:FAILED";
+    expect(parseRouteStep(line)?.status).toBe("FAILED");
+  });
+
+  test("parses SKIPPED status", () => {
+    const line = "ROUTE_STEP|step-03|Skip|ACTION:skipped|OBSERVATION:n/a|STATUS:SKIPPED";
+    expect(parseRouteStep(line)?.status).toBe("SKIPPED");
+  });
+
+  test("returns null when fewer than 6 parts", () => {
+    expect(parseRouteStep("ROUTE_STEP|step-01|title|ACTION:did")).toBeNull();
+    expect(parseRouteStep("")).toBeNull();
+  });
+
+  test("defaults to FAILED for unrecognized status", () => {
+    const line = "ROUTE_STEP|step-01|Title|ACTION:did|OBSERVATION:saw|STATUS:UNKNOWN";
+    expect(parseRouteStep(line)?.status).toBe("FAILED");
+  });
+
+  test("strips ACTION: and OBSERVATION: prefixes", () => {
+    const line = "ROUTE_STEP|step-01|Title|ACTION:my action|OBSERVATION:my observation|STATUS:PASSED";
+    const result = parseRouteStep(line);
+    expect(result?.action).toBe("my action");
+    expect(result?.observation).toBe("my observation");
+  });
+});
+