ccqa 0.1.0

package/src/cli/trace.ts

@@ -0,0 +1,244 @@
+import { readFile, writeFile, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { Command } from "commander";
+import { buildTraceSystemPrompt, buildTracePrompt, generateSessionName } from "../prompts/trace.ts";
+import { invokeClaudeStreaming } from "../claude/invoke.ts";
+import type { SDKMessage } from "@anthropic-ai/claude-agent-sdk";
+import { ensureCcqaDir, parseSpecPath, readSpecFile, saveRoute, saveTraceActions, getSetupDir } from "../store/index.ts";
+import { parseTestSpec } from "../spec/parser.ts";
+import type { Route, RouteStep, TraceAction, TraceCommand, AssertType, ParsedStatusLine } from "../types.ts";
+import * as log from "./logger.ts";
+
+export const traceCommand = new Command("trace")
+  .argument("<feature/spec>", "Spec to trace (e.g. tasks/create-and-complete)")
+  .description("Run agent-browser, verify assertions, and record structured actions")
+  .action(async (specPath: string) => {
+    const { featureName, specName } = parseSpecPath(specPath);
+    await runTrace(featureName, specName);
+  });
+
+async function runTrace(featureName: string, specName: string): Promise<void> {
+  log.header("trace", `${featureName}/${specName}`);
+
+  await ensureCcqaDir();
+
+  const specContent = await readSpecFile(featureName, specName);
+  const spec = parseTestSpec(specContent);
+  const hasSetups = (spec.setups?.length ?? 0) > 0;
+
+  log.meta("spec", spec.title);
+  log.meta("url", spec.baseUrl);
+  if (hasSetups) log.meta("setups", spec.setups!.map((s) => s.name).join(", "));
+  log.meta("steps", spec.steps.length);
+  log.blank();
+
+  // Generate a session name to share between setup execution and trace
+  const sessionName = generateSessionName();
+
+  // Run setups before tracing (same session)
+  if (hasSetups) {
+    log.info("Running setup procedures...");
+    await runSetups(
+      spec.setups as Array<{ name: string; params?: Record<string, string> }>,
+      sessionName,
+    );
+    log.blank();
+  }
+
+  const systemPrompt = buildTraceSystemPrompt(spec, {
+    sessionName,
+    skipCookiesClear: hasSetups,
+  });
+  const prompt = buildTracePrompt(spec);
+
+  log.info("Running agent-browser session...");
+  log.blank();
+
+  const routeSteps: RouteStep[] = [];
+  let overallStatus: "passed" | "failed" = "passed";
+  const traceActions: TraceAction[] = [];
+
+  const { isError } = await invokeClaudeStreaming(
+    {
+      prompt,
+      systemPrompt,
+      allowedTools: ["Bash(*)", "Read", "Grep", "Glob"],
+      env: { AGENT_BROWSER_SESSION: sessionName },
+      onAbAction: (abAction: string) => {
+        const action = parseAbAction(abAction);
+        if (action) traceActions.push(action);
+      },
+      onAbActionFailed: () => {
+        traceActions.pop();
+      },
+    },
+    (msg: SDKMessage) => {
+      if (msg.type !== "assistant") return;
+
+      for (const block of msg.message.content ?? []) {
+        if (block.type !== "text" || !block.text) continue;
+        const text = block.text;
+
+        const statusLine = parseStatusLine(text);
+        if (statusLine) log.step(statusLine.type, statusLine.stepId, statusLine.detail);
+
+        for (const line of text.split("\n")) {
+          const trimmed = line.trim();
+          if (trimmed.startsWith("ROUTE_STEP|")) {
+            const routeStep = parseRouteStep(trimmed);
+            if (routeStep) {
+              routeSteps.push(routeStep);
+              if (routeStep.status === "FAILED") overallStatus = "failed";
+            }
+          } else if (trimmed.startsWith("AB_ACTION|snapshot|") || trimmed.startsWith("AB_ACTION|assert|")) {
+            const action = parseAbAction(trimmed);
+            if (action) traceActions.push(action);
+          }
+        }
+      }
+    },
+  );
+
+  if (isError) overallStatus = "failed";
+
+  const timestamp = new Date().toISOString();
+  const route: Route = { specName, timestamp, status: overallStatus, steps: routeSteps };
+
+  const [routePath, actionsPath] = await Promise.all([
+    saveRoute(featureName, specName, route),
+    saveTraceActions(featureName, specName, traceActions),
+  ]);
+
+  log.blank();
+  log.meta("route", routePath);
+  log.meta("saved", actionsPath);
+  log.meta("actions", traceActions.length);
+  log.meta("status", overallStatus.toUpperCase());
+  log.hint(`run 'ccqa generate ${featureName}/${specName}' to generate a test script`);
+}
+
+/**
+ * Execute setup procedures by running their test.spec.ts via vitest with a fixed session name.
+ * Creates a temporary runner script that sets the session and imports each setup's test body.
+ */
+async function runSetups(
+  setups: Array<{ name: string; params?: Record<string, string> }>,
+  sessionName: string,
+): Promise<void> {
+  for (const ref of setups) {
+    log.info(`  setup: ${ref.name}`);
+
+    const scriptPath = join(getSetupDir(ref.name), "test.spec.ts");
+    let script = await readFile(scriptPath, "utf-8").catch(() => {
+      throw new Error(`Setup test script not found: ${scriptPath}. Run \`ccqa generate-setup ${ref.name}\` first.`);
+    });
+
+    // Replace placeholders with params
+    for (const [key, value] of Object.entries(ref.params ?? {})) {
+      script = script.replaceAll(`{{${key}}}`, value);
+    }
+
+    // Fix the session name to share with the trace phase
+    script = script.replace(
+      /process\.env\.AGENT_BROWSER_SESSION\s*=\s*`.+`;/,
+      `process.env.AGENT_BROWSER_SESSION = ${JSON.stringify(sessionName)};`,
+    );
+
+    // Write temp file, run vitest, clean up
+    const tmpPath = join(getSetupDir(ref.name), `_run.spec.ts`);
+    await writeFile(tmpPath, script, "utf-8");
+
+    try {
+      const proc = Bun.spawn(["bunx", "vitest", "run", tmpPath], {
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      const [stdout, stderr, exitCode] = await Promise.all([
+        new Response(proc.stdout).text(),
+        new Response(proc.stderr).text(),
+        proc.exited,
+      ]);
+      process.stdout.write(stdout);
+      if (stderr) process.stderr.write(stderr);
+
+      if (exitCode !== 0) {
+        throw new Error(`Setup '${ref.name}' failed (exit ${exitCode})`);
+      }
+    } finally {
+      await unlink(tmpPath).catch(() => {});
+    }
+  }
+}
+
+export function parseStatusLine(text: string): ParsedStatusLine | null {
+  for (const line of text.split("\n")) {
+    const match = line.match(/^(STEP_START|STEP_DONE|ASSERTION_FAILED|STEP_SKIPPED|RUN_COMPLETED)\|([^|]*)\|(.*)$/);
+    if (match) {
+      return {
+        type: match[1] as ParsedStatusLine["type"],
+        stepId: match[2] ?? "",
+        detail: match[3] ?? "",
+      };
+    }
+  }
+  return null;
+}
+
+export function parseRouteStep(line: string): RouteStep | null {
+  const parts = line.split("|");
+  if (parts.length < 6) return null;
+
+  const title = parts[2] ?? "";
+  const action = (parts[3] ?? "").replace(/^ACTION:/, "").trim();
+  const observation = (parts[4] ?? "").replace(/^OBSERVATION:/, "").trim();
+  const statusRaw = (parts[5] ?? "").replace(/^STATUS:/, "").trim();
+
+  const status = (["PASSED", "FAILED", "SKIPPED"] as const).find((s) => s === statusRaw) ?? "FAILED";
+  return { title, action, observation, status };
+}
+
+export function parseAbAction(line: string): TraceAction | null {
+  if (!line.startsWith("AB_ACTION|")) return null;
+  const parts = line.split("|");
+  const command = parts[1] as TraceCommand | undefined;
+
+  switch (command) {
+    case "cookies_clear":
+      return { command };
+    case "open":
+      return { command, value: parts[2] };
+    case "press":
+      return { command, value: parts[2] };
+    case "scroll":
+      return { command, direction: parts[2], pixels: parts[3] };
+    case "snapshot":
+      return { command, observation: parts[2] };
+    case "assert":
+      return {
+        command,
+        assertType: parts[2] as AssertType,
+        selector: parts[3] || undefined,
+        value: parts[4] || undefined,
+        observation: parts[5] || undefined,
+      };
+    case "click":
+    case "dblclick":
+    case "check":
+    case "uncheck":
+    case "hover":
+      return { command, selector: parts[2], label: parts[3] };
+    case "wait": {
+      const isTextWait = parts[2] === "--text";
+      const selector = isTextWait ? `text=${parts[3]}` : parts[2];
+      return { command, selector, label: isTextWait ? parts[4] : parts[3] };
+    }
+    case "fill":
+    case "type":
+    case "select":
+      return { command, selector: parts[2], value: parts[3], label: parts[4] };
+    case "drag":
+      return { command, selector: parts[2], target: parts[3], label: parts[4] };
+    default:
+      return null;
+  }
+}

package/src/codegen/actions-to-script.ts

@@ -0,0 +1,188 @@
+import type { TraceAction } from "../types.ts";
+
+/**
+ * Converts recorded trace actions into a vitest-compatible test.spec.ts.
+ * Uses child_process.spawnSync with explicit argument arrays to avoid shell quoting issues.
+ * agent-browser bin is resolved via import.meta.resolve to avoid hardcoded absolute paths.
+ */
+export interface SetupScript {
+  name: string;
+  body: string;
+}
+
+export function actionsToScript(actions: TraceAction[], title: string, setupScripts?: SetupScript[]): string {
+  // Resolve the helpers path relative to this file so it works from any cwd
+  const helpersPath = new URL("../runtime/test-helpers.ts", import.meta.url).pathname;
+
+  const imports = [
+    `import { test } from "vitest";`,
+    `import { spawnSync } from "node:child_process";`,
+    `import { ab, abWait, abAssertTextVisible, abAssertVisible, abAssertNotVisible, abAssertUrl, abAssertEnabled, abAssertDisabled, abAssertChecked, abAssertUnchecked } from ${JSON.stringify(helpersPath)};`,
+    "",
+    `// Single session shared across all tests — reset per run via cookies clear in first test`,
+    `process.env.AGENT_BROWSER_SESSION = \`ccqa-run-\${Date.now()}\`;`,
+    "",
+  ];
+
+  const parts: string[] = [...imports];
+
+  // Setup tests (same session — cookies clear in first setup ensures clean state)
+  if (setupScripts?.length) {
+    for (const setup of setupScripts) {
+      parts.push(
+        `test("setup: ${setup.name}", () => {`,
+        setup.body,
+        "}, 3 * 60 * 1000);",
+        "",
+      );
+    }
+  }
+
+  // Main test (same session — setup state is preserved, no cookies clear)
+  const testLines = actionsToLines(actions);
+  const body = testLines.map((l) => `  ${l}`).join("\n");
+  parts.push(
+    `test(${JSON.stringify(title)}, () => {`,
+    body,
+    "}, 5 * 60 * 1000);",
+    "",
+  );
+
+  return parts.join("\n");
+}
+
+/** Commands that interact with page elements and need the page to be loaded */
+const ELEMENT_COMMANDS = new Set<string>(["click", "dblclick", "fill", "type", "check", "uncheck", "select", "hover", "drag"]);
+
+function actionsToLines(actions: TraceAction[]): string[] {
+  const lines: string[] = [];
+  let prevLine: string | null = null;
+  let prevCommand: string | null = null;
+  for (const action of actions) {
+    const line = actionToLine(action);
+    if (line === null) continue;
+    if (line === prevLine) continue;
+    // After 'open', always insert a sleep — page load is guaranteed to be needed
+    if (prevCommand === "open" && ELEMENT_COMMANDS.has(action.command)) {
+      lines.push(`spawnSync("sleep", ["3"], { stdio: "inherit" });`);
+    }
+    lines.push(line);
+    prevLine = line;
+    prevCommand = action.command;
+  }
+  return lines;
+}
+
+/** Returns true if a selector is a session-specific @ref that cannot be replayed. */
+function isRefSelector(selector: string | undefined): boolean {
+  return typeof selector === "string" && /^@/.test(selector.trim());
+}
+
+function actionToLine(action: TraceAction): string | null {
+  // Skip actions that use @ref selectors — they are session-specific and not replayable
+  if ("selector" in action && isRefSelector(action.selector)) return null;
+
+  switch (action.command) {
+    case "cookies_clear":
+      return `ab("cookies", "clear");`;
+
+    case "open": {
+      // Strip stray surrounding quotes that can appear when agent-browser is called with quoted URL
+      const url = (action.value ?? "").replace(/^["']|["']$/g, "");
+      return `ab("open", ${j(url)});`;
+    }
+
+    case "snapshot":
+      return action.observation ? `// ${action.observation}` : null;
+
+    case "click":
+      return `ab("click", ${j(action.selector!)});`;
+
+    case "dblclick":
+      return `ab("dblclick", ${j(action.selector!)});`;
+
+    case "fill":
+      return `ab("fill", ${j(action.selector!)}, ${j(action.value!)});`;
+
+    case "type":
+      return `ab("fill", ${j(action.selector!)}, ${j(action.value!)});`;
+
+    case "check":
+      return `ab("check", ${j(action.selector!)});`;
+
+    case "uncheck":
+      return `ab("uncheck", ${j(action.selector!)});`;
+
+    case "press":
+      return `ab("press", ${j(action.value!)});`;
+
+    case "select":
+      return `ab("select", ${j(action.selector!)}, ${j(action.value!)});`;
+
+    case "hover":
+      return `ab("hover", ${j(action.selector!)});`;
+
+    case "scroll": {
+      const args = [action.direction ?? "down", ...(action.pixels ? [action.pixels] : [])];
+      return `ab("scroll", ${args.map(j).join(", ")});`;
+    }
+
+    case "drag":
+      return `ab("drag", ${j(action.selector!)}, ${j(action.target!)});`;
+
+    case "wait": {
+      const sel = action.selector!;
+      // Numeric waits represent sleep durations (from auto-fix)
+      if (/^\d+$/.test(sel)) return `spawnSync("sleep", [${j(sel)}], { stdio: "inherit" });`;
+      return `abWait(${j(sel)});`;
+    }
+
+    case "assert": {
+      // LLM may omit selector/value fields and put the text in observation instead
+      // Fall back to observation when the specific field is missing
+      const val = action.value ?? action.observation;
+      const sel = action.selector ?? action.observation;
+      const comment = action.observation ? `// Assert: ${action.observation}` : null;
+      let assertLine: string | null = null;
+      switch (action.assertType) {
+        case "text_visible":
+          if (val) assertLine = `abAssertTextVisible(${j(val)});`;
+          break;
+        case "text_not_visible":
+          if (val) assertLine = `abAssertNotVisible(${j("text=" + val)}, 180_000);`;
+          break;
+        case "element_visible":
+          if (sel) assertLine = `abAssertVisible(${j(sel)});`;
+          break;
+        case "element_not_visible":
+          if (sel) assertLine = `abAssertNotVisible(${j(sel)});`;
+          break;
+        case "url_contains":
+          if (val) assertLine = `abAssertUrl(${j(val)});`;
+          break;
+        case "element_enabled":
+          // is enabled is unreliable with text= and [aria-label=] selectors that may not exist in DOM
+          if (sel && !sel.startsWith("text=") && !sel.startsWith("[aria-label=")) assertLine = `abAssertEnabled(${j(sel)});`;
+          break;
+        case "element_disabled":
+          // is enabled is unreliable with text= and [aria-label=] selectors that may not exist in DOM
+          if (sel && !sel.startsWith("text=") && !sel.startsWith("[aria-label=")) assertLine = `abAssertDisabled(${j(sel)});`;
+          break;
+        case "element_checked":
+          if (sel) assertLine = `abAssertChecked(${j(sel)});`;
+          break;
+        case "element_unchecked":
+          if (sel) assertLine = `abAssertUnchecked(${j(sel)});`;
+          break;
+      }
+      if (comment && assertLine) return `${comment}\n  ${assertLine}`;
+      return assertLine ?? comment;
+    }
+
+    default:
+      return null;
+  }
+}
+
+/** JSON.stringify — produces a quoted string literal safe for embedding in TS source. */
+const j = (s: string) => JSON.stringify(s);

package/src/prompts/codegen.ts

@@ -0,0 +1,73 @@
+import type { TraceAction } from "../types.ts";
+
+export function buildAutoFixPrompt(script: string, failureLog: string): string {
+  return `You are analyzing a failing E2E test script. The test fails because some browser actions execute before the page has finished loading or navigating.
+
+Your task: identify which line numbers need a sleep/wait inserted BEFORE them to fix timing issues.
+
+## Rules
+- ONLY identify lines where a sleep is needed — do NOT suggest any other changes
+- Common patterns that need a sleep:
+  - After \`ab("open", ...)\` when the next line interacts with elements (fill, click, etc.)
+  - After \`ab("press", "Enter")\` or \`ab("click", ...)\` when a page navigation occurs before the next action
+  - After any action that triggers a redirect or page reload
+- Look at the error log to identify WHICH lines failed, then determine if a sleep before that line would fix it
+- If a \`spawnSync("sleep", ...)\` already exists before a failing line, suggest increasing its duration instead
+- Output ONLY a JSON array of objects, no explanation, no markdown code fences
+
+## Output format
+Each object has:
+- "line": the 1-based line number to insert a sleep BEFORE
+- "seconds": recommended sleep duration (typically 3-5)
+- "reason": very short explanation (e.g., "page navigation after form submit")
+
+If a sleep already exists and needs to be increased:
+- "line": the line number of the existing sleep
+- "increase_to": the new duration in seconds
+- "reason": explanation
+
+Example output:
+[{"line": 15, "seconds": 3, "reason": "page navigation after press Enter"}, {"line": 22, "increase_to": 5, "reason": "slow page load"}]
+
+If no fixes are needed, return: []
+
+## Test Script (with line numbers)
+${script.split("\n").map((l, i) => `${i + 1}: ${l}`).join("\n")}
+
+## Failure Log
+${failureLog.slice(0, 3000)}`;
+}
+
+export function buildCleanupPrompt(actions: TraceAction[]): string {
+  const lines = actions
+    .map((a, i) => {
+      const parts = [`${i + 1}. ${a.command}`];
+      if (a.assertType) parts.push(`assertType="${a.assertType}"`);
+      if (a.selector) parts.push(`selector="${a.selector}"`);
+      if (a.value) parts.push(`value="${a.value}"`);
+      if (a.observation) parts.push(`→ ${a.observation}`);
+      return parts.join(" ");
+    })
+    .join("\n");
+
+  return `You are given a list of browser actions recorded during an E2E test trace.
+The trace contains noise: failed attempts, redundant retries, and duplicate operations recorded because the agent explored multiple strategies.
+
+Your task: return a **cleaned-up JSON array** of TraceAction objects that represents the minimal, correct sequence of actions needed to reproduce the test.
+
+Each TraceAction object has the following shape (use EXACTLY these field names):
+{ "command": "...", "assertType": "...", "selector": "...", "value": "...", "label": "...", "observation": "..." }
+Only include fields that are present in the original action. The "command" field is required. For assert actions, "assertType" is also required.
+
+Rules:
+- Remove actions that were failed attempts superseded by a later successful action (e.g., if \`fill selector="text=Foo"\` was followed by \`fill selector="[placeholder='Foo']"\`, keep only the latter)
+- Remove duplicate fill operations on the same field (keep only the last successful fill for each field)
+- For \`click\` and \`fill\` actions: if the selector starts with \`text=\`, it is a failed attempt — remove it (text= selectors only work with the wait command, not click/fill)
+- Keep all snapshot actions — they serve as comments/observations in the generated test
+- Keep all assert actions — they are the test's verification points and must not be removed
+- Do NOT invent new actions or change values
+- Output ONLY a valid JSON array, no explanation, no markdown code fences
+
+## Recorded Actions
+${lines}`;
+}