@gethmy/harness 1.0.0

package/src/verification.ts

@@ -0,0 +1,711 @@
+import { type ChildProcess, execFileSync, spawn } from "node:child_process";
+import type { HarmonyApiClient } from "@gethmy/mcp/src/api-client.js";
+import type { VerificationConfig } from "./exec-types.js";
+import { log } from "./log.js";
+import { spawnRunArgs } from "./pm.js";
+import {
+  buildCommand,
+  formatFixCommand,
+  lintCommand,
+  supportsDevServer,
+  testCommand,
+} from "./project-type.js";
+import { findDeletedTestFiles } from "./revert-guard.js";
+
+const TAG = "verification";
+
+/**
+ * Output cap for the verification steps' `execFileSync` calls. Node defaults to
+ * 1 MB, and on overflow `execFileSync` kills the child and throws regardless of
+ * exit code (`ENOBUFS`, `status: null`) — so a *passing* suite that merely
+ * printed a lot landed in the failure path and its ordinary green lines were
+ * parsed as assertion failures (#701). A verbose reporter (`jest --verbose`, a
+ * console.log-heavy suite, a CI reporter) clears 1 MB easily.
+ */
+const MAX_OUTPUT_BUFFER = 64 * 1024 * 1024;
+
+export interface VerificationResult {
+  passed: boolean;
+  buildErrors: string[];
+  /**
+   * Failures from the repo's own test suite (#688). Non-empty blocks the branch
+   * before Review — a run that breaks tests is a failed attempt. Empty when the
+   * repo has no resolvable test command or the step is disabled.
+   */
+  testFailures: string[];
+  lintWarnings: string[];
+  reviewFindings: string[];
+  /**
+   * Branch reverts already-merged work — currently: test/spec files it deletes
+   * relative to current main (#408). Non-empty blocks the branch before Review.
+   */
+  revertWarnings: string[];
+}
+
+// ============ PUBLIC API ============
+
+export async function runVerification(
+  worktreePath: string,
+  config: VerificationConfig,
+  workerId: number,
+): Promise<VerificationResult> {
+  const result: VerificationResult = {
+    passed: true,
+    buildErrors: [],
+    testFailures: [],
+    lintWarnings: [],
+    reviewFindings: [],
+    revertWarnings: [],
+  };
+
+  if (config.verification.revertGuard) {
+    log.info(TAG, `[worker:${workerId}] Checking for reverted merged work...`);
+    const deletedTests = findDeletedTestFiles(
+      worktreePath,
+      config.worktree.baseBranch,
+    );
+    if (deletedTests.length > 0) {
+      result.revertWarnings = deletedTests.map(
+        (f) =>
+          `Branch deletes test file '${f}' relative to current ${config.worktree.baseBranch} — ` +
+          "likely an accidental revert of already-merged work. Restore the test or rebase on current main.",
+      );
+      log.warn(
+        TAG,
+        `[worker:${workerId}] Revert guard tripped: ${deletedTests.length} deleted test file(s)`,
+      );
+      // Deleting a bound regression test removes the safety net — block before
+      // Review regardless of build/lint outcome (#408).
+      result.passed = false;
+    } else {
+      log.info(TAG, `[worker:${workerId}] Revert guard passed`);
+    }
+  }
+
+  if (config.verification.build) {
+    log.info(TAG, `[worker:${workerId}] Running build...`);
+    result.buildErrors = runBuild(worktreePath, config.verification.timeout);
+    if (result.buildErrors.length > 0) {
+      log.warn(
+        TAG,
+        `[worker:${workerId}] Build failed with ${result.buildErrors.length} error(s)`,
+      );
+      result.passed = false;
+    } else {
+      log.info(TAG, `[worker:${workerId}] Build passed`);
+    }
+  }
+
+  // Tests run only against a repo that compiles. A broken build fails the suite
+  // for reasons the test output can't explain, and the auto-fix loop re-runs
+  // verification from the top anyway — so once the build is fixed the suite
+  // still gates the branch.
+  if (config.verification.test && result.buildErrors.length === 0) {
+    log.info(TAG, `[worker:${workerId}] Running tests...`);
+    result.testFailures = runTests(
+      worktreePath,
+      config.verification.testTimeout,
+    );
+    if (result.testFailures.length > 0) {
+      log.warn(
+        TAG,
+        `[worker:${workerId}] Tests failed with ${result.testFailures.length} failure(s)`,
+      );
+      // A broken suite is a broken branch — block before Review (#688).
+      result.passed = false;
+    } else {
+      log.info(TAG, `[worker:${workerId}] Tests passed`);
+    }
+  }
+
+  if (config.verification.lint) {
+    log.info(TAG, `[worker:${workerId}] Running lint...`);
+    result.lintWarnings = runLint(worktreePath, config.verification.timeout);
+    if (result.lintWarnings.length > 0) {
+      log.warn(
+        TAG,
+        `[worker:${workerId}] Lint found ${result.lintWarnings.length} issue(s)`,
+      );
+      // Lint warnings alone don't block — only build errors block
+    } else {
+      log.info(TAG, `[worker:${workerId}] Lint passed`);
+    }
+  }
+
+  if (config.verification.deepReview) {
+    log.info(TAG, `[worker:${workerId}] Running deep review...`);
+    result.reviewFindings = await runDeepReview(worktreePath, config, workerId);
+    if (result.reviewFindings.length > 0) {
+      log.warn(
+        TAG,
+        `[worker:${workerId}] Deep review found ${result.reviewFindings.length} finding(s)`,
+      );
+    } else {
+      log.info(TAG, `[worker:${workerId}] Deep review passed`);
+    }
+  }
+
+  return result;
+}
+
+export function runBuild(worktreePath: string, timeout: number): string[] {
+  const command = buildCommand(worktreePath);
+  if (!command) {
+    log.warn(
+      TAG,
+      `No known build toolchain for ${worktreePath} — skipping build`,
+    );
+    return [];
+  }
+  try {
+    execFileSync(command.cmd, command.args, {
+      cwd: worktreePath,
+      timeout,
+      stdio: "pipe",
+      maxBuffer: MAX_OUTPUT_BUFFER,
+    });
+    return [];
+  } catch (err: unknown) {
+    return parseErrorOutput(err);
+  }
+}
+
+/**
+ * Run the repo's own test suite. Returns the failures (empty = passed, or no
+ * suite to run). A non-zero exit ALWAYS yields at least one entry — an
+ * unparsable failure must never read as a pass (#688).
+ */
+export function runTests(worktreePath: string, timeout: number): string[] {
+  const command = testCommand(worktreePath);
+  if (!command) {
+    log.warn(
+      TAG,
+      `No test command for detected toolchain in ${worktreePath} — skipping tests`,
+    );
+    return [];
+  }
+  try {
+    execFileSync(command.cmd, command.args, {
+      cwd: worktreePath,
+      timeout,
+      stdio: "pipe",
+      maxBuffer: MAX_OUTPUT_BUFFER,
+    });
+    return [];
+  } catch (err: unknown) {
+    // The suite's own output is the diagnostic — put its tail in the run log,
+    // not only the parsed lines that become subtasks.
+    const output = combineOutput(err);
+    log.warn(
+      TAG,
+      `Test run failed:\n${output.slice(-4000) || "(no output captured)"}`,
+    );
+    return parseTestFailures(err, timeout);
+  }
+}
+
+/**
+ * Run the repo's write-mode auto-fixer over the worktree so deterministic
+ * formatter drift is fixed in place (#691). The completion pipeline calls this
+ * *before* `commitUncommittedChanges` + the pre-verify push, NOT from inside
+ * `runVerification` — verification runs after that push, so formatting there
+ * would never reach origin (and thus never clear CI's blocking lint gate). The
+ * fixed files are folded into the commit by `commitUncommittedChanges` and
+ * pushed. Best-effort: a repo with no declared fixer is a no-op, and a fixer
+ * that exits non-zero (e.g. it wrote what it could but flagged an unfixable
+ * rule) is logged, never thrown. This does NOT change the lint-warn-only
+ * policy — the later `runLint` still reports whatever the fixer left behind.
+ */
+export function runFormatFix(
+  worktreePath: string,
+  timeout: number,
+  workerId: number,
+): void {
+  const command = formatFixCommand(worktreePath);
+  if (!command) return;
+  try {
+    execFileSync(command.cmd, command.args, {
+      cwd: worktreePath,
+      timeout,
+      stdio: "pipe",
+      maxBuffer: MAX_OUTPUT_BUFFER,
+    });
+    log.info(
+      TAG,
+      `[worker:${workerId}] Auto-formatted worktree before commit/push`,
+    );
+  } catch (err: unknown) {
+    log.warn(
+      TAG,
+      `[worker:${workerId}] Auto-format step exited non-zero (non-fatal): ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+}
+
+export function runLint(worktreePath: string, timeout: number): string[] {
+  const command = lintCommand(worktreePath);
+  if (!command) {
+    log.info(
+      TAG,
+      `No lint step for detected toolchain in ${worktreePath} — skipping lint`,
+    );
+    return [];
+  }
+  try {
+    execFileSync(command.cmd, command.args, {
+      cwd: worktreePath,
+      timeout,
+      stdio: "pipe",
+      maxBuffer: MAX_OUTPUT_BUFFER,
+    });
+    return [];
+  } catch (err: unknown) {
+    return parseErrorOutput(err);
+  }
+}
+
+export async function runDeepReview(
+  worktreePath: string,
+  config: VerificationConfig,
+  workerId: number,
+): Promise<string[]> {
+  // Deep review boots a `dev` server and probes it over HTTP — only Node
+  // repos support that. Skip for Swift/iOS and unknown toolchains.
+  if (!supportsDevServer(worktreePath)) {
+    log.info(
+      TAG,
+      `[worker:${workerId}] Detected non-web toolchain — skipping deep review`,
+    );
+    return [];
+  }
+
+  const port = config.verification.devServerBasePort + workerId;
+  let devServer: ChildProcess | null = null;
+
+  try {
+    // Start dev server in background
+    const [cmd, args] = spawnRunArgs("dev", "--port", String(port));
+    devServer = spawn(cmd, args, {
+      cwd: worktreePath,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    // Wait for dev server to be ready, then confirm it answers HTTP.
+    try {
+      await waitForDevServer(devServer, 30_000);
+      await probeDevServer(port);
+    } catch (err) {
+      log.error(
+        TAG,
+        `Dev server did not become ready: ${err instanceof Error ? err.message : err}`,
+      );
+      return [];
+    }
+
+    // Get diff for review context
+    let diff = "";
+    try {
+      // Without a raised cap a diff over 1 MB throws and silently degrades to
+      // "(unable to retrieve diff)" — reviewing the change with no change (#701).
+      diff = execFileSync(
+        "git",
+        ["diff", `origin/${config.worktree.baseBranch}..HEAD`],
+        {
+          cwd: worktreePath,
+          encoding: "utf-8",
+          timeout: 30_000,
+          maxBuffer: MAX_OUTPUT_BUFFER,
+        },
+      );
+    } catch {
+      diff = "(unable to retrieve diff)";
+    }
+
+    // Spawn Claude for review
+    const reviewPrompt = [
+      "You are reviewing code changes for quality and correctness.",
+      `A dev server is running at http://localhost:${port}.`,
+      "Review the following diff and report any issues found.",
+      "Output ONLY a numbered list of findings, one per line.",
+      "If no issues, output: No issues found.",
+      "",
+      "```diff",
+      diff.slice(0, 50_000),
+      "```",
+    ].join("\n");
+
+    const leanSources = config.claude.leanSettingSources;
+    const output = execFileSync(
+      "claude",
+      [
+        "--print",
+        "--model",
+        "sonnet",
+        "--max-turns",
+        "10",
+        // Lean spawn — skip project CLAUDE.md/@-imports (#348).
+        ...(leanSources ? ["--setting-sources", leanSources] : []),
+        "--",
+        reviewPrompt,
+      ],
+      {
+        cwd: worktreePath,
+        encoding: "utf-8",
+        timeout: config.verification.timeout,
+        stdio: "pipe",
+        maxBuffer: MAX_OUTPUT_BUFFER,
+      },
+    );
+
+    return parseReviewFindings(output);
+  } catch (err) {
+    log.error(
+      TAG,
+      `Deep review failed: ${err instanceof Error ? err.message : err}`,
+    );
+    return [];
+  } finally {
+    if (devServer && !devServer.killed) {
+      devServer.kill("SIGTERM");
+    }
+  }
+}
+
+export function attemptAutoFix(
+  worktreePath: string,
+  config: VerificationConfig,
+  errors: string[],
+): void {
+  const errorSummary = errors.slice(0, 20).join("\n");
+  const fixPrompt = [
+    "The following build, test, and lint failures were found after implementing a feature.",
+    "Fix the source files to resolve them.",
+    "Do NOT commit build artifacts or modify files in dist/.",
+    "Fix source files only.",
+    // Without this the cheapest way to make a failing suite pass is to delete
+    // or neuter the test — which ships the regression it caught (#688).
+    "For a failing test: fix the code under test. Do NOT delete, skip, or weaken",
+    "a test to make it pass — unless the test itself is provably wrong, and then",
+    "say so explicitly.",
+    "",
+    "Failures:",
+    "```",
+    errorSummary,
+    "```",
+  ].join("\n");
+
+  const leanSources = config.claude.leanSettingSources;
+  const args = [
+    "--print",
+    "--model",
+    config.claude.model,
+    "--max-turns",
+    "50",
+    "--allowedTools",
+    "Bash,Read,Write,Edit,Glob,Grep",
+    // Lean spawn — concrete build/lint errors, no project docs needed (#348).
+    ...(leanSources ? ["--setting-sources", leanSources] : []),
+    "--",
+    fixPrompt,
+  ];
+
+  log.info(TAG, "Spawning Claude for auto-fix...");
+  // A long fix run prints well past 1 MB; without a raised cap the overflow
+  // throws and aborts the auto-fix loop even when the fix itself worked (#701).
+  execFileSync("claude", args, {
+    cwd: worktreePath,
+    timeout: config.verification.timeout,
+    stdio: "pipe",
+    maxBuffer: MAX_OUTPUT_BUFFER,
+  });
+}
+
+export interface RecoveryInfo {
+  /** Remote ref where the failed attempt was pushed. */
+  branchName: string;
+  /** Public URL of the branch (GitHub/GitLab/Bitbucket tree view), if known. */
+  branchUrl: string | null;
+}
+
+export async function reportFindings(
+  client: HarmonyApiClient,
+  cardId: string,
+  result: VerificationResult,
+  recovery?: RecoveryInfo | null,
+): Promise<void> {
+  const items: string[] = [];
+
+  if (recovery) {
+    const cmd = `git fetch && git checkout ${recovery.branchName}`;
+    const url = recovery.branchUrl ? ` (${recovery.branchUrl})` : "";
+    items.push(`Recovery: \`${cmd}\`${url}`);
+  }
+
+  // Revert warnings first — a deleted regression test is the most severe
+  // finding (it can mask a reintroduced bug), so it leads the subtask list.
+  for (const warn of result.revertWarnings) {
+    items.push(`Revert: ${warn}`);
+  }
+  for (const err of result.buildErrors) {
+    items.push(`Build: ${err}`);
+  }
+  for (const err of result.testFailures) {
+    items.push(`Test: ${err}`);
+  }
+  for (const err of result.lintWarnings) {
+    items.push(`Lint: ${err}`);
+  }
+  for (const finding of result.reviewFindings) {
+    items.push(`Review: ${finding}`);
+  }
+
+  const maxSubtasks = 10;
+  const overflow = items.length - maxSubtasks;
+  const toCreate = items.slice(0, maxSubtasks);
+
+  await Promise.all(
+    toCreate.map(async (item) => {
+      const title = item.length > 120 ? `${item.slice(0, 117)}...` : item;
+      try {
+        await client.createSubtask(cardId, title);
+      } catch (err) {
+        log.error(
+          TAG,
+          `Failed to create subtask: ${err instanceof Error ? err.message : err}`,
+        );
+      }
+    }),
+  );
+
+  if (overflow > 0) {
+    try {
+      await client.createSubtask(cardId, `...and ${overflow} more issues`);
+    } catch {
+      // best-effort
+    }
+  }
+
+  log.info(
+    TAG,
+    `Reported ${Math.min(items.length, maxSubtasks)} finding(s) as subtasks on card ${cardId}`,
+  );
+}
+
+// ============ HELPERS ============
+
+function combineOutput(err: unknown): string {
+  const stderr =
+    (err as { stderr?: Buffer | string })?.stderr?.toString() ?? "";
+  const stdout =
+    (err as { stdout?: Buffer | string })?.stdout?.toString() ?? "";
+  return `${stderr}\n${stdout}`;
+}
+
+function parseErrorOutput(err: unknown): string[] {
+  const combined = combineOutput(err);
+
+  const lines = combined
+    .split("\n")
+    .map((l) => l.trim())
+    .filter(
+      (l) =>
+        l.length > 0 &&
+        (l.includes("error") ||
+          l.includes("Error") ||
+          l.includes("✖") ||
+          l.includes("×")),
+    )
+    .map((l) => (l.length > 200 ? `${l.slice(0, 197)}...` : l));
+
+  // If we couldn't parse specific error lines, return the whole output truncated
+  if (lines.length === 0 && combined.trim().length > 0) {
+    return [combined.trim().slice(0, 200)];
+  }
+
+  return lines;
+}
+
+/**
+ * Lines a test runner uses to mark a failure. Deliberately broader than
+ * {@link parseErrorOutput}'s `error`-only filter: vitest/bun/jest report a
+ * failing test as `FAIL`, `✗`, `(fail)` or an assertion diff, and none of
+ * those contain the word "error".
+ */
+const TEST_FAILURE_LINE =
+  /(\bFAIL\b|\(fail\)|✗|✘|×|✖|\bfailed\b|\bfailing\b|AssertionError|\bexpect(ed)?\b|\berror\b)/i;
+
+const MAX_TEST_FAILURE_LINES = 20;
+
+/**
+ * Turn a failed test run into reportable failures. Guarantees a non-empty
+ * result: the caller only reaches here on a non-zero exit, so returning `[]`
+ * would silently promote a broken branch to Review.
+ */
+function parseTestFailures(err: unknown, timeout: number): string[] {
+  const e = err as { code?: string | number };
+
+  // execFileSync surfaces its own timeout as ETIMEDOUT (SIGTERM'd child, no
+  // usable output) — report it as itself, not as a phantom assertion failure,
+  // so the operator knows to raise the cap.
+  if (e?.code === "ETIMEDOUT") {
+    return [
+      `Test run exceeded the ${timeout}ms limit and was killed — raise agent.verification.testTimeout or narrow the suite`,
+    ];
+  }
+  if (e?.code === "ENOENT") {
+    return [
+      "Test runner not found — could not execute the repo's test command",
+    ];
+  }
+
+  // Output blew past maxBuffer: execFileSync killed the child and threw, so the
+  // exit code is gone and the captured output is a truncated prefix. Whether the
+  // suite actually passed is unknowable here — report the infrastructure limit,
+  // never the green lines that prefix happens to contain (#701).
+  if (e?.code === "ENOBUFS") {
+    return [
+      `Test output exceeded the ${MAX_OUTPUT_BUFFER / (1024 * 1024)}MB capture limit and the run was killed — ` +
+        "the suite's real result is unknown. Quieten the reporter or raise the limit.",
+    ];
+  }
+
+  const combined = combineOutput(err);
+  const lines = combined
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l.length > 0 && TEST_FAILURE_LINE.test(l))
+    .map((l) => (l.length > 200 ? `${l.slice(0, 197)}...` : l));
+
+  const unique = [...new Set(lines)].slice(0, MAX_TEST_FAILURE_LINES);
+  if (unique.length > 0) return unique;
+
+  // Exited non-zero but nothing matched — fall back to the raw tail rather
+  // than reporting a pass.
+  const tail = combined.trim().slice(-200);
+  return [tail.length > 0 ? tail : "Tests failed (no output captured)"];
+}
+
+function parseReviewFindings(output: string): string[] {
+  if (output.toLowerCase().includes("no issues found")) {
+    return [];
+  }
+
+  return output
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => /^\d+[.)]/.test(l))
+    .map((l) => l.replace(/^\d+[.)]\s*/, ""))
+    .filter((l) => l.length > 0);
+}
+
+export class DevServerReadinessError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "DevServerReadinessError";
+  }
+}
+
+/**
+ * Wait for a dev server to signal readiness on stdout/stderr.
+ *
+ * Rejects (does NOT resolve) on timeout or process error — callers that
+ * need the server to be live for correctness (e.g. the review worker)
+ * must not proceed without a confirmed signal. If the server dies before
+ * becoming ready, we reject with the exit details.
+ */
+export function waitForDevServer(
+  proc: ChildProcess,
+  timeout: number,
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const cleanup = () => {
+      proc.stdout?.off("data", onData);
+      proc.stderr?.off("data", onData);
+      proc.off("error", onError);
+      proc.off("exit", onExit);
+      clearTimeout(timer);
+    };
+
+    const settleResolve = () => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      resolve();
+    };
+    const settleReject = (err: Error) => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      reject(err);
+    };
+
+    const timer = setTimeout(() => {
+      settleReject(
+        new DevServerReadinessError(
+          `dev server did not signal readiness within ${timeout}ms`,
+        ),
+      );
+    }, timeout);
+
+    const onData = (data: Buffer) => {
+      const text = data.toString();
+      if (
+        text.includes("ready") ||
+        text.includes("localhost") ||
+        text.includes("Local:")
+      ) {
+        settleResolve();
+      }
+    };
+
+    const onError = (err: Error) => {
+      settleReject(err);
+    };
+
+    const onExit = (code: number | null, signal: NodeJS.Signals | null) => {
+      settleReject(
+        new DevServerReadinessError(
+          `dev server exited before becoming ready (code=${code ?? "?"}, signal=${signal ?? "?"})`,
+        ),
+      );
+    };
+
+    proc.stdout?.on("data", onData);
+    proc.stderr?.on("data", onData);
+    proc.on("error", onError);
+    proc.on("exit", onExit);
+  });
+}
+
+/**
+ * Verify the dev server actually responds to HTTP GET before treating
+ * it as ready. Listening alone isn't enough — a framework can bind but
+ * still crash on the first request.
+ */
+export async function probeDevServer(
+  port: number,
+  timeoutMs = 5000,
+): Promise<void> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(`http://localhost:${port}/`, {
+      signal: controller.signal,
+    });
+    if (!res.ok && res.status >= 500) {
+      throw new DevServerReadinessError(
+        `dev server returned ${res.status} on probe`,
+      );
+    }
+  } catch (err) {
+    if (err instanceof DevServerReadinessError) throw err;
+    throw new DevServerReadinessError(
+      `dev server probe failed: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  } finally {
+    clearTimeout(timer);
+  }
+}