huddle-cli 0.1.0 → 0.1.3

package/dist/orchestrator/qa.js

@@ -15,11 +15,24 @@ import fs from "node:fs";
 import path from "node:path";
 import { launchAgent, stopChild } from "./launcher.js";
 import { createWorktree, removeWorktree } from "../sandbox/worktree.js";
+import { onFinished } from "../platform.js";
 import { isBigGoal } from "./planner.js";
+import { acceptanceList } from "./goal.js";
 export const QA_ROUNDS = 2;
 /** Bugs one round may file — QA should report the serious ones, not a wishlist. */
 export const QA_MAX_BUGS = 6;
 const QA_TIMEOUT_MS = 15 * 60_000;
+/**
+ * A first clean round ends QA only if its check took at least this long
+ * (HUDDLE_QA_MIN_CLEAN_S overrides); a quicker one gets a second opinion from
+ * another agent. Found live (capstone round 5): cursor's round-1 check came
+ * back clean in 90s, QA stopped there, and 2 of the hidden checks failed
+ * on bugs a goal-by-goal check would have hit at once.
+ */
+export function qaMinCleanMs() {
+    const s = Number(process.env.HUDDLE_QA_MIN_CLEAN_S);
+    return Number.isFinite(s) && s >= 0 ? s * 1000 : 5 * 60_000;
+}
 /**
  * QA on or off for this goal: an explicit --qa / --no-qa wins; otherwise on
  * for big goals (the same heuristic as the design council), off for small
@@ -42,11 +55,19 @@ function git(cwd, args) {
 }
 const cap = (s, n) => (s.length > n ? s.slice(0, n) + "…" : s);
 const str = (v, n) => cap(String(v ?? "").trim(), n);
+/** A title cut at a word, not mid-word (found live: "… and counts i…"). */
+const capWords = (s, n) => {
+    if (s.length <= n)
+        return s;
+    const cut = s.slice(0, n);
+    const sp = cut.lastIndexOf(" ");
+    return (sp > n * 0.6 ? cut.slice(0, sp) : cut).replace(/[\s,;:—-]+$/, "") + "…";
+};
 /** Who wrote each file since `since`: the newest `huddle(<agent>): …` commit that touched it. */
 export function authorsOf(repoRoot, since, files) {
     const out = new Map();
     for (const f of files) {
-        const subjects = git(repoRoot, ["log", "--format=%s", `${since}..HEAD`, "--", f]).split("\n");
+        const subjects = git(repoRoot, ["log", "--format=%s", `${since}..HEAD`, "--", f]).split(/\r?\n/);
         const who = subjects.map((s) => /^huddle\(([^)]+)\): /.exec(s)?.[1]).find(Boolean);
         if (who)
             out.set(f, who);
@@ -56,7 +77,7 @@ export function authorsOf(repoRoot, since, files) {
 /** Every agent with a merged commit since `since`. */
 function authorsSince(repoRoot, since) {
     const n = new Map();
-    for (const s of git(repoRoot, ["log", "--format=%s", `${since}..HEAD`]).split("\n")) {
+    for (const s of git(repoRoot, ["log", "--format=%s", `${since}..HEAD`]).split(/\r?\n/)) {
         const who = /^huddle\(([^)]+)\): /.exec(s)?.[1];
         if (who)
             n.set(who, (n.get(who) ?? 0) + 1);
@@ -111,18 +132,23 @@ ${o.stat || "(none)"}
 
 THE DIFF (capped):
 ${o.diff || "(empty)"}
-${o.earlier.length ? `\nALREADY REPORTED in earlier rounds (their fixes merged or are in flight) — report one again only if it is STILL broken:\n${o.earlier.map((b) => `- ${b.title}${b.files.length ? ` (${b.files.join(", ")})` : ""}`).join("\n")}\n` : ""}
+${o.checklist ? `\nACCEPTANCE CHECKLIST — the GOAL's requirements, one by one:\n${o.checklist}\n` : ""}${o.earlier.length ? `\nALREADY REPORTED in earlier rounds (their fixes merged or are in flight) — report one again only if it is STILL broken:\n${o.earlier.map((b) => `- ${b.title}${b.files.length ? ` (${b.files.join(", ")})` : ""}`).join("\n")}\n` : ""}
+The GOAL is the spec you check against. The team's contract (HUDDLE_CONTRACT.md), README, task specs and tests are NOT: where any of them says something the goal doesn't, the goal wins, and code that follows them instead is a bug (e.g. a generic "invalid arguments" error where the goal gives an exact message for that case). Passing tests prove only what their authors thought of.
+
 DO, in order (huddle_progress after each step):
 1. Run the full test suite: ${o.testCmd ? `\`${o.testCmd}\`` : "(no test command detected — look for one, or skip)"}.
-2. Use it the way a user would, per the goal: run the app / CLI / functions with real inputs, edge cases, bad input.
+2. Go through the ACCEPTANCE CHECKLIST item by item: run the app / CLI / API the way a user would and compare the output with the goal byte for byte. For each item use the smallest input that triggers it, and its edge cases:
+   - every exact error message: trigger it with the input left out entirely (no argument, no flag, no field) AND given but empty or invalid; check the stream (stdout vs stderr), the exit code and the HTTP status
+   - every parser / import rule: special characters inside a quoted field AND inside an unquoted one, escapes, blank lines, extra or missing fields, rows that must be skipped, and the exact counts
+   - every exact output format: spacing, ordering, the empty case, the final newline
 3. Read the diff for flaws: bugs, modules from different agents that don't fit together, missing requirements, crashes.
    Start with the tasks marked NOT reviewed above: nobody but their author has read that code.
 4. For EACH concrete, reproducible defect: huddle_report_bug(title, repro, expected, actual, files).
    - When a test can show it, write a regression test in this repo's test style that FAILS now (and will pass once fixed),
      then pass its path as test_file — the fix task gets it. Don't fix the bug yourself.
    - Only real defects, at most ${o.maxBugs}, most serious first. No style nits, no "nice to have".
-5. huddle_done(task_id, summary): what you checked, and "N bugs filed" or "clean".
-Don't commit. If everything works, say so — a clean round ends QA.`;
+5. huddle_done(task_id, summary): which checklist items you ran, and "N bugs filed" or "clean".
+Don't commit. If everything works, say so — but only after every checklist item was run.`;
 }
 export async function runQa(o) {
     const { db, room, repoRoot, signal, onEvent } = o;
@@ -139,21 +165,28 @@ export async function runQa(o) {
     }
     // Who checks this round: someone who didn't write the code, rotating; an
     // author only when everyone wrote some (fewest commits first).
+    // After a quick clean round, the second opinion comes from someone else (when there is anyone).
+    let secondOpinion = new Set();
     const pick = () => {
         const wrote = authorsSince(repoRoot, o.startHead);
-        return local
+        const others = local.filter((w) => !secondOpinion.has(w.id));
+        return (others.length ? others : local)
             .map((w, i) => ({ w, i, author: wrote.has(w.id), n: wrote.get(w.id) ?? 0, u: used.get(w.id) ?? 0 }))
             .sort((a, b) => Number(a.author) - Number(b.author) || a.u - b.u || a.n - b.n || a.i - b.i)
             .slice(0, Math.max(1, o.perRound ?? 1))
             .map(({ w, author }) => ({ w, author }));
     };
     const worktrees = new Map(); // QA agent -> its worktree (to read a test_file)
+    const checklist = acceptanceList(o.goal);
+    const minCleanMs = o.minCleanMs ?? qaMinCleanMs();
+    let cleanRounds = 0;
+    let slowest = 0; // the longest finished check in this round
     let round = 0;
     let roundBugs = 0;
     let finished = 0; // checks in this round that ran to the end
     /** huddle_report_bug while a round runs: validate, dedupe, cap, file a routed fix task. */
     const onBug = (raw, from) => {
-        const title = str(raw.title, 120);
+        const title = capWords(String(raw.title ?? "").trim().replace(/\s+/g, " "), 120);
         const repro = str(raw.repro, 2000);
         if (!title || !repro)
             return { ok: false, error: "title and repro are required — exact steps or a command that shows the bug" };
@@ -200,7 +233,7 @@ export async function runQa(o) {
             .filter(Boolean)
             .join("\n");
         const task = db.addTask({
-            title: cap(`Fix: ${title}`, 120),
+            title: capWords(`Fix: ${title}`, 120),
             spec,
             prefers: ["quickfix", ...(assignee ? [`@${assignee}`] : [])],
             files: [...files, ...(testCode && !testFile.startsWith("(") ? [testFile] : [])],
@@ -243,6 +276,7 @@ export async function runQa(o) {
                 approvals: o.approvals,
                 worktree: wt,
             });
+            o.budget?.track(h.meter, h.agentId);
         }
         catch (e) {
             db.setTaskStatus(task.id, "failed");
@@ -275,23 +309,34 @@ export async function runQa(o) {
             worktrees.delete(w.id);
             const bugs = roundBugsBy(w.id) - before;
             const reason = code === 0 ? undefined : signal?.aborted ? "stopped by you" : `exited with code ${code}`;
-            if (code === 0)
+            if (code === 0) {
                 finished++;
-            db.publish("qa.done", { agent_id: w.id, task_id: task.id, payload: { round, ok: code === 0, bugs, ms } });
-            onEvent({ type: "checked", agent: w.id, round, ok: code === 0, bugs, ms, tokens: h.meter.tokens, reason });
+                slowest = Math.max(slowest, ms);
+            }
+            db.publish("qa.done", { agent_id: w.id, task_id: task.id, payload: { round, ok: code === 0, bugs, ms, tokens: h.meter.tokens } });
+            onEvent({ type: "checked", agent: w.id, round, ok: code === 0, bugs, ms, tokens: h.meter.tokens, reason, task: task.id });
             resolve();
         };
-        h.child.on("exit", end);
-        h.child.on("error", () => end(-1));
+        // Its last stdout lines (token usage) can still be in the pipe at 'exit'.
+        onFinished(h.child, end);
     });
     const roundBugsBy = (id) => filed.filter((b) => b.round === round && b.by === id).length;
     const prev = room.bugs;
     try {
         for (round = 1; round <= rounds && !signal?.aborted; round++) {
+            if (o.budget?.stopping) {
+                result.budget = true;
+                const reason = `the token budget is nearly spent — QA round ${round} not started`;
+                db.publish("qa.skipped", { payload: { round, reason } });
+                onEvent({ type: "skipped", reason });
+                break;
+            }
             roundBugs = 0;
             finished = 0;
+            slowest = 0;
             result.rounds = round;
             const who = pick();
+            secondOpinion = new Set();
             who.forEach(({ w }) => used.set(w.id, (used.get(w.id) ?? 0) + 1));
             // authors: checkers who also wrote some of it (everyone did), so the story doesn't claim fresh eyes.
             db.publish("qa.round", { payload: { round, agents: who.map((x) => x.w.id), authors: who.filter((x) => x.author).map((x) => x.w.id) } });
@@ -308,7 +353,7 @@ export async function runQa(o) {
             const stat = git(repoRoot, ["diff", "--stat", `${o.startHead}..HEAD`]);
             const diff = cap(git(repoRoot, ["diff", "--no-color", `${o.startHead}..HEAD`]), 20_000);
             room.bugs = onBug;
-            await Promise.all(who.map(({ w, author }) => check(w, author, qaPrompt({ goal: o.goal, round, testCmd: o.testCmd(repoRoot), tasks, stat, diff, earlier: filed.filter((b) => b.round < round), author, maxBugs }))));
+            await Promise.all(who.map(({ w, author }) => check(w, author, qaPrompt({ goal: o.goal, round, testCmd: o.testCmd(repoRoot), tasks, stat, diff, earlier: filed.filter((b) => b.round < round), author, maxBugs, checklist }))));
             room.bugs = prev;
             if (signal?.aborted)
                 break;
@@ -322,6 +367,17 @@ export async function runQa(o) {
                 continue;
             }
             if (!roundBugs) {
+                // A first clean round, quick, with nothing ever filed: one agent's word isn't enough.
+                // Trusted: after fixes (it re-checked them), a second clean, the last round, or a long check.
+                const trusted = result.bugs > 0 || cleanRounds > 0 || round === rounds || slowest >= minCleanMs;
+                cleanRounds++;
+                if (!trusted) {
+                    secondOpinion = new Set(who.map(({ w }) => w.id));
+                    const reason = `clean in ${Math.round(slowest / 1000)}s on the first look — another agent checks again`;
+                    db.publish("qa.suspect", { payload: { round, ms: slowest, reason } });
+                    onEvent({ type: "suspect", round, ms: slowest, reason });
+                    continue;
+                }
                 db.publish("qa.clean", { payload: { round } });
                 onEvent({ type: "clean", round });
                 result.clean = true;