@rsaglobaltech/specgate 0.8.1 → 0.10.0

package/dist/scripts/cli/commands/harness/RunCommand.js

@@ -5,7 +5,7 @@
  *
  * A spec-driven repo is already an environment for an agent: `plan` is the
  * task queue, the feature file + AI_RULES.md are the per-task context,
- * `validate --strict-tdd` + the project test command are the reward signal,
+ * `validate --strict` + the project test command are the reward signal,
  * and `done` is the state transition. This command is the missing
  * orchestration layer — it runs plan → context → agent → verify → done
  * without a human copy-pasting prompts.
@@ -14,7 +14,7 @@
  * `harness/REQ-NNN` branch:
  *   1. Build a self-contained prompt (Gherkin + AI_RULES + paths + retry feedback).
  *   2. Shell out to the user-configured agent ({prompt_file} placeholder).
- *   3. Gate: `validate --strict-tdd` + the project test command.
+ *   3. Gate: `validate --strict` + the project test command.
  *   4. Green → `done REQ-NNN` + commit. Red → retry N times feeding the failure.
  *   5. Emit a pass/fail/attempts report.
  *
@@ -73,6 +73,7 @@ const GherkinQuality_1 = require("../../../../packages/core/src/domain/GherkinQu
 const HarnessConfigFile_1 = require("../../../../packages/core/src/infrastructure/HarnessConfigFile");
 const agent_1 = require("../../../lib/agent");
 const WriteScope_1 = require("../../../../packages/core/src/domain/WriteScope");
+const GateCommandIntegrity_1 = require("../../../../packages/core/src/domain/GateCommandIntegrity");
 const EmptyAttempt_1 = require("../../../../packages/core/src/domain/EmptyAttempt");
 const DeclaredArtifacts_1 = require("../../../../packages/core/src/domain/DeclaredArtifacts");
 const ResumeState_1 = require("../../../../packages/core/src/domain/ResumeState");
@@ -81,6 +82,8 @@ const ProfileMatch_1 = require("../../../../packages/core/src/domain/ProfileMatc
 const TraceabilityFormat_1 = require("../../../../packages/core/src/domain/TraceabilityFormat");
 const RequirementPlan_1 = require("../../../../packages/core/src/domain/RequirementPlan");
 const RunBudget_1 = require("../../../../packages/core/src/domain/RunBudget");
+const worktree_lock_1 = require("../../../harness/worktree-lock");
+const AdversarialProbe_1 = require("../../../../packages/core/src/domain/AdversarialProbe");
 const CucumberMessages_1 = require("../../../../packages/core/src/domain/CucumberMessages");
 const HarnessRun_1 = require("../../../../packages/core/src/domain/HarnessRun");
 // Three levels up from dist/scripts/cli/commands/harness is dist/scripts,
@@ -145,6 +148,10 @@ function usage() {
         "  --budget-seconds <n>   Wall-clock ceiling for the whole run. On reaching it\n" +
         "                         the run stops cleanly and still reports what it did.\n" +
         "  --max-requirements <n> Attempt at most n requirements in this run.\n" +
+        "  --verbose              Stream the gate command's output as it runs, on\n" +
+        "                         stderr. Without it the output only appears when the\n" +
+        "                         gate fails, so a run that takes minutes shows\n" +
+        "                         nothing and a hung test looks like a working one.\n" +
         "  --skip-not-ready       Skip requirements an agent could not succeed at — no\n" +
         "                         feature, an unrunnable scenario, unmet dependencies,\n" +
         "                         Deprecated or Needs Clarification. Default: warn and\n" +
@@ -177,6 +184,7 @@ function parseArgs(argv) {
         resume: false,
         format: "text",
         dryRun: false,
+        verbose: false,
         strictArtifacts: false,
         skipNotReady: false,
         budgetSeconds: 0,
@@ -246,6 +254,9 @@ function parseArgs(argv) {
         else if (token === "--format") {
             args.format = argv[++i] || "";
         }
+        else if (token === "--verbose") {
+            args.verbose = true;
+        }
         else if (token === "--dry-run") {
             args.dryRun = true;
         }
@@ -323,7 +334,7 @@ function runPlan(projectDir) {
 // can easily produce >1 MB of dependency-download log, and `spawnSync`'s
 // default 1 MB ceiling otherwise kills the gate with ENOBUFS.
 const SUBPROCESS_MAX_BUFFER = 64 * 1024 * 1024;
-/** Run the gate (validate --strict-tdd, then the optional test command). */
+/** Run the gate (validate --strict, then the optional test command). */
 /**
  * What Cucumber says it did, when it can be asked (F5).
  *
@@ -358,7 +369,13 @@ function readGateMessages(worktreeDir, reportPath) {
     return run.parsed ? run : null;
 }
 function runGate(worktreeDir, testCmd, timeoutMs, req = {}, settings = {}) {
-    const validate = (0, node_child_process_1.spawnSync)(process.execPath, [VALIDATE_SCRIPT, worktreeDir, "--strict-tdd"], {
+    // `--delivering`: the requirement is still Draft while its gate runs, and a
+    // Draft row owes no files. Without it the gate would skip the one
+    // requirement this attempt exists to deliver.
+    const validateArgs = [VALIDATE_SCRIPT, worktreeDir, "--strict"];
+    if (req.requirement)
+        validateArgs.push("--delivering", req.requirement);
+    const validate = (0, node_child_process_1.spawnSync)(process.execPath, validateArgs, {
         encoding: "utf8",
         timeout: timeoutMs,
         maxBuffer: SUBPROCESS_MAX_BUFFER,
@@ -366,7 +383,7 @@ function runGate(worktreeDir, testCmd, timeoutMs, req = {}, settings = {}) {
     if (validate.status !== 0) {
         return {
             ok: false,
-            stage: "validate --strict-tdd",
+            stage: "validate --strict",
             output: validate.stdout + validate.stderr,
             hint: "",
         };
@@ -384,12 +401,23 @@ function runGate(worktreeDir, testCmd, timeoutMs, req = {}, settings = {}) {
         // dependencies has no node_modules here. Verified the hard way: an agent
         // spent its first attempt installing them and timed out. The gate command
         // is the right place to say so, since only the project knows how.
+        // `--verbose` streams instead of capturing, and streams to **fd 2**.
+        //
+        // `spawnSync` cannot do both: the buffers only exist once the child has
+        // exited, which is the moment streaming stops being useful. Streaming is
+        // the point of the flag — during a run that takes minutes there is nothing
+        // to watch, and a gate that hangs looks exactly like a gate that is working.
+        //
+        // Stderr rather than inherited stdout, because `--format json` promises one
+        // parseable document on stdout. Letting the test command write there is
+        // precisely H18: prose and JSON sharing a stream.
         const test = (0, node_child_process_1.spawnSync)(resolved, {
             shell: true,
             cwd: worktreeDir,
             encoding: "utf8",
             timeout: timeoutMs,
             maxBuffer: SUBPROCESS_MAX_BUFFER,
+            stdio: settings.verbose ? ["ignore", 2, 2] : undefined,
         });
         if (test.status !== 0) {
             // Name the command. A gate that silently does the wrong thing — running
@@ -399,7 +427,11 @@ function runGate(worktreeDir, testCmd, timeoutMs, req = {}, settings = {}) {
             return {
                 ok: false,
                 stage: `test command: ${resolved}`,
-                output: test.stdout + test.stderr,
+                // Streamed output is not captured, so say where it went rather than
+                // recording an empty string a reader would take for "printed nothing".
+                output: settings.verbose
+                    ? "(streamed above — run without --verbose to capture it in the report)"
+                    : test.stdout + test.stderr,
             };
         }
         // Exit zero is where the old gate stopped. This is the part that catches a
@@ -470,6 +502,75 @@ function discardWorktreeChanges(worktreeDir) {
     // reviewer's edits must not discard the evidence.
     git(worktreeDir, ["clean", "-fd", "-e", EmptyAttempt_1.PROMPT_ARCHIVE_DIR]);
 }
+/**
+ * Let an adversary try to break a green implementation (D3).
+ *
+ * Sequence, and the order is the whole design: the gate has already passed, so
+ * the work is committable; the adversary writes a test in the worktree; the
+ * project's own test command runs again; whatever the adversary wrote is
+ * discarded so the branch carries the implementer's work alone.
+ *
+ * Never throws and never changes the verdict. The gate stays the only judge —
+ * an adversary can always assert a behaviour the requirement never promised,
+ * and a failing probe that blocked the run would stop correct work.
+ */
+function runAdversarialProbe(worktreeDir, req, settings, timeoutMs, promptFor) {
+    const profile = settings.adversaryProfile;
+    const command = settings.profileAgents[profile] || settings.agent;
+    if (!command)
+        return AdversarialProbe_1.NO_PROBE;
+    const instruction = `The implementation below already passes its gate. Your job is to try to ` +
+        `**break** it: write one additional test, in the project's existing test ` +
+        `style, for a case ${req.requirement} should handle and might not — an ` +
+        `empty input, a boundary, a duplicate, a failure path. Write the test and ` +
+        `nothing else. Do not modify the implementation, the specification or any ` +
+        `existing test.`;
+    const promptFile = path.join(os.tmpdir(), `csda-probe-${req.requirement}-${crypto.randomBytes(4).toString("hex")}.md`);
+    try {
+        fs.writeFileSync(promptFile, promptFor(instruction), "utf8");
+        const agent = (0, node_child_process_1.spawnSync)((0, HarnessRun_1.substituteAgentCommand)(command, promptFile), {
+            shell: true,
+            cwd: worktreeDir,
+            encoding: "utf8",
+            timeout: timeoutMs,
+            maxBuffer: SUBPROCESS_MAX_BUFFER,
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        const wrote = git(worktreeDir, ["status", "--porcelain"])
+            .stdout.split("\n")
+            .map((line) => line.slice(3).trim())
+            .filter((line) => line && !line.startsWith(EmptyAttempt_1.PROMPT_ARCHIVE_DIR));
+        if (wrote.length === 0) {
+            // Nothing written is not a failure: an adversary that cannot think of a
+            // case is telling you something mildly reassuring.
+            return { ran: true, broke: false, detail: "The adversary wrote no test.", wrote: [] };
+        }
+        const after = runGate(worktreeDir, settings.testCmd, timeoutMs, req, settings);
+        return {
+            ran: true,
+            broke: !after.ok,
+            detail: [`${agent.stdout || ""}${agent.stderr || ""}`.trim(), after.output || ""]
+                .filter(Boolean)
+                .join("\n\n")
+                .slice(0, 4000),
+            wrote,
+        };
+    }
+    catch {
+        // Insurance that fails is still only insurance; it must not take the run.
+        return AdversarialProbe_1.NO_PROBE;
+    }
+    finally {
+        try {
+            fs.rmSync(promptFile, { force: true });
+        }
+        catch {
+            /* a temp file nobody will read again */
+        }
+        // The branch carries the implementer's work and nothing else.
+        discardWorktreeChanges(worktreeDir);
+    }
+}
 /**
  * Did the agent edit the contract it was being judged against? (A1, closes H16)
  *
@@ -560,6 +661,52 @@ function checkWriteScopeInWorktree(worktreeDir, settings) {
             `(\`specgate change new\`), reviewed by a person — not in this attempt.`,
     };
 }
+/**
+ * Did the agent rewrite the definition the gate command depends on? (#167)
+ *
+ * The write-scope guard covers the contract — spec.md, AI_RULES.md, features/**
+ * — and not the file that defines the command the gate runs. Measured with a
+ * stub agent that wrote no implementation at all and only replaced
+ * `"test": "node --test tests/*.test.js"` with `"echo 'all good'"` over a red
+ * suite: `✅ REQ-001 pass (1 attempt)`.
+ *
+ * This does not protect the manifest — adding a dependency mid-implementation
+ * is legitimate, and a guard that blocks it gets switched off. It compares one
+ * value: the script the gate actually invokes, at the base commit and now.
+ *
+ * Returns null when there is nothing to report, including when the gate command
+ * is one whose meaning does not live in a file we can read.
+ */
+function checkGateCommandInWorktree(worktreeDir, baseRef, testCmd) {
+    if (!testCmd)
+        return null;
+    const probe = (0, GateCommandIntegrity_1.gateCommandIntegrity)(testCmd, "{}", "{}");
+    if (!probe.script)
+        return null;
+    const manifest = probe.script.manifest;
+    const show = git(worktreeDir, ["show", `${baseRef}:${manifest}`]);
+    const before = show.status === 0 ? show.stdout : null;
+    const currentPath = path.join(worktreeDir, manifest);
+    const after = fs.existsSync(currentPath) ? fs.readFileSync(currentPath, "utf8") : null;
+    const result = (0, GateCommandIntegrity_1.gateCommandIntegrity)(testCmd, before, after);
+    if (!result.checked || !result.changed)
+        return null;
+    return {
+        manifest,
+        script: result.script.script,
+        message: `agent_rewrote_gate_command — the gate runs \`${testCmd}\`, and this attempt ` +
+            `changed what that means.\n\n` +
+            `  ${manifest} → scripts.${result.script.script}\n` +
+            `  before: ${result.before === null ? "(absent)" : result.before}\n` +
+            `  after:  ${result.after === null ? "(absent)" : result.after}\n\n` +
+            `The command the gate runs is part of the contract this requirement is ` +
+            `judged against, exactly like the scenario is. Weakening it makes a green ` +
+            `gate meaningless.\n\n` +
+            `Revert that script and make the suite pass as it stands. If the command is ` +
+            `genuinely wrong, that is a change for a person to make — not something to ` +
+            `fix inside the attempt being measured by it.`,
+    };
+}
 /**
  * Every path the agent wrote in this worktree — added and modified alike.
  *
@@ -614,6 +761,10 @@ function attemptRequirement(req, ctx) {
     const attemptLog = [];
     /** The reviewer's findings from the previous attempt, fed into this prompt. */
     let reviewFindings = "";
+    // Once per requirement, not once per attempt: probing an implementation that
+    // already failed tells you nothing, and probing every retry multiplies the
+    // cost of the one step in the loop that is pure insurance.
+    let probe = AdversarialProbe_1.NO_PROBE;
     const firstAttempt = Math.max(1, (ctx.resumeAt && ctx.resumeAt.attempt) || 1);
     for (let attempt = firstAttempt; attempt <= settings.maxAttempts; attempt += 1) {
         info(`${req.requirement}: attempt ${attempt}/${settings.maxAttempts}`);
@@ -651,6 +802,10 @@ function attemptRequirement(req, ctx) {
                     reviewFindings: step.advisory ? undefined : reviewFindings || undefined,
                     attempt,
                     maxAttempts: settings.maxAttempts,
+                    // Read from the worktree, which carries the matrix at the base
+                    // commit — the precedent must be work already accepted, not a
+                    // sibling requirement this same run happens to be writing.
+                    withPrecedents: settings.promptPrecedents === true,
                 });
                 const promptFile = path.join(os.tmpdir(), `csda-harness-prompt-${req.requirement}-${crypto.randomBytes(4).toString("hex")}.md`);
                 fs.writeFileSync(promptFile, prompt, "utf8");
@@ -744,7 +899,22 @@ function attemptRequirement(req, ctx) {
             record("write-scope");
             continue;
         }
-        const gate = runGate(worktreeDir, settings.testCmd, timeoutMs, req, settings);
+        // Same reason as the write-scope guard, one level up: a gate command the
+        // agent has just loosened cannot fail either.
+        const gateCmd = checkGateCommandInWorktree(worktreeDir, "HEAD", settings.testCmd);
+        if (gateCmd) {
+            previousFailure = gateCmd.message;
+            warn(`${req.requirement}: the agent rewrote the gate command`);
+            warn(`  ${gateCmd.manifest} → scripts.${gateCmd.script}`);
+            record("write-scope");
+            continue;
+        }
+        // `verbose` is a run flag, not a project setting, so it travels on the
+        // context rather than in `harness.config.yaml`.
+        const gate = runGate(worktreeDir, settings.testCmd, timeoutMs, req, {
+            ...settings,
+            verbose: ctx.verbose,
+        });
         if (!gate.ok) {
             previousFailure =
                 `Gate failed at: ${gate.stage}\n\n` + (gate.hint ? `⚠ ${gate.hint}\n\n` : "") + gate.output;
@@ -793,8 +963,37 @@ function attemptRequirement(req, ctx) {
             record("commit");
             continue;
         }
+        // The adversary runs **after** the commit, and the order is not cosmetic:
+        // before it, `git status` shows the implementer's uncommitted work, so
+        // discarding "what the probe wrote" would throw the implementation away and
+        // report a pass over an empty branch. That is the H19 family — a green
+        // verdict over work that is not there — and it was reintroduced here once.
+        if ((0, AdversarialProbe_1.shouldProbe)({
+            profile: settings.adversaryProfile,
+            gatePassed: true,
+            alreadyProbed: probe.ran,
+        })) {
+            info(`${req.requirement}: probing with '${settings.adversaryProfile}'`);
+            probe = runAdversarialProbe(worktreeDir, req, settings, timeoutMs, (instruction) => (0, prompt_1.buildPrompt)(req, worktreeDir, {
+                promptPrefix: settings.promptPrefix,
+                hint: instruction,
+                withPrecedents: settings.promptPrecedents === true,
+            }));
+            if (probe.broke) {
+                const finding = (0, AdversarialProbe_1.probeFinding)(req.requirement, probe);
+                warn(`${req.requirement}: ${finding.message} [${finding.code}]`);
+                warn(`  fix: ${finding.fix}`);
+            }
+        }
         record("pass");
-        return { result: "pass", attempts: attempt, attemptLog };
+        return {
+            result: "pass",
+            attempts: attempt,
+            attemptLog,
+            // The probe never changes the verdict; it travels beside it so
+            // `harness report` and the run record can show what it found.
+            ...(probe.ran ? { adversarial: probe } : {}),
+        };
     }
     // Every attempt is spent. Commit what the agent produced anyway, on the
     // branch, before the worktree is removed.
@@ -1290,7 +1489,11 @@ function processRequirement(req, ctx) {
         const args = resuming
             ? ["worktree", "add", dir, branch]
             : ["worktree", "add", "-b", branch, dir, baseRef];
-        const add = git(projectDir, args);
+        // Serialised across processes: `--concurrency` gives each requirement its
+        // own process, and two `git worktree add` calls in one repository race on
+        // the half-written `.git/worktrees/<name>/` entry. Only the creation is
+        // inside the lock — the agent and the gate stay parallel.
+        const add = (0, worktree_lock_1.withWorktreeLock)(projectDir, () => git(projectDir, args));
         if (add.status !== 0) {
             return {
                 requirement: req.requirement,
@@ -1824,6 +2027,7 @@ class RunCommand extends command_1.BaseCommand {
                     const prompt = (0, prompt_1.buildPrompt)(req, projectDir, {
                         promptPrefix: settings.promptPrefix,
                         hint: hintByReq.get(req.requirement),
+                        withPrecedents: settings.promptPrecedents === true,
                     });
                     process.stdout.write(`\n${"═".repeat(72)}\n${req.requirement} (${req.category}) → branch harness/${req.requirement}\n${"═".repeat(72)}\n`);
                     process.stdout.write(prompt + "\n");
@@ -1865,6 +2069,7 @@ class RunCommand extends command_1.BaseCommand {
                 budgetSeconds: args.budgetSeconds,
                 maxRequirements: args.maxRequirements,
                 strictArtifacts: args.strictArtifacts,
+                verbose: args.verbose,
             };
             const startedAt = new Date().toISOString();
             const results = await runLevels(pending, ctx, {