@mmerterden/multi-agent-pipeline 20.2.0 → 20.3.0

package/pipeline/scripts/evidence-gate.mjs

@@ -24,6 +24,30 @@
 //                      state.visualEvidence.required is true.
 //   --success-pattern  override the default success regex for the claim type (marker claims only)
 //   --failure-pattern  override the default failure regex for the claim type (marker claims only)
+//   --stack <id>       read the build/test markers from that adapter in
+//                      stack-adapters.json instead of the defaults below, and
+//                      for a test claim also read the test counts
+//                      (test-summary.mjs): zero tests executed or any failed
+//                      test contradicts the pass. The unknown adapter has no
+//                      success markers, so present evidence without a failure
+//                      marker is reported not-applicable rather than failed.
+//                      Without --stack the defaults apply unchanged. An
+//                      adapter's `skipped` markers (a test task that did not
+//                      run) count as zero tests when the log carries no counts.
+//   --junit <path>     JUnit XML file or directory, and
+//   --xcresult <bundle> / --xcresult-json <file>
+//                      where to read test counts when the log prints none
+//                      (Gradle on success, xcodebuild -quiet).
+//   --state <path>     agent-state.json for the gate ledger. While the quality
+//                      gates are active (MULTI_AGENT_UNATTENDED=1, or a run
+//                      in autopilot mode) every verdict is appended to
+//                      state.gates[] as `evidence-gate/<claim>`
+//                      (gate-ledger.mjs); without it nothing is written.
+//
+// Quality gates active, a test claim needs a positive executed-test count, from
+// the log or from --junit / --xcresult: markers alone, however green, do not
+// say how many tests ran. Attended, counts that cannot be read leave the
+// markers to decide, as they always did.
 //
 // The manual claim reads a JSON document instead of scanning for markers:
 //   { "criteria": [ { "spec", "observed", "verdict": "pass|fail|not-tested", "reason"?, ... } ], ... }
@@ -35,7 +59,13 @@
 //   1  -  pass claim is UNVERIFIED (no evidence / empty / shows failure / no success marker) -> caller must NOT record passed
 //   2  -  usage error
 
+import { spawnSync } from "node:child_process";
 import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+import { gateContext } from "./gate-ledger.mjs";
+
+const LEDGER = fileURLToPath(new URL("./gate-ledger.mjs", import.meta.url));
 
 // Failure markers are kept DEFINITIVE (not broad) because failure now overrides
 // success (see the decision logic): a bare "error:" or stray "FAILED" in an
@@ -44,11 +74,15 @@ import { existsSync, readFileSync, statSync, writeFileSync } from "node:fs";
 // build log that merely mentions "SUCCESS" once would otherwise sneak through.
 const DEFAULT_MARKERS = {
   build: {
-    success: /BUILD SUCCEEDED|Build succeeded/,
+    success: /BUILD SUCCEEDED|Build succeeded|BUILD SUCCESSFUL/,
     failure: /BUILD FAILED|Build failed|fatal error:|\*\* BUILD FAILED \*\*/,
   },
   test: {
     success: /\bPASS\b|0 failures|Test Suite.*passed|tests? passed|all tests passed|✓/i,
+    // pytest markers, matched case-sensitively: the per-test PASSED verdict and
+    // the "N passed" summary. Under the /i flag above, PASSED would also match
+    // any sentence that says "passed", and a zero count is not a success.
+    successExact: /\bPASSED\b|\b[1-9][0-9]* passed\b/,
     failure: /\bFAILED\b|✗|[1-9][0-9]*\s+failures?|[1-9][0-9]*\s+failing/i,
   },
   review: {
@@ -109,14 +143,59 @@ function parseArgs(argv) {
 // discards what has not drained, while writeFileSync to fd 1 returns only once
 // the bytes are gone. The verdict is short today; the verdict is also the only
 // thing a caller reads, so losing half of it is the one failure that matters.
-function fail(msg) {
-  writeFileSync(1, `${JSON.stringify({ ok: false, code: 1, reason: msg })}\n`);
+let verdictExtra = {};
+// Only while the quality gates are active (lib/unattended.mjs, gatesActive):
+// one synchronous append, so the verdict and its ledger entry
+// cannot come apart. A ledger problem is reported and never changes the verdict.
+let ledgerClaim = null;
+let ledgerState = null;
+function ledger(verdict, reason, extra) {
+  if (!ledgerClaim) return;
+  const ctx = gateContext({ statePath: ledgerState });
+  if (!ctx.active) return;
+  const detail = { reason, ...(extra.summary ? { summary: extra.summary } : {}) };
+  if (verdictExtra.stack) detail.stack = verdictExtra.stack;
+  const argv = [LEDGER, "append", "--gate", `evidence-gate/${ledgerClaim}`, "--verdict", verdict];
+  argv.push("--detail", JSON.stringify(detail));
+  if (ctx.statePath) argv.push("--state", ctx.statePath);
+  const r = spawnSync(process.execPath, argv, { encoding: "utf8", timeout: 30000 });
+  if (r.status !== 0) process.stderr.write(r.stderr || "gate-ledger: append failed\n");
+}
+function fail(msg, extra = {}) {
+  writeFileSync(
+    1,
+    `${JSON.stringify({ ok: false, code: 1, reason: msg, ...verdictExtra, ...extra })}\n`,
+  );
+  ledger("fail", msg, extra);
   process.exit(1);
 }
-function ok(msg) {
-  writeFileSync(1, `${JSON.stringify({ ok: true, code: 0, reason: msg })}\n`);
+function ok(msg, extra = {}, { nothingToProve = false } = {}) {
+  writeFileSync(
+    1,
+    `${JSON.stringify({ ok: true, code: 0, reason: msg, ...verdictExtra, ...extra })}\n`,
+  );
+  const na = extra.notApplicable === true;
+  // A status that is not a pass claim is honest, but it is not a pass either.
+  ledger(na ? "not-applicable" : nothingToProve ? "fail" : "pass", msg, extra);
   process.exit(0);
 }
+/** The extra count sources named on the command line, read lazily. */
+function* countSources(ts) {
+  const path = (flag) => (typeof args[flag] === "string" ? args[flag] : null);
+  if (path("junit")) yield ts.summarizeJunit(path("junit"));
+  if (path("xcresult")) yield ts.summarizeXcresult(path("xcresult"));
+  if (path("xcresult-json")) {
+    try {
+      yield ts.fromXcresultJson(
+        JSON.parse(readFileSync(path("xcresult-json"), "utf8")),
+        path("xcresult-json"),
+      );
+    } catch {
+      yield null;
+    }
+  }
+}
+
 function usage(msg) {
   console.error(`evidence-gate: ${msg}`);
   process.exit(2);
@@ -210,11 +289,31 @@ if (!claim || !CLAIMS.includes(claim)) {
 if (typeof status !== "string") {
   usage("--status is required");
 }
+ledgerClaim = claim;
+ledgerState = typeof args.state === "string" ? args.state : null;
 
 // Only a "passed" claim needs proof. failed / pending / anything else is honest
 // about not succeeding, so it passes through untouched.
 if (status !== "passed") {
-  ok(`status="${status}" is not a pass claim; nothing to substantiate`);
+  ok(
+    `status="${status}" is not a pass claim; nothing to substantiate`,
+    {},
+    { nothingToProve: true },
+  );
+}
+
+let adapter = null;
+let adapterMarkers = null;
+if (args.stack !== undefined && !JSON_CLAIMS.has(claim)) {
+  if (args.stack === true) usage("--stack needs an adapter id");
+  const { getAdapter, compileMarkers } = await import("./_stack-adapter.mjs");
+  try {
+    adapter = getAdapter(String(args.stack), { strict: true });
+    adapterMarkers = compileMarkers(adapter, claim);
+  } catch (err) {
+    usage(err.message);
+  }
+  verdictExtra = { stack: adapter.id };
 }
 
 const evidence = args.evidence;
@@ -241,20 +340,34 @@ if (JSON_CLAIMS.has(claim)) {
   gateManual(claim, evidence, body, args["require-screenshot"] === true);
 }
 
+const notApplicable = adapter !== null && adapter.applicable === false;
+
 const successRe = userRegExp(args["success-pattern"], DEFAULT_MARKERS[claim].success, "success");
 const failureRe = userRegExp(args["failure-pattern"], DEFAULT_MARKERS[claim].failure, "failure");
+const userSuccess = args["success-pattern"] && args["success-pattern"] !== true;
+const userFailure = args["failure-pattern"] && args["failure-pattern"] !== true;
+const successList = userSuccess || !adapterMarkers ? [successRe] : adapterMarkers.success;
+const failureList = userFailure || !adapterMarkers ? [failureRe] : adapterMarkers.failure;
 
-// Zero-count summaries ("0 failed", "0 tests failed out of 10", "0 failures")
-// are success statements, not failure markers. The default \bFAILED\b marker
+// Zero-count summaries ("0 failed", "0 tests failed out of 10", "0 failures",
+// "failed=0", "failed: 0") are success statements, not failure markers. The default \bFAILED\b marker
 // would otherwise flip a passing log to fail - and failure is decisive. Only
 // applied when the DEFAULT failure marker is in play; a caller-supplied
 // --failure-pattern is matched against the raw body untouched.
-const ZERO_FAIL_SUMMARY = /\b0\s+(?:\w+\s+){0,2}(?:failed|failures?|failing)\b/gi;
+// The `failed: 0` / `failed=0` form counts only as an item of a summary list:
+// at the start of a line, after a separator, or after another `key=N` item, and
+// followed by the end of the line, a separator or another `key=N` item. So
+// "Tests: 5 passed, failed: 0" is a success statement and
+// "Assertion failed: 0 == expected" stays a failure.
+const ZERO_FAIL_SUMMARY =
+  /\b0\s+(?:\w+\s+){0,2}(?:failed|failures?|failing)\b|(?:^|[,;|(]|\b\w+\s*[=:]\s*\d+\s)\s*(?:failed|failures?|failing)\s*[=:]\s*0(?!\d)(?=\s*(?:[,;|)]|$)|\s+\w+\s*[=:]\s*\d)/gim;
 const usingDefaultFailure = !args["failure-pattern"] || args["failure-pattern"] === true;
 const failureBody = usingDefaultFailure ? body.replace(ZERO_FAIL_SUMMARY, "") : body;
 
-const showsFailure = failureRe.test(failureBody);
-const showsSuccess = successRe.test(body);
+const showsFailure = failureList.some((re) => re.test(failureBody));
+const usingDefaultSuccess = !userSuccess && !adapterMarkers;
+const successExact = usingDefaultSuccess ? DEFAULT_MARKERS[claim].successExact : undefined;
+const showsSuccess = successList.some((re) => re.test(body)) || (successExact?.test(body) ?? false);
 
 // Failure is DECISIVE. A log that shows a definitive failure marker cannot
 // substantiate a "passed" claim no matter what else it contains - this closes
@@ -262,11 +375,78 @@ const showsSuccess = successRe.test(body);
 if (showsFailure) {
   fail(`${claim} evidence shows a failure marker  -  pass claim contradicted by ${evidence}`);
 }
+
+// An unknown stack has no success markers to look for. Evidence that exists and
+// shows no definitive failure is all this gate can say about it.
+if (notApplicable && !userSuccess) {
+  ok(
+    `${claim} evidence present with no failure marker; stack "${adapter.id}" has no success markers (not-applicable)`,
+    { notApplicable: true },
+  );
+}
+
+// A runner's success line does not say how many tests it covers. With an
+// adapter, or with the quality gates active, the counts are read too: zero
+// executed or any failed test contradicts the pass. Counts that cannot be read
+// from the log are looked for in --junit / --xcresult; still unread, the gates
+// fail the claim and an attended run leaves the markers to decide.
+let summary;
+const gated = gateContext({ statePath: ledgerState }).active;
+if (claim === "test" && (adapterMarkers || gated)) {
+  const ts = await import("./test-summary.mjs");
+  summary = ts.summarize(body, {
+    parsers: adapterMarkers ? adapter.testSummary.parsers : undefined,
+    source: evidence,
+  });
+  if (summary.executed === null) {
+    for (const extra of countSources(ts)) {
+      if (extra && extra.executed !== null) {
+        summary = extra;
+        break;
+      }
+    }
+  }
+  if (summary.executed === null && adapterMarkers?.skipped.some((re) => re.test(body))) {
+    summary = {
+      ...summary,
+      executed: 0,
+      passed: 0,
+      failed: 0,
+      status: "no-tests",
+      reason: "the runner reports the test task skipped",
+    };
+  }
+  if (summary.executed === null && gated) {
+    fail(
+      `${claim} evidence carries no positive executed-test count (${summary.reason || "counts unparsed"}); with the quality gates active a test pass needs one, from the log or from --junit / --xcresult`,
+      { summary },
+    );
+  }
+  if (summary.executed === 0) {
+    fail(`${claim} evidence shows no tests executed  -  pass claim unverified by ${evidence}`, {
+      summary,
+    });
+  }
+  if (summary.failed > 0) {
+    fail(
+      `${claim} evidence counts ${summary.failed} failed test(s)  -  pass claim contradicted by ${evidence}`,
+      { summary },
+    );
+  }
+}
+const summaryExtra = summary ? { summary } : {};
 // No failure, but also no success marker -> unproven -> default-FAIL.
 if (!showsSuccess) {
   fail(
     `${claim} evidence does not contain a success marker  -  pass claim unverified by ${evidence}`,
+    summaryExtra,
   );
 }
 
-ok(`${claim} pass claim substantiated by ${evidence}`);
+if (summary && summary.executed === null) {
+  ok(
+    `${claim} pass claim substantiated by ${evidence} (markers only; counts unparsed: ${summary.reason})`,
+    summaryExtra,
+  );
+}
+ok(`${claim} pass claim substantiated by ${evidence}`, summaryExtra);

package/pipeline/scripts/feedback-send.mjs

@@ -30,6 +30,7 @@ import { execFileSync } from "node:child_process";
 import { readFileSync, existsSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
+import { hostCommand } from "../lib/user-facing.mjs";
 
 const ENDPOINT_DEFAULT = "https://mmerterden.com/api/feedback/ingest";
 const TIMEOUT_MS = 8000;
@@ -109,7 +110,8 @@ function endpointAllowed(endpoint) {
     return false;
   }
   if (u.protocol === "https:") return true;
-  return u.protocol === "http:" && ["localhost", "127.0.0.1", "::1"].includes(u.hostname);
+  // URL.hostname returns an IPv6 literal bracketed ("[::1]"); accept both forms.
+  return u.protocol === "http:" && ["localhost", "127.0.0.1", "::1", "[::1]"].includes(u.hostname);
 }
 
 async function main() {
@@ -152,7 +154,7 @@ async function main() {
   const token = resolveToken();
   if (!token) {
     process.stderr.write(
-      "ERROR: no ingest token. Run /multi-agent:update once (it registers one) or /multi-agent:setup.\n",
+      `ERROR: no ingest token. Run ${hostCommand("update", import.meta.dirname)} once (it registers one) or ${hostCommand("setup", import.meta.dirname)}.\n`,
     );
     process.exit(1);
   }