@mmerterden/multi-agent-pipeline 10.7.4 → 10.9.0

package/pipeline/scripts/smoke-verify-by-test.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+# smoke-verify-by-test.sh
+#
+# Verifies the Phase 4 Step 3.7 verify-by-test contract:
+#   1. phase-4-review.md documents Step 3.7 with evidence-gate invocation + feature-doc pointer
+#   2. refs/features/verify-by-test.md exists and covers the verdict table + red-test handoff
+#   3. prefs.schema.json exposes global.verifyByTest.{enabled,maxFindings,model,stepTimeoutSec}
+#   4. verifyByTest.enabled defaults to false (opt-in, no surprise cost)
+#   5. triage-output.schema.json is v3.2.0 with the $defs.verification result enum
+#   6. validate-triage.mjs accepts a valid `confirmed` verification and rejects bad ones
+#   7. phase-3-dev.md documents the redTests rework re-entry
+#
+# Exit 0 = all pass, 1 = any failure.
+
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+PHASE4_DOC="$ROOT/pipeline/commands/multi-agent/refs/phases/phase-4-review.md"
+PHASE3_DOC="$ROOT/pipeline/commands/multi-agent/refs/phases/phase-3-dev.md"
+FEATURE_DOC="$ROOT/pipeline/commands/multi-agent/refs/features/verify-by-test.md"
+PREFS_SCHEMA="$ROOT/pipeline/schemas/prefs.schema.json"
+TRIAGE_SCHEMA="$ROOT/pipeline/schemas/triage-output.schema.json"
+VALIDATOR="$ROOT/pipeline/scripts/validate-triage.mjs"
+
+pass=0
+fail=0
+failures=()
+record_pass() { pass=$((pass + 1)); printf '  \033[0;32mPASS\033[0m %s\n' "$1"; }
+record_fail() { fail=$((fail + 1)); failures+=("$1"); printf '  \033[0;31mFAIL\033[0m %s\n' "$1"; }
+
+printf '→ smoke-verify-by-test: Phase 4 Step 3.7 contract\n'
+
+# 1. Phase 4 doc documents Step 3.7
+if [ ! -f "$PHASE4_DOC" ]; then
+  record_fail "phase-4-review.md missing"
+else
+  if grep -qF "3.7 Verify-by-test" "$PHASE4_DOC"; then
+    record_pass "phase-4-review.md documents Step 3.7"
+  else
+    record_fail "phase-4-review.md missing Step 3.7 section"
+  fi
+  if grep -qF "evidence-gate.mjs --claim test --status passed" "$PHASE4_DOC"; then
+    record_pass "Step 3.7 downgrade is evidence-gated"
+  else
+    record_fail "Step 3.7 must gate downgrades via evidence-gate.mjs --claim test --status passed"
+  fi
+  if grep -qF "refs/features/verify-by-test.md" "$PHASE4_DOC"; then
+    record_pass "phase-4-review.md points to the feature doc"
+  else
+    record_fail "phase-4-review.md must reference refs/features/verify-by-test.md"
+  fi
+  if grep -qF "review.verify_by_test" "$PHASE4_DOC"; then
+    record_pass "Step 3.7 emits review.verify_by_test telemetry"
+  else
+    record_fail "Step 3.7 must document the review.verify_by_test metric"
+  fi
+fi
+
+# 2. Feature doc exists with verdict + handoff coverage
+if [ ! -f "$FEATURE_DOC" ]; then
+  record_fail "refs/features/verify-by-test.md missing"
+else
+  for token in "not-reproduced" "inconclusive" "redTests" "Off by default"; do
+    if grep -qF "$token" "$FEATURE_DOC"; then
+      record_pass "feature doc covers '$token'"
+    else
+      record_fail "feature doc missing '$token'"
+    fi
+  done
+fi
+
+# 3. Prefs schema exposes verifyByTest knobs
+for prop in enabled maxFindings model stepTimeoutSec; do
+  if jq -e ".properties.global.properties.verifyByTest.properties.${prop}" "$PREFS_SCHEMA" >/dev/null 2>&1; then
+    record_pass "prefs schema exposes verifyByTest.${prop}"
+  else
+    record_fail "prefs schema missing verifyByTest.${prop}"
+  fi
+done
+
+# 4. Off by default  -  preserves existing-user baseline
+if jq -e '.properties.global.properties.verifyByTest.properties.enabled
+          | has("default") and .default == false' "$PREFS_SCHEMA" >/dev/null 2>&1; then
+  record_pass "verifyByTest.enabled defaults to false (opt-in)"
+else
+  record_fail "verifyByTest.enabled must default to false"
+fi
+
+# 5. Triage schema version + verification enum
+schema_version=$(jq -r '.version // empty' "$TRIAGE_SCHEMA")
+if [ "$schema_version" = "3.2.0" ]; then
+  record_pass "triage-output schema version is 3.2.0"
+else
+  record_fail "triage-output schema version should be 3.2.0 (was: ${schema_version:-missing})"
+fi
+if jq -e '.["$defs"].verification.properties.result.enum
+          | (index("confirmed") != null
+             and index("not-reproduced") != null
+             and index("inconclusive") != null)' "$TRIAGE_SCHEMA" >/dev/null 2>&1; then
+  record_pass "schema verification.result enum complete"
+else
+  record_fail "schema \$defs.verification.result enum must be confirmed/not-reproduced/inconclusive"
+fi
+
+# 6. Behavioral validator round-trips
+valid_fixture='{"accepted":[{"severity":"blocking","file":"Sources/Auth/Login.swift","line":42,"issue":"expired token accepted as valid","fix":"reject tokens past expiry in validateToken()","reviewer":"fable","verification":{"result":"confirmed","testRef":"AuthTests/LoginTests/testExpiredTokenRejected","evidencePath":".pipeline/verify-1.test.log"}}],"deferred":[],"rejected":[],"approved":false}'
+if printf '%s' "$valid_fixture" | node "$VALIDATOR" - >/dev/null 2>&1; then
+  record_pass "validator accepts confirmed verification with testRef+evidencePath"
+else
+  record_fail "validator rejected a valid confirmed verification"
+fi
+
+bad_result_fixture='{"accepted":[{"severity":"blocking","file":"a.swift","line":1,"issue":"some issue here","fix":"do the fix","reviewer":"fable","verification":{"result":"maybe"}}],"deferred":[],"rejected":[],"approved":false}'
+if printf '%s' "$bad_result_fixture" | node "$VALIDATOR" - >/dev/null 2>&1; then
+  record_fail "validator must reject verification.result 'maybe'"
+else
+  record_pass "validator rejects bad verification.result"
+fi
+
+missing_ref_fixture='{"accepted":[{"severity":"blocking","file":"a.swift","line":1,"issue":"some issue here","fix":"do the fix","reviewer":"fable","verification":{"result":"confirmed"}}],"deferred":[],"rejected":[],"approved":false}'
+if printf '%s' "$missing_ref_fixture" | node "$VALIDATOR" - >/dev/null 2>&1; then
+  record_fail "validator must reject confirmed verification without testRef/evidencePath"
+else
+  record_pass "validator rejects confirmed verification lacking testRef/evidencePath"
+fi
+
+# Reviewer enum parity: fable (Claude Code default) and gpt (Copilot CLI) accepted
+fable_fixture='{"accepted":[{"severity":"important","file":"a.swift","line":1,"issue":"some issue here","fix":"do the fix","reviewer":"gpt"}],"deferred":[],"rejected":[],"approved":true}'
+if printf '%s' "$fable_fixture" | node "$VALIDATOR" - >/dev/null 2>&1; then
+  record_pass "validator accepts schema-allowed reviewers (fable/gpt)"
+else
+  record_fail "validator must accept reviewer values fable and gpt (schema v3.1.0 parity)"
+fi
+
+# 7. Phase 3 doc documents the red-test rework re-entry
+if grep -qF "verifyByTest.redTests" "$PHASE3_DOC"; then
+  record_pass "phase-3-dev.md documents redTests rework re-entry"
+else
+  record_fail "phase-3-dev.md must document verifyByTest.redTests re-entry"
+fi
+
+printf '\n══ verify-by-test smoke: %d passed, %d failed ══\n' "$pass" "$fail"
+if [ "$fail" -gt 0 ]; then
+  printf '\nFailures:\n'
+  for msg in "${failures[@]}"; do printf '  - %s\n' "$msg"; done
+  exit 1
+fi
+exit 0

package/pipeline/scripts/update-check.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# update-check.sh  -  cached advisory version check (Phase 0 Step 0.6)
+#
+# Compares the locally installed pipeline version against the npm registry's
+# dist-tags.latest. Cached with a TTL so at most one network call per TTL
+# window; the call is bounded by a short timeout and every failure path is
+# silent  -  this script NEVER blocks or fails the pipeline.
+#
+# stdout: "<local>|<latest>" when a newer version exists, nothing otherwise.
+# Exit code: always 0 (advisory contract).
+#
+# Usage:
+#   bash pipeline/scripts/update-check.sh                 # auto-detect local version
+#   bash pipeline/scripts/update-check.sh --local 10.8.0  # explicit local version
+#   bash pipeline/scripts/update-check.sh --ttl-hours 24  # cache window (default 24)
+#   bash pipeline/scripts/update-check.sh --force         # ignore cache
+#
+# Cache file: ~/.claude/logs/multi-agent/.update-check ("epoch|latest").
+# Registry read is a plain curl  -  never `npm view` (a user-level .npmrc scope
+# mapping can silently reroute npm to a different registry; curl cannot lie).
+
+set -euo pipefail
+
+PKG="@mmerterden/multi-agent-pipeline"
+REGISTRY_URL="https://registry.npmjs.org/${PKG/\//%2F}"
+CACHE_FILE="${UPDATE_CHECK_CACHE:-$HOME/.claude/logs/multi-agent/.update-check}"
+TTL_HOURS=24
+LOCAL_VERSION=""
+FORCE=0
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --local) LOCAL_VERSION="${2:-}"; shift 2 ;;
+    --ttl-hours) TTL_HOURS="${2:-24}"; shift 2 ;;
+    --force) FORCE=1; shift ;;
+    *) shift ;;
+  esac
+done
+
+# Local version: explicit arg, else read from the pipeline repo clone.
+if [ -z "$LOCAL_VERSION" ]; then
+  for candidate in "$HOME/multi-agent-pipeline" "$HOME/dev/multi-agent-pipeline" "$HOME/projects/multi-agent-pipeline"; do
+    if [ -f "$candidate/package.json" ]; then
+      LOCAL_VERSION=$(node -p "require('$candidate/package.json').version" 2>/dev/null || true)
+      [ -n "$LOCAL_VERSION" ] && break
+    fi
+  done
+fi
+[ -z "$LOCAL_VERSION" ] && exit 0  # cannot determine local version -> silent no-op
+
+now=$(date +%s)
+latest=""
+
+# Fresh cache?
+if [ "$FORCE" -eq 0 ] && [ -f "$CACHE_FILE" ]; then
+  cached_epoch=$(cut -d'|' -f1 "$CACHE_FILE" 2>/dev/null || echo 0)
+  cached_latest=$(cut -d'|' -f2 "$CACHE_FILE" 2>/dev/null || echo "")
+  case "$cached_epoch" in (*[!0-9]*|"") cached_epoch=0 ;; esac
+  if [ $((now - cached_epoch)) -lt $((TTL_HOURS * 3600)) ] && [ -n "$cached_latest" ]; then
+    latest="$cached_latest"
+  fi
+fi
+
+# Stale or missing cache -> one bounded registry call (silent on any failure).
+if [ -z "$latest" ]; then
+  latest=$(curl -sm 3 "$REGISTRY_URL" 2>/dev/null \
+    | { command -v jq >/dev/null 2>&1 && jq -r '."dist-tags".latest // empty' \
+        || sed -n 's/.*"latest":"\([^"]*\)".*/\1/p'; } | head -1) || true
+  [ -z "$latest" ] && exit 0
+  mkdir -p "$(dirname "$CACHE_FILE")" 2>/dev/null || exit 0
+  printf '%s|%s\n' "$now" "$latest" > "$CACHE_FILE" 2>/dev/null || true
+fi
+
+[ "$latest" = "$LOCAL_VERSION" ] && exit 0
+
+# Update available only when latest sorts strictly ABOVE local (a dev machine
+# running ahead of the registry must not see an "update" prompt).
+highest=$(printf '%s\n%s\n' "$LOCAL_VERSION" "$latest" | sort -t. -k1,1n -k2,2n -k3,3n | tail -1)
+if [ "$highest" = "$latest" ]; then
+  printf '%s|%s\n' "$LOCAL_VERSION" "$latest"
+fi
+exit 0

package/pipeline/scripts/validate-diff-risk.mjs

@@ -23,6 +23,7 @@ const ALLOWED_SIGNALS = new Set([
   "complexity_delta",
   "ui_critical",
   "migration",
+  "test_lines_removed",
 ]);
 
 function readInput() {
@@ -48,7 +49,7 @@ function validate(obj) {
   if (typeof obj !== "object" || obj === null || Array.isArray(obj)) {
     return ["root must be an object"];
   }
-  if (obj.schemaVersion !== "1.0.0") errors.push(`schemaVersion must be "1.0.0", got ${JSON.stringify(obj.schemaVersion)}`);
+  if (obj.schemaVersion !== "1.1.0") errors.push(`schemaVersion must be "1.1.0", got ${JSON.stringify(obj.schemaVersion)}`);
 
   if (typeof obj.task !== "object" || obj.task === null) {
     errors.push("task must be an object");

package/pipeline/scripts/validate-triage.mjs

@@ -23,9 +23,10 @@
 
 import { readFileSync } from "node:fs";
 
-const ALLOWED_REVIEWERS = new Set(["opus", "sonnet"]);
+const ALLOWED_REVIEWERS = new Set(["fable", "opus", "sonnet", "gpt"]);
 const ALLOWED_SEVERITIES = new Set(["blocking", "important", "suggestion"]);
 const ALLOWED_CONSENSUS_VERDICTS = new Set(["unanimous-pass", "unanimous-block", "split", "unverified"]);
+const ALLOWED_VERIFICATION_RESULTS = new Set(["confirmed", "not-reproduced", "inconclusive"]);
 const OVER_REJECT_THRESHOLD = 0.8;
 const OVER_REJECT_MIN_FINDINGS = 5;
 
@@ -64,13 +65,41 @@ function validateRawFinding(f, label, errors) {
   if (typeof f.issue !== "string" || f.issue.length < 4) {
     errors.push(`${label}: issue must be a string ≥4 chars`);
   }
+  if (f.verification !== undefined) {
+    validateVerification(f.verification, `${label}.verification`, errors);
+  }
+}
+
+// v3.2.0 verify-by-test outcome (Phase 4 Step 3.7). Optional; when present:
+// result is required, and confirmed/not-reproduced additionally require
+// testRef + evidencePath (the empirical claims must be traceable).
+function validateVerification(v, label, errors) {
+  if (typeof v !== "object" || v === null || Array.isArray(v)) {
+    errors.push(`${label}: must be an object when present`);
+    return;
+  }
+  if (!ALLOWED_VERIFICATION_RESULTS.has(v.result)) {
+    errors.push(`${label}: bad result "${v.result}" (allowed: confirmed|not-reproduced|inconclusive)`);
+    return;
+  }
+  if (v.result === "confirmed" || v.result === "not-reproduced") {
+    if (typeof v.testRef !== "string" || v.testRef.length === 0) {
+      errors.push(`${label}: testRef required and non-empty when result is "${v.result}"`);
+    }
+    if (typeof v.evidencePath !== "string" || v.evidencePath.length === 0) {
+      errors.push(`${label}: evidencePath required and non-empty when result is "${v.result}"`);
+    }
+  }
+  if (v.note !== undefined && typeof v.note !== "string") {
+    errors.push(`${label}: note must be a string when present`);
+  }
 }
 
 function validateAccepted(f, i, errors) {
   validateRawFinding(f, `accepted[${i}]`, errors);
   if (!ALLOWED_REVIEWERS.has(f.reviewer)) {
     errors.push(
-      `accepted[${i}]: reviewer must be "opus" or "sonnet" (got ${JSON.stringify(f.reviewer)}; haiku was removed in v2.1.0)`,
+      `accepted[${i}]: reviewer must be one of fable|opus|sonnet|gpt (got ${JSON.stringify(f.reviewer)}; haiku was removed in v2.1.0)`,
     );
   }
   if (typeof f.fix !== "string" || f.fix.length < 4) {