bigpowers 2.89.0 → 2.90.0

package/scripts/lib/ab_metrics.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+# story: e58s01 e58s02 e58s03 e58s06
+"""A/B eval adapter output parsers: turn an agent CLI's stdout into cost, token and turn counts.
+
+claude-json      one result object (claude -p --output-format json)
+opencode-jsonl   one event per line (opencode run --format json); step_finish carries tokens + cost
+pi-json          one event per line (pi -p --mode json); assistant message_end carries usage + cost
+none             no metrics
+
+tap_score reads `node --test` summaries for a task's partial-credit score (e58s03).
+"""
+from __future__ import annotations
+
+import json
+import re
+
+
+def empty_metrics() -> dict:
+    return {"cost_usd": None, "input_tokens": None, "output_tokens": None, "turns": None,
+            "models": None, "agent_reported_error": None}
+
+
+def json_lines(stdout: str) -> list[dict]:
+    events = []
+    for line in stdout.strip().splitlines():
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(obj, dict):
+            events.append(obj)
+    return events
+
+
+def parse_metrics(parser: str, stdout: str) -> dict:
+    if parser == "claude-json":
+        return parse_claude_json(stdout)
+    if parser == "opencode-jsonl":
+        return parse_opencode_jsonl(stdout)
+    if parser == "pi-json":
+        return parse_pi_json(stdout)
+    return empty_metrics()
+
+
+def parse_claude_json(stdout: str) -> dict:
+    metrics = empty_metrics()
+    result = next((e for e in reversed(json_lines(stdout)) if "num_turns" in e), None)
+    if result is None:
+        return metrics
+    input_tokens, output_tokens = token_totals(result)
+    metrics.update({
+        "cost_usd": result.get("total_cost_usd"),
+        "input_tokens": input_tokens,
+        "output_tokens": output_tokens,
+        "turns": result.get("num_turns"),
+        "models": sorted(result.get("modelUsage") or {}) or None,
+        "agent_reported_error": bool(result.get("is_error")),
+    })
+    return metrics
+
+
+def token_totals(result: dict) -> tuple[int | None, int | None]:
+    """Whole-tree tokens from modelUsage (incl. subagents and cache), else main-loop usage."""
+    per_model = [m for m in (result.get("modelUsage") or {}).values() if "inputTokens" in m]
+    if per_model:
+        inputs = sum(m.get("inputTokens", 0) + m.get("cacheReadInputTokens", 0)
+                     + m.get("cacheCreationInputTokens", 0) for m in per_model)
+        return inputs, sum(m.get("outputTokens", 0) for m in per_model)
+    usage = result.get("usage") or {}
+    if "input_tokens" not in usage:
+        return None, usage.get("output_tokens")
+    inputs = (usage["input_tokens"] + usage.get("cache_read_input_tokens", 0)
+              + usage.get("cache_creation_input_tokens", 0))
+    return inputs, usage.get("output_tokens")
+
+
+def parse_opencode_jsonl(stdout: str) -> dict:
+    """Sum step_finish parts: tokens = {input, output, reasoning, cache: {read, write}}, cost in USD."""
+    metrics = empty_metrics()
+    events = json_lines(stdout)
+    steps = [(e.get("part") or {}) for e in events if e.get("type") == "step_finish"]
+    errored = any(e.get("type") == "error" for e in events)
+    if not steps:
+        metrics["agent_reported_error"] = errored or None
+        return metrics
+    cost = inputs = outputs = 0.0
+    for part in steps:
+        tokens = part.get("tokens") or {}
+        cache = tokens.get("cache") or {}
+        cost += part.get("cost") or 0
+        inputs += tokens.get("input", 0) + cache.get("read", 0) + cache.get("write", 0)
+        outputs += tokens.get("output", 0) + tokens.get("reasoning", 0)
+    metrics.update({"cost_usd": round(cost, 6), "input_tokens": int(inputs), "output_tokens": int(outputs),
+                    "turns": len(steps), "agent_reported_error": errored})
+    return metrics
+
+
+def parse_pi_json(stdout: str) -> dict:
+    metrics = empty_metrics()
+    events = json_lines(stdout)
+    replies = [e["message"] for e in events if e.get("type") == "message_end"
+               and (e.get("message") or {}).get("role") == "assistant"]
+    if not replies:
+        return metrics
+    aborted = any(e.get("type") == "agent_settled" and e.get("aborted") for e in events)
+    cost = inputs = outputs = 0.0
+    for msg in replies:
+        usage = msg.get("usage") or {}
+        cost += (usage.get("cost") or {}).get("total") or 0
+        inputs += usage.get("input", 0) + usage.get("cacheRead", 0) + usage.get("cacheWrite", 0)
+        outputs += usage.get("output", 0)
+    errored = aborted or any(m.get("stopReason") in ("error", "aborted") for m in replies)
+    metrics.update({"cost_usd": round(cost, 6), "input_tokens": int(inputs), "output_tokens": int(outputs),
+                    "turns": len(replies), "agent_reported_error": errored})
+    return metrics
+
+
+def tap_score(stdout: str) -> float | None:
+    """Share of tests passing over every TAP summary (`node --test --test-reporter=tap`: piped
+    output is the spec reporter otherwise). Cancelled tests (timeouts) count as failed."""
+    passed = sum(int(n) for n in re.findall(r"^# pass (\d+)", stdout, re.M))
+    failed = sum(int(n) for n in re.findall(r"^# (?:fail|cancelled) (\d+)", stdout, re.M))
+    return round(passed / (passed + failed), 4) if passed + failed else None

package/scripts/lib/ab_progress.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# story: e58s05
+"""Live view of an A/B eval while it runs: reads every ab-live.jsonl under the given folders
+(one per shard) and prints runs, pass rate, score and cost per task and arm, each arm's delta
+against vanilla with its bootstrap CI, and how those deltas evolved as runs accumulated.
+
+Usage: python3 scripts/lib/ab_progress.py <out-dir>... [--watch SECONDS]
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from ab_report import aggregate, mean  # type: ignore[import]  # noqa: E402
+
+CHECKPOINTS = (5, 10, 20, 30, 50, 75, 100)
+
+
+def load(dirs: list[Path]) -> list[dict]:
+    records = []
+    for folder in dirs:
+        for live in sorted(folder.rglob("ab-live.jsonl")):
+            records += [json.loads(line) for line in live.read_text().splitlines() if line.strip()]
+    return records
+
+
+def ordered(values: list[str], preferred: str) -> list[str]:
+    unique = sorted(set(values))
+    return [preferred] + [v for v in unique if v != preferred] if preferred in unique else unique
+
+
+def fmt(value, places=2) -> str:
+    return "-" if value is None else f"{value:.{places}f}"
+
+
+def table(records: list[dict], tasks: list[str], arms: list[str]) -> list[str]:
+    lines = [f"{'task':<8} {'arm':<10} {'runs':>5} {'pass':>6} {'score':>6} {'cost$':>7} {'errors':>6}"]
+    for task in tasks:
+        for arm in arms:
+            runs = [r for r in records if r["task"] == task and r["arm"] == arm]
+            if runs:
+                lines.append(f"{task:<8} {arm:<10} {len(runs):>5} {fmt(mean([float(r['pass']) for r in runs])):>6} "
+                             f"{fmt(mean([r.get('score') for r in runs])):>6} {fmt(mean([r['cost_usd'] for r in runs]), 3):>7} "
+                             f"{sum(r['agent_error'] for r in runs):>6}")
+    return lines
+
+
+def delta_line(name: str, d: dict) -> str:
+    line = f"  {name} - vanilla: pass {d['pass_rate']:+.2f} {d['ci95']}"
+    if "score" in d:
+        line += f"  score {d['score']:+.3f} {d['score_ci95']}"
+    return line
+
+
+def evolution(records: list[dict], tasks: list[str], arms: list[str]) -> list[str]:
+    """Deltas using only runs with index <= each checkpoint: shows the CI narrowing over time."""
+    top = max(r["run"] for r in records)
+    lines = []
+    for cp in [c for c in CHECKPOINTS if c < top] + [top]:
+        subset = [r for r in records if r["run"] <= cp]
+        for name, d in aggregate(subset, arms, tasks)["deltas_vs_vanilla"].items():
+            lines.append(f"  runs<={cp:<4}{delta_line(name, d)[1:]}")
+    return lines
+
+
+def render(records: list[dict]) -> str:
+    tasks = sorted({r["task"] for r in records})
+    arms = ordered([r["arm"] for r in records], "vanilla")
+    models = sorted({str(r.get("model")) for r in records})
+    out = [f"A/B progress — {len(records)} runs, model {', '.join(models)}, at {datetime.now():%H:%M:%S}", ""]
+    out += table(records, tasks, arms)
+    result = aggregate(records, arms, tasks)
+    if result["deltas_vs_vanilla"]:
+        out += ["", "deltas (all runs so far, 95% CI):"]
+        out += [delta_line(n, d) for n, d in result["deltas_vs_vanilla"].items()]
+        out += ["", "evolution (pass and score delta by run index):"] + evolution(records, tasks, arms)
+    return "\n".join(out)
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description="Live A/B eval progress across shards")
+    p.add_argument("dirs", nargs="+", type=Path)
+    p.add_argument("--watch", type=int, help="repeat every N seconds")
+    args = p.parse_args(argv)
+    while True:
+        records = load(args.dirs)
+        if not records:
+            print("no ab-live.jsonl found under: " + ", ".join(map(str, args.dirs)))
+            return 2
+        print(render(records), flush=True)
+        if not args.watch:
+            return 0
+        time.sleep(args.watch)
+        print()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/lib/ab_report.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# story: e58s01 e58s03 e58s04
+"""A/B eval aggregation and reports — pure functions over run records (no agent, no subprocess
+except reading the bigpowers git SHA). Split from ab-eval.py to keep both under the 300-line cap."""
+from __future__ import annotations
+
+import json
+import random
+import subprocess
+from datetime import date
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+BOOTSTRAP_RESAMPLES = 1000
+BOOTSTRAP_SEED = 58
+
+
+# ── aggregation ──────────────────────────────────────────────────────────────
+
+def mean(values: list) -> float | None:
+    vals = [v for v in values if v is not None]
+    return round(sum(vals) / len(vals), 4) if vals else None
+
+
+def bootstrap_delta_ci(treatment: list[float], control: list[float]) -> list[float]:
+    """Seeded 95% CI of mean(treatment) - mean(control); bools or scores in [0, 1]."""
+    rng = random.Random(BOOTSTRAP_SEED)
+    deltas = []
+    for _ in range(BOOTSTRAP_RESAMPLES):
+        t = [rng.choice(treatment) for _ in treatment]
+        c = [rng.choice(control) for _ in control]
+        deltas.append(sum(t) / len(t) - sum(c) / len(c))
+    deltas.sort()
+    lo = deltas[int(0.025 * BOOTSTRAP_RESAMPLES)]
+    hi = deltas[int(0.975 * BOOTSTRAP_RESAMPLES) - 1]
+    return [round(lo, 4), round(hi, 4)]
+
+
+def arm_summary(runs: list[dict]) -> dict:
+    return {
+        "runs": len(runs),
+        "pass_rate": mean([float(r["pass"]) for r in runs]),
+        "mean_score": mean([r.get("score") for r in runs]),
+        "tamper_rate": mean([float(r["tampered"]) for r in runs]),
+        "agent_error_rate": mean([float(r["agent_error"]) for r in runs]),
+        "mean_cost_usd": mean([r["cost_usd"] for r in runs]),
+        "mean_input_tokens": mean([r["input_tokens"] for r in runs]),
+        "mean_output_tokens": mean([r["output_tokens"] for r in runs]),
+        "mean_turns": mean([r["turns"] for r in runs]),
+        "mean_duration_s": mean([r["duration_s"] for r in runs]),
+        "mean_diff_lines": mean([r["diff_lines"] for r in runs]),
+    }
+
+
+def aggregate(records: list[dict], arms: list[str], tasks: list[str]) -> dict:
+    by_arm = {a: [r for r in records if r["arm"] == a] for a in arms}
+    summary = {a: arm_summary(runs) for a, runs in by_arm.items()}
+    per_task = {t: {a: mean([float(r["pass"]) for r in by_arm[a] if r["task"] == t]) for a in arms}
+                for t in tasks}
+    per_task_score = {t: {a: mean([r.get("score") for r in by_arm[a] if r["task"] == t]) for a in arms}
+                      for t in tasks}
+    deltas = {a: arm_delta(by_arm, summary, a) for a in arms if a != "vanilla"} if "vanilla" in arms else {}
+    return {"arms_summary": summary, "per_task": per_task, "per_task_score": per_task_score,
+            "delta": deltas.get("bigpowers"), "deltas_vs_vanilla": deltas}
+
+
+def arm_delta(by_arm: dict, summary: dict, arm: str) -> dict:
+    """arm - vanilla: pass rate and score with seeded bootstrap CIs, tamper rate and cost (e58s04)."""
+    tr, va = summary[arm], summary["vanilla"]
+    cost = None
+    if tr["mean_cost_usd"] is not None and va["mean_cost_usd"] is not None:
+        cost = round(tr["mean_cost_usd"] - va["mean_cost_usd"], 4)
+    delta = {
+        "pass_rate": round(tr["pass_rate"] - va["pass_rate"], 4),
+        "ci95": bootstrap_delta_ci([r["pass"] for r in by_arm[arm]], [r["pass"] for r in by_arm["vanilla"]]),
+        "tamper_rate": round(tr["tamper_rate"] - va["tamper_rate"], 4),
+        "cost_usd": cost,
+    }
+    scores = {a: [r["score"] for r in by_arm[a] if r.get("score") is not None] for a in (arm, "vanilla")}
+    if scores[arm] and scores["vanilla"]:
+        delta["score"] = round(tr["mean_score"] - va["mean_score"], 4)
+        delta["score_ci95"] = bootstrap_delta_ci(scores[arm], scores["vanilla"])
+    return delta
+
+
+# ── reporting ────────────────────────────────────────────────────────────────
+
+def bigpowers_sha() -> str:
+    res = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=REPO_ROOT, capture_output=True, text=True)
+    return res.stdout.strip() or "unknown"
+
+
+def write_reports(out: Path, meta: dict, result: dict, records: list[dict]) -> Path:
+    out.mkdir(parents=True, exist_ok=True)
+    today = date.today().isoformat()
+    report = out / f"AB-{today}.yaml"
+    report.write_text(yaml.safe_dump({**meta, **result}, sort_keys=False))
+    with (out / f"ab-runs-{today}.jsonl").open("w") as f:
+        for rec in records:
+            f.write(json.dumps({**rec, "bigpowers_sha": meta["bigpowers_sha"]}) + "\n")
+    return report
+
+
+def print_summary(result: dict) -> None:
+    print(f"{'arm':<10} {'pass':>6} {'score':>6} {'tamper':>7} {'cost$':>8} {'turns':>6} {'secs':>7}")
+    for arm, s in result["arms_summary"].items():
+        fmt = lambda v, p=2: "-" if v is None else f"{v:.{p}f}"
+        print(f"{arm:<10} {fmt(s['pass_rate']):>6} {fmt(s['mean_score']):>6} {fmt(s['tamper_rate']):>7} "
+              f"{fmt(s['mean_cost_usd'], 3):>8} {fmt(s['mean_turns'], 1):>6} {fmt(s['mean_duration_s'], 0):>7}")
+    for arm, d in result.get("deltas_vs_vanilla", {}).items():
+        line = f"delta {arm:<10} - vanilla: pass {d['pass_rate']:+.2f} CI {d['ci95']}"
+        if "score" in d:
+            line += f"  score {d['score']:+.2f} CI {d['score_ci95']}"
+        print(line)

package/scripts/lib/golden-suite-gates.sh

@@ -79,6 +79,8 @@ GOLDEN_GATES=(
   "specs-parse:bash scripts/validate-specs-yaml.sh:false"
   # The runner itself: failing output is shown, skips are named and reasoned.
   "golden-gate-runner:bash scripts/test-golden-gate-runner.sh:false"
+  # Pipefail SIGPIPE races: fix shapes hold, git output never feeds grep -q/head.
+  "pipefail-pipe-race:bash scripts/test-pipefail-pipe-race.sh:false"
   # One pipeline, release needs: every gate, pinned runner and actions.
   "ci-pipeline-contract-selftest:bash scripts/test-ci-pipeline-contract.sh --self-test:false"
   "ci-pipeline-contract:bash scripts/test-ci-pipeline-contract.sh:false"

package/scripts/lib/parallel-review-worktrees.sh

@@ -11,7 +11,9 @@ cd "$REPO_ROOT"
 BASE=".bigpowers/worktrees/review-${CHECK}"
 mkdir -p "$(dirname "$BASE")"
 
-if git worktree list | grep -q "$BASE"; then
+# Capture first: `git worktree list | grep -q` is a pipefail race (BUG-2026-11-02-pipefail-pipe-race).
+WORKTREES="$(git worktree list)"
+if grep -q "$BASE" <<<"$WORKTREES"; then
   git worktree remove --force "$BASE" 2>/dev/null || true
 fi
 

package/scripts/lib/plan-consistency-check.sh

@@ -69,7 +69,7 @@ for tasks in "${TASKS[@]+"${TASKS[@]}"}"; do
   sid="$(echo "$base" | grep -oE 'e[0-9]+s[0-9]+' | head -1)"
   [[ -n "$sid" ]] || continue
 
-  spec_name="$(grep -E '^spec:' "$tasks" | head -1 | sed -E 's/^spec:[[:space:]]*//; s/^"(.*)"$/\1/')"
+  spec_name="$(grep -E '^spec:' "$tasks" | head -1 | sed -E 's/^spec:[[:space:]]*//; s/^"(.*)"$/\1/' || true)"
   spec="$CAPSULE/$spec_name"
   if [[ -z "$spec_name" || ! -f "$spec" ]]; then
     report CRITICAL "Story $sid tasks.yaml declares no valid spec: file"
@@ -92,7 +92,9 @@ for tasks in "${TASKS[@]+"${TASKS[@]}"}"; do
     report CRITICAL "Story $sid tasks.yaml missing runnable verify: commands"
   else
     verify_cmd="$("$PYTHON_BIN" "$SCRIPT_LIB_DIR/extract-story-verify.py" "$REPO_ROOT" 2>/dev/null \
-      | awk -F'\t' -v s="$sid" '$1 == s { print $4; exit }')"
+      | awk -F'\t' -v s="$sid" '$1 == s && !found { print $4; found = 1 }')"
+    # awk reads to the end: exiting early closes the pipe, Python fails to flush and exits 120,
+    # and pipefail turns that into a random failure of this check.
     if [[ -n "$verify_cmd" ]]; then
       is_executable_verify "$verify_cmd" \
         || report CRITICAL "Story $sid verify: is prose, not a runnable command: $verify_cmd"

package/scripts/lib/run-benchmark.py

@@ -60,26 +60,44 @@ def scenario_pass_rate(scenario: dict, runs: int, with_skill: bool) -> tuple[flo
     if not command:
         return (-1.0, "missing_command")
     if not with_skill:
-        command = f"test -f /dev/null"  # bare agent: code graders cannot pass without skill context
+        # No agent runs here, so there is no honest without-skill number (e58s01).
+        # Real with/without deltas come from scripts/run-ab-eval.sh.
+        return (-1.0, "unmeasured")
     passes = sum(1 for _ in range(runs) if run_code_grader(command, REPO_ROOT))
     return (passes / runs, "code")
 
 
-def pass_at_k(scenarios: list[dict], runs: int, with_skill: bool) -> float:
+def pass_at_k(scenarios: list[dict], runs: int, with_skill: bool) -> float | None:
     total_weight = 0.0
     weighted = 0.0
     for sc in scenarios:
         rate, status = scenario_pass_rate(sc, runs, with_skill)
-        if status == "pending_agent_eval":
+        if status in ("pending_agent_eval", "unmeasured"):
             continue
         weight = float(sc.get("weight", 1.0))
         total_weight += weight
         weighted += weight * max(rate, 0.0)
     if total_weight == 0:
-        return 0.0
+        return None
     return round(weighted / total_weight, 2)
 
 
+def delta_of(with_score: float | None, without_score: float | None) -> float | None:
+    if with_score is None or without_score is None:
+        return None
+    return round(with_score - without_score, 2)
+
+
+def delta_summary(val_delta: float | None) -> str:
+    if val_delta is None:
+        return "UNMEASURED: no without-skill agent run — use scripts/run-ab-eval.sh for a real delta"
+    if val_delta < 0:
+        return f"REGRESSION: validation Δ {val_delta} — do NOT ship"
+    if val_delta < 0.05:
+        return f"WARN: validation Δ {val_delta} — marginal contribution"
+    return f"OK: validation Δ {val_delta}"
+
+
 def partition_scenarios(scenarios: list[dict]) -> tuple[list[dict], list[dict]]:
     train, validation = [], []
     for sc in scenarios:
@@ -93,13 +111,13 @@ def partition_scenarios(scenarios: list[dict]) -> tuple[list[dict], list[dict]]:
 
 def build_split_report(scenarios: list[dict], runs: int) -> dict:
     if not scenarios:
-        return {"with_skill": 0.0, "without_skill": 0.0, "delta": 0.0, "scenarios": []}
+        return {"with_skill": None, "without_skill": None, "delta": None, "scenarios": []}
     with_score = pass_at_k(scenarios, runs, with_skill=True)
     without_score = pass_at_k(scenarios, runs, with_skill=False)
     return {
         "with_skill": with_score,
         "without_skill": without_score,
-        "delta": round(with_score - without_score, 2),
+        "delta": delta_of(with_score, without_score),
         "scenarios": [sc["id"] for sc in scenarios],
     }
 
@@ -119,13 +137,15 @@ def scenario_details(scenarios: list[dict], runs: int) -> list[dict]:
                 }
             )
             continue
+        with_value = round(max(with_rate, 0.0), 2)
+        without_value = None if without_rate < 0 else round(without_rate, 2)
         rows.append(
             {
                 "id": sc["id"],
                 "split": sc.get("split", "validation"),
-                "with_pass_rate": round(max(with_rate, 0.0), 2),
-                "without_pass_rate": round(max(without_rate, 0.0), 2),
-                "delta": round(max(with_rate, 0.0) - max(without_rate, 0.0), 2),
+                "with_pass_rate": with_value,
+                "without_pass_rate": without_value,
+                "delta": delta_of(with_value, without_value),
                 "weight": sc.get("weight", 1.0),
             }
         )
@@ -180,13 +200,7 @@ def run_benchmark(skill: str, baseline: bool = False) -> dict:
         shutil.copy(yaml_path, REPORTS_DIR / f"BASELINE-{skill}.yaml")
         shutil.copy(json_path, REPORTS_DIR / f"baseline-{skill}.json")
         print(f"Pinned baseline for {skill}")
-    val_delta = report["validation"]["delta"]
-    if val_delta < 0:
-        print(f"REGRESSION: validation Δ {val_delta} — do NOT ship")
-    elif val_delta < 0.05:
-        print(f"WARN: validation Δ {val_delta} — marginal contribution")
-    else:
-        print(f"OK: validation Δ {val_delta}")
+    print(delta_summary(report["validation"]["delta"]))
     return report
 
 

package/scripts/run-ab-eval.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# story: e58s01
+# run-ab-eval.sh — run the same tasks with and without bigpowers; grade only the outcome.
+# Advisory: exits 0 after a completed run, 2 on configuration errors. See specs/benchmarks/SCHEMA.md.
+set -euo pipefail
+source "$(dirname "${BASH_SOURCE[0]}")/lib/python-env.sh"
+exec "$PYTHON" "$(dirname "${BASH_SOURCE[0]}")/lib/ab-eval.py" "$@"

package/scripts/run-smoke.sh

@@ -46,7 +46,8 @@ run_single_url_check() {
 
   # Assert content signal
   if [ -n "$content_signal" ]; then
-    if echo "$body" | grep -qiE "$content_signal"; then
+    # Here-string, not `echo "$body" | grep -q`: a large body races pipefail (BUG-2026-11-02-pipefail-pipe-race).
+    if grep -qiE "$content_signal" <<<"$body"; then
       echo "  PASS: body matches \"$content_signal\""
     else
       echo "  FAIL: body does not match \"$content_signal\""
@@ -165,7 +166,7 @@ run_parsed_check() {
 
   # Assert content signal
   if [ -n "$content_signal" ]; then
-    if echo "$body" | grep -qiE "$content_signal"; then
+    if grep -qiE "$content_signal" <<<"$body"; then
       echo "  PASS: body matches \"$content_signal\""
       checks_passed=$((checks_passed + 1))
     else

package/scripts/run-story-verify.sh

@@ -82,6 +82,13 @@ run_story_verify() {
   fi
   echo "FAIL: $sid [$src]$note — $cmd"
   echo "      output: $(echo "$RUN_OUT" | tail -3 | tr '\n' ' ')"
+  # Name the failing test. unittest/pytest footers ("FAILED (failures=1)") say a test
+  # broke but not which one, so a red gate is undiagnosable from CI logs alone
+  # (BUG-2026-11-03-story-verify-output-hides-failing-test).
+  local detail
+  detail="$(echo "$RUN_OUT" | grep -E '^(FAIL|ERROR): |\.\.\. (FAIL|ERROR)$|^AssertionError|^[A-Za-z]*Error: |^Traceback' \
+    | head -3 | tr '\n' ' ')"
+  [ -n "$detail" ] && echo "      detail: $detail"
   FAIL=$((FAIL + 1))
   return 1
 }

package/scripts/test-pipefail-pipe-race.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# story: BUG-2026-11-02-pipefail-pipe-race
+# test-pipefail-pipe-race.sh — regression guard for SIGPIPE races under pipefail.
+#
+# `producer | grep -q` (or `| head`) fails at random under `set -o pipefail`:
+# the reader exits early, the producer dies with SIGPIPE (141), and pipefail
+# reports the pipeline as failed — a found match reads as "not found".
+#
+#   (a) anti-vacuity: the racy shape really fails on this host
+#   (b) each fix shape used in the repo passes RUNS times with a long producer
+#   (c) static guard: no pipefail script pipes git output into an early-exit reader
+#
+# Usage: bash scripts/test-pipefail-pipe-race.sh
+
+set -euo pipefail
+source "$(dirname "${BASH_SOURCE[0]}")/lib/skill-common.sh"
+resolve_repo_root
+cd "$REPO_ROOT"
+
+source scripts/lib/test-assertions.sh
+TA_PASS=0
+TA_FAIL=0
+RUNS=20
+
+# ~2 MB of output: far past any pipe buffer, so an early reader always wins.
+long_producer() { seq 1 300000; }
+
+# Runs "$@" RUNS times; prints how many runs returned non-zero.
+count_failures() {
+  local i fails=0
+  for ((i = 0; i < RUNS; i++)); do
+    "$@" || fails=$((fails + 1))
+  done
+  echo "$fails"
+}
+
+racy_grep_q()      { long_producer | grep -q '^1$'; }
+fixed_capture()    { local out; out="$(long_producer)"; grep -q '^1$' <<<"$out"; }
+fixed_nonempty()   { [[ -n "$(long_producer)" ]]; }
+fixed_awk_limit()  { long_producer | awk -v limit=3 'NR <= limit' >/dev/null; }
+fixed_sed_range()  { long_producer | sed -n '1,5p' >/dev/null; }
+
+echo "=== test-pipefail-pipe-race.sh ==="
+
+fails=$(count_failures racy_grep_q)
+[[ "$fails" -gt 0 ]] \
+  && ta_pass "(a) racy 'producer | grep -q' fails under pipefail ($fails/$RUNS)" \
+  || ta_fail "(a) racy shape never failed — test cannot detect the race on this host"
+
+for fix in fixed_capture fixed_nonempty fixed_awk_limit fixed_sed_range; do
+  fails=$(count_failures "$fix")
+  [[ "$fails" -eq 0 ]] \
+    && ta_pass "(b) $fix: 0/$RUNS failures" \
+    || ta_fail "(b) $fix: $fails/$RUNS failures"
+done
+
+# run-skill-verify.sh holds `git diff ... | head -20` as a fixture string of
+# fail-open verify directives, not as a command it runs.
+hits=$(grep -rlE 'pipefail' scripts hooks --include='*.sh' \
+  | grep -v '^scripts/run-skill-verify\.sh$' \
+  | xargs grep -nE 'git [a-z-]+[^|]*\|[[:space:]]*(grep -[a-zA-Z]*[qm]|head)' \
+  | grep -vE '^[^:]+:[0-9]+:[[:space:]]*#' || true)
+[[ -z "$hits" ]] \
+  && ta_pass "(c) no pipefail script pipes git output into grep -q/-m or head" \
+  || ta_fail "(c) git output piped into an early-exit reader:"$'\n'"$hits"
+
+echo "$TA_PASS passed, $TA_FAIL failed"
+[[ "$TA_FAIL" -eq 0 ]]

package/scripts/validate-doctrine.sh

@@ -55,7 +55,7 @@ else
   grep -rE "PLAN\.md|STATE\.md|PROJECT\.md|CONTEXT\.md|SUMMARY\.md|VERIFICATION\.md|RESEARCH\.md|RELEASE-PLAN\.md" \
     docs/ 2>/dev/null \
     | grep -v "docs/archive/" \
-    | grep -v "specs/archive" | head -20 >&2
+    | grep -v "specs/archive" | sed -n '1,20p' >&2  # sed reads to the end; `head` races pipefail (BUG-2026-11-02-pipefail-pipe-race)
 fi
 
 # ── Epic 1: Skill count consistency ───────────────────────────────────────────
@@ -94,7 +94,7 @@ else
     2>/dev/null \
     | grep -v "specs/archive\|specs/epics/archive\|docs/archive/\|\.git\|node_modules\|\.gemini\|\.cursor" \
     | grep -v "CHANGELOG\.md\|specs/verifications/reports/\|PLAN-evolve-structure\.md" \
-    | head -10 >&2
+    | sed -n '1,10p' >&2
 fi
 
 # ── Epic 3: BCP slop guard ────────────────────────────────────────────────────
@@ -107,7 +107,7 @@ if [[ "$BCP_SLOP" -eq 0 ]]; then
 else
   doctrine_fail "'Build Commit Point' found ($BCP_SLOP hits) — canonical term is 'Business Complexity Points'; see docs/references/bcp.md"
   grep -ri "build commit point" . 2>/dev/null \
-    | grep -v "specs/archive\|\.git\|node_modules\|\.gemini\|\.cursor\|validate-doctrine\.sh" | head -5 >&2
+    | grep -v "specs/archive\|\.git\|node_modules\|\.gemini\|\.cursor\|validate-doctrine\.sh" | sed -n '1,5p' >&2
 fi
 
 # ── Epic 4: SKILL.md size cap ─────────────────────────────────────────────────