bigpowers 2.73.4 → 2.73.6

package/scripts/run-golden-suite.sh

@@ -1,492 +1,8 @@
 #!/usr/bin/env bash
 # run-golden-suite.sh — Deterministic golden suite runner
-#
-# Runs compliance → deterministic gates → (optional) agent-driven golden
-# stories. Captures results, writes timestamped YAML report.
-# Supports --dry-run, --baseline, --check-size, and --agent.
-#
-# Usage:
-#   bash scripts/run-golden-suite.sh                    # deterministic gates only
-#   bash scripts/run-golden-suite.sh --agent            # deterministic + agent stories
-#   bash scripts/run-golden-suite.sh --agent --dry-run  # validate YAMLs only
-#   bash scripts/run-golden-suite.sh --dry-run          # print plan only
-#   bash scripts/run-golden-suite.sh --baseline         # run + pin baseline
-
 set -euo pipefail
 source "$(dirname "${BASH_SOURCE[0]}")/lib/python-env.sh"
-
-REPORT_DIR="specs/benchmarks/reports"
-BASELINE_FILE="$REPORT_DIR/BASELINE-GOLDEN.yaml"
-GOLDEN_DIR="specs/benchmarks/golden"
-DRY_RUN=false
-BASELINE_MODE=false
-CHECK_SIZE=false
-AGENT_MODE=false
-
-for arg in "$@"; do
-  case "$arg" in
-    --dry-run) DRY_RUN=true ;;
-    --baseline) BASELINE_MODE=true ;;
-    --check-size) CHECK_SIZE=true ;;
-    --agent) AGENT_MODE=true ;;
-    *) echo "Unknown flag: $arg"; echo "Usage: run-golden-suite.sh [--agent] [--dry-run] [--baseline] [--check-size]"; exit 2 ;;
-  esac
-done
-
-# ── Gate definitions ──────────────────────────────────────────────────
-# Each gate: name, command, optional: true (won't fail suite if missing)
-GATES=(
-  "compliance:npm run compliance:false"
-  "g04-selftest:bash scripts/golden-g04-selftest.sh:true"
-  "g06-migrate-version:bash tests/run-golden-migrate-version.sh:true"
-  "g07-negative-path:bash scripts/golden-g07-negative-path.sh:false"
-  "g08-anti-vacuity:bash scripts/golden-g08-anti-vacuity.sh:false"
-  "g09-yaml-roundtrip:bash scripts/golden-g09-yaml-roundtrip.sh:false"
-  "g10-trace-anti-vacuity:bash scripts/golden-g10-trace-anti-vacuity.sh:false"
-  "g11-gitignore-venv:bash scripts/golden-g11-gitignore-venv.sh:false"
-  "specs-parse:bash scripts/validate-specs-yaml.sh:false"
-)
-
-GREEN='\033[0;32m'
-RED='\033[0;31m'
-YELLOW='\033[0;33m'
-NC='\033[0m'
-
-# ── Agent mode: gh-aw check ────────────────────────────────────────────
-
-if $AGENT_MODE; then
-  if ! command -v gh-aw &>/dev/null && ! command -v gh &>/dev/null; then
-    echo -e "${YELLOW}WARNING: gh-aw not installed — agent gate skipped${NC}" >&2
-    AGENT_MODE=false  # degrade gracefully per 6b
-  fi
-fi
-
-# ── Agent dry-run: validate YAMLs ──────────────────────────────────────
-
-agent_dry_run() {
-  local yaml_count=0
-  local yaml_ok=0
-  local yaml_fail=0
-
-  echo "Agent Golden Stories — DRY RUN"
-  echo "Golden YAML directory: $GOLDEN_DIR"
-
-  if [[ ! -d "$GOLDEN_DIR" ]]; then
-    echo -e "${YELLOW}WARN: $GOLDEN_DIR not found${NC}"
-    return 0
-  fi
-
-  for yaml_file in "$GOLDEN_DIR"/g-*.yaml; do
-    [[ -f "$yaml_file" ]] || continue
-    (( yaml_count += 1 ))
-    story_id=$(basename "$yaml_file" .yaml)
-
-    if $PYTHON -c "import yaml; yaml.safe_load(open('$yaml_file'))" 2>/dev/null; then
-      local grader_type=$($PYTHON -c "import yaml; d=yaml.safe_load(open('$yaml_file')); print(d.get('grader',{}).get('type',''))" 2>/dev/null)
-      local pass_k=$($PYTHON -c "import yaml; d=yaml.safe_load(open('$yaml_file')); pk=d.get('pass_at_k',{}); print(f\"{pk.get('k',0)}-of-{pk.get('threshold',0)}\")" 2>/dev/null)
-      echo -e "  ${GREEN}OK${NC}   $story_id — grader=$grader_type, pass@k=$pass_k"
-      (( yaml_ok += 1 ))
-    else
-      echo -e "  ${RED}FAIL${NC} $story_id — YAML parse error"
-      (( yaml_fail += 1 ))
-    fi
-  done
-
-  echo ""
-  echo "Agent YAML validation: $yaml_ok/$yaml_count parseable"
-  if [[ "$yaml_fail" -gt 0 ]]; then
-    echo -e "${RED}$yaml_fail YAML(s) failed to parse${NC}"
-    return 1
-  fi
-  return 0
-}
-
-# ── Dry run ───────────────────────────────────────────────────────────
-
-if $DRY_RUN; then
-  if $AGENT_MODE; then
-    echo "Golden Suite — DRY RUN (deterministic + agent)"
-    echo ""
-    echo "Deterministic gates (in order):"
-    for gate in "${GATES[@]}"; do
-      IFS=':' read -r name cmd optional <<< "$gate"
-      echo "  - $name: $cmd"
-    done
-    echo ""
-    agent_dry_run
-    exit $?
-  else
-    echo "Golden Suite — DRY RUN"
-    echo "Gates (in order):"
-    for gate in "${GATES[@]}"; do
-      IFS=':' read -r name cmd optional <<< "$gate"
-      echo "  - $name: $cmd"
-    done
-    echo "Report: $REPORT_DIR/GOLDEN-YYYY-MM-DD.yaml"
-    echo "OK (dry run)"
-    exit 0
-  fi
-fi
-
-# ── Setup ─────────────────────────────────────────────────────────────
-
-mkdir -p "$REPORT_DIR"
-
-TODAY=$(date +%Y-%m-%d)
-TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
-GIT_COMMIT=$(git rev-parse HEAD 2>/dev/null || echo "unknown")
-
-if $BASELINE_MODE; then
-  REPORT_FILE="$BASELINE_FILE"
-else
-  REPORT_FILE="$REPORT_DIR/GOLDEN-${TODAY}.yaml"
-fi
-
-# ── Run deterministic gates ────────────────────────────────────────────
-# (Run first — agent stories execute only if these all pass)
-
-PASS_COUNT=0
-FAIL_COUNT=0
-SKIP_COUNT=0
-GATE_RESULTS=()
-
-for gate in "${GATES[@]}"; do
-  IFS=':' read -r name cmd is_optional <<< "$gate"
-
-  echo -n "[$name] "
-
-  # Check if command exists (for scripts)
-  main_cmd=$(echo "$cmd" | awk '{print $1}')
-  if [[ "$main_cmd" == "bash" ]]; then
-    script_path=$(echo "$cmd" | awk '{print $2}')
-    if [[ ! -f "$script_path" ]]; then
-      if [[ "$is_optional" == "true" ]]; then
-        echo -e "${YELLOW}SKIP${NC} (script not yet implemented)"
-        GATE_RESULTS+=("  - name: $name")
-        GATE_RESULTS+=("    exit_code: skipped")
-        GATE_RESULTS+=("    duration_seconds: 0")
-        GATE_RESULTS+=("    status: skipped")
-        GATE_RESULTS+=("    note: script not found: $script_path")
-        (( SKIP_COUNT += 1 ))
-        continue
-      else
-        echo -e "${RED}FAIL${NC} (script not found: $script_path)"
-        GATE_RESULTS+=("  - name: $name")
-        GATE_RESULTS+=("    exit_code: 127")
-        GATE_RESULTS+=("    duration_seconds: 0")
-        GATE_RESULTS+=("    status: fail")
-        GATE_RESULTS+=("    error: script not found: $script_path")
-        (( FAIL_COUNT += 1 ))
-        continue
-      fi
-    fi
-  fi
-
-  # Run the gate and capture output
-  START_TIME=$(date +%s%N)
-  GATE_OUTPUT=$(eval "$cmd" 2>&1) && GATE_EXIT=$? || GATE_EXIT=$?
-  END_TIME=$(date +%s%N)
-  DURATION_MS=$(( (END_TIME - START_TIME) / 1000000 ))
-  DURATION_SEC=$(echo "scale=2; $DURATION_MS / 1000" | bc 2>/dev/null || echo "0")
-
-  if [[ "$GATE_EXIT" -eq 0 ]]; then
-    echo -e "${GREEN}PASS${NC} (${DURATION_SEC}s)"
-    GATE_RESULTS+=("  - name: $name")
-    GATE_RESULTS+=("    exit_code: 0")
-    GATE_RESULTS+=("    duration_seconds: $DURATION_SEC")
-    GATE_RESULTS+=("    status: pass")
-    (( PASS_COUNT += 1 ))
-  else
-    echo -e "${RED}FAIL${NC} (${DURATION_SEC}s, exit $GATE_EXIT)"
-    GATE_RESULTS+=("  - name: $name")
-    GATE_RESULTS+=("    exit_code: $GATE_EXIT")
-    GATE_RESULTS+=("    duration_seconds: $DURATION_SEC")
-    GATE_RESULTS+=("    status: fail")
-    (( FAIL_COUNT += 1 ))
-  fi
-done
-
-# ── Agent-driven golden stories (e42s04) ──────────────────────────────
-
-AGENT_PASS=0
-AGENT_FAIL=0
-AGENT_SKIP=0
-AGENT_FLAKE=0
-AGENT_TOTAL=0
-AGENT_RESULTS=()
-DETERMINISTIC_PASSED=false
-
-if [[ "$FAIL_COUNT" -eq 0 ]]; then
-  DETERMINISTIC_PASSED=true
-fi
-
-if $AGENT_MODE && $DETERMINISTIC_PASSED; then
-  echo ""
-  echo "── Agent-Driven Golden Stories ──"
-
-  if [[ ! -d "$GOLDEN_DIR" ]]; then
-    echo -e "${YELLOW}WARN: $GOLDEN_DIR not found — agent stories skipped${NC}"
-    AGENT_RESULTS+=("  status: skipped")
-    AGENT_RESULTS+=("  reason: golden directory not found")
-    (( AGENT_SKIP += 1 ))
-  else
-    for yaml_file in "$GOLDEN_DIR"/g-*.yaml; do
-      [[ -f "$yaml_file" ]] || continue
-      story_id=$(basename "$yaml_file" .yaml)
-      echo -n "[$story_id] "
-
-      # Read pass@k policy
-      PASS_K=$($PYTHON -c "import yaml; d=yaml.safe_load(open('$yaml_file')); print(d.get('pass_at_k',{}).get('k',3))" 2>/dev/null || echo 3)
-      PASS_THRESHOLD=$($PYTHON -c "import yaml; d=yaml.safe_load(open('$yaml_file')); print(d.get('pass_at_k',{}).get('threshold',2))" 2>/dev/null || echo 2)
-
-      story_pass=0
-      story_fail=0
-      story_flake_count=0
-
-      for ((attempt=1; attempt<=PASS_K; attempt++)); do
-        # Invoke gh-aw for this golden story
-        # In CI, the workflow file is pre-compiled. Locally, check if .lock.yml exists.
-        LOCK_FILE=".github/workflows/e42-golden-deepseek.lock.yml"
-        if [[ ! -f "$LOCK_FILE" ]]; then
-          echo -e "${YELLOW}SKIP${NC} (no compiled workflow for $story_id)"
-          break
-        fi
-
-        # gh-aw run would go here. For now, check if workflow exists and is valid.
-        # The actual agent execution is triggered via gh workflow run in CI;
-        # local runs validate YAML definitions only.
-        if $PYTHON -c "
-import yaml
-d = yaml.safe_load(open('$yaml_file'))
-assert d.get('grader',{}).get('type') == 'code', 'grader must be code'
-assert d['grader'].get('command'), 'grader command required'
-assert d.get('pass_at_k',{}).get('k') == $PASS_K
-assert d['pass_at_k'].get('threshold') == $PASS_THRESHOLD
-" 2>/dev/null; then
-          # Simulated: YAML valid → 1 pass
-          (( story_pass += 1 ))
-        else
-          (( story_fail += 1 ))
-        fi
-      done
-
-      if [[ "$story_pass" -eq 0 && "$story_fail" -eq 0 ]]; then
-        echo -e "${YELLOW}SKIP${NC}"
-        AGENT_RESULTS+=("  - id: $story_id")
-        AGENT_RESULTS+=("    status: skipped")
-        AGENT_RESULTS+=("    pass_at_k: \"0/$PASS_K\"")
-        (( AGENT_SKIP += 1 ))
-      elif [[ "$story_pass" -ge "$PASS_THRESHOLD" ]]; then
-        flake_note=""
-        if [[ "$story_fail" -gt 0 ]]; then
-          flake_note=" (flake: $story_fail)"
-          (( AGENT_FLAKE += story_fail ))
-        fi
-        echo -e "${GREEN}PASS${NC}${flake_note} (pass@k: $story_pass/$PASS_K)"
-        AGENT_RESULTS+=("  - id: $story_id")
-        AGENT_RESULTS+=("    status: pass")
-        AGENT_RESULTS+=("    pass_at_k: \"$story_pass/$PASS_K\"")
-        AGENT_RESULTS+=("    flake_count: $story_fail")
-        (( AGENT_PASS += 1 ))
-      else
-        echo -e "${RED}FAIL${NC} (pass@k: $story_pass/$PASS_K, threshold $PASS_THRESHOLD)"
-        AGENT_RESULTS+=("  - id: $story_id")
-        AGENT_RESULTS+=("    status: fail")
-        AGENT_RESULTS+=("    pass_at_k: \"$story_pass/$PASS_K\"")
-        AGENT_RESULTS+=("    flake_count: $story_fail")
-        (( AGENT_FAIL += 1 ))
-      fi
-    done
-  fi
-elif $AGENT_MODE && ! $DETERMINISTIC_PASSED; then
-  echo ""
-  echo -e "${YELLOW}── Agent stories skipped: deterministic gates failed ──${NC}"
-  AGENT_RESULTS+=("  status: skipped")
-  AGENT_RESULTS+=("  reason: deterministic gates failed")
-fi
-
-# ── Combined summary ──────────────────────────────────────────────────
-
-TOTAL=$((PASS_COUNT + FAIL_COUNT + SKIP_COUNT))
-AGENT_TOTAL=$((AGENT_PASS + AGENT_FAIL + AGENT_SKIP))
-
-if [[ "$FAIL_COUNT" -gt 0 ]]; then
-  OVERALL="fail"
-elif [[ "$AGENT_FAIL" -gt 0 ]]; then
-  OVERALL="fail"
-else
-  OVERALL="pass"
-fi
-
-echo ""
-echo "══════════════════════════════════════════"
-echo "           GOLDEN SUITE VERDICT"
-echo "══════════════════════════════════════════"
-echo ""
-echo "Deterministic gates:"
-echo -e "  ${PASS_COUNT}/${TOTAL} passed"
-if [[ "$SKIP_COUNT" -gt 0 ]]; then
-  echo -e "  ${SKIP_COUNT} skipped"
-fi
-if [[ "$FAIL_COUNT" -gt 0 ]]; then
-  echo -e "  ${RED}${FAIL_COUNT} failed${NC}"
-fi
-
-if $AGENT_MODE; then
-  echo ""
-  echo "Agent-driven stories:"
-  echo -e "  ${AGENT_PASS}/${AGENT_TOTAL} passed"
-  if [[ "$AGENT_FAIL" -gt 0 ]]; then
-    echo -e "  ${RED}${AGENT_FAIL} failed${NC}"
-  fi
-  if [[ "$AGENT_SKIP" -gt 0 ]]; then
-    echo -e "  ${AGENT_SKIP} skipped"
-  fi
-  if [[ "$AGENT_FLAKE" -gt 0 ]]; then
-    echo -e "  ${YELLOW}${AGENT_FLAKE} flake(s) detected${NC} (absorbed by pass@k 2-of-3)"
-  fi
-fi
-
-echo ""
-echo -e "Combined verdict: $([[ "$OVERALL" == "pass" ]] && echo -e "${GREEN}PASS${NC}" || echo -e "${RED}FAIL${NC}")"
-echo "Report: $REPORT_FILE"
-echo "══════════════════════════════════════════"
-
-# ── Write report ──────────────────────────────────────────────────────
-
-PASS_RATE=$(echo "scale=2; $PASS_COUNT / $TOTAL" | bc 2>/dev/null || echo "0")
-AGENT_PASS_RATE="n/a"
-if [[ "$AGENT_TOTAL" -gt 0 ]]; then
-  AGENT_PASS_RATE=$(echo "scale=2; $AGENT_PASS / $AGENT_TOTAL" | bc 2>/dev/null || echo "0")
-fi
-
-cat > "$REPORT_FILE" << YAML_EOF
-timestamp: "$TIMESTAMP"
-git_commit: "$GIT_COMMIT"
-suite: "golden-combined"
-mode: "$([ "$BASELINE_MODE" = true ] && echo 'baseline' || echo 'daily')"
-deterministic:
-  total: $TOTAL
-  passed: $PASS_COUNT
-  failed: $FAIL_COUNT
-  skipped: $SKIP_COUNT
-  pass_rate: $PASS_RATE
-gates:
-$(printf '%s\n' "${GATE_RESULTS[@]}")
-agent_stories:
-  total: $AGENT_TOTAL
-  passed: $AGENT_PASS
-  failed: $AGENT_FAIL
-  skipped: $AGENT_SKIP
-  flake_count: $AGENT_FLAKE
-  pass_rate: $AGENT_PASS_RATE
-summary:
-  combined_verdict: "$OVERALL"
-YAML_EOF
-
-# ── OKF verification-report (e45s02) ────────────────────────────────────
-OKF_DIR="specs/verifications/reports"
-mkdir -p "$OKF_DIR"
-OKF_FILE="$OKF_DIR/GOLDEN-${TODAY}.okf.md"
-THRESHOLD=100
-GATE_STATUS="$OVERALL"
-
-cat > "$OKF_FILE" << OKF_EOF
----
-okf_kind: verification-report
-okf_version: "0.1"
-score: ${PASS_RATE}
-gate_status: "${GATE_STATUS}"
-threshold: ${THRESHOLD}
-total_pass: ${PASS_COUNT}
-total_fail: ${FAIL_COUNT}
-total_skip: ${SKIP_COUNT}
-generated_by: scripts/run-golden-suite.sh
-generated_at: "${TIMESTAMP}"
-git_commit: "${GIT_COMMIT}"
----
-
-# Golden Suite — ${TODAY}
-
-**Score:** ${PASS_RATE} (${PASS_COUNT}/${TOTAL} passed)
-**Gate:** ${GATE_STATUS} | **Threshold:** ${THRESHOLD}%
-
-See \`${REPORT_FILE}\` for the full YAML report.
-OKF_EOF
-
-echo "OKF report: $OKF_FILE"
-
-# ── Size budget (placeholder for e31s04) ──────────────────────────────
-
-if $BASELINE_MODE; then
-  echo "" >> "$REPORT_FILE"
-  echo "# Skill byte counts (baseline for e31s04 size budget)" >> "$REPORT_FILE"
-  echo "skill_sizes:" >> "$REPORT_FILE"
-  for skill_md in skills/*/SKILL.md; do
-    skill_name=$(basename "$(dirname "$skill_md")")
-    skill_bytes=$(wc -c < "$skill_md" | tr -d ' ')
-    echo "  $skill_name: $skill_bytes" >> "$REPORT_FILE"
-  done
-  echo "Baseline pinned: $REPORT_FILE"
-fi
-
-# ── Size budget check (e31s04) ───────────────────────────────────────
-
-if $CHECK_SIZE; then
-  WARN_THRESHOLD=20
-  FAIL_THRESHOLD=50
-
-  if [[ ! -f "$BASELINE_FILE" ]]; then
-    echo "No baseline found. Run --baseline first."
-    exit 1
-  fi
-
-  echo ""
-  echo "Size Budget Check (+${WARN_THRESHOLD}% warn, +${FAIL_THRESHOLD}% fail):"
-  SIZE_WARNINGS=0
-  SIZE_FAILURES=0
-  NEW_SKILLS=0
-
-  for skill_md in skills/*/SKILL.md; do
-    skill_name=$(basename "$(dirname "$skill_md")")
-    current_bytes=$(wc -c < "$skill_md" | tr -d ' ')
-
-    # Get baseline size (best-effort grep from YAML)
-    baseline_bytes=$(grep "^  $skill_name:" "$BASELINE_FILE" 2>/dev/null | awk '{print $2}')
-
-    if [[ -z "$baseline_bytes" ]]; then
-      echo "  ${YELLOW}NEW${NC}  $skill_name: ${current_bytes}b (not in baseline)"
-      (( NEW_SKILLS += 1 )) || true
-      continue
-    fi
-
-    if [[ "$baseline_bytes" -eq 0 ]]; then
-      baseline_bytes=1  # avoid division by zero
-    fi
-
-    pct=$(( (current_bytes - baseline_bytes) * 100 / baseline_bytes ))
-
-    if [[ "$pct" -ge "$FAIL_THRESHOLD" ]]; then
-      echo "  ${RED}FAIL${NC} $skill_name: ${current_bytes}b (+${pct}%)"
-      (( SIZE_FAILURES += 1 )) || true
-    elif [[ "$pct" -ge "$WARN_THRESHOLD" ]]; then
-      echo "  ${YELLOW}WARN${NC} $skill_name: ${current_bytes}b (+${pct}%)"
-      (( SIZE_WARNINGS += 1 )) || true
-    else
-      echo "  ${GREEN}OK${NC}   $skill_name: ${current_bytes}b"
-    fi
-  done
-
-  echo ""
-  echo "Size summary: $SIZE_WARNINGS warnings, $SIZE_FAILURES failures, $NEW_SKILLS new"
-
-  if [[ "$SIZE_FAILURES" -gt 0 ]]; then
-    exit 1
-  fi
-  exit 0
-fi
-
-if [[ "$OVERALL" == "fail" ]]; then
-  exit 1
-fi
-exit 0
+GOLDEN_GENERATED_BY="scripts/run-golden-suite.sh"
+GOLDEN_USAGE_NAME="run-golden-suite.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/lib/golden-suite-run.sh"
+golden_suite_main "$@"