@mmerterden/multi-agent-pipeline 12.7.0 → 12.9.0

package/pipeline/scripts/smoke-run-aggregator.sh

@@ -1,216 +0,0 @@
-#!/usr/bin/env bash
-# smoke-run-aggregator.sh  -  v7.8.0 Paket C: unified telemetry aggregator.
-#
-# Validates run-aggregator.mjs reads phase-tracker + otel-spans + cost table
-# and emits a normalized pipeline-run.jsonl with:
-#   1. Time-sorted events
-#   2. Correct cost computation per cost-table.json
-#   3. Span passthrough with task_id stamping
-#   4. Summary as last event with totals
-#   5. Unknown model graceful degradation
-#   6. --summary-only mode
-#   7. Read-only contract (no writes to source files)
-#   8. Missing inputs handled with clear errors
-#   9. Idempotent (running twice produces identical output)
-
-set -uo pipefail
-
-REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
-SCRIPT="$REPO_ROOT/pipeline/scripts/run-aggregator.mjs"
-
-PASS=0
-FAIL=0
-pass() { PASS=$((PASS + 1)); echo "  ✓ $1"; }
-fail() { FAIL=$((FAIL + 1)); echo "  ✗ $1"; }
-
-cleanup() { [ -n "${TMP:-}" ] && rm -rf "$TMP"; }
-trap cleanup EXIT
-TMP=$(mktemp -d)
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 1. Setup synthetic task dir + tracker + spans"
-cat > "$TMP/phase-tracker.json" <<'EOF'
-{
-  "task_id": "PROJ-100",
-  "started_at": "2026-04-27T10:00:00Z",
-  "phases": [
-    {"id":"0","name":"Init","status":"completed","started_at":"2026-04-27T10:00:00Z","completed_at":"2026-04-27T10:00:30Z","tokens_in":1000,"tokens_out":200,"model":"sonnet","subs":[]},
-    {"id":"1","name":"Analysis","status":"completed","started_at":"2026-04-27T10:00:30Z","completed_at":"2026-04-27T10:05:00Z","tokens_in":50000,"tokens_out":3000,"model":"opus","subs":[{"id":"1.0","name":"Explore","status":"completed"}]},
-    {"id":"3","name":"Dev","status":"failed","started_at":"2026-04-27T10:10:00Z","completed_at":"2026-04-27T10:30:00Z","tokens_in":10000,"tokens_out":2000,"model":"sonnet","subs":[]}
-  ]
-}
-EOF
-cat > "$TMP/otel-spans.jsonl" <<'EOF'
-{"name":"phase.update","ts":"2026-04-27T10:00:30Z","attributes":{"multi_agent.status":"completed"}}
-{"name":"phase.tokens","ts":"2026-04-27T10:25:00Z","attributes":{"multi_agent.tokens_in_delta":10000}}
-EOF
-[ -f "$TMP/phase-tracker.json" ] && pass "synthetic phase-tracker.json written"
-[ -f "$TMP/otel-spans.jsonl" ] && pass "synthetic otel-spans.jsonl written"
-
-# Compute hash of source files BEFORE running aggregator
-HASH_BEFORE=$(md5 -q "$TMP/phase-tracker.json" "$TMP/otel-spans.jsonl" 2>/dev/null || \
-              md5sum "$TMP/phase-tracker.json" "$TMP/otel-spans.jsonl" 2>/dev/null | awk '{print $1}' | tr -d '\n')
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 2. Run aggregator → pipeline-run.jsonl produced"
-node "$SCRIPT" --task-dir "$TMP" 2>/dev/null
-if [ -f "$TMP/pipeline-run.jsonl" ]; then
-  pass "pipeline-run.jsonl created"
-else
-  fail "pipeline-run.jsonl not created"
-fi
-
-LINE_COUNT=$(wc -l < "$TMP/pipeline-run.jsonl" | tr -d ' ')
-if [ "$LINE_COUNT" -ge 8 ]; then
-  pass "output has $LINE_COUNT events (≥ 8 expected)"
-else
-  fail "too few events: $LINE_COUNT"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 3. Read-only contract  -  source files untouched"
-HASH_AFTER=$(md5 -q "$TMP/phase-tracker.json" "$TMP/otel-spans.jsonl" 2>/dev/null || \
-             md5sum "$TMP/phase-tracker.json" "$TMP/otel-spans.jsonl" 2>/dev/null | awk '{print $1}' | tr -d '\n')
-if [ "$HASH_BEFORE" = "$HASH_AFTER" ]; then
-  pass "source files unchanged after aggregation"
-else
-  fail "source files were modified (hash drift)"
-fi
-
-# Static check: aggregator script does not write to phase-tracker or otel-spans
-if grep -qE "writeFileSync.*phase-tracker|writeFileSync.*otel-spans" "$SCRIPT"; then
-  fail "run-aggregator.mjs writes to source telemetry files (read-only contract violated)"
-else
-  pass "no writes to source telemetry files in aggregator"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 4. Time-sorted events"
-# Extract ts values and verify sorted (lexicographic = time-sorted for ISO 8601)
-TSS=$(grep -oE '"ts":"[^"]*"' "$TMP/pipeline-run.jsonl" | sed 's/"ts":"//;s/"//')
-if [ "$TSS" = "$(echo "$TSS" | sort)" ]; then
-  pass "events time-sorted (ISO ts ascending)"
-else
-  fail "events out of time order"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 5. Cost computation correct"
-# sonnet 1000+200 = 0.001*3 + 0.0002*15 = 0.003 + 0.003 = 0.006
-# opus 50000+3000 = 0.05*5 + 0.003*25 = 0.25 + 0.075 = 0.325
-# sonnet 10000+2000 = 0.01*3 + 0.002*15 = 0.03 + 0.03 = 0.060
-# total = 0.006 + 0.325 + 0.060 = 0.391
-SUMMARY=$(tail -1 "$TMP/pipeline-run.jsonl")
-TOTAL=$(echo "$SUMMARY" | grep -oE '"total_cost_usd":[^,}]*' | sed 's/"total_cost_usd"://')
-if [ "$TOTAL" = "0.391" ]; then
-  pass "total_cost_usd = $TOTAL (matches expected)"
-else
-  fail "total_cost_usd = $TOTAL (expected 0.391)"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 6. Summary as last event"
-LAST_KIND=$(tail -1 "$TMP/pipeline-run.jsonl" | grep -oE '"kind":"[^"]*"' | head -1)
-if [ "$LAST_KIND" = '"kind":"summary"' ]; then
-  pass "summary is the final event"
-else
-  fail "last event is not summary: $LAST_KIND"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 7. Phase status counted correctly"
-COMPLETED=$(echo "$SUMMARY" | grep -oE '"phases_completed":[0-9]+' | grep -oE '[0-9]+')
-FAILED=$(echo "$SUMMARY" | grep -oE '"phases_failed":[0-9]+' | grep -oE '[0-9]+')
-if [ "$COMPLETED" = "2" ] && [ "$FAILED" = "1" ]; then
-  pass "phases_completed=$COMPLETED phases_failed=$FAILED (matches fixture)"
-else
-  fail "phase counts wrong: completed=$COMPLETED failed=$FAILED (expected 2/1)"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 8. Span passthrough with task_id stamping"
-SPAN_COUNT=$(grep -c '"kind":"span"' "$TMP/pipeline-run.jsonl" || true)
-if [ "$SPAN_COUNT" = "2" ]; then
-  pass "all 2 spans passed through"
-else
-  fail "span passthrough count wrong: $SPAN_COUNT (expected 2)"
-fi
-if grep -q '"kind":"span".*"task_id":"PROJ-100"' "$TMP/pipeline-run.jsonl"; then
-  pass "spans stamped with task_id"
-else
-  fail "spans missing task_id stamp"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 9. --summary-only mode"
-SUMMARY_OUT=$(node "$SCRIPT" --task-dir "$TMP" --stdout --summary-only 2>/dev/null)
-SUMMARY_LINES=$(echo "$SUMMARY_OUT" | grep -c "^{" || true)
-if [ "$SUMMARY_LINES" = "1" ]; then
-  pass "--summary-only emits exactly 1 line"
-else
-  fail "--summary-only emitted $SUMMARY_LINES lines"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 10. Unknown model graceful degradation"
-cat > "$TMP/phase-tracker.json" <<'EOF'
-{
-  "task_id": "PROJ-200",
-  "started_at": "2026-04-27T11:00:00Z",
-  "phases": [
-    {"id":"0","name":"Init","status":"completed","started_at":"2026-04-27T11:00:00Z","completed_at":"2026-04-27T11:00:10Z","tokens_in":500,"tokens_out":50,"model":"made-up-model","subs":[]}
-  ]
-}
-EOF
-rm -f "$TMP/otel-spans.jsonl"
-node "$SCRIPT" --task-dir "$TMP" --stdout 2>/dev/null > "$TMP/run.jsonl"
-if grep -q '"cost_unknown_model":true' "$TMP/run.jsonl"; then
-  pass "unknown model marked with cost_unknown_model flag"
-else
-  fail "unknown model not flagged"
-fi
-SUMMARY=$(tail -1 "$TMP/run.jsonl")
-if echo "$SUMMARY" | grep -q '"unknown_model_phases":1'; then
-  pass "summary records unknown_model_phases count"
-else
-  fail "summary missing unknown_model_phases counter"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 11. Missing task-dir errors clearly"
-set +e
-node "$SCRIPT" --task-dir /nonexistent/path 2>"$TMP/err.txt"
-EC=$?
-set -e
-if [ "$EC" -ne 0 ] && grep -qE "not found" "$TMP/err.txt"; then
-  pass "missing task-dir exits $EC with 'not found' error"
-else
-  fail "missing task-dir error handling weak (exit=$EC)"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo "→ 12. Idempotent  -  second run produces identical content"
-# Restore the first fixture
-cat > "$TMP/phase-tracker.json" <<'EOF'
-{
-  "task_id": "PROJ-100",
-  "started_at": "2026-04-27T10:00:00Z",
-  "phases": [
-    {"id":"0","name":"Init","status":"completed","started_at":"2026-04-27T10:00:00Z","completed_at":"2026-04-27T10:00:30Z","tokens_in":1000,"tokens_out":200,"model":"sonnet","subs":[]}
-  ]
-}
-EOF
-node "$SCRIPT" --task-dir "$TMP" 2>/dev/null
-H1=$(md5 -q "$TMP/pipeline-run.jsonl" 2>/dev/null || md5sum "$TMP/pipeline-run.jsonl" | awk '{print $1}')
-node "$SCRIPT" --task-dir "$TMP" 2>/dev/null
-H2=$(md5 -q "$TMP/pipeline-run.jsonl" 2>/dev/null || md5sum "$TMP/pipeline-run.jsonl" | awk '{print $1}')
-if [ "$H1" = "$H2" ]; then
-  pass "two consecutive runs produce identical output"
-else
-  fail "output drift between runs ($H1 vs $H2)"
-fi
-
-# ──────────────────────────────────────────────────────────────────────────
-echo ""
-echo "══ run-aggregator smoke: $PASS passed, $FAIL failed ══"
-[ "$FAIL" -eq 0 ]

package/pipeline/scripts/smoke-run-metrics.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-# smoke-run-metrics.sh  -  run-metrics.mjs extracts the right outcome numbers
-# from an agent-state.json. This is the evidence-collection harness: run it
-# after each real task and the corpus becomes the real-world validation.
-#
-# Exit 0 = all pass, 1 = any failure.
-
-set -uo pipefail
-
-ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
-M="$ROOT/pipeline/scripts/run-metrics.mjs"
-FIX="$ROOT/pipeline/eval/run-metrics-fixture.json"
-PASS=0; FAIL=0
-pass() { PASS=$((PASS+1)); echo "  ✓ $1"; }
-fail() { FAIL=$((FAIL+1)); echo "  ✗ $1"; }
-command -v node >/dev/null 2>&1 || { echo "error: node required" >&2; exit 127; }
-command -v jq >/dev/null 2>&1 || { echo "error: jq required" >&2; exit 127; }
-[ -f "$M" ] || { echo "FAIL: run-metrics.mjs missing" >&2; exit 1; }
-[ -f "$FIX" ] || { echo "FAIL: fixture missing" >&2; exit 1; }
-
-out=$(node "$M" --state "$FIX" 2>/dev/null)
-
-check() { # jq-expr desc
-  if jq -e "$1" >/dev/null 2>&1 <<< "$out"; then pass "$2"; else fail "$2 (got: $(jq -c "$3" <<< "$out" 2>/dev/null))"; fi
-}
-
-echo "→ 1. core outcome fields"
-check '.status == "complete"' "status parsed" '.status'
-check '.reviewIterations == 2' "review iterations counted (rework loop)" '.reviewIterations'
-check '.firstPassClean == false' "first-pass-clean detects iteration-1 blocker" '.firstPassClean'
-check '.buildPassed == true' "build outcome aggregated across repos" '.buildPassed'
-
-echo ""
-echo "→ 2. review signal-to-noise"
-check '.reviewSignal.rawFindings == 3' "raw reviewer findings summed" '.reviewSignal.rawFindings'
-check '.reviewSignal.acceptedAll == 1' "accepted summed across iterations" '.reviewSignal.acceptedAll'
-check '.reviewSignal.acceptedRatio == 0.33' "accepted ratio (signal-to-noise)" '.reviewSignal.acceptedRatio'
-
-echo ""
-echo "→ 3. consensus surfaced"
-check '.consensusVerdict == "unanimous-pass"' "consensus verdict extracted" '.consensusVerdict'
-
-echo ""
-echo "→ 4. stdin + missing-field resilience"
-echo '{"taskId":"X","status":"failed"}' | node "$M" --state - >/dev/null 2>&1 && pass "minimal state via stdin does not crash" || fail "stdin/minimal state crashed"
-node "$M" 2>/dev/null; [ "$?" = "2" ] && pass "no --state -> usage exit 2" || fail "missing --state wrong exit"
-
-echo ""
-echo "══ run-metrics smoke: $PASS passed, $FAIL failed ══"
-[ "$FAIL" -eq 0 ]

package/pipeline/scripts/smoke-search.sh

@@ -1,187 +0,0 @@
-#!/usr/bin/env bash
-# smoke-search.sh  -  WS v5.1.0 log-search self-verification.
-#
-# Verifies (on synthetic fixture tree, never touches real ~/.claude/logs/):
-#   1. Script executable, --help shows banner.
-#   2. Plain query returns matching tasks.
-#   3. --project filter narrows results.
-#   4. --since 7d drops old files.
-#   5. --phase N limits search to phase section.
-#   6. --limit caps results.
-#   7. JSON output is valid JSON with matched_tasks count.
-#   8. TSV output has header + one row per task.
-#   9. No-match returns exit 1 + friendly message.
-#  10. Ranking: recent matches score higher than old ones.
-
-set -uo pipefail
-
-HERE="$(cd "$(dirname "$0")" && pwd)"
-ROOT="$(cd "$HERE/../.." && pwd)"
-SEARCH="$ROOT/pipeline/scripts/search-logs.sh"
-
-PASS=0
-FAIL=0
-pass() { PASS=$((PASS+1)); echo "  ✓ $1"; }
-fail() { FAIL=$((FAIL+1)); echo "  ✗ $1"; }
-
-need() { command -v "$1" >/dev/null 2>&1 || { echo "error: $1 required" >&2; exit 127; }; }
-need jq
-
-FIX=$(mktemp -d)
-trap 'rm -rf "$FIX"' EXIT
-
-# --- Fixture tree --------------------------------------------------------
-
-# Shape: $FIX/<project>/<task_id>/agent-log.md
-
-mkdir -p "$FIX/app/PROJ-1234" "$FIX/app/PROJ-1337" "$FIX/api/API-42" "$FIX/app/PROJ-OLD"
-
-cat > "$FIX/app/PROJ-1234/agent-log.md" <<'EOF'
-# PROJ-1234  -  Add flight price filter
-
-**Branch**: feature/PROJ-1234-flight-filter | **Project**: app
-
-## Phase 1: Analysis
-Explored the flight booking module.
-
-## Phase 3: Dev
-Wrote FlightFilterView. Null check added for optional price range.
-
-## Phase 4: Review
-0 blocking, 2 important, 5 suggestion.
-EOF
-
-cat > "$FIX/app/PROJ-1337/agent-log.md" <<'EOF'
-# PROJ-1337  -  Guard null in BookingViewModel
-
-**Branch**: bugfix/PROJ-1337-null-deref | **Project**: app
-
-## Phase 3: Dev
-Null-deref fixed at BookingViewModel.swift:142 via guard let.
-Added regression test.
-
-## Phase 4: Review
-Reviewer flagged the null check as important. Blocking.
-EOF
-
-cat > "$FIX/api/API-42/agent-log.md" <<'EOF'
-# API-42  -  Rate limit header mismatch
-
-**Branch**: bugfix/API-42-rate-limit | **Project**: api
-
-## Phase 3: Dev
-Fixed X-RateLimit-Remaining header emission.
-EOF
-
-cat > "$FIX/app/PROJ-OLD/agent-log.md" <<'EOF'
-# PROJ-OLD  -  Old task
-
-**Branch**: feature/PROJ-OLD-archive | **Project**: app
-
-## Phase 3: Dev
-Contains the word null but from long ago.
-EOF
-# make PROJ-OLD old: 60 days ago
-OLD_EPOCH=$(date -v-60d +%s 2>/dev/null || date -d "-60 days" +%s)
-touch -t "$(date -r "$OLD_EPOCH" +%Y%m%d%H%M 2>/dev/null || date -d "@$OLD_EPOCH" +%Y%m%d%H%M)" "$FIX/app/PROJ-OLD/agent-log.md"
-
-# --- 1. Executable + help -----------------------------------------------
-
-echo "→ 1. search-logs.sh executable + help"
-[ -x "$SEARCH" ] && pass "search-logs.sh is executable" || fail "search-logs.sh not executable"
-bash "$SEARCH" --help 2>&1 | head -1 | grep -q "search-logs" && pass "--help banner present" || fail "--help banner missing"
-
-# --- 2. Plain query -----------------------------------------------------
-
-echo ""
-echo "→ 2. plain query returns matching tasks"
-OUT=$(bash "$SEARCH" --root "$FIX" "null" 2>&1)
-echo "$OUT" | grep -q "PROJ-1337" && pass "PROJ-1337 (high match count) returned" || fail "PROJ-1337 missing"
-echo "$OUT" | grep -q "PROJ-1234" && pass "PROJ-1234 (single match) returned" || fail "PROJ-1234 missing"
-echo "$OUT" | grep -q "API-42" && fail "API-42 wrongly returned (no 'null')" || pass "API-42 correctly excluded (no 'null')"
-
-# --- 3. --project filter ------------------------------------------------
-
-echo ""
-echo "→ 3. --project filter narrows results"
-OUT=$(bash "$SEARCH" --root "$FIX" --project api "null" 2>&1)
-echo "$OUT" | grep -q "no matches" && pass "api project has no 'null' → correctly empty" || fail "api project filter leaked"
-OUT=$(bash "$SEARCH" --root "$FIX" --project app "null" 2>&1)
-echo "$OUT" | grep -q "PROJ-1234" && pass "app project filter includes PROJ-1234" || fail "app project filter broken"
-
-# --- 4. --since filter --------------------------------------------------
-
-echo ""
-echo "→ 4. --since 7d drops old tasks"
-OUT=$(bash "$SEARCH" --root "$FIX" --since 7d "null" 2>&1)
-echo "$OUT" | grep -q "PROJ-OLD" && fail "PROJ-OLD (60d) wrongly included with --since 7d" || pass "PROJ-OLD correctly dropped with --since 7d"
-echo "$OUT" | grep -q "PROJ-1337" && pass "recent PROJ-1337 still included with --since 7d" || fail "recent task lost with --since 7d"
-
-# --- 5. --phase filter --------------------------------------------------
-
-echo ""
-echo "→ 5. --phase 4 limits to phase section"
-OUT=$(bash "$SEARCH" --root "$FIX" --phase 4 "blocking" 2>&1)
-echo "$OUT" | grep -q "PROJ-1337" && pass "--phase 4 matches Phase 4 content" || fail "--phase 4 broke"
-
-# --- 6. --limit -----------------------------------------------------
-
-echo ""
-echo "→ 6. --limit caps results"
-OUT=$(bash "$SEARCH" --root "$FIX" --limit 1 "null" 2>&1)
-count=$(echo "$OUT" | grep -c "^PROJ-\|^API-\|^FIX-")
-# just verify command runs and limits in JSON (safer check)
-OUT=$(bash "$SEARCH" --root "$FIX" --limit 1 "null" --json 2>&1)
-count=$(echo "$OUT" | jq -r '.tasks | length')
-[ "$count" -eq 1 ] && pass "--limit 1 returns exactly 1 task" || fail "--limit 1 returned $count tasks"
-
-# --- 7. JSON output -----------------------------------------------------
-
-echo ""
-echo "→ 7. JSON output valid"
-OUT=$(bash "$SEARCH" --root "$FIX" "null" --json 2>&1)
-if echo "$OUT" | jq . >/dev/null 2>&1; then
-  pass "JSON is valid"
-else
-  fail "JSON invalid"
-fi
-MATCHED=$(echo "$OUT" | jq -r '.matched_tasks')
-[ "$MATCHED" -ge 2 ] && pass "JSON matched_tasks ≥ 2 (got $MATCHED)" || fail "JSON matched_tasks = $MATCHED"
-
-# --- 8. TSV output ------------------------------------------------------
-
-echo ""
-echo "→ 8. TSV output has header + rows"
-OUT=$(bash "$SEARCH" --root "$FIX" "null" --tsv 2>&1)
-echo "$OUT" | head -1 | grep -q "^score" && pass "TSV has header row" || fail "TSV missing header"
-BODY_LINES=$(echo "$OUT" | tail -n +2 | wc -l | tr -d ' ')
-[ "$BODY_LINES" -ge 2 ] && pass "TSV has ≥ 2 body rows" || fail "TSV body rows = $BODY_LINES"
-
-# --- 9. No-match path ---------------------------------------------------
-
-echo ""
-echo "→ 9. no-match → exit 1 + friendly message"
-set +e
-bash "$SEARCH" --root "$FIX" "nonexistent-string-xyz" > /tmp/search-out 2>&1
-RC=$?
-set -e
-[ "$RC" -eq 1 ] && pass "no-match exit code = 1" || fail "expected 1, got $RC"
-grep -q "no matches" /tmp/search-out && pass "no-match message shown" || fail "no-match message missing"
-
-# --- 10. Ranking: recent > old ------------------------------------------
-
-echo ""
-echo "→ 10. ranking prefers recent matches (score = matches × recency)"
-# PROJ-1234: 1 match, recent → score=100 (100% weight)
-# PROJ-1337: multiple matches, recent → score>100
-# PROJ-OLD: 1 match, 60d old → score=20 (20% weight)
-OUT=$(bash "$SEARCH" --root "$FIX" "null" --json 2>&1)
-FIRST_ID=$(echo "$OUT" | jq -r '.tasks[0].task_id')
-FIRST_SCORE=$(echo "$OUT" | jq -r '.tasks[0].score')
-[ "$FIRST_ID" = "PROJ-1337" ] && pass "highest score is PROJ-1337 ($FIRST_SCORE)" || fail "expected PROJ-1337 on top, got $FIRST_ID"
-
-# --- Summary ------------------------------------------------------------
-
-echo ""
-echo "══ search smoke: $PASS passed, $FAIL failed ══"
-[ "$FAIL" -eq 0 ]