@mmerterden/multi-agent-pipeline 12.7.0 → 12.8.0

package/pipeline/scripts/smoke-agent-log-cost.sh

@@ -1,262 +0,0 @@
-#!/usr/bin/env bash
-# smoke-agent-log-cost.sh  -  v8.3.0
-#
-# Verifies the agent-log Cost Breakdown pipeline:
-#   1. render-agent-log-cost.sh exits 2 when no tracker data available
-#   2. Renders 4-column markdown (Phase | Model | Tokens in | Tokens out | Est. USD)
-#      from a hand-built object-shaped phase-tracker.json
-#   3. Renders identical block from the array-shaped phase-tracker.json that
-#      phase-tracker.sh actually writes
-#   4. Unknown model degrades to " - " with footnote, never blocks
-#   5. log-metric.sh forwards tokens + model to tracker when LOG_METRIC_FORWARD_TO_TRACKER=1
-#   6. log-format.md ref doc declares the Tokens column + Cost Breakdown section
-#
-# Exit 0 = all pass, 1 = any failure.
-
-set -euo pipefail
-
-ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
-RENDERER="$ROOT/pipeline/scripts/render-agent-log-cost.sh"
-TRACKER="$ROOT/pipeline/scripts/phase-tracker.sh"
-LOG_METRIC="$ROOT/pipeline/scripts/log-metric.sh"
-TABLE="$ROOT/pipeline/scripts/cost-table.json"
-LOG_FORMAT_DOC="$ROOT/pipeline/multi-agent-refs/phases/log-format.md"
-
-pass=0
-fail=0
-failures=()
-record_pass() { pass=$((pass + 1)); printf '  \033[0;32mPASS\033[0m %s\n' "$1"; }
-record_fail() { fail=$((fail + 1)); failures+=("$1"); printf '  \033[0;31mFAIL\033[0m %s\n' "$1"; }
-
-printf '→ smoke-agent-log-cost (v8.3.0): agent-log Cost Breakdown contract\n'
-
-[ -x "$RENDERER" ] || { record_fail "renderer not executable: $RENDERER"; exit 1; }
-
-# --- 1: missing-data path returns exit 2 ---
-tmp=$(mktemp -d)
-pushd "$tmp" >/dev/null
-set +e
-bash "$RENDERER" GHOST-1 >/dev/null 2>&1
-rc=$?
-set -e
-if [ "$rc" -eq 2 ]; then
-  record_pass "renderer exits 2 when no tracker data"
-else
-  record_fail "renderer should exit 2 with no data (got $rc)"
-fi
-popd >/dev/null
-rm -rf "$tmp"
-
-# --- 2: object-shaped fixture renders 4-column markdown ---
-tmp=$(mktemp -d)
-pushd "$tmp" >/dev/null
-mkdir -p .worktrees/DEMO-1
-cat > .worktrees/DEMO-1/phase-tracker.json <<'JSON'
-{"taskId":"DEMO-1","phases":{
-  "1":{"name":"1. Analysis","model":"opus","tokens_in":12480,"tokens_out":3210},
-  "3":{"name":"3. Dev","model":"sonnet","tokens_in":42310,"tokens_out":9880},
-  "4":{"name":"4. Review","model":"opus","tokens_in":8200,"tokens_out":1500}
-}}
-JSON
-out=$(bash "$RENDERER" DEMO-1 2>/dev/null)
-if grep -qE '^\| Phase \| Model \| Tokens in \| Tokens out \| Est\. USD \|' <<< "$out"; then
-  record_pass "object-shape: header row rendered"
-else
-  record_fail "object-shape: header row missing"
-fi
-if grep -q '## Cost Breakdown' <<< "$out"; then
-  record_pass "object-shape: Cost Breakdown section heading present"
-else
-  record_fail "object-shape: section heading missing"
-fi
-if grep -qE '\| 1\. Analysis \| opus \| 12,480 \| 3,210 \| \$' <<< "$out"; then
-  record_pass "object-shape: Phase 1 row formatted with comma-grouped tokens + USD"
-else
-  record_fail "object-shape: Phase 1 row formatting wrong"
-  printf '%s\n' "$out" | head -10 >&2
-fi
-if grep -qE '\| \*\*Total\*\* .* \| \*\*\$' <<< "$out"; then
-  record_pass "object-shape: Total row rendered"
-else
-  record_fail "object-shape: Total row missing"
-fi
-popd >/dev/null
-rm -rf "$tmp"
-
-# --- 3: array-shaped fixture (the shape phase-tracker.sh actually writes) ---
-tmp=$(mktemp -d)
-pushd "$tmp" >/dev/null
-mkdir -p .worktrees/PROJ-2
-cat > .worktrees/PROJ-2/phase-tracker.json <<'JSON'
-{"task_id":"PROJ-2","started_at":"2026-05-05T10:00:00Z","phases":[
-  {"id":"1","name":"Analysis","status":"completed","model":"opus","tokens_in":10000,"tokens_out":2000,"subs":[]},
-  {"id":"3","name":"Dev","status":"completed","model":"sonnet","tokens_in":50000,"tokens_out":8000,"subs":[]}
-]}
-JSON
-out=$(bash "$RENDERER" PROJ-2 2>/dev/null)
-if grep -qE '\| Analysis \| opus \| 10,000 \| 2,000 \| \$' <<< "$out"; then
-  record_pass "array-shape: Phase 1 row rendered correctly"
-else
-  record_fail "array-shape: Phase 1 row wrong"
-  printf '%s\n' "$out" | head -10 >&2
-fi
-if grep -qE '\| Dev \| sonnet \| 50,000 \| 8,000 \| \$' <<< "$out"; then
-  record_pass "array-shape: Phase 3 row rendered correctly"
-else
-  record_fail "array-shape: Phase 3 row wrong"
-fi
-popd >/dev/null
-rm -rf "$tmp"
-
-# --- 4: unknown model degrades gracefully ---
-tmp=$(mktemp -d)
-pushd "$tmp" >/dev/null
-mkdir -p .worktrees/UNK-1
-cat > .worktrees/UNK-1/phase-tracker.json <<'JSON'
-{"task_id":"UNK-1","phases":[
-  {"id":"3","name":"Dev","model":"made-up-9000","tokens_in":1000,"tokens_out":200,"subs":[]}
-]}
-JSON
-out=$(bash "$RENDERER" UNK-1 2>/dev/null)
-if grep -qF '| made-up-9000 |' <<< "$out" && grep -qF '|  -  |' <<< "$out"; then
-  record_pass "unknown model row renders with USD as  - "
-else
-  record_fail "unknown model handling broken"
-fi
-if grep -q 'cost-table.json' <<< "$out"; then
-  record_pass "unknown model footnote points to cost-table.json"
-else
-  record_fail "unknown model footnote missing"
-fi
-popd >/dev/null
-rm -rf "$tmp"
-
-# --- 5: log-metric forwarder writes to phase-tracker ---
-tmp=$(mktemp -d)
-export METRICS_FILE="$tmp/metrics.jsonl"
-export TRACKER_FILE="$tmp/tracker.json"
-# Bootstrap tracker with one phase
-bash "$TRACKER" init "FWD-1" >/dev/null 2>&1 || true
-# init writes to $HOME/.claude/logs/multi-agent/FWD-1/tracker-state.json  -  override via TRACKER_FILE
-cat > "$TRACKER_FILE" <<'JSON'
-{"task_id":"FWD-1","started_at":"2026-05-05T00:00:00Z","phases":[
-  {"id":"4","name":"Review","status":"in_progress","subs":[]}
-]}
-JSON
-LOG_METRIC_FORWARD_TO_TRACKER=1 TRACKER_QUIET=1 \
-  bash "$LOG_METRIC" FWD-1 4 review.reviewer_call \
-  model=opus tokens_in=1234 tokens_out=567 tokens_cached=400 duration_ms=1000 >/dev/null 2>&1 || true
-if jq -e '.phases[0].tokens_in == 1234 and .phases[0].tokens_out == 567 and .phases[0].model == "opus"' "$TRACKER_FILE" >/dev/null 2>&1; then
-  record_pass "log-metric forwards tokens + model to tracker"
-else
-  record_fail "log-metric forwarder did not update tracker"
-  cat "$TRACKER_FILE" >&2
-fi
-# End-to-end: tokens_cached must flow log-metric -> phase-tracker -> tracker state
-if jq -e '.phases[0].tokens_cached == 400' "$TRACKER_FILE" >/dev/null 2>&1; then
-  record_pass "log-metric forwards tokens_cached end-to-end"
-else
-  record_fail "tokens_cached did not reach the tracker"
-  cat "$TRACKER_FILE" >&2
-fi
-# A malformed tokens_cached must NOT poison the call (in/out must still land).
-cat > "$TRACKER_FILE" <<'JSON'
-{"task_id":"FWD-2","phases":[{"id":"4","name":"Review","status":"in_progress","subs":[]}]}
-JSON
-LOG_METRIC_FORWARD_TO_TRACKER=1 TRACKER_QUIET=1 \
-  bash "$LOG_METRIC" FWD-2 4 review.reviewer_call \
-  model=opus tokens_in=999 tokens_out=111 tokens_cached=abc duration_ms=1 >/dev/null 2>&1 || true
-if jq -e '.phases[0].tokens_in == 999 and .phases[0].tokens_out == 111 and (.phases[0].tokens_cached // 0) == 0' "$TRACKER_FILE" >/dev/null 2>&1; then
-  record_pass "malformed tokens_cached sanitized to 0; in/out preserved"
-else
-  record_fail "malformed tokens_cached poisoned the tokens call"
-  cat "$TRACKER_FILE" >&2
-fi
-unset METRICS_FILE TRACKER_FILE
-rm -rf "$tmp"
-
-# --- 5b: cached tokens are priced at the cache-read rate + surfaced ---
-tmp=$(mktemp -d)
-pushd "$tmp" >/dev/null
-mkdir -p .worktrees/CACHE-1
-cat > .worktrees/CACHE-1/phase-tracker.json <<'JSON'
-{"task_id":"CACHE-1","phases":[
-  {"id":"1","name":"Analysis","model":"opus","tokens_in":20000,"tokens_out":3000,"tokens_cached":15000,"subs":[]},
-  {"id":"3","name":"Dev","model":"sonnet","tokens_in":60000,"tokens_out":10000,"subs":[]}
-]}
-JSON
-out=$(bash "$RENDERER" CACHE-1 2>/dev/null)
-# Cache-exclusive: tokens_in (20000) and tokens_cached (15000) are disjoint.
-# 20000 fresh @5 + 15000 cached @0.50 + 3000 out @25 = 0.1825 -> floor-to-cents $0.18.
-# Without the cache discount (cached billed @5) it would be 0.25 -> $0.25, so the
-# discounted rate is what drives the row down to $0.18.
-if grep -qE '\| Analysis \| opus \| 20,000 \| 3,000 \| \$0\.18 \|' <<< "$out"; then
-  record_pass "cached tokens priced at cache-read rate (opus row = \$0.18)"
-else
-  record_fail "cached-token pricing wrong"
-  printf '%s\n' "$out" | head -12 >&2
-fi
-if grep -q 'Cache reads: 15,000 tokens' <<< "$out"; then
-  record_pass "cache-read footer surfaced when tokens_cached present"
-else
-  record_fail "cache-read footer missing"
-fi
-if grep -qE 'Top cost driver:' <<< "$out"; then
-  record_pass "top cost driver line rendered"
-else
-  record_fail "top cost driver line missing"
-fi
-popd >/dev/null
-rm -rf "$tmp"
-
-# --- 5c: no cache footer when no cached tokens (object-shape regression) ---
-tmp=$(mktemp -d)
-pushd "$tmp" >/dev/null
-mkdir -p .worktrees/NOCACHE-1
-cat > .worktrees/NOCACHE-1/phase-tracker.json <<'JSON'
-{"task_id":"NOCACHE-1","phases":[
-  {"id":"3","name":"Dev","model":"sonnet","tokens_in":50000,"tokens_out":8000,"subs":[]}
-]}
-JSON
-out=$(bash "$RENDERER" NOCACHE-1 2>/dev/null)
-if grep -q 'Cache reads:' <<< "$out"; then
-  record_fail "cache-read footer should be absent when no cached tokens"
-else
-  record_pass "no cache-read footer when tokens_cached absent"
-fi
-# Sonnet 50000 @3 + 8000 @15 = 0.27 -> $0.27 (unchanged from pre-cache behavior)
-if grep -qE '\| Dev \| sonnet \| 50,000 \| 8,000 \| \$0\.27 \|' <<< "$out"; then
-  record_pass "no-cache USD unchanged (backward-compatible)"
-else
-  record_fail "no-cache USD changed unexpectedly"
-  printf '%s\n' "$out" | head -12 >&2
-fi
-popd >/dev/null
-rm -rf "$tmp"
-
-# --- 6: log-format.md declares Tokens + Cost Breakdown ---
-if grep -q 'Tokens (in/out)' "$LOG_FORMAT_DOC"; then
-  record_pass "log-format.md declares Tokens (in/out) column"
-else
-  record_fail "log-format.md missing Tokens column declaration"
-fi
-if grep -q '## Cost Breakdown' "$LOG_FORMAT_DOC"; then
-  record_pass "log-format.md declares Cost Breakdown section"
-else
-  record_fail "log-format.md missing Cost Breakdown section"
-fi
-if grep -q 'render-agent-log-cost.sh' "$LOG_FORMAT_DOC"; then
-  record_pass "log-format.md references render-agent-log-cost.sh"
-else
-  record_fail "log-format.md does not reference renderer"
-fi
-
-# --- Summary ---
-total=$((pass + fail))
-printf '\n→ smoke-agent-log-cost: %d/%d passed\n' "$pass" "$total"
-if [ "$fail" -ne 0 ]; then
-  printf '\nFailures:\n'
-  for f in "${failures[@]}"; do printf '  - %s\n' "$f"; done
-  exit 1
-fi
-exit 0

package/pipeline/scripts/smoke-agent-model-routing.sh

@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-# smoke-agent-model-routing.sh  -  v6.1.D
-#
-# Asserts per-persona model routing contract:
-#   1. Every agent file in pipeline/agents/ has a `preferredModel` frontmatter field.
-#   2. `preferredModel` is one of: fable | opus | sonnet | haiku.
-#   3. `model:` (Claude-Code-native) matches `preferredModel` (single source of truth).
-#   4. Orchestrator docs mention `CLAUDE_CODE_SUBAGENT_MODEL` and `preferredModel`.
-#   5. Phase 4 review doc references the per-dispatch override mechanism.
-#
-# Exit 0 = all assertions pass, 1 = any failure.
-
-set -euo pipefail
-
-ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
-AGENTS_DIR="$ROOT/pipeline/agents"
-ORCH_SKILL="$ROOT/pipeline/skills/shared/core/multi-agent/SKILL.md"
-ORCH_CMD="$ROOT/pipeline/commands/multi-agent/SKILL.md"
-PHASE4="$ROOT/pipeline/multi-agent-refs/phases/phase-4-review.md"
-
-pass=0
-fail=0
-failures=()
-
-record_pass() { pass=$((pass + 1)); printf '  \033[0;32mPASS\033[0m %s\n' "$1"; }
-record_fail() { fail=$((fail + 1)); failures+=("$1"); printf '  \033[0;31mFAIL\033[0m %s\n' "$1"; }
-
-printf '→ smoke-agent-model-routing (v6.1.D): per-persona model routing contract\n'
-
-# --- 1/2/3: per-agent frontmatter ---
-expected=(code-reviewer explorer ios-architect android-architect backend-architect security-auditor)
-for name in "${expected[@]}"; do
-  f="$AGENTS_DIR/$name.md"
-  if [ ! -f "$f" ]; then
-    record_fail "$name.md missing"
-    continue
-  fi
-  pref=$(awk '/^preferredModel:/ {print $2; exit}' "$f" | tr -d '"'"'")
-  mdl=$(awk '/^model:/ {print $2; exit}' "$f" | tr -d '"'"'")
-  case "$pref" in
-    fable|opus|sonnet|haiku)
-      record_pass "$name has preferredModel=$pref"
-      ;;
-    *)
-      record_fail "$name preferredModel invalid or missing (got '$pref')"
-      ;;
-  esac
-  if [ -n "$pref" ] && [ "$pref" != "$mdl" ]; then
-    record_fail "$name model/preferredModel mismatch (model=$mdl preferredModel=$pref)"
-  elif [ -n "$pref" ]; then
-    record_pass "$name model == preferredModel ($pref)"
-  fi
-  if grep -q '^modelRationale:' "$f"; then
-    record_pass "$name has modelRationale"
-  else
-    record_fail "$name missing modelRationale"
-  fi
-done
-
-# --- 4: orchestrator SKILL.md references ---
-if grep -q 'CLAUDE_CODE_SUBAGENT_MODEL' "$ORCH_SKILL" && grep -q 'preferredModel' "$ORCH_SKILL"; then
-  record_pass "orchestrator SKILL.md documents dispatch contract"
-else
-  record_fail "orchestrator SKILL.md missing dispatch contract"
-fi
-
-# --- 4b: orchestrator command file references ---
-if grep -q 'preferredModel' "$ORCH_CMD"; then
-  record_pass "commands/multi-agent/SKILL.md references preferredModel"
-else
-  record_fail "commands/multi-agent/SKILL.md missing preferredModel reference"
-fi
-
-# --- 5: Phase 4 review doc references override ---
-if grep -q 'PHASE_MODEL_OVERRIDE\|preferredModel' "$PHASE4"; then
-  record_pass "phase-4-review.md documents per-reviewer override"
-else
-  record_fail "phase-4-review.md missing per-reviewer override doc"
-fi
-
-printf '\n══ agent-model-routing smoke: %d passed, %d failed ══\n' "$pass" "$fail"
-if [ "$fail" -gt 0 ]; then
-  printf '\nFailures:\n'
-  for msg in "${failures[@]}"; do printf '  - %s\n' "$msg"; done
-  exit 1
-fi
-exit 0

package/pipeline/scripts/smoke-ask-choice.sh

@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# smoke-ask-choice.sh  -  the portable picker fallback used on platforms without
-# a native AskUserQuestion. Exercises default-by-index, default-by-label,
-# non-interactive first-option, label-vs-index resolution, and usage errors.
-
-set -uo pipefail
-
-REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
-ASK="$REPO_ROOT/pipeline/lib/ask-choice.sh"
-
-PASS=0
-FAIL=0
-pass() { PASS=$((PASS + 1)); echo "  ✓ $1"; }
-fail() { FAIL=$((FAIL + 1)); echo "  ✗ $1"; }
-
-echo "→ 1. usage error with <3 args (exit 64)"
-bash "$ASK" "only a question" >/dev/null 2>&1
-[ "$?" -eq 64 ] && pass "too-few-args exits 64" || fail "too-few-args should exit 64"
-
-echo "→ 2. ASK_CHOICE_DEFAULT by 1-based index"
-out=$(ASK_CHOICE_DEFAULT=2 bash "$ASK" "Approve this plan?" "Approve" "Cancel" "Edit" 2>/dev/null)
-[ "$out" = "Cancel" ] && pass "index default 2 -> 'Cancel'" || fail "index default gave '$out' (expected Cancel)"
-
-echo "→ 3. ASK_CHOICE_DEFAULT by label (case-insensitive)"
-out=$(ASK_CHOICE_DEFAULT=approve bash "$ASK" "Approve this plan?" "Approve" "Cancel" 2>/dev/null)
-[ "$out" = "Approve" ] && pass "label default 'approve' -> 'Approve'" || fail "label default gave '$out'"
-
-echo "→ 4. out-of-range / unmatched default falls through to first option (non-TTY)"
-out=$(ASK_CHOICE_DEFAULT=99 bash "$ASK" "Pick" "A" "B" </dev/null 2>/dev/null)
-[ "$out" = "A" ] && pass "bad default + no TTY -> first option 'A'" || fail "bad default gave '$out'"
-
-echo "→ 5. non-interactive (no TTY, no default) selects first option, never blocks"
-out=$(bash "$ASK" "Continue?" "Yes" "No" </dev/null 2>/dev/null)
-[ "$out" = "Yes" ] && pass "no TTY -> first option 'Yes'" || fail "no TTY gave '$out'"
-
-echo "→ 6. stdout carries ONLY the label (menu/prompt go to stderr)"
-out=$(ASK_CHOICE_DEFAULT=1 bash "$ASK" "Q" "Only" "Other" 2>/dev/null)
-[ "$out" = "Only" ] && pass "stdout is clean single label" || fail "stdout not clean: '$out'"
-
-echo ""
-echo "══ ask-choice smoke: $PASS passed, $FAIL failed ══"
-[ "$FAIL" -eq 0 ]

package/pipeline/scripts/smoke-autopilot-circuit-breaker.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# smoke-autopilot-circuit-breaker.sh  -  wiring contract for the autopilot
-# circuit-breaker feature: the doc exists, enumerates the five trip conditions,
-# and is referenced from the modes "what never skips" list.
-#
-# Exit 0 = all pass, 1 = any failure.
-
-set -uo pipefail
-
-ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
-DOC="$ROOT/pipeline/multi-agent-refs/features/autopilot-circuit-breaker.md"
-MODES="$ROOT/pipeline/multi-agent-refs/phases/modes.md"
-
-PASS=0; FAIL=0
-pass() { PASS=$((PASS+1)); echo "  ✓ $1"; }
-fail() { FAIL=$((FAIL+1)); echo "  ✗ $1"; }
-
-[ -f "$DOC" ] && pass "feature doc exists" || fail "feature doc missing"
-[ -f "$MODES" ] && pass "modes doc exists" || fail "modes doc missing"
-
-echo "→ five trip conditions documented"
-for t in "No-progress stall" "Identical repeated failure" "Rework storm" "Cost drift" "Merge/rebase conflict"; do
-  grep -Fq "$t" "$DOC" && pass "trigger: $t" || fail "trigger missing: $t"
-done
-
-echo "→ action + safety invariants"
-grep -Fq "resume" "$DOC" && pass "documents resume path" || fail "no resume path"
-grep -Eiq "never auto-resolve|does not paper over|no force-anything" "$DOC" && pass "no-self-heal invariant present" || fail "missing no-self-heal invariant"
-
-echo "→ wired into modes 'what never skips'"
-grep -Fq "autopilot-circuit-breaker.md" "$MODES" && pass "modes references the feature doc" || fail "modes does not reference the feature"
-grep -Fiq "Circuit-breaker" "$MODES" && pass "modes lists circuit-breaker under never-skip" || fail "modes missing circuit-breaker bullet"
-
-echo ""
-echo "══ autopilot-circuit-breaker smoke: $PASS passed, $FAIL failed ══"
-[ "$FAIL" -eq 0 ]