autonomous-coding-toolkit 1.0.0

package/scripts/tests/test-run-plan-routing.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../lib/run-plan-parser.sh"
+source "$SCRIPT_DIR/../lib/run-plan-routing.sh"
+
+FAILURES=0
+TESTS=0
+
+assert_eq() {
+    local desc="$1" expected="$2" actual="$3"
+    TESTS=$((TESTS + 1))
+    if [[ "$expected" != "$actual" ]]; then
+        echo "FAIL: $desc"
+        echo "  expected: $expected"
+        echo "  actual:   $actual"
+        FAILURES=$((FAILURES + 1))
+    else
+        echo "PASS: $desc"
+    fi
+}
+
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# Plan with clear parallel batches
+cat > "$WORK/parallel-plan.md" << 'PLAN'
+## Batch 1: Foundation
+
+**Files:**
+- Create: `src/lib.sh`
+
+### Task 1: Create lib
+Write lib.
+
+## Batch 2: Feature A
+
+**Files:**
+- Create: `src/feature-a.sh`
+context_refs: src/lib.sh
+
+### Task 2: Build feature A
+
+## Batch 3: Feature B
+
+**Files:**
+- Create: `src/feature-b.sh`
+context_refs: src/lib.sh
+
+### Task 3: Build feature B
+
+## Batch 4: Integration
+
+**Files:**
+- Modify: `src/feature-a.sh`
+- Modify: `src/feature-b.sh`
+context_refs: src/feature-a.sh, src/feature-b.sh
+
+### Task 4: Wire together
+PLAN
+
+# Test dependency graph building
+deps=$(build_dependency_graph "$WORK/parallel-plan.md")
+assert_eq "dep graph: B2 depends on B1" "true" "$(echo "$deps" | jq '.["2"] | contains(["1"])')"
+assert_eq "dep graph: B3 depends on B1" "true" "$(echo "$deps" | jq '.["3"] | contains(["1"])')"
+assert_eq "dep graph: B4 depends on B2 and B3" "true" "$(echo "$deps" | jq '.["4"] | (contains(["2"]) and contains(["3"]))')"
+
+# Test parallelism score
+score=$(compute_parallelism_score "$WORK/parallel-plan.md")
+TESTS=$((TESTS + 1))
+if [[ "$score" -gt 40 ]]; then
+    echo "PASS: parallelism score: $score > 40 (batches 2,3 can run parallel)"
+else
+    echo "FAIL: parallelism score: $score <= 40"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# Test mode recommendation
+mode=$(recommend_execution_mode "$score" "false" 21)
+assert_eq "recommend: team for high score" "team" "$mode"
+
+# Sequential plan (each batch depends on previous)
+cat > "$WORK/sequential-plan.md" << 'PLAN'
+## Batch 1: Setup
+
+**Files:**
+- Create: `src/main.sh`
+
+### Task 1: Setup
+
+## Batch 2: Extend
+
+**Files:**
+- Modify: `src/main.sh`
+context_refs: src/main.sh
+
+### Task 2: Extend
+
+## Batch 3: Finalize
+
+**Files:**
+- Modify: `src/main.sh`
+context_refs: src/main.sh
+
+### Task 3: Finalize
+PLAN
+
+score=$(compute_parallelism_score "$WORK/sequential-plan.md")
+TESTS=$((TESTS + 1))
+if [[ "$score" -lt 30 ]]; then
+    echo "PASS: sequential plan score: $score < 30"
+else
+    echo "FAIL: sequential plan score: $score >= 30"
+    FAILURES=$((FAILURES + 1))
+fi
+
+mode=$(recommend_execution_mode "$score" "false" 21)
+assert_eq "recommend: headless for low score" "headless" "$mode"
+
+# Test model routing
+model=$(classify_batch_model "$WORK/parallel-plan.md" 1)
+assert_eq "model: batch with Create files = sonnet" "sonnet" "$model"
+
+# Verification batch
+cat > "$WORK/verify-plan.md" << 'PLAN'
+## Batch 1: Verify everything
+
+### Task 1: Run all tests
+
+**Step 1: Run tests**
+Run: `bash scripts/tests/run-all-tests.sh`
+
+**Step 2: Check line counts**
+Run: `wc -l scripts/*.sh`
+PLAN
+
+model=$(classify_batch_model "$WORK/verify-plan.md" 1)
+assert_eq "model: batch with only Run commands = haiku" "haiku" "$model"
+
+# === jq timeout guard (bug #15) ===
+# Verify the jq call in compute_parallelism_score is wrapped with timeout
+TESTS=$((TESTS + 1))
+if grep -q 'timeout 30 jq' "$SCRIPT_DIR/../lib/run-plan-routing.sh"; then
+    echo "PASS: jq calls wrapped with timeout 30"
+else
+    echo "FAIL: jq calls should be wrapped with timeout 30"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# Verify timeout actually terminates a slow jq process
+# Use a FIFO that blocks forever to simulate corrupted/circular graph data
+TESTS=$((TESTS + 1))
+fifo="$WORK/slow-fifo"
+mkfifo "$fifo"
+# Keep FIFO open in background so jq blocks on read (not EOF)
+sleep 60 > "$fifo" &
+sleep_pid=$!
+start=$(date +%s)
+timeout 2 jq -r '.test' < "$fifo" 2>/dev/null || true
+end=$(date +%s)
+elapsed=$((end - start))
+kill "$sleep_pid" 2>/dev/null || true
+wait "$sleep_pid" 2>/dev/null || true
+if [[ "$elapsed" -le 4 ]]; then
+    echo "PASS: timeout kills hanging jq within 4s"
+else
+    echo "FAIL: timeout did not kill hanging jq (took ${elapsed}s)"
+    FAILURES=$((FAILURES + 1))
+fi
+
+echo ""
+echo "Results: $((TESTS - FAILURES))/$TESTS passed"
+if [[ $FAILURES -gt 0 ]]; then
+    echo "FAILURES: $FAILURES"
+    exit 1
+fi
+echo "ALL PASSED"

package/scripts/tests/test-run-plan-scoring.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../lib/run-plan-parser.sh"
+source "$SCRIPT_DIR/../lib/run-plan-scoring.sh"
+
+FAILURES=0
+TESTS=0
+
+assert_eq() {
+    local desc="$1" expected="$2" actual="$3"
+    TESTS=$((TESTS + 1))
+    if [[ "$expected" != "$actual" ]]; then
+        echo "FAIL: $desc"
+        echo "  expected: $expected"
+        echo "  actual:   $actual"
+        FAILURES=$((FAILURES + 1))
+    else
+        echo "PASS: $desc"
+    fi
+}
+
+# Gate failed = score 0
+score=$(score_candidate 0 50 100 0 0 0)
+assert_eq "score: gate failed = 0" "0" "$score"
+
+# Gate passed, good metrics
+score=$(score_candidate 1 50 100 2 0 0)
+TESTS=$((TESTS + 1))
+if [[ "$score" -gt 0 ]]; then
+    echo "PASS: score: gate passed = positive ($score)"
+else
+    echo "FAIL: score: gate passed should be positive ($score)"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# More tests = higher score
+score_a=$(score_candidate 1 50 100 0 0 0)
+score_b=$(score_candidate 1 80 100 0 0 0)
+TESTS=$((TESTS + 1))
+if [[ "$score_b" -gt "$score_a" ]]; then
+    echo "PASS: score: more tests = higher score ($score_b > $score_a)"
+else
+    echo "FAIL: score: more tests should be higher ($score_b <= $score_a)"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# Lesson violations = penalty
+score_clean=$(score_candidate 1 50 100 0 0 0)
+score_dirty=$(score_candidate 1 50 100 0 2 0)
+TESTS=$((TESTS + 1))
+if [[ "$score_clean" -gt "$score_dirty" ]]; then
+    echo "PASS: score: lesson violations penalized ($score_clean > $score_dirty)"
+else
+    echo "FAIL: score: lesson violations not penalized ($score_clean <= $score_dirty)"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# select_winner picks highest score
+winner=$(select_winner "500 300 700 0")
+assert_eq "select_winner: picks index of highest" "2" "$winner"
+
+# select_winner returns -1 when all zero
+winner=$(select_winner "0 0 0")
+assert_eq "select_winner: all zero = -1 (no winner)" "-1" "$winner"
+
+# === classify_batch_type ===
+
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# Test: Create files = new-file
+cat > "$WORK/plan-new.md" << 'PLAN'
+## Batch 1: Setup
+- Create: `src/lib.py`
+- Create: `src/util.py`
+- Test: `tests/test_lib.py`
+
+**Step 1:** Write files
+PLAN
+
+result=$(classify_batch_type "$WORK/plan-new.md" 1)
+assert_eq "classify: Create dominant = new-file" "new-file" "$result"
+
+# Test: Modify only = refactoring
+cat > "$WORK/plan-refactor.md" << 'PLAN'
+## Batch 1: Refactor auth
+- Modify: `src/auth.py:20-50`
+- Modify: `src/session.py:10-30`
+
+**Step 1:** Update auth
+PLAN
+
+result=$(classify_batch_type "$WORK/plan-refactor.md" 1)
+assert_eq "classify: Modify only = refactoring" "refactoring" "$result"
+
+# Test: Run commands only = test-only
+cat > "$WORK/plan-test.md" << 'PLAN'
+## Batch 1: Verify
+Run: pytest tests/ -v
+Run: bash scripts/quality-gate.sh --project-root .
+
+**Step 1:** Run tests
+PLAN
+
+result=$(classify_batch_type "$WORK/plan-test.md" 1)
+assert_eq "classify: Run only = test-only" "test-only" "$result"
+
+# Test: Integration title = integration
+cat > "$WORK/plan-integ.md" << 'PLAN'
+## Batch 1: Integration Wiring
+- Modify: `src/main.py`
+- Create: `src/glue.py`
+
+**Step 1:** Wire components
+PLAN
+
+result=$(classify_batch_type "$WORK/plan-integ.md" 1)
+assert_eq "classify: integration title = integration" "integration" "$result"
+
+# === get_prompt_variants ===
+
+# Test: no history = vanilla first
+result=$(get_prompt_variants "new-file" "/nonexistent/outcomes.json" 3)
+first_line=$(echo "$result" | head -1)
+assert_eq "variants: first is vanilla" "vanilla" "$first_line"
+
+# Test: returns exactly N lines
+count=$(echo "$result" | wc -l)
+assert_eq "variants: returns N lines" "3" "$count"
+
+# Test: with learned history, slot 2 picks winner
+cat > "$WORK/outcomes.json" << 'JSON'
+[{"batch_type": "new-file", "prompt_variant": "check all imports before running tests", "won": true, "score": 500}]
+JSON
+
+result=$(get_prompt_variants "new-file" "$WORK/outcomes.json" 3)
+second_line=$(echo "$result" | sed -n '2p')
+assert_eq "variants: learned winner in slot 2" "check all imports before running tests" "$second_line"
+
+echo ""
+echo "Results: $((TESTS - FAILURES))/$TESTS passed"
+if [[ $FAILURES -gt 0 ]]; then
+    echo "FAILURES: $FAILURES"
+    exit 1
+fi
+echo "ALL PASSED"

package/scripts/tests/test-run-plan-state.sh

@@ -0,0 +1,261 @@
+#!/usr/bin/env bash
+# Test state manager functions
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../lib/run-plan-state.sh"
+
+FAILURES=0
+TESTS=0
+
+assert_eq() {
+    local desc="$1" expected="$2" actual="$3"
+    TESTS=$((TESTS + 1))
+    if [[ "$expected" != "$actual" ]]; then
+        echo "FAIL: $desc"
+        echo "  expected: $expected"
+        echo "  actual:   $actual"
+        FAILURES=$((FAILURES + 1))
+    else
+        echo "PASS: $desc"
+    fi
+}
+
+# --- Temp dir for worktree simulation ---
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# --- Test: init_state creates the file ---
+init_state "$WORK" "docs/plans/2026-02-20-feature.md" "headless"
+
+TESTS=$((TESTS + 1))
+if [[ -f "$WORK/.run-plan-state.json" ]]; then
+    echo "PASS: init_state creates state file"
+else
+    echo "FAIL: init_state should create .run-plan-state.json"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# --- Test: read_state_field reads plan_file ---
+val=$(read_state_field "$WORK" "plan_file")
+assert_eq "read plan_file" "docs/plans/2026-02-20-feature.md" "$val"
+
+# --- Test: read_state_field reads mode ---
+val=$(read_state_field "$WORK" "mode")
+assert_eq "read mode" "headless" "$val"
+
+# --- Test: read_state_field reads current_batch ---
+val=$(read_state_field "$WORK" "current_batch")
+assert_eq "read current_batch (initial)" "1" "$val"
+
+# --- Test: read_state_field reads started_at ---
+val=$(read_state_field "$WORK" "started_at")
+TESTS=$((TESTS + 1))
+if [[ -n "$val" && "$val" != "null" ]]; then
+    echo "PASS: started_at is set"
+else
+    echo "FAIL: started_at should be a non-null timestamp"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# --- Test: completed_batches starts empty ---
+val=$(read_state_field "$WORK" "completed_batches")
+assert_eq "completed_batches starts empty" "[]" "$val"
+
+# --- Test: last_quality_gate starts null ---
+val=$(read_state_field "$WORK" "last_quality_gate")
+assert_eq "last_quality_gate starts null" "null" "$val"
+
+# --- Test: complete_batch updates state ---
+complete_batch "$WORK" 1 42
+
+val=$(read_state_field "$WORK" "current_batch")
+assert_eq "current_batch after completing batch 1" "2" "$val"
+
+val=$(read_state_field "$WORK" "completed_batches")
+assert_eq "completed_batches has batch 1" "[1]" "$val"
+
+val=$(jq -r '.test_counts["1"]' "$WORK/.run-plan-state.json")
+assert_eq "test_count for batch 1" "42" "$val"
+
+# --- Test: multiple complete_batch calls accumulate ---
+complete_batch "$WORK" 2 55
+
+val=$(read_state_field "$WORK" "current_batch")
+assert_eq "current_batch after completing batch 2" "3" "$val"
+
+val=$(read_state_field "$WORK" "completed_batches")
+assert_eq "completed_batches has both" "[1,2]" "$val"
+
+val=$(jq -r '.test_counts["2"]' "$WORK/.run-plan-state.json")
+assert_eq "test_count for batch 2" "55" "$val"
+
+# Previous batch 1 count still there
+val=$(jq -r '.test_counts["1"]' "$WORK/.run-plan-state.json")
+assert_eq "test_count for batch 1 still intact" "42" "$val"
+
+# --- Test: get_previous_test_count ---
+val=$(get_previous_test_count "$WORK")
+assert_eq "previous test count after batch 2" "55" "$val"
+
+# --- Test: get_previous_test_count with no completions ---
+WORK2=$(mktemp -d)
+trap 'rm -rf "$WORK" "$WORK2"' EXIT
+init_state "$WORK2" "plan.md" "team"
+
+val=$(get_previous_test_count "$WORK2")
+assert_eq "previous test count with no completions" "0" "$val"
+
+# --- Test: set_quality_gate ---
+set_quality_gate "$WORK" 2 "true" 55
+
+val=$(jq -r '.last_quality_gate.batch' "$WORK/.run-plan-state.json")
+assert_eq "quality gate batch" "2" "$val"
+
+val=$(jq -r '.last_quality_gate.passed' "$WORK/.run-plan-state.json")
+assert_eq "quality gate passed" "true" "$val"
+
+val=$(jq -r '.last_quality_gate.test_count' "$WORK/.run-plan-state.json")
+assert_eq "quality gate test_count" "55" "$val"
+
+# Verify timestamp exists
+val=$(jq -r '.last_quality_gate.timestamp' "$WORK/.run-plan-state.json")
+TESTS=$((TESTS + 1))
+if [[ -n "$val" && "$val" != "null" ]]; then
+    echo "PASS: quality gate has timestamp"
+else
+    echo "FAIL: quality gate should have timestamp"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# --- Test: set_quality_gate with failed gate ---
+set_quality_gate "$WORK" 3 "false" 50
+
+val=$(jq -r '.last_quality_gate.batch' "$WORK/.run-plan-state.json")
+assert_eq "failed quality gate batch" "3" "$val"
+
+val=$(jq -r '.last_quality_gate.passed' "$WORK/.run-plan-state.json")
+assert_eq "failed quality gate passed" "false" "$val"
+
+# --- Test: complete_batch stores duration ---
+WORK3=$(mktemp -d)
+trap 'rm -rf "$WORK" "$WORK2" "$WORK3"' EXIT
+init_state "$WORK3" "plan.md" "headless"
+complete_batch "$WORK3" 1 42 120
+
+duration=$(jq -r '.durations["1"]' "$WORK3/.run-plan-state.json")
+assert_eq "complete_batch: stores duration" "120" "$duration"
+
+# --- Test: duration defaults to 0 when not provided ---
+complete_batch "$WORK3" 2 55
+duration=$(jq -r '.durations["2"]' "$WORK3/.run-plan-state.json")
+assert_eq "complete_batch: duration defaults to 0" "0" "$duration"
+
+# --- Test: init_state includes durations object ---
+val=$(jq -r '.durations | type' "$WORK3/.run-plan-state.json")
+assert_eq "init_state: has durations object" "object" "$val"
+
+# --- Test: complete_batch with non-numeric batch_num ('final') ---
+WORK4=$(mktemp -d)
+trap 'rm -rf "$WORK" "$WORK2" "$WORK3" "$WORK4"' EXIT
+init_state "$WORK4" "plan.md" "headless"
+complete_batch "$WORK4" 1 42
+complete_batch "$WORK4" "final" 50
+
+val=$(jq -r '.test_counts["final"]' "$WORK4/.run-plan-state.json")
+assert_eq "complete_batch: non-numeric batch 'final' stores test count" "50" "$val"
+
+val=$(jq -r '.durations["final"]' "$WORK4/.run-plan-state.json")
+assert_eq "complete_batch: non-numeric batch 'final' stores duration" "0" "$val"
+
+val=$(jq -r '.completed_batches | last' "$WORK4/.run-plan-state.json")
+assert_eq "complete_batch: non-numeric batch 'final' in completed_batches" "final" "$val"
+
+# Numeric batches still work after non-numeric
+val=$(jq -r '.test_counts["1"]' "$WORK4/.run-plan-state.json")
+assert_eq "complete_batch: numeric batch still intact after non-numeric" "42" "$val"
+
+# --- Test: get_previous_test_count returns -1 when key missing ---
+WORK5=$(mktemp -d)
+trap 'rm -rf "$WORK" "$WORK2" "$WORK3" "$WORK4" "$WORK5"' EXIT
+init_state "$WORK5" "plan.md" "headless"
+# Manually add a batch to completed_batches without a corresponding test_count entry
+jq '.completed_batches += [1]' "$WORK5/.run-plan-state.json" > "$WORK5/.tmp.json" && mv "$WORK5/.tmp.json" "$WORK5/.run-plan-state.json"
+
+val=$(get_previous_test_count "$WORK5")
+assert_eq "get_previous_test_count: returns -1 when key missing" "-1" "$val"
+
+# --- Test: set_quality_gate with non-numeric batch_num ('final') ---
+WORK6=$(mktemp -d)
+trap 'rm -rf "$WORK" "$WORK2" "$WORK3" "$WORK4" "$WORK5" "$WORK6"' EXIT
+init_state "$WORK6" "plan.md" "headless"
+complete_batch "$WORK6" 1 42
+set_quality_gate "$WORK6" "final" "true" 99
+
+val=$(jq -r '.last_quality_gate.batch' "$WORK6/.run-plan-state.json")
+assert_eq "set_quality_gate: non-numeric batch 'final' stored" "final" "$val"
+
+val=$(jq -r '.last_quality_gate.passed' "$WORK6/.run-plan-state.json")
+assert_eq "set_quality_gate: non-numeric batch passed=true" "true" "$val"
+
+val=$(jq -r '.last_quality_gate.test_count' "$WORK6/.run-plan-state.json")
+assert_eq "set_quality_gate: non-numeric batch test_count" "99" "$val"
+
+# --- Test: end-to-end complete_batch 'final' then get_previous_test_count ---
+WORK7=$(mktemp -d)
+trap 'rm -rf "$WORK" "$WORK2" "$WORK3" "$WORK4" "$WORK5" "$WORK6" "$WORK7"' EXIT
+init_state "$WORK7" "plan.md" "headless"
+complete_batch "$WORK7" 1 42
+complete_batch "$WORK7" "final" 99
+
+val=$(get_previous_test_count "$WORK7")
+assert_eq "e2e: complete_batch 'final' then get_previous_test_count returns 99" "99" "$val"
+
+# --- Test: init_state includes costs object ---
+WORK_COST=$(mktemp -d)
+trap 'rm -rf "$WORK" "$WORK2" "$WORK3" "$WORK4" "$WORK5" "$WORK6" "$WORK7" "$WORK_COST" "$WORK8"' EXIT
+init_state "$WORK_COST" "plan.md" "headless"
+
+val=$(jq -r '.costs | type' "$WORK_COST/.run-plan-state.json")
+assert_eq "init_state: has costs object" "object" "$val"
+
+val=$(jq -r '.total_cost_usd' "$WORK_COST/.run-plan-state.json")
+assert_eq "init_state: total_cost_usd starts at 0" "0" "$val"
+
+# --- Test: set_quality_gate normalizes truthy values (#8) ---
+WORK8=$(mktemp -d)
+trap 'rm -rf "$WORK" "$WORK2" "$WORK3" "$WORK4" "$WORK5" "$WORK6" "$WORK7" "$WORK_COST" "$WORK8"' EXIT
+init_state "$WORK8" "plan.md" "headless"
+
+# "1" should normalize to true
+set_quality_gate "$WORK8" 1 "1" 10
+val=$(jq -r '.last_quality_gate.passed' "$WORK8/.run-plan-state.json")
+assert_eq "set_quality_gate: '1' normalizes to true" "true" "$val"
+
+# "0" should normalize to false
+set_quality_gate "$WORK8" 1 "0" 10
+val=$(jq -r '.last_quality_gate.passed' "$WORK8/.run-plan-state.json")
+assert_eq "set_quality_gate: '0' normalizes to false" "false" "$val"
+
+# "yes" should normalize to true
+set_quality_gate "$WORK8" 1 "yes" 10
+val=$(jq -r '.last_quality_gate.passed' "$WORK8/.run-plan-state.json")
+assert_eq "set_quality_gate: 'yes' normalizes to true" "true" "$val"
+
+# "no" should normalize to false
+set_quality_gate "$WORK8" 1 "no" 10
+val=$(jq -r '.last_quality_gate.passed' "$WORK8/.run-plan-state.json")
+assert_eq "set_quality_gate: 'no' normalizes to false" "false" "$val"
+
+# "random" should normalize to false
+set_quality_gate "$WORK8" 1 "random" 10
+val=$(jq -r '.last_quality_gate.passed' "$WORK8/.run-plan-state.json")
+assert_eq "set_quality_gate: 'random' normalizes to false" "false" "$val"
+
+echo ""
+echo "Results: $((TESTS - FAILURES))/$TESTS passed"
+if [[ $FAILURES -gt 0 ]]; then
+    echo "FAILURES: $FAILURES"
+    exit 1
+fi
+echo "ALL PASSED"