autonomous-coding-toolkit 1.0.0

package/scripts/tests/test-thompson-sampling.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/test-helpers.sh"
+
+# Source the library under test
+source "$SCRIPT_DIR/../lib/thompson-sampling.sh"
+
+# --- Test: thompson_sample returns float in [0,1] ---
+sample=$(thompson_sample 10 5)
+TESTS=$((TESTS + 1))
+# Check it's a valid float between 0 and 1 using bc
+is_valid=$(echo "$sample >= 0 && $sample <= 1" | bc -l 2>/dev/null || echo "0")
+if [[ "$is_valid" == "1" ]]; then
+    echo "PASS: thompson_sample 10 5 returns float in [0,1] (got $sample)"
+else
+    echo "FAIL: thompson_sample 10 5 returned '$sample' (expected float in [0,1])"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# --- Test: thompson_route returns "mab" when perf file missing ---
+route_missing=$(thompson_route "new-file" "/tmp/nonexistent-perf-$$.json")
+assert_eq "missing perf file → mab" "mab" "$route_missing"
+
+# --- Test: thompson_route returns "mab" when type has < 5 data points ---
+TEST_TMPDIR=$(mktemp -d)
+trap 'rm -rf "$TEST_TMPDIR"' EXIT
+
+cat > "$TEST_TMPDIR/perf.json" <<'JSON'
+{
+  "new-file": {"superpowers": {"wins": 2, "losses": 1}, "ralph": {"wins": 1, "losses": 0}},
+  "refactoring": {"superpowers": {"wins": 0, "losses": 0}, "ralph": {"wins": 0, "losses": 0}},
+  "integration": {"superpowers": {"wins": 5, "losses": 2}, "ralph": {"wins": 3, "losses": 4}},
+  "test-only": {"superpowers": {"wins": 0, "losses": 0}, "ralph": {"wins": 0, "losses": 0}},
+  "calibration_count": 0,
+  "calibration_complete": false
+}
+JSON
+
+route_few=$(thompson_route "new-file" "$TEST_TMPDIR/perf.json")
+assert_eq "< 5 data points → mab" "mab" "$route_few"
+
+# --- Test: thompson_route returns "mab" for integration type (always explore) ---
+cat > "$TEST_TMPDIR/perf-int.json" <<'JSON'
+{
+  "new-file": {"superpowers": {"wins": 10, "losses": 2}, "ralph": {"wins": 3, "losses": 9}},
+  "refactoring": {"superpowers": {"wins": 10, "losses": 2}, "ralph": {"wins": 3, "losses": 9}},
+  "integration": {"superpowers": {"wins": 10, "losses": 2}, "ralph": {"wins": 3, "losses": 9}},
+  "test-only": {"superpowers": {"wins": 10, "losses": 2}, "ralph": {"wins": 3, "losses": 9}},
+  "calibration_count": 10,
+  "calibration_complete": true
+}
+JSON
+
+route_int=$(thompson_route "integration" "$TEST_TMPDIR/perf-int.json")
+assert_eq "integration type → always mab" "mab" "$route_int"
+
+# --- Test: thompson_route returns winning strategy with strong signal ---
+# superpowers: 15 wins / 3 losses = 83% win rate, 18 total points
+# ralph: 3 wins / 15 losses = 17% win rate
+# Run 10 times and check majority goes to superpowers
+superpowers_count=0
+for i in $(seq 1 10); do
+    result=$(thompson_route "new-file" "$TEST_TMPDIR/perf-int.json")
+    if [[ "$result" == "superpowers" ]]; then
+        superpowers_count=$((superpowers_count + 1))
+    fi
+done
+
+TESTS=$((TESTS + 1))
+if [[ "$superpowers_count" -ge 7 ]]; then
+    echo "PASS: strong signal routes to winner (superpowers won $superpowers_count/10)"
+else
+    echo "FAIL: strong signal should route to superpowers ≥7/10 times (got $superpowers_count/10)"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# --- Test: init_strategy_perf creates valid JSON ---
+init_file="$TEST_TMPDIR/init-perf.json"
+init_strategy_perf "$init_file"
+
+TESTS=$((TESTS + 1))
+if [[ -f "$init_file" ]] && jq . "$init_file" > /dev/null 2>&1; then
+    echo "PASS: init_strategy_perf creates valid JSON"
+else
+    echo "FAIL: init_strategy_perf did not create valid JSON"
+    FAILURES=$((FAILURES + 1))
+fi
+
+# Check all 4 batch types present
+for bt in "new-file" "refactoring" "integration" "test-only"; do
+    has_type=$(jq --arg bt "$bt" 'has($bt)' "$init_file" 2>/dev/null)
+    assert_eq "init has $bt type" "true" "$has_type"
+done
+
+# Check calibration fields
+has_cal_count=$(jq 'has("calibration_count")' "$init_file" 2>/dev/null)
+assert_eq "init has calibration_count" "true" "$has_cal_count"
+
+has_cal_complete=$(jq 'has("calibration_complete")' "$init_file" 2>/dev/null)
+assert_eq "init has calibration_complete" "true" "$has_cal_complete"
+
+# --- Test: update_strategy_perf increments wins/losses ---
+update_file="$TEST_TMPDIR/update-perf.json"
+init_strategy_perf "$update_file"
+
+# Superpowers wins a new-file batch
+update_strategy_perf "$update_file" "new-file" "superpowers"
+sp_wins=$(jq '."new-file".superpowers.wins' "$update_file" 2>/dev/null)
+assert_eq "superpowers wins incremented" "1" "$sp_wins"
+
+# Ralph loses (the other side)
+ralph_losses=$(jq '."new-file".ralph.losses' "$update_file" 2>/dev/null)
+assert_eq "ralph losses incremented" "1" "$ralph_losses"
+
+# Do it again
+update_strategy_perf "$update_file" "new-file" "superpowers"
+sp_wins2=$(jq '."new-file".superpowers.wins' "$update_file" 2>/dev/null)
+assert_eq "superpowers wins incremented again" "2" "$sp_wins2"
+
+# --- Test: thompson_route spread-too-close returns "mab" ---
+# When both strategies have similar win rates (spread < 15%), route to mab.
+# superpowers: 6W/4L = 60%, ralph: 5W/5L = 50% → spread = 10% < 15% → "mab"
+cat > "$TEST_TMPDIR/perf-close.json" <<'JSON'
+{
+  "new-file": {"superpowers": {"wins": 6, "losses": 4}, "ralph": {"wins": 5, "losses": 5}},
+  "refactoring": {"superpowers": {"wins": 6, "losses": 4}, "ralph": {"wins": 5, "losses": 5}},
+  "integration": {"superpowers": {"wins": 6, "losses": 4}, "ralph": {"wins": 5, "losses": 5}},
+  "test-only": {"superpowers": {"wins": 6, "losses": 4}, "ralph": {"wins": 5, "losses": 5}},
+  "calibration_count": 20,
+  "calibration_complete": true
+}
+JSON
+
+route_close=$(thompson_route "new-file" "$TEST_TMPDIR/perf-close.json")
+assert_eq "spread < 15% → mab (too close to call)" "mab" "$route_close"
+
+report_results

package/scripts/tests/test-validate-all.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Test validate-all.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/test-helpers.sh"
+
+VALIDATOR="$SCRIPT_DIR/../validate-all.sh"
+
+# === Test: Runs on actual toolkit with --warn and reports summary ===
+output=$(bash "$VALIDATOR" --warn 2>&1 || echo "EXIT:$?")
+assert_contains "actual toolkit --warn: shows summary" "validators passed" "$output"
+assert_not_contains "actual toolkit --warn: exit 0" "EXIT:" "$output"
+
+# === Test: Reports individual validator names ===
+output=$(bash "$VALIDATOR" --warn 2>&1 || echo "EXIT:$?")
+assert_contains "shows lessons" "validate-lessons" "$output"
+assert_contains "shows skills" "validate-skills" "$output"
+assert_contains "shows commands" "validate-commands" "$output"
+assert_contains "shows plugin" "validate-plugin" "$output"
+assert_contains "shows hooks" "validate-hooks" "$output"
+
+# === Test: Actual toolkit passes strict (no --warn) ===
+output=$(bash "$VALIDATOR" 2>&1 || echo "EXIT:$?")
+assert_contains "actual toolkit strict: shows summary" "validators passed" "$output"
+assert_not_contains "actual toolkit strict: exit 0" "EXIT:" "$output"
+
+# === Test: --help exits 0 ===
+output=$(bash "$VALIDATOR" --help 2>&1 || echo "EXIT:$?")
+assert_contains "--help: shows usage" "Usage:" "$output"
+assert_not_contains "--help: exit 0" "EXIT:" "$output"
+
+# === Test: No args (empty PASS_ARGS) works under set -u (#23) ===
+# This specifically tests that empty array expansion doesn't error under set -u
+output=$(bash "$VALIDATOR" --warn 2>&1; echo "EXIT:$?")
+assert_contains "empty PASS_ARGS: succeeds" "EXIT:0" "$output"
+
+# === Test: Reports FAIL and exits 1 when a validator fails ===
+# Create a controlled fixture with an invalid skill to trigger failure
+TMP_DIR=$(mktemp -d)
+trap 'rm -rf "$TMP_DIR"' EXIT
+# Create a minimal toolkit structure with a broken skill
+mkdir -p "$TMP_DIR/skills/broken-skill"
+# Empty dir = missing SKILL.md → validate-skills fails
+# Create stub validators that pass, except skills points to our broken fixture
+mkdir -p "$TMP_DIR/scripts"
+for v in validate-lessons validate-commands validate-plugin validate-hooks; do
+    echo '#!/usr/bin/env bash' > "$TMP_DIR/scripts/${v}.sh"
+    echo 'echo "'"$v"': PASS"; exit 0' >> "$TMP_DIR/scripts/${v}.sh"
+done
+# validate-skills uses SKILLS_DIR env — point to broken fixture
+echo '#!/usr/bin/env bash' > "$TMP_DIR/scripts/validate-skills.sh"
+echo 'echo "broken-skill: Missing SKILL.md"; echo "validate-skills: FAIL"; exit 1' >> "$TMP_DIR/scripts/validate-skills.sh"
+# Copy validate-all.sh to the temp dir
+cp "$VALIDATOR" "$TMP_DIR/scripts/validate-all.sh"
+output=$(bash "$TMP_DIR/scripts/validate-all.sh" 2>&1 || echo "EXIT:$?")
+assert_contains "reports failures: exit 1" "EXIT:1" "$output"
+assert_contains "reports failures: shows Failed" "Failed:" "$output"
+
+report_results

package/scripts/tests/test-validate-commands.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Test validate-commands.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/test-helpers.sh"
+
+VALIDATOR="$SCRIPT_DIR/../validate-commands.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# Helper: create a command file
+create_command() {
+    local name="$1" content="$2"
+    mkdir -p "$WORK/commands"
+    echo "$content" > "$WORK/commands/$name"
+}
+
+# Helper: run validator against temp commands dir
+run_validator() {
+    local exit_code=0
+    COMMANDS_DIR="$WORK/commands" bash "$VALIDATOR" "$@" 2>&1 || exit_code=$?
+    echo "EXIT:$exit_code"
+}
+
+# === Test: Valid command passes ===
+rm -rf "$WORK/commands"
+create_command "my-command.md" '---
+description: "A test command"
+---
+Body text here.'
+
+output=$(run_validator)
+assert_contains "valid command: PASS" "validate-commands: PASS" "$output"
+assert_contains "valid command: exit 0" "EXIT:0" "$output"
+
+# === Test: Missing description field fails ===
+rm -rf "$WORK/commands"
+create_command "no-desc.md" '---
+argument-hint: "<thing>"
+---
+Body.'
+
+output=$(run_validator)
+assert_contains "missing description: reports violation" "Missing required field: description" "$output"
+assert_contains "missing description: exit 1" "EXIT:1" "$output"
+
+# === Test: Missing frontmatter start fails ===
+rm -rf "$WORK/commands"
+create_command "no-front.md" 'description: "No frontmatter markers"
+Some body.'
+
+output=$(run_validator)
+assert_contains "missing ---: reports violation" "First line must be '---'" "$output"
+assert_contains "missing ---: exit 1" "EXIT:1" "$output"
+
+# === Test: Missing closing --- fails ===
+rm -rf "$WORK/commands"
+create_command "no-close.md" '---
+description: "Unclosed frontmatter"
+Body text without closing delimiter.'
+
+output=$(run_validator)
+assert_contains "missing close ---: reports violation" "Frontmatter not closed" "$output"
+assert_contains "missing close ---: exit 1" "EXIT:1" "$output"
+
+# === Test: --warn exits 0 even with violations ===
+rm -rf "$WORK/commands"
+create_command "warn-test.md" '---
+argument-hint: "<thing>"
+---
+Body.'
+
+output=$(run_validator --warn)
+assert_contains "--warn: still reports violation" "Missing required field: description" "$output"
+assert_contains "--warn: exits 0" "EXIT:0" "$output"
+
+# === Test: --help exits 0 ===
+output=$(run_validator --help)
+assert_contains "--help: shows usage" "Usage:" "$output"
+assert_contains "--help: exits 0" "EXIT:0" "$output"
+
+# === Test: Missing commands directory fails ===
+rm -rf "$WORK/commands"
+output=$(COMMANDS_DIR="$WORK/nonexistent" bash "$VALIDATOR" 2>&1 || echo "EXIT:$?")
+assert_contains "missing dir: error message" "commands directory not found" "$output"
+assert_contains "missing dir: exit 1" "EXIT:1" "$output"
+
+report_results

package/scripts/tests/test-validate-hooks.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# Test validate-hooks.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/test-helpers.sh"
+
+VALIDATOR="$SCRIPT_DIR/../validate-hooks.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# Helper: create hooks dir with hooks.json
+create_hooks() {
+    local content="$1"
+    mkdir -p "$WORK/hooks"
+    echo "$content" > "$WORK/hooks/hooks.json"
+}
+
+# Helper: create a script file (optionally executable)
+create_script() {
+    local path="$1" executable="${2:-true}"
+    mkdir -p "$(dirname "$WORK/$path")"
+    echo '#!/usr/bin/env bash' > "$WORK/$path"
+    if [[ "$executable" == "true" ]]; then chmod +x "$WORK/$path"; fi
+}
+
+# Helper: run validator against temp dir
+run_validator() {
+    local exit_code=0
+    HOOKS_DIR="$WORK/hooks" TOOLKIT_ROOT="$WORK" bash "$VALIDATOR" "$@" 2>&1 || exit_code=$?
+    echo "EXIT:$exit_code"
+}
+
+# === Test: Valid hooks.json with existing executable script passes ===
+create_hooks '{
+  "hooks": {
+    "Stop": [{"hooks": [{"type": "command", "command": "${CLAUDE_PLUGIN_ROOT}/hooks/stop-hook.sh"}]}]
+  }
+}'
+create_script "hooks/stop-hook.sh"
+
+output=$(run_validator)
+assert_contains "valid hooks: PASS" "validate-hooks: PASS" "$output"
+assert_contains "valid hooks: exit 0" "EXIT:0" "$output"
+
+# === Test: Nonexistent script fails ===
+create_hooks '{
+  "hooks": {
+    "Stop": [{"hooks": [{"type": "command", "command": "${CLAUDE_PLUGIN_ROOT}/hooks/missing.sh"}]}]
+  }
+}'
+
+output=$(run_validator)
+assert_contains "missing script: reports violation" "script not found" "$output"
+assert_contains "missing script: exit 1" "EXIT:1" "$output"
+
+# === Test: Non-executable script fails ===
+create_hooks '{
+  "hooks": {
+    "Stop": [{"hooks": [{"type": "command", "command": "${CLAUDE_PLUGIN_ROOT}/hooks/not-exec.sh"}]}]
+  }
+}'
+create_script "hooks/not-exec.sh" "false"
+
+output=$(run_validator)
+assert_contains "not executable: reports violation" "not executable" "$output"
+assert_contains "not executable: exit 1" "EXIT:1" "$output"
+
+# === Test: Invalid JSON fails ===
+create_hooks '{invalid json'
+
+output=$(run_validator)
+assert_contains "invalid JSON: error" "hooks.json is not valid JSON" "$output"
+assert_contains "invalid JSON: exit 1" "EXIT:1" "$output"
+
+# === Test: Missing hooks.json fails ===
+rm -rf "$WORK/hooks"
+output=$(HOOKS_DIR="$WORK/nonexistent" TOOLKIT_ROOT="$WORK" bash "$VALIDATOR" 2>&1 || echo "EXIT:$?")
+assert_contains "missing dir: error message" "hooks directory not found" "$output"
+assert_contains "missing dir: exit 1" "EXIT:1" "$output"
+
+# === Test: --warn exits 0 even with violations ===
+create_hooks '{
+  "hooks": {
+    "Stop": [{"hooks": [{"type": "command", "command": "${CLAUDE_PLUGIN_ROOT}/hooks/missing.sh"}]}]
+  }
+}'
+
+output=$(run_validator --warn)
+assert_contains "--warn: still reports violation" "script not found" "$output"
+assert_contains "--warn: exits 0" "EXIT:0" "$output"
+
+# === Test: --help exits 0 ===
+output=$(run_validator --help)
+assert_contains "--help: shows usage" "Usage:" "$output"
+assert_contains "--help: exits 0" "EXIT:0" "$output"
+
+report_results

package/scripts/tests/test-validate-lessons.sh

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+# Test validate-lessons.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/test-helpers.sh"
+
+VALIDATOR="$SCRIPT_DIR/../validate-lessons.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# Helper: create a lesson file in the temp lessons dir
+create_lesson() {
+    local name="$1" content="$2"
+    local dir="$WORK/lessons"
+    mkdir -p "$dir"
+    echo "$content" > "$dir/$name"
+}
+
+# Helper: run validator against temp lessons dir
+run_validator() {
+    local exit_code=0
+    LESSONS_DIR="$WORK/lessons" bash "$VALIDATOR" "$@" 2>&1 || exit_code=$?
+    echo "EXIT:$exit_code"
+}
+
+# === Test: Valid lesson passes ===
+rm -rf "$WORK/lessons"
+create_lesson "0001-test.md" '---
+id: 1
+title: "Test lesson"
+severity: blocker
+languages: [python]
+pattern:
+  type: syntactic
+  regex: "foo"
+---
+Body text here.'
+
+output=$(run_validator)
+assert_contains "valid lesson: PASS" "validate-lessons: PASS" "$output"
+assert_contains "valid lesson: exit 0" "EXIT:0" "$output"
+
+# === Test: Missing --- start line fails ===
+rm -rf "$WORK/lessons"
+create_lesson "0002-bad-start.md" 'id: 1
+title: "No frontmatter marker"
+severity: blocker
+languages: [python]
+---'
+
+output=$(run_validator)
+assert_contains "missing ---: reports violation" "First line must be" "$output"
+assert_contains "missing ---: FAIL" "FAIL" "$output"
+assert_contains "missing ---: exit 1" "EXIT:1" "$output"
+
+# === Test: Missing required field fails ===
+rm -rf "$WORK/lessons"
+create_lesson "0003-missing-field.md" '---
+id: 3
+severity: blocker
+languages: [python]
+pattern:
+  type: semantic
+---'
+
+output=$(run_validator)
+assert_contains "missing title: reports violation" "Missing required field: title" "$output"
+assert_contains "missing title: exit 1" "EXIT:1" "$output"
+
+# === Test: Duplicate IDs fail ===
+rm -rf "$WORK/lessons"
+create_lesson "0004-dup-a.md" '---
+id: 42
+title: "First"
+severity: blocker
+languages: [python]
+pattern:
+  type: semantic
+---'
+create_lesson "0005-dup-b.md" '---
+id: 42
+title: "Second"
+severity: blocker
+languages: [python]
+pattern:
+  type: semantic
+---'
+
+output=$(run_validator)
+assert_contains "duplicate IDs: reports violation" "Duplicate lesson ID: 42" "$output"
+assert_contains "duplicate IDs: exit 1" "EXIT:1" "$output"
+
+# === Test: Invalid severity fails ===
+rm -rf "$WORK/lessons"
+create_lesson "0006-bad-severity.md" '---
+id: 6
+title: "Bad severity"
+severity: critical
+languages: [python]
+pattern:
+  type: semantic
+---'
+
+output=$(run_validator)
+assert_contains "invalid severity: reports violation" "Invalid severity" "$output"
+assert_contains "invalid severity: exit 1" "EXIT:1" "$output"
+
+# === Test: Syntactic without regex fails ===
+rm -rf "$WORK/lessons"
+create_lesson "0007-no-regex.md" '---
+id: 7
+title: "Missing regex"
+severity: blocker
+languages: [python]
+pattern:
+  type: syntactic
+---'
+
+output=$(run_validator)
+assert_contains "syntactic no regex: reports violation" "Syntactic lesson missing regex field" "$output"
+assert_contains "syntactic no regex: exit 1" "EXIT:1" "$output"
+
+# === Test: --warn exits 0 even with violations ===
+rm -rf "$WORK/lessons"
+create_lesson "0008-warn-test.md" '---
+id: 8
+severity: blocker
+languages: [python]
+pattern:
+  type: semantic
+---'
+
+output=$(run_validator --warn)
+assert_contains "--warn: still reports violation" "Missing required field: title" "$output"
+assert_contains "--warn: exits 0" "EXIT:0" "$output"
+
+# === Test: --help exits 0 ===
+output=$(run_validator --help)
+assert_contains "--help: shows usage" "Usage:" "$output"
+assert_contains "--help: exits 0" "EXIT:0" "$output"
+
+# === Test: Missing lessons directory fails ===
+SAVE_DIR="$WORK/lessons"
+rm -rf "$WORK/lessons"
+output=$(LESSONS_DIR="$WORK/nonexistent" bash "$VALIDATOR" 2>&1 || echo "EXIT:$?")
+assert_contains "missing dir: error message" "lessons directory not found" "$output"
+assert_contains "missing dir: exit 1" "EXIT:1" "$output"
+
+report_results