autonomous-coding-toolkit 1.0.0

package/scripts/tests/test-validate-plan-quality.sh

@@ -0,0 +1,235 @@
+#!/usr/bin/env bash
+# Test plan quality scorecard
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+VALIDATE="$SCRIPT_DIR/../validate-plan-quality.sh"
+
+FAILURES=0
+TESTS=0
+
+assert_eq() {
+    local desc="$1" expected="$2" actual="$3"
+    TESTS=$((TESTS + 1))
+    if [[ "$expected" != "$actual" ]]; then
+        echo "FAIL: $desc (expected: $expected, got: $actual)"
+        FAILURES=$((FAILURES + 1))
+    else
+        echo "PASS: $desc"
+    fi
+}
+
+assert_contains() {
+    local desc="$1" needle="$2" haystack="$3"
+    TESTS=$((TESTS + 1))
+    if [[ "$haystack" != *"$needle"* ]]; then
+        echo "FAIL: $desc"
+        echo "  expected to contain: $needle"
+        FAILURES=$((FAILURES + 1))
+    else
+        echo "PASS: $desc"
+    fi
+}
+
+TMPDIR_ROOT=$(mktemp -d)
+trap 'rm -rf "$TMPDIR_ROOT"' EXIT
+
+# =============================================================================
+# High-quality plan (should score >= 60)
+# =============================================================================
+
+HIGH_QUALITY="$TMPDIR_ROOT/high-quality.md"
+cat > "$HIGH_QUALITY" << 'EOF'
+# High Quality Plan
+
+## Batch 1: Setup (Tasks 1-2)
+
+### Task 1: Create Data Model
+
+**Files:**
+- Create: `src/models.py`
+- Test: `tests/test_models.py`
+
+**Step 1: Write the failing test**
+
+Write `tests/test_models.py` with assertions that verify Model class works.
+
+**Step 2: Implement**
+
+Create `src/models.py` with the Model class.
+
+### Task 2: Add Validation
+
+**Files:**
+- Modify: `src/models.py`
+
+Add validation. Write test first, then implement. Should reject invalid input.
+
+## Batch 2: Integration (Tasks 3-4)
+
+### Task 3: Wire Components
+
+**Files:**
+- Create: `src/pipeline.py`
+- Test: `tests/test_pipeline.py`
+
+Connect parser to transformer. Verify end-to-end with test that checks output format.
+
+### Task 4: CLI Entry Point
+
+**Files:**
+- Create: `src/cli.py`
+- Test: `tests/test_cli.py`
+
+Add CLI. Test argument parsing. Should handle missing file gracefully.
+EOF
+
+output=$(bash "$VALIDATE" "$HIGH_QUALITY" 2>&1)
+exit_code=$?
+
+assert_eq "high quality plan passes" "0" "$exit_code"
+assert_contains "high quality shows PASSED" "PASSED" "$output"
+
+# =============================================================================
+# Low-quality plan (should score < 60 with low min-score threshold)
+# =============================================================================
+
+LOW_QUALITY="$TMPDIR_ROOT/low-quality.md"
+cat > "$LOW_QUALITY" << 'EOF'
+# Low Quality Plan
+
+## Batch 1: Do Everything
+
+Build the whole thing. Make it work. Deploy it.
+
+## Batch 2: More Stuff
+
+Do more stuff. Fix what broke in batch 1.
+Depends on batch 3 being done first.
+
+## Batch 3: Final Things
+
+Finish everything. Clean up.
+EOF
+
+output=$(bash "$VALIDATE" "$LOW_QUALITY" 2>&1 || true)
+# Low quality: no tasks, no files, no tests, forward ref in batch 2 → batch 3
+assert_contains "low quality shows scores" "Scorecard" "$output"
+
+# With min-score=90 this should definitely fail
+exit_code=0
+bash "$VALIDATE" "$LOW_QUALITY" --min-score 90 >/dev/null 2>&1 || exit_code=$?
+assert_eq "low quality fails at min-score 90" "1" "$exit_code"
+
+# =============================================================================
+# JSON output mode
+# =============================================================================
+
+json_output=$(bash "$VALIDATE" "$HIGH_QUALITY" --json 2>&1)
+assert_contains "json has score field" '"score"' "$json_output"
+assert_contains "json has dimensions" '"dimensions"' "$json_output"
+assert_contains "json has passed field" '"passed"' "$json_output"
+assert_contains "json has task_granularity" '"task_granularity"' "$json_output"
+assert_contains "json has weight" '"weight"' "$json_output"
+
+# =============================================================================
+# Error cases
+# =============================================================================
+
+# No file
+exit_code=0
+bash "$VALIDATE" 2>/dev/null || exit_code=$?
+assert_eq "no file arg exits 1" "1" "$exit_code"
+
+# Missing file
+exit_code=0
+bash "$VALIDATE" "$TMPDIR_ROOT/nonexistent.md" 2>/dev/null || exit_code=$?
+assert_eq "missing file exits 1" "1" "$exit_code"
+
+# File with no batches
+NO_BATCHES="$TMPDIR_ROOT/no-batches.md"
+echo "# Just a title" > "$NO_BATCHES"
+exit_code=0
+bash "$VALIDATE" "$NO_BATCHES" 2>/dev/null || exit_code=$?
+assert_eq "no batches exits 1" "1" "$exit_code"
+
+# =============================================================================
+# Custom min-score
+# =============================================================================
+
+# High quality with min-score=101 should fail (max score is 100)
+exit_code=0
+bash "$VALIDATE" "$HIGH_QUALITY" --min-score 101 >/dev/null 2>&1 || exit_code=$?
+assert_eq "min-score 101 is impossible to pass" "1" "$exit_code"
+
+# High quality with min-score=1 should pass
+exit_code=0
+bash "$VALIDATE" "$HIGH_QUALITY" --min-score 1 >/dev/null 2>&1 || exit_code=$?
+assert_eq "min-score 1 is easy to pass" "0" "$exit_code"
+
+# =============================================================================
+# Dimension-specific tests
+# =============================================================================
+
+# Forward dependency reference
+FORWARD_REF="$TMPDIR_ROOT/forward-ref.md"
+cat > "$FORWARD_REF" << 'EOF'
+# Forward Reference Plan
+
+## Batch 1: First
+
+### Task 1: Setup
+
+**Files:**
+- Create: `src/setup.py`
+
+This task needs batch 2 to be done first. Check tests pass.
+
+## Batch 2: Second
+
+### Task 2: Build
+
+**Files:**
+- Create: `src/build.py`
+
+Build the thing. Verify it works.
+EOF
+
+json=$(bash "$VALIDATE" "$FORWARD_REF" --json 2>&1)
+dep_score=$(echo "$json" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['dimensions']['dependency_ordering']['score'])")
+assert_eq "forward ref lowers dependency score" "50" "$dep_score"
+
+# Oversized batch (> 5 tasks)
+BIG_BATCH="$TMPDIR_ROOT/big-batch.md"
+cat > "$BIG_BATCH" << 'EOF'
+# Big Batch Plan
+
+## Batch 1: Everything
+
+### Task 1: A
+Do A. Check it works.
+### Task 2: B
+Do B. Verify output.
+### Task 3: C
+Do C. Test the result.
+### Task 4: D
+Do D. Assert correctness.
+### Task 5: E
+Do E. Should pass all tests.
+### Task 6: F
+Do F. Confirm it works.
+### Task 7: G
+Do G. Must be correct.
+EOF
+
+json=$(bash "$VALIDATE" "$BIG_BATCH" --json 2>&1)
+size_score=$(echo "$json" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['dimensions']['batch_size']['score'])")
+assert_eq "oversized batch gets 0 on batch_size" "0" "$size_score"
+
+echo ""
+echo "Results: $((TESTS - FAILURES))/$TESTS passed"
+if [[ $FAILURES -gt 0 ]]; then
+    echo "FAILURES: $FAILURES"
+    exit 1
+fi
+echo "ALL PASSED"

package/scripts/tests/test-validate-plans.sh

@@ -0,0 +1,187 @@
+#!/usr/bin/env bash
+# Test validate-plans.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/test-helpers.sh"
+
+VALIDATOR="$SCRIPT_DIR/../validate-plans.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# Helper: create a plan file
+create_plan() {
+    local name="$1" content="$2"
+    mkdir -p "$WORK/plans"
+    printf '%s\n' "$content" > "$WORK/plans/$name"
+}
+
+# Helper: run validator against temp plans dir
+run_validator() {
+    local exit_code=0
+    PLANS_DIR="$WORK/plans" bash "$VALIDATOR" "$@" 2>&1 || exit_code=$?
+    echo "EXIT:$exit_code"
+}
+
+# === Test: Valid plan passes ===
+rm -rf "$WORK/plans"
+create_plan "2026-01-01-test-plan.md" '# Test Plan
+
+## Batch 1: Setup
+
+### Task 1: Do thing one
+
+Some content.
+
+### Task 2: Do thing two
+
+More content.
+
+## Batch 2: Implementation
+
+### Task 3: Do thing three
+
+Final content.'
+
+output=$(run_validator)
+assert_contains "valid plan: PASS" "validate-plans: PASS" "$output"
+assert_contains "valid plan: exit 0" "EXIT:0" "$output"
+
+# === Test: No batches fails (explicit file) ===
+rm -rf "$WORK/plans"
+create_plan "2026-01-01-no-batches.md" '# A Plan
+
+This plan has no batch headers at all.
+
+Just some text.'
+
+exit_code=0
+output=$(bash "$VALIDATOR" "$WORK/plans/2026-01-01-no-batches.md" 2>&1) || exit_code=$?
+output="${output}
+EXIT:${exit_code}"
+assert_contains "no batches: reports violation" "No batches found" "$output"
+assert_contains "no batches: exit 1" "EXIT:1" "$output"
+
+# === Test: Empty batch (no tasks) fails ===
+rm -rf "$WORK/plans"
+create_plan "2026-01-01-empty-batch.md" '# Plan
+
+## Batch 1: Setup
+
+### Task 1: Do something
+
+Content.
+
+## Batch 2: Empty
+
+No tasks here.
+
+## Batch 3: More
+
+### Task 2: Do another thing
+
+Content.'
+
+output=$(run_validator)
+assert_contains "empty batch: reports violation" "has no tasks" "$output"
+assert_contains "empty batch: exit 1" "EXIT:1" "$output"
+
+# === Test: Non-sequential batch numbers fails ===
+rm -rf "$WORK/plans"
+create_plan "2026-01-01-nonseq.md" '# Plan
+
+## Batch 1: First
+
+### Task 1: Do thing
+
+Content.
+
+## Batch 3: Skipped Two
+
+### Task 2: Do another
+
+Content.'
+
+output=$(run_validator)
+assert_contains "non-sequential: reports violation" "expected Batch 2" "$output"
+assert_contains "non-sequential: exit 1" "EXIT:1" "$output"
+
+# === Test: Design docs (no Batch headers) are skipped ===
+rm -rf "$WORK/plans"
+create_plan "2026-01-01-design.md" '# Design Doc
+
+This is a design document, not a plan. No Batch headers.'
+create_plan "2026-01-01-plan.md" '# Plan
+
+## Batch 1: Setup
+
+### Task 1: Do thing
+
+Content.'
+
+output=$(run_validator)
+assert_contains "design doc skipped: PASS" "validate-plans: PASS" "$output"
+assert_contains "design doc skipped: exit 0" "EXIT:0" "$output"
+
+# === Test: Single file argument validates just that file ===
+rm -rf "$WORK/plans"
+create_plan "2026-01-01-good.md" '# Plan
+
+## Batch 1: Setup
+
+### Task 1: Do thing
+
+Content.'
+
+exit_code=0
+output=$(bash "$VALIDATOR" "$WORK/plans/2026-01-01-good.md" 2>&1) || exit_code=$?
+output="${output}
+EXIT:${exit_code}"
+assert_contains "single file arg: PASS" "validate-plans: PASS" "$output"
+assert_contains "single file arg: exit 0" "EXIT:0" "$output"
+
+# === Test: --warn exits 0 even with violations ===
+rm -rf "$WORK/plans"
+create_plan "2026-01-01-bad.md" '# Plan
+
+## Batch 1: Setup
+
+No tasks here.'
+
+output=$(run_validator --warn)
+assert_contains "--warn: still reports violation" "has no tasks" "$output"
+assert_contains "--warn: exits 0" "EXIT:0" "$output"
+
+# === Test: --help exits 0 ===
+output=$(run_validator --help)
+assert_contains "--help: shows usage" "Usage:" "$output"
+assert_contains "--help: exits 0" "EXIT:0" "$output"
+
+# === Test: Task on next batch header line not counted for previous batch (#26) ===
+# This tests the sed range fix — when a "### Task" line is the first line of
+# the next batch (immediately after "## Batch N"), it must not be counted for
+# the previous batch.
+rm -rf "$WORK/plans"
+create_plan "2026-01-01-adjacent.md" '# Plan
+
+## Batch 1: Setup
+
+Some content but no tasks here.
+
+## Batch 2: Implementation
+
+### Task 1: The only task
+
+Content.'
+
+output=$(run_validator)
+assert_contains "adjacent batch: batch 1 has no tasks" "Batch 1 has no tasks" "$output"
+assert_contains "adjacent batch: FAIL" "FAIL" "$output"
+
+# === Test: Missing plans directory fails ===
+rm -rf "$WORK/plans"
+output=$(PLANS_DIR="$WORK/nonexistent" bash "$VALIDATOR" 2>&1 || echo "EXIT:$?")
+assert_contains "missing dir: error message" "plans directory not found" "$output"
+assert_contains "missing dir: exit 1" "EXIT:1" "$output"
+
+report_results

package/scripts/tests/test-validate-plugin.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Test validate-plugin.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/test-helpers.sh"
+
+VALIDATOR="$SCRIPT_DIR/../validate-plugin.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+
+# Helper: create plugin files
+create_plugin() {
+    local plugin_json="$1" marketplace_json="$2"
+    mkdir -p "$WORK/.claude-plugin"
+    echo "$plugin_json" > "$WORK/.claude-plugin/plugin.json"
+    echo "$marketplace_json" > "$WORK/.claude-plugin/marketplace.json"
+}
+
+# Helper: run validator against temp dir
+run_validator() {
+    local exit_code=0
+    PLUGIN_DIR="$WORK/.claude-plugin" bash "$VALIDATOR" "$@" 2>&1 || exit_code=$?
+    echo "EXIT:$exit_code"
+}
+
+# === Test: Matching name and version passes ===
+create_plugin \
+    '{"name":"my-toolkit","version":"1.0.0","description":"Test"}' \
+    '{"name":"my-toolkit","plugins":[{"name":"my-toolkit","version":"1.0.0"}]}'
+
+output=$(run_validator)
+assert_contains "matching: PASS" "validate-plugin: PASS" "$output"
+assert_contains "matching: exit 0" "EXIT:0" "$output"
+
+# === Test: Mismatched name fails ===
+create_plugin \
+    '{"name":"toolkit-a","version":"1.0.0","description":"Test"}' \
+    '{"name":"toolkit-b","plugins":[{"name":"toolkit-b","version":"1.0.0"}]}'
+
+output=$(run_validator)
+assert_contains "name mismatch: reports violation" "name mismatch" "$output"
+assert_contains "name mismatch: exit 1" "EXIT:1" "$output"
+
+# === Test: Mismatched version fails ===
+create_plugin \
+    '{"name":"my-toolkit","version":"1.0.0","description":"Test"}' \
+    '{"name":"my-toolkit","plugins":[{"name":"my-toolkit","version":"2.0.0"}]}'
+
+output=$(run_validator)
+assert_contains "version mismatch: reports violation" "version mismatch" "$output"
+assert_contains "version mismatch: exit 1" "EXIT:1" "$output"
+
+# === Test: Missing plugin.json fails ===
+rm -rf "$WORK/.claude-plugin"
+mkdir -p "$WORK/.claude-plugin"
+echo '{"name":"x","plugins":[{"name":"x","version":"1.0.0"}]}' > "$WORK/.claude-plugin/marketplace.json"
+
+output=$(run_validator)
+assert_contains "missing plugin.json: error" "plugin.json not found" "$output"
+assert_contains "missing plugin.json: exit 1" "EXIT:1" "$output"
+
+# === Test: Missing marketplace.json fails ===
+rm -rf "$WORK/.claude-plugin"
+mkdir -p "$WORK/.claude-plugin"
+echo '{"name":"x","version":"1.0.0"}' > "$WORK/.claude-plugin/plugin.json"
+
+output=$(run_validator)
+assert_contains "missing marketplace.json: error" "marketplace.json not found" "$output"
+assert_contains "missing marketplace.json: exit 1" "EXIT:1" "$output"
+
+# === Test: Invalid JSON in plugin.json fails ===
+create_plugin '{invalid json' '{"name":"x","plugins":[{"name":"x","version":"1.0.0"}]}'
+
+output=$(run_validator)
+assert_contains "invalid plugin.json: error" "plugin.json is not valid JSON" "$output"
+assert_contains "invalid plugin.json: exit 1" "EXIT:1" "$output"
+
+# === Test: Invalid JSON in marketplace.json fails ===
+create_plugin '{"name":"x","version":"1.0.0"}' '{invalid json'
+
+output=$(run_validator)
+assert_contains "invalid marketplace.json: error" "marketplace.json is not valid JSON" "$output"
+assert_contains "invalid marketplace.json: exit 1" "EXIT:1" "$output"
+
+# === Test: --warn exits 0 even with violations ===
+create_plugin \
+    '{"name":"a","version":"1.0.0"}' \
+    '{"name":"b","plugins":[{"name":"b","version":"1.0.0"}]}'
+
+output=$(run_validator --warn)
+assert_contains "--warn: still reports violation" "name mismatch" "$output"
+assert_contains "--warn: exits 0" "EXIT:0" "$output"
+
+# === Test: --help exits 0 ===
+output=$(run_validator --help)
+assert_contains "--help: shows usage" "Usage:" "$output"
+assert_contains "--help: exits 0" "EXIT:0" "$output"
+
+# === Test: Missing plugin directory fails ===
+rm -rf "$WORK/.claude-plugin"
+output=$(PLUGIN_DIR="$WORK/nonexistent" bash "$VALIDATOR" 2>&1 || echo "EXIT:$?")
+assert_contains "missing dir: error message" "plugin directory not found" "$output"
+assert_contains "missing dir: exit 1" "EXIT:1" "$output"
+
+report_results