agentic-loop 3.3.0 → 3.4.0

package/bin/ralph.sh

@@ -56,6 +56,7 @@ source "$RALPH_LIB/loop.sh"
 source "$RALPH_LIB/verify.sh"
 source "$RALPH_LIB/prd.sh"
 source "$RALPH_LIB/signs.sh"
+source "$RALPH_LIB/test.sh"
 
 # Run auto-config if config.json was just created
 if [[ "${_ralph_needs_autoconfig:-}" == "true" ]]; then
@@ -104,6 +105,12 @@ main() {
       fi
       run_verification "$1"
       ;;
+    test)
+      ralph_test "$@"
+      ;;
+    coverage)
+      ralph_test_coverage "$@"
+      ;;
     sign)
       ralph_sign "$@"
       ;;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentic-loop",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "description": "Autonomous AI coding loop - PRD-driven development with Claude Code",
   "author": "Allie Jones <allie@allthrive.ai>",
   "license": "MIT",

package/ralph/test.sh

@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+# shellcheck shell=bash
+# test.sh - Comprehensive test runner for nightly builds
+#
+# Runs full test suite + all PRD testSteps from completed stories.
+# Use this in nightly CI jobs, not on every PR.
+
+# Run comprehensive tests (for nightly CI)
+ralph_test() {
+  local mode="${1:-all}"
+
+  echo ""
+  print_info "=== Ralph Nightly Test Suite ==="
+  echo ""
+
+  local failed=0
+  local total=0
+  local passed=0
+
+  case "$mode" in
+    all)
+      run_full_test_suite || failed=1
+      run_all_prd_tests || failed=1
+      ;;
+    unit)
+      run_full_test_suite || failed=1
+      ;;
+    prd)
+      run_all_prd_tests || failed=1
+      ;;
+    *)
+      echo "Usage: ralph test [all|unit|prd]"
+      echo ""
+      echo "Modes:"
+      echo "  all   - Run unit tests + all PRD testSteps (default)"
+      echo "  unit  - Run only unit tests"
+      echo "  prd   - Run only PRD testSteps from completed stories"
+      return 1
+      ;;
+  esac
+
+  echo ""
+  if [[ $failed -eq 0 ]]; then
+    print_success "=== All nightly tests passed ==="
+    return 0
+  else
+    print_error "=== Nightly tests failed ==="
+    return 1
+  fi
+}
+
+# Run the full test suite
+run_full_test_suite() {
+  echo "--- Unit Tests ---"
+  echo ""
+
+  local test_cmd
+  test_cmd=$(get_config '.checks.testCommand' "")
+
+  if [[ -z "$test_cmd" ]]; then
+    # Auto-detect test command
+    if [[ -f "package.json" ]] && grep -q '"test"' package.json; then
+      test_cmd="npm test"
+    elif [[ -f "pytest.ini" ]] || [[ -f "pyproject.toml" ]]; then
+      test_cmd="pytest -v"
+    elif [[ -f "Cargo.toml" ]]; then
+      test_cmd="cargo test"
+    elif [[ -f "go.mod" ]]; then
+      test_cmd="go test -v ./..."
+    else
+      print_warning "No test command found, skipping unit tests"
+      return 0
+    fi
+  fi
+
+  echo "Running: $test_cmd"
+  echo ""
+
+  if eval "$test_cmd"; then
+    print_success "Unit tests passed"
+    return 0
+  else
+    print_error "Unit tests failed"
+    return 1
+  fi
+}
+
+# Run all PRD testSteps from all stories (completed and incomplete)
+run_all_prd_tests() {
+  echo ""
+  echo "--- PRD Test Steps ---"
+  echo ""
+
+  if [[ ! -f "$RALPH_DIR/prd.json" ]]; then
+    print_warning "No PRD found, skipping PRD tests"
+    return 0
+  fi
+
+  local failed=0
+  local total=0
+  local passed=0
+
+  # Get all stories
+  local stories
+  stories=$(jq -r '.stories[].id' "$RALPH_DIR/prd.json" 2>/dev/null)
+
+  if [[ -z "$stories" ]]; then
+    echo "No stories found in PRD"
+    return 0
+  fi
+
+  while IFS= read -r story_id; do
+    [[ -z "$story_id" ]] && continue
+
+    local story_title
+    story_title=$(jq -r --arg id "$story_id" '.stories[] | select(.id==$id) | .title' "$RALPH_DIR/prd.json")
+
+    echo "[$story_id] $story_title"
+
+    local test_steps
+    test_steps=$(jq -r --arg id "$story_id" '.stories[] | select(.id==$id) | .testSteps[]?' "$RALPH_DIR/prd.json" 2>/dev/null)
+
+    if [[ -z "$test_steps" ]]; then
+      echo "  (no testSteps)"
+      continue
+    fi
+
+    while IFS= read -r step; do
+      [[ -z "$step" ]] && continue
+      ((total++))
+
+      echo -n "  $step... "
+
+      if eval "$step" >/dev/null 2>&1; then
+        print_success "passed"
+        ((passed++))
+      else
+        print_error "failed"
+        ((failed++))
+      fi
+    done <<< "$test_steps"
+
+    echo ""
+  done <<< "$stories"
+
+  echo "PRD Tests: $passed/$total passed"
+
+  [[ $failed -gt 0 ]] && return 1
+  return 0
+}
+
+# Generate test coverage report
+ralph_test_coverage() {
+  echo ""
+  print_info "=== Test Coverage Report ==="
+  echo ""
+
+  # Python coverage
+  if [[ -f "pytest.ini" ]] || [[ -f "pyproject.toml" ]]; then
+    local backend_dir
+    backend_dir=$(get_config '.directories.backend' ".")
+
+    echo "Running pytest with coverage..."
+    if (cd "$backend_dir" && pytest --cov --cov-report=term-missing 2>/dev/null); then
+      return 0
+    else
+      print_warning "Coverage report failed (pytest-cov may not be installed)"
+      return 1
+    fi
+  fi
+
+  # JS/TS coverage
+  if [[ -f "package.json" ]] && grep -q '"test:coverage"' package.json; then
+    echo "Running npm test:coverage..."
+    npm run test:coverage
+    return $?
+  fi
+
+  print_warning "No coverage tool detected"
+  return 0
+}

package/ralph/verify/tests.sh

@@ -2,6 +2,106 @@
 # shellcheck shell=bash
 # tests.sh - Test verification module for ralph
 
+# Check that new/modified source files have corresponding test files
+# This catches the case where Claude writes code but forgets tests
+verify_test_files_exist() {
+  local story_type="${RALPH_STORY_TYPE:-general}"
+
+  # Skip for frontend stories (handled differently with .test.tsx pattern)
+  [[ "$story_type" == "frontend" ]] && return 0
+
+  echo -n "    Test files exist for new code... "
+
+  # Get list of modified Python files (excluding tests themselves)
+  local modified_files
+  modified_files=$(git diff --name-only HEAD~1 2>/dev/null | grep '\.py$' | grep -v 'test_' | grep -v '_test\.py' | grep -v '/tests/' || true)
+
+  # If no Python files modified, skip
+  if [[ -z "$modified_files" ]]; then
+    print_success "skipped (no new Python files)"
+    return 0
+  fi
+
+  local missing_tests=()
+  local checked=0
+
+  while IFS= read -r src_file; do
+    [[ -z "$src_file" ]] && continue
+    [[ ! -f "$src_file" ]] && continue
+
+    # Skip __init__.py, migrations, config files
+    [[ "$src_file" == *"__init__.py" ]] && continue
+    [[ "$src_file" == *"/migrations/"* ]] && continue
+    [[ "$src_file" == *"/alembic/"* ]] && continue
+    [[ "$src_file" == *"config"* ]] && continue
+    [[ "$src_file" == *"settings"* ]] && continue
+
+    ((checked++))
+
+    # Determine expected test file location
+    local base_name dir_name test_file
+    base_name=$(basename "$src_file" .py)
+    dir_name=$(dirname "$src_file")
+
+    # Common patterns: tests/test_foo.py or foo_test.py
+    local possible_tests=(
+      "$dir_name/tests/test_${base_name}.py"
+      "$dir_name/test_${base_name}.py"
+      "${dir_name}/tests/${base_name}_test.py"
+      "tests/test_${base_name}.py"
+      "tests/${base_name}_test.py"
+    )
+
+    # Check for backend dir patterns
+    local backend_dir
+    backend_dir=$(get_config '.directories.backend' "")
+    if [[ -n "$backend_dir" ]]; then
+      possible_tests+=(
+        "$backend_dir/tests/test_${base_name}.py"
+        "$backend_dir/tests/${base_name}_test.py"
+      )
+    fi
+
+    local found=false
+    for test_path in "${possible_tests[@]}"; do
+      if [[ -f "$test_path" ]]; then
+        found=true
+        break
+      fi
+    done
+
+    if [[ "$found" == "false" ]]; then
+      missing_tests+=("$src_file")
+    fi
+  done <<< "$modified_files"
+
+  if [[ ${#missing_tests[@]} -eq 0 ]]; then
+    print_success "passed ($checked files checked)"
+    return 0
+  else
+    print_error "missing tests"
+    echo ""
+    echo "    The following files need test files:"
+    for file in "${missing_tests[@]}"; do
+      local base_name
+      base_name=$(basename "$file" .py)
+      echo "      $file → test_${base_name}.py"
+    done
+    echo ""
+    echo "    Create test files for new code before completing the story."
+
+    # Save for failure context
+    {
+      echo "Missing test files for new code:"
+      for file in "${missing_tests[@]}"; do
+        echo "  $file"
+      done
+    } > "$RALPH_DIR/last_test_existence_failure.log"
+
+    return 1
+  fi
+}
+
 # Run unit tests
 run_unit_tests() {
   local log_file

package/ralph/verify.sh

@@ -33,12 +33,15 @@ run_verification() {
   fi
 
   # ========================================
-  # STEP 2: Run unit tests
+  # STEP 2: Verify tests exist + run them
   # ========================================
   if [[ $failed -eq 0 ]]; then
     echo ""
     echo "  [2/3] Running tests..."
-    if ! run_unit_tests; then
+    # First check that test files exist for new code
+    if ! verify_test_files_exist; then
+      failed=1
+    elif ! run_unit_tests; then
       failed=1
     fi
   fi

package/templates/PROMPT.md

@@ -22,9 +22,21 @@ For each story, you must:
 
 ### 2. Write Tests
 
-- Write unit tests for the business logic
-- Write tests that validate acceptance criteria
-- Test error cases and edge cases
+**Every new code file MUST have a corresponding test file.**
+
+For **backend** stories (Python/API):
+- New file `foo.py` → create `tests/test_foo.py`
+- Test each public function/method
+- Test error cases (invalid input, missing data, API failures)
+- Test edge cases (empty lists, None values, boundary conditions)
+- Use pytest fixtures for database/API mocking
+
+For **frontend** stories (TypeScript/React):
+- New component `Foo.tsx` → create `Foo.test.tsx`
+- Test rendering, user interactions, error states
+- Test loading states and empty states
+
+**Do NOT skip tests.** If you create code without tests, verification will fail.
 
 ### 3. Verify It Actually Works