agentic-loop 3.19.0 → 3.21.0

package/ralph/utils.sh

@@ -2,6 +2,18 @@
 # shellcheck shell=bash
 # utils.sh - Shared utility functions for ralph
 
+# Get utils.sh directory to locate package.json
+_UTILS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+_PACKAGE_JSON="$_UTILS_DIR/../package.json"
+
+# Version constant (read from package.json)
+RALPH_VERSION="$(jq -r '.version' "$_PACKAGE_JSON" 2>/dev/null || echo "unknown")"
+readonly RALPH_VERSION
+
+# Loop protocol version (for stream-json activity feed)
+RALPH_LOOP_VERSION="2"
+readonly RALPH_LOOP_VERSION
+
 # Constants - Output limits
 readonly MAX_LOG_LINES=30
 readonly MAX_PROGRESS_LINES=10
@@ -16,7 +28,6 @@ readonly MAX_SIGN_DEDUP_EXISTING=20
 # Constants - Timeouts (centralized to avoid magic numbers)
 readonly ITERATION_DELAY_SECONDS=0
 readonly DEFAULT_TIMEOUT_SECONDS=600
-readonly DEFAULT_MAX_ITERATIONS=20
 readonly CODE_REVIEW_TIMEOUT_SECONDS=120
 readonly BROWSER_TIMEOUT_SECONDS=60
 readonly BROWSER_PAGE_TIMEOUT_MS=30000
@@ -140,38 +151,24 @@ get_config() {
   echo "$default"
 }
 
-# Get a field from a story in prd.json
-# Usage: get_story_field "STORY-001" ".type" "general"
-get_story_field() {
-  local story_id="$1"
-  local field="$2"
-  local default="${3:-}"
-  local prd="$RALPH_DIR/prd.json"
-
-  if [[ -f "$prd" ]]; then
-    local value
-    value=$(jq -r --arg id "$story_id" ".stories[] | select(.id==\$id) | $field // empty" "$prd" 2>/dev/null)
-    if [[ -n "$value" && "$value" != "null" ]]; then
-      echo "$value"
-      return
+# Migrate deprecated config fields in-place
+# Safe to call multiple times - only writes if changes are needed
+_migrate_config() {
+  local config="$RALPH_DIR/config.json"
+  [[ ! -f "$config" ]] && return 0
+
+  # testUrlBase -> urls.frontend
+  local legacy_url
+  legacy_url=$(jq -r '.testUrlBase // empty' "$config" 2>/dev/null)
+  if [[ -n "$legacy_url" ]]; then
+    local current_frontend
+    current_frontend=$(jq -r '.urls.frontend // empty' "$config" 2>/dev/null)
+    if [[ -z "$current_frontend" ]]; then
+      jq --arg url "$legacy_url" '.urls.frontend = $url | del(.testUrlBase)' "$config" > "${config}.tmp" && mv "${config}.tmp" "$config"
+    else
+      jq 'del(.testUrlBase)' "$config" > "${config}.tmp" && mv "${config}.tmp" "$config"
     fi
   fi
-  echo "$default"
-}
-
-# Clear a failure log file
-# Usage: clear_failure_log "lint"  # clears last_lint_failure.log
-clear_failure_log() {
-  local log_name="$1"
-  rm -f "$RALPH_DIR/last_${log_name}_failure.log"
-}
-
-# Append content to a failure log file
-# Usage: log_failure "lint" "Error details here"
-log_failure() {
-  local log_name="$1"
-  local content="$2"
-  echo "$content" >> "$RALPH_DIR/last_${log_name}_failure.log"
 }
 
 # Deep merge user config with project config
@@ -431,33 +428,17 @@ validate_command() {
 
   # Block obviously dangerous patterns (defense-in-depth, not security boundary)
   local dangerous_patterns=(
-    # Destructive file operations
     'rm[[:space:]]+-rf[[:space:]]+/'              # rm -rf /
     'rm[[:space:]]+-rf[[:space:]]+~'              # rm -rf ~ (home dir)
-    'rm[[:space:]]+-rf[[:space:]]+\*'             # rm -rf *
-    'rm[[:space:]]+-rf[[:space:]]+\.\.'           # rm -rf ..
     'rm[[:space:]].*--no-preserve-root'           # rm with --no-preserve-root
-    # Remote code execution
     'curl.*\|.*bash'                               # curl | bash
     'curl.*\|.*sh[[:space:]]*$'                    # curl | sh
     'wget.*\|.*bash'                               # wget | bash
     'wget.*\|.*sh[[:space:]]*$'                    # wget | sh
-    'curl.*>[[:space:]]*/tmp/.*&&.*bash'           # curl > /tmp/x && bash
-    # Code injection
-    '\$\([^)]*eval'                                # $(eval ...)
     'eval[[:space:]]+\$'                           # eval $var
-    'eval[[:space:]]+["\x27]'                      # eval "..." or eval '...'
-    # System damage
     '>[[:space:]]*/dev/sd'                         # write to disk devices
     '>[[:space:]]*/dev/nvme'                       # write to nvme devices
     'mkfs\.'                                       # format filesystems
-    'dd[[:space:]]+if='                            # dd commands
-    ':(){:|:&};:'                                  # fork bomb
-    # Credential theft
-    'cat.*\.ssh/id_'                               # read SSH keys
-    'cat.*/etc/shadow'                             # read shadow file
-    'cat.*\.aws/credentials'                       # read AWS creds
-    'cat.*\.env'                                   # read env files (often has secrets)
   )
 
   for pattern in "${dangerous_patterns[@]}"; do
@@ -471,25 +452,6 @@ validate_command() {
   return 0
 }
 
-# Validate a URL is safe (http/https only, no internal IPs in production)
-validate_url() { # public-api
-  local url="$1"
-
-  # Must start with http:// or https://
-  if [[ ! "$url" =~ ^https?:// ]]; then
-    print_error "Invalid URL scheme (must be http or https): $url"
-    return 1
-  fi
-
-  # Block file:// and other dangerous schemes
-  if [[ "$url" =~ ^(file|ftp|data|javascript): ]]; then
-    print_error "Dangerous URL scheme: $url"
-    return 1
-  fi
-
-  return 0
-}
-
 # Safely execute a command (validates first, uses bash -c instead of eval)
 safe_exec() {
   local cmd="$1"
@@ -710,6 +672,92 @@ fix_hardcoded_paths() {
   fi
 }
 
+# Detect the type of project based on files present
+# Returns one of: fullstack, rust, go, elixir, hugo, fastmcp, django, fastapi, python, node, minimal
+detect_project_type() {
+  # Check for fullstack patterns first (more specific)
+  if [[ -d "frontend" && -d "core" ]]; then
+    echo "fullstack"; return
+  elif [[ -d "frontend" && -d "backend" ]]; then
+    echo "fullstack"; return
+  elif [[ -d "apps" ]]; then
+    echo "fullstack"; return
+  fi
+
+  # Single-language projects
+  if [[ -f "Cargo.toml" ]]; then echo "rust"; return; fi
+  if [[ -f "go.mod" ]]; then echo "go"; return; fi
+  if [[ -f "mix.exs" ]]; then echo "elixir"; return; fi
+
+  # Hugo (Go static site generator)
+  for _hc in "hugo.toml" "hugo.yaml" "hugo.json" "config.toml"; do
+    if [[ -f "$_hc" ]] && [[ -d "content" || -d "layouts" || -d "themes" ]]; then
+      echo "hugo"; return
+    fi
+  done
+
+  # Python framework variants (most specific first)
+  if [[ -f "pyproject.toml" ]]; then
+    if grep -qiE "(fastmcp|\"fastmcp\"|'fastmcp')" pyproject.toml 2>/dev/null; then
+      echo "fastmcp"; return
+    elif grep -qiE "(django|\"django\"|'django')" pyproject.toml 2>/dev/null || [[ -f "manage.py" ]]; then
+      echo "django"; return
+    elif grep -qiE "(fastapi|\"fastapi\"|'fastapi')" pyproject.toml 2>/dev/null; then
+      echo "fastapi"; return
+    else
+      echo "python"; return
+    fi
+  elif [[ -f "requirements.txt" || -f "setup.py" ]]; then
+    if [[ -f "requirements.txt" ]]; then
+      if grep -qi 'fastmcp' requirements.txt 2>/dev/null; then echo "fastmcp"; return; fi
+      if grep -qi 'django' requirements.txt 2>/dev/null || [[ -f "manage.py" ]]; then echo "django"; return; fi
+      if grep -qi 'fastapi' requirements.txt 2>/dev/null; then echo "fastapi"; return; fi
+    fi
+    echo "python"; return
+  elif [[ -f "manage.py" ]]; then
+    echo "django"; return
+  fi
+
+  if [[ -f "package.json" ]]; then echo "node"; return; fi
+
+  echo "minimal"
+}
+
+# Detect framework type for CLAUDE.md generation
+# Returns one of: fastmcp, fastapi, django, react, or empty string
+detect_framework_type() {
+  if [[ -f "pyproject.toml" ]]; then
+    if grep -qiE "(fastmcp|\"fastmcp\"|'fastmcp')" pyproject.toml 2>/dev/null; then
+      echo "fastmcp"; return
+    elif grep -qiE "(fastapi|\"fastapi\"|'fastapi')" pyproject.toml 2>/dev/null; then
+      echo "fastapi"; return
+    elif grep -qiE "(django|\"django\"|'django')" pyproject.toml 2>/dev/null || [[ -f "manage.py" ]]; then
+      echo "django"; return
+    fi
+  elif [[ -f "requirements.txt" ]]; then
+    if grep -qi 'fastmcp' requirements.txt 2>/dev/null; then echo "fastmcp"; return; fi
+    if grep -qi 'fastapi' requirements.txt 2>/dev/null; then echo "fastapi"; return; fi
+    if grep -qi 'django' requirements.txt 2>/dev/null || [[ -f "manage.py" ]]; then echo "django"; return; fi
+  elif [[ -f "manage.py" ]]; then
+    echo "django"; return
+  fi
+
+  # Check for React in frontend package.json
+  local pkg="package.json"
+  local fe_dir=""
+  [[ -d "frontend" ]] && fe_dir="frontend"
+  [[ -d "client" ]] && fe_dir="client"
+  [[ -d "web" ]] && fe_dir="web"
+  [[ -d "apps/web" ]] && fe_dir="apps/web"
+  [[ -n "$fe_dir" && -f "${fe_dir}/package.json" ]] && pkg="${fe_dir}/package.json"
+
+  if [[ -f "$pkg" ]] && grep -q '"react"' "$pkg" 2>/dev/null; then
+    echo "react"; return
+  fi
+
+  echo ""
+}
+
 # Detect Python runner (uv, poetry, pipenv, or plain python)
 detect_python_runner() {
   local search_dir="${1:-.}"
@@ -732,7 +780,7 @@ detect_python_runner() {
     return 0
   fi
 
-  # Default to plain command (assumes activated venv or global)
+  # Default to plain command (no runner detected)
   echo ""
   return 0
 }
@@ -761,9 +809,11 @@ detect_migration_tool() {
     return 0
   fi
 
-  # Django
+  # Django (use python3 for macOS compatibility when no runner detected)
   if [[ -f "$search_dir/manage.py" ]] && [[ -d "$search_dir" ]] && find "$search_dir" -type d -name "migrations" -print -quit | grep -q .; then
-    echo "cd $search_dir && ${py_runner}${py_runner:+ }python manage.py migrate"
+    local python_cmd="python3"
+    [[ -n "$py_runner" ]] && python_cmd="python"
+    echo "cd $search_dir && ${py_runner}${py_runner:+ }${python_cmd} manage.py migrate"
     return 0
   fi
 
@@ -812,6 +862,85 @@ find_all_migration_tools() {
   printf '%s\n' "${tools[@]}" | sort -u
 }
 
+# Validate batch assignments in a PRD file
+# Returns 0 if valid (or no batches), 1 with error messages if invalid
+# Checks: all-or-none batch field, dependency ordering, file overlap within batch
+validate_batch_assignments() {
+  local prd_file="$1"
+  local errors=""
+  local error_count=0
+
+  # Count stories with and without batch
+  local total_stories with_batch without_batch
+  total_stories=$(jq '.stories | length' "$prd_file" 2>/dev/null || echo "0")
+  with_batch=$(jq '[.stories[] | select(.batch != null)] | length' "$prd_file" 2>/dev/null || echo "0")
+  without_batch=$((total_stories - with_batch))
+
+  # No batches at all — valid, nothing to check
+  [[ "$with_batch" -eq 0 ]] && return 0
+
+  # Some but not all stories have batch — error
+  if [[ "$without_batch" -gt 0 ]]; then
+    local missing_ids
+    missing_ids=$(jq -r '[.stories[] | select(.batch == null) | .id] | join(", ")' "$prd_file" 2>/dev/null)
+    errors+="batch field missing on: $missing_ids\n"
+    error_count=$((error_count + 1))
+  fi
+
+  # Check dependency ordering: no dependsOn pointing to same or later batch
+  local dep_violations
+  dep_violations=$(jq -r '
+    .stories as $all |
+    [.stories[] | . as $s |
+      ($s.dependsOn // [])[] as $dep |
+      ($all[] | select(.id == $dep)) as $dep_story |
+      select($dep_story.batch != null and $s.batch != null and $dep_story.batch >= $s.batch) |
+      "\($s.id) (batch \($s.batch)) depends on \($dep) (batch \($dep_story.batch))"
+    ] | .[]' "$prd_file" 2>/dev/null)
+
+  if [[ -n "$dep_violations" ]]; then
+    while IFS= read -r violation; do
+      [[ -z "$violation" ]] && continue
+      errors+="$violation\n"
+      error_count=$((error_count + 1))
+    done <<< "$dep_violations"
+  fi
+
+  # Check file overlap within same batch (create/modify only)
+  local file_overlaps
+  file_overlaps=$(jq -r '
+    . as $root |
+    [$root.stories[] | select(.batch != null) | .batch] | unique | .[] as $b |
+    [$root.stories[] | select(.batch == $b)] |
+    if length < 2 then empty
+    else
+      . as $stories |
+      range(length) as $i | range($i+1; length) as $j |
+      $stories[$i] as $a | $stories[$j] as $b_story |
+      (($a.files.create // []) + ($a.files.modify // [])) as $files_a |
+      (($b_story.files.create // []) + ($b_story.files.modify // [])) as $files_b |
+      ($files_a | map(select(. as $f | $files_b | any(. == $f)))) as $shared |
+      select(($shared | length) > 0) |
+      "batch \($b): \($a.id) and \($b_story.id) share files: \($shared | join(", "))"
+    end
+  ' "$prd_file" 2>/dev/null)
+
+  if [[ -n "$file_overlaps" ]]; then
+    while IFS= read -r overlap; do
+      [[ -z "$overlap" ]] && continue
+      errors+="$overlap\n"
+      error_count=$((error_count + 1))
+    done <<< "$file_overlaps"
+  fi
+
+  if [[ $error_count -gt 0 ]]; then
+    echo -e "$errors"
+    return 1
+  fi
+
+  return 0
+}
+
 # Ensure database migrations are applied before verification
 # Migration commands are idempotent - they no-op if nothing pending
 run_migrations_if_needed() {

package/ralph/verify/tests.sh

@@ -195,7 +195,9 @@ run_unit_tests() {
     if [[ -f "package.json" ]] && grep -q '"test"' package.json; then
       test_cmd="npm test"
     elif [[ -f "pytest.ini" ]] || [[ -f "pyproject.toml" ]]; then
-      test_cmd="pytest"
+      local py_runner
+      py_runner=$(detect_python_runner ".")
+      test_cmd="${py_runner}${py_runner:+ }pytest"
     elif [[ -f "Cargo.toml" ]]; then
       test_cmd="cargo test"
     elif [[ -f "go.mod" ]]; then
@@ -217,8 +219,35 @@ run_unit_tests() {
   else
     print_error "failed"
     echo ""
-    echo "    Output (last $MAX_LOG_LINES lines):"
-    tail -"$MAX_LOG_LINES" "$log_file" | sed 's/^/      /'
+
+    # Check for missing tool (uv, poetry, pytest, etc.)
+    if grep -qiE "command not found|no such file or directory" "$log_file" 2>/dev/null; then
+      local missing_tool=""
+      if grep -qi "uv.*command not found\|uv:.*not found" "$log_file" 2>/dev/null; then
+        missing_tool="uv"
+      elif grep -qi "poetry.*command not found\|poetry:.*not found" "$log_file" 2>/dev/null; then
+        missing_tool="poetry"
+      elif grep -qi "pytest.*command not found\|pytest:.*not found" "$log_file" 2>/dev/null; then
+        missing_tool="pytest"
+      fi
+
+      if [[ -n "$missing_tool" ]]; then
+        echo "    '$missing_tool' is not installed."
+        echo ""
+        echo "    Run setup to auto-detect and configure your project tools:"
+        echo "      npx agentic-loop setup"
+        echo ""
+        echo "    See Step 3 of the Getting Started guide:"
+        echo "      docs/GETTING-STARTED.md"
+      else
+        echo "    Output (last $MAX_LOG_LINES lines):"
+        tail -"$MAX_LOG_LINES" "$log_file" | sed 's/^/      /'
+      fi
+    else
+      echo "    Output (last $MAX_LOG_LINES lines):"
+      tail -"$MAX_LOG_LINES" "$log_file" | sed 's/^/      /'
+    fi
+
     cp "$log_file" "$RALPH_DIR/last_test_failure.log"
     rm -f "$log_file"
     return 1
@@ -228,9 +257,9 @@ run_unit_tests() {
 # Expand config placeholders in a string
 # Usage: _expand_config_vars "curl {config.urls.backend}/api"
 # Expands any {config.X.Y} placeholder from .ralph/config.json via jq.
-# Known placeholders have fallback paths for backward compatibility:
+# Known placeholders with fallback paths:
 #   {config.urls.backend}  -> .urls.backend // .api.baseUrl
-#   {config.urls.frontend} -> .urls.frontend // .testUrlBase
+#   {config.urls.frontend} -> .urls.frontend
 _expand_config_vars() {
   local input="$1"
   local config="$RALPH_DIR/config.json"
@@ -249,7 +278,7 @@ _expand_config_vars() {
 
   if [[ "$result" == *"{config.urls.frontend}"* ]]; then
     local val
-    val=$(jq -r '.urls.frontend // .testUrlBase // empty' "$config" 2>/dev/null)
+    val=$(jq -r '.urls.frontend // empty' "$config" 2>/dev/null)
     [[ -n "$val" ]] && result="${result//\{config.urls.frontend\}/$val}"
   fi
 
@@ -276,6 +305,17 @@ _expand_config_vars() {
   echo "$result"
 }
 
+# Check if a command string contains unresolved auth placeholder variables
+# Returns 0 (true) if auth placeholders found, 1 (false) if clean
+_has_auth_placeholder() {
+  local cmd="$1"
+  local auth_vars='TOKEN|API_KEY|JWT|AUTH_TOKEN|BEARER_TOKEN|ACCESS_TOKEN|SECRET|PASSWORD|CREDENTIALS|API_SECRET|AUTH'
+  [[ "$cmd" =~ \$($auth_vars)[^A-Za-z_] ]] && return 0
+  [[ "$cmd" =~ \$($auth_vars)$ ]] && return 0
+  [[ "$cmd" =~ \$\{($auth_vars)\} ]] && return 0
+  return 1
+}
+
 # Verify PRD acceptance criteria / test steps
 verify_prd_criteria() {
   local story="$1"
@@ -296,13 +336,21 @@ verify_prd_criteria() {
   # Clear previous PRD failure log
   rm -f "$prd_failure_log"
 
-  local step_index=0
+  local step_index=-1
   while IFS= read -r step; do
     [[ -z "$step" ]] && continue
 
     # Expand config placeholders (e.g., {config.urls.backend})
     local expanded_step
     expanded_step=$(_expand_config_vars "$step")
+    ((step_index++)) || true
+
+    # Skip steps with unresolved auth placeholders — don't fail, just warn
+    if _has_auth_placeholder "$expanded_step"; then
+      echo -n "    $expanded_step... "
+      print_warning "skipped (uses auth placeholder variable — set a real value in env or config)"
+      continue
+    fi
 
     echo -n "    $expanded_step... "
 
@@ -311,8 +359,16 @@ verify_prd_criteria() {
     else
       print_error "failed"
       echo ""
-      echo "    Output:"
-      tail -"$MAX_OUTPUT_PREVIEW_LINES" "$log_file" | sed 's/^/      /'
+
+      # Check for connection refused — give actionable guidance
+      if grep -qiE "connection refused|couldn.t connect|failed to connect" "$log_file" 2>/dev/null; then
+        local dev_cmd
+        dev_cmd=$(get_config '.commands.dev' "docker compose up -d")
+        echo "    Server not running. Start it before running Ralph: $dev_cmd"
+      else
+        echo "    Output:"
+        tail -"$MAX_OUTPUT_PREVIEW_LINES" "$log_file" | sed 's/^/      /'
+      fi
 
       # Save failure details for retry context
       {
@@ -325,7 +381,6 @@ verify_prd_criteria() {
 
       failed=1
     fi
-    ((step_index++)) || true
   done <<< "$test_steps"
 
   rm -f "$log_file"

package/templates/PROMPT.md

@@ -158,6 +158,8 @@ After completing the story:
    - No commented-out code
    - All tests passing
 
+4. **Then stop.** Your session should end here. Ralph will automatically run verification (lint, tests, build) and mark the story as passed or failed. You do not need to mark anything — just finish implementing and stop.
+
 ---
 
 ## Rules
@@ -166,18 +168,22 @@ After completing the story:
 2. **Follow the PRD** - It has all the context you need
 3. **Read before coding** - Understand existing patterns first
 4. **Test frequently** - Run tests after each significant change
-5. **NEVER edit prd.json** - Ralph handles story completion
+5. **Don't edit prd.json** - You can read it for context, but Ralph manages story state (passes, retryCount, skipped). After your session ends, Ralph runs verification and marks the story. Editing these fields has no effect — Ralph resets them before verifying.
 6. **Don't give up** - If verification fails, fix and retry
 
 ---
 
 ## If Blocked
 
-If you encounter a blocker you cannot resolve:
+If you encounter a blocker you cannot resolve after 2-3 attempts:
+
 1. Document the issue in `.ralph/progress.txt`
 2. Note what you tried and why it didn't work
-3. Suggest potential solutions
-4. Do NOT mark the story as passing
+3. Signal Ralph to skip this story:
+   ```bash
+   echo "BLOCKED: [reason]" > .ralph/.blocked
+   ```
+4. Then stop. Ralph will stop the loop so the issue can be resolved.
 
 ---