@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.4

package/eval/scaffold/shell/run-claude-loop.sh

@@ -0,0 +1,181 @@
+#!/usr/bin/env bash
+# Multi-turn claude driver for interactive workflow eval.
+#
+# Runs the primary (subject) claude with the task prompt, then when it pauses
+# at a decision point (asks the user a question), drives a lightweight
+# "user-simulator" turn to answer, feeding the answer back via --resume.
+# Both calls share the same container HOME so session state persists.
+#
+# Usage: run-claude-loop.sh <prompt|@prompt-file> [--max-turns N] [--model MODEL]
+#   --max-turns N             Maximum number of subject<->simulator round trips (default 12)
+#   --model MODEL             Model for both subject and simulator
+#   --simulator-prompt-file   File containing the simulator system prompt
+#   --continue-prompt TEXT    Nudge used when the workflow should continue
+#   --decision-pattern TEXT   Extra case-insensitive substring to treat as a decision point
+#
+# Env: ANTHROPIC_AUTH_TOKEN / ANTHROPIC_BASE_URL / ANTHROPIC_MODEL for the proxy.
+# Stdout: concatenated stream-json from every subject turn (for event extraction).
+# Stderr: driver progress log.
+
+set -uo pipefail
+
+PROMPT_ARG="${1:?usage: run-claude-loop.sh <prompt|@prompt-file> [--max-turns N]}"
+shift || true
+
+if [[ "$PROMPT_ARG" == @* ]]; then
+    PROMPT="$(cat "${PROMPT_ARG#@}")"
+else
+    PROMPT="$PROMPT_ARG"
+fi
+
+MAX_TURNS=12
+MODEL="${ANTHROPIC_MODEL:-}"
+SIMULATOR_PROMPT=""
+CONTINUE_PROMPT="Please continue with the next phase of the comet workflow."
+DECISION_PATTERNS=()
+PLUGIN_ARGS=()
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --max-turns) MAX_TURNS="$2"; shift 2 ;;
+        --model) MODEL="$2"; shift 2 ;;
+        --plugin-dir)
+            PLUGIN_ARGS+=(--plugin-dir "$2")
+            shift 2
+            ;;
+        --simulator-prompt-file)
+            SIMULATOR_PROMPT="$(cat "$2")"
+            shift 2
+            ;;
+        --continue-prompt) CONTINUE_PROMPT="$2"; shift 2 ;;
+        --decision-pattern)
+            DECISION_PATTERNS+=("$2")
+            shift 2
+            ;;
+        *) shift ;;
+    esac
+done
+
+MODEL_FLAG=()
+[[ -n "$MODEL" ]] && MODEL_FLAG=(--model "$MODEL")
+
+# Detect whether the last subject turn is waiting on the user. comet decision
+# points present as a question / confirmation request with no pending tool call.
+is_decision_point() {
+    local text="$1"
+    local pattern
+    for pattern in "${DECISION_PATTERNS[@]}"; do
+        if [[ -n "$pattern" ]] && echo "$text" | grep -qiF -- "$pattern"; then
+            return 0
+        fi
+    done
+    # Heuristics: question marks, explicit "confirm/choose/proceed/continue" asks.
+    if echo "$text" | grep -qiE '\?|confirm|choose|proceed|continue|approve|select an option|which (option|approach|name)|enter the|provide|would you|shall we|do you want|preferred'; then
+        return 0
+    fi
+    return 1
+}
+
+# Drive the user-simulator: given the subject's last message, produce a concise
+# affirmative/clarifying reply that lets the workflow proceed.
+simulate_user() {
+    local subject_text="$1"
+    local sim_prompt
+    if [[ -n "$SIMULATOR_PROMPT" ]]; then
+        sim_prompt=$(cat <<EOF
+${SIMULATOR_PROMPT}
+
+Assistant's message:
+"""
+${subject_text:0:3000}
+"""
+EOF
+)
+    else
+        sim_prompt=$(cat <<EOF
+You are simulating a developer user in an automated eval. The AI assistant below is running the Comet development workflow and has paused to ask you something. Read its message and reply with a SHORT (1-3 sentences) response that:
+- Approves the proposed approach / name / plan when asked to confirm
+- Picks the most reasonable default option when asked to choose
+- Asks for clarification only if the question is truly ambiguous about WHAT to build
+Never refuse; always let the workflow move forward. Do not write code or files.
+
+Assistant's message:
+"""
+${subject_text:0:3000}
+"""
+EOF
+)
+    fi
+    # Run the simulator as a one-shot print call (separate session).
+    claude -p "$sim_prompt" "${MODEL_FLAG[@]}" --dangerously-skip-permissions 2>/dev/null
+}
+
+SESSION_ID=""
+COMBINED_OUT=""
+TURN=0
+
+while [[ $TURN -lt $MAX_TURNS ]]; do
+    TURN=$((TURN + 1))
+    echo "[loop] turn $TURN/$MAX_TURNS" >&2
+
+    if [[ -z "$SESSION_ID" ]]; then
+        # First turn: start fresh with the task prompt.
+        RAW=$(claude -p "$PROMPT" "${PLUGIN_ARGS[@]}" "${MODEL_FLAG[@]}" \
+            --output-format stream-json --verbose \
+            --dangerously-skip-permissions 2>/dev/null) || RAW=""
+    else
+        # Subsequent turns: resume the session with the simulated user reply.
+        RAW=$(claude -p "$USER_REPLY" "${PLUGIN_ARGS[@]}" "${MODEL_FLAG[@]}" \
+            --resume "$SESSION_ID" \
+            --output-format stream-json --verbose \
+            --dangerously-skip-permissions 2>/dev/null) || RAW=""
+    fi
+
+    COMBINED_OUT="${COMBINED_OUT}${RAW}"$'\n'
+
+    # Extract session id and the final assistant text from this turn.
+    SESSION_ID=$(echo "$RAW" | grep -oE '"session_id":\s*"[^"]+"' | head -1 | sed -E 's/.*"session_id":\s*"([^"]+)".*/\1/') || true
+
+    # Pull the result text (type=result) for decision-point detection.
+    RESULT_TEXT=$(echo "$RAW" | grep '"type": *"result"' | tail -1 | python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('result',''))" 2>/dev/null || echo "")
+    # Fallback: last assistant text block.
+    if [[ -z "$RESULT_TEXT" ]]; then
+        RESULT_TEXT=$(echo "$RAW" | grep '"type": *"assistant"' | tail -1 | python3 -c "
+import json,sys
+try:
+    d=json.loads(sys.stdin.read())
+    c=d.get('message',{}).get('content',[])
+    print(' '.join(i.get('text','') for i in c if isinstance(i,dict) and i.get('type')=='text'))
+except: print('')
+" 2>/dev/null || echo "")
+    fi
+
+    if [[ -z "$RESULT_TEXT" ]]; then
+        echo "[loop] no result text; ending" >&2
+        break
+    fi
+
+    if is_decision_point "$RESULT_TEXT"; then
+        echo "[loop] decision point detected; simulating user reply" >&2
+        USER_REPLY=$(simulate_user "$RESULT_TEXT")
+        if [[ -z "$USER_REPLY" ]]; then
+            USER_REPLY="Yes, please proceed with the recommended option."
+        fi
+        echo "[loop] simulated reply (${#USER_REPLY} chars)" >&2
+        continue
+    fi
+
+    # Not a decision point: check if workflow looks complete.
+    if echo "$RESULT_TEXT" | grep -qiE 'archive(d)? complete|workflow complete|change archived|all (5|five) phases|finished|done'; then
+        echo "[loop] workflow appears complete; ending" >&2
+        break
+    fi
+
+    # If the result has no question and isn't complete, the subject likely finished
+    # its turn naturally — try one more nudge to keep going, then stop.
+    echo "[loop] no decision point and not complete; nudging to continue" >&2
+    USER_REPLY="$CONTINUE_PROMPT"
+done
+
+echo "[loop] finished after $TURN turns" >&2
+# Emit the combined stream-json on stdout for the harness to parse.
+printf '%s' "$COMBINED_OUT"

package/eval/scaffold/shell/setup.sh

@@ -0,0 +1,259 @@
+#!/usr/bin/env bash
+# Test environment setup utilities
+# Source this file or run directly
+#
+# Commands:
+#   ./setup.sh verify <env_dir> [required_files...]
+#   ./setup.sh create-temp [prefix]
+#   ./setup.sh cleanup <temp_dir>
+#   ./setup.sh write-skill <test_dir> <skill_name> <content_file> [scripts_dir] [source_dir]
+#   ./setup.sh write-claude-md <test_dir> <content_file>
+#   ./setup.sh copy-env <test_dir> <env_dir>
+
+set -eo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/common.sh"
+source "$SCRIPT_DIR/docker.sh"
+
+# =============================================================================
+# VERIFICATION
+# =============================================================================
+
+verify_environment() {
+    local env_dir="$1"
+    shift
+    local required_files=("$@")
+    local errors=()
+
+    # Default required files
+    if [[ ${#required_files[@]} -eq 0 ]]; then
+        required_files=("Dockerfile" "requirements.txt")
+    fi
+
+    # Check Claude CLI
+    if ! command -v claude &> /dev/null; then
+        errors+=("Claude Code CLI not found. Install from: https://claude.ai/code")
+    fi
+
+    # Check Docker
+    if ! check_docker; then
+        errors+=("Docker not available")
+    fi
+
+    # Check API keys
+    if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then
+        errors+=("ANTHROPIC_API_KEY not set")
+    fi
+    if [[ -z "${OPENAI_API_KEY:-}" ]]; then
+        errors+=("OPENAI_API_KEY not set")
+    fi
+
+    # Check environment directory
+    if [[ ! -d "$env_dir" ]]; then
+        errors+=("Environment directory not found: $env_dir")
+    else
+        for file in "${required_files[@]}"; do
+            if [[ ! -f "$env_dir/$file" ]]; then
+                errors+=("Missing: $env_dir/$file")
+            fi
+        done
+    fi
+
+    # Report errors
+    if [[ ${#errors[@]} -gt 0 ]]; then
+        log_error "Environment verification failed:"
+        for error in "${errors[@]}"; do
+            echo "  - $error" >&2
+        done
+        return 1
+    fi
+
+    log_success "Environment verified (Docker, Claude CLI, API keys)"
+    return 0
+}
+
+# =============================================================================
+# TEMP DIRECTORY MANAGEMENT
+# =============================================================================
+
+create_temp_dir() {
+    local prefix="${1:-claude_test_}"
+    mktemp -d -t "${prefix}XXXXXXXX"
+}
+
+cleanup_temp_dir() {
+    local dir="$1"
+
+    # Safety check: only remove if it looks like our temp dir
+    if [[ "$dir" != *"claude_test_"* && "$dir" != */tmp/* && "$dir" != */var/folders/* ]]; then
+        log_warn "Refusing to clean up $dir (not a temp directory)"
+        return 1
+    fi
+
+    if [[ -d "$dir" ]]; then
+        log_info "Cleaning up $dir..."
+        rm -rf "$dir"
+        log_success "Cleaned up"
+    fi
+}
+
+# =============================================================================
+# SKILL SETUP
+# =============================================================================
+
+write_skill() {
+    local test_dir="$1"
+    local skill_name="$2"
+    local content_file="$3"
+    local scripts_dir="${4:-}"
+    local source_dir="${5:-}"
+
+    local skill_dir="$test_dir/.claude/skills/$skill_name"
+    mkdir -p "$skill_dir"
+
+    if [[ -n "$source_dir" && -d "$source_dir" ]]; then
+        cp -r "$source_dir"/. "$skill_dir"/
+        log_info "Copied skill package to $skill_name/"
+    fi
+
+    # Write SKILL.md
+    if [[ -f "$content_file" ]]; then
+        cp "$content_file" "$skill_dir/SKILL.md"
+    else
+        # Content passed as string (for piping)
+        echo "$content_file" > "$skill_dir/SKILL.md"
+    fi
+
+    # Copy scripts if provided
+    if [[ -n "$scripts_dir" && -d "$scripts_dir" ]]; then
+        rm -rf "$skill_dir/scripts"
+        cp -r "$scripts_dir" "$skill_dir/scripts"
+        chmod -R u+x "$skill_dir/scripts" 2>/dev/null || true
+        log_info "Copied scripts to $skill_name/scripts/"
+    fi
+
+    echo "$skill_dir/SKILL.md"
+}
+
+write_claude_md() {
+    local test_dir="$1"
+    local content_file="$2"
+
+    local claude_dir="$test_dir/.claude"
+    mkdir -p "$claude_dir"
+
+    if [[ -f "$content_file" ]]; then
+        cp "$content_file" "$claude_dir/CLAUDE.md"
+        cp "$content_file" "$test_dir/CLAUDE.md"
+    else
+        # Content passed as string
+        echo "$content_file" > "$claude_dir/CLAUDE.md"
+        echo "$content_file" > "$test_dir/CLAUDE.md"
+    fi
+
+    local size
+    size=$(wc -c < "$claude_dir/CLAUDE.md" | tr -d ' ')
+    log_success "Created CLAUDE.md ($size chars)"
+}
+
+# =============================================================================
+# ENVIRONMENT COPY
+# =============================================================================
+
+copy_environment() {
+    local test_dir="$1"
+    local env_dir="$2"
+
+    if [[ ! -d "$env_dir" ]]; then
+        log_error "Environment directory not found: $env_dir"
+        return 1
+    fi
+
+    # Merge (rsync-like) instead of clobber: copy each top-level item, and for
+    # directories that already exist in test_dir (e.g. .claude/, openspec/),
+    # merge contents recursively so injected skills/commands are preserved.
+    for item in "$env_dir"/*; do
+        [[ -e "$item" ]] || continue
+        local name
+        name=$(basename "$item")
+        local dest="$test_dir/$name"
+        if [[ -d "$item" && -d "$dest" ]]; then
+            # Both dirs: copy contents into existing dest (merge).
+            cp -r "$item"/. "$dest"/ 2>/dev/null || cp -r "$item"/* "$dest"/ 2>/dev/null || true
+        else
+            cp -r "$item" "$dest"
+        fi
+    done
+
+    log_success "Copied environment from $(basename "$env_dir")/ (merged)"
+}
+
+# =============================================================================
+# CLI MODE
+# =============================================================================
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    cmd="${1:-help}"
+    shift || true
+
+    case "$cmd" in
+        verify)
+            env_dir="${1:-}"
+            if [[ -z "$env_dir" ]]; then
+                die "Usage: $0 verify <env_dir> [required_files...]"
+            fi
+            shift
+            verify_environment "$env_dir" "$@"
+            ;;
+        create-temp)
+            prefix="${1:-claude_test_}"
+            create_temp_dir "$prefix"
+            ;;
+        cleanup)
+            dir="${1:-}"
+            if [[ -z "$dir" ]]; then
+                die "Usage: $0 cleanup <temp_dir>"
+            fi
+            cleanup_temp_dir "$dir"
+            ;;
+        write-skill)
+            test_dir="${1:-}"
+            skill_name="${2:-}"
+            content="${3:-}"
+            scripts_dir="${4:-}"
+            source_dir="${5:-}"
+            if [[ -z "$test_dir" || -z "$skill_name" || -z "$content" ]]; then
+                die "Usage: $0 write-skill <test_dir> <skill_name> <content_file> [scripts_dir] [source_dir]"
+            fi
+            write_skill "$test_dir" "$skill_name" "$content" "$scripts_dir" "$source_dir"
+            ;;
+        write-claude-md)
+            test_dir="${1:-}"
+            content="${2:-}"
+            if [[ -z "$test_dir" || -z "$content" ]]; then
+                die "Usage: $0 write-claude-md <test_dir> <content_file>"
+            fi
+            write_claude_md "$test_dir" "$content"
+            ;;
+        copy-env)
+            test_dir="${1:-}"
+            env_dir="${2:-}"
+            if [[ -z "$test_dir" || -z "$env_dir" ]]; then
+                die "Usage: $0 copy-env <test_dir> <env_dir>"
+            fi
+            copy_environment "$test_dir" "$env_dir"
+            ;;
+        help|*)
+            echo "Usage: $0 <command> [args...]"
+            echo ""
+            echo "Commands:"
+            echo "  verify <env_dir> [files...]     Verify environment (Docker, Claude, keys)"
+            echo "  create-temp [prefix]            Create temp directory"
+            echo "  cleanup <temp_dir>              Remove temp directory"
+            echo "  write-skill <dir> <name> <file> [scripts] [source]  Write skill to .claude/skills/"
+            echo "  write-claude-md <dir> <file>    Write CLAUDE.md"
+            echo "  copy-env <dir> <env_dir>        Copy environment files"
+            ;;
+    esac
+fi

package/eval/simulator-instruction.md

@@ -0,0 +1,9 @@
+You are simulating a developer user in an automated eval. The AI assistant is
+running the Comet development workflow and has paused to ask you something. Read
+its message and reply with a SHORT (1-3 sentences) response that:
+
+- Approves the proposed approach / name / plan when asked to confirm
+- Picks the most reasonable default option when asked to choose
+- Asks for clarification only if the question is truly ambiguous about WHAT to build
+
+Never refuse; always let the workflow move forward. Do not write code or files.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rpamis/comet",
-  "version": "0.4.0-beta.3",
-  "description": "Agent Skill Harness Phase-Guarded Automation From Idea To Archive",
+  "version": "0.4.0-beta.4",
+  "description": "Agent Skill Harness For Turning Ideas Into Evaluated Workflows",
   "keywords": [
     "comet",
     "openspec",
@@ -22,6 +22,28 @@
     "dist",
     "bin",
     "assets",
+    "eval",
+    "!eval/.venv",
+    "!eval/.uv-cache",
+    "!eval/.cache",
+    "!eval/.tmp",
+    "!eval/.ruff_cache",
+    "!eval/uv.lock",
+    "!eval/**/__pycache__",
+    "!eval/.pytest_cache",
+    "!eval/.pytest_cache*",
+    "!eval/.pytest-basetemp-*",
+    "!eval/.pytest-tmp-*",
+    "!eval/local/logs",
+    "!eval/local/.pytest_docker_build.lock",
+    "!eval/**/logs",
+    "!eval/**/.cache",
+    "!eval/**/.ruff_cache",
+    "!eval/**/.tmp",
+    "!eval/**/.pytest_cache",
+    "!eval/**/.pytest_cache*",
+    "!eval/**/.pytest-basetemp-*",
+    "!eval/**/.pytest-tmp-*",
     "scripts/install/postinstall.js",
     "!dist/**/*.test.js",
     "!dist/**/__tests__"