agent-petri-dish 0.1.0

data/lib/petri_dish/runner.rb

@@ -0,0 +1,330 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "shellwords"
+require "tempfile"
+require_relative "config"
+require_relative "environment"
+require_relative "hook_log"
+require_relative "results_builder"
+require_relative "transcript"
+
+module PetriDish
+  class StartupFailure < StandardError; end
+  class AuthFailure < StandardError; end
+
+  class Runner
+    POLL_INTERVAL = 2 # seconds
+    STARTUP_DEADLINE = 60 # seconds, must see a SessionStart event by then or fail loud
+    AUTH_ERROR_PATTERN = %r{API Error: 40[13]|Please run /login|Invalid authentication credentials|authentication_error}
+
+    def initialize(test_name, cultures_dir:, results_dir:, deny: false, debug: false, keep: false)
+      @cultures_dir = cultures_dir
+      @results_dir = results_dir
+      @test_dir = File.join(cultures_dir, test_name)
+      @config = Config.new(@test_dir)
+      @deny = deny
+      @debug = debug
+      @keep = keep
+      @tmux_session = "test-#{@config.name}"
+    end
+
+    def run!
+      preflight!
+      ensure_env!
+      run_prepare!
+
+      env = Environment.new(@config.environment[:name])
+
+      # Re-trust after prepare. The setup-time trust call runs before the
+      # work_dir exists, so File.realpath falls back to expand_path and stores
+      # the trust key under /tmp/... — but Claude resolves symlinks and looks
+      # under /private/tmp/... on macOS. Re-trusting here, with the path now
+      # existing, lets realpath canonicalize properly.
+      env.trust!(@config.runtime[:work_dir])
+
+      results_dir = create_results_dir
+      signal_file = File.join(results_dir, "signal")
+      transcript_path = File.join(results_dir, "transcript.log")
+
+      env.clear_hook_log!
+
+      prompt = build_prompt(signal_file)
+      launch_tmux!(prompt)
+
+      failure = nil
+      begin
+        poll_for_completion(signal_file, env)
+      rescue Interrupt
+        log "Interrupted by user"
+      rescue StartupFailure, AuthFailure => e
+        failure = e
+      end
+
+      # Capture transcript first. If we failed, tmux may be dying and we
+      # want whatever output Claude printed (auth error, version mismatch, etc.).
+      transcript = Transcript.new(@tmux_session)
+      transcript.save!(transcript_path) if tmux_alive?
+
+      # Collect artifacts
+      hook_log_dest = File.join(results_dir, "hook-events.jsonl")
+      FileUtils.cp(env.hook_log_path, hook_log_dest) if File.exist?(env.hook_log_path)
+
+      if failure
+        # Tear down tmux now so the error message isn't competing with a live session.
+        system("tmux kill-session -t #{@tmux_session} 2>/dev/null")
+        report_failure!(failure, transcript_path, results_dir)
+        exit 1
+      end
+
+      # Build results
+      if File.exist?(hook_log_dest)
+        builder = ResultsBuilder.new(hook_log_dest, transcript_path, results_dir)
+        builder.build!
+      end
+
+      # Teardown
+      unless @keep
+        system("tmux kill-session -t #{@tmux_session} 2>/dev/null")
+      end
+
+      # Summary
+      results_file = File.join(results_dir, "results.md")
+      log ""
+      log "Run complete: #{@config.name}"
+      log "  Results:    #{File.exist?(results_file) ? results_file : '(not written)'}"
+      log "  Hook log:   #{File.exist?(hook_log_dest) ? hook_log_dest : '(not captured)'}"
+      log "  Transcript: #{File.exist?(transcript_path) ? transcript_path : '(not captured)'}"
+      if @keep
+        log "  tmux:       tmux attach -t #{@tmux_session}"
+      end
+    end
+
+    private
+
+    def run_prepare!
+      commands = @config.runtime[:prepare] || []
+      return if commands.empty?
+
+      log "Running #{commands.length} prepare step(s)"
+      commands.each do |cmd|
+        log "  prepare: #{cmd}"
+        raise "prepare step failed: #{cmd}" unless system(cmd)
+      end
+    end
+
+    def preflight!
+      %w[claude tmux].each do |cmd|
+        unless system("command -v #{cmd} > /dev/null 2>&1")
+          raise "Required command not found: #{cmd}"
+        end
+      end
+    end
+
+    def ensure_env!
+      env = Environment.new(@config.environment[:name])
+      unless env.exists?
+        raise "Environment '#{@config.environment[:name]}' not found. Run: petri-dish setup #{@config.name}"
+      end
+    end
+
+    def create_results_dir
+      timestamp = Time.now.strftime("%Y-%m-%dT%H-%M-%S")
+      base = File.join(@results_dir, @config.name, timestamp)
+      # Guard against collisions: fast cultures (~20-40s) can finish multiple
+      # runs in the same second. Append a counter so a later run never clobbers
+      # an earlier one's dir (and races its half-written hook-events.jsonl).
+      dir = base
+      counter = 1
+      while File.exist?(dir)
+        counter += 1
+        dir = "#{base}-#{counter}"
+      end
+      FileUtils.mkdir_p(dir)
+      dir
+    end
+
+    def build_prompt(signal_file)
+      parts = []
+
+      # Preamble
+      if @config.runtime[:preamble]
+        preamble_path = resolve_preamble(@config.runtime[:preamble])
+        if preamble_path
+          parts << File.read(preamble_path)
+          parts << "\n---\n"
+        end
+      end
+
+      # Main prompt
+      prompt_path = @config.prompt_path
+      raise "Prompt not found: #{prompt_path}" unless prompt_path.exist?
+      parts << prompt_path.read
+
+      # Part suffix
+      if @config.runtime[:part_suffix]
+        parts << "\n---\n"
+        parts << "**INSTRUCTION: #{@config.runtime[:part_suffix]}**"
+      end
+
+      # Signal file injection
+      if @config.runtime[:inject_results_file]
+        parts << "\n---\n"
+        parts << "**SIGNAL_FILE: #{signal_file}**"
+      end
+
+      parts.join("\n")
+    end
+
+    def launch_tmux!(prompt)
+      system("tmux kill-session -t #{@tmux_session} 2>/dev/null")
+
+      work_dir = @config.runtime[:work_dir]
+      env_dir = Environment.new(@config.environment[:name]).env_path
+
+      prompt_file = Tempfile.new(["petri-dish-prompt-", ".md"], Dir.tmpdir)
+      prompt_file.write(prompt)
+      prompt_file.close
+
+      model_flag = @config.runtime[:model] ? " --model #{@config.runtime[:model]}" : ""
+      launcher = Tempfile.new(["petri-dish-launcher-", ".sh"], Dir.tmpdir)
+      launcher.write(<<~SH)
+        #!/usr/bin/env bash
+        exec env CLAUDE_CONFIG_DIR='#{env_dir}' ENABLE_CLAUDEAI_MCP_SERVERS='false' claude#{model_flag} "$(cat '#{prompt_file.path}')"
+      SH
+      launcher.close
+      File.chmod(0o755, launcher.path)
+
+      @_prompt_file = prompt_file
+      @_launcher = launcher
+
+      system("tmux new-session -d -s #{@tmux_session} -c #{work_dir.shellescape}")
+      system("tmux send-keys -t #{@tmux_session} #{launcher.path.shellescape} Enter")
+
+      log "Claude launched in tmux session '#{@tmux_session}'"
+      log "Attach to watch: tmux attach -t #{@tmux_session}"
+
+      sleep 3
+    end
+
+    def poll_for_completion(signal_file, env)
+      timeout = @config.runtime[:timeout]
+      start = Time.now
+      hook_log_lines_seen = 0
+      startup_confirmed = false
+
+      log "Polling for completion (timeout: #{timeout}s)"
+
+      loop do
+        unless tmux_alive?
+          unless startup_confirmed
+            raise StartupFailure, "tmux session exited before any SessionStart hook event fired"
+          end
+          break
+        end
+
+        if File.exist?(signal_file) && File.size(signal_file) > 0
+          log "Signal file detected. Session complete."
+          break
+        end
+
+        # Watch for SessionStart in the hook log, firmest signal that Claude
+        # actually booted (vs. exited on auth failure / version mismatch / etc.).
+        if !startup_confirmed && File.exist?(env.hook_log_path)
+          if File.read(env.hook_log_path).include?('"hook_event_name":"SessionStart"')
+            startup_confirmed = true
+            log "Session started (SessionStart event observed)"
+          end
+        end
+
+        # Once booted, watch the visible pane for mid-session auth failures.
+        # Claude fires SessionStart before its first API call, so an OAuth-stale
+        # session can confirm startup, hit 401/403, then sit waiting for input.
+        if startup_confirmed && (match = AUTH_ERROR_PATTERN.match(Transcript.new(@tmux_session).capture_visible))
+          raise AuthFailure, "API auth error mid-session: #{match[0]}"
+        end
+
+        elapsed = Time.now - start
+        if !startup_confirmed && elapsed >= STARTUP_DEADLINE
+          raise StartupFailure, "no SessionStart event within #{STARTUP_DEADLINE}s (tmux still alive, Claude may be stuck at a login prompt)"
+        end
+
+        if elapsed >= timeout
+          log "Timeout reached (#{timeout}s). Stopping."
+          break
+        end
+
+        # Debug: print new hook events as they arrive
+        if @debug && File.exist?(env.hook_log_path)
+          lines = File.readlines(env.hook_log_path)
+          lines[hook_log_lines_seen..].each do |line|
+            data = JSON.parse(line) rescue next
+            p = data["payload"]
+            event = p["hook_event_name"]
+            tool = p["tool_name"] || ""
+            cmd = p.dig("tool_input", "command") || ""
+            $stderr.puts "\e[2m[hook] #{event} #{tool} #{cmd[0..60]}\e[0m"
+          end
+          hook_log_lines_seen = lines.size
+        end
+
+        sleep POLL_INTERVAL
+      end
+    end
+
+    def report_failure!(err, transcript_path, results_dir)
+      label = err.is_a?(AuthFailure) ? "AUTH FAILURE" : "STARTUP FAILURE"
+      marker_file = err.is_a?(AuthFailure) ? "AUTH_FAILURE.txt" : "STARTUP_FAILURE.txt"
+
+      $stderr.puts ""
+      $stderr.puts "\e[31m[runner] #{label}\e[0m"
+      $stderr.puts "  #{err.message}"
+      $stderr.puts ""
+      $stderr.puts "  Likely causes:"
+      $stderr.puts "    - Stale OAuth credentials in the cenv environment."
+      $stderr.puts "    - Claude Code version installed for this env can't auth."
+      $stderr.puts "    - claude binary missing from PATH inside tmux."
+      $stderr.puts ""
+      $stderr.puts "  Recovery:"
+      $stderr.puts "    petri-dish setup --clean #{@config.name}"
+      $stderr.puts "    petri-dish setup #{@config.name}"
+      $stderr.puts "    cenv login #{@config.environment[:name]}  # if a /login prompt was shown"
+      $stderr.puts ""
+      if File.exist?(transcript_path) && File.size(transcript_path) > 0
+        $stderr.puts "  Captured tmux output: #{transcript_path}"
+        last = File.read(transcript_path).lines.last(20).join
+        $stderr.puts "  --- last 20 lines ---"
+        $stderr.puts last.gsub(/^/, "  ")
+      else
+        $stderr.puts "  No tmux output captured (session may have died before printing)."
+      end
+      $stderr.puts ""
+      # Drop a marker so the empty results dir is self-explanatory later.
+      File.write(File.join(results_dir, marker_file), "#{err.message}\n")
+    end
+
+    def resolve_preamble(name)
+      # Legacy configs use paths like "lib/preambles/sandbox.md". Strip the prefix
+      # so the bare name maps to gem-bundled lib/petri_dish/preambles/<name>.md, but
+      # still allow user-provided overrides in the cultures_dir.
+      bare = name.sub(%r{\Alib/preambles/}, "")
+
+      candidates = [
+        File.join(@cultures_dir, name),
+        File.join(@cultures_dir, bare),
+        File.join(PetriDish.root, "lib", "petri_dish", "preambles", bare)
+      ]
+
+      candidates.find { |p| File.exist?(p) }
+    end
+
+    def tmux_alive?
+      system("tmux has-session -t #{@tmux_session} 2>/dev/null")
+    end
+
+    def log(msg)
+      puts "\e[32m[runner]\e[0m #{msg}"
+    end
+  end
+end

data/lib/petri_dish/transcript.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module PetriDish
+  class Transcript
+    ANSI_REGEX = /\e\[[0-9;]*[a-zA-Z]/
+
+    def initialize(tmux_session)
+      @tmux_session = tmux_session
+    end
+
+    def capture_pane(visible_only: false)
+      flag = visible_only ? "" : "-S -"
+      raw = `tmux capture-pane -t #{@tmux_session} -p #{flag} 2>/dev/null`
+      strip_ansi(raw)
+    end
+
+    def capture_visible
+      capture_pane(visible_only: true)
+    end
+
+    def save!(output_path)
+      content = capture_pane
+      File.write(output_path, content)
+      log "Transcript saved to #{output_path}"
+    end
+
+    private
+
+    def strip_ansi(text)
+      text.gsub(ANSI_REGEX, "")
+    end
+
+    def log(msg)
+      puts "\e[32m[transcript]\e[0m #{msg}"
+    end
+  end
+end

data/lib/petri_dish/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module PetriDish
+  VERSION = "0.1.0"
+end

data/lib/petri_dish.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "petri_dish/version"
+
+module PetriDish
+  # Root directory of the installed gem. Used to locate ship-with assets
+  # (hooks, preambles). Distinct from user-provided cultures_dir and results_dir.
+  def self.root
+    File.expand_path("..", __dir__)
+  end
+end
+
+require_relative "petri_dish/config"
+require_relative "petri_dish/environment"
+require_relative "petri_dish/hook_log"
+require_relative "petri_dish/results_builder"
+require_relative "petri_dish/runner"
+require_relative "petri_dish/transcript"
+require_relative "petri_dish/cli"

data/scripts/analyze-hooks.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""Analyze hook event log and correlate with sidecar results."""
+import json
+import sys
+from datetime import datetime
+
+def parse_ts(ts_str):
+    """Parse ISO timestamp to datetime."""
+    # Handle both millisecond and second precision
+    for fmt in ("%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"):
+        try:
+            return datetime.strptime(ts_str, fmt)
+        except ValueError:
+            continue
+    return None
+
+def main():
+    hook_file = sys.argv[1] if len(sys.argv) > 1 else "/tmp/petri-dish-hooks.jsonl"
+
+    # Parse hook events
+    entries = []
+    with open(hook_file) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            data = json.loads(line)
+            entries.append(data)
+
+    # Pair Pre/Post by tool_use_id
+    pairs = {}
+    for entry in entries:
+        ts = parse_ts(entry["ts"])
+        p = entry["payload"]
+        event = p["hook_event_name"]
+        tid = p.get("tool_use_id")
+
+        if event == "Notification":
+            ntype = p.get("notification_type", "unknown")
+            msg = p.get("message", "")
+            print(f"  NOTIFICATION: type={ntype} msg={msg[:60]}")
+            continue
+
+        if not tid:
+            continue
+
+        if event == "PreToolUse":
+            cmd = p.get("tool_input", {}).get("command", "")
+            tool = p.get("tool_name", "")
+            pairs[tid] = {"pre_ts": ts, "cmd": cmd, "tool": tool}
+        elif event == "PostToolUse" and tid in pairs:
+            pairs[tid]["post_ts"] = ts
+
+    # Prompted commands from results.md (PROMPTED_ALLOWED)
+    prompted_cmds = [
+        "for f in a.txt b.txt c.txt; do echo $f; done",
+        "for f in *.txt; do cat $f; done",
+        "echo $(whoami)",
+        "echo $(ls /tmp)",
+        "echo {1..5}",
+        "echo file-{a,b,c}.txt",
+        "cat <(echo hello)",
+        "{ echo a; echo b; }",
+        "echo $HOME",
+        "echo $PATH",
+        "echo $MY_CUSTOM_VAR",
+    ]
+
+    print()
+    print(f"{'#':<4} {'Command':<52} {'Delta':>7}  {'Sidecar':<18}")
+    print("-" * 90)
+
+    i = 0
+    for tid, p in pairs.items():
+        if "post_ts" not in p:
+            continue
+        if p["tool"] != "Bash":
+            continue
+
+        i += 1
+        cmd = p["cmd"]
+        delta = (p["post_ts"] - p["pre_ts"]).total_seconds()
+
+        was_prompted = cmd in prompted_cmds
+        sidecar = "PROMPTED" if was_prompted else "silent"
+
+        # Truncate command for display
+        cmd_display = cmd[:50] + "..." if len(cmd) > 50 else cmd
+
+        print(f"{i:<4} {cmd_display:<52} {delta:>6.2f}s  {sidecar:<18}")
+
+    # Summary stats
+    prompted_deltas = []
+    silent_deltas = []
+    for tid, p in pairs.items():
+        if "post_ts" not in p or p["tool"] != "Bash":
+            continue
+        delta = (p["post_ts"] - p["pre_ts"]).total_seconds()
+        if p["cmd"] in prompted_cmds:
+            prompted_deltas.append(delta)
+        else:
+            silent_deltas.append(delta)
+
+    print()
+    print("Summary:")
+    if silent_deltas:
+        print(f"  Silent commands:   avg={sum(silent_deltas)/len(silent_deltas):.2f}s  "
+              f"min={min(silent_deltas):.2f}s  max={max(silent_deltas):.2f}s  n={len(silent_deltas)}")
+    if prompted_deltas:
+        print(f"  Prompted commands: avg={sum(prompted_deltas)/len(prompted_deltas):.2f}s  "
+              f"min={min(prompted_deltas):.2f}s  max={max(prompted_deltas):.2f}s  n={len(prompted_deltas)}")
+
+if __name__ == "__main__":
+    main()

data/scripts/hook-block-pattern.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# PreToolUse hook: block any Bash command matching a regex pattern.
+#
+# Environment:
+#   HOOK_BLOCK_PATTERN - regex matched against tool_input.command (required)
+#   PETRIDISH_HOOK_LOG_FILE      - JSONL path to log every invocation (optional)
+#
+# Reads the PreToolUse JSON payload on stdin. If the command matches
+# HOOK_BLOCK_PATTERN, emits a JSON decision denying the call with
+# exit 0 (documented mechanism). Otherwise exits 0 silently.
+
+set -u
+
+LOG_FILE="${PETRIDISH_HOOK_LOG_FILE:-/tmp/petri-hooks.jsonl}"
+PATTERN="${HOOK_BLOCK_PATTERN:-}"
+
+PAYLOAD=$(cat)
+
+# Log every invocation with timestamp
+TS=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ" 2>/dev/null || date -u +"%Y-%m-%dT%H:%M:%SZ")
+printf '{"ts":"%s","payload":%s}\n' "$TS" "$PAYLOAD" >> "$LOG_FILE"
+
+# No pattern configured: let everything through
+if [ -z "$PATTERN" ]; then
+  exit 0
+fi
+
+# Only evaluate Bash tool calls
+TOOL_NAME=$(printf '%s' "$PAYLOAD" | python3 -c "import json,sys; print(json.load(sys.stdin).get('tool_name',''))")
+if [ "$TOOL_NAME" != "Bash" ]; then
+  exit 0
+fi
+
+COMMAND=$(printf '%s' "$PAYLOAD" | python3 -c "import json,sys; print(json.load(sys.stdin).get('tool_input',{}).get('command',''))")
+
+if printf '%s' "$COMMAND" | grep -qE "$PATTERN"; then
+  cat <<JSON
+{
+  "hookSpecificOutput": {
+    "hookEventName": "PreToolUse",
+    "permissionDecision": "deny",
+    "permissionDecisionReason": "Blocked by policy hook: command matches HOOK_BLOCK_PATTERN"
+  }
+}
+JSON
+  exit 0
+fi
+
+exit 0

data/scripts/hook-logger.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Hook logger: reads JSON from stdin, adds timestamp, appends to JSONL file.
+# Usage: Configured as a Claude Code hook command.
+#
+# Environment:
+#   PETRIDISH_HOOK_LOG_FILE - path to append JSONL entries (default: /tmp/petri-hooks.jsonl)
+
+LOG_FILE="${PETRIDISH_HOOK_LOG_FILE:-/tmp/petri-hooks.jsonl}"
+TS=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ" 2>/dev/null || date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+# Read stdin (the hook JSON payload)
+PAYLOAD=$(cat)
+
+# Wrap with timestamp and write as one JSONL line
+printf '{"ts":"%s","payload":%s}\n' "$TS" "$PAYLOAD" >> "$LOG_FILE"
+
+# Exit 0: no decision, let normal flow continue
+exit 0

data/scripts/inspect-session.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# inspect-session.sh — mine a Claude Code session JSONL for LSP-relevant content.
+#
+# Usage:
+#   scripts/inspect-session.sh <path-to-session.jsonl>
+#
+# The session JSONL lives at:
+#   ~/.local/share/cenv/<env>/projects/<cwd-slug>/<session-id>.jsonl
+#
+# Reports: attachment types, user-message content matching diagnostic patterns,
+# and LSP tool invocations (operation + position).
+
+set -euo pipefail
+
+SESSION="${1:?usage: inspect-session.sh <session.jsonl>}"
+
+if [ ! -f "$SESSION" ]; then
+  echo "error: session file not found: $SESSION" >&2
+  exit 1
+fi
+
+echo "=== Session: $SESSION ==="
+echo
+
+echo "=== Entry type counts ==="
+jq -r '.type' "$SESSION" | sort | uniq -c
+echo
+
+echo "=== Attachment types ==="
+jq -r 'select(.type == "attachment") | .attachment.type // (.attachment | keys | join(","))' "$SESSION" \
+  | sort | uniq -c
+echo
+
+echo "=== User-role content matching diagnostic/lsp patterns ==="
+jq -r 'select(.type == "user") | .message.content | if type == "array" then map(.text // (.content // "") | tostring) | join(" | ") else . end' "$SESSION" \
+  | grep -iE 'diagnost|lsp |offense|warning:|error:' \
+  | head -20 || echo "(no matches)"
+echo
+
+echo "=== LSP tool invocations (assistant tool_use entries) ==="
+jq -r 'select(.type == "assistant") | .message.content | if type == "array" then map(select(.type == "tool_use" and .name == "LSP")) else [] end | .[] | "\(.input.operation // "?") @ \(.input.filePath // "?"):\(.input.line // "?"):\(.input.character // "?")"' "$SESSION" \
+  || echo "(no LSP tool calls)"
+echo
+
+echo "=== Any 'lsp_diagnostics' or 'lspDiagnostics' anywhere in raw content ==="
+grep -ciE 'lsp_diagnostics|lspDiagnostics' "$SESSION" || true
+echo
+
+echo "=== Tool counts (PreToolUse equivalent — assistant tool_use by name) ==="
+jq -r 'select(.type == "assistant") | .message.content | if type == "array" then map(select(.type == "tool_use") | .name) else [] end | .[]' "$SESSION" \
+  | sort | uniq -c