rspec-agents 0.1.0

data/lib/rspec/agents/cli.rb

@@ -0,0 +1,342 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module RSpec
+  module Agents
+    # Unified CLI for rspec-agents
+    #
+    # Commands:
+    #   run      - Single-process execution (default)
+    #   parallel - Parallel execution with worker processes
+    #   worker   - Internal: run as a worker subprocess
+    #
+    # @example Single process
+    #   CLI.run(["spec/"])
+    #   CLI.run(["run", "spec/"])
+    #
+    # @example Parallel
+    #   CLI.run(["parallel", "-w", "4", "spec/"])
+    #
+    # @example Worker (internal)
+    #   CLI.run(["worker"])
+    #
+    class CLI
+      COMMANDS = %w[run parallel worker render].freeze
+
+      def self.run(argv)
+        new(argv).run
+      end
+
+      def initialize(argv)
+        @argv = argv.dup
+      end
+
+      def run
+        command, args = parse_command(@argv)
+
+        case command
+        when "run"
+          run_single(args)
+        when "parallel"
+          run_parallel(args)
+        when "worker"
+          run_worker(args)
+        when "render"
+          run_render(args)
+        else
+          # Default to single-process run
+          run_single(@argv)
+        end
+      end
+
+      private
+
+      def parse_command(argv)
+        return [nil, argv] if argv.empty?
+
+        first = argv.first
+
+        # Auto-detect parallel mode if -w/--workers flag is present
+        if has_parallel_flag?(argv)
+          # If first arg is explicit command, consume it; otherwise keep all args
+          if COMMANDS.include?(first)
+            return ["parallel", argv[1..]]
+          else
+            return ["parallel", argv]
+          end
+        end
+
+        # Original logic for explicit commands or default to single-process
+        if COMMANDS.include?(first)
+          [first, argv[1..]]
+        elsif first.start_with?("-")
+          # Flag, not a command - default to run
+          [nil, argv]
+        else
+          # Path or unknown - default to run
+          [nil, argv]
+        end
+      end
+
+      def has_parallel_flag?(argv)
+        argv.any? { |arg| arg == "-w" || arg.start_with?("--workers") }
+      end
+
+      # =========================================================================
+      # Single-process mode
+      # =========================================================================
+
+      def run_single(args)
+        options = parse_single_options(args)
+
+        runner = Runners::TerminalRunner.new(
+          output:    $stdout,
+          color:     options[:color],
+          json_path: options[:json_path],
+          html_path: options[:html_path]
+        )
+        runner.run(options[:paths])
+      end
+
+      def parse_single_options(args)
+        options = { paths: [], color: nil, json_path: nil, html_path: nil }
+
+        parser = OptionParser.new do |opts|
+          opts.banner = "Usage: rspec-agents [run] [options] [paths...]"
+          opts.separator ""
+          opts.separator "Run specs in a single process with terminal output."
+          opts.separator ""
+          opts.separator "Options:"
+
+          opts.on("--[no-]color", "Force color on/off (default: auto)") do |v|
+            options[:color] = v
+          end
+
+          opts.on("--ui MODE", [:interactive, :interleaved, :quiet],
+                  "Output mode (ignored in single-process mode)") do |_mode|
+            # Accepted for CLI compatibility with parallel mode, but ignored
+          end
+
+          opts.on("--json PATH", "Save JSON run data to file") do |path|
+            options[:json_path] = path
+          end
+
+          opts.on("--html PATH", "Render HTML report to path") do |path|
+            options[:html_path] = path
+          end
+
+          opts.on("-h", "--help", "Show this help") do
+            puts opts
+            exit 0
+          end
+        end
+
+        remaining = parser.parse(args)
+        options[:paths] = remaining.empty? ? ["spec"] : remaining
+        options
+      end
+
+      # =========================================================================
+      # Parallel mode
+      # =========================================================================
+
+      def run_parallel(args)
+        options = parse_parallel_options(args)
+
+        runner = Runners::ParallelTerminalRunner.new(
+          worker_count: options[:workers],
+          fail_fast:    options[:fail_fast],
+          output:       $stdout,
+          color:        options[:color],
+          json_path:    options[:json_path],
+          html_path:    options[:html_path],
+          ui_mode:      options[:ui_mode]
+        )
+        runner.run(options[:paths])
+      end
+
+      def parse_parallel_options(args)
+        options = {
+          workers:   4,
+          fail_fast: false,
+          paths:     [],
+          color:     nil,
+          json_path: nil,
+          html_path: nil,
+          ui_mode:   nil
+        }
+
+        parser = OptionParser.new do |opts|
+          opts.banner = "Usage: rspec-agents parallel [options] [paths...]"
+          opts.separator ""
+          opts.separator "Run specs in parallel across multiple worker processes."
+          opts.separator ""
+          opts.separator "Options:"
+
+          opts.on("-w", "--workers COUNT", Integer, "Number of workers (default: 4)") do |w|
+            options[:workers] = w
+          end
+
+          opts.on("--fail-fast", "Stop on first failure") do
+            options[:fail_fast] = true
+          end
+
+          opts.on("--[no-]color", "Force color on/off (default: auto)") do |v|
+            options[:color] = v
+          end
+
+          opts.on("--ui MODE", [:interactive, :interleaved, :quiet],
+                  "Output mode: interactive, interleaved, quiet (default: auto)") do |mode|
+            options[:ui_mode] = mode
+          end
+
+          opts.on("--json PATH", "Save JSON run data to file") do |path|
+            options[:json_path] = path
+          end
+
+          opts.on("--html PATH", "Render HTML report to path") do |path|
+            options[:html_path] = path
+          end
+
+          opts.on("-h", "--help", "Show this help") do
+            puts opts
+            exit 0
+          end
+        end
+
+        remaining = parser.parse(args)
+        options[:paths] = remaining.empty? ? ["spec"] : remaining
+        options
+      end
+
+      # =========================================================================
+      # Worker mode (internal - used by parallel controller)
+      # =========================================================================
+
+      def run_worker(_args)
+        require "json"
+
+        # Get RPC socket from environment
+        fd_str = ENV["RPC_SOCKET_FD"]
+        unless fd_str
+          $stderr.puts("ERROR: RPC_SOCKET_FD not set (worker mode requires controller)")
+          exit 1
+        end
+
+        fd = fd_str.to_i
+        socket = IO.for_fd(fd, mode: "r+")
+        socket.sync = true
+
+        worker_index = ENV["WORKER_INDEX"] || "?"
+        $stderr.puts("[worker-#{worker_index}] started")
+
+        runner = nil
+        running = true
+
+        while running && (line = socket.gets)
+          begin
+            handle_worker_message(socket, line, worker_index) do |example_ids|
+              runner ||= Runners::HeadlessRunner.new(rpc_output: socket)
+              runner.run(example_ids)
+            end
+          rescue JSON::ParserError => e
+            $stderr.puts("[worker-#{worker_index}] JSON parse error: #{e.message}")
+          rescue => e
+            $stderr.puts("[worker-#{worker_index}] Error: #{e.class}: #{e.message}")
+            $stderr.puts(e.backtrace.first(5).join("\n"))
+          end
+        end
+
+        $stderr.puts("[worker-#{worker_index}] exiting")
+        0
+      end
+
+      def handle_worker_message(socket, line, worker_index)
+        msg = JSON.parse(line.chomp, symbolize_names: true)
+
+        case msg[:action]
+        when "__shutdown__"
+          send_worker_response(socket, msg[:id], { status: "shutting_down" })
+          return false # Signal to stop
+
+        when "run_specs"
+          $stderr.puts("[worker-#{worker_index}] running #{msg[:example_ids]&.size || 0} examples")
+          result = yield(msg[:example_ids] || [])
+          send_worker_response(socket, msg[:id], result)
+
+        else
+          send_worker_response(socket, msg[:id], { error: "unknown action: #{msg[:action]}" })
+        end
+
+        true # Continue running
+      end
+
+      def send_worker_response(socket, request_id, payload)
+        response = payload.merge(reply_to: request_id)
+        socket.puts(response.to_json)
+        socket.flush
+      end
+
+      # =========================================================================
+      # Render mode - generate HTML report from JSON file
+      # =========================================================================
+
+      def run_render(args)
+        options = parse_render_options(args)
+
+        unless options[:json_path]
+          $stderr.puts("Error: JSON file path is required")
+          $stderr.puts("Usage: rspec-agents render <json_file> [--html PATH]")
+          return 1
+        end
+
+        unless File.exist?(options[:json_path])
+          $stderr.puts("Error: JSON file not found: #{options[:json_path]}")
+          return 1
+        end
+
+        output_path = Serialization::TestSuiteRenderer.from_json_file(
+          options[:json_path],
+          output_path: options[:html_path]
+        )
+
+        if output_path
+          puts "HTML report written to: #{output_path}"
+          0
+        else
+          $stderr.puts("Error: Failed to render HTML report")
+          1
+        end
+      end
+
+      def parse_render_options(args)
+        options = { json_path: nil, html_path: nil }
+
+        parser = OptionParser.new do |opts|
+          opts.banner = "Usage: rspec-agents render <json_file> [options]"
+          opts.separator ""
+          opts.separator "Render an HTML report from a JSON run file."
+          opts.separator ""
+          opts.separator "Options:"
+
+          opts.on("--html PATH", "Output HTML path (default: tmp/rspec_agents_debug.html)") do |path|
+            options[:html_path] = path
+          end
+
+          opts.on("-h", "--help", "Show this help") do
+            puts opts
+            exit 0
+          end
+        end
+
+        remaining = parser.parse(args)
+        options[:json_path] = remaining.first
+        options
+      end
+    end
+
+    # Zeitwerk expects cli.rb to define Cli, but we use CLI (conventional for command-line interfaces)
+    Cli = CLI
+  end
+end

data/lib/rspec/agents/conversation.rb

@@ -0,0 +1,308 @@
+module RSpec
+  module Agents
+    # Tracks the state of a conversation during test execution
+    # Maintains messages, turns, and topic history
+    class Conversation
+      attr_reader :messages, :turns, :topic_history, :evaluation_results
+
+      # @param event_bus [EventBus, nil] Optional event bus for emitting events
+      def initialize(event_bus: nil)
+        @event_bus = event_bus
+        @example_id = Thread.current[:rspec_agents_example_id]
+        @messages = []
+        @turns = []
+        @topic_history = []  # Array of { topic: Symbol, turns: Array<Turn> }
+        @current_topic = nil
+        @turns_per_topic = Hash.new(0)
+        @evaluation_results = []  # Array of EvaluationResult objects
+      end
+
+      def conversation
+        self
+      end
+
+      # Add a user message to the conversation
+      #
+      # @param text [String] Message text
+      # @param metadata [Hash] Optional metadata
+      # @param source [Symbol] Message source (:simulator, :script, :unknown)
+      # @return [Message] The added message
+      def add_user_message(text, metadata: {}, source: :unknown)
+        message = Message.new(
+          role:     :user,
+          content:  text,
+          metadata: metadata
+        )
+        @messages << message
+
+        emit(Events::UserMessage.new(
+          example_id:  @example_id,
+          turn_number: @turns.size + 1,
+          text:        text,
+          source:      source,
+          time:        Time.now
+        ))
+
+        message
+      end
+
+      # Add an agent response to the conversation
+      # Creates a turn with the last user message
+      #
+      # @param response [AgentResponse] The agent's response
+      # @return [Turn] The created turn
+      def add_agent_response(response)
+        message = Message.new(
+          role:       :agent,
+          content:    response.text,
+          tool_calls: response.tool_calls,
+          metadata:   response.metadata.to_h
+        )
+        @messages << message
+
+        # Find the last user message to pair with this response
+        last_user_msg = @messages.reverse.find(&:user?)
+        last_user_message = last_user_msg&.content
+
+        # Create turn
+        turn = Turn.new(last_user_message, response, topic: @current_topic)
+        @turns << turn
+
+        # Update topic tracking
+        if @current_topic
+          @turns_per_topic[@current_topic] += 1
+          add_turn_to_topic_history(turn)
+        end
+
+        emit(Events::AgentResponse.new(
+          example_id:  @example_id,
+          turn_number: @turns.size,
+          text:        response.text,
+          tool_calls:  response.tool_calls.map { |tc| tc.respond_to?(:to_h) ? tc.to_h : tc },
+          metadata:    response.metadata.respond_to?(:to_h) ? response.metadata.to_h : (response.metadata || {}),
+          time:        Time.now
+        ))
+
+        turn
+      end
+
+      # Set the current topic
+      #
+      # @param topic_name [Symbol] The topic name
+      # @param trigger [Symbol, nil] What triggered the topic change
+      def set_topic(topic_name, trigger: nil)
+        return if @current_topic == topic_name
+
+        previous = @current_topic
+        @current_topic = topic_name
+
+        # Start new topic history entry
+        @topic_history << { topic: topic_name, turns: [] }
+
+        emit(Events::TopicChanged.new(
+          example_id:  @example_id,
+          turn_number: @turns.size,
+          from_topic:  previous,
+          to_topic:    topic_name,
+          trigger:     trigger || :unknown,
+          time:        Time.now
+        ))
+      end
+
+      # Get the current topic
+      #
+      # @return [Symbol, nil]
+      attr_reader :current_topic
+
+      # Get the number of turns that have occurred in a specific topic
+      #
+      # @param topic_name [Symbol] The topic name
+      # @return [Integer]
+      def turns_in_topic(topic_name)
+        @turns_per_topic[topic_name.to_sym]
+      end
+
+      # Get all turns that occurred in a specific topic
+      #
+      # @param topic_name [Symbol] The topic name
+      # @return [Array<Turn>]
+      def turns_for_topic(topic_name)
+        entry = @topic_history.find { |h| h[:topic] == topic_name }
+        entry ? entry[:turns] : []
+      end
+
+      # Get all tool calls across all turns
+      #
+      # @return [Array<ToolCall>]
+      def all_tool_calls
+        @turns.flat_map(&:tool_calls)
+      end
+
+      # Check if a specific tool was called
+      #
+      # @param name [Symbol, String] Tool name
+      # @return [Boolean]
+      def has_tool_call?(name)
+        all_tool_calls.any? { |tc| tc.name == name.to_sym }
+      end
+
+      # Count how many times a tool was called
+      #
+      # @param name [Symbol, String] Tool name
+      # @return [Integer]
+      def called_tool?(name)
+        all_tool_calls.count { |tc| tc.name == name.to_sym }
+      end
+
+      # Find tool calls by name
+      #
+      # @param name [Symbol, String] Tool name
+      # @param params [Hash, nil] Optional parameter filter
+      # @return [Array<ToolCall>]
+      def find_tool_calls(name, params: nil)
+        calls = all_tool_calls.select { |tc| tc.name == name.to_sym }
+        return calls unless params
+
+        calls.select { |tc| tc.matches_params?(params) }
+      end
+
+      # Get the last agent response
+      #
+      # @return [AgentResponse, nil]
+      def last_agent_response
+        @turns.last&.agent_response
+      end
+
+      # Get the last user message
+      #
+      # @return [String, nil]
+      def last_user_message
+        @messages.reverse.find(&:user?)&.content
+      end
+
+      # Get the last turn
+      #
+      # @return [Turn, nil]
+      def last_turn
+        @turns.last
+      end
+
+      # Get the number of turns
+      #
+      # @return [Integer]
+      def turn_count
+        @turns.count
+      end
+
+      # Check if conversation is empty
+      #
+      # @return [Boolean]
+      def empty?
+        @messages.empty?
+      end
+
+      # Record an evaluation result (soft or hard)
+      #
+      # @param mode [Symbol] :soft or :hard
+      # @param type [Symbol] Type of assertion (:quality, :grounding, :tool_call, etc.)
+      # @param description [String] Human-readable description
+      # @param passed [Boolean] Whether evaluation passed
+      # @param failure_message [String, nil] Reason for failure
+      # @param metadata [Hash] Additional context
+      # @return [EvaluationResult] The recorded result
+      def record_evaluation(mode:, type:, description:, passed:, failure_message: nil, metadata: {})
+        turn_number = @turns.size
+        enriched_metadata = metadata.merge(
+          turn_number: turn_number,
+          topic:       @current_topic
+        )
+
+        result = EvaluationResult.new(
+          mode:            mode,
+          type:            type,
+          description:     description,
+          passed:          passed,
+          failure_message: failure_message,
+          metadata:        enriched_metadata
+        )
+        @evaluation_results << result
+
+        emit(Events::EvaluationRecorded.new(
+          example_id:      @example_id,
+          turn_number:     turn_number,
+          mode:            mode,
+          type:            type,
+          description:     description,
+          passed:          passed,
+          failure_message: failure_message,
+          metadata:        metadata,
+          time:            Time.now
+        ))
+
+        result
+      end
+
+      # Get soft evaluation results only
+      #
+      # @return [Array<EvaluationResult>]
+      def soft_evaluations
+        @evaluation_results.select(&:soft?)
+      end
+
+      # Get hard evaluation results only
+      #
+      # @return [Array<EvaluationResult>]
+      def hard_evaluations
+        @evaluation_results.select(&:hard?)
+      end
+
+      # Format messages for sending to agent
+      # Returns messages in the format expected by agent adapters
+      # Note: role is converted to string for API compatibility
+      #
+      # @return [Array<Hash>]
+      def messages_for_agent
+        @messages.map do |m|
+          { role: m.role.to_s, content: m.content }
+        end
+      end
+
+      # Reset the conversation state
+      def reset!
+        @messages.clear
+        @turns.clear
+        @topic_history.clear
+        @current_topic = nil
+        @turns_per_topic.clear
+        @evaluation_results.clear
+      end
+
+      def to_h
+        {
+          messages:           @messages.map(&:to_h),
+          turns:              @turns.map(&:to_h),
+          topic_history:      @topic_history,
+          current_topic:      @current_topic,
+          evaluation_results: @evaluation_results.map(&:to_h)
+        }
+      end
+
+      def inspect
+        "#<#{self.class.name} turns=#{@turns.count} messages=#{@messages.count}>"
+      end
+
+      private
+
+      def emit(event)
+        @event_bus&.publish(event)
+      end
+
+      def add_turn_to_topic_history(turn)
+        return if @topic_history.empty?
+
+        current_entry = @topic_history.last
+        current_entry[:turns] << turn if current_entry[:topic] == @current_topic
+      end
+    end
+  end
+end