rspec-agents 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 499cd9647c7ce1702786bb9d597089a0e9f8d3c56587dd6e9b0e2283ca512dba
-  data.tar.gz: d5d9774ebd6ebf22bb13faf952081c240740c2b2623f277c04de575788361be7
+  metadata.gz: f32aaad8fd16cbd4acd7406452a9b070fbde786ce582fe82964ed846a9547420
+  data.tar.gz: 8132da0b079c695d7f763d22bb65527672856dd1b7f4acf1659f5f580afd1769
 SHA512:
-  metadata.gz: 21bdfb0dd7a6c3ab6442901dbf81827403f2b480d9bfa099b2474ad7b8f20bc20935ebb98e31c88cebe9be41746d630049070ac02fecc684a1b84d07ae505df1
-  data.tar.gz: b77860c9f87960fcbbda50188347ef7a05af71b15eb9e59e7e15ea3bb04c14b3fc276115271e96fb883172986d8b53e378a822e08eadfc6d8493b4b4cd6ce804
+  metadata.gz: 102605b1124b6b0c0f052465e3347ccd5779e3218c797e909a702c8ff282fca8b015100b8002f830637cfe38c55a9e2defe7447dcff0a3f06601c61d9de56141
+  data.tar.gz: 732cc6a14e67137fadfaf66bd960a81443a500dada2f206ab462220d8945543594fe0777fe6eac3c061210ca664e8a399a5bb5df087a5283a7187ce927589d9e

data/bin/rspec-agents

@@ -4,8 +4,7 @@
 # Unified CLI for rspec-agents
 #
 # Usage:
-#   rspec-agents [run] [options] [paths...]     # Single-process (default)
-#   rspec-agents parallel [options] [paths...]  # Parallel with workers
+#   rspec-agents [options] [paths...]           # Run specs (use -w for parallel)
 #   rspec-agents render <json_file> [options]   # Render HTML from JSON
 #   rspec-agents worker                         # Internal: worker mode
 

data/lib/rspec/agents/cli.rb

@@ -6,23 +6,24 @@ module RSpec
   module Agents
     # Unified CLI for rspec-agents
     #
-    # Commands:
-    #   run      - Single-process execution (default)
-    #   parallel - Parallel execution with worker processes
-    #   worker   - Internal: run as a worker subprocess
+    # Subcommands:
+    #   render  - Generate HTML report from JSON file
+    #   worker  - Internal: run as a worker subprocess
+    #
+    # Without a subcommand, runs specs directly. Pass -w/--workers to
+    # enable parallel execution.
     #
     # @example Single process
     #   CLI.run(["spec/"])
-    #   CLI.run(["run", "spec/"])
     #
     # @example Parallel
-    #   CLI.run(["parallel", "-w", "4", "spec/"])
+    #   CLI.run(["-w", "4", "spec/"])
     #
     # @example Worker (internal)
     #   CLI.run(["worker"])
     #
     class CLI
-      COMMANDS = %w[run parallel worker render].freeze
+      SUBCOMMANDS = %w[worker render].freeze
 
       def self.run(argv)
         new(argv).run
@@ -33,159 +34,86 @@ module RSpec
       end
 
       def run
-        command, args = parse_command(@argv)
+        command, args = extract_subcommand(@argv)
 
         case command
-        when "run"
-          run_single(args)
-        when "parallel"
-          run_parallel(args)
         when "worker"
           run_worker(args)
         when "render"
           run_render(args)
         else
-          # Default to single-process run
-          run_single(@argv)
+          run_specs(args)
         end
       end
 
       private
 
-      def parse_command(argv)
+      def extract_subcommand(argv)
         return [nil, argv] if argv.empty?
 
         first = argv.first
-
-        # Auto-detect parallel mode if -w/--workers flag is present
-        if has_parallel_flag?(argv)
-          # If first arg is explicit command, consume it; otherwise keep all args
-          if COMMANDS.include?(first)
-            return ["parallel", argv[1..]]
-          else
-            return ["parallel", argv]
-          end
-        end
-
-        # Original logic for explicit commands or default to single-process
-        if COMMANDS.include?(first)
+        if SUBCOMMANDS.include?(first)
           [first, argv[1..]]
-        elsif first.start_with?("-")
-          # Flag, not a command - default to run
-          [nil, argv]
         else
-          # Path or unknown - default to run
           [nil, argv]
         end
       end
 
-      def has_parallel_flag?(argv)
-        argv.any? { |arg| arg == "-w" || arg.start_with?("--workers") }
-      end
-
       # =========================================================================
-      # Single-process mode
+      # Spec execution (single-process or parallel via -w)
       # =========================================================================
 
-      def run_single(args)
-        options = parse_single_options(args)
-
-        runner = Runners::TerminalRunner.new(
-          output:     $stdout,
-          color:      options[:color],
-          json_path:  options[:json_path],
-          html_path:  options[:html_path],
-          upload_url: options[:upload_url]
-        )
-        runner.run(options[:paths])
-      end
-
-      def parse_single_options(args)
-        options = { paths: [], color: nil, json_path: nil, html_path: nil, upload_url: nil }
-
-        parser = OptionParser.new do |opts|
-          opts.banner = "Usage: rspec-agents [run] [options] [paths...]"
-          opts.separator ""
-          opts.separator "Run specs in a single process with terminal output."
-          opts.separator ""
-          opts.separator "Options:"
-
-          opts.on("--[no-]color", "Force color on/off (default: auto)") do |v|
-            options[:color] = v
-          end
-
-          opts.on("--ui MODE", [:interactive, :interleaved, :quiet],
-                  "Output mode (ignored in single-process mode)") do |_mode|
-            # Accepted for CLI compatibility with parallel mode, but ignored
-          end
-
-          opts.on("--json PATH", "Save JSON run data to file") do |path|
-            options[:json_path] = path
-          end
-
-          opts.on("--html PATH", "Render HTML report to path") do |path|
-            options[:html_path] = path
-          end
-
-          opts.on("--upload [URL]", "Upload run data to agents-studio (default: http://localhost:9292)") do |url|
-            options[:upload_url] = url || "http://localhost:9292"
-          end
-
-          opts.on("-h", "--help", "Show this help") do
-            puts opts
-            exit 0
-          end
+      def run_specs(args)
+        options = parse_run_options(args)
+
+        if options[:workers]
+          runner = Runners::ParallelTerminalRunner.new(
+            worker_count: options[:workers],
+            fail_fast:    options[:fail_fast],
+            output:       $stdout,
+            color:        options[:color],
+            json_path:    options[:json_path],
+            html_path:    options[:html_path],
+            ui_mode:      options[:ui_mode],
+            upload_url:   options[:upload_url]
+          )
+        else
+          runner = Runners::TerminalRunner.new(
+            output:     $stdout,
+            color:      options[:color],
+            json_path:  options[:json_path],
+            html_path:  options[:html_path],
+            upload_url: options[:upload_url]
+          )
         end
 
-        remaining = parser.parse(args)
-        options[:paths] = remaining.empty? ? ["spec"] : remaining
-        options
-      end
-
-      # =========================================================================
-      # Parallel mode
-      # =========================================================================
-
-      def run_parallel(args)
-        options = parse_parallel_options(args)
-
-        runner = Runners::ParallelTerminalRunner.new(
-          worker_count: options[:workers],
-          fail_fast:    options[:fail_fast],
-          output:       $stdout,
-          color:        options[:color],
-          json_path:    options[:json_path],
-          html_path:    options[:html_path],
-          ui_mode:      options[:ui_mode],
-          upload_url:   options[:upload_url]
-        )
         runner.run(options[:paths])
       end
 
-      def parse_parallel_options(args)
+      def parse_run_options(args)
         options = {
-          workers:    4,
-          fail_fast:  false,
-          paths:      [],
-          color:      nil,
-          json_path:  nil,
-          html_path:  nil,
-          ui_mode:    nil,
+          paths:     [],
+          workers:   nil,
+          fail_fast: false,
+          color:     nil,
+          json_path: nil,
+          html_path: nil,
+          ui_mode:   nil,
           upload_url: nil
         }
 
         parser = OptionParser.new do |opts|
-          opts.banner = "Usage: rspec-agents parallel [options] [paths...]"
+          opts.banner = "Usage: rspec-agents [options] [paths...]"
           opts.separator ""
-          opts.separator "Run specs in parallel across multiple worker processes."
+          opts.separator "Run specs with terminal output. Pass -w to run in parallel."
           opts.separator ""
           opts.separator "Options:"
 
-          opts.on("-w", "--workers COUNT", Integer, "Number of workers (default: 4)") do |w|
+          opts.on("-w", "--workers COUNT", Integer, "Run in parallel with COUNT workers") do |w|
             options[:workers] = w
           end
 
-          opts.on("--fail-fast", "Stop on first failure") do
+          opts.on("--fail-fast", "Stop on first failure (parallel mode)") do
             options[:fail_fast] = true
           end
 
@@ -194,7 +122,7 @@ module RSpec
           end
 
           opts.on("--ui MODE", [:interactive, :interleaved, :quiet],
-                  "Output mode: interactive, interleaved, quiet (default: auto)") do |mode|
+                  "Output mode: interactive, interleaved, quiet (parallel mode, default: auto)") do |mode|
             options[:ui_mode] = mode
           end
 

data/lib/rspec/agents/runners/parallel_terminal_runner.rb

@@ -3,7 +3,7 @@
 require "async"
 require "fileutils"
 require_relative "../parallel/ui/ui_factory"
-require_relative "run_data_uploader"
+require_relative "streaming_run_data_uploader"
 
 module RSpec
   module Agents
@@ -94,6 +94,12 @@ module RSpec
           executor.on_event { |type, event| route_event_to_ui(type, event) }
           executor.on_progress { |c, t, f| @ui.on_progress(completed: c, total: t, failures: f) }
 
+          # Wire up streaming upload if --upload specified
+          if @upload_url
+            streaming_uploader = StreamingRunDataUploader.new(url: @upload_url, output: @output)
+            attach_streaming_upload(streaming_uploader, executor)
+          end
+
           # Start UI
           @ui.on_run_started(worker_count: @worker_count, example_count: examples.size)
           @ui.start_input_handling
@@ -114,8 +120,11 @@ module RSpec
               print_failed_examples_filter
             end
 
-            # Save outputs
-            save_outputs(executor.run_data) if @json_path || @html_path || @upload_url
+            # Finish streaming upload (blocks until queue drained)
+            streaming_uploader&.finish
+
+            # Save file outputs
+            save_outputs(executor.run_data) if @json_path || @html_path
 
             result&.success? ? 0 : 1
           ensure
@@ -204,6 +213,15 @@ module RSpec
           "#{COLORS[color]}#{text}#{COLORS[:reset]}"
         end
 
+        def attach_streaming_upload(uploader, executor)
+          executor.on_event do |type, _event|
+            uploader.start(executor.run_data) if type == "SuiteStarted"
+          end
+          executor.on_example_completed do |event, run_data|
+            uploader.upload_example(event, run_data)
+          end
+        end
+
         def save_outputs(run_data)
           return unless run_data
 
@@ -216,10 +234,6 @@ module RSpec
           if @html_path
             Serialization::TestSuiteRenderer.render(run_data, output_path: @html_path)
           end
-
-          if @upload_url
-            RunDataUploader.new(url: @upload_url, output: @output).upload(run_data)
-          end
         end
 
         def populate_rendered_extensions(run_data)

data/lib/rspec/agents/runners/streaming_run_data_uploader.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module RSpec
+  module Agents
+    module Runners
+      # Uploads run data to an agents-studio webapp incrementally, one example at a time.
+      # Each upload sends a minimal RunData containing run metadata + just that example.
+      # The receiving /api/import endpoint is idempotent with respect to run_id, merging
+      # examples into the same run record.
+      #
+      # Uses a background thread with a Queue so uploads never block spec execution.
+      #
+      # @example
+      #   uploader = StreamingRunDataUploader.new(url: "http://localhost:9292", output: $stdout)
+      #   uploader.start(run_data)
+      #   executor.on_example_completed { |event, rd| uploader.upload_example(event, rd) }
+      #   # ... after suite finishes ...
+      #   uploader.finish
+      #
+      class StreamingRunDataUploader
+        TIMEOUT = 10 # seconds per request
+
+        # @param url [String] Base URL of the agents-studio webapp
+        # @param output [IO] Output stream for status messages
+        def initialize(url:, output: $stdout)
+          @url = url.chomp("/")
+          @output = output
+          @queue = Queue.new
+          @failed_examples = {}
+          @mutex = Mutex.new
+          @worker_thread = nil
+          @http = nil
+          @uploaded_count = 0
+          @run_id = nil
+        end
+
+        # Whether the uploader has been started.
+        def started?
+          !!@run_id
+        end
+
+        # Start the background upload worker.
+        # Idempotent — safe to call multiple times; only the first call takes effect.
+        # @param run_data [Serialization::RunData] The run data (used for metadata)
+        def start(run_data)
+          return if started?
+
+          @run_id = run_data.run_id
+          @run_metadata = {
+            run_id:     run_data.run_id,
+            started_at: run_data.started_at,
+            seed:       run_data.seed,
+            git_commit: run_data.git_commit
+          }
+          @output.puts "Streaming run data to #{@url}..."
+          @worker_thread = Thread.new { process_queue }
+        end
+
+        # Enqueue a single example for upload.
+        # Meant to be called from an on_example_completed callback.
+        # @param event [Events::ExamplePassed, Events::ExampleFailed, Events::ExamplePending]
+        # @param run_data [Serialization::RunData] Current run data
+        def upload_example(event, run_data)
+          return unless @run_id
+
+          example_data = run_data&.example(event.example_id)
+          return unless example_data
+
+          @queue.push([:upload, { example_id: event.example_id, example_data: example_data }])
+        end
+
+        # Signal the worker to finish: retry failed examples, then drain and stop.
+        # Blocks until the worker thread completes (up to 60s).
+        def finish
+          return unless @worker_thread
+
+          @queue.push([:finish, nil])
+          @worker_thread.join(60)
+          close_http
+
+          if @uploaded_count > 0
+            @output.puts "Upload complete: #{@uploaded_count} example#{"s" unless @uploaded_count == 1} streamed"
+          end
+
+          if @failed_examples.any?
+            @output.puts "Warning: #{@failed_examples.size} example(s) failed to upload"
+          end
+        end
+
+        private
+
+        def process_queue
+          loop do
+            action, payload = @queue.pop
+
+            case action
+            when :upload
+              do_upload_example(payload)
+            when :finish
+              retry_failed_examples
+              break
+            end
+          end
+        rescue => e
+          @output.puts "Upload worker error: #{e.message}"
+        end
+
+        def do_upload_example(payload)
+          example_data = payload[:example_data]
+          stable_id = example_data.stable_id || payload[:example_id]
+
+          # Build a minimal RunData with just this one example
+          body = build_single_example_payload(example_data)
+          post("/api/import", body)
+
+          @mutex.synchronize do
+            @uploaded_count += 1
+            @failed_examples.delete(stable_id)
+          end
+        rescue => e
+          @mutex.synchronize do
+            @failed_examples[stable_id] = payload
+          end
+        end
+
+        def retry_failed_examples
+          failed = @mutex.synchronize { @failed_examples.dup }
+          failed.each do |stable_id, payload|
+            do_upload_example(payload)
+          rescue
+            # Leave in @failed_examples for final warning
+          end
+        end
+
+        def build_single_example_payload(example_data)
+          {
+            run_id:     @run_metadata[:run_id],
+            started_at: serialize_time(@run_metadata[:started_at]),
+            seed:       @run_metadata[:seed],
+            git_commit: @run_metadata[:git_commit],
+            scenarios:  {},
+            examples:   {
+              example_data.id => example_data.to_h
+            }
+          }
+        end
+
+        def serialize_time(time)
+          return nil unless time
+          time.is_a?(Time) ? time.iso8601(3) : time
+        end
+
+        def http_connection
+          @http ||= begin
+            uri = URI.parse(@url)
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = (uri.scheme == "https")
+            http.open_timeout = TIMEOUT
+            http.read_timeout = TIMEOUT
+            http.start
+            http
+          end
+        end
+
+        def close_http
+          @http&.finish
+        rescue
+          # Ignore close errors
+        ensure
+          @http = nil
+        end
+
+        def post(path, body)
+          uri = URI.parse("#{@url}#{path}")
+          request = Net::HTTP::Post.new(uri.path)
+          request["Content-Type"] = "application/json"
+          request.body = JSON.generate(body)
+
+          response = http_connection.request(request)
+          unless response.is_a?(Net::HTTPSuccess)
+            raise "HTTP #{response.code}: #{response.body}"
+          end
+          response
+        rescue Errno::ECONNREFUSED, Errno::ECONNRESET, IOError, Net::OpenTimeout, Net::ReadTimeout => e
+          close_http # Reset connection on transport errors
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/rspec/agents/runners/terminal_runner.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "fileutils"
-require_relative "run_data_uploader"
+require_relative "streaming_run_data_uploader"
 
 module RSpec
   module Agents
@@ -50,10 +50,19 @@ module RSpec
           # Wire up event handling for terminal display
           executor.on_event { |type, event| handle_event(type, event) }
 
+          # Wire up streaming upload if --upload specified
+          if @upload_url
+            streaming_uploader = StreamingRunDataUploader.new(url: @upload_url, output: @output)
+            attach_streaming_upload(streaming_uploader, executor)
+          end
+
           result = executor.execute(Array(files_or_args))
 
-          # Save outputs after run completes
-          save_outputs(executor.run_data) if @json_path || @html_path || @upload_url
+          # Finish streaming upload (blocks until queue drained)
+          streaming_uploader&.finish
+
+          # Save file outputs after run completes
+          save_outputs(executor.run_data) if @json_path || @html_path
 
           result.success? ? 0 : 1
         end
@@ -166,6 +175,15 @@ module RSpec
           "#{COLORS[color]}#{text}#{COLORS[:reset]}"
         end
 
+        def attach_streaming_upload(uploader, executor)
+          executor.on_event do |type, _event|
+            uploader.start(executor.run_data) if type == "SuiteStarted"
+          end
+          executor.on_example_completed do |event, run_data|
+            uploader.upload_example(event, run_data)
+          end
+        end
+
         def save_outputs(run_data)
           return unless run_data
 
@@ -178,10 +196,6 @@ module RSpec
           if @html_path
             Serialization::TestSuiteRenderer.render(run_data, output_path: @html_path)
           end
-
-          if @upload_url
-            RunDataUploader.new(url: @upload_url, output: @output).upload(run_data)
-          end
         end
 
         def populate_rendered_extensions(run_data)

data/lib/rspec/agents/turn_executor.rb

@@ -55,6 +55,14 @@ module RSpec
         @conversation.current_topic
       end
 
+      # Get tool calls from the current response, optionally filtered by name
+      # @param name [Symbol, String, nil] Optional tool name to filter by
+      # @return [Array<ToolCall>]
+      def tool_calls(name = nil)
+        calls = @current_response&.tool_calls || []
+        name ? calls.select { |tc| tc.name == name.to_sym } : calls
+      end
+
       # Check if current turn is in expected topic
       #
       # @param expected_topic [Symbol] Expected topic name

data/lib/rspec/agents/version.rb

@@ -2,6 +2,6 @@
 
 module RSpec
   module Agents
-    VERSION = "0.1.2"
+    VERSION = "0.1.3"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rspec-agents
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Frederik Fix
@@ -186,7 +186,7 @@ files:
 - lib/rspec/agents/prompt_builders/user_simulation.rb
 - lib/rspec/agents/runners/headless_runner.rb
 - lib/rspec/agents/runners/parallel_terminal_runner.rb
-- lib/rspec/agents/runners/run_data_uploader.rb
+- lib/rspec/agents/runners/streaming_run_data_uploader.rb
 - lib/rspec/agents/runners/terminal_runner.rb
 - lib/rspec/agents/runners/user_simulator.rb
 - lib/rspec/agents/scenario.rb

data/lib/rspec/agents/runners/run_data_uploader.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-require "net/http"
-require "uri"
-require "json"
-
-module RSpec
-  module Agents
-    module Runners
-      # Uploads run data to an agents-studio webapp via HTTP POST.
-      # Used by TerminalRunner and ParallelTerminalRunner when --upload is specified.
-      #
-      # @example
-      #   uploader = RunDataUploader.new(url: "http://localhost:9292")
-      #   uploader.upload(run_data) # => true/false
-      #
-      class RunDataUploader
-        TIMEOUT = 30 # seconds
-
-        # @param url [String] Base URL of the agents-studio webapp
-        # @param output [IO] Output stream for status messages
-        def initialize(url:, output: $stdout)
-          @url = url.chomp("/")
-          @output = output
-        end
-
-        # Upload run data to the webapp.
-        # @param run_data [Serialization::RunData] The run data to upload
-        # @return [Boolean] true if upload succeeded
-        def upload(run_data)
-          return false unless run_data
-
-          uri = URI.parse("#{@url}/api/import")
-          json_body = JSON.generate(run_data.to_h)
-
-          @output.puts "Uploading run data to #{@url}..."
-
-          http = Net::HTTP.new(uri.host, uri.port)
-          http.use_ssl = (uri.scheme == "https")
-          http.open_timeout = TIMEOUT
-          http.read_timeout = TIMEOUT
-
-          request = Net::HTTP::Post.new(uri.path)
-          request["Content-Type"] = "application/json"
-          request.body = json_body
-
-          response = http.request(request)
-
-          if response.is_a?(Net::HTTPSuccess)
-            result = JSON.parse(response.body)
-            @output.puts "Upload complete: #{result["example_count"]} examples " \
-                         "(#{result["passed_count"]} passed, #{result["failed_count"]} failed)"
-            true
-          else
-            @output.puts "Upload failed (HTTP #{response.code}): #{response.body}"
-            false
-          end
-        rescue Errno::ECONNREFUSED
-          @output.puts "Upload failed: could not connect to #{@url} (is agents-studio running?)"
-          false
-        rescue Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout
-          @output.puts "Upload failed: connection to #{@url} timed out"
-          false
-        rescue StandardError => e
-          @output.puts "Upload failed: #{e.message}"
-          false
-        end
-      end
-    end
-  end
-end