agent-harness 0.5.6 → 0.5.8

data/lib/agent_harness/command_executor.rb

@@ -39,23 +39,47 @@ module AgentHarness
     #
     # @param command [Array<String>, String] command to execute
     # @param timeout [Integer, nil] timeout in seconds
+    # @param idle_timeout [Integer, Float, nil] idle timeout in seconds based on output activity
     # @param env [Hash] environment variables
     # @param stdin_data [String, nil] data to send to stdin
+    # @param on_stdout_chunk [Proc, nil] callback for stdout chunks as they are produced
+    # @param on_stderr_chunk [Proc, nil] callback for stderr chunks as they are produced
+    # @param on_heartbeat [Proc, nil] callback invoked periodically while the command is running
+    # @param heartbeat_interval [Integer, Float] heartbeat interval in seconds
+    # @param observer [Object, nil] optional observer responding to
+    #   +on_stdout_chunk+, +on_stderr_chunk+, and +on_heartbeat+
     # @return [Result] execution result
     # @raise [TimeoutError] if the command times out
-    def execute(command, timeout: nil, env: {}, stdin_data: nil)
+    # @raise [IdleTimeoutError] if the command exceeds the idle timeout
+    def execute(command, timeout: nil, idle_timeout: nil, env: {}, stdin_data: nil,
+      on_stdout_chunk: nil, on_stderr_chunk: nil, on_heartbeat: nil,
+      heartbeat_interval: 1.0, observer: nil)
+      validate_duration!(timeout, name: :timeout, allow_nil: true)
+      validate_duration!(idle_timeout, name: :idle_timeout, allow_nil: true)
+      validate_duration!(heartbeat_interval, name: :heartbeat_interval, allow_nil: true)
+
       cmd_array = normalize_command(command)
       cmd_string = cmd_array.shelljoin
 
-      log_debug("Executing command", command: cmd_string, timeout: timeout)
+      log_debug("Executing command",
+        command: cmd_string,
+        timeout: timeout,
+        idle_timeout: idle_timeout)
 
       start_time = Time.now
 
-      stdout, stderr, status = if timeout
-        execute_with_timeout(cmd_array, timeout: timeout, env: env, stdin_data: stdin_data)
-      else
-        execute_without_timeout(cmd_array, env: env, stdin_data: stdin_data)
-      end
+      stdout, stderr, status = execute_streaming(
+        cmd_array,
+        timeout: timeout,
+        idle_timeout: idle_timeout,
+        env: env,
+        stdin_data: stdin_data,
+        on_stdout_chunk: on_stdout_chunk,
+        on_stderr_chunk: on_stderr_chunk,
+        on_heartbeat: on_heartbeat,
+        heartbeat_interval: heartbeat_interval,
+        observer: observer
+      )
 
       duration = Time.now - start_time
 
@@ -102,47 +126,444 @@ module AgentHarness
 
     private
 
-    def execute_with_timeout(cmd_array, timeout:, env:, stdin_data:)
-      stdout = ""
-      stderr = ""
-      status = nil
+    def execute_streaming(cmd_array, timeout:, idle_timeout:, env:, stdin_data:,
+      on_stdout_chunk:, on_stderr_chunk:, on_heartbeat:, heartbeat_interval:, observer:)
+      stdout = +""
+      stderr = +""
 
-      Timeout.timeout(timeout) do
-        Open3.popen3(env, *cmd_array) do |stdin, stdout_io, stderr_io, wait_thr|
-          if stdin_data
-            stdin.write(stdin_data)
+      Open3.popen3(env, *cmd_array, pgroup: true) do |stdin, stdout_io, stderr_io, wait_thr|
+        unless selectable_streams?(stdin, stdout_io, stderr_io)
+          return execute_buffered(
+            stdin,
+            stdout_io,
+            stderr_io,
+            wait_thr,
+            stdin_data: stdin_data,
+            stdout: stdout,
+            stderr: stderr,
+            timeout: timeout,
+            idle_timeout: idle_timeout,
+            cmd_array: cmd_array,
+            on_stdout_chunk: on_stdout_chunk,
+            on_stderr_chunk: on_stderr_chunk,
+            on_heartbeat: on_heartbeat,
+            heartbeat_interval: heartbeat_interval,
+            observer: observer
+          )
+        end
+
+        start_time = monotonic_time
+        last_output_at = start_time
+        last_heartbeat_at = start_time
+        stdin_buffer = stdin_data.is_a?(String) ? stdin_data : stdin_data.to_s
+        stdin_offset = 0
+        streams = {
+          stdout_io => [stdout, on_stdout_chunk, :on_stdout_chunk],
+          stderr_io => [stderr, on_stderr_chunk, :on_stderr_chunk]
+        }
+
+        until streams.empty? && stdin.nil?
+          ready = IO.select(streams.keys, stdin ? [stdin] : nil, nil, 0)
+
+          if ready
+            stdin, stdin_offset, last_output_at = process_ready_streams(
+              ready,
+              streams: streams,
+              stdin: stdin,
+              stdin_buffer: stdin_buffer,
+              stdin_offset: stdin_offset,
+              last_output_at: last_output_at,
+              observer: observer,
+              wait_thr: wait_thr
+            )
+
+            if process_exited?(wait_thr)
+              stdin, last_output_at = finalize_exited_process(
+                stdin, streams, observer, last_output_at, wait_thr,
+                timeout: timeout, idle_timeout: idle_timeout, start_time: start_time, cmd_array: cmd_array
+              )
+              next
+            end
+
+            now = monotonic_time
+            if should_emit_heartbeat?(on_heartbeat, observer, heartbeat_interval, now - last_heartbeat_at)
+              emit_heartbeat(
+                on_heartbeat,
+                observer,
+                elapsed: now - start_time,
+                idle_for: now - last_output_at,
+                wait_thr: wait_thr
+              )
+              last_heartbeat_at = now
+            end
+
+            check_wall_timeout!(timeout, now - start_time, wait_thr, cmd_array)
+            check_idle_timeout!(idle_timeout, now - last_output_at, wait_thr, cmd_array)
+            next
           end
-          stdin.close
 
-          # Read output streams
-          stdout = stdout_io.read
-          stderr = stderr_io.read
-          status = wait_thr.value
+          if process_exited?(wait_thr)
+            stdin, last_output_at = finalize_exited_process(
+              stdin, streams, observer, last_output_at, wait_thr,
+              timeout: timeout, idle_timeout: idle_timeout, start_time: start_time, cmd_array: cmd_array
+            )
+            next
+          end
+
+          now = monotonic_time
+          check_wall_timeout!(timeout, now - start_time, wait_thr, cmd_array)
+          check_idle_timeout!(idle_timeout, now - last_output_at, wait_thr, cmd_array)
+
+          if should_emit_heartbeat?(on_heartbeat, observer, heartbeat_interval, now - last_heartbeat_at)
+            emit_heartbeat(
+              on_heartbeat,
+              observer,
+              elapsed: now - start_time,
+              idle_for: now - last_output_at,
+              wait_thr: wait_thr
+            )
+            last_heartbeat_at = now
+          end
+
+          ready = IO.select(
+            streams.keys,
+            stdin ? [stdin] : nil,
+            nil,
+            select_timeout(
+              timeout,
+              idle_timeout,
+              heartbeat_interval,
+              elapsed: now - start_time,
+              idle_for: now - last_output_at,
+              heartbeat_age: now - last_heartbeat_at,
+              heartbeat_requested: on_heartbeat || observer_responds_to?(observer, :on_heartbeat)
+            )
+          )
+
+          unless ready
+            if process_exited?(wait_thr)
+              stdin, last_output_at = finalize_exited_process(
+                stdin, streams, observer, last_output_at, wait_thr,
+                timeout: timeout, idle_timeout: idle_timeout, start_time: start_time, cmd_array: cmd_array
+              )
+            end
+            next
+          end
+
+          stdin, stdin_offset, last_output_at = process_ready_streams(
+            ready,
+            streams: streams,
+            stdin: stdin,
+            stdin_buffer: stdin_buffer,
+            stdin_offset: stdin_offset,
+            last_output_at: last_output_at,
+            observer: observer,
+            wait_thr: wait_thr
+          )
+
+          if process_exited?(wait_thr)
+            stdin, last_output_at = finalize_exited_process(
+              stdin, streams, observer, last_output_at, wait_thr,
+              timeout: timeout, idle_timeout: idle_timeout, start_time: start_time, cmd_array: cmd_array
+            )
+          end
         end
+
+        # Supervise the wait for process exit so a child that closed its
+        # stdio but keeps running cannot hang past the configured timeouts.
+        unless process_exited?(wait_thr)
+          supervise_process_exit(
+            wait_thr,
+            timeout: timeout,
+            idle_timeout: idle_timeout,
+            start_time: start_time,
+            last_output_at: last_output_at,
+            cmd_array: cmd_array,
+            on_heartbeat: on_heartbeat,
+            observer: observer,
+            heartbeat_interval: heartbeat_interval,
+            last_heartbeat_at: last_heartbeat_at
+          )
+        end
+
+        [stdout, stderr, wait_thr.value]
       end
+    end
 
-      [stdout, stderr, status]
+    def log_debug(message, **context)
+      @logger&.debug("[AgentHarness::CommandExecutor] #{message}: #{context.inspect}")
+    end
+
+    def selectable_streams?(*streams)
+      streams.all? { |stream| stream.is_a?(IO) }
+    end
+
+    def execute_buffered(stdin, stdout_io, stderr_io, wait_thr, stdin_data:, stdout:, stderr:, timeout:, idle_timeout:,
+      cmd_array:, on_stdout_chunk:, on_stderr_chunk:, on_heartbeat:, heartbeat_interval:, observer:)
+      validate_buffered_execution_support!(
+        idle_timeout: idle_timeout,
+        on_heartbeat: on_heartbeat,
+        heartbeat_interval: heartbeat_interval,
+        observer: observer
+      )
+
+      result = lambda do
+        write_stdin_buffered(stdin, stdin_data)
+
+        stdout_chunk = stdout_io.read.to_s
+        stderr_chunk = stderr_io.read.to_s
+
+        unless stdout_chunk.empty?
+          stdout << stdout_chunk
+          emit_chunk(on_stdout_chunk, observer, :on_stdout_chunk, stdout_chunk, wait_thr: wait_thr)
+        end
+
+        unless stderr_chunk.empty?
+          stderr << stderr_chunk
+          emit_chunk(on_stderr_chunk, observer, :on_stderr_chunk, stderr_chunk, wait_thr: wait_thr)
+        end
+
+        [stdout, stderr, wait_thr.value]
+      end
+
+      return result.call unless timeout
+
+      Timeout.timeout(timeout) do
+        result.call
+      end
     rescue Timeout::Error
+      terminate_process(wait_thr)
       raise TimeoutError, "Command timed out after #{timeout} seconds: #{cmd_array.first}"
     end
 
-    def execute_without_timeout(cmd_array, env:, stdin_data:)
-      Open3.popen3(env, *cmd_array) do |stdin, stdout_io, stderr_io, wait_thr|
-        if stdin_data
-          stdin.write(stdin_data)
+    def write_stdin_buffered(stdin, stdin_data)
+      return unless stdin
+
+      stdin.write(stdin_data.to_s)
+      close_stream(stdin)
+    rescue Errno::EPIPE, IOError
+      close_stream(stdin)
+    end
+
+    def write_stdin_nonblock(stdin, stdin_buffer, stdin_offset)
+      return stdin_buffer.bytesize if stdin_offset >= stdin_buffer.bytesize
+
+      chunk = stdin_buffer.byteslice(stdin_offset, 4096)
+      written = stdin.write_nonblock(chunk, exception: false)
+      return stdin_offset if written == :wait_writable
+
+      stdin_offset + written
+    end
+
+    def process_ready_streams(ready, streams:, stdin:, stdin_buffer:, stdin_offset:, last_output_at:, observer:, wait_thr:)
+      ready[1]&.each do |io|
+        stdin_offset = write_stdin_nonblock(io, stdin_buffer, stdin_offset)
+        next unless stdin_offset >= stdin_buffer.bytesize
+
+        close_stream(io)
+        stdin = nil
+      rescue Errno::EPIPE, IOError
+        close_stream(io)
+        stdin = nil
+      end
+
+      ready[0]&.each do |io|
+        chunk = io.read_nonblock(4096, exception: false)
+
+        case chunk
+        when :wait_readable
+          next
+        when nil
+          streams.delete(io)
+          io.close
+        else
+          buffer, callback, observer_method = streams.fetch(io) do
+            raise KeyError, "Unexpected ready stream for command execution"
+          end
+          buffer << chunk
+          last_output_at = monotonic_time
+          emit_chunk(callback, observer, observer_method, chunk, wait_thr: wait_thr)
         end
-        stdin.close
+      end
+
+      [stdin, stdin_offset, last_output_at]
+    end
+
+    def close_stream(stream)
+      return unless stream
+
+      stream.close
+    rescue IOError
+      nil
+    end
+
+    def process_exited?(wait_thr)
+      !wait_thr.join(0).nil?
+    end
+
+    # Drain remaining output from an exited process's streams.
+    #
+    # Uses nonblocking reads with timeout supervision so that descendant
+    # processes holding stdout/stderr open cannot hang past the configured
+    # wall-clock or idle timeout.
+    def finalize_exited_process(stdin, streams, observer, last_output_at, wait_thr,
+      timeout: nil, idle_timeout: nil, start_time: nil, cmd_array: nil)
+      close_stream(stdin) if stdin
+
+      until streams.empty?
+        ready = IO.select(streams.keys, nil, nil, 0.1)
+
+        if ready
+          ready[0].each do |io|
+            chunk = io.read_nonblock(4096, exception: false)
+
+            case chunk
+            when :wait_readable
+              next
+            when nil
+              streams.delete(io)
+              close_stream(io)
+            else
+              buffer, callback, observer_method = streams.fetch(io)
+              buffer << chunk
+              last_output_at = monotonic_time
+              emit_chunk(callback, observer, observer_method, chunk, wait_thr: wait_thr)
+            end
+          end
+        end
+
+        now = monotonic_time
+        check_wall_timeout!(timeout, now - start_time, wait_thr, cmd_array) if start_time
+        check_idle_timeout!(idle_timeout, now - last_output_at, wait_thr, cmd_array)
+      end
+
+      [nil, last_output_at]
+    end
+
+    # Poll for process exit with timeout and heartbeat supervision.
+    # Called after all streams are closed when the child is still running.
+    def supervise_process_exit(wait_thr, timeout:, idle_timeout:, start_time:,
+      last_output_at:, cmd_array:, on_heartbeat:, observer:,
+      heartbeat_interval:, last_heartbeat_at:)
+      loop do
+        break if process_exited?(wait_thr)
 
-        stdout = stdout_io.read
-        stderr = stderr_io.read
-        status = wait_thr.value
+        now = monotonic_time
+        check_wall_timeout!(timeout, now - start_time, wait_thr, cmd_array)
+        check_idle_timeout!(idle_timeout, now - last_output_at, wait_thr, cmd_array)
 
-        [stdout, stderr, status]
+        if should_emit_heartbeat?(on_heartbeat, observer, heartbeat_interval, now - last_heartbeat_at)
+          emit_heartbeat(
+            on_heartbeat,
+            observer,
+            elapsed: now - start_time,
+            idle_for: now - last_output_at,
+            wait_thr: wait_thr
+          )
+          last_heartbeat_at = now
+        end
+
+        wait_timeout = select_timeout(
+          timeout, idle_timeout, heartbeat_interval,
+          elapsed: now - start_time,
+          idle_for: now - last_output_at,
+          heartbeat_age: now - last_heartbeat_at,
+          heartbeat_requested: on_heartbeat || observer_responds_to?(observer, :on_heartbeat)
+        )
+        # Sleep briefly before re-checking, bounded by the next deadline
+        sleep([wait_timeout || 0.1, 0.1].min)
       end
     end
 
-    def log_debug(message, **context)
-      @logger&.debug("[AgentHarness::CommandExecutor] #{message}: #{context.inspect}")
+    def validate_duration!(value, name:, allow_nil: false)
+      return if allow_nil && value.nil?
+      return if value.is_a?(Numeric) && value.positive?
+
+      raise InvalidDurationError, "#{name} must be a positive number"
+    end
+
+    def validate_buffered_execution_support!(idle_timeout:, on_heartbeat:, heartbeat_interval:, observer:)
+      heartbeat_requested = on_heartbeat || observer_responds_to?(observer, :on_heartbeat)
+      unsupported_supervision = idle_timeout || (heartbeat_requested && !heartbeat_interval.nil?)
+      return unless unsupported_supervision
+
+      raise ArgumentError, "Buffered command execution does not support idle timeouts or heartbeats"
+    end
+
+    def monotonic_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def check_wall_timeout!(timeout, elapsed, wait_thr, cmd_array)
+      return unless timeout && elapsed >= timeout
+
+      terminate_process(wait_thr)
+      raise TimeoutError, "Command timed out after #{timeout} seconds: #{cmd_array.first}"
+    end
+
+    def check_idle_timeout!(idle_timeout, idle_for, wait_thr, cmd_array)
+      return unless idle_timeout && idle_for >= idle_timeout
+
+      terminate_process(wait_thr)
+      raise IdleTimeoutError,
+        "Command exceeded idle timeout after #{idle_timeout} seconds: #{cmd_array.first}"
+    end
+
+    def should_emit_heartbeat?(callback, observer, heartbeat_interval, heartbeat_age)
+      return false unless callback || observer_responds_to?(observer, :on_heartbeat)
+      return false if heartbeat_interval.nil?
+
+      heartbeat_age >= heartbeat_interval
+    end
+
+    def emit_chunk(callback, observer, observer_method, chunk, wait_thr: nil)
+      callback&.call(chunk)
+      observer.public_send(observer_method, chunk) if observer_responds_to?(observer, observer_method)
+    rescue
+      terminate_process(wait_thr) if wait_thr
+      raise
+    end
+
+    def emit_heartbeat(callback, observer, elapsed:, idle_for:, wait_thr: nil)
+      callback&.call(elapsed: elapsed, idle_for: idle_for)
+      observer.on_heartbeat(elapsed: elapsed, idle_for: idle_for) if observer_responds_to?(observer, :on_heartbeat)
+    rescue
+      terminate_process(wait_thr) if wait_thr
+      raise
+    end
+
+    def observer_responds_to?(observer, method_name)
+      observer&.respond_to?(method_name)
+    end
+
+    def select_timeout(timeout, idle_timeout, heartbeat_interval, elapsed:, idle_for:, heartbeat_age:, heartbeat_requested:)
+      timeouts = []
+      timeouts << (timeout - elapsed) if timeout
+      timeouts << (idle_timeout - idle_for) if idle_timeout
+      timeouts << (heartbeat_interval - heartbeat_age) if heartbeat_requested && heartbeat_interval
+
+      min_timeout = timeouts.min
+      return nil unless min_timeout
+
+      [min_timeout, 0].max
+    end
+
+    def terminate_process(wait_thr)
+      pid = wait_thr.pid
+      signal_process("TERM", pid)
+      Timeout.timeout(1) { wait_thr.join }
+    rescue Errno::ESRCH, Timeout::Error
+      begin
+        signal_process("KILL", pid)
+      rescue Errno::ESRCH
+        nil
+      end
+    end
+
+    def signal_process(signal, pid)
+      Process.kill(signal, -pid)
+    rescue Errno::ESRCH
+      Process.kill(signal, pid)
     end
   end
 end

data/lib/agent_harness/docker_command_executor.rb

@@ -35,12 +35,20 @@ module AgentHarness
     #
     # @param command [Array<String>, String] command to execute
     # @param timeout [Integer, nil] timeout in seconds
+    # @param idle_timeout [Integer, Float, nil] idle timeout in seconds based on output activity
     # @param env [Hash] environment variables to set in the container
     # @param stdin_data [String, nil] data to send to stdin
     # @return [Result] execution result
-    def execute(command, timeout: nil, env: {}, stdin_data: nil)
+    def execute(command, timeout: nil, idle_timeout: nil, env: {}, stdin_data: nil, **execution_options)
       docker_cmd = build_docker_command(command, env: env, stdin_data: stdin_data)
-      super(docker_cmd, timeout: timeout, env: {}, stdin_data: stdin_data)
+      super(
+        docker_cmd,
+        timeout: timeout,
+        idle_timeout: idle_timeout,
+        env: {},
+        stdin_data: stdin_data,
+        **execution_options
+      )
     end
 
     # Check if a binary exists inside the container
@@ -62,11 +70,23 @@ module AgentHarness
 
     def build_docker_command(command, env:, stdin_data:)
       cmd = ["docker", "exec"]
+      unset_env_keys = []
 
-      env.each { |key, value| cmd.push("--env", "#{key}=#{value}") }
+      env.each do |key, value|
+        if value.nil?
+          unset_env_keys << key
+          next
+        end
+
+        cmd.push("--env", "#{key}=#{value}")
+      end
       cmd.push("-i") if stdin_data
 
       cmd.push(@container_id)
+      unless unset_env_keys.empty?
+        cmd.push("env")
+        unset_env_keys.each { |key| cmd.push("-u", key) }
+      end
 
       cmd.concat(normalize_command(command))
     end

data/lib/agent_harness/error_taxonomy.rb

@@ -39,6 +39,11 @@ module AgentHarness
         action: :retry_with_backoff,
         retryable: true
       },
+      idle_timeout: {
+        description: "Operation exceeded idle timeout",
+        action: :escalate,
+        retryable: false
+      },
       sandbox_failure: {
         description: "Sandbox setup failed",
         action: :escalate,
@@ -58,6 +63,9 @@ module AgentHarness
       # @param patterns [Hash<Symbol, Array<Regexp>>] provider-specific patterns
       # @return [Symbol] error category
       def classify(error, patterns = {})
+        return :idle_timeout if error.is_a?(IdleTimeoutError)
+        return :timeout if error.is_a?(TimeoutError)
+
         message = error.message.to_s.downcase
 
         # Check provider-specific patterns first
@@ -112,6 +120,8 @@ module AgentHarness
 
       def classify_generic(message)
         case message
+        when /idle.?timeout/i
+          :idle_timeout
         when /rate.?limit|too many requests|429/i
           :rate_limited
         when /quota|usage.?limit|billing/i

data/lib/agent_harness/errors.rb

@@ -22,6 +22,11 @@ module AgentHarness
   # Execution errors
   class TimeoutError < Error; end
 
+  # Raised when a duration argument is invalid (non-positive)
+  class InvalidDurationError < ArgumentError; end
+
+  class IdleTimeoutError < TimeoutError; end
+
   class CommandExecutionError < Error; end
 
   # Rate limiting and circuit breaker errors