process_executer 3.2.4 → 4.0.0

data/lib/process_executer/monitored_pipe.rb

@@ -5,22 +5,58 @@ require 'io/wait'
 require 'track_open_instances'
 
 module ProcessExecuter
-  # Write data sent through a pipe to a destination
+  # Acts as a pipe that writes the data written to it to one or more destinations
+  #
+  # {ProcessExecuter::MonitoredPipe} was created to expand the output redirection
+  # options for
+  # [Process.spawn](https://docs.ruby-lang.org/en/3.4/Process.html#method-c-spawn)
+  # and methods derived from it within the `ProcessExecuter` module.
+  #
+  # This class's initializer accepts any redirection destination supported by
+  # [Process.spawn](https://docs.ruby-lang.org/en/3.4/Process.html#method-c-spawn)
+  # (this is the `value` part of the file redirection option described in [the File
+  # Redirection section of
+  # `Process.spawn`](https://docs.ruby-lang.org/en/3.4/Process.html#module-Process-label-File+Redirection+-28File+Descriptor-29).
+  #
+  # In addition to the standard redirection destinations, {ProcessExecuter::MonitoredPipe} also
+  # supports these additional types of destinations:
+  #
+  # - **Arbitrary Writers**
+  #
+  #   You can redirect subprocess output to any Ruby object that implements the
+  #   `#write` method. This is particularly useful for:
+  #
+  #     - capturing command output in in-memory buffers like `StringIO`
+  #     - sending command output to custom logging objects that do not have a file
+  #       descriptor
+  #     - processing with a streaming parser to parse and process command output as
+  #       the command is running
+  #
+  # - **Multiple Destinations**
+  #
+  #   MonitoredPipe supports duplicating (or "teeing") output to multiple
+  #   destinations simultaneously. This is achieved by providing a redirection
+  #   destination in the form `[:tee, destination1, destination2, ...]`, where each
+  #   `destination` can be any value that `MonitoredPipe` itself supports (including
+  #   another tee or MonitoredPipe).
   #
   # When a new MonitoredPipe is created, a pipe is created (via IO.pipe) and
-  # a thread is created to read data written to the pipe.
+  # a thread is created to read data written to the pipe. As data is read from the pipe,
+  # it is written to the destination provided in the MonitoredPipe initializer.
   #
   # If the destination raises an exception, the monitoring thread will exit, the
   # pipe will be closed, and the exception will be saved in `#exception`.
   #
-  # `#close` must be called to ensure that (1) the pipe is closed, (2) all data is
-  # read from the pipe and written to the destination, and (3) the monitoring thread is
-  # killed.
+  # > **⚠️ WARNING**
+  # >
+  # > `#close` must be called to ensure that (1) the pipe is closed, (2) all data is
+  #   read from the pipe and written to the destination, and (3) the monitoring thread is
+  #   killed.
   #
-  # @example Collect pipe data into a string
+  # @example Collect pipe data into a StringIO object
   #   pipe_data = StringIO.new
   #   begin
-  #     pipe = MonitoredPipe.new(pipe_data)
+  #     pipe = ProcessExecuter::MonitoredPipe.new(pipe_data)
   #     pipe.write("Hello World")
   #   ensure
   #     pipe.close
@@ -31,14 +67,27 @@ module ProcessExecuter
   #   pipe_data_string = StringIO.new
   #   pipe_data_file = File.open("pipe_data.txt", "w")
   #   begin
-  #     pipe = MonitoredPipe.new(pipe_data_string, pipe_data_file)
+  #     pipe = ProcessExecuter::MonitoredPipe.new([:tee, pipe_data_string, pipe_data_file])
   #     pipe.write("Hello World")
   #   ensure
   #     pipe.close
   #   end
   #   pipe_data_string.string #=> "Hello World"
+  #   # It is your responsibility to close the file you opened
+  #   pipe_data_file.close
   #   File.read("pipe_data.txt") #=> "Hello World"
   #
+  # @example Using a MonitoredPipe with Process.spawn
+  #   stdout_buffer = StringIO.new
+  #   begin
+  #     stdout_pipe = ProcessExecuter::MonitoredPipe.new(stdout_buffer)
+  #     pid = Process.spawn('echo Hello World', out: stdout_pipe)
+  #     _waited_pid, status = Process.wait2(pid)
+  #   ensure
+  #     stdout_pipe.close
+  #   end
+  #   stdout_buffer.string #=> "Hello World\n"
+  #
   # @api public
   #
   class MonitoredPipe
@@ -46,14 +95,28 @@ module ProcessExecuter
 
     # Create a new monitored pipe
     #
-    # Creates a IO.pipe and starts a monitoring thread to read data written to the pipe.
+    # Creates an IO.pipe and starts a monitoring thread to read data written to the
+    # pipe.
     #
     # @example
-    #   data_collector = StringIO.new
-    #   pipe = ProcessExecuter::MonitoredPipe.new(data_collector)
+    #   redirection_destination = StringIO.new
+    #   pipe = ProcessExecuter::MonitoredPipe.new(redirection_destination)
     #
-    # @param redirection_destination [Array<#write>] as data is read from the pipe,
+    # @param redirection_destination [Object] as data is read from the pipe,
     #   it is written to this destination
+    #
+    #   Accepts any redirection destination supported by
+    #   [`Process.spawn`](https://docs.ruby-lang.org/en/3.4/Process.html#method-c-spawn).
+    #   This is the `value` part of the file redirection option described in [the
+    #   File Redirection section of
+    #   `Process.spawn`](https://docs.ruby-lang.org/en/3.4/Process.html#module-Process-label-File+Redirection+-28File+Descriptor-29).
+    #
+    #   In addition to the standard redirection destinations, `MonitoredPipe` also
+    #   accepts (1) another monitored pipe, (2) any object that implements a `#write` method and
+    #   (3) an array in the form `[:tee, destination1, destination2, ...]` where each
+    #   `destination` can be any value that `MonitoredPipe` itself supports (including
+    #   another tee or MonitoredPipe).
+    #
     # @param chunk_size [Integer] the size of the chunks to read from the pipe
     #
     def initialize(redirection_destination, chunk_size: 100_000)
@@ -65,6 +128,13 @@ module ProcessExecuter
       @condition_variable = ConditionVariable.new
       @chunk_size = chunk_size
       @pipe_reader, @pipe_writer = IO.pipe
+
+      # Set the encoding of the pipe reader to ASCII_8BIT. This is not strictly
+      # necessary since read_nonblock always returns a String where encoding is
+      # Encoding::ASCII_8BIT, but it is a good practice to explicitly set the
+      # encoding.
+      pipe_reader.set_encoding(Encoding::ASCII_8BIT)
+
       @state = :open
       @thread = start_monitoring_thread
 
@@ -116,8 +186,6 @@ module ProcessExecuter
     #
     # @return [IO] the write end of the pipe
     #
-    # @api private
-    #
     def to_io
       pipe_writer
     end
@@ -134,8 +202,6 @@ module ProcessExecuter
     #
     # @return [Integer] the file descriptor for the write end of the pipe
     #
-    # @api private
-    #
     def fileno
       pipe_writer.fileno
     end
@@ -156,7 +222,7 @@ module ProcessExecuter
     #
     # @return [Integer] the number of bytes written to the pipe
     #
-    # @api private
+    # @raise [IOError] if the pipe is not open
     #
     def write(data)
       mutex.synchronize do
@@ -174,7 +240,7 @@ module ProcessExecuter
     #   require 'stringio'
     #   data_collector = StringIO.new
     #   pipe = ProcessExecuter::MonitoredPipe.new(data_collector)
-    #   pipe.chunk_size #=> 1000
+    #   pipe.chunk_size #=> 100_000
     #
     # @return [Integer] the size of the chunks to read from the pipe
     #
@@ -188,12 +254,45 @@ module ProcessExecuter
     #   require 'stringio'
     #   data_collector = StringIO.new
     #   pipe = ProcessExecuter::MonitoredPipe.new(data_collector)
-    #   pipe.destination #=>
+    #   pipe.destination #=> #<ProcessExecuter::Destinations::Writer>
     #
-    # @return [Array<ProcessExecuter::Destination::Base>]
+    # @return [ProcessExecuter::Destinations::DestinationBase]
     #
     attr_reader :destination
 
+    # @!attribute [r]
+    #
+    # The state of the pipe
+    #
+    # Must be either `:open`, `:closing`, or `:closed`
+    #
+    # * `:open` - the pipe is open and data can be written to it
+    # * `:closing` - the pipe is being closed and data can no longer be written to it
+    # * `:closed` - the pipe is closed and data can no longer be written to it
+    #
+    # @example
+    #   pipe = ProcessExecuter::MonitoredPipe.new($stdout)
+    #   pipe.state #=> :open
+    #   pipe.close
+    #   pipe.state #=> :closed
+    #
+    # @return [Symbol] the state of the pipe
+    #
+    attr_reader :state
+
+    # @!attribute [r]
+    #
+    # The exception raised by a destination
+    #
+    # If an exception is raised by a destination, it is stored here. Otherwise, it is `nil`.
+    #
+    # @example
+    #   pipe.exception #=> nil
+    #
+    # @return [Exception, nil] the exception raised by a destination or `nil` if no exception was raised
+    #
+    attr_reader :exception
+
     # @!attribute [r]
     #
     # The thread that monitors the pipe
@@ -205,6 +304,9 @@ module ProcessExecuter
     #   pipe.thread #=> #<Thread:0x00007f8b1a0b0e00>
     #
     # @return [Thread]
+    #
+    # @api private
+    #
     attr_reader :thread
 
     # @!attribute [r]
@@ -216,6 +318,9 @@ module ProcessExecuter
     #   pipe.pipe_reader #=> #<IO:fd 11>
     #
     # @return [IO]
+    #
+    # @api private
+    #
     attr_reader :pipe_reader
 
     # @!attribute [r]
@@ -227,40 +332,10 @@ module ProcessExecuter
     #   pipe.pipe_writer #=> #<IO:fd 12>
     #
     # @return [IO] the write end of the pipe
-    attr_reader :pipe_writer
-
-    # @!attribute [r]
-    #
-    # The state of the pipe
-    #
-    # Must be either `:open`, `:closing`, or `:closed`
-    #
-    # * `:open` - the pipe is open and data can be written to it
-    # * `:closing` - the pipe is being closed and data can no longer be written to it
-    # * `:closed` - the pipe is closed and data can no longer be written to it
     #
-    # @example
-    #   pipe = ProcessExecuter::MonitoredPipe.new($stdout)
-    #   pipe.state #=> :open
-    #   pipe.close
-    #   pipe.state #=> :closed
-    #
-    # @return [Symbol] the state of the pipe
-    #
-    attr_reader :state
-
-    # @!attribute [r]
-    #
-    # The exception raised by a destination
-    #
-    # If an exception is raised by a destination, it is stored here. Otherwise, it is `nil`.
-    #
-    # @example
-    #   pipe.exception #=> nil
-    #
-    # @return [Exception, nil] the exception raised by a destination or `nil` if no exception was raised
+    # @api private
     #
-    attr_reader :exception
+    attr_reader :pipe_writer
 
     private
 
@@ -332,19 +407,13 @@ module ProcessExecuter
     # @return [void]
     # @api private
     def monitor_pipe
+      # read_nonblock always returns a String where encoding is Encoding::ASCII_8BIT
       new_data = pipe_reader.read_nonblock(chunk_size)
       write_data(new_data)
     rescue IO::WaitReadable
       pipe_reader.wait_readable(0.001)
     end
 
-    # # Check if the writer is a file descriptor
-    # #
-    # # @param writer [#write] the writer to check
-    # # @return [Boolean] true if the writer is a file descriptor
-    # # @api private
-    # def file_descriptor?(writer) = writer.is_a?(Integer) || writer.is_a?(Symbol)
-
     # Write the data read from the pipe to the destination
     #
     # If an exception is raised by a writer, set the state to `:closing`

data/lib/process_executer/options/base.rb

@@ -14,35 +14,54 @@ module ProcessExecuter
     #       # Call super to include options defined in the parent class
     #       [
     #         *super,
-    #         ProcessExecuter::Options::OptionDefinition.new(:option1),
-    #         ProcessExecuter::Options::OptionDefinition.new(:option2)
+    #         ProcessExecuter::Options::OptionDefinition.new(
+    #           :option1, default: '', validator: method(:assert_is_string)
+    #           ),
+    #         ProcessExecuter::Options::OptionDefinition.new(
+    #           :option2, default: '', validator: method(:assert_is_string)
+    #         ),
+    #         ProcessExecuter::Options::OptionDefinition.new(
+    #           :option3, default: '', validator: method(:assert_is_string)
+    #         )
     #       ]
     #     end
+    #     def assert_is_string(key, value)
+    #       return if value.is_a?(String)
+    #       errors << "#{key} must be a String but was #{value}"
+    #     end
     #   end
-    #   options_hash = { options1: 'value1', option2: 'value2' }
-    #   options = MyOptions.new(options_hash)
+    #   options = MyOptions.new(option1: 'value1', option2: 'value2')
     #   options.option1 # => 'value1'
     #   options.option2 # => 'value2'
     #
+    # @example invalid option values
+    #   begin
+    #     options = MyOptions.new(option1: 1, option2: 2)
+    #   rescue ProcessExecuter::ArgumentError => e
+    #     e.message #=> "option1 must be a String but was 1\noption2 must be a String but was 2"
+    #   end
+    #
     # @api public
     class Base
       # Create a new Options object
       #
-      # @example
-      #   options = ProcessExecuter::Options.new(out: $stdout, err: $stderr, timeout_after: 10)
+      # Normally you would use a subclass instead of instantiating this class
+      # directly.
       #
-      # @param options [Hash] Process.spawn options plus additional options listed below.
+      # @example
+      #   options = MyOptions.new(option1: 'value1', option2: 'value2')
       #
-      #   See [Process.spawn](https://ruby-doc.org/core/Process.html#method-c-spawn)
-      #   for a list of valid options that can be passed to `Process.spawn`.
+      # @example with invalid option values
+      #   begin
+      #     options = MyOptions.new(option1: 1, option2: 2)
+      #   rescue ProcessExecuter::ArgumentError => e
+      #     e.message #=> "option1 must be a String but was 1\noption2 must be a String but was 2"
+      #   end
       #
-      # @option options [Integer, Float, nil] :timeout_after
-      #   Number of seconds to wait for the process to terminate. Any number
-      #   may be used, including Floats to specify fractional seconds. A value of 0 or nil
-      #   will allow the process to run indefinitely.
+      # @param options_hash [Hash] a hash of options
       #
-      def initialize(**options)
-        @options = allowed_options.transform_values(&:default).merge(options)
+      def initialize(**options_hash)
+        @options_hash = allowed_options.transform_values(&:default).merge(options_hash)
         @errors = []
         assert_no_unknown_options
         define_accessor_methods
@@ -51,15 +70,20 @@ module ProcessExecuter
 
       # All the allowed options as a hash whose keys are the option names
       #
-      # The returned hash what is returned from `define_options` but with the
-      # option names as keys. The values are instances of `OptionDefinition`.
+      # The returned hash what is returned from `define_options` but with the option
+      # names as keys. The values are instances of `OptionDefinition`.
       #
       # The returned hash is frozen and cannot be modified.
       #
       # @example
-      #   options.allowed_options # => { timeout_after: #<OptionDefinition>, ... }
+      #   options = MyOptions.new(option1: 'value1', option2: 'value2')
+      #   options.allowed_options # => {
+      #     option1: #<OptionDefinition>,
+      #     option2: #<OptionDefinition>
+      #   }
       #
-      # @return [Hash]
+      # @return [Hash<Symbol, ProcessExecuter::Options::OptionDefinition>] A hash
+      #   where keys are option names and values are their definitions.
       #
       def allowed_options
         @allowed_options ||=
@@ -69,94 +93,98 @@ module ProcessExecuter
       end
 
       # A string representation of the object that includes the options
+      #
       # @example
-      #   options = ProcessExecuter::Options.new(option1: 'value1', option2: 'value2')
-      #   options.to_s # => #<ProcessExecuter::Options:0x00007f8f9b0b3d20 option1: "value1", option2: "value2">'
+      #   options = MyOptions.new(option1: 'value1', option2: 'value2')
+      #   options.to_s # => #<MyOptions option1: "value1", option2: "value2">'
+      #
       # @return [String]
+      #
       def to_s
         "#{super.to_s[0..-2]} #{inspect}>"
       end
 
       # A string representation of the options
+      #
       # @example
-      #   options = ProcessExecuter::Options.new(option1: 'value1', option2: 'value2')
+      #   options = MyOptions.new(option1: 'value1', option2: 'value2')
       #   options.inspect # => '{:option1=>"value1", :option2=>"value2"}'
+      #
       # @return [String]
+      #
       def inspect
-        options.inspect
+        options_hash.inspect
       end
 
       # A hash representation of the options
+      #
       # @example
-      #   options = ProcessExecuter::Options.new(option1: 'value1', option2: 'value2')
+      #   options = MyOptions.new(option1: 'value1', option2: 'value2')
       #   options.to_h # => { option1: "value1", option2: "value2" }
+      #
       # @return [Hash]
+      #
       def to_h
-        @options.dup
+        options_hash.dup
       end
 
       # Iterate over each option with an object
+      #
       # @example
-      #   options = ProcessExecuter::Options.new(option1: 'value1', option2: 'value2')
+      #   options = MyOptions.new(option1: 'value1', option2: 'value2')
       #   options.each_with_object({}) { |(option_key, option_value), obj| obj[option_key] = option_value }
       #   # => { option1: "value1", option2: "value2" }
-      # @yield [option_key, option_value, obj]
-      # @return [void]
+      #
+      # @yield [key_value, obj]
+      #
+      # @yieldparam key_value [Array<Object, Object>] An array containing the option key and its value
+      #
+      # @yieldparam obj [Object] The object passed to the block.
+      #
+      # @return [Object] the obj passed to the block
+      #
       def each_with_object(obj, &)
-        @options.each_with_object(obj, &)
+        options_hash.each_with_object(obj, &)
       end
 
-      # Merge the given options into the current options
+      # Merge the given options into the current options object
+      #
+      # Subsequent hashes' values overwrite earlier ones for the same key.
+      #
       # @example
-      #   options = ProcessExecuter::Options.new(option1: 'value1', option2: 'value2')
-      #   options.merge!(option2: 'new_value2', option3: 'value3')
-      #   options.option2 # => 'new_value2'
-      #   options.option3 # => 'value3'
+      #   options = MyOptions.new(option1: 'value1', option2: 'value2')
+      #   h1 = { option2: 'new_value2' }
+      #   h2 = { option3: 'value3' }
+      #   options.merge!(h1, h2) => {option1: "value1", option2: "new_value2", option3: "value3"}
       #
-      # @param other_options [Hash] the options to merge into the current options
-      # @return [void]
-      def merge!(**other_options)
-        @options.merge!(other_options)
-      end
-
-      # A shallow copy of self with options copied but not the values they reference
+      # @param other_options_hashes [Array<Hash>] zero of more hashes to merge into the current options
       #
-      # If any keyword arguments are given, the copy will be created with the
-      # respective option values updated.
+      # @return [self] the current options object with the merged options
       #
-      # @example
-      #   options_hash = { option1: 'value1', option2: 'value2' }
-      #   options = ProcessExecuter::MyOptions.new(options_hash)
-      #   copy = options.with(option1: 'new_value1')
-      #   copy.option1 # => 'new_value1'
-      #   copy.option2 # => 'value2'
-      #   options.option1 # => 'value1'
-      #   options.option2 # => 'value2'
-      #
-      # @options_hash [Hash] the options to merge into the current options
-      # @return [self.class]
+      # @api public
       #
-      def with(**options_hash)
-        self.class.new(**@options, **options_hash)
+      def merge!(*other_options_hashes)
+        options_hash.merge!(*other_options_hashes)
       end
 
-      # The list of validation errors
-      #
-      # Validators should add an error messages to this array.
+      # Returns a new options object formed by merging self with each of other_hashes
       #
       # @example
-      #   options = ProcessExecuter::Options::RunOptions.new(timeout_after: 'not_a_number', raise_errors: 'yes')
-      #   #=> raises an Argument error with the following message:
-      #       timeout_after must be nil or a non-negative real number but was "not_a_number"
-      #       raise_errors must be true or false but was "yes""
-      #    errors # => [
-      #      "timeout_after must be nil or a non-negative real number but was \"not_a_number\"",
-      #      "raise_errors must be true or false but was \"yes\""
-      #    ]
+      #   options = MyOptions.new(option1: 'value1', option2: 'value2')
+      #   options.object_id # => 1025
+      #   h1 = { option2: 'new_value2' }
+      #   h2 = { option3: 'value3' }
+      #   merged_options = options.merge(h1, h2)
+      #   merged_options.object_id # => 1059
       #
-      # @return [Array<String>]
-      # @api private
-      attr_reader :errors
+      # @param other_options_hashes [Array<Hash>] the options to merge into the current options
+      #
+      # @return [self.class]
+      #
+      def merge(*other_options_hashes)
+        merged_options = other_options_hashes.reduce(options_hash, :merge)
+        self.class.new(**merged_options)
+      end
 
       protected
 
@@ -185,27 +213,37 @@ module ProcessExecuter
 
       private
 
+      # The list of validation errors
+      #
+      # Validators should add error messages to this array.
+      #
+      # @return [Array<String>]
+      #
+      # @api private
+      #
+      attr_reader :errors
+
       # @!attribute [r]
       #
       # A hash of all options keyed by the option name
       #
-      # @return [Hash{Symbol => Object}]
+      # @return [Hash<Object, Object>]
       #
       # @api private
       #
-      attr_reader :options
+      attr_reader :options_hash
 
       # Raise an argument error for invalid option values
       # @return [void]
-      # @raise [ArgumentError] if any invalid option values are found
+      # @raise [ProcessExecuter::ArgumentError] if any invalid option values are found
       # @api private
       def validate_options
-        options.each_key do |option_key|
+        options_hash.each_key do |option_key|
           validator = allowed_options[option_key]&.validator
-          instance_exec(&validator.to_proc) unless validator.nil?
+          instance_exec(option_key, send(option_key), &validator.to_proc) unless validator.nil?
         end
 
-        raise ArgumentError, errors.join("\n") unless errors.empty?
+        raise ProcessExecuter::ArgumentError, errors.join("\n") unless errors.empty?
       end
 
       # Define accessor methods for each option
@@ -214,26 +252,24 @@ module ProcessExecuter
       def define_accessor_methods
         allowed_options.each_key do |option|
           define_singleton_method(option) do
-            @options[option]
+            options_hash[option]
           end
         end
       end
 
       # Determine if the options hash contains any unknown options
       # @return [void]
-      # @raise [ArgumentError] if the options hash contains any unknown options
+      # @raise [ProcessExecuter::ArgumentError] if the options hash contains any unknown options
       # @api private
       def assert_no_unknown_options
-        unknown_options = options.keys.reject { |key| valid_option?(key) }
+        unknown_options = options_hash.keys.reject { |key| valid_option?(key) }
 
         return if unknown_options.empty?
 
-        # :nocov: SimpleCov on JRuby reports the last with the last argument line is not covered
         raise(
           ArgumentError,
-          "Unknown option#{unknown_options.count > 1 ? 's' : ''}: #{unknown_options.join(', ')}"
+          "Unknown option#{'s' if unknown_options.count > 1}: #{unknown_options.join(', ')}"
         )
-        # :nocov:
       end
     end
   end

data/lib/process_executer/options/option_definition.rb

@@ -29,6 +29,10 @@ module ProcessExecuter
 
       # A method or proc that validates the option
       #
+      # A callable that receives `option_key`, `option_value` and is executed in the
+      # context of the options instance. It should add messages to an `errors` array
+      # if validation fails.
+      #
       # @example
       #   option = ProcessExecuter::Options::OptionDefinition.new(
       #     :timeout_after, validator: method(:validate_timeout_after)
@@ -43,7 +47,7 @@ module ProcessExecuter
       #
       # @example
       #   option = ProcessExecuter::Options::OptionDefinition.new(
-      #     :timeout_after, default: 10, validator: -> { timeout_after.is_a?(Numeric) }
+      #     :timeout_after, default: 10, validator: ->(_k, _v) { timeout_after.is_a?(Numeric) }
       #   )
       #
       def initialize(name, default: nil, validator: nil)