subprocess 1.3.2 → 1.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 28480023adb6bb4e7bff7096e766e6be65ff9421
-  data.tar.gz: 3ad723848b5a45e44fad49d967c0787d1d24f108
+SHA256:
+  metadata.gz: b4d97562256aef790c8c0a24420d8b6509375cc023809aaa833310d2e3c46442
+  data.tar.gz: 9b91371b9e3e9a2af35431f69c4d3a795f02697063675378676bdb2f8cd7a262
 SHA512:
-  metadata.gz: 566d18db86adb9950509f92d4efa608475c9c9d9ccc174e11c06ee2fa128eadc6c8fbc22ab49404abed394c930a0750a4de5eb489dcc8761a72c4fb3b6a043e0
-  data.tar.gz: f9c2d1307de9dfcf067eeda605182456872dc16ef0948348db488fd0103d080ea296cba486b9b5448c712fb188cb23e1feacb4ce8f08c7647b1be766205a02c0
+  metadata.gz: 7c757567b28d5e36cffd5ea13b705012ab790b445d757a1744a76d3eef375bfa6bc48259e0f0e744a351ee8455a79264a74dd9b6bcd6275c7fdd864b5b2ec82f
+  data.tar.gz: fd3dd1fdbda8bcf00a2a96a7855610d2c9ff7fb2a6ec1d8f24751df80723f050444dbccd8bd93d3090210e4c7c6be033a008d46e7af550a0181fff0146a0329d

data/lib/subprocess.rb

@@ -1,4 +1,3 @@
-require 'fileutils'
 require 'thread'
 require 'set'
 
@@ -31,7 +30,7 @@ module Subprocess
   #
   # @return [::Process::Status] The exit status of the process
   #
-  # @see {Process#initialize}
+  # @see Process#initialize
   def self.call(cmd, opts={}, &blk)
     Process.new(cmd, opts, &blk).wait
   end
@@ -63,7 +62,7 @@ module Subprocess
   #   was terminated with an error or was killed by a signal)
   # @return [::Process::Status] The exit status of the process
   #
-  # @see {Process#initialize}
+  # @see Process#initialize
   def self.check_call(cmd, opts={}, &blk)
     status = Process.new(cmd, opts, &blk).wait
     raise NonZeroExit.new(cmd, status) unless status.success?
@@ -80,7 +79,7 @@ module Subprocess
   #   was terminated with an error or was killed by a signal)
   # @return [String] The contents of `stdout`
   #
-  # @see {Process#initialize}
+  # @see Process#initialize
   def self.check_output(cmd, opts={}, &blk)
     opts[:stdout] = PIPE
     child = Process.new(cmd, opts, &blk)
@@ -162,6 +161,22 @@ module Subprocess
     end
   end
 
+  # Error class representing a timeout during a call to `communicate`
+  class CommunicateTimeout < StandardError
+    # @!attribute [r] stdout
+    #   @return [String] Content read from stdout before the timeout
+    # @!attribute [r] stderr
+    #   @return [String] Content read from stderr before the timeout
+    attr_reader :stdout, :stderr
+
+    def initialize(cmd, stdout, stderr)
+      @stdout = stdout
+      @stderr = stderr
+
+      super("Timeout communicating with `#{cmd.join(' ')}`")
+    end
+  end
+
   # A child process. The preferred way of spawning a subprocess is through the
   # functions on {Subprocess} (especially {Subprocess::check_call} and
   # {Subprocess::check_output}).
@@ -187,7 +202,7 @@ module Subprocess
     #
     # @param [Array<String>] cmd The command to run and its arguments (in the
     #   style of an `argv` array). Unlike Python's subprocess module, `cmd`
-    #   cannnot be a String.
+    #   cannot be a String.
     #
     # @option opts [IO, Fixnum, String, Subprocess::PIPE, nil] :stdin The `IO`,
     #   file descriptor number, or file name to use for the process's standard
@@ -224,7 +239,7 @@ module Subprocess
     #   in conjunction with {Subprocess::check_call}.
     # @yieldparam process [Process] The process that was just spawned.
     def initialize(cmd, opts={}, &blk)
-      raise ArgumentError, "cmd must be an Array" unless Array === cmd
+      raise ArgumentError, "cmd must be an Array of strings" unless Array === cmd
       raise ArgumentError, "cmd cannot be empty" if cmd.empty?
 
       @command = cmd
@@ -244,8 +259,6 @@ module Subprocess
 
       @pid = fork do
         begin
-          FileUtils.cd(opts[:cwd]) if opts[:cwd]
-
           ::STDIN.reopen(@child_stdin) if @child_stdin
           ::STDOUT.reopen(@child_stdout) if @child_stdout
           if opts[:stderr] == STDOUT
@@ -257,24 +270,44 @@ module Subprocess
           # Set up a new environment if we're requested to do so.
           if opts[:env]
             ENV.clear
-            ENV.update(opts[:env])
+            begin
+              ENV.update(opts[:env])
+            rescue TypeError => e
+              raise ArgumentError, "`env` option must be a hash where all keys and values are strings (#{e})"
+            end
           end
 
           # Call the user back, maybe?
-          opts[:preexec_fn].call if opts[:preexec_fn]
+          if opts[:preexec_fn]
+            if opts[:cwd]
+              Dir.chdir(opts[:cwd], &opts[:preexec_fn])
+            else
+              opts[:preexec_fn].call
+            end
+          end
 
           options = {close_others: true}.merge(opts.fetch(:exec_opts, {}))
           if opts[:retain_fds]
             retained_fds.each { |fd| options[fd] = fd }
           end
+          if opts[:cwd]
+            # We use the chdir option to `exec` since wrapping the
+            # `exec` in a Dir.chdir block caused these sporadic errors on macOS:
+            # Too many open files - getcwd (Errno::EMFILE)
+            options[:chdir] = opts[:cwd]
+          end
 
-          # Ruby's Kernel#exec will call an exec(3) variant if called with two
-          # or more arguments, but when called with just a single argument will
-          # spawn a subshell with that argument as the command. Since we always
-          # want to call exec(3), we use the third exec form, which passes a
-          # [cmdname, argv0] array as its first argument and never invokes a
-          # subshell.
-          exec([cmd[0], cmd[0]], *cmd[1..-1], options)
+          begin
+            # Ruby's Kernel#exec will call an exec(3) variant if called with two
+            # or more arguments, but when called with just a single argument will
+            # spawn a subshell with that argument as the command. Since we always
+            # want to call exec(3), we use the third exec form, which passes a
+            # [cmdname, argv0] array as its first argument and never invokes a
+            # subshell.
+            exec([cmd[0], cmd[0]], *cmd[1..-1], options)
+          rescue TypeError => e
+            raise ArgumentError, "cmd must be an Array of strings (#{e})"
+          end
 
         rescue Exception => e
           # Dump all errors up to the parent through the control socket
@@ -337,7 +370,7 @@ module Subprocess
     #   condition (`EOFError` or `EPIPE`).
     def drain_fd(fd, buf=nil)
       loop do
-        tmp = fd.read_nonblock(4096)
+        tmp = fd.read_nonblock(4096).force_encoding(fd.external_encoding)
         buf << tmp unless buf.nil?
       end
     rescue EOFError, Errno::EPIPE
@@ -348,43 +381,48 @@ module Subprocess
       false
     end
 
-    # Write the (optional) input to the process's `stdin`. Also, read (and
-    # buffer in memory) the contents of `stdout` and `stderr`. Do this all using
-    # `IO::select`, so we don't deadlock due to full pipe buffers.
+    # Write the (optional) input to the process's `stdin` and read the contents of
+    # `stdout` and `stderr`. If a block is provided, stdout and stderr are yielded as they
+    # are read. Otherwise they are buffered in memory and returned when the process
+    # exits. Do this all using `IO::select`, so we don't deadlock due to full pipe
+    # buffers.
     #
     # This is only really useful if you set some of `:stdin`, `:stdout`, and
     # `:stderr` to {Subprocess::PIPE}.
     #
     # @param [String] input A string to feed to the child's standard input.
-    # @return [Array<String>] An array of two elements: the data read from the
+    # @param [Numeric] timeout_s Raise {Subprocess::CommunicateTimeout} if communicate
+    #   does not finish after timeout_s seconds.
+    # @yieldparam [String] stdout Data read from stdout since the last yield
+    # @yieldparam [String] stderr Data read from stderr since the last yield
+    # @return [Array(String, String), nil] An array of two elements: the data read from the
     #   child's standard output and standard error, respectively.
-    def communicate(input=nil)
+    #   Returns nil if a block is provided.
+    def communicate(input=nil, timeout_s=nil)
       raise ArgumentError if !input.nil? && @stdin.nil?
 
       stdout, stderr = "", ""
-      stdout_encoding = @stdout.external_encoding if @stdout
-      stderr_encoding = @stderr.external_encoding if @stderr
 
       input = input.dup unless input.nil?
 
       @stdin.close if (input.nil? || input.empty?) && !@stdin.nil?
 
-      self_read, self_write = IO.pipe
-      self.class.catching_sigchld(pid, self_write) do
-        wait_r = [@stdout, @stderr, self_read].compact
+      timeout_at = Time.now + timeout_s if timeout_s
+
+      self.class.catching_sigchld(pid) do |global_read, self_read|
+        wait_r = [@stdout, @stderr, self_read, global_read].compact
         wait_w = [input && @stdin].compact
-        loop do
-          ready_r, ready_w = select(wait_r, wait_w)
-
-          # If the child exits, we still have to be sure to read any data left
-          # in the pipes. So we poll the child, drain all the pipes, and *then*
-          # check @status.
-          #
-          # It's very important that we do not call poll between draining the
-          # pipes and checking @status. If we did, we open a race condition
-          # where the child writes to stdout and exits in that brief window,
-          # causing us to lose that data.
-          poll
+        done = false
+        while !done
+          # If the process has exited, we want to drain any remaining output before returning
+          if poll
+            ready_r = wait_r - [self_read, global_read]
+            ready_w = []
+            done = true
+          else
+            ready_r, ready_w = select_until(wait_r, wait_w, [], timeout_at)
+            raise CommunicateTimeout.new(@command, stdout, stderr) if ready_r.nil?
+          end
 
           if ready_r.include?(@stdout)
             if drain_fd(@stdout, stdout)
@@ -398,6 +436,13 @@ module Subprocess
             end
           end
 
+          if ready_r.include?(global_read)
+            if drain_fd(global_read)
+              raise "Unexpected internal error -- someone closed the global self-pipe!"
+            end
+            self.class.wakeup_sigchld
+          end
+
           if ready_r.include?(self_read)
             if drain_fd(self_read)
               raise "Unexpected internal error -- someone closed our self-pipe!"
@@ -405,10 +450,24 @@ module Subprocess
           end
 
           if ready_w.include?(@stdin)
+            written = 0
             begin
               written = @stdin.write_nonblock(input)
             rescue EOFError # Maybe I shouldn't catch this...
             rescue Errno::EINTR
+            rescue IO::WaitWritable
+              # On OS X, a pipe can raise EAGAIN even after select indicates
+              # that it is writable. Once the process consumes from the pipe,
+              # the next write should succeed and we should make forward progress.
+              # Until then, treat this as not writing any bytes and continue looping.
+              # For details see: https://github.com/stripe/subprocess/pull/22
+              nil
+            rescue Errno::EPIPE
+              # The other side of the pipe closed before we could
+              # write all of our input. This can happen if the
+              # process exits prematurely.
+              @stdin.close
+              wait_w.delete(@stdin)
             end
             input[0...written] = ''
             if input.empty?
@@ -417,19 +476,20 @@ module Subprocess
             end
           end
 
-          break if @status
-
-          # If there's nothing left to wait for, we're done!
-          break if wait_r.length == 0 && wait_w.length == 0
+          if block_given? && !(stderr.empty? && stdout.empty?)
+            yield stdout, stderr
+            stdout, stderr = "", ""
+          end
         end
       end
 
       wait
 
-      stdout.force_encoding(stdout_encoding) if stdout_encoding
-      stderr.force_encoding(stderr_encoding) if stderr_encoding
-
-      [stdout, stderr]
+      if block_given?
+        nil
+      else
+        [stdout, stderr]
+      end
     end
 
     # Does exactly what it says on the box.
@@ -482,19 +542,58 @@ module Subprocess
       end
     end
 
+    # Call IO.select timing out at Time `timeout_at`. If `timeout_at` is nil, never times out.
+    def select_until(read_array, write_array, err_array, timeout_at)
+      if !timeout_at
+        return IO.select(read_array, write_array, err_array)
+      end
+
+      remaining = (timeout_at - Time.now)
+      return nil if remaining <= 0
+
+      IO.select(read_array, write_array, err_array, remaining)
+    end
+
     @sigchld_mutex = Mutex.new
     @sigchld_fds = {}
     @sigchld_old_handler = nil
+    @sigchld_global_write = nil
+    @sigchld_global_read = nil
+    @sigchld_pipe_pid = nil
+
+    def self.handle_sigchld
+      # We'd like to just notify everything in `@sigchld_fds`, but
+      # ruby signal handlers are not executed atomically with respect
+      # to other Ruby threads, so reading it is racy. We can't grab
+      # `@sigchld_mutex`, because signal execution blocks the main
+      # thread, and so we'd deadlock if the main thread currently
+      # holds it.
+      #
+      # Instead, we keep a long-lived notify self-pipe that we select
+      # on inside `communicate`, and we task `communicate` with
+      # grabbing the lock and fanning out the wakeups.
+      begin
+        @sigchld_global_write.write_nonblock("\x00")
+      rescue Errno::EWOULDBLOCK, Errno::EAGAIN
+        nil # ignore
+      end
+    end
 
     # Wake up everyone. We can't tell who we should wake up without `wait`ing,
     # and we want to let the process itself do that. In practice, we're not
     # likely to have that many in-flight subprocesses, so this is probably not a
     # big deal.
-    def self.handle_sigchld
-      @sigchld_fds.values.each do |fd|
-        begin
-          fd.write_nonblock("\x00")
-        rescue Errno::EWOULDBLOCK, Errno::EAGAIN
+    def self.wakeup_sigchld
+      @sigchld_mutex.synchronize do
+        @sigchld_fds.values.each do |fd|
+          begin
+            fd.write_nonblock("\x00")
+          rescue Errno::EWOULDBLOCK, Errno::EAGAIN
+            # If the pipe is full, the other end will be woken up
+            # regardless when it next reads, so it's fine to skip the
+            # write (the pipe is a wakeup channel, and doesn't contain
+            # meaningful data).
+          end
         end
       end
     end
@@ -503,6 +602,14 @@ module Subprocess
       @sigchld_mutex.synchronize do
         @sigchld_fds[pid] = fd
         if @sigchld_fds.length == 1
+          if @sigchld_global_write.nil? || @sigchld_pipe_pid != ::Process.pid
+            # Check the PID so that if we fork we will re-open the
+            # pipe. It's important that a fork parent and child don't
+            # share this pipe, because if they do they risk stealing
+            # each others' wakeups.
+            @sigchld_pipe_pid = ::Process.pid
+            @sigchld_global_read, @sigchld_global_write = IO.pipe
+          end
           @sigchld_old_handler = Signal.trap('SIGCHLD') {handle_sigchld}
         end
       end
@@ -517,11 +624,15 @@ module Subprocess
       end
     end
 
-    def self.catching_sigchld(pid, fd)
-      register_pid(pid, fd)
-      yield
-    ensure
-      unregister_pid(pid)
+    def self.catching_sigchld(pid)
+      IO.pipe do |self_read, self_write|
+        begin
+          register_pid(pid, self_write)
+          yield @sigchld_global_read, self_read
+        ensure
+          unregister_pid(pid)
+        end
+      end
     end
   end
 end

data/lib/subprocess/version.rb

@@ -1,3 +1,3 @@
 module Subprocess
-  VERSION = '1.3.2'
+  VERSION = '1.5.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: subprocess
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.5.3
 platform: ruby
 authors:
 - Carl Jackson
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-17 00:00:00.000000000 Z
+date: 2020-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -89,10 +89,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A port of Python's subprocess module to Ruby
 test_files: []
-has_rdoc: