right_popen 1.0.21 → 1.1.3

data/lib/right_popen/linux/process.rb

@@ -1,5 +1,5 @@
 #--  -*- mode: ruby; encoding: utf-8 -*-
-# Copyright: Copyright (c) 2011 RightScale, Inc.
+# Copyright: Copyright (c) 2011-2013 RightScale, Inc.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -23,20 +23,87 @@
 
 require 'etc'
 
+require ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', 'process_base'))
+require ::File.expand_path(::File.join(::File.dirname(__FILE__), '..', 'process_status'))
+
 module RightScale
   module RightPopen
-    class Process
-      attr_reader :pid, :stdin, :stdout, :stderr, :status_fd
-      attr_accessor :status
-
-      def initialize(parameters={})
-        parameters[:locale] = true unless parameters.has_key?(:locale)
-        @parameters = parameters
-        @status_fd = nil
+    class Process < ProcessBase
+
+      def initialize(options={})
+        super(options)
+      end
+
+      # Determines if the process is still running.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if running
+      def alive?
+        raise ProcessError.new('Process not started') unless @pid
+        unless @status
+          begin
+            ignored, status = ::Process.waitpid2(@pid, ::Process::WNOHANG)
+            @status = status
+          rescue
+            wait_for_exit_status
+          end
+        end
+        @status.nil?
+      end
+
+      # Linux must only read streams that are selected for read, even on child
+      # death. the issue is that a child process can (inexplicably) close one of
+      # the streams but continue writing to the other and this will cause the
+      # parent to hang reading the stream until the child goes away.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if draining all
+      def drain_all_upon_death?
+        false
+      end
+
+      # @return [Array] escalating termination signals for this platform
+      def signals_for_interrupt
+        ['INT', 'TERM', 'KILL']
       end
 
-      def fork(cmd)
-        @cmd = cmd
+      # blocks waiting for process exit status.
+      #
+      # === Return
+      # @return [ProcessStatus] exit status
+      def wait_for_exit_status
+        raise ProcessError.new('Process not started') unless @pid
+        unless @status
+          begin
+            ignored, status = ::Process.waitpid2(@pid)
+            @status = status
+          rescue
+            # ignored
+          end
+        end
+        @status
+      end
+
+      # spawns (forks) a child process using given command and handler target in
+      # linux-specific manner.
+      #
+      # must be overridden and override must call super.
+      #
+      # === Parameters
+      # @param [String|Array] cmd as shell command or binary to execute
+      # @param [Object] target that implements all handlers (see TargetProxy)
+      #
+      # === Return
+      # @return [TrueClass] always true
+      def spawn(cmd, target)
+        super(cmd, target)
+
+        # garbage collect any open file descriptors from past executions before
+        # forking to prevent them being inherited. also reduces memory footprint
+        # since forking will duplicate everything in memory for child process.
+        ::GC.start
+
+        # create pipes.
         stdin_r, stdin_w = IO.pipe
         stdout_r, stdout_w = IO.pipe
         stderr_r, stderr_w = IO.pipe
@@ -45,24 +112,28 @@ module RightScale
         [stdin_r, stdin_w, stdout_r, stdout_w,
          stderr_r, stderr_w, status_r, status_w].each {|fdes| fdes.sync = true}
 
-        @pid = Kernel::fork do
+        @pid = ::Kernel::fork do
           begin
             stdin_w.close
-            STDIN.reopen stdin_r
+            ::STDIN.reopen stdin_r
 
             stdout_r.close
-            STDOUT.reopen stdout_w
+            ::STDOUT.reopen stdout_w
 
             stderr_r.close
-            STDERR.reopen stderr_w
+            ::STDERR.reopen stderr_w
 
             status_r.close
-            status_w.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
-
-            ObjectSpace.each_object(IO) do |io|
-              if ![STDIN, STDOUT, STDERR, status_w].include?(io)
-            	  io.close unless io.closed?
-            	end
+            status_w.fcntl(::Fcntl::F_SETFD, ::Fcntl::FD_CLOEXEC)
+
+            unless @options[:inherit_io]
+              ::ObjectSpace.each_object(IO) do |io|
+                if ![::STDIN, ::STDOUT, ::STDERR, status_w].include?(io)
+                  # be careful to not allow streams in a bad state from the
+                  # parent process to prevent child process running.
+                  (io.close rescue nil) unless (io.closed? rescue true)
+                end
+              end
             end
 
             if group = get_group
@@ -75,24 +146,34 @@ module RightScale
               ::Process.uid = user
             end
 
-            Dir.chdir(@parameters[:directory]) if @parameters[:directory]
-
-            ENV["LC_ALL"] = "C" if @parameters[:locale]
+            if umask = get_umask
+              ::File.umask(umask)
+            end
 
-            @parameters[:environment].each do |key,value|
-              ENV[key.to_s] = value.to_s
-            end if @parameters[:environment]
+            # avoid chdir when pwd is already correct due to asinine printed
+            # warning from chdir block for what is basically a no-op.
+            working_directory = @options[:directory]
+            if working_directory &&
+               ::File.expand_path(working_directory) != ::File.expand_path(::Dir.pwd)
+              ::Dir.chdir(working_directory)
+            end
 
-            File.umask(get_umask) if @parameters[:umask]
+            environment_hash = {}
+            environment_hash['LC_ALL'] = 'C' if @options[:locale]
+            environment_hash.merge!(@options[:environment]) if @options[:environment]
+            environment_hash.each do |key, value|
+              ::ENV[key.to_s] = value.to_s if value
+            end
 
             if cmd.kind_of?(Array)
+              cmd = cmd.map { |c| c.to_s } #exec only likes string arguments
               exec(*cmd)
             else
-              exec("sh", "-c", cmd)
+              exec('sh', '-c', cmd.to_s)  # allows shell commands for cmd string
             end
-            raise 'forty-two' 
-          rescue Exception => e
-            Marshal.dump(e, status_w)
+            raise 'Unreachable code'
+          rescue ::Exception => e
+            ::Marshal.dump(e, status_w)
           end
           status_w.close
           exit!
@@ -106,11 +187,21 @@ module RightScale
         @stdout = stdout_r
         @stderr = stderr_r
         @status_fd = status_r
+        start_timer
+        true
+      rescue
+        # catch-all for failure to spawn process ensuring a non-nil status. the
+        # PID most likely is nil but the exit handler can be invoked for async.
+        safe_close_io
+        @status = ::RightScale::RightPopen::ProcessStatus.new(@pid, 1)
+        raise
       end
 
+      # @deprecated this seems like test harness code smell, not production code.
       def wait_for_exec
+        warn 'WARNING: RightScale::RightPopen::Process#wait_for_exec is deprecated in lib and will be moved to spec'
         begin
-          e = Marshal.load @status_fd
+          e = ::Marshal.load(@status_fd)
           # thus meaning that the process failed to exec...
           @stdin.close
           @stdout.close
@@ -126,28 +217,29 @@ module RightScale
       private
 
       def get_user
-        user = @parameters[:user] || nil
-        unless user.kind_of?(Integer)
-          user = Etc.getpwnam(user).uid if user
+        if user = @options[:user]
+          user = Etc.getpwnam(user).uid unless user.kind_of?(Integer)
         end
         user
       end
 
       def get_group
-        group = @parameters[:group] || nil
-        unless group.kind_of?(Integer)
-          group = Etc.getgrnam(group).gid if group
+        if group = @options[:group]
+          group = Etc.getgrnam(group).gid unless group.kind_of?(Integer)
         end
         group
       end
 
       def get_umask
-        if @parameters[:umask].respond_to?(:oct)
-          value = @parameters[:umask].oct
-        else
-          value = @parameters[:umask].to_i
+        if umask = @options[:umask]
+          if umask.respond_to?(:oct)
+            umask = umask.oct
+          else
+            umask = umask.to_i
+          end
+          umask = umask & 007777
         end
-        value & 007777
+        umask
       end
     end
   end

data/lib/right_popen/linux/utilities.rb

@@ -26,6 +26,8 @@ require File.expand_path(File.join(File.dirname(__FILE__), "accumulator"))
 
 module RightScale
   module RightPopen
+
+    # @deprecated this seems like test harness code smell, not production code
     module Utilities
       module_function
 
@@ -77,14 +79,15 @@ module RightScale
       alias_method :spawn, :run_collecting_output
 
       def run_with_blocks(cmd, stdin_block, stdout_block, stderr_block, parameters={})
+        warn 'WARNING: RightScale::RightPopen::Utilities are deprecated and will be removed.'
         process = Process.new(parameters)
-        process.fork(cmd)
+        process.spawn(cmd, ::RightScale::RightPopen::TargetProxy.new(parameters))
         process.wait_for_exec
         a = Accumulator.new(process,
                             [process.stdout, process.stderr], [stdout_block, stderr_block],
                             [process.stdin], [stdin_block])
         a.run_to_completion
-        process.status[1]
+        a.status[1]
       end
     end
   end

data/lib/right_popen/process_base.rb

@@ -0,0 +1,339 @@
+#--  -*- mode: ruby; encoding: utf-8 -*-
+# Copyright: Copyright (c) 2013 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'thread'
+
+module RightScale
+  module RightPopen
+    class ProcessBase
+
+      class ProcessError < Exception; end
+
+      attr_reader :pid, :stdin, :stdout, :stderr, :status_fd, :status
+      attr_reader :start_time, :stop_time, :channels_to_finish
+
+      # === Parameters
+      # @param [Hash] options see RightScale.popen3_async for details
+      def initialize(options={})
+        @options = options
+        @stdin = nil
+        @stdout = nil
+        @stderr = nil
+        @status_fd = nil
+        @last_interrupt = nil
+        @pid = nil
+        @start_time = nil
+        @stop_time = nil
+        @watch_directory = nil
+        @size_limit_bytes = nil
+        @cmd = nil
+        @target = nil
+        @status = nil
+        @channels_to_finish = nil
+        @needs_watching = !!(
+          @options[:timeout_seconds] ||
+          @options[:size_limit_bytes] ||
+          @options[:watch_handler])
+      end
+
+      # Determines if the process is still running.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if running
+      def alive?
+        raise NotImplementedError, 'Must be overridden'
+      end
+
+      # Determines whether or not to drain all open streams upon death of child
+      # or else only those where IO.select indicates data available. This
+      # decision is platform-specific.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if draining all
+      def drain_all_upon_death?
+        raise NotImplementedError, 'Must be overridden'
+      end
+
+      # Determines if this process needs to be watched (beyond waiting for the
+      # process to exit).
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if needs watching
+      def needs_watching?; @needs_watching; end
+
+      # Determines if timeout on child process has expired, if any.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if timer expired
+      def timer_expired?
+        !!(@stop_time && Time.now >= @stop_time)
+      end
+
+      # Determines if total size of files created by child process has exceeded
+      # the limit specified, if any.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if size limit exceeded
+      def size_limit_exceeded?
+        if @watch_directory
+          globbie = ::File.join(@watch_directory, '**/*')
+          size = 0
+          ::Dir.glob(globbie) do |f|
+            size += ::File.stat(f).size rescue 0 if ::File.file?(f)
+            break if size > @size_limit_bytes
+          end
+          size > @size_limit_bytes
+        else
+          false
+        end
+      end
+
+      # @return [TrueClass|FalseClass] interrupted as true if child process was interrupted by watcher
+      def interrupted?; !!@last_interrupt; end
+
+      # Performs all process operations in synchronous fashion. It is possible
+      # for errors or callback behavior to conditionally short-circuit the
+      # synchronous operations.
+      #
+      # === Parameters
+      # @param [String|Array] cmd as shell command or binary to execute
+      # @param [Object] target that implements all handlers (see TargetProxy)
+      #
+      # === Return
+      # @return [TrueClass] always true
+      def sync_all(cmd, target)
+        spawn(cmd, target)
+        sync_exit_with_target if sync_pid_with_target
+        true
+      end
+
+      # Spawns a child process using given command and handler target in a
+      # platform-independant manner.
+      #
+      # must be overridden and override must call super.
+      #
+      # === Parameters
+      # @param [String|Array] cmd as shell command or binary to execute
+      # @param [Object] target that implements all handlers (see TargetProxy)
+      #
+      # === Return
+      # @return [TrueClass] always true
+      def spawn(cmd, target)
+        @cmd = cmd
+        @target = target
+        @kill_time = nil
+        @pid = nil
+        @status = nil
+        @last_interrupt = nil
+        @channels_to_finish = nil
+
+        if @size_limit_bytes = @options[:size_limit_bytes]
+          @watch_directory = @options[:watch_directory] || @options[:directory] || ::Dir.pwd
+        end
+      end
+
+      # Performs initial handler callbacks before consuming I/O. Represents any
+      # code that must not be invoked twice (unlike sync_exit_with_target).
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true to begin watch, false to abandon
+      def sync_pid_with_target
+        # early handling in case caller wants to stream to/from the pipes
+        # directly (as in a classic popen3/4 scenario).
+        @target.pid_handler(@pid)
+        if input_text = @options[:input]
+          @stdin.write(input_text)
+        end
+
+        # one-time initialization of the stateful channels_to_finish hash to
+        # allow for multiple invocations of the sync_exit_with_target with a
+        # possible abandon in between.
+        #
+        # note that calling IO.select on pipes which have already had all
+        # of their output consumed can cause segfault (in Ubuntu?) so it is
+        # important to keep track of when all I/O has been consumed.
+        @channels_to_finish = [
+          [:stdout_handler, @stdout],
+          [:stderr_handler, @stderr],
+        ]
+        @channels_to_finish << [:status_fd, @status_fd] if @status_fd
+
+        # sync watch_handler has the option to abandon watch as soon as child
+        # process comes alive and before streaming any output.
+        if @target.watch_handler(self)
+          # can close stdin if not returning control to caller.
+          @stdin.close rescue nil
+          return true
+        else
+          # caller is reponsible for draining and/or closing all pipes. this can
+          # be accomplished by explicity calling either sync_exit_with_target or
+          # safe_close_io plus wait_for_exit_status (if data has been read from
+          # the I/O streams). it is unsafe to read some data and then call
+          # sync_exit_with_target because IO.select may segfault if all of the
+          # data in a stream has been already consumed.
+          return false
+        end
+      rescue
+        safe_close_io
+        raise
+      end
+
+      # Monitors I/O from child process and directly notifies target of any
+      # events. Blocks until child exits.
+      #
+      # === Return
+      # @return [TrueClass] always true
+      def sync_exit_with_target
+        abandon = false
+        last_exception = nil
+        begin
+          while true
+            channels_to_watch = @channels_to_finish.map { |ctf| ctf.last }
+            ready = ::IO.select(channels_to_watch, nil, nil, 0.1) rescue nil
+            dead = !alive?
+            channels_to_read = ready && ready.first
+            if dead && drain_all_upon_death?
+              # finish reading all dead channels.
+              channels_to_read = @channels_to_finish.map { |ctf| ctf.last }
+            end
+            if channels_to_read
+              channels_to_read.each do |channel|
+                index = @channels_to_finish.index { |ctf| ctf.last == channel }
+                key = @channels_to_finish[index].first
+                data = dead ? channel.gets(nil) : channel.gets
+                if data
+                  if key == :status_fd
+                    last_exception = ::Marshal.load(data)
+                  else
+                    @target.method(key).call(data)
+                  end
+                else
+                  # nothing on channel indicates EOF
+                  @channels_to_finish.delete_at(index)
+                end
+              end
+            end
+            if dead
+              break
+            elsif (interrupted? || timer_expired? || size_limit_exceeded?)
+              interrupt
+            elsif abandon = !@target.watch_handler(self)
+              return true  # bypass any remaining callbacks
+            end
+          end
+          wait_for_exit_status
+          @target.timeout_handler if timer_expired?
+          @target.size_limit_handler if size_limit_exceeded?
+          @target.exit_handler(@status)
+
+          # re-raise exception from fork, if any.
+          case last_exception
+          when nil
+            # all good
+          when ::Exception
+            raise last_exception
+          else
+            raise "Unknown failure: saw #{last_exception.inspect} on status channel."
+          end
+        ensure
+          # abandon will not close I/O objects; caller takes responsibility via
+          # process object passed to watch_handler. if anyone calls interrupt
+          # then close I/O regardless of abandon to try to force child to die.
+          safe_close_io if !abandon || interrupted?
+        end
+        true
+      end
+
+      # blocks waiting for process exit status.
+      #
+      # === Return
+      # @return [ProcessStatus] exit status
+      def wait_for_exit_status
+        raise NotImplementedError, 'Must be overridden'
+      end
+
+      # @return [Array] escalating termination signals for this platform
+      def signals_for_interrupt
+        raise NotImplementedError, 'Must be overridden'
+      end
+
+      # Interrupts the running process (without abandoning watch) in increasing
+      # degrees of signalled severity.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if process was alive and interrupted, false if dead before (first) interrupt
+      def interrupt
+        while alive?
+          if !@kill_time || Time.now >= @kill_time
+            # soft then hard interrupt (assumed to be called periodically until
+            # process is gone).
+            sigs = signals_for_interrupt
+            if @last_interrupt
+              last_index = sigs.index(@last_interrupt)
+              next_interrupt = sigs[last_index + 1]
+            else
+              next_interrupt = sigs.first
+            end
+            unless next_interrupt
+              raise ::RightScale::RightPopen::ProcessBase::ProcessError
+                    'Unable to kill child process'
+            end
+            @last_interrupt = next_interrupt
+
+            # kill
+            result = ::Process.kill(next_interrupt, @pid) rescue nil
+            if result
+              @kill_time = Time.now + 3 # more seconds until next attempt
+              break
+            end
+          end
+        end
+        interrupted?
+      end
+
+      # Safely closes any open I/O objects associated with this process.
+      #
+      # === Return
+      # @return [TrueClass] alway true
+      def safe_close_io
+        @stdin.close rescue nil if @stdin && !@stdin.closed?
+        @stdout.close rescue nil if @stdout && !@stdout.closed?
+        @stderr.close rescue nil if @stderr && !@stderr.closed?
+        @status_fd.close rescue nil if @status_fd && !@status_fd.closed?
+        true
+      end
+
+      protected
+
+      def start_timer
+        # start timer when process comes alive (ruby processes are slow to
+        # start in Windows, etc.).
+        raise ProcessError.new("Process not started") unless @pid
+        @start_time = ::Time.now
+        @stop_time  = @options[:timeout_seconds] ?
+                      (@start_time + @options[:timeout_seconds]) :
+                      nil
+      end
+    end
+  end
+end