right_popen 1.0.21-x86-mswin32-60 → 1.1.3-x86-mswin32-60

data/README.rdoc

@@ -15,6 +15,7 @@ documentation.
 Also use the built-in issues tracker (https://github.com/rightscale/right_popen/issues)
 to report issues.
 
+Maintained by the RightScale Teal Team
 
 == USAGE
 
@@ -47,13 +48,14 @@ to report issues.
   EM.run do
     EM.next_tick do
       command = "ruby -e \"puts 'some stdout text'; $stderr.puts 'some stderr text'\; exit 99\""
-      RightScale.popen3(:command        => command,
-                        :target         => self,
-                        :environment    => nil,
-                        :pid_handler    => :on_pid,
-                        :stdout_handler => :on_read_stdout,
-                        :stderr_handler => :on_read_stderr,
-                        :exit_handler   => :on_exit)
+      RightScale::RightPopen.popen3_async(
+        command,
+        :target         => self,
+        :environment    => nil,
+        :pid_handler    => :on_pid,
+        :stdout_handler => :on_read_stdout,
+        :stderr_handler => :on_read_stderr,
+        :exit_handler   => :on_exit)
     end
     timer = EM::PeriodicTimer.new(0.1) do
       if @exit_status
@@ -109,7 +111,7 @@ the RightPopen tests:
 
 <b>RightPopen</b>
 
-Copyright:: Copyright (c) 2010 RightScale, Inc. 
+Copyright:: Copyright (c) 2010-2013 RightScale, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/right_popen.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009 RightScale Inc
+# Copyright (c) 2009-2013 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,46 +21,141 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-# RightScale.popen3 allows running external processes aynchronously
-# while still capturing their standard and error outputs.
-# It relies on EventMachine for most of its internal mechanisms.
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'target_proxy'))
 
-if RUBY_PLATFORM =~ /mswin/
-  require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'win32', 'right_popen'))
-else
-  require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'linux', 'right_popen'))
+# TEAL FIX: this seems like test harness code smell, not production code. it
+# should be removed in next major revision. unfortunately we cannot remove these
+# require statements without breaking any code that depends on them.
+unless RUBY_PLATFORM =~ /mswin|mingw/
+  require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'linux', 'accumulator'))
+  require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'linux', 'utilities'))
 end
 
 module RightScale
 
-  # Spawn process to run given command asynchronously, hooking all three
-  # standard streams of the child process.
+  # see popen3_async for details.
   #
-  # Streams the command's stdout and stderr to the given handlers. Time-
-  # ordering of bytes sent to stdout and stderr is not preserved.
-  #
-  # Calls given exit handler upon command process termination, passing in the
-  # resulting Process::Status.
-  #
-  # All handlers must be methods exposed by the given target.
+  # @deprecated in favor of sync vs. async methods in RightPopen namespace.
   #
   # === Parameters
-  # options[:command](String or Array):: Command to execute, including any arguments as a single string or an array of command and arguments
-  # options[:environment](Hash):: Hash of environment variables values keyed by name
-  # options[:input](String):: Input string that will get streamed into child's process stdin
-  # options[:target](Object):: object defining handler methods to be called, optional (no handlers can be defined if not specified)
-  # options[:pid_handler](String):: PID notification handler method name, optional
-  # options[:stdout_handler](String):: Stdout handler method name, optional
-  # options[:stderr_handler](String):: Stderr handler method name, optional
-  # options[:exit_handler](String):: Exit handler method name, optional
+  # @param [Hash] options for execution
   #
   # === Returns
-  # true:: always true
+  # @return [TrueClass] always true
   def self.popen3(options)
-    raise "EventMachine reactor must be started" unless EM.reactor_running?
-    raise "Missing command" unless options[:command]
-    raise "Missing target" unless options[:target] || !options[:stdout_handler] && !options[:stderr_handler] && !options[:exit_handler] && !options[:pid_handler]
-    return RightScale.popen3_imp(options)
+    warn 'WARNING: RightScale.popen3 is deprecated in favor of RightScale::RightPopen.popen3_async'
+    options = options.dup
+    cmd = options.dup.delete(:command)
+    raise ::ArgumentError.new("Missing command") unless cmd
+    ::RightScale::RightPopen.popen3_async(options[:command], options)
   end
 
+  module RightPopen
+
+    # see popen3_async for details.
+    DEFAULT_POPEN3_OPTIONS = {
+      :environment      => nil,
+      :exit_handler     => nil,
+      :group            => nil,
+      :inherit_io       => false,
+      :input            => nil,
+      :locale           => true,
+      :pid_handler      => nil,
+      :size_limit_bytes => nil,
+      :stderr_handler   => nil,
+      :stdout_handler   => nil,
+      :target           => nil,
+      :timeout_seconds  => nil,
+      :umask            => nil,
+      :user             => nil,
+      :watch_handler    => nil,
+      :watch_directory  => nil,
+    }
+
+    # Loads the specified implementation.
+    #
+    # === Parameters
+    # @param [Symbol|String] synchronicity to load
+    #
+    # === Return
+    # @return [TrueClass] always true
+    def self.require_popen3_impl(synchronicity)
+      case RUBY_PLATFORM
+      when /mswin/
+        platform = 'win32'
+      when /mingw/
+        platform = 'mingw'
+      else
+        platform = 'linux'
+      end
+      require ::File.expand_path(::File.join(::File.dirname(__FILE__), 'right_popen', platform, synchronicity.to_s))
+    end
+
+    # Spawns a process to run given command synchronously. This is similar to
+    # the Ruby backtick but also supports streaming I/O, process watching, etc.
+    # Does not require any evented library to use.
+    #
+    # Streams the command's stdout and stderr to the given handlers. Time-
+    # ordering of bytes sent to stdout and stderr is not preserved.
+    #
+    # Calls given exit handler upon command process termination, passing in the
+    # resulting Process::Status.
+    #
+    # All handlers must be methods exposed by the given target.
+    #
+    # === Parameters
+    # @param [Hash] options see popen3_async for details
+    #
+    # === Returns
+    # @return [TrueClass] always true
+    def self.popen3_sync(cmd, options)
+      options = DEFAULT_POPEN3_OPTIONS.dup.merge(options)
+      require_popen3_impl(:popen3_sync)
+      ::RightScale::RightPopen.popen3_sync_impl(
+        cmd, ::RightScale::RightPopen::TargetProxy.new(options), options)
+    end
+
+    # Spawns a process to run given command asynchronously, hooking all three
+    # standard streams of the child process. Implementation requires the
+    # eventmachine gem.
+    #
+    # Streams the command's stdout and stderr to the given handlers. Time-
+    # ordering of bytes sent to stdout and stderr is not preserved.
+    #
+    # Calls given exit handler upon command process termination, passing in the
+    # resulting Process::Status.
+    #
+    # All handlers must be methods exposed by the given target.
+    #
+    # === Parameters
+    # @param [Hash] options for execution
+    # @option options [Hash] :environment variables values keyed by name
+    # @option options [Symbol] :exit_handler target method called on exit
+    # @option options [Integer|String] :group or gid for forked process (linux only)
+    # @option options [TrueClass|FalseClass] :inherit_io set to true to share all IO objects with forked process or false to close shared IO objects (default) (linux only)
+    # @option options [String] :input string that will get streamed into child's process stdin
+    # @option options [TrueClass|FalseClass] :locale set to true to export LC_ALL=C in the forked environment (default) or false to use default locale (linux only)
+    # @option options [Symbol] :pid_handler target method called with process ID (PID)
+    # @option options [Integer] :size_limit_bytes for total size of watched directory after which child process will be interrupted
+    # @option options [Symbol] :stderr_handler target method called as error text is received
+    # @option options [Symbol] :stdout_handler target method called as output text is received
+    # @option options [Object] :target object defining handler methods to be called (no handlers can be defined if not specified)
+    # @option options [Numeric] :timeout_seconds after which child process will be interrupted
+    # @option options [Integer|String] :umask for files created by process (linux only)
+    # @option options [Integer|String] :user or uid for forked process (linux only)
+    # @option options [Symbol] :watch_handler called periodically with process during watch; return true to continue, false to abandon (sync only)
+    # @option options [String] :watch_directory to monitor for child process writing files
+    #
+    # === Returns
+    # @return [TrueClass] always true
+    def self.popen3_async(cmd, options)
+      options = DEFAULT_POPEN3_OPTIONS.dup.merge(options)
+      require_popen3_impl(:popen3_async)
+      unless ::EM.reactor_running?
+        raise ::ArgumentError, "EventMachine reactor must be running."
+      end
+      ::RightScale::RightPopen.popen3_async_impl(
+        cmd, ::RightScale::RightPopen::TargetProxy.new(options), options)
+    end
+  end
 end

data/lib/right_popen/process_base.rb

@@ -0,0 +1,339 @@
+#--  -*- mode: ruby; encoding: utf-8 -*-
+# Copyright: Copyright (c) 2013 RightScale, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'thread'
+
+module RightScale
+  module RightPopen
+    class ProcessBase
+
+      class ProcessError < Exception; end
+
+      attr_reader :pid, :stdin, :stdout, :stderr, :status_fd, :status
+      attr_reader :start_time, :stop_time, :channels_to_finish
+
+      # === Parameters
+      # @param [Hash] options see RightScale.popen3_async for details
+      def initialize(options={})
+        @options = options
+        @stdin = nil
+        @stdout = nil
+        @stderr = nil
+        @status_fd = nil
+        @last_interrupt = nil
+        @pid = nil
+        @start_time = nil
+        @stop_time = nil
+        @watch_directory = nil
+        @size_limit_bytes = nil
+        @cmd = nil
+        @target = nil
+        @status = nil
+        @channels_to_finish = nil
+        @needs_watching = !!(
+          @options[:timeout_seconds] ||
+          @options[:size_limit_bytes] ||
+          @options[:watch_handler])
+      end
+
+      # Determines if the process is still running.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if running
+      def alive?
+        raise NotImplementedError, 'Must be overridden'
+      end
+
+      # Determines whether or not to drain all open streams upon death of child
+      # or else only those where IO.select indicates data available. This
+      # decision is platform-specific.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if draining all
+      def drain_all_upon_death?
+        raise NotImplementedError, 'Must be overridden'
+      end
+
+      # Determines if this process needs to be watched (beyond waiting for the
+      # process to exit).
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if needs watching
+      def needs_watching?; @needs_watching; end
+
+      # Determines if timeout on child process has expired, if any.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if timer expired
+      def timer_expired?
+        !!(@stop_time && Time.now >= @stop_time)
+      end
+
+      # Determines if total size of files created by child process has exceeded
+      # the limit specified, if any.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if size limit exceeded
+      def size_limit_exceeded?
+        if @watch_directory
+          globbie = ::File.join(@watch_directory, '**/*')
+          size = 0
+          ::Dir.glob(globbie) do |f|
+            size += ::File.stat(f).size rescue 0 if ::File.file?(f)
+            break if size > @size_limit_bytes
+          end
+          size > @size_limit_bytes
+        else
+          false
+        end
+      end
+
+      # @return [TrueClass|FalseClass] interrupted as true if child process was interrupted by watcher
+      def interrupted?; !!@last_interrupt; end
+
+      # Performs all process operations in synchronous fashion. It is possible
+      # for errors or callback behavior to conditionally short-circuit the
+      # synchronous operations.
+      #
+      # === Parameters
+      # @param [String|Array] cmd as shell command or binary to execute
+      # @param [Object] target that implements all handlers (see TargetProxy)
+      #
+      # === Return
+      # @return [TrueClass] always true
+      def sync_all(cmd, target)
+        spawn(cmd, target)
+        sync_exit_with_target if sync_pid_with_target
+        true
+      end
+
+      # Spawns a child process using given command and handler target in a
+      # platform-independant manner.
+      #
+      # must be overridden and override must call super.
+      #
+      # === Parameters
+      # @param [String|Array] cmd as shell command or binary to execute
+      # @param [Object] target that implements all handlers (see TargetProxy)
+      #
+      # === Return
+      # @return [TrueClass] always true
+      def spawn(cmd, target)
+        @cmd = cmd
+        @target = target
+        @kill_time = nil
+        @pid = nil
+        @status = nil
+        @last_interrupt = nil
+        @channels_to_finish = nil
+
+        if @size_limit_bytes = @options[:size_limit_bytes]
+          @watch_directory = @options[:watch_directory] || @options[:directory] || ::Dir.pwd
+        end
+      end
+
+      # Performs initial handler callbacks before consuming I/O. Represents any
+      # code that must not be invoked twice (unlike sync_exit_with_target).
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true to begin watch, false to abandon
+      def sync_pid_with_target
+        # early handling in case caller wants to stream to/from the pipes
+        # directly (as in a classic popen3/4 scenario).
+        @target.pid_handler(@pid)
+        if input_text = @options[:input]
+          @stdin.write(input_text)
+        end
+
+        # one-time initialization of the stateful channels_to_finish hash to
+        # allow for multiple invocations of the sync_exit_with_target with a
+        # possible abandon in between.
+        #
+        # note that calling IO.select on pipes which have already had all
+        # of their output consumed can cause segfault (in Ubuntu?) so it is
+        # important to keep track of when all I/O has been consumed.
+        @channels_to_finish = [
+          [:stdout_handler, @stdout],
+          [:stderr_handler, @stderr],
+        ]
+        @channels_to_finish << [:status_fd, @status_fd] if @status_fd
+
+        # sync watch_handler has the option to abandon watch as soon as child
+        # process comes alive and before streaming any output.
+        if @target.watch_handler(self)
+          # can close stdin if not returning control to caller.
+          @stdin.close rescue nil
+          return true
+        else
+          # caller is reponsible for draining and/or closing all pipes. this can
+          # be accomplished by explicity calling either sync_exit_with_target or
+          # safe_close_io plus wait_for_exit_status (if data has been read from
+          # the I/O streams). it is unsafe to read some data and then call
+          # sync_exit_with_target because IO.select may segfault if all of the
+          # data in a stream has been already consumed.
+          return false
+        end
+      rescue
+        safe_close_io
+        raise
+      end
+
+      # Monitors I/O from child process and directly notifies target of any
+      # events. Blocks until child exits.
+      #
+      # === Return
+      # @return [TrueClass] always true
+      def sync_exit_with_target
+        abandon = false
+        last_exception = nil
+        begin
+          while true
+            channels_to_watch = @channels_to_finish.map { |ctf| ctf.last }
+            ready = ::IO.select(channels_to_watch, nil, nil, 0.1) rescue nil
+            dead = !alive?
+            channels_to_read = ready && ready.first
+            if dead && drain_all_upon_death?
+              # finish reading all dead channels.
+              channels_to_read = @channels_to_finish.map { |ctf| ctf.last }
+            end
+            if channels_to_read
+              channels_to_read.each do |channel|
+                index = @channels_to_finish.index { |ctf| ctf.last == channel }
+                key = @channels_to_finish[index].first
+                data = dead ? channel.gets(nil) : channel.gets
+                if data
+                  if key == :status_fd
+                    last_exception = ::Marshal.load(data)
+                  else
+                    @target.method(key).call(data)
+                  end
+                else
+                  # nothing on channel indicates EOF
+                  @channels_to_finish.delete_at(index)
+                end
+              end
+            end
+            if dead
+              break
+            elsif (interrupted? || timer_expired? || size_limit_exceeded?)
+              interrupt
+            elsif abandon = !@target.watch_handler(self)
+              return true  # bypass any remaining callbacks
+            end
+          end
+          wait_for_exit_status
+          @target.timeout_handler if timer_expired?
+          @target.size_limit_handler if size_limit_exceeded?
+          @target.exit_handler(@status)
+
+          # re-raise exception from fork, if any.
+          case last_exception
+          when nil
+            # all good
+          when ::Exception
+            raise last_exception
+          else
+            raise "Unknown failure: saw #{last_exception.inspect} on status channel."
+          end
+        ensure
+          # abandon will not close I/O objects; caller takes responsibility via
+          # process object passed to watch_handler. if anyone calls interrupt
+          # then close I/O regardless of abandon to try to force child to die.
+          safe_close_io if !abandon || interrupted?
+        end
+        true
+      end
+
+      # blocks waiting for process exit status.
+      #
+      # === Return
+      # @return [ProcessStatus] exit status
+      def wait_for_exit_status
+        raise NotImplementedError, 'Must be overridden'
+      end
+
+      # @return [Array] escalating termination signals for this platform
+      def signals_for_interrupt
+        raise NotImplementedError, 'Must be overridden'
+      end
+
+      # Interrupts the running process (without abandoning watch) in increasing
+      # degrees of signalled severity.
+      #
+      # === Return
+      # @return [TrueClass|FalseClass] true if process was alive and interrupted, false if dead before (first) interrupt
+      def interrupt
+        while alive?
+          if !@kill_time || Time.now >= @kill_time
+            # soft then hard interrupt (assumed to be called periodically until
+            # process is gone).
+            sigs = signals_for_interrupt
+            if @last_interrupt
+              last_index = sigs.index(@last_interrupt)
+              next_interrupt = sigs[last_index + 1]
+            else
+              next_interrupt = sigs.first
+            end
+            unless next_interrupt
+              raise ::RightScale::RightPopen::ProcessBase::ProcessError
+                    'Unable to kill child process'
+            end
+            @last_interrupt = next_interrupt
+
+            # kill
+            result = ::Process.kill(next_interrupt, @pid) rescue nil
+            if result
+              @kill_time = Time.now + 3 # more seconds until next attempt
+              break
+            end
+          end
+        end
+        interrupted?
+      end
+
+      # Safely closes any open I/O objects associated with this process.
+      #
+      # === Return
+      # @return [TrueClass] alway true
+      def safe_close_io
+        @stdin.close rescue nil if @stdin && !@stdin.closed?
+        @stdout.close rescue nil if @stdout && !@stdout.closed?
+        @stderr.close rescue nil if @stderr && !@stderr.closed?
+        @status_fd.close rescue nil if @status_fd && !@status_fd.closed?
+        true
+      end
+
+      protected
+
+      def start_timer
+        # start timer when process comes alive (ruby processes are slow to
+        # start in Windows, etc.).
+        raise ProcessError.new("Process not started") unless @pid
+        @start_time = ::Time.now
+        @stop_time  = @options[:timeout_seconds] ?
+                      (@start_time + @options[:timeout_seconds]) :
+                      nil
+      end
+    end
+  end
+end