right_popen 1.0.21 → 1.1.3

data/README.rdoc

@@ -15,6 +15,7 @@ documentation.
 Also use the built-in issues tracker (https://github.com/rightscale/right_popen/issues)
 to report issues.
 
+Maintained by the RightScale Teal Team
 
 == USAGE
 
@@ -47,13 +48,14 @@ to report issues.
   EM.run do
     EM.next_tick do
       command = "ruby -e \"puts 'some stdout text'; $stderr.puts 'some stderr text'\; exit 99\""
-      RightScale.popen3(:command        => command,
-                        :target         => self,
-                        :environment    => nil,
-                        :pid_handler    => :on_pid,
-                        :stdout_handler => :on_read_stdout,
-                        :stderr_handler => :on_read_stderr,
-                        :exit_handler   => :on_exit)
+      RightScale::RightPopen.popen3_async(
+        command,
+        :target         => self,
+        :environment    => nil,
+        :pid_handler    => :on_pid,
+        :stdout_handler => :on_read_stdout,
+        :stderr_handler => :on_read_stderr,
+        :exit_handler   => :on_exit)
     end
     timer = EM::PeriodicTimer.new(0.1) do
       if @exit_status
@@ -109,7 +111,7 @@ the RightPopen tests:
 
 <b>RightPopen</b>
 
-Copyright:: Copyright (c) 2010 RightScale, Inc. 
+Copyright:: Copyright (c) 2010-2013 RightScale, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/right_popen.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009 RightScale Inc
+# Copyright (c) 2009-2013 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,46 +21,141 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-# RightScale.popen3 allows running external processes aynchronously
-# while still capturing their standard and error outputs.
-# It relies on EventMachine for most of its internal mechanisms.
+require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'target_proxy'))
 
-if RUBY_PLATFORM =~ /mswin/
-  require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'win32', 'right_popen'))
-else
-  require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'linux', 'right_popen'))
+# TEAL FIX: this seems like test harness code smell, not production code. it
+# should be removed in next major revision. unfortunately we cannot remove these
+# require statements without breaking any code that depends on them.
+unless RUBY_PLATFORM =~ /mswin|mingw/
+  require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'linux', 'accumulator'))
+  require File.expand_path(File.join(File.dirname(__FILE__), 'right_popen', 'linux', 'utilities'))
 end
 
 module RightScale
 
-  # Spawn process to run given command asynchronously, hooking all three
-  # standard streams of the child process.
+  # see popen3_async for details.
   #
-  # Streams the command's stdout and stderr to the given handlers. Time-
-  # ordering of bytes sent to stdout and stderr is not preserved.
-  #
-  # Calls given exit handler upon command process termination, passing in the
-  # resulting Process::Status.
-  #
-  # All handlers must be methods exposed by the given target.
+  # @deprecated in favor of sync vs. async methods in RightPopen namespace.
   #
   # === Parameters
-  # options[:command](String or Array):: Command to execute, including any arguments as a single string or an array of command and arguments
-  # options[:environment](Hash):: Hash of environment variables values keyed by name
-  # options[:input](String):: Input string that will get streamed into child's process stdin
-  # options[:target](Object):: object defining handler methods to be called, optional (no handlers can be defined if not specified)
-  # options[:pid_handler](String):: PID notification handler method name, optional
-  # options[:stdout_handler](String):: Stdout handler method name, optional
-  # options[:stderr_handler](String):: Stderr handler method name, optional
-  # options[:exit_handler](String):: Exit handler method name, optional
+  # @param [Hash] options for execution
   #
   # === Returns
-  # true:: always true
+  # @return [TrueClass] always true
   def self.popen3(options)
-    raise "EventMachine reactor must be started" unless EM.reactor_running?
-    raise "Missing command" unless options[:command]
-    raise "Missing target" unless options[:target] || !options[:stdout_handler] && !options[:stderr_handler] && !options[:exit_handler] && !options[:pid_handler]
-    return RightScale.popen3_imp(options)
+    warn 'WARNING: RightScale.popen3 is deprecated in favor of RightScale::RightPopen.popen3_async'
+    options = options.dup
+    cmd = options.dup.delete(:command)
+    raise ::ArgumentError.new("Missing command") unless cmd
+    ::RightScale::RightPopen.popen3_async(options[:command], options)
   end
 
+  module RightPopen
+
+    # see popen3_async for details.
+    DEFAULT_POPEN3_OPTIONS = {
+      :environment      => nil,
+      :exit_handler     => nil,
+      :group            => nil,
+      :inherit_io       => false,
+      :input            => nil,
+      :locale           => true,
+      :pid_handler      => nil,
+      :size_limit_bytes => nil,
+      :stderr_handler   => nil,
+      :stdout_handler   => nil,
+      :target           => nil,
+      :timeout_seconds  => nil,
+      :umask            => nil,
+      :user             => nil,
+      :watch_handler    => nil,
+      :watch_directory  => nil,
+    }
+
+    # Loads the specified implementation.
+    #
+    # === Parameters
+    # @param [Symbol|String] synchronicity to load
+    #
+    # === Return
+    # @return [TrueClass] always true
+    def self.require_popen3_impl(synchronicity)
+      case RUBY_PLATFORM
+      when /mswin/
+        platform = 'win32'
+      when /mingw/
+        platform = 'mingw'
+      else
+        platform = 'linux'
+      end
+      require ::File.expand_path(::File.join(::File.dirname(__FILE__), 'right_popen', platform, synchronicity.to_s))
+    end
+
+    # Spawns a process to run given command synchronously. This is similar to
+    # the Ruby backtick but also supports streaming I/O, process watching, etc.
+    # Does not require any evented library to use.
+    #
+    # Streams the command's stdout and stderr to the given handlers. Time-
+    # ordering of bytes sent to stdout and stderr is not preserved.
+    #
+    # Calls given exit handler upon command process termination, passing in the
+    # resulting Process::Status.
+    #
+    # All handlers must be methods exposed by the given target.
+    #
+    # === Parameters
+    # @param [Hash] options see popen3_async for details
+    #
+    # === Returns
+    # @return [TrueClass] always true
+    def self.popen3_sync(cmd, options)
+      options = DEFAULT_POPEN3_OPTIONS.dup.merge(options)
+      require_popen3_impl(:popen3_sync)
+      ::RightScale::RightPopen.popen3_sync_impl(
+        cmd, ::RightScale::RightPopen::TargetProxy.new(options), options)
+    end
+
+    # Spawns a process to run given command asynchronously, hooking all three
+    # standard streams of the child process. Implementation requires the
+    # eventmachine gem.
+    #
+    # Streams the command's stdout and stderr to the given handlers. Time-
+    # ordering of bytes sent to stdout and stderr is not preserved.
+    #
+    # Calls given exit handler upon command process termination, passing in the
+    # resulting Process::Status.
+    #
+    # All handlers must be methods exposed by the given target.
+    #
+    # === Parameters
+    # @param [Hash] options for execution
+    # @option options [Hash] :environment variables values keyed by name
+    # @option options [Symbol] :exit_handler target method called on exit
+    # @option options [Integer|String] :group or gid for forked process (linux only)
+    # @option options [TrueClass|FalseClass] :inherit_io set to true to share all IO objects with forked process or false to close shared IO objects (default) (linux only)
+    # @option options [String] :input string that will get streamed into child's process stdin
+    # @option options [TrueClass|FalseClass] :locale set to true to export LC_ALL=C in the forked environment (default) or false to use default locale (linux only)
+    # @option options [Symbol] :pid_handler target method called with process ID (PID)
+    # @option options [Integer] :size_limit_bytes for total size of watched directory after which child process will be interrupted
+    # @option options [Symbol] :stderr_handler target method called as error text is received
+    # @option options [Symbol] :stdout_handler target method called as output text is received
+    # @option options [Object] :target object defining handler methods to be called (no handlers can be defined if not specified)
+    # @option options [Numeric] :timeout_seconds after which child process will be interrupted
+    # @option options [Integer|String] :umask for files created by process (linux only)
+    # @option options [Integer|String] :user or uid for forked process (linux only)
+    # @option options [Symbol] :watch_handler called periodically with process during watch; return true to continue, false to abandon (sync only)
+    # @option options [String] :watch_directory to monitor for child process writing files
+    #
+    # === Returns
+    # @return [TrueClass] always true
+    def self.popen3_async(cmd, options)
+      options = DEFAULT_POPEN3_OPTIONS.dup.merge(options)
+      require_popen3_impl(:popen3_async)
+      unless ::EM.reactor_running?
+        raise ::ArgumentError, "EventMachine reactor must be running."
+      end
+      ::RightScale::RightPopen.popen3_async_impl(
+        cmd, ::RightScale::RightPopen::TargetProxy.new(options), options)
+    end
+  end
 end

data/lib/right_popen/linux/accumulator.rb

@@ -23,10 +23,13 @@
 
 module RightScale
   module RightPopen
+
+    # @deprecated this seems like test harness code smell, not production code
     class Accumulator
       READ_CHUNK_SIZE = 4096
 
       def initialize(process, inputs, read_callbacks, outputs, write_callbacks)
+        warn 'WARNING: RightScale::RightPopen::Accumulator is deprecated and will be removed.'
         @process = process
         @inputs = inputs
         @outputs = outputs
@@ -45,10 +48,17 @@ module RightScale
         @status = nil
       end
 
+      def status
+        unless @status
+          @status = ::Process.waitpid2(@process.pid, ::Process::WNOHANG)
+        end
+        @status
+      end
+
       def tick(sleep_time = 0.1)
-        return true unless @process.status.nil?
+        return true unless @status.nil?
 
-        @process.status = status = ::Process.waitpid2(@process.pid, ::Process::WNOHANG)
+        status
 
         inputs = @inputs.dup
         outputs = @outputs.dup
@@ -93,7 +103,7 @@ module RightScale
           end
         end unless ready.nil? || ready[1].nil?
 
-        return !@process.status.nil?
+        return !@status.nil?
       end
 
       def number_waiting_on
@@ -103,7 +113,7 @@ module RightScale
       def cleanup
         @inputs.each {|p| p.close unless p.closed? }
         @outputs.each {|p| p.close unless p.closed? }
-        @process.status = ::Process.waitpid2(@process.pid) if @process.status.nil?
+        @status = ::Process.waitpid2(@process.pid) if @status.nil?
       end
 
       def run_to_completion(sleep_time=0.1)

data/lib/right_popen/linux/{right_popen.rb → popen3_async.rb}

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009 RightScale Inc
+# Copyright (c) 2009-2013 RightScale Inc
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,17 +21,13 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #++
 
-# RightScale.popen3 allows running external processes aynchronously
-# while still capturing their standard and error outputs.
-# It relies on EventMachine for most of its internal mechanisms.
-
 require 'rubygems'
 require 'eventmachine'
+
 require File.expand_path(File.join(File.dirname(__FILE__), "process"))
-require File.expand_path(File.join(File.dirname(__FILE__), "accumulator"))
-require File.expand_path(File.join(File.dirname(__FILE__), "utilities"))
 
-module RightScale
+module RightScale::RightPopen
+
   # ensure uniqueness of handler to avoid confusion.
   raise "#{StatusHandler.name} is already defined" if defined?(StatusHandler)
 
@@ -63,8 +59,8 @@ module RightScale
 
     def unbind
       if @data.size > 0
-        e = Marshal.load @data
-        raise (Exception === e ? e : "unknown failure: saw #{e} on status channel")
+        e = ::Marshal.load(@data)
+        raise (::Exception === e ? e : "unknown failure: saw #{e} on status channel")
       end
     end
   end
@@ -80,11 +76,11 @@ module RightScale
       # itself.
       @handle = file_handle
       @target = target
-      @handler = handler
+      @data_handler = @target.method(handler)
     end
 
     def receive_data(data)
-      @target.method(@handler).call(data) if @handler
+      @data_handler.call(data)
     end
 
     def drain_and_close
@@ -123,62 +119,82 @@ module RightScale
     end
   end
 
-  # Forks process to run given command asynchronously, hooking all three
-  # standard streams of the child process.
-  #
-  # === Parameters
-  # options[:pid_handler](Symbol):: Token for pid handler method name.
-  #
-  # See RightScale.popen3
-  def self.popen3_imp(options)
-    GC.start # To garbage collect open file descriptors from past executions
-    EM.next_tick do
-      process = RightPopen::Process.new(:environment => options[:environment] || {})
-      process.fork(options[:command])
-
-      handlers = []
-      handlers << EM.attach(process.status_fd, StatusHandler, process.status_fd)
-      handlers << EM.attach(process.stderr, PipeHandler, process.stderr, options[:target],
-                            options[:stderr_handler])
-      handlers << EM.attach(process.stdout, PipeHandler, process.stdout, options[:target],
-                            options[:stdout_handler])
-      handlers << EM.attach(process.stdin, InputHandler, process.stdin, options[:input])
-
-      options[:target].method(options[:pid_handler]).call(process.pid) if options.key? :pid_handler
-
-      handle_exit(process.pid, 0.1, handlers, options)
+  # See RightScale.popen3_async for details
+  def self.popen3_async_impl(cmd, target, options)
+    # always create eventables on the main EM thread by using next_tick. this
+    # prevents synchronization problems between EM threads.
+    ::EM.next_tick do
+      process = nil
+      begin
+        # create process.
+        process = ::RightScale::RightPopen::Process.new(options)
+        process.spawn(cmd, target)
+
+        # connect EM eventables to open streams.
+        handlers = []
+        handlers << ::EM.attach(process.status_fd, ::RightScale::RightPopen::StatusHandler, process.status_fd)
+        handlers << ::EM.attach(process.stderr, ::RightScale::RightPopen::PipeHandler, process.stderr, target, :stderr_handler)
+        handlers << ::EM.attach(process.stdout, ::RightScale::RightPopen::PipeHandler, process.stdout, target, :stdout_handler)
+        handlers << ::EM.attach(process.stdin, ::RightScale::RightPopen::InputHandler, process.stdin, options[:input])
+
+        target.pid_handler(process.pid)
+
+        # initial watch callback.
+        #
+        # note that we cannot abandon async watch; callback needs to interrupt
+        # in this case
+        target.watch_handler(process)
+
+        # periodic watcher.
+        watch_process(process, 0.1, target, handlers)
+      rescue
+        # we can't raise from the main EM thread or it will stop EM.
+        # the spawn method will signal the exit handler but not the
+        # pid handler in this case since there is no pid. any action
+        # (logging, etc.) associated with the failure will have to be
+        # driven by the exit handler.
+        target.exit_handler(process.status) rescue nil if target && process
+      end
     end
     true
   end
 
-  # Wait for process to exit and then call exit handler
-  # If no exit detected, double the wait time up to a maximum of 2 seconds
+  # watches process for exit or interrupt criteria. doubles the wait time up to
+  # a maximum of 1 second for next wait.
   #
   # === Parameters
-  # pid(Integer):: Process identifier
-  # wait_time(Fixnum):: Amount of time to wait before checking status
-  # handlers(Array):: Handlers for status, stderr, stdout, and stdin
-  # options[:exit_handler](Symbol):: Handler to be called when process exits
-  # options[:target](Object):: Object initiating command execution
+  # @param [Process] process that was run
+  # @param [Numeric] wait_time as seconds to wait before checking status
+  # @param [Object] target for handler calls
+  # @param [Array] handlers used by eventmachine for status, stderr, stdout, and stdin
   #
   # === Return
   # true:: Always return true
-  def self.handle_exit(pid, wait_time, handlers, options)
-    EM::Timer.new(wait_time) do
-      if value = Process.waitpid2(pid, Process::WNOHANG)
-        ignored, status = value
-        first_exception = nil
-        handlers.each do |h|
-          begin
-            h.drain_and_close
-          rescue Exception => e
-            first_exception = e unless first_exception
+  def self.watch_process(process, wait_time, target, handlers)
+    ::EM::Timer.new(wait_time) do
+      begin
+        if process.alive?
+          if process.timer_expired? || process.size_limit_exceeded?
+            process.interrupt
+          else
+            # cannot abandon async watch; callback needs to interrupt in this case
+            target.watch_handler(process)
           end
+          watch_process(process, [wait_time * 2, 1].min, target, handlers)
+        else
+          handlers.each { |h| h.drain_and_close rescue nil }
+          process.wait_for_exit_status
+          target.timeout_handler rescue nil if process.timer_expired?
+          target.size_limit_handler rescue nil if process.size_limit_exceeded?
+          target.exit_handler(process.status) rescue nil
         end
-        options[:target].method(options[:exit_handler]).call(status) if options[:exit_handler]
-        raise first_exception if first_exception
-      else
-        handle_exit(pid, [wait_time * 2, 1].min, handlers, options)
+      rescue
+        # we can't raise from the main EM thread or it will stop EM.
+        # the spawn method will signal the exit handler but not the
+        # pid handler in this case since there is no pid. any action
+        # (logging, etc.) associated with the failure will have to be
+        # driven by the exit handler.
+        target.exit_handler(process.status) rescue nil if target && process
       end
     end
     true

data/lib/right_popen/linux/popen3_sync.rb

@@ -0,0 +1,35 @@
+#--
+# Copyright (c) 2013 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require File.expand_path(File.join(File.dirname(__FILE__), "process"))
+
+module RightScale::RightPopen
+
+  # See RightScale.popen3_sync for details
+  def self.popen3_sync_impl(cmd, target, options)
+    process = ::RightScale::RightPopen::Process.new(options)
+    process.sync_all(cmd, target)
+    true
+  end
+
+end