right-popen 1.0.0-x86-mswin32-60

data/ext/win32/right_popen.h

@@ -0,0 +1,32 @@
+///////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2010 RightScale Inc
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+///////////////////////////////////////////////////////////////////////////////
+
+#include "ruby.h"
+#include "rubysig.h"
+#include "rubyio.h"
+
+#include <malloc.h>
+#include <io.h>
+#include <string.h>
+#include <fcntl.h>
+#include <sys/stat.h>

data/lib/right_popen.rb

@@ -0,0 +1,57 @@
+#
+# Copyright (c) 2009 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+# RightScale.popen3 allows running external processes aynchronously
+# while still capturing their standard and error outputs.
+# It relies on EventMachine for most of its internal mechanisms.
+
+if RUBY_PLATFORM =~ /mswin/
+  require File.expand_path(File.join(File.dirname(__FILE__), 'win32', 'right_popen'))
+else
+  require File.expand_path(File.join(File.dirname(__FILE__), 'linux', 'right_popen'))
+end
+
+module RightScale
+
+  # Closes all open I/O objects in the object space, keeping any provided as
+  # arguments.
+  #
+  # === Parameters
+  # keep(Array):: array of I/O objects to keep.
+  def self.close_all_fd(keep = [])
+    keep = [keep] unless keep.is_a? Array
+    keep += [$stdin, $stdout, $stderr]
+    keep = keep.collect { |io| io.fileno }
+
+    ObjectSpace.each_object(IO) do |io|
+      unless io.closed? || keep.include?(io.fileno)
+        begin
+          io.close
+        rescue Exception
+        #do nothing
+        end
+      end
+    end
+  end
+
+end

data/lib/win32/right_popen.rb

@@ -0,0 +1,234 @@
+#
+# Copyright (c) 2009 RightScale Inc
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+
+# RightScale.popen3 allows running external processes aynchronously
+# while still capturing their standard and error outputs.
+# It relies on EventMachine for most of its internal mechanisms.
+
+require 'rubygems'
+begin
+  gem 'eventmachine', '=0.12.8.1'  # patched version for Windows-only socket close fix
+rescue Gem::LoadError
+  gem 'eventmachine', '=0.12.8'  # notify_readable is deprecated in 0.12.10
+end
+require 'eventmachine'
+require 'win32/process'
+
+require File.join(File.dirname(__FILE__), 'right_popen.so')  # win32 native code
+
+module RightScale
+
+  # Provides an eventmachine callback handler for the stdout stream.
+  module StdOutHandler
+
+    # Quacks like Process::Status, which we cannot instantiate ourselves because
+    # has no public new method. RightScale::popen3 needs this because the
+    # 'win32/process' gem currently won't return Process::Status objects but
+    # only returns a [pid, exitstatus] value.
+    class Status
+      # Process ID
+      attr_reader :pid
+
+      # Process exit code
+      attr_reader :exitstatus
+
+      # === Parameters
+      # pid(Integer): Process ID.
+      # 
+      # exitstatus(Integer): Process exit code
+      def initialize(pid, exitstatus)
+        @pid = pid
+        @exitstatus = exitstatus
+      end
+
+      # Simulates Process::Status.exited?
+      #
+      # === Returns
+      # true in all cases because this object cannot be returned until the
+      # process exits
+      def exited?
+        return true
+      end
+
+      # Simulates Process::Status.success?
+      #
+      # === Returns
+      # true in if the process returned zero as its exit code
+      def success?
+        return @exitstatus ? (0 == @exitstatus) : true;
+      end
+    end
+
+    # === Parameters
+    # target(Object): Object defining handler methods to be called.
+    #
+    # stdout_handler(String): Token for stdout handler method name.
+    #
+    # exit_handler(String): Token for exit handler method name.
+    #
+    # stderr_eventable(Connector): EM object representing stderr handler.
+    #
+    # stream_out(IO): Standard output stream.
+    #
+    # pid(Integer): Child process ID.
+    def initialize(target, stdout_handler, exit_handler, stderr_eventable, stream_out, pid)
+      @target = target
+      @stdout_handler = stdout_handler
+      @exit_handler = exit_handler
+      @stderr_eventable = stderr_eventable
+      @stream_out = stream_out
+      @pid = pid
+      @status = nil
+    end
+
+    # Callback from EM to asynchronously read the stdout stream. Note that this
+    # callback mechanism is deprecated after EM v0.12.8
+    def notify_readable
+      data = RightPopen.async_read(@stream_out)
+      receive_data(data) if (data && data.length > 0)
+      detach unless data
+    end
+
+    # Callback from EM to receive data, which we also use to handle the
+    # asynchronous data we read ourselves.
+    def receive_data(data)
+      @target.method(@stdout_handler).call(data) if @stdout_handler
+    end
+
+    # Override of Connection.get_status() for Windows implementation.
+    def get_status
+      unless @status
+        begin
+          @status = Status.new(@pid, Process.waitpid2(@pid)[1])
+        rescue Process::Error
+          # process is gone, which means we have no recourse to retrieve the
+          # actual exit code; let's be optimistic.
+          @status = Status.new(@pid, 0)
+        end
+      end
+      return @status
+    end
+
+    # Callback from EM to unbind.
+    def unbind
+      # We force the attached stderr handler to go away so that
+      # we don't end up with a broken pipe
+      @stderr_eventable.force_detach if @stderr_eventable
+      @target.method(@exit_handler).call(get_status) if @exit_handler
+      @stream_out.close
+    end
+  end
+
+  # Provides an eventmachine callback handler for the stderr stream.
+  module StdErrHandler
+
+    # === Parameters
+    # target(Object): Object defining handler methods to be called.
+    #
+    # stderr_handler(String): Token for stderr handler method name.
+    # 
+    # stream_err(IO): Standard error stream.
+    def initialize(target, stderr_handler, stream_err)
+      @target = target
+      @stderr_handler = stderr_handler
+      @stream_err = stream_err
+      @unbound = false
+    end
+
+    # Callback from EM to asynchronously read the stderr stream. Note that this
+    # callback mechanism is deprecated after EM v0.12.8
+    def notify_readable
+      data = RightPopen.async_read(@stream_err)
+      receive_data(data) if (data && data.length > 0)
+      detach unless data
+    end
+
+    # Callback from EM to receive data, which we also use to handle the
+    # asynchronous data we read ourselves.
+    def receive_data(data)
+      @target.method(@stderr_handler).call(data)
+    end
+
+    # Callback from EM to unbind.
+    def unbind
+      @unbound = true
+      @stream_err.close
+    end
+
+    # Forces detachment of the stderr handler on EM's next tick.
+    def force_detach
+      # Use next tick to prevent issue in EM where descriptors list
+      # gets out-of-sync when calling detach in an unbind callback
+      EM.next_tick { detach unless @unbound }
+    end
+  end
+
+  # Creates a child process and connects event handlers to the standard output
+  # and error streams used by the created process. Connectors use named pipes
+  # and asynchronous I/O in the native Windows implementation.
+  # 
+  # Streams the command's stdout and stderr to the given handlers. Time-
+  # ordering of bytes sent to stdout and stderr is not preserved.
+  #
+  # Calls given exit handler upon command process termination, passing in the
+  # resulting Process::Status.
+  #
+  # All handlers must be methods exposed by the given target.
+  #
+  # === Parameters
+  # cmd(String): command to execute, including any arguments.
+  #
+  # target(Object): object defining handler methods to be called.
+  #
+  # stdout_handler(String): token for stdout handler method name.
+  #
+  # stderr_handler(String): token for stderr handler method name.
+  #
+  # exit_handler(String): token for exit handler method name.
+  def self.popen3(cmd, target, stdout_handler = nil, stderr_handler = nil, exit_handler = nil)
+    raise "EventMachine reactor must be started" unless EM.reactor_running?
+
+    # launch cmd and request asynchronous output (which is only provided by
+    # the RightScale version of win32/open3 gem).
+    mode = "t"
+    show_window = false
+    asynchronous_output = true
+    stream_in, stream_out, stream_err, pid = RightPopen.popen4(cmd, mode, show_window, asynchronous_output)
+
+    # close input immediately.
+    stream_in.close
+
+    # attach handlers to event machine and let it monitor incoming data. the
+    # streams aren't used directly by the connectors except that they are closed
+    # on unbind.
+    stderr_eventable = EM.attach(stream_err, StdErrHandler, target, stderr_handler, stream_err) if stderr_handler
+    EM.attach(stream_out, StdOutHandler, target, stdout_handler, exit_handler, stderr_eventable, stream_out, pid)
+
+    # note that control returns to the caller, but the launched cmd continues
+    # running and sends output to the handlers. the caller is not responsible
+    # for waiting for the process to terminate or closing streams as the
+    # attached eventables will handle this automagically. notification will be
+    # sent to the exit_handler on process termination.
+  end
+
+end

data/right-popen.gemspec

@@ -0,0 +1,59 @@
+require 'rubygems'
+
+spec = Gem::Specification.new do |spec|
+  is_windows = RUBY_PLATFORM =~ /mswin/
+
+  spec.name      = 'right-popen'
+  spec.version   = '1.0.0'
+  spec.authors   = ['Scott Messier', 'Raphael Simon']
+  spec.email     = 'scott@rightscale.com'
+  spec.homepage  = 'https://github.com/rightscale/right_popen'
+  if is_windows
+    spec.platform = 'x86-mswin32-60'
+  else
+    spec.platform  = Gem::Platform::RUBY
+  end
+  spec.summary   = 'Provides a platform-independent popen implementation'
+  spec.has_rdoc = true
+  spec.rdoc_options = ["--main", "README.rdoc", "--title", "RightPopen"]
+  spec.extra_rdoc_files = ["README.rdoc"]
+  spec.required_ruby_version = '>= 1.8.6'
+  spec.rubyforge_project = %q{right-popen}
+
+  spec.description = <<-EOF
+RightPopen allows running external processes aynchronously while still
+capturing their standard and error outputs. It relies on EventMachine for most
+of its internal mechanisms. The linux implementation is valid for any linux
+platform but there is also a native implementation for Windows platforms.
+EOF
+
+  if is_windows
+    extension_dir = "ext,"
+  else
+    extension_dir = ""
+  end
+  candidates = Dir.glob("{#{extension_dir}lib,spec}/**/*") +
+               ["LICENSE", "README.rdoc", "Rakefile", "right-popen.gemspec"]
+  candidates = candidates.delete_if do |item|
+    item.include?("Makefile") || item.include?(".obj") || item.include?(".pdb") || item.include?(".def") || item.include?(".exp") || item.include?(".lib")
+  end
+  candidates = candidates.delete_if do |item|
+    if is_windows
+      item.include?("/linux/")
+    else
+      item.include?("/win32/")
+    end
+  end
+  spec.files = candidates.sort!
+
+  # Current implementation doesn't support > 0.12.8, but support any patched 0.12.8.x versions.
+  spec.add_runtime_dependency(%q<eventmachine>, [">= 0.12.8", "< 0.12.9"])
+  if is_windows
+    spec.add_runtime_dependency(%q<win32-process>, [">= 0.6.1"])
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+   Gem.manage_gems if Gem::RubyGemsVersion.to_f < 1.0
+   Gem::Builder.new(spec).build
+end

data/spec/produce_mixed_output.rb

@@ -0,0 +1,8 @@
+count = ARGV[0] ? ARGV[0].to_i : 1
+
+count.times do |i|
+  $stderr.puts "stderr #{i}" if 0 == i % 10
+  $stdout.puts "stdout #{i}"
+end
+
+exit 99

data/spec/produce_output.rb

@@ -0,0 +1,2 @@
+$stdout.puts ARGV[0]
+$stderr.puts ARGV[1]

data/spec/produce_status.rb

@@ -0,0 +1 @@
+exit ARGV[0].to_i

data/spec/produce_stderr_only.rb

@@ -0,0 +1,5 @@
+count = ARGV[0] ? ARGV[0].to_i : 1
+
+count.times do |i|
+  $stderr.puts "stderr #{i}"
+end

data/spec/produce_stdout_only.rb

@@ -0,0 +1,5 @@
+count = ARGV[0] ? ARGV[0].to_i : 1
+
+count.times do |i|
+  $stdout.puts "stdout #{i}"
+end

data/spec/right_popen_spec.rb

@@ -0,0 +1,143 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+require 'right_popen'
+
+RUBY_CMD             = 'ruby'
+STANDARD_MESSAGE     = 'Standard message'
+ERROR_MESSAGE        = 'Error message'
+EXIT_STATUS          = 146
+LARGE_OUTPUT_COUNTER = 1000
+
+describe 'RightScale::popen3' do
+
+  def on_read_stdout(data)
+    @stdoutput << data
+  end
+
+  def on_read_stderr(data)
+    @stderr << data
+  end
+
+  def on_exit(status)
+    @done = status
+  end
+
+  before(:each) do
+    @done      = false
+    @stdoutput = ''
+    @stderr    = ''
+  end
+
+  it 'should redirect output' do
+    EM.next_tick do
+      cmd = "\"#{RUBY_CMD}\" \"#{File.expand_path(File.join(File.dirname(__FILE__), 'produce_output.rb'))}\" \"#{STANDARD_MESSAGE}\" \"#{ERROR_MESSAGE}\""
+      RightScale.popen3(cmd, self, :on_read_stdout, :on_read_stderr, :on_exit)
+    end
+    EM.run do
+      timer = EM::PeriodicTimer.new(0.1) do
+        if @done
+          timer.cancel
+          @stdoutput.should == STANDARD_MESSAGE + "\n"
+          @stderr.should == ERROR_MESSAGE + "\n"
+          EM.stop
+        end
+      end
+    end
+  end
+
+  it 'should return the right status' do
+    EM.next_tick do
+      cmd = "\"#{RUBY_CMD}\" \"#{File.expand_path(File.join(File.dirname(__FILE__), 'produce_status.rb'))}\" #{EXIT_STATUS}"
+      RightScale.popen3(cmd, self, nil, nil, :on_exit)
+    end
+    EM.run do
+      timer = EM::PeriodicTimer.new(0.1) do
+        if @done
+          timer.cancel
+          @done.exitstatus.should == EXIT_STATUS
+          EM.stop
+        end
+      end
+    end
+  end
+
+  it 'should preserve the integrity of stdout when stderr is unavailable' do
+    # manually bump count up for more aggressive multi-processor testing, lessen
+    # for a quick smoke test
+    count = LARGE_OUTPUT_COUNTER
+    EM.next_tick do
+      cmd = "\"#{RUBY_CMD}\" \"#{File.expand_path(File.join(File.dirname(__FILE__), 'produce_stdout_only.rb'))}\" #{count}"
+      RightScale.popen3(cmd, self, :on_read_stdout, :on_read_stderr, :on_exit)
+    end
+    EM.run do
+      timer = EM::PeriodicTimer.new(0.1) do
+        if @done
+          timer.cancel
+          results = []
+          count.times do |i|
+            results << "stdout #{i}"
+          end
+          @stdoutput.should == results.join("\n") + "\n"
+          results = []
+          @stderr.should == ''
+          EM.stop
+        end
+      end
+    end
+  end
+
+  it 'should preserve the integrity of stderr when stdout is unavailable' do
+    # manually bump count up for more aggressive multi-processor testing, lessen
+    # for a quick smoke test
+    count = LARGE_OUTPUT_COUNTER
+    EM.next_tick do
+      cmd = "\"#{RUBY_CMD}\" \"#{File.expand_path(File.join(File.dirname(__FILE__), 'produce_stderr_only.rb'))}\" #{count}"
+      RightScale.popen3(cmd, self, :on_read_stdout, :on_read_stderr, :on_exit)
+    end
+    EM.run do
+      timer = EM::PeriodicTimer.new(0.1) do
+        if @done
+          timer.cancel
+          results = []
+          count.times do |i|
+            results << "stderr #{i}"
+          end
+          @stderr.should == results.join("\n") + "\n"
+          results = []
+          @stdoutput.should == ''
+          EM.stop
+        end
+      end
+    end
+  end
+
+  it 'should preserve the integrity of stdout and stderr despite interleaving' do
+    # manually bump count up for more aggressive multi-processor testing, lessen
+    # for a quick smoke test
+    count = LARGE_OUTPUT_COUNTER
+    EM.next_tick do
+      cmd = "\"#{RUBY_CMD}\" \"#{File.expand_path(File.join(File.dirname(__FILE__), 'produce_mixed_output.rb'))}\" #{count}"
+      RightScale.popen3(cmd, self, :on_read_stdout, :on_read_stderr, :on_exit)
+    end
+    EM.run do
+      timer = EM::PeriodicTimer.new(0.1) do
+        if @done
+          timer.cancel
+          results = []
+          count.times do |i|
+            results << "stdout #{i}"
+          end
+          @stdoutput.should == results.join("\n") + "\n"
+          results = []
+          count.times do |i|
+            (results << "stderr #{i}") if 0 == i % 10
+          end
+          @stderr.should == results.join("\n") + "\n"
+          results = []
+          @done.exitstatus.should == 99
+          EM.stop
+        end
+      end
+    end
+  end
+
+end