rubysh 0.0.4 → 0.0.5

data/lib/rubysh/error.rb

@@ -17,5 +17,6 @@ module Rubysh
     class AlreadyClosedError < BaseError; end
     class AlreadyRunError < BaseError; end
     class BadExitError < BaseError; end
+    class ECHILDError < BaseError; end
   end
 end

data/lib/rubysh/redirect.rb

@@ -15,7 +15,7 @@ module Rubysh
 
     attr_accessor :source, :direction, :target
 
-    def initialize(source, direction, target)
+    def initialize(source, direction, target, opts=nil)
       unless VALID_DIRECTIONS.include?(direction)
         raise Rubysh::Error::BaseError.new("Direction must be one of #{VALID_DIRECTIONS.join(', ')}, not #{direction.inspect}")
       end
@@ -31,6 +31,7 @@ module Rubysh
       @source = source
       @target = target
       @direction = direction
+      @opts = opts || {}
     end
 
     def printable_source
@@ -124,7 +125,8 @@ module Rubysh
         :buffer => StringIO.new,
         :target_name => target_name,
         :read_pos => 0,
-        :subprocess_fd_number => Util.to_fileno(source)
+        :subprocess_fd_number => Util.to_fileno(source),
+        :tee => @opts[:tee]
       }
     end
 

data/lib/rubysh/runner.rb

@@ -108,8 +108,8 @@ module Rubysh
     # API for running/waiting
     def run_async
       raise Rubysh::Error::AlreadyRunError.new("You have already run this #{self.class} instance. Cannot run again. You can run its command directly though, which will create a fresh #{self.class} instance.") unless @runner_state == :initialized
-      prepare_io
       @command.start_async(self)
+      prepare_io
       @runner_state = :started
       self
     end
@@ -218,6 +218,14 @@ module Rubysh
       end
     end
 
+    def subprocesses
+      # Not everything with state has a subprocess
+      @state.map do |object, target|
+        next unless object.kind_of?(BaseCommand)
+        target.fetch(:subprocess)
+      end.compact
+    end
+
     def do_wait
       return unless @runner_state == :parallel_io_ran
       @command.wait(self)
@@ -239,7 +247,7 @@ module Rubysh
     # Can't build this in the prepare stage because pipes aren't built
     # there.
     def prepare_io
-      @parallel_io = Subprocess::ParallelIO.new(readers, writers)
+      @parallel_io = Subprocess::PidAwareParallelIO.new(readers, writers, subprocesses)
       @parallel_io.on_read do |target_name, data|
         state = @targets[target_name]
         buffer = state[:buffer]
@@ -248,6 +256,9 @@ module Rubysh
           buffer.close_write
         else
           Rubysh.log.debug("Just read #{data.inspect} on #{target_name.inspect}")
+          tee = state[:tee]
+          tee.write(data) if tee
+
           # Seek to end
           buffer.pos = buffer.length
           buffer.write(data)

data/lib/rubysh/subprocess/parallel_io.rb

@@ -1,6 +1,7 @@
 class Rubysh::Subprocess
   class ParallelIO
     module EOF; end
+    class NothingAvailable < StandardError; end
 
     # readers/writers should be hashes mapping {fd => name}
     def initialize(readers, writers)
@@ -65,16 +66,25 @@ class Rubysh::Subprocess
       end
     end
 
-    def run_once
+    # This method is a stub so it can be extended in subclasses
+    def run_once(timeout=nil)
+      run_select_loop(timeout)
+    end
+
+    def run_select_loop(timeout)
       potential_readers = available_readers
       potential_writers = available_writers
 
       begin
-        ready_readers, ready_writers, _ = IO.select(potential_readers, potential_writers)
+        selected = IO.select(potential_readers, potential_writers, nil, timeout)
       rescue Errno::EINTR
         retry
+      else
+        raise NothingAvailable unless selected
       end
 
+      ready_readers, ready_writers, _ = selected
+
       ready_readers.each do |reader|
         read_available(reader)
       end
@@ -84,6 +94,13 @@ class Rubysh::Subprocess
       end
     end
 
+    def consume_all_available
+      begin
+        loop {run_select_loop(0)}
+      rescue NothingAvailable
+      end
+    end
+
     def read_available(reader)
       begin
         data = reader.read_nonblock(4096)
@@ -106,7 +123,7 @@ class Rubysh::Subprocess
     def issue_reader_callback(reader, data)
       if @on_read
         name = reader_name(reader)
-        @on_read.call(name, data)
+        @on_read.call(name, data) if name
       end
     end
 
@@ -168,7 +185,7 @@ class Rubysh::Subprocess
     def issue_writer_callback(writer, data, remaining)
       if @on_write
         name = writer_name(writer)
-        @on_write.call(name, data, remaining)
+        @on_write.call(name, data, remaining) if name
       end
     end
 

data/lib/rubysh/subprocess/pid_aware_parallel_io.rb

@@ -0,0 +1,103 @@
+require 'set'
+require 'thread'
+
+class Rubysh::Subprocess
+  # We can't actually rely on an EOF once our subprocess has died,
+  # since it may have forked and a child inherited the parent's
+  # fds. (This happens, for example, when using SSH's ControlPersist.)
+  #
+  # E.g. try Rubysh.run('ruby', 'bad.rb', Rubysh.>) with:
+  #   # cat bad.rb
+  #   fork {sleep 1000}
+  class PidAwareParallelIO < ParallelIO
+    @pids_mutex = Mutex.new
+    @parallel_ios = {}
+    @old_sigchld_handler = nil
+
+    def self.register_parallel_io(parallel_io, breaker_writer)
+      @pids_mutex.synchronize do
+        register_sigchld_handler if @parallel_ios.length == 0
+        @parallel_ios[parallel_io] = breaker_writer
+
+        # This is needed in case the SIGCHLD is handled before the
+        # writer is stored.
+        trigger_breaker(breaker_writer)
+      end
+    end
+
+    def self.deregister_parallel_io(parallel_io)
+      @pids_mutex.synchronize do
+        @parallel_ios.delete(parallel_io)
+        deregister_sigchld_handler if @parallel_ios.length == 0
+      end
+    end
+
+    def self.handle_sigchld
+      # It's ok for this operation to race against other
+      # threads. Break loop on all currently active selectors. This
+      # could in theory cause a thundering herd, but it's probably not
+      # worth the work to defend against.
+      @parallel_ios.values.each {|writer| trigger_breaker(writer)}
+    end
+
+    def self.trigger_breaker(writer)
+      begin
+        writer.write_nonblock('a') unless writer.closed?
+      rescue Errno::EAGAIN, Errno::EPIPE
+      end
+    end
+
+    def self.register_sigchld_handler
+      @old_sigchld_handler = Signal.trap('CHLD') {handle_sigchld}
+      # MRI returns nil for a DEFAULT handler, but it also treats nil
+      # as IGNORE.
+      @old_sigchld_handler ||= 'DEFAULT'
+    end
+
+    def self.deregister_sigchld_handler
+      Signal.trap('CHLD', @old_sigchld_handler)
+    end
+
+    # readers/writers should be hashes mapping {fd => name}
+    def initialize(readers, writers, subprocesses)
+      @breaker_reader, @breaker_writer = IO.pipe
+      @subprocesses = subprocesses
+
+      readers = readers.dup
+      readers[@breaker_reader] = nil
+      super(readers, writers)
+
+      register_subprocesses
+    end
+
+    def register_subprocesses
+      self.class.register_parallel_io(self, @breaker_writer)
+    end
+
+    def run_once(timeout=nil)
+      @subprocesses.each do |subprocess|
+        subprocess.wait(true)
+      end
+
+      # All subprocesses have exited! We're done here.
+      if @subprocesses.all?(&:status)
+        finalize_all
+        return
+      end
+
+      super
+    end
+
+    def finalize_all
+      @breaker_writer.close
+
+      # We're guaranteed that if a process exited, all of its bytes
+      # are immediately available to us.
+      consume_all_available
+
+      available_readers.each {|reader| reader.close}
+      available_writers.each {|writer| writer.close}
+      self.class.deregister_parallel_io(self)
+    end
+  end
+end

data/lib/rubysh/subprocess.rb

@@ -4,6 +4,7 @@ require 'timeout'
 require 'thread'
 
 require 'rubysh/subprocess/parallel_io'
+require 'rubysh/subprocess/pid_aware_parallel_io'
 require 'rubysh/subprocess/pipe_wrapper'
 
 module Rubysh
@@ -73,7 +74,13 @@ module Rubysh
 
     def do_wait(nonblock=false)
       flags = nonblock ? Process::WNOHANG : nil
-      return nil unless result = Process.waitpid2(@pid, flags)
+      begin
+        result = Process.waitpid2(@pid, flags)
+      rescue Errno::ECHILD => e
+        raise Rubysh::Error::ECHILDError.new("No unreaped process #{@pid}. This could indicate a bug in Rubysh, but more likely means you have something in your codebase which is wait(2)ing on subprocesses.")
+      end
+
+      return unless result
 
       pid, @status = result
       Rubysh.assert(pid == @pid,

data/lib/rubysh/triple_less_than.rb

@@ -3,9 +3,12 @@ require 'tempfile'
 module Rubysh
   # Looks like bash always buffers <<< to disk
   class TripleLessThan < BaseDirective
+    attr_reader :fd, :literal
+
     class Shell < BaseDirective
-      def initialize(fd)
+      def initialize(fd, opts)
         @fd = fd
+        @opts = opts
       end
 
       def <(literal=:stdin)

data/lib/rubysh/version.rb

@@ -1,3 +1,3 @@
 module Rubysh
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

data/lib/rubysh.rb

@@ -94,8 +94,13 @@ module Rubysh
     command.run(&blk)
   end
 
+  def self.run_async(*args, &blk)
+    command = Rubysh::Command.new(args)
+    command.run_async(&blk)
+  end
+
   def self.check_call(*args, &blk)
-    command = Rubysh::Command.new(*args)
+    command = Rubysh::Command.new(args)
     command.check_call(&blk)
   end
 
@@ -123,22 +128,47 @@ module Rubysh
     FD.new(2)
   end
 
-  def self.>(target=:stdout)
-    Redirect.new(1, '>', target)
+  def self.>(target=nil, opts=nil)
+    # Might want to DRY this logic up at some point. Right now seems
+    # like it'd just sacrifice clarity though.
+    if !opts && target.kind_of?(Hash)
+      opts = target
+      target = nil
+    end
+    target = :stdout
+
+    Redirect.new(1, '>', target, opts)
   end
 
-  def self.>>(target=:stdout)
-    Redirect.new(1, '>>', target)
+  def self.>>(target=nil, opts=nil)
+    if !opts && target.kind_of?(Hash)
+      opts = target
+      target = nil
+    end
+    target = :stdout
+
+    Redirect.new(1, '>>', target, opts)
   end
 
-  def self.<(target=:stdin)
-    Redirect.new(0, '<', target)
+  def self.<(target=nil, opts=nil)
+    if !opts && target.kind_of?(Hash)
+      opts = target
+      target = nil
+    end
+    target = :stdin
+
+    Redirect.new(0, '<', target, opts)
   end
 
   # Hack to implement <<<
-  def self.<<(fd=nil)
+  def self.<<(fd=nil, opts=nil)
+    if !opts && fd.kind_of?(Hash)
+      opts = fd
+      fd = nil
+    end
+
     fd ||= FD.new(0)
-    TripleLessThan::Shell.new(fd)
+    TripleLessThan::Shell.new(fd, opts)
   end
 
   # Internal utility methods

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubysh
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-12 00:00:00.000000000 Z
+date: 2013-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -68,6 +68,7 @@ files:
 - lib/rubysh/runner.rb
 - lib/rubysh/subprocess.rb
 - lib/rubysh/subprocess/parallel_io.rb
+- lib/rubysh/subprocess/pid_aware_parallel_io.rb
 - lib/rubysh/subprocess/pipe_wrapper.rb
 - lib/rubysh/triple_less_than.rb
 - lib/rubysh/util.rb