subprocess 0.1.6

data/features/step_definitions/popen_over_ssh_steps.rb

@@ -0,0 +1,29 @@
+
+Given /^I have a new remote Subprocess instance initialized with "([^\"]*)"$/ do |command|
+  @popen_remote = Subprocess::PopenRemote.new(command, 'localhost', 'popen', 10, :password => 'popen')
+end
+
+When /^I invoke the run method of said remote subprocess$/ do
+  @popen_remote.run
+end
+
+When /^I invoke the wait method of said remote subprocess$/ do
+  @popen_remote.wait
+end
+
+Then /^the remote instances exit status is "([^\"]*)"$/ do |exitstatus|
+  @popen_remote.status[:exitstatus].should == exitstatus.to_i
+end
+
+Then /^the remote instances stdout matches "([^\"]*)"$/ do |stdout|
+  @popen_remote.stdout.should match(stdout)
+end
+
+Then /^the remote instances stderr matches "([^\"]*)"$/ do |stderr|
+  @popen_remote.stderr.should match(stderr)
+end
+
+Then /^the remote instance should have a numerical pid$/ do
+  @popen_remote.pid.should be_a_kind_of Fixnum
+end
+

data/features/step_definitions/popen_over_ssh_without_blocking_steps.rb

@@ -0,0 +1,30 @@
+
+Given /^I have a new remote nonblocking subprocess that takes a long time to run$/ do
+  @popen = Subprocess::PopenRemote.new('sleep 3 && exit 1', 'localhost', nil, 10, 'popen', :password => 'popen')
+  @popen.should_not be_nil
+end
+
+When /^I invoke the run method of said nonblocking remote subprocess$/ do
+  start_time = Time.now.to_i
+  @popen.run
+  @total_time = Time.now.to_i - start_time
+end
+
+Then /^the remote nonblocking subprocess should not block$/ do
+  @total_time.should be_close(0, 2) 
+end
+
+Then /^the remote nonblocking subprocess should report its run status$/ do
+  @popen.should respond_to(:running?)
+end
+
+Then /^the remote nonblocking subprocess should support being waited on till complete$/ do
+  @popen.wait
+end
+
+Then /^the remote nonblocking subprocess should have status info$/ do
+  @popen.status[:exitstatus].should be_kind_of Numeric
+  @popen.status.should be_a_kind_of Hash
+end
+
+

data/features/step_definitions/popen_remote_fails_with_invalid_auth_dat_steps.rb

@@ -0,0 +1,19 @@
+Given /^I have a new remote subproces with invalid username$/ do
+  @popen_remote = Subprocess::PopenRemote.new('tail -n50 /var/log/daemon.log',
+                    'localhost', 'someadminnamethatshouldneverexist', 300,
+                    :password => 'somebadpasswordnooneuses')
+end
+
+Given /^invalid password$/ do
+  # by leaving this empty its an auto pass and really just serves to make feature read better
+end
+
+When /^I run the remote subprocess$/ do
+  @popen_remote.run
+end
+
+Then /^the remote subprocess should return an error$/ do
+  @popen_remote.wait
+  #@popen_remote.stderr.should_not be_nil
+end
+

data/features/step_definitions/popen_reports_runtime_steps.rb

@@ -0,0 +1,13 @@
+Given /^I have a new subprocess that takes 3 seconds$/ do
+  @popen = Subprocess::Popen.new('sleep 3')
+end
+
+When /^I wait on said 3 second process to complete$/ do
+  @popen.run
+  @popen.wait
+end
+
+Then /^the subprocess should report a run time of around 3 seconds$/ do
+  @popen.run_time.should be_close(3, 0.2)
+end
+

data/features/step_definitions/popen_running_steps.rb

@@ -0,0 +1,12 @@
+Given /^I have a new subprocess that runs fast$/ do
+  @popen = Subprocess::Popen.new('echo 1')
+end
+
+When /^I invoke the run method of said fast subprocess$/ do
+  @popen.run
+end
+
+Then /^the subprocess should report running as false without waiting$/ do
+  @popen.running?.should be_false
+end
+

data/features/step_definitions/popen_steps.rb

@@ -0,0 +1,34 @@
+Given /^I have a new Subprocess instance initialized with "([^\"]*)"$/ do |command|
+  @popen = Subprocess::Popen.new(command)
+end
+
+When /^I invoke the run method of said subprocess$/ do
+  @popen.run
+end
+
+When /^I invoke the wait method of said subprocess$/ do
+  @popen.wait
+end
+
+Then /^the instance should have a status attribute$/ do
+  @popen.status.should be_a_kind_of Hash
+end
+
+Then /^the instances exit status is "([^\"]*)"$/ do |exitstatus|
+  @popen.status[:exitstatus].should == exitstatus.to_i
+end
+
+Then /^the instances stdout matches "([^\"]*)"$/ do |stdout|
+  @popen.stdout.should match(stdout)
+end
+
+Then /^the instances stderr matches "([^\"]*)"$/ do |stderr|
+  @popen.stderr.should match(stderr)
+end
+
+Then /^the instance should have a numerical pid$/ do
+  @popen.pid.should be_a_kind_of Fixnum
+end
+
+
+

data/features/step_definitions/popen_with_timeout_steps.rb

@@ -0,0 +1,24 @@
+Given /^I have a new subprocess that takes more than 5 seconds to run$/ do
+  @popen = Subprocess::Popen.new('sleep 10', 5)
+end
+
+Given /^I have a new subprocess that takes less than 5 seconds to run$/ do
+  @popen = Subprocess::Popen.new('sleep 1', 5)
+end
+
+When /^I invoke the run method of said subprocess with timeout$/ do
+  @popen.run
+  @popen.wait
+end
+
+Given /^I set a timeout of 5 seconds$/ do
+end
+
+Then /^the subprocess should exit with exitcode 1$/ do
+  @popen.status[:exitstatus].should == 1
+end
+
+Then /^the subprocess should complete fine$/ do
+  @popen.running?.should be_false
+end
+

data/features/step_definitions/popen_without_blocking_steps.rb

@@ -0,0 +1,33 @@
+
+Given /^I have a new subprocess that takes a long time to run$/ do 
+  @popen = Subprocess::Popen.new('sleep 3 && exit 1')
+  @popen.should_not be_nil
+end
+
+When /^I invoke the run method of said nonblocking subprocess$/ do
+  start_time = Time.now.to_i
+  @popen.run
+  @total_time = Time.now.to_i - start_time
+end
+
+Then /^the subprocess should not block$/ do
+  # this should pass if we backgrounded cause it shouldn't take more than 
+  # 1 second to get here but the command should take 3 seconds to run
+  @total_time.should be_close(0, 2) 
+end
+
+Then /^the subprocess should report its run status$/ do
+  @popen.should respond_to(:running?)
+end
+
+Then /^the subprocess should support being waited on till complete$/ do
+  @popen.wait
+  @popen.status[:exitstatus].should be_kind_of Numeric
+end
+
+Then /^the subprocess should have status info$/ do
+  @popen.status.should be_a_kind_of Hash
+end
+
+
+

data/features/support/common.rb

@@ -0,0 +1,29 @@
+module CommonHelpers
+  def in_tmp_folder(&block)
+    FileUtils.chdir(@tmp_root, &block)
+  end
+
+  def in_project_folder(&block)
+    project_folder = @active_project_folder || @tmp_root
+    FileUtils.chdir(project_folder, &block)
+  end
+
+  def in_home_folder(&block)
+    FileUtils.chdir(@home_path, &block)
+  end
+
+  def force_local_lib_override(project_name = @project_name)
+    rakefile = File.read(File.join(project_name, 'Rakefile'))
+    File.open(File.join(project_name, 'Rakefile'), "w+") do |f|
+      f << "$:.unshift('#{@lib_path}')\n"
+      f << rakefile
+    end
+  end
+
+  def setup_active_project_folder project_name
+    @active_project_folder = File.join(@tmp_root, project_name)
+    @project_name = project_name
+  end
+end
+
+World(CommonHelpers)

data/features/support/env.rb

@@ -0,0 +1,15 @@
+require File.dirname(__FILE__) + "/../../lib/subprocess"
+
+gem 'cucumber'
+require 'cucumber'
+gem 'rspec', '<= 1.3.0'
+require 'spec'
+
+Before do
+  @tmp_root = File.dirname(__FILE__) + "/../../tmp"
+  @home_path = File.expand_path(File.join(@tmp_root, "home"))
+  @lib_path  = File.expand_path(File.dirname(__FILE__) + "/../../lib")
+  FileUtils.rm_rf   @tmp_root
+  FileUtils.mkdir_p @home_path
+  ENV['HOME'] = @home_path
+end

data/features/support/matchers.rb

@@ -0,0 +1,11 @@
+module Matchers
+  def contain(expected)
+    simple_matcher("contain #{expected.inspect}") do |given, matcher|
+      matcher.failure_message = "expected #{given.inspect} to contain #{expected.inspect}"
+      matcher.negative_failure_message = "expected #{given.inspect} not to contain #{expected.inspect}"
+      given.index expected
+    end
+  end
+end
+
+World(Matchers)

data/lib/core_ext/hash.rb

@@ -0,0 +1,14 @@
+
+class Hash
+  # str8 from activesupport core_ext/hash/keys.rb
+  def symbolize_keys
+    inject({}) do |options, (key, value)|
+      options[(key.to_sym rescue key) || key] = value
+      options
+    end
+  end
+  def symbolize_keys!
+    self.replace(self.symbolize_keys)
+  end
+end
+

data/lib/core_ext/process_status.rb

@@ -0,0 +1,14 @@
+
+module Process
+  class Status
+    def to_hash
+      { :exited? => exited?, :exitstatus => exitstatus, :pid => pid, 
+        :stopped? => stopped?, :stopsig => stopsig, :success? => success?,
+        :termsig => termsig, :timed_out? => false }
+    end
+    def to_json
+      to_hash.to_json
+    end
+  end
+end
+

data/lib/subprocess/popen.rb

@@ -0,0 +1,188 @@
+module Subprocess
+  class Popen
+    include Timeout
+
+    attr_accessor :command, :stdout, :stderr, :status, :timeout
+
+    def initialize(command, timeout=300)
+      self.command = command
+      self.timeout = timeout
+      @running = false
+      @ipc_parsed = false
+    end
+
+    def running?
+      # return false if no parent pid or if running is already false
+      return false unless @parent_pid
+      return @running unless @running
+
+      begin
+        # see if the process is running or not
+        Process.kill(0, @parent_pid)
+        # since we didn't error then we have a pid running
+        # so lets see if is over after less than .5 seconds
+        begin
+          Timeout::timeout(0.5) do
+            @parent_pid, parent_status = Process.wait2(@parent_pid, 0)
+          end
+        rescue Timeout::Error
+          # wait timed out so so pid is stll running
+          @running = true
+        end
+        # no timeout so pid is finished
+        @running = false
+      rescue Errno::ESRCH
+        # Process.kill says the pid is not found
+        @running = false
+      end
+
+      # parse the child status if pid is complete
+      parse_ipc_pipe unless (@running or @ipc_parsed)
+      @running
+    end
+
+    def run_time
+      defined?(:@status) ? @status[:run_time] : false
+    end
+
+    def pid
+      defined?(:@status) ? @status[:pid] : false
+    end
+
+    def perform
+      # delayed job anyone?
+      run unless running?
+      wait
+    end
+
+    def run
+      setup_pipes
+      # record the time, set running and fork off the command
+      @start_time = Time.now.to_f
+      @running = true
+      @parent_pid = fork_parent
+
+      # close up the pipes
+      [ @stdin_rd, @stdout_wr, @stderr_wr, @ipc_wr ].each do |p|
+        p.close
+      end
+
+      # make sure sure the stdin_wr handle is sync
+      @stdin_wr.sync = true
+    end
+
+    def wait
+      # block until the process completes
+      @parent_pid, parent_status = Process.wait2(@parent_pid)
+      @running = false
+      parse_ipc_pipe unless @ipc_parsed
+    end
+
+    def active_record?
+      Module.constants.include?("ActiveRecord")
+    end
+
+    private
+    def setup_pipes
+      # setup stream pipes and a pipe for interprocess communication
+      @stdin_rd, @stdin_wr = IO::pipe
+      @stdout_rd, @stdout_wr = IO::pipe
+      @stderr_rd, @stderr_wr = IO::pipe
+      @ipc_rd, @ipc_wr = IO::pipe
+    end
+
+    def ar_remove_connection
+      ActiveRecord::Base.remove_connection
+    end
+    
+    def ar_establish_connection(dbconfig)
+      ActiveRecord::Base.establish_connection(dbconfig)
+    end
+
+    def fork_parent
+      dbconfig = ar_remove_connection if active_record?
+      pid = Kernel.fork {
+        ar_establish_connection(dbconfig) if active_record?
+        begin
+          redirect_stdstreams
+          report_child_status_to_parent(fork_child)
+          teardown_pipes
+        ensure
+          ar_remove_connection if active_record?
+        end
+      }
+      ar_establish_connection(dbconfig) if active_record?
+      pid
+    end
+
+    def redirect_stdstreams
+      # close the pipes we don't need
+      [ @stdin_wr, @stdout_rd, @stderr_rd ].each do |p|
+        p.close
+      end
+
+      # redirect this forks std streams into pipes
+      STDIN.reopen(@stdin_rd)
+      STDOUT.reopen(@stdout_wr)
+      STDERR.reopen(@stderr_wr)
+    end
+
+    def fork_child
+      child_status = Hash.new
+      child_pid = nil
+      start_time = Time.now.to_f
+      begin
+        Timeout::timeout(self.timeout) do
+          dbconfig = ar_remove_connection if active_record?
+          begin
+            child_pid = Kernel.fork{ 
+              fork_child_exec 
+            }
+          ensure
+            ar_establish_connection(dbconfig) if active_record?
+          end
+          child_status = Process.wait2(child_pid)[1]
+        end
+      rescue Timeout::Error
+        begin
+          Process.kill('KILL', child_pid)
+        rescue Errno::ESRCH
+        end
+        child_status[:exitstatus] = 1
+        child_status[:timed_out?] = true
+      end
+      child_status = child_status.to_hash
+      child_status[:run_time] = Time.now.to_f - start_time
+      child_status
+    end
+
+    def report_child_status_to_parent(child_status)
+      @ipc_wr.write child_status.to_json
+      @ipc_wr.flush
+    end
+
+    def teardown_pipes
+      # close pipes
+      [ @stdin_rd, @stdout_wr, @stderr_wr, @ipc_wr ].each do |p|
+        p.close
+      end
+    end
+
+    def fork_child_exec
+      Kernel.exec(@command)
+    end
+
+    def parse_ipc_pipe
+      begin
+        @status = JSON.parse(@ipc_rd.read).symbolize_keys
+      rescue StandardError => err
+        @status = { :exitstatus => 1, :timed_out? => false, :error => err }
+      end
+      @stdout, @stderr = @stdout_rd.read.chomp, @stderr_rd.read.chomp
+      @stdout_rd.close; @stderr_rd.close; @ipc_rd.close
+      @ipc_parsed = true
+    end
+
+  end # class Popen
+end # module Subprocess
+

data/lib/subprocess/popen_factory.rb

@@ -0,0 +1,63 @@
+require 'erb'
+
+module Subprocess
+  module PopenFactoryMixin
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+    
+    module ClassMethods
+
+      def def_command(name, command, class_meths=true)
+        name = class_meths ? "self.#{name}" : name
+        instance_eval "
+          def #{name}(*args)
+            Subprocess::Popen.new('#{command}', *args)
+          end
+        ", __FILE__, __LINE__
+      end
+      
+      def def_dynamic_command(name, command, class_meths=true)
+        name = class_meths ? "self.#{name}" : name
+        instance_eval "
+          def #{name}(data, *args)
+            command = lambda do |data|
+              \"#{command}\"
+            end
+            Subprocess::Popen.new(command.call(data), *args)
+          end
+        ", __FILE__, __LINE__
+      end
+    end # ClassMethods
+  end # PopenFactoryMixin
+
+  module PopenRemoteFactoryMixin
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+    
+    module ClassMethods
+
+      def def_command(name, command, hostname_var, username_var, ssh_params_var, timeout=300)
+        instance_eval do 
+          define_method name.to_sym do
+            eval("Subprocess::PopenRemote.new('#{command}', #{hostname_var}, #{username_var}, timeout, *#{ssh_params_var})")
+          end
+        end
+      end
+
+      def def_dynamic_command(name, command, hostname_var, username_var, ssh_params_var, timeout=300)
+        instance_eval do 
+          define_method name.to_sym do |data|
+            cmd = ERB.new(command).result(binding)
+            eval("Subprocess::PopenRemote.new(cmd, #{hostname_var}, #{username_var}, timeout, *#{ssh_params_var})")
+          end
+        end
+      end
+    end # ClassMethods
+  end # PopenRemoteFactoryMixin
+
+end # Subprocess
+

data/lib/subprocess/popen_remote.rb

@@ -0,0 +1,64 @@
+
+module Subprocess
+  
+  class PopenRemote < Popen
+    attr_accessor :hostname, :ssh_params
+
+    def initialize(command, hostname, username, timeout=300, *ssh_params)
+      @command = command
+      @timeout = timeout
+      @running = false
+      @ipc_parsed = false
+
+      @hostname = hostname
+      @username = username
+      @ssh_params = ssh_params
+    end
+
+    private
+    def log_to_stderr_and_exit(msg)
+      $stderr.write("Net::SSH error: #{@hostname} #{msg}")
+      exit 1
+    end
+
+    def fork_child_exec
+      exit_status = 0
+      begin
+        ssh = Net::SSH.start(@hostname, @username, *@ssh_params) do |ssh|
+          ssh.open_channel do |channel|
+            channel.exec(@command) do |chan, success|
+              log_to_stderr_and_exit('failed to open exec channel') unless success
+
+              channel.on_data do |chan, data|
+                $stdout.write data
+              end
+
+              channel.on_extended_data do |chan, type, data|
+                $stderr.write data
+              end
+
+              channel.on_request('exit-status') do |chan, data|
+                exit_status = data.read_long.to_i
+              end
+
+            end
+          end
+          ssh.loop
+        end
+      rescue Net::SSH::AuthenticationFailed
+        log_to_stderr_and_exit("authentication failure\n")
+      rescue Errno::ECONNREFUSED
+        log_to_stderr_and_exit("connection refused\n")
+      rescue Errno::ETIMEDOUT
+        log_to_stderr_and_exit("connection timeout\n")
+      rescue Errno::EHOSTUNREACH
+        log_to_stderr_and_exit("unreachable\n")
+      rescue StandardError => error
+        log_to_stderr_and_exit("error: #{error.message}\n")
+      end
+      exit exit_status
+    end
+
+  end # class PopenRemote
+end # module Subprocess
+

data/lib/subprocess/popen_sequence.rb

@@ -0,0 +1,57 @@
+
+module Subprocess
+  class PopenSequenceError < StandardError; end
+  class PopenSequence
+    require 'forwardable'
+    extend Forwardable
+    attr_reader :running, :queue, :complete, :incomplete, :failed
+
+    def_delegators :@last, :stdout, :stderr, :status
+
+    def initialize(queue=[])
+      @queue = queue
+      @incomplete = queue
+      @complete = []
+      @failed = []
+      @running = false
+      @completed = false
+      @last = nil
+    end
+
+    def include?(item)
+      @queue.include?(item)
+    end
+
+    def [](index)
+      @queue[index]
+    end
+
+    def add_popen(popen)
+      raise PopenSequenceError, 
+        "appending a completed sequence is not allowed" if @completed
+      @incomplete << popen
+    end
+    alias :<< :add_popen
+
+    def perform
+      @running = true
+      @failure = false
+      @queue.each do |popen|
+        @last = popen
+        popen.perform
+        popen.status[:exitstatus] == 0 ? @complete << popen : 
+                                        (@failed << popen; break)
+      end
+      @incomplete = @incomplete - @complete
+      @incomplete = @incomplete - @failed
+      @running = false
+      @completed = true
+    end
+
+    def completed?
+      @completed
+    end
+
+  end
+end
+

data/lib/subprocess.rb

@@ -0,0 +1,23 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+begin
+  require 'net/ssh'
+  require 'json'
+rescue LoadError
+  require 'rubygems'
+  require 'net/ssh'
+  require 'json'
+end
+require 'timeout'
+
+module Subprocess
+  VERSION = '0.1.6'
+end
+
+require 'core_ext/hash'
+require 'core_ext/process_status'
+require 'subprocess/popen'
+require 'subprocess/popen_remote'
+require 'subprocess/popen_sequence'
+require 'subprocess/popen_factory'