costan-zerg_support 0.0.9

data/CHANGELOG

@@ -0,0 +1,17 @@
+v0.0.9. Changed ProcTable wrapper to deal with API change in 0.8.0.
+
+v0.0.8. Extension emulation works in Windows without a build environment.
+
+v0.0.7. Windows support.
+
+v0.0.6. Event Machine protocols for talking to zergling daemons.
+
+v0.0.5. Added forgotten spawn.rb to Manifest.
+
+v0.0.4. Polished implementation and tests for process spawning.
+
+v0.0.3. Preliminary implementation of process spawning.
+
+v0.0.2. Process management.
+
+v0.0.1. Initial release. Support code for zerg* gem installation.

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2008 Victor Costan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Manifest

@@ -0,0 +1,22 @@
+CHANGELOG
+lib/zerg_support/event_machine/connection_mocks.rb
+lib/zerg_support/event_machine/frame_protocol.rb
+lib/zerg_support/event_machine/object_protocol.rb
+lib/zerg_support/gems.rb
+lib/zerg_support/open_ssh.rb
+lib/zerg_support/process.rb
+lib/zerg_support/spawn.rb
+lib/zerg_support.rb
+LICENSE
+Manifest
+Rakefile
+README
+RUBYFORGE
+test/fork_tree.rb
+test/test_connection_mocks.rb
+test/test_frame_protocol.rb
+test/test_gems.rb
+test/test_object_protocol.rb
+test/test_open_ssh.rb
+test/test_process.rb
+test/test_spawn.rb

data/RUBYFORGE

@@ -0,0 +1,40 @@
+Quickstart for Rubyforge:
+
+1) Get the code
+git clone git@github.com:costan/zerg_support.git
+
+2) Install the rubyforge gem
+gem install rubyforge
+
+3) Save your rubyforge.org login information
+rubyforge setup
+
+4) Get a login cookie
+rubyforge login
+
+5) Get project configuration from rubyforge
+rubyforge config zerglings
+
+6) Create a package to release under
+rubyforge create_package zerglings zerg_support
+
+7) Install the echoe gem (required for building this gem)
+gem install echoe
+
+8) Release the gem (finally!)
+rake release
+
+Releasing a new gemspec to Github
+
+1) Build the gem
+rake package
+
+2) Copy the spec
+cp pkg/zerg_support-*/zerg_support.gemspec .
+
+3) Commit the spec
+git add zerg_support.gemspec
+git commit -m "New gemspec, for Github distribution."
+
+4) Push to Github
+git push

data/Rakefile

@@ -0,0 +1,26 @@
+require 'rubygems'
+gem 'echoe'
+require 'echoe'
+
+Echoe.new('zerg_support') do |p|
+  p.project = 'zerglings' # rubyforge project
+  
+  p.author = 'Victor Costan'
+  p.email = 'victor@zergling.net'
+  p.summary = 'Support libraries used by Zergling.Net deployment code.'
+  p.url = 'http://www.zergling.net'
+  
+  p.need_tar_gz = !Platform.windows?
+  p.need_zip = !Platform.windows?
+  p.rdoc_pattern = /^(lib|bin|tasks|ext)|^BUILD|^README|^CHANGELOG|^TODO|^LICENSE|^COPYING$/
+  
+  p.development_dependencies = ["echoe >=3.0.2",
+                                "event_machine >=0.12.2",
+                                "flexmock >=0.8.3",
+                               ]
+end
+
+if $0 == __FILE__
+  Rake.application = Rake::Application.new
+  Rake.application.run
+end

data/lib/zerg_support/event_machine/connection_mocks.rb

@@ -0,0 +1,46 @@
+# Mocks the sending end of an EventMachine connection.
+# The sent data is concatenated in a string available by calling #string.
+class Zerg::Support::EventMachine::SendMock  
+  attr_reader :string
+  
+  def initialize
+    @string = ''
+  end
+  
+  def send_data(data)
+    @string << data
+  end
+end
+
+# Mocks the receiving end of an EventMachine connection.
+# The data to be received is passed as an array of strings to the constructor.
+# Calling #replay mocks receiving the data.
+class Zerg::Support::EventMachine::ReceiveMock
+  attr_accessor :strings
+  attr_accessor :objects
+  
+  def initialize(strings = [''])
+    @strings = strings
+    @objects = []
+  end
+  
+  # Simulates receiving all the given strings as data from Event Machine.
+  def replay
+    @strings.each { |str| receive_data str }
+    self
+  end
+
+  #:nodoc:
+  def receive__object(object)
+    @objects << object
+  end
+  
+  # Declares the name of the object to be received. For instance, a frame
+  # protocol would use :frame for name. This generates a receive_frame method,
+  # and a frames accessor.
+  def self.object_name(name)
+    alias_method "receive_#{name}".to_sym, :receive__object
+    return if name == :object
+    alias_method "#{name}s".to_sym, :objects
+  end
+end

data/lib/zerg_support/event_machine/frame_protocol.rb

@@ -0,0 +1,68 @@
+#:nodoc: namespace
+module Zerg::Support::EventMachine
+  
+# Event Machine protocol for sending and receiving discrete-sized frames
+module FrameProtocol  
+  #:nodoc: This is called by Event Machine when TCP stream data is available.
+  def receive_data(data)
+    @frame_protocol_varsize ||= ''
+
+    i = 0
+    loop do
+      while @frame_protocol_buffer.nil? and i < data.size
+        @frame_protocol_varsize << data[i]
+        if (data[i] & 0x80) == 0
+          @frame_protocol_bytes_left =
+              FrameProtocol.decode_natural @frame_protocol_varsize
+          @frame_protocol_buffer = ''
+        end
+        i += 1
+      end
+
+      return if @frame_protocol_buffer.nil?
+      break if @frame_protocol_bytes_left > data.size - i
+
+      receive_frame @frame_protocol_buffer + data[i, @frame_protocol_bytes_left]
+      @frame_protocol_varsize, @frame_protocol_buffer = '', nil
+      i += @frame_protocol_bytes_left
+    end
+
+    @frame_protocol_buffer << data[i..-1]
+    @frame_protocol_bytes_left -= data.size-i
+  end
+  
+  # Override to process incoming frames.
+  def receive_frame(frame_data); end
+
+  # Sends a frame via the underlying Event Machine TCP stream.
+  def send_frame(frame_data)
+    encoded_length = FrameProtocol.encode_natural(frame_data.length)
+    send_data encoded_length + frame_data
+  end
+  
+  #:nodoc: Encodes a natural (non-negative) integer into a string.
+  def self.encode_natural(number)
+    string = ''
+    loop do
+      number, byte = number.divmod(0x80)
+      string << (byte | ((number > 0) ? 0x80 : 0x00))
+      break if number == 0
+    end
+    string
+  end
+  
+  #:nodoc: Decodes a natural (non-negative) integer from a string.
+  def self.decode_natural(string)
+    number = 0
+    multiplier = 1
+    string.each_byte do |byte|
+      more, number_bits = byte.divmod 0x80
+      number += number_bits * multiplier
+      break if more == 0
+      multiplier *= 0x80
+    end
+    return number
+  end  
+end
+
+end # namespace Zerg::Support::EventMachine

data/lib/zerg_support/event_machine/object_protocol.rb

@@ -0,0 +1,24 @@
+require 'yaml'
+
+#:nodoc: namespace
+module Zerg::Support::EventMachine
+
+# Event Machine protocol for sending serializable objects.
+module ObjectProtocol
+  include FrameProtocol
+
+  # Send a serialized object.
+  def send_object(object)
+    send_frame YAML.dump(object)
+  end
+  
+  #:nodoc: Processes an incoming frame and de-serializes the object in it.
+  def receive_frame(frame_data)
+    receive_object YAML.load(frame_data)
+  end
+  
+  # Override to process incoming objects.
+  def receive_object(object); end
+end
+
+end # namespace Zerg::Support::EventMachine

data/lib/zerg_support/gems.rb

@@ -0,0 +1,60 @@
+# methods used in gem installation hooks
+module Zerg::Support::Gems
+  # called by ensure_on_path for Windows systems
+  def self.ensure_on_windows_path(bin_file)
+    bat_file = File.expand_path(File.join(ENV["WINDIR"],
+                                File.basename(bin_file) + ".bat"))
+    begin
+      File.open(bat_file, 'w') do |f|
+        f.write <<END_BATCH
+@ECHO OFF
+IF NOT "%~f0" == "~f0" GOTO :WinNT
+@"ruby.exe" "#{File.expand_path(bin_file)}" %1 %2 %3 %4 %5 %6 %7 %8 %9
+GOTO :EOF
+:WinNT
+@"ruby.exe" "#{File.expand_path(bin_file)}" %*
+END_BATCH
+      end
+    #rescue
+      # if anything goes wrong we probably don't have permissions (hi Vista?)
+    end    
+  end
+  
+  # called by ensure_on_path for UNIX systems
+  def self.ensure_on_unix_path(bin_file)
+    path = "/usr/bin/#{File.basename bin_file}"
+    begin
+      # using a link so the gem can be updated and the link still works
+      FileUtils.ln_s(bin_file, path, :force)
+    rescue
+      # if anything goes wrong we probably don't have permissions
+      # oh well at least we tried
+    end
+  end
+  
+  # ensures that bin_file can be invoked from a shell
+  def self.ensure_on_path(bin_script)
+    caller_trace = Kernel.caller.first
+    caller_match = /^(.*)\:\d+\:in /.match(caller_trace) ||
+                   /^(.*)\:\d+$/.match(caller_trace)
+    bin_file = File.expand_path caller_match[1] + '/../../../bin/' + bin_script
+    # this is a cheat to get the binary in the right place on stubborn Debians
+    if RUBY_PLATFORM =~ /win/ and RUBY_PLATFORM !~ /darwin/
+      ensure_on_windows_path bin_file
+    else
+      ensure_on_unix_path bin_file
+    end
+  end
+  
+  # tricks rubygems into believeing that the extension compiled and worked out
+  def self.emulate_extension_install(extension_name)
+    File.open('Makefile', 'w') { |f| f.write "all:\n\ninstall:\n\n" }
+    File.open('make', 'w') do |f|
+      f.write '#!/bin/sh'
+      f.chmod f.stat.mode | 0111
+    end
+    File.open(extension_name + '.so', 'w') {}
+    File.open(extension_name + '.dll', 'w') {}
+    File.open('nmake.bat', 'w') { |f| }
+  end
+end

data/lib/zerg_support/open_ssh.rb

@@ -0,0 +1,104 @@
+require 'base64'
+require 'openssl'
+
+# Tools for managing openssh cryptographic material
+module Zerg::Support::OpenSSH
+  # Extracts the keys from a file of known_hosts format.  
+  def self.known_hosts_keys(io)
+    io.each_line do |line|
+      
+    end
+  end
+  
+  # The components in a openssh .pub / known_host RSA public key.
+  RSA_COMPONENTS = ['ssh-rsa', :e, :n]
+  # The components in a openssh .pub / known_host DSA public key.
+  DSA_COMPONENTS = ['ssh-dss', :p, :q, :g, :pub_key]
+
+  # Encodes a key's public part in the format found in .pub & known_hosts files.
+  def self.encode_pubkey(key)
+    case key
+    when OpenSSL::PKey::RSA
+      components = RSA_COMPONENTS
+    when OpenSSL::PKey::DSA
+      components = DSA_COMPONENTS
+    else
+      raise "Unsupported key type #{key.class.name}"
+    end
+    components.map! { |c| c.kind_of?(Symbol) ? encode_mpi(key.send(c)) : c }
+    # ruby tries to be helpful and adds new lines every 60 bytes :(
+    [pack_pubkey_components(components)].pack('m').gsub("\n", '')
+  end
+  
+  # Decodes an openssh public key from the format of .pub & known_hosts files.
+  def self.decode_pubkey(string)
+    components = unpack_pubkey_components Base64.decode64(string)
+    case components.first
+    when RSA_COMPONENTS.first
+      ops = RSA_COMPONENTS.zip components
+      key = OpenSSL::PKey::RSA.new
+    when DSA_COMPONENTS.first
+      ops = DSA_COMPONENTS.zip components
+      key = OpenSSL::PKey::DSA.new
+    else
+      raise "Unsupported key type #{components.first}"
+    end
+    ops.each do |o|
+      next unless o.first.kind_of? Symbol
+      key.send "#{o.first}=", decode_mpi(o.last)
+    end
+    return key
+  end
+
+  # Loads a serialized key from an IO instance (File, StringIO).
+  def self.load_key(io)
+    key_from_string io.read
+  end
+
+  # Reads a serialized key from a string.
+  def self.key_from_string(serialized_key)
+    header = first_line serialized_key
+    if header.index 'RSA'
+      OpenSSL::PKey::RSA.new serialized_key
+    elsif header.index 'DSA'
+      OpenSSL::PKey::DSA.new serialized_key
+    else
+      raise 'Unknown key type'
+    end
+  end
+
+  # Extracts the first line of a string.
+  def self.first_line(string)
+    string[0, string.index(/\r|\n/) || string.len]
+  end  
+  
+  # Unpacks the string components in an openssh-encoded pubkey.
+  def self.unpack_pubkey_components(str)
+    cs = []
+    i = 0
+    while i < str.length
+      len = str[i, 4].unpack('N').first
+      cs << str[i + 4, len]
+      i += 4 + len
+    end
+    return cs
+  end
+  
+  # Packs string components into an openssh-encoded pubkey.
+  def self.pack_pubkey_components(strings)
+    (strings.map { |s| [s.length].pack('N') }).zip(strings).flatten.join
+  end
+  
+  # Decodes an openssh-mpi-encoded integer.
+  def self.decode_mpi(mpi_str)
+    mpi_str.unpack('C*').inject(0) { |acc, c| (acc << 8) | c }
+  end  
+  
+  # Encodes an openssh-mpi-encoded integer.
+  def self.encode_mpi(n)
+    chars, n = [], n.to_i
+    chars << (n & 0xff) and n >>= 8 while n != 0
+    chars << 0 if chars.empty? or chars.last >= 0x80
+    chars.reverse.pack('C*')
+  end  
+end

data/lib/zerg_support/process.rb

@@ -0,0 +1,225 @@
+require 'time'
+
+# process management
+module Zerg::Support::Process
+  @@no_multiple_pids = false
+  
+  unless RUBY_PLATFORM =~ /win/ and RUBY_PLATFORM !~ /darwin/
+    # Translates process info given by Sys::ProcTable into our own nicer format.
+    def self.xlate_process_info(low_info)
+      {
+        :pid => low_info.pid,
+        :parent_pid => low_info.ppid,
+        :real_uid => low_info.ruid || -1,
+        :real_gid => low_info.rgid || -1,
+        :start_time => low_info.start,
+        :nice => low_info.nice || 0,
+        :priority => low_info.priority || 0,
+        :syscall_priority => low_info.user_priority || 0,
+        :resident_size => (low_info.id_rss || 0) * 1024,
+        :code_size => (low_info.ix_rss || 0) * 1024,
+        :virtual_size => (low_info.is_rss || 0) * 1024,
+        :percent_cpu => low_info.pctcpu,
+        :percent_ram => low_info.pctmem,
+        :state => low_info.state,
+        :command_line => low_info.cmdline
+      }
+    end
+    
+  else
+    #:nodoc:
+    def self.xlate_process_info(low_info)
+      state = low_info.execution_state.to_s
+      state << 's' if low_info.session_id == 0
+      
+      {
+        :pid => low_info.pid,
+        :parent_pid => low_info.ppid,
+        :real_uid => -1,
+        :real_gid => -1,
+        :start_time => low_info.creation_date,
+        :nice => 0,
+        :priority => low_info.priority || 0,
+        :syscall_priority => 0,
+        :resident_size => low_info.working_set_size || 0,
+        :code_size => 0,
+        :virtual_size => low_info.virtual_size || 0,
+        :percent_cpu => 0,
+        :percent_ram => 0,
+        :state => state,
+        :command_line => low_info.cmdline || low_info.comm || ''
+      }
+    end    
+  end
+  
+  # Collects information about a single process. Returns nil 
+  def self.process_info(pid)
+    pinfo = ps pid
+    pinfo ? xlate_process_info(pinfo) : nil 
+  end
+  
+  # Collects information about processes with the given pids.
+  # Returns all processes if no list of pids is given.
+  def self.processes(pids = nil)
+    if @@no_multiple_pids and !pids.empty
+      pids.map { |pid| process_info pid }
+    else
+      begin
+        if pids
+          ps_result = ps(pids)
+        else
+          ps_result = ps
+        end
+        ps_result.map { |pinfo| xlate_process_info pinfo }
+      rescue TypeError
+        # we're using the real sys-proctable, and its ps doesn't like multiple
+        # arguments
+        @@no_multiple_pids = true
+        processes pids
+      end
+    end
+  end
+  
+  # Collects information about processes with the given pids.
+  # The information is returned indexed by the processes' pids.
+  def self.processes_by_id(pids = nil)
+    retval = {}
+    self.processes(pids).each { |pinfo| retval[pinfo[:pid]] = pinfo } 
+    return retval
+  end
+  
+  # Returns information about the descendants of the process with the given pid.
+  def self.process_tree(*root_pids)
+    procs_by_ppid = {}
+    proc_list = self.processes_by_id
+    proc_list.each do |pid, pinfo|
+      procs_by_ppid[pinfo[:parent_pid]] ||= []
+      procs_by_ppid[pinfo[:parent_pid]] << pinfo
+    end
+    
+    proc_queue = root_pids.map { |pid| proc_list[pid] }.select { |pinfo| pinfo }
+    
+    index = 0
+    while index < proc_queue.length
+      pid, index = proc_queue[index][:pid], index + 1
+      next unless procs_by_ppid.has_key? pid
+      proc_queue += procs_by_ppid[pid]
+    end
+    return proc_queue
+  end
+  
+  
+  if RUBY_PLATFORM =~ /win/ and RUBY_PLATFORM !~ /darwin/
+    #:nodoc: Wrapper around Process.kill that works on all platforms.
+    def self.kill_primitive(pid, force = false)
+      Process.kill(force ? 9 : 4, pid)
+    end
+  else
+    #:nodoc: Wrapper around Process.kill that works on all platforms.
+    def self.kill_primitive(pid, force = false)
+      Process.kill(force ? 'KILL' : 'TERM', pid)      
+    end
+  end
+  
+  # Kills the process with the given pid.
+  def self.kill(pid)
+    begin
+      self.kill_primitive pid, false
+      Thread.new(pid) do |victim_pid|
+        Kernel.sleep 0.2
+        self.kill_primitive pid, true
+      end
+    rescue
+      # we probably don't have the right to kill the process
+      print "#{$!.class.name}: #{$!}\n"
+    end    
+  end
+  
+  # Kills all the processes descending from the process with the given pid.
+  def self.kill_tree(root_pid)
+    self.process_tree(root_pid).each do |pinfo|
+      self.kill pinfo[:pid]
+    end
+  end
+end
+
+
+## Backend for process information listing
+
+begin
+  raise 'Use ps' unless RUBY_PLATFORM =~ /win/ and RUBY_PLATFORM !~ /darwin/
+  
+  require 'time'
+  require 'sys/proctable'
+  
+  module Zerg::Support::Process
+    def self.ps(pid = nil)
+      Sys::ProcTable.ps pid
+    end
+  end 
+rescue Exception
+  # Emulate the sys-proctable gem using the ps command.
+  # This will be slower than having native syscalls, but at least it doesn't
+  # crash ruby. (yes, sys-proctable 0.7.6, I mean you!)
+    
+  #:nodoc: all
+  module Zerg::Support::ProcTable
+    class ProcInfo
+      def initialize(pid, ppid, ruid, rgid, uid, gid, start, nice, rss, rssize,
+                     text_size, vsz, user_time, total_time, pctcpu, pctmem,
+                     priority, user_priority, state, cmdline)
+        @pid, @ppid, @ruid, @rgid, @uid, @gid, @nice, @priority,
+             @user_priority = *([pid, ppid, ruid, rgid, uid, gid, nice,
+                                 priority, user_priority].map { |s| s.to_i })
+        @start = Time.parse start
+        @ix_rss, @id_rss, @is_rss = text_size.to_f, rssize.to_f, vsz.to_f
+        @pctcpu, @pctmem = *([pctcpu, pctmem].map { |s| s.to_f })
+        @cmdline = cmdline
+        
+        # TODO(victor): translate UNIX strings into something else
+        @state = state
+      end
+      attr_reader :pid, :ppid, :ruid, :rgid, :uid, :gid, :nice, :priority,
+                  :user_priority, :start, :ix_rss, :id_rss, :is_rss, :pctcpu,
+                  :pctmem, :state, :cmdline
+    end
+
+    @@ps_root_cmdline = 'ps -o pid,ppid,ruid,rgid,uid,gid' +
+                        ',lstart="STARTED_________________________________"' +
+                        ',nice,rss,rssize,tsiz,vsz,utime,cputime,pcpu="PCPU_"' +
+                        ',pmem="PMEM_",pri,usrpri,stat,command'
+                        
+    @@ps_all_cmdline = @@ps_root_cmdline + ' -A'
+    @@ps_some_cmdline = @@ps_root_cmdline + ' -p '
+
+    def self.ps(pid = nil)
+      pids = pid ? (pid.kind_of?(Enumerable) ? pid : [pid]) : []
+      retval = []
+      ps_cmdline = pids.empty? ? @@ps_all_cmdline :
+                                 @@ps_some_cmdline + pids.join(',')
+      ps_output = Kernel.` ps_cmdline
+      header_splits = nil
+      ps_output.each_line do |pline|
+        if header_splits
+          # result line, break it up
+          retval << ProcInfo.new(*(header_splits.map { |hs| pline[hs].strip }))
+        else
+          # first line, compute headers
+          lengths = pline.split(/\S\s/).map { |h| h.length + 2 }
+          header_splits, sum = [], 0
+          lengths.each_index do |i|
+            header_splits[i] = Range.new sum, sum += lengths[i], true
+          end
+          header_splits[-1] = Range.new header_splits[-1].begin, -1, false          
+        end
+      end
+      return pids.length == 1 ? retval.first : retval
+    end
+  end
+  
+  module Zerg::Support::Process
+    def self.ps(pid = nil)
+      Zerg::Support::ProcTable.ps pid
+    end
+  end   
+end