fairchild-poolparty 1.2.12 → 1.3.5

data/lib/poolparty/net/init.rb

@@ -1,16 +0,0 @@
-class Object
-  #TODO: deprecate. use RemoterBase.available_bases instead
-  def remote_bases
-    PoolParty::Remote.available
-  end
-end
-
-# Load the core net libraries.
-# These are neccessary for any of the remoter_bases to function.
-require ::File.join(::File.dirname(__FILE__),'remoter_base.rb')
-require ::File.join(::File.dirname(__FILE__),'remote_instance.rb')
-
-Dir["#{::File.dirname(__FILE__)}/remoter_bases/*/*.rb"].each do |base|
-  name = ::File.basename(base, ::File.extname(base))
-  require base
-end

data/lib/poolparty/net/remote_instance.rb

@@ -1,93 +0,0 @@
-module PoolParty  
-  module Remote
-    
-    class RemoteInstance
-      include Dslify
-      
-      dsl_methods :name,        # Name of the remote instance (internal usage)
-                  :ip,          # Ip of the remote instance
-                  :internal_ip, # Internal ip of the remote instance
-                  :public_ip,
-                  :status,       # Status of the remote instance
-                  :launch_time,
-                  :keypair_name
-      
-      def initialize(opts={})
-        opts.choose{|k,v| dsl_options.has_key? k}
-        set_vars_from_options(opts) if opts.is_a?(Hash)
-        on_init
-      end
-      
-      ## hash like methods
-      # TODO: move these into a module, or into dslify
-      # include Enumerable
-      def each
-        dsl_options.each{ |k,v| yield k,v }
-      end
-      
-      def [](k)
-        if dsl_options.has_key? k
-          dsl_options[k]
-        else
-          nil
-        end
-      end
-      
-      def []=(k,v)
-        dsl_options[k] = v
-      end
-      
-      def has_key?(key)
-        dsl_options.has_key?(key)
-      end
-      
-      def keys
-        dsl_options.keys
-      end
-         
-      def values
-        dsl_options.values
-      end
-      ##end of hash like methods
-      
-      # Is this instance running?
-      def running?
-        !(status =~ /running/).nil?
-      end
-      # Is this instance pending?
-      def pending?
-        !(status =~ /pending/).nil?
-      end
-      # Is this instance terminating?
-      def terminating?
-        !(status =~ /shutting/).nil?
-      end
-      # Has this instance been terminated?
-      def terminated?
-        !(status =~ /terminated/).nil?
-      end
-      
-      def elapsed_runtime
-        Time.now.to_i - launching_time.to_time.to_i
-      end
-      
-      # Callback
-      def on_init
-      end
-      
-      # The remote instances is only valid if there is an ip and a name
-      def valid?
-        (ip.nil? || name.nil?) ? false : true
-      end
-      
-      # This is how we get the current load of the instance
-      # The approach of this may change entirely, but the usage of
-      # it will always be the same
-      def load
-        current_load ||= 0.0  #NOTE MF: returning 0.0 seems like a bad idea here.  should return nil if we dont have a real value
-      end
-      
-    end
-    
-  end
-end

data/lib/poolparty/net/remoter/cloud_control.rb

@@ -1,24 +0,0 @@
-require "ping"
-
-module PoolParty
-  module Remote
-    
-    # # Are too few instances running?
-    def are_too_few_instances_running?
-      nodes(:status => "running").size < minimum_instances.to_i
-    end
-    # # Are there more instances than allowed?
-    def are_too_many_instances_running?
-      nodes(:status => "running").size > maximum_instances.to_i
-    end
-
-    def list_of_nodes_exceeding_minimum_runtime
-      nodes(:status => "running").reject{|i| i.elapsed_runtime < minimum_runtime}
-    end
-    
-    def are_any_nodes_exceeding_minimum_runtime?
-      !list_of_nodes_exceeding_minimum_runtime.blank?
-    end
-    
-  end
-end

data/lib/poolparty/net/remoter/connections.rb

@@ -1,177 +0,0 @@
-require 'rubygems'
-require 'net/ssh'
-
-module PoolParty
-  module Remote
-    
-    include ::PoolParty::Pinger
-    
-    def target_host(dns_or_ip=nil)
-      dns_or_ip ? @target_host=dns_or_ip : @target_host
-    end
-    
-    # Simply shell out and call ssh, simple, reliable and fewest dependencies, but slow
-    def simplest_run_remote(host=target_host, command=[], extra_ssh_ops={})
-      command = command.compact.join(' && ') if command.is_a? Array
-      cmd = "ssh #{host} #{ssh_options(extra_ssh_ops)} '#{command}'"
-      puts "\n--------------------------------\nrunning_remote:\n #{cmd}\n"
-      puts %x{#{cmd}}
-    end
-    
-    def run_remote( hst, cmds )
-      netssh hst, cmds
-    end
-    
-    def ssh_into(inst, extra_ssh_ops={})
-      ip =  ((inst.respond_to?(:has_key) && inst.has_key?(:ip)) || inst.respond_to?(:ip)) ? inst.ip : inst
-      cmd = extra_ssh_ops.delete(:cmd) if extra_ssh_ops.has_key?(:cmd)
-      Kernel.system("ssh #{ssh_options(extra_ssh_ops)} #{ip}%s" % [cmd ? " #{cmd}" : ""])
-    end
-    
-    def ssh_options(opts={})
-      o = {"-i" => keypair.full_filepath,
-           "-l" => user,
-           "-o" =>"StrictHostKeyChecking=no"
-           }.merge(opts)
-      o.collect{ |k,v| "#{k} #{v}"}.join(' ')
-    end
-    
-    def rsync( source_path, destination_path, rsync_opts=['-v -a'] )
-      dputs "rsync -e 'ssh #{ssh_options}' #{rsync_opts.join(' ')} #{source_path}  root@#{target_host}:#{destination_path}"
-      out = %x{ rsync -e 'ssh #{ssh_options}' #{rsync_opts.join(' ')} #{source_path}  root@#{target_host}:#{destination_path} }
-      puts out if debugging?
-    end
-    
-    # Run commands on the local machine, i.e. your laptop, not the server
-    def run_local(commands)
-      commands.each do |cmd|
-        puts `#{cmd}`
-      end
-    end
-    
-    def commands
-      @commands ||= Array.new
-    end
-
-    # TODO: make extendable multithreaded version
-    def execute!(cmds=commands)
-      netssh(
-        [cmds.compact.join(' && ')],
-        :host=>target_host, :user=>'root')
-      # commands.each {|c| run_remote(c, target_host) }
-    end
-    
-    def netssh(cmds=[], opts={})
-      user = opts.delete(:user) || 'root' #user rescue 'root'
-      host = opts.delete(:host) || target_host
-      ssh_options_hash = {:keys         => [keypair.full_filepath],
-                          :auth_methods => 'publickey',
-                          :paranoid     => false,
-                          :timeout      => 3.minutes,
-                          :user         => user
-                           }.merge(opts)
-      # ssh_options_hash[:verbose]=:debug if debugging?
-      puts "connecting to ssh with options = #{ssh_options_hash.inspect}"
-
-      Net::SSH.start(host, user, ssh_options_hash) do |ssh|
-        cmds.each do |command|
-          $stderr.print command if debugging
-          ssh.exec!(command) do |ch, stream, data|
-            if stream == :stdout
-             print data
-            else
-              $stderr.print "#{host} stderr => #{data}"
-            end
-          end
-        end
-      end
-    end
-    
-    
-##########################################################################################
-# TODO: Delete deprecated commands below here
-    
-    def rsync_storage_files_to_command(remote_instance)
-      #TODO: rsync_to_command("#{Default.storage_directory}/", Default.remote_storage_path, remote_storage_path) if remote_instance
-      "#{rsync_command} #{Default.storage_directory}/ #{remote_instance.ip}:#{Default.remote_storage_path}" if remote_instance
-    end
-    # rsync a file to a node.  By default to the master node.
-    def rsync_to_command(source, target=source, remote_instance=master)
-      "#{rsync_command} #{source} #{remote_instance.ip}:#{target}"
-    end
-    def run_command_on_command(cmd="ls -l", remote_instance=nil)
-      vputs "Running #{cmd} on #{remote_instance.name == %x[hostname].chomp ? "self (master)" : "#{remote_instance.name}"}"
-      (remote_instance.nil? || remote_instance.name == %x[hostname].chomp) ? %x[#{cmd}] : "#{ssh_command(remote_instance)} '#{cmd}'"
-    end
-    def ssh_command(remote_instance)
-      "#{ssh_string} #{remote_instance.ip}"
-    end
-    # Generic commandable strings
-    def ssh_string
-      (["ssh"] << ssh_array).join(" ")
-    end
-    # Array of ssh options
-    # Includes StrictHostKeyChecking to no
-    # Ssh with the user in Base
-    # And including the keypair_path
-    # "-l '#{Default.user}'", 
-    def ssh_array
-      ["-o StrictHostKeyChecking=no", "-l #{Default.user}", '-i "'+full_keypair_path+'"']
-    end
-    def scp_array
-      ["-o StrictHostKeyChecking=no", '-i "'+full_keypair_path+'"']
-    end
-    def rsync_command
-      "rsync -azP --exclude cache -e '#{ssh_string} -l #{Default.user}'"
-    end
-    def remote_ssh_array
-      ["-o StrictHostKeyChecking=no", "-l '#{Default.user}'", '-i "'+remote_keypair_path+'"']
-    end
-    def remote_ssh_string
-      (["ssh"] << remote_ssh_array).join(" ")
-    end
-    def remote_rsync_command
-      "rsync -azP --exclude cache -e '#{remote_ssh_string}'"
-    end
-          
-    def scp_to_command(source, dest=target, remote_instance=master)
-      #TODO: check if source is Directory and add -r if it is
-      "scp #{source} #{remote_instance.ip}:#{dest} #{scp_array.join(' ')}"
-    end
-    
-    # Rsync a file or directory to a node.  Rsync to master by default
-    def rsync_to(source, target=source, num=0)
-      str = "#{rsync_to_command(source, target, get_instance_by_number( num ))}"
-      dputs "Running: #{str}"
-      verbose ?  Kernel.system(str) : hide_output {Kernel.system str}
-    end
-    
-    # Rsync command to the instance
-    def rsync_storage_files_to(instance=nil)
-      hide_output {Kernel.system "#{rsync_storage_files_to_command(instance)}" if instance}
-    end
-    # Take the rsync command and execute it on the system
-    # if there is an instance given
-    def run_command_on(cmd, instance=nil)        
-      Kernel.system "#{run_command_on_command(cmd, instance)}"
-    end
-    
-    # Ssh into the instance given
-    # def ssh_into(instance)
-    #   cmd = "#{ssh_command(instance)}"
-    #   vputs "Running #{cmd}"
-    #   Kernel.system cmd if instance
-    # end
-    # Find the instance by the number given
-    # and then ssh into the instance
-    def ssh_into_instance_number(num=0)
-      ssh_into( get_instance_by_number( num ) )
-    end
-    
-    # Run command on the instance by the number
-    def run_command_on_instance_number(cmd="ls -l", num=0)
-      run_command_on(cmd, get_instance_by_number( num ) )
-    end
-    
-  end
-end

data/lib/poolparty/net/remoter/interactive.rb

@@ -1,62 +0,0 @@
-module PoolParty
-  module Remote
-    
-    # Select a list of instances based on their status
-    def nodes(hsh={}, with_neighborhood_default=false)
-      unordered = begin
-        if with_neighborhood_default
-          list_of_instances(with_neighborhood_default).select_with_hash(hsh)
-        else
-          kname = (hsh[:keypair_name] ||self.keypair_name || self.keypair_name || keypair.basename)
-          # Added keypair to filter on either keypair_name or keypair response
-          key_condition = {:keypair_name => kname, :keypair => kname, :key_name => kname}
-          # if hsh.delete(:uncached)
-          #   @nodes = describe_instances.select_with_hash(conditions)
-          # else
-          #   @nodes ||= describe_instances.select_with_hash(conditions)
-          # end
-          # 
-          results = describe_instances.select_with_hash(key_condition)
-          results.select_with_hash(hsh)
-        end
-      end
-      
-      unordered.sort_by(&:launch_time) # provide consistent sorting for nodes 
-    end
-    
-    # Select the list of instances, either based on the neighborhoods
-    # loaded from /etc/poolparty/neighborhood.json
-    # or by the remote_base on keypair
-    def list_of_instances(with_neighborhood_default=true)
-      return @list_of_instances if @list_of_instances
-      @containing_cloud = self
-      n = false ? Neighborhoods.load_default : nil
-      @list_of_instances = ((n.nil? || n.instances.empty?) ? _list_of_instances : n.instances)
-    end
-
-    private
-    # Cache the instances_by_status here
-    def _nodes
-      @_nodes ||= {}
-    end
-    
-    # List the instances for the current key pair, regardless of their states
-    # If no keypair is passed, select them all
-    def _list_of_instances(select={})      
-      @describe_instances ||= remote_base.describe_instances(dsl_options).select_with_hash(select)
-    end
-    
-    # If the cloud is starting an instance, it will not be listed in 
-    # the running instances, so we need to keep track of the instance
-    # that is being started so we can add it to the neighborhood list
-    def started_instance
-      @started_instance ||= []
-    end
-
-    # Reset the cache of descriptions
-    def reset_remoter_base!
-      @_nodes = @list_of_instances = @describe_instances = nil
-    end
-    
-  end
-end

data/lib/poolparty/net/remoter_base.rb

@@ -1,262 +0,0 @@
-=begin rdoc
-  The base for Remote Bases
-  
-  By extending this class, you can easily add remoters to 
-  PoolParty. There are 4 methods that the remote base needs to implement
-  in order to be compatible.
-  
-  The four methods are:
-    launch_new_instance!
-    terminate_instance(id)
-    describe_instance(id)
-    describe_instances
-  
-  After your remote base is written, make sure to register the base outside the context
-  of the remote base, like so:
-    register_remote_base :remote_base_name
-  
-=end
-module PoolParty
-
-  module Remote
-    
-    def self.available
-      @available ||= []
-    end
-    
-    # This class is the base class for all remote types, such as ec2
-    # Everything remoting-wise is derived from this class
-    class RemoterBase
-      include Dslify
-      include ::PoolParty::Remote
-      include ::PoolParty::Pinger
-      
-      # All inheriting remoter bases should override these default_options
-      dsl_methods :keypair,
-                  :image_id,
-                  :keypair_name
-        
-      def initialize(opts={}, &block)
-        opts.each {|k,v| opts[k] = v.call if v.respond_to?(:call) }
-        set_vars_from_options opts
-        instance_eval(&block) if block
-      end
-      
-      def self.inherited(subclass)
-        unless Remote.available.include?(subclass)
-          Remote.available << subclass
-        end
-      end
-      
-      def cloud(n=nil)
-        if n.nil?
-          @cloud
-        else
-          @cloud = n
-        end
-      end
-      
-      def keypair
-        cloud.keypair
-      end
-      
-      def key_name
-        keypair.basename
-      end
-      
-      # def evaluate_proc_options(opts)
-      #   if opts.respond_to?(:call)
-      #     opts.call
-      #   elsif opts.respond_to? :each
-      #     if opts.respond_to?(:values) && opts.respond_to?(:keys)
-      #       opts.each {|k,v| opts[k] = evaluate_proc_options v }
-      #     elsif opts.respond_to?(:each_with_index)
-      #       opts.each_with_index{|o,i| opts[i] = evaluate_proc_options o}
-      #     end
-      #   end
-      #   opts
-      # end
-      
-      #TODO: deprecate these two methods, use Remote.available
-      def self.available_bases
-        Remote.available.collect{|b| b.name.split('::').pop.snake_case.to_sym}
-      end
-      
-      # Returns a :remote_base_name symbol for the remoter base name
-      def self.symbol
-        name.top_level_class.to_sym
-      end
-      
-      # def method_missing(meth, *args, &block)
-      #   if @cloud
-      #     @cloud.send meth, *args, &block rescue super
-      #   else
-      #     super
-      #   end
-      # end
-      
-      # Required methods
-      # The next methods are required on all RemoteInstance types
-      # If your RemoteInstance type does not overwrite the following methods
-      # An exception will be raised and poolparty will explode into tiny little 
-      # pieces. Don't forget to overwrite these methods
-      # Launch a new instance
-      def self.launch_new_instance!(o={})
-        new(o).launch_new_instance!(o)
-      end
-      def launch_new_instance!(o={})
-        raise RemoteException.new(:method_not_defined, "launch_new_instance!")        
-      end
-      
-      # Terminate an instance by id
-      def self.terminate_instance!(o={})
-        new(o).terminate_instance!(o)
-      end
-      def terminate_instance!(o={})        
-        raise RemoteException.new(:method_not_defined, "terminate_instance!")
-      end
-      
-      # Describe an instance's status.  Should return a hash like object
-      #Required keys are:
-      # :image_id
-      # :keypair_name
-      # :instance_id
-      # :status
-      def self.describe_instance(o={})
-        new(o).describe_instance(o) 
-      end
-      def describe_instance(o={})
-        raise RemoteException.new(:method_not_defined, "describe_instance")
-      end
-      
-      # Get instances
-      # The instances must return an object responding to each
-      # Each yielded object must respond to [:status]
-      def self.describe_instances(o={})
-        new(o).describe_instances(o)
-      end
-      def describe_instances(o={})        
-        raise RemoteException.new(:method_not_defined, "describe_instances")
-      end
-      
-      # TODO: Rename and modularize the @inst.status =~ /pending/ so that it works on all 
-      # remoter_bases
-      # 1.) Launches a new instance, 
-      # 2.) Waits for the instance to get an ip address
-      # 3.) Waits for port 22 to be open
-      # 4.) Calls call_after_launch_instance callbacks
-      # 5.) Executes passed &block, if any
-      def launch_instance!(o={}, &block)
-        @cloud = clouds[o[:cloud_name] || o[:name]]
-        o[:keypair_name] = @cloud.keypair.basename
-        @inst = launch_new_instance!( dsl_options.merge(o) )
-        sleep(2)
-        
-        dputs "#{@cloud.name} launched instance checking for ip..."
-        
-        # Wait for 10 minutes for the instance to gain an ip if it doesn't already have one
-        500.times do |i|
-          if @inst
-            break if @inst[:ip] && @inst[:ip] =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
-            break if @inst[:public_ip] && @inst[:public_ip] =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/            
-            sleep(2)
-            @inst = describe_instance(@inst)
-            dprint "."
-          else
-            @inst = describe_instances.last
-            dprint "."
-          end
-        end        
-        dputs "Found an ip"
-        dputs "#{@cloud.name} Launched instance #{@inst[:ip]}"
-        dputs "   waiting for it to respond"
-        
-        # Try for 10 minutes to pint port 22 
-        500.times do |i|
-          dprint "."
-          if ping_port(@inst[:ip], 22)
-            dputs ""
-            @cloud.started_instance = @inst
-            
-            @cloud.call_after_launch_instance_callbacks(@inst)
-            block.call(@inst) if block
-            
-            return @inst
-          end
-          sleep(2)
-        end
-        raise "Instance not responding at #{inst.ip}"
-      end
-      
-      def self.launch_instance!(o={}, &block)
-        new(o, &block).launch_instance!
-      end
-      
-      def terminate_youngest_instance!(o={})
-        terminate_instance!(:instance_id => nodes(:status => "running").last.instance_id)
-      end
-
-      # Called after an instance is launched
-      def after_launch_instance(instance=nil)
-        puts "after_launch_instance in remoter_base"
-      end
-      
-      def remoter_base_options(n=nil)
-        if n.nil?
-          dsl_options[:remoter_base_options]
-        else
-          dsl_options[:remoter_base_options] = remote_base.dsl_options.choose do |k,v|
-            v && (v.respond_to?(:empty) ? !v.empty?: true)
-          end
-        end
-      end
-
-      #TODO: Remove
-      # def self.when_instance_is_responding(inst, &block)
-      #   if ping_port(inst.ip, 22)
-      #     block.call if block
-      #   else
-      #     raise "Instance not responding at #{inst.ip}"
-      #   end
-      # end
-      # def when_instance_is_responding(inst, &block);self.class.when_instance_is_responding;end
-      
-      # TODO: BAD FORM, already defined in connections.rb. Fix this, ASAP
-      # def self.ping_port(host, port=22, retry_times=400)
-      #   connected = false
-      #   retry_times.times do |i|
-      #     begin
-      #       break if connected = TCPSocket.new(host, port).is_a?(TCPSocket)
-      #     rescue Exception => e
-      #       sleep(2)
-      #     end
-      #   end
-      #   connected
-      # end
-      
-      # After launch callback
-      # This is called after a new instance is launched
-      def after_launched(force=false)        
-      end
-      
-      # Before shutdown callback
-      # This is called before the cloud is contracted
-      def before_shutdown
-      end
-      
-      def to_s
-        self.class.name
-      end
-      
-      def to_hash
-        dsl_options
-      end
-      
-    end
-    
-  end
-end
-
-Dir["#{File.dirname(__FILE__)}/remoter/*.rb"].each do |remoter_module| 
-  require remoter_module
-end