fairchild-poolparty 1.1.5 → 1.2.12

data/lib/poolparty/net/remoter/interactive.rb

@@ -1,43 +1,49 @@
 module PoolParty
   module Remote
     
-    # #DEPRECATE We'll stub the ip to be the master ip for ease and accessibility
-    # def ip(i=nil)
-    #   puts "DEPRECATED:  ip will only be callable against a RemoteInstance in the next release."
-    #   i ? options[:ip] = i : (master ? master.ip : options[:ip])
-    # end
-    # #DEPRECATE: get the master instance
-    def master
-      puts "DEPRECATED:  'master' is deprecated and will be removed in the next major release."
-      get_instance_by_number(0)
-    end
-    
     # Select a list of instances based on their status
-    def nodes(hsh={})
-      # _nodes[hsh] ||= 
-      list_of_instances.select_with_hash(hsh)
-    end
-    
-    # Cache the instances_by_status here
-    def _nodes
-      @_nodes ||= {}
+    def nodes(hsh={}, with_neighborhood_default=false)
+      unordered = begin
+        if with_neighborhood_default
+          list_of_instances(with_neighborhood_default).select_with_hash(hsh)
+        else
+          kname = (hsh[:keypair_name] ||self.keypair_name || self.keypair_name || keypair.basename)
+          # Added keypair to filter on either keypair_name or keypair response
+          key_condition = {:keypair_name => kname, :keypair => kname, :key_name => kname}
+          # if hsh.delete(:uncached)
+          #   @nodes = describe_instances.select_with_hash(conditions)
+          # else
+          #   @nodes ||= describe_instances.select_with_hash(conditions)
+          # end
+          # 
+          results = describe_instances.select_with_hash(key_condition)
+          results.select_with_hash(hsh)
+        end
+      end
+      
+      unordered.sort_by(&:launch_time) # provide consistent sorting for nodes 
     end
     
     # Select the list of instances, either based on the neighborhoods
     # loaded from /etc/poolparty/neighborhood.json
     # or by the remote_base on keypair
-    def list_of_instances
+    def list_of_instances(with_neighborhood_default=true)
       return @list_of_instances if @list_of_instances
       @containing_cloud = self
-      n = Neighborhoods.load_default
-      @list_of_instances = (n.empty? ? _list_of_instances : n.instances)
+      n = false ? Neighborhoods.load_default : nil
+      @list_of_instances = ((n.nil? || n.instances.empty?) ? _list_of_instances : n.instances)
     end
 
     private
+    # Cache the instances_by_status here
+    def _nodes
+      @_nodes ||= {}
+    end
+    
     # List the instances for the current key pair, regardless of their states
     # If no keypair is passed, select them all
-    def _list_of_instances(select={})
-      @describe_instances ||= remote_base.describe_instances(options).select_with_hash(select)
+    def _list_of_instances(select={})      
+      @describe_instances ||= remote_base.describe_instances(dsl_options).select_with_hash(select)
     end
     
     # If the cloud is starting an instance, it will not be listed in 

data/lib/poolparty/net/remoter_base.rb

@@ -16,24 +16,76 @@
     register_remote_base :remote_base_name
   
 =end
-
 module PoolParty
 
-  module Remote    
+  module Remote
+    
+    def self.available
+      @available ||= []
+    end
+    
     # This class is the base class for all remote types, such as ec2
     # Everything remoting-wise is derived from this class
     class RemoterBase
-      include               Dslify
-      include  ::PoolParty::Remote
+      include Dslify
+      include ::PoolParty::Remote
+      include ::PoolParty::Pinger
       
-      attr_reader :cloud
+      # All inheriting remoter bases should override these default_options
+      dsl_methods :keypair,
+                  :image_id,
+                  :keypair_name
+        
+      def initialize(opts={}, &block)
+        opts.each {|k,v| opts[k] = v.call if v.respond_to?(:call) }
+        set_vars_from_options opts
+        instance_eval(&block) if block
+      end
       
-      def initialize(prnt, opts={}, &block)
-        dsl_options prnt.options.merge(opts) if prnt && prnt.respond_to?(:options)
-        instance_eval &block if block
-        @cloud = prnt
+      def self.inherited(subclass)
+        unless Remote.available.include?(subclass)
+          Remote.available << subclass
+        end
+      end
+      
+      def cloud(n=nil)
+        if n.nil?
+          @cloud
+        else
+          @cloud = n
+        end
+      end
+      
+      def keypair
+        cloud.keypair
+      end
+      
+      def key_name
+        keypair.basename
+      end
+      
+      # def evaluate_proc_options(opts)
+      #   if opts.respond_to?(:call)
+      #     opts.call
+      #   elsif opts.respond_to? :each
+      #     if opts.respond_to?(:values) && opts.respond_to?(:keys)
+      #       opts.each {|k,v| opts[k] = evaluate_proc_options v }
+      #     elsif opts.respond_to?(:each_with_index)
+      #       opts.each_with_index{|o,i| opts[i] = evaluate_proc_options o}
+      #     end
+      #   end
+      #   opts
+      # end
+      
+      #TODO: deprecate these two methods, use Remote.available
+      def self.available_bases
+        Remote.available.collect{|b| b.name.split('::').pop.snake_case.to_sym}
+      end
+      
+      # Returns a :remote_base_name symbol for the remoter base name
+      def self.symbol
+        name.top_level_class.to_sym
       end
-
       
       # def method_missing(meth, *args, &block)
       #   if @cloud
@@ -49,33 +101,39 @@ module PoolParty
       # An exception will be raised and poolparty will explode into tiny little 
       # pieces. Don't forget to overwrite these methods
       # Launch a new instance
-      def self.launch_new_instance!(cld, o={})
-        new(cld, o).launch_new_instance!(o)
+      def self.launch_new_instance!(o={})
+        new(o).launch_new_instance!(o)
       end
       def launch_new_instance!(o={})
         raise RemoteException.new(:method_not_defined, "launch_new_instance!")        
       end
       
       # Terminate an instance by id
-      def self.terminate_instance!(cld, o={})
-        new(cld, o).terminate_instance!(o)
+      def self.terminate_instance!(o={})
+        new(o).terminate_instance!(o)
       end
       def terminate_instance!(o={})        
         raise RemoteException.new(:method_not_defined, "terminate_instance!")
       end
       
-      # Describe an instance's status
-      def self.describe_instance(cld, o={})
-        new(cld, o).describe_instance(o) 
+      # Describe an instance's status.  Should return a hash like object
+      #Required keys are:
+      # :image_id
+      # :keypair_name
+      # :instance_id
+      # :status
+      def self.describe_instance(o={})
+        new(o).describe_instance(o) 
       end
       def describe_instance(o={})
         raise RemoteException.new(:method_not_defined, "describe_instance")
       end
       
       # Get instances
-      # The instances must have a status associated with them on the hash
-      def self.describe_instances(cld, o={})
-        new(cld, o).describe_instances(o)
+      # The instances must return an object responding to each
+      # Each yielded object must respond to [:status]
+      def self.describe_instances(o={})
+        new(o).describe_instances(o)
       end
       def describe_instances(o={})        
         raise RemoteException.new(:method_not_defined, "describe_instances")
@@ -83,34 +141,46 @@ module PoolParty
       
       # TODO: Rename and modularize the @inst.status =~ /pending/ so that it works on all 
       # remoter_bases
+      # 1.) Launches a new instance, 
+      # 2.) Waits for the instance to get an ip address
+      # 3.) Waits for port 22 to be open
+      # 4.) Calls call_after_launch_instance callbacks
+      # 5.) Executes passed &block, if any
       def launch_instance!(o={}, &block)
-        @inst = launch_new_instance!( o )
+        @cloud = clouds[o[:cloud_name] || o[:name]]
+        o[:keypair_name] = @cloud.keypair.basename
+        @inst = launch_new_instance!( dsl_options.merge(o) )
         sleep(2)
         
-        cloud.dputs "#{cloud.name} launched instance checking for ip..."
+        dputs "#{@cloud.name} launched instance checking for ip..."
         
         # Wait for 10 minutes for the instance to gain an ip if it doesn't already have one
         500.times do |i|
-          break if @inst[:ip] =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
-          sleep(2)
-          @inst = describe_instance(@inst)
-          cloud.dprint "."
+          if @inst
+            break if @inst[:ip] && @inst[:ip] =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
+            break if @inst[:public_ip] && @inst[:public_ip] =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/            
+            sleep(2)
+            @inst = describe_instance(@inst)
+            dprint "."
+          else
+            @inst = describe_instances.last
+            dprint "."
+          end
         end        
-        cloud.dputs "Found an ip"
-        cloud.dputs "#{@cloud.name} Launched instance #{@inst[:ip]}"
-        cloud.dputs "   waiting for it to respond"
+        dputs "Found an ip"
+        dputs "#{@cloud.name} Launched instance #{@inst[:ip]}"
+        dputs "   waiting for it to respond"
         
         # Try for 10 minutes to pint port 22 
         500.times do |i|
-          cloud.dprint "."
+          dprint "."
           if ping_port(@inst[:ip], 22)
-            cloud.dputs ""
-            cloud.started_instance = @inst
+            dputs ""
+            @cloud.started_instance = @inst
             
-            cloud.call_after_launch_instance_callbacks(@inst)
+            @cloud.call_after_launch_instance_callbacks(@inst)
             block.call(@inst) if block
-            # after_launch_instance(@inst)
-            cloud.call_after_launch_instance_callbacks(@inst)
+            
             return @inst
           end
           sleep(2)
@@ -118,14 +188,28 @@ module PoolParty
         raise "Instance not responding at #{inst.ip}"
       end
       
-      def self.launch_instance!(cld, o={}, &block)
-        new(cld, o, &block).launch_instance!
+      def self.launch_instance!(o={}, &block)
+        new(o, &block).launch_instance!
+      end
+      
+      def terminate_youngest_instance!(o={})
+        terminate_instance!(:instance_id => nodes(:status => "running").last.instance_id)
       end
 
       # Called after an instance is launched
       def after_launch_instance(instance=nil)
         puts "after_launch_instance in remoter_base"
       end
+      
+      def remoter_base_options(n=nil)
+        if n.nil?
+          dsl_options[:remoter_base_options]
+        else
+          dsl_options[:remoter_base_options] = remote_base.dsl_options.choose do |k,v|
+            v && (v.respond_to?(:empty) ? !v.empty?: true)
+          end
+        end
+      end
 
       #TODO: Remove
       # def self.when_instance_is_responding(inst, &block)
@@ -138,17 +222,17 @@ module PoolParty
       # def when_instance_is_responding(inst, &block);self.class.when_instance_is_responding;end
       
       # TODO: BAD FORM, already defined in connections.rb. Fix this, ASAP
-      def self.ping_port(host, port=22, retry_times=400)
-        connected = false
-        retry_times.times do |i|
-          begin
-            break if connected = TCPSocket.new(host, port).is_a?(TCPSocket)
-          rescue Exception => e
-            sleep(2)
-          end
-        end
-        connected
-      end
+      # def self.ping_port(host, port=22, retry_times=400)
+      #   connected = false
+      #   retry_times.times do |i|
+      #     begin
+      #       break if connected = TCPSocket.new(host, port).is_a?(TCPSocket)
+      #     rescue Exception => e
+      #       sleep(2)
+      #     end
+      #   end
+      #   connected
+      # end
       
       # After launch callback
       # This is called after a new instance is launched
@@ -160,6 +244,14 @@ module PoolParty
       def before_shutdown
       end
       
+      def to_s
+        self.class.name
+      end
+      
+      def to_hash
+        dsl_options
+      end
+      
     end
     
   end

data/lib/poolparty/net/remoter_bases/ec2/ec2.rb

@@ -25,7 +25,7 @@ end
   
 class String
   def convert_from_ec2_to_ip
-    self.gsub(/.compute-1.amazonaws.com*/, '').gsub(/ec2-/, '').gsub(/-/, '.')
+    self.match(/-(\d+-\d+-\d+-\d+)\./) ? $1.gsub(/-/, '.') : self
   end
   def parse_datetime
     DateTime.parse( self.chomp ) rescue self
@@ -35,95 +35,130 @@ end
 module PoolParty    
   module Remote
     class Ec2 < Remote::RemoterBase
+      require "#{::File.dirname(__FILE__)}/ec2_remote_instance"
+      
+      dsl_methods :elastic_ips,           # An array of the elastic ips
+                  :ebs_volume_id         # The volume id of an ebs volume
       
       default_options({
-        :image_id => 'ami-bf5eb9d6',
+        :image_id           => 'ami-bf5eb9d6',
+        :ami                => 'ami-bf5eb9d6',  #Deprecated, but here for backwards compatability
         # :key_name => ::File.basename(keypair.is_a?(String) ? keypair : keypair.full_filepath),
-        :instance_type => 'm1.small', # or 'm1.large', 'm1.xlarge', 'c1.medium', or 'c1.xlarge'
-        :addressing_type => "public",
-        :availabilty_zone => "us-east-1a",
-        :security_group => ["default"]
+        :instance_type      => 'm1.small', # or 'm1.large', 'm1.xlarge', 'c1.medium', or 'c1.xlarge'
+        :addressing_type    => "public",
+        :availability_zone  => "us-east-1a",
+        :access_key         => ENV['AWS_ACCESS_KEY'],
+        :secret_access_key  => ENV['AWS_SECRET_ACCESS_KEY'],
+        :security_group     => ["default"],
+        :keypair_name       => nil,
+        :key_name           => nil
         })
+        
+      # alias to image_id
+      # def ami(n=nil)
+      #   if n.nil?
+      #     image_id
+      #   else
+      #     image_id n
+      #   end
+      # end
       
       # Requires a hash of options
-      def self.launch_new_instance!(parent_cloud, o)
-        new(parent_cloud, o).launch_new_instance!
+      def self.launch_new_instance!(o)
+        new(o).launch_new_instance!
       end
       
       # TODO: Fix the key_name issue
       # Start a new instance with the given options
       def launch_new_instance!(o={})
-        raise "You must pass a keypair to launch an instance, or else you will not be able to login. options = #{o.inspect}" if !cloud.keypair
-        o.merge!( options ).merge!(:key_name=>keypair.basename)
+        set_vars_from_options o
+        keypair_name ||= o[:keypair_name] || keypair || (clouds[o[:cloud_name]].keypair.basename if o[:cloud_name])
+        raise "You must pass a keypair to launch an instance, or else you will not be able to login. options = #{o.inspect}" if !keypair_name 
+        o.merge!( dsl_options.merge(:key_name=>keypair_name, :group_id => security_group) )
         instance = ec2(o).run_instances(o)
+        
         begin
-          h = EC2ResponseObject.get_hash_from_response(instance.instancesSet.item.first)
+          h = EC2ResponseObject.describe_instance(instance)
           #h = instance.instancesSet.item.first
         rescue Exception => e
-          h = EC2ResponseObject.get_hash_from_response(instance) rescue instance
+          vputs "There was an error:\n\t#{e.inspect}"
+          h = EC2ResponseObject.describe_instance(instance) rescue instance
           # h = instance
         end
-        h
+        Ec2RemoteInstance.new(h)
       end
+      
       # Terminate an instance by id
       def terminate_instance!(o={})
         ec2(o).terminate_instances(:instance_id => o[:instance_id])
       end
+      
       # Describe an instance's status
       def describe_instance(o={})
-        return describe_instances.first if o[:instance_id].nil?
+        return describe_instances.first if o.empty? || o[:instance_id].nil?
         describe_instances.detect {|a| a[:name] == o[:instance_id] || a[:ip] == o[:instance_id] || a[:instance_id] == o[:instance_id] }
       end
+      
       def describe_instances(o={})
-        id = 0
-        get_instances_description(options.merge(o)).each_with_index do |h,i|          
-          if h[:status] == "running"
-            inst_name = id == 0 ? "master" : "node#{id}"
-            id += 1
-          else
-            inst_name = "#{h[:status]}_node#{i}"
-          end
-          h.merge!({
-            :name => inst_name,
-            :hostname => h[:ip],
-            :ip => h[:ip].convert_from_ec2_to_ip,
-            :index => i,  #TODO MF get the instance id from the aws result instead
-            :launching_time => (h[:launching_time])
-          })
-        end.compact.sort {|a,b| a[:index] <=> b[:index] }
+        ec2_instants = EC2ResponseObject.describe_instances(ec2.describe_instances)
+        insts = ec2_instants.select_with_hash(o) if !o.empty?
+        ec2_remote_instances = ec2_instants.collect{|i| Ec2RemoteInstance.new(i)}
+        ec2_remote_instances.sort {|a,b| a[:ami_launch_index] <=> b[:ami_launch_index] }
       end
       
+      # TODO: Clean up this method and remove hostnames
+      # def describe_instances(o={})
+      #   id = 0
+      #   set_vars_from_options(dsl_options.merge(o))
+      #   get_instances_description(dsl_options).each_with_index do |h,i|
+      #     if h[:status] == "running"
+      #       inst_name = id == 0 ? "master" : "node#{id}"
+      #       id += 1
+      #     else
+      #       inst_name = "#{h[:status]}_node#{i}"
+      #     end
+      #     h.merge!({
+      #       :name           => inst_name,
+      #       :hostname       => h[:ip],
+      #       :ip             => h[:ip].convert_from_ec2_to_ip,
+      #       # :internal_ip    => h.ip
+      #       :index          => i,  #TODO get the instance id from the aws result instead
+      #       :launching_time => (h[:launching_time])
+      #     })
+      #   end.compact.sort {|a,b| a[:index] <=> b[:index] }
+      # end
+      
       # ===================================
       # = Ec2 Specific methods below here =
       # ===================================
       
       # return or create a new base EC2 connection object that will actually connect to ec2
       def ec2(o={})
-        @ec2 ||= EC2::Base.new( :access_key_id => o[:access_key], 
-                                :secret_access_key => o[:secret_access_key]
+        @ec2 ||= EC2::Base.new( :access_key_id => o[:access_key] || get_access_key, 
+                                :secret_access_key => o[:secret_access_key] || get_secret_access_key
                               )
       end
       def self.ec2(o)
-        @ec2 ||= self.class.ec2(o)
+        @ec2 ||= EC2::Base.new( :access_key_id => o[:access_key], 
+                                :secret_access_key => o[:secret_access_key]
+                              )
       end
       
       # Get the ec2 description for the response in a hash format
       def get_instances_description(o={})
         #TODO: only use keypair.full_filepath
-        key_hash = {:keypair => ::File.basename(keypair.is_a?(String) ? keypair : keypair.full_filepath)}
-        EC2ResponseObject.get_descriptions(ec2(o).describe_instances).select_with_hash(key_hash)
+        set_vars_from_options dsl_options.merge(o)
+        key_hash = {:keypair => self.keypair_name}
+        out = EC2ResponseObject.get_descriptions(ec2(dsl_options).describe_instances)
+        out = keypair_name ? out.select_with_hash(key_hash) : out
       end
       def get_descriptions(o={})
         self.class.get_descriptions(o)
       end
-      
-      def keypair
-        cloud.keypair
-      end
-      
+            
       # Class method helpers
       def aws_keys
-        unless @access_key && @secret_access_key          
+        unless @access_key && @secret_access_key
           aws_keys = {}
           aws_keys = YAML::load( File.open('/etc/poolparty/aws_keys.yml') ) rescue 'No aws_keys.yml file.   Will try to use enviornment variables'
           @access_key ||= aws_keys[:access_key] || ENV['AMAZON_ACCESS_KEY_ID'] || ENV['AWS_ACCESS_KEY']
@@ -137,16 +172,6 @@ module PoolParty
           associate_address(inst)
         end
       end
-    
-      # Attach a volume to the instance
-      # DEPRECATE this relies on master.  master will be removed in next major release.  This method will be in ec2_remote_instance instead, or require an instance id
-      def attach_volume(instance=nil)
-        if ebs_volume_id
-          vputs "Attaching volume #{ebs_volume_id} to the master at #{ebs_volume_device}"
-          instance = master        
-          ec2.attach_volume(:volume_id => ebs_volume_id, :instance_id => instance.instance_id, :device => ebs_volume_device) if ebs_volume_id && ebs_volume_mount_point
-        end
-      end
       
       # Associate an address with the instance using ec2
       # Get the next_unused_elastic_ip
@@ -156,6 +181,16 @@ module PoolParty
         if ip = next_unused_elastic_ip
           vputs "Associating #{instance.instance_id} with #{ip}"
           ec2.associate_address(:instance_id => instance.instance_id, :public_ip => ip)
+          
+          # Try for 10 minutes to pint port 22 
+          500.times do |i|
+            dprint "."
+            if ping_port(ip, 22)
+              instance[:ip] = ip
+              return true
+            end
+            sleep(2)
+          end 
         end
       end
       
@@ -167,21 +202,23 @@ module PoolParty
       # intersection of the unused ips and those, find the first one available
       # and return that, otherwise, return the first elastic ip available
       def next_unused_elastic_ip
-        # [{"instanceId"=>nil, "publicIp"=>"174.129.212.93"}, {"instanceId"=>nil, "publicIp"=>"174.129.212.94"}]
-        if addressesSet = ec2(options).describe_addresses["addressesSet"]
-          begin
-            empty_addresses = addressesSet["item"].select {|i| i["instanceId"].nil? }
-            ips = empty_addresses.map {|addr| addr["publicIp"]}
-            if cloud.elastic_ips?
-              ips_to_use = cloud.elastic_ips & ips
-              ips_to_use.first
-            else
-              ips.first
+        if elastic_ips.empty?
+          nil
+        else
+          # [{"instanceId"=>nil, "publicIp"=>"174.129.212.93"}, {"instanceId"=>nil, "publicIp"=>"174.129.212.94"}]
+          if addressesSet = ec2(dsl_options).describe_addresses(:public_ip => elastic_ips)["addressesSet"]
+            begin
+              empty_addresses = addressesSet["item"].select {|i| i["instanceId"].nil? }
+              ips = empty_addresses.map {|addr| addr["publicIp"]}
+              if elastic_ips
+                ips_to_use = elastic_ips & ips
+                ips_to_use.first
+              end
+            rescue Exception => e
+              puts "Error: #{e}"
+              nil
             end
-          rescue Exception => e
-            puts "Error: #{e}"
-            nil
-          end          
+          end
         end
       end
 
@@ -217,7 +254,7 @@ module PoolParty
       end
     
       def custom_minimum_runnable_options
-        [:ami, :availabilty_zone, :security_group]
+        [:ami, :availability_zone, :security_group]
       end
 
       # Hook
@@ -237,10 +274,21 @@ module PoolParty
           # "/usr/bin/gem install --no-ri --no-rdoc amazon-ec2.gem 2>&1"
         ]
       end
-
-      def custom_configure_tasks_for(o)
-        [
-        ]
+      
+      def get_access_key(n=nil)
+        if n.nil?
+          dsl_options[:access_key] ||= Default.access_key
+        else
+          self.access_key = n
+        end
+      end
+      
+      def get_secret_access_key(n=nil)
+        if n.nil?
+          dsl_options[:secret_access_key] ||= Default.secret_access_key
+        else
+          self.secret_access_key = n
+        end
       end
 
       def reset_base!
@@ -249,4 +297,4 @@ module PoolParty
     end
         
   end
-end
+end