poolparty 0.2.84 → 1.2.2

data/lib/poolparty/net/remoter_base.rb

@@ -16,149 +16,152 @@
     register_remote_base :remote_base_name
   
 =end
-module PoolParty
 
-  def register_remote_base(*args)
-    args.each do |arg|
-      (remote_bases << "#{arg}".downcase.to_sym)
-    end
-  end
-  
-  def remote_bases
-    $remote_bases ||= []
-  end
+module PoolParty
 
   module Remote    
-    # This class is the base class for all remote types
+    # This class is the base class for all remote types, such as ec2
     # Everything remoting-wise is derived from this class
-    module RemoterBaseMethods
+    class RemoterBase
+      include               Dslify
+      include  ::PoolParty::Remote
+      
+      attr_reader :cloud
+      
+      def initialize(prnt, opts={}, &block)
+        dsl_options prnt.options.merge(opts) if prnt && prnt.respond_to?(:options)
+        instance_eval &block if block
+        @cloud = prnt
+      end
+      
+      def self.inherited(arg)
+        base_name = "#{arg}".downcase.top_level_class.to_sym
+        (remote_bases << base_name) unless remote_bases.include?(base_name)
+      end
+
+      
+      # def method_missing(meth, *args, &block)
+      #   if @cloud
+      #     @cloud.send meth, *args, &block rescue super
+      #   else
+      #     super
+      #   end
+      # end
+      
       # Required methods
       # The next methods are required on all RemoteInstance types
       # If your RemoteInstance type does not overwrite the following methods
       # An exception will be raised and poolparty will explode into tiny little 
       # pieces. Don't forget to overwrite these methods
       # Launch a new instance
-      def launch_new_instance!
-        raise RemoteException.new(:method_not_defined, "launch_new_instance!")
+      def self.launch_new_instance!(cld, o={})
+        new(cld, o).launch_new_instance!(o)
       end
+      def launch_new_instance!(o={})
+        raise RemoteException.new(:method_not_defined, "launch_new_instance!")        
+      end
+      
       # Terminate an instance by id
-      def terminate_instance!(id=nil)
+      def self.terminate_instance!(cld, o={})
+        new(cld, o).terminate_instance!(o)
+      end
+      def terminate_instance!(o={})        
         raise RemoteException.new(:method_not_defined, "terminate_instance!")
       end
+      
       # Describe an instance's status
-      def describe_instance(id=nil)
+      def self.describe_instance(cld, o={})
+        new(cld, o).describe_instance(o) 
+      end
+      def describe_instance(o={})
         raise RemoteException.new(:method_not_defined, "describe_instance")
       end
+      
       # Get instances
       # The instances must have a status associated with them on the hash
-      def describe_instances
+      def self.describe_instances(cld, o={})
+        new(cld, o).describe_instances(o)
+      end
+      def describe_instances(o={})        
         raise RemoteException.new(:method_not_defined, "describe_instances")
       end
       
-    end
-    module RemoterBase
-      # The following methods are inherent on the RemoterBase
-      # If you need to overwrite these methods, do so with caution
-      # Listing methods
-      def list_of_running_instances(list = list_of_nonterminated_instances)
-        list.select {|a| a.running? }
-      end
-      # Get a list of the pending instances
-      def list_of_pending_instances(list = list_of_nonterminated_instances)
-        list.select {|a| a.pending? }
-      end
-      # list of shutting down instances
-      def list_of_terminating_instances(list = remote_instances_list)
-        list.reject {|i| true if !i.terminating? }
-      end
-      # Get the instances that are non-master instances
-      def nonmaster_nonterminated_instances(list = list_of_nonterminated_instances)
-        list_of_nonterminated_instances.reject {|i| i.master? }
-      end
-      # list all the nonterminated instances
-      def list_of_nonterminated_instances(list = remote_instances_list)
-        list.reject {|i| i.terminating? || i.terminated? }
-      end
-      # get the master instance
-      def master
-        get_instance_by_number(0)
-      end
-      # Get instance by number
-      def get_instance_by_number(i=0, list = remote_instances_list)
-        name = (i.zero? ? "master" : "node#{i}")
-        list.select {|i| i.name == name }.first
-      end
-      # A callback before the configuration task takes place
-      def before_configuration_tasks        
-      end
-      def remote_instances_list        
-        @containing_cloud = self
-        # puts "> #{@containing_cloud} #{@describe_instances.nil?}"
-        list_of_instances(keypair).collect {|h| PoolParty::Remote::RemoteInstance.new(h, @containing_cloud) }
-      end
-      # List the instances for the current key pair, regardless of their states
-      # If no keypair is passed, select them all
-      def list_of_instances(keyp=nil)
-        tmp_key = (keyp ? keyp : keypair).to_s
-        unless @describe_instances
-          tmpInstanceList = describe_instances.select {|a| a if tmp_key ? a[:keypair] == tmp_key : true }
-          has_master = !tmpInstanceList.select {|a| a[:name] == "master" }.empty?          
-          if has_master
-            @describe_instances = tmpInstanceList
-          else
-            @id = 0
-            running = select_from_instances_on_status(/running/, tmpInstanceList)
-            pending = select_from_instances_on_status(/pending/, tmpInstanceList)
-            terminated = select_from_instances_on_status(/shutting/, tmpInstanceList)
+      # TODO: Rename and modularize the @inst.status =~ /pending/ so that it works on all 
+      # remoter_bases
+      def launch_instance!(o={}, &block)
+        @inst = launch_new_instance!( o )
+        sleep(2)
+        
+        cloud.dputs "#{cloud.name} launched instance checking for ip..."
+        
+        # Wait for 10 minutes for the instance to gain an ip if it doesn't already have one
+        500.times do |i|
+          break if @inst[:ip] =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
+          sleep(2)
+          @inst = describe_instance(@inst)
+          cloud.dprint "."
+        end        
+        cloud.dputs "Found an ip"
+        cloud.dputs "#{@cloud.name} Launched instance #{@inst[:ip]}"
+        cloud.dputs "   waiting for it to respond"
+        
+        # Try for 10 minutes to pint port 22 
+        500.times do |i|
+          cloud.dprint "."
+          if ping_port(@inst[:ip], 22)
+            cloud.dputs ""
+            cloud.started_instance = @inst
             
-            running = running.map do |inst|
-              inst[:name] = (@id == 0 ? "master" : "node#{@id}")
-              @id += 1
-              inst
-            end.sort_by {|a| a[:index] }
+            cloud.call_after_launch_instance_callbacks(@inst)
+            block.call(@inst) if block
             
-            @describe_instances = [running, pending, terminated].flatten
+            return @inst
           end
+          sleep(2)
         end
-        @describe_instances
-      end
-      # Select the instances based on their status
-      def select_from_instances_on_status(status=/running/, list=[])
-        list.select {|a| a[:status] =~ status}
-      end
-      # Helpers
-      def create_keypair
+        raise "Instance not responding at #{inst.ip}"
       end
-      # Reset the cache of descriptions
-      def reset_remoter_base!
-        @describe_instances = nil
+      
+      def self.launch_instance!(cld, o={}, &block)
+        new(cld, o, &block).launch_instance!
       end
-      def self.included(other)
-        PoolParty.register_remote_base(self.class.to_s.downcase.to_sym)
+
+      # Called after an instance is launched
+      def after_launch_instance(instance=nil)
+        puts "after_launch_instance in remoter_base"
       end
+
+      #TODO: Remove
+      # def self.when_instance_is_responding(inst, &block)
+      #   if ping_port(inst.ip, 22)
+      #     block.call if block
+      #   else
+      #     raise "Instance not responding at #{inst.ip}"
+      #   end
+      # end
+      # def when_instance_is_responding(inst, &block);self.class.when_instance_is_responding;end
       
-      # Callback after loaded
-      def loaded_remoter_base        
+      # TODO: BAD FORM, already defined in connections.rb. Fix this, ASAP
+      def self.ping_port(host, port=22, retry_times=400)
+        connected = false
+        retry_times.times do |i|
+          begin
+            break if connected = TCPSocket.new(host, port).is_a?(TCPSocket)
+          rescue Exception => e
+            sleep(2)
+          end
+        end
+        connected
       end
       
-      # Custom minimum runnable options
-      # Extend the minimum runnable options that are necessary
-      # for poolparty to run on the remote base
-      def custom_minimum_runnable_options
-        []
+      # After launch callback
+      # This is called after a new instance is launched
+      def after_launched(force=false)        
       end
-            
-      # Custom installation tasks
-      # Allow the remoter bases to attach their own tasks on the 
-      # installation process
-      def custom_install_tasks_for(a=nil)
-        []
-      end
-      # Custom configure tasks
-      # Allows the remoter bases to attach their own
-      # custom configuration tasks to the configuration process
-      def custom_configure_tasks_for(a=nil)
-        []
+      
+      # Before shutdown callback
+      # This is called before the cloud is contracted
+      def before_shutdown
       end
       
     end
@@ -166,4 +169,6 @@ module PoolParty
   end
 end
 
-Dir["#{File.dirname(__FILE__)}/remote_bases/*.rb"].each {|base| require base }
+Dir["#{File.dirname(__FILE__)}/remoter/*.rb"].each do |remoter_module| 
+  require remoter_module
+end

data/lib/poolparty/net/remoter_bases/ec2/ec2.rb

@@ -0,0 +1,245 @@
+=begin rdoc
+  EC2 Remoter Base
+  
+  This serves as the basis for running PoolParty on Amazon's ec2 cloud.
+=end
+require "date"
+require "#{::File.dirname(__FILE__)}/ec2_response_object"
+
+begin
+  require 'EC2'
+rescue LoadError
+  puts <<-EOM
+Error: In order to use ec2, you need to install the amazon-ec2 gem
+
+Ec2 is the default remoter base for PoolParty. If you intend on using
+a different remoter base, specify it with:
+
+using :remoter_name
+
+in your config file, otherwise, to continue install grempe-amazon-ec2 with
+
+sudo gem install grempe-amazon-ec2 --source http://gems.github.com
+EOM
+end
+  
+class String
+  def convert_from_ec2_to_ip
+    self.gsub(/.compute-1.amazonaws.com*/, '').gsub(/ec2-/, '').gsub(/-/, '.')
+  end
+  def parse_datetime
+    DateTime.parse( self.chomp ) rescue self
+  end
+end
+  
+module PoolParty    
+  module Remote
+    class Ec2 < Remote::RemoterBase
+      
+      default_options({
+        :image_id => 'ami-bf5eb9d6',
+        # :key_name => ::File.basename(keypair.is_a?(String) ? keypair : keypair.full_filepath),
+        :instance_type => 'm1.small', # or 'm1.large', 'm1.xlarge', 'c1.medium', or 'c1.xlarge'
+        :addressing_type => "public",
+        :availabilty_zone => "us-east-1a",
+        :security_group => ["default"]
+        })
+      
+      # Requires a hash of options
+      def self.launch_new_instance!(parent_cloud, o)
+        new(parent_cloud, o).launch_new_instance!
+      end
+      
+      # TODO: Fix the key_name issue
+      # Start a new instance with the given options
+      def launch_new_instance!(o={})
+        raise "You must pass a keypair to launch an instance, or else you will not be able to login. options = #{o.inspect}" if !cloud.keypair
+        o.merge!( options ).merge!(:key_name=>keypair.basename)
+        instance = ec2(o).run_instances(o)
+        begin
+          h = EC2ResponseObject.get_hash_from_response(instance.instancesSet.item.first)
+          #h = instance.instancesSet.item.first
+        rescue Exception => e
+          h = EC2ResponseObject.get_hash_from_response(instance) rescue instance
+          # h = instance
+        end
+        h
+      end
+      # Terminate an instance by id
+      def terminate_instance!(o={})
+        ec2(o).terminate_instances(:instance_id => o[:instance_id])
+      end
+      # Describe an instance's status
+      def describe_instance(o={})
+        return describe_instances.first if o[:instance_id].nil?
+        describe_instances.detect {|a| a[:name] == o[:instance_id] || a[:ip] == o[:instance_id] || a[:instance_id] == o[:instance_id] }
+      end
+      # TODO: Clean up this method and remove hostnames
+      def describe_instances(o={})
+        id = 0
+        get_instances_description(options.merge(o)).each_with_index do |h,i|          
+          if h[:status] == "running"
+            inst_name = id == 0 ? "master" : "node#{id}"
+            id += 1
+          else
+            inst_name = "#{h[:status]}_node#{i}"
+          end
+          h.merge!({
+            :name => inst_name,
+            :hostname => h[:ip],
+            :ip => h[:ip].convert_from_ec2_to_ip,
+            :index => i,  #TODO MF get the instance id from the aws result instead
+            :launching_time => (h[:launching_time])
+          })
+        end.compact.sort {|a,b| a[:index] <=> b[:index] }
+      end
+      
+      # ===================================
+      # = Ec2 Specific methods below here =
+      # ===================================
+      
+      # return or create a new base EC2 connection object that will actually connect to ec2
+      def ec2(o={})
+        @ec2 ||= EC2::Base.new( :access_key_id => o[:access_key], 
+                                :secret_access_key => o[:secret_access_key]
+                              )
+      end
+      def self.ec2(o)
+        @ec2 ||= EC2::Base.new( :access_key_id => o[:access_key], 
+                                :secret_access_key => o[:secret_access_key]
+                              )
+      end
+      
+      # Get the ec2 description for the response in a hash format
+      def get_instances_description(o={})
+        #TODO: only use keypair.full_filepath
+        key_hash = {:keypair => ::File.basename(keypair.is_a?(String) ? keypair : keypair.full_filepath)}
+        EC2ResponseObject.get_descriptions(ec2(o).describe_instances).select_with_hash(key_hash)
+      end
+      def get_descriptions(o={})
+        self.class.get_descriptions(o)
+      end
+      
+      def keypair
+        cloud.keypair
+      end
+      
+      # Class method helpers
+      def aws_keys
+        unless @access_key && @secret_access_key          
+          aws_keys = {}
+          aws_keys = YAML::load( File.open('/etc/poolparty/aws_keys.yml') ) rescue 'No aws_keys.yml file.   Will try to use enviornment variables'
+          @access_key ||= aws_keys[:access_key] || ENV['AMAZON_ACCESS_KEY_ID'] || ENV['AWS_ACCESS_KEY']
+          @secret_access_key ||= aws_keys[:secret_access_key] || ENV['AMAZON_SECRET_ACCESS_KEY'] || ENV['AWS_SECRET_ACCESS_KEY']
+        end
+        [@access_key, @secret_access_key]
+      end
+      
+      def after_launch_instance(inst)
+        if inst
+          associate_address(inst)
+        end
+      end
+      
+      # Associate an address with the instance using ec2
+      # Get the next_unused_elastic_ip
+      # and if there is one, associate the instance to the 
+      # public ip
+      def associate_address(instance=nil)
+        if ip = next_unused_elastic_ip
+          vputs "Associating #{instance.instance_id} with #{ip}"
+          ec2.associate_address(:instance_id => instance.instance_id, :public_ip => ip)
+        end
+      end
+      
+      # Get the next usable elastic ip
+      # First, get the list of addresses from ec2 that the client
+      # has access to, then select only the ones that are not associated
+      # with an instance.
+      # If the cloud has set elastic_ips to use, then, using the 
+      # intersection of the unused ips and those, find the first one available
+      # and return that, otherwise, return the first elastic ip available
+      def next_unused_elastic_ip
+        # [{"instanceId"=>nil, "publicIp"=>"174.129.212.93"}, {"instanceId"=>nil, "publicIp"=>"174.129.212.94"}]
+        if addressesSet = ec2(options).describe_addresses["addressesSet"]
+          begin
+            empty_addresses = addressesSet["item"].select {|i| i["instanceId"].nil? }
+            ips = empty_addresses.map {|addr| addr["publicIp"]}
+            if cloud.elastic_ips?
+              ips_to_use = cloud.elastic_ips & ips
+              ips_to_use.first
+            else
+              ips.first
+            end
+          rescue Exception => e
+            puts "Error: #{e}"
+            nil
+          end          
+        end
+      end
+
+      # Help create a keypair for the cloud
+      # This is a helper to create the keypair and add them to the cloud for you
+      def create_keypair
+        return false unless keypair
+        unless ::File.exists?( new_keypair_path )
+          FileUtils.mkdir_p ::File.dirname( new_keypair_path )
+          vputs "Creating keypair: #{keypair} in #{new_keypair_path}"
+          Kernel.system "ec2-add-keypair #{keypair} > #{new_keypair_path} && chmod 600 #{new_keypair_path}"
+        end
+      end
+    
+      # wrapper for remote base to perform a snapshot backup for the ebs volume
+      def create_snapshot
+        return nil if ebs_volume_id.nil?
+        ec2.create_snapshot(:volume_id => ebs_volume_id)
+      end
+    
+      def has_cert_and_key?
+        pub_key && private_key
+      end
+      # The keys are used only for puppet certificates
+      # and are only used for EC2.
+      # Public key 
+      def pub_key
+        @pub_key ||= ENV["EC2_CERT"] ? ENV["EC2_CERT"] : nil
+      end
+      # Private key
+      def private_key
+        @private_key ||= ENV["EC2_PRIVATE_KEY"] ? ENV["EC2_PRIVATE_KEY"] : nil
+      end
+    
+      def custom_minimum_runnable_options
+        [:ami, :availabilty_zone, :security_group]
+      end
+
+      # Hook
+      #TODO#: Change this so they match with the cap tasks
+      def custom_install_tasks_for(o)        
+        [
+          # "if [ -z $(grep -v '#' /etc/hosts | grep '#{o.name}') ]; then echo \"$(curl http://169.254.169.254/latest/meta-data/public-ipv4) #{o.name}\" >> /etc/hosts; fi",
+          "if [ -z \"$(grep -v '#' /etc/hosts | grep '#{o.name}')\" ]; then echo '127.0.0.1 #{o.name}' >> /etc/hosts; fi",
+          "hostname #{o.name}",
+          "echo #{o.name} > /etc/hostname"
+        ]
+      end
+    
+      def after_install_tasks_for(o)
+        [
+          # "cd /var/poolparty && wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1",
+          # "/usr/bin/gem install --no-ri --no-rdoc amazon-ec2.gem 2>&1"
+        ]
+      end
+
+      def custom_configure_tasks_for(o)
+        [
+        ]
+      end
+
+      def reset_base!
+        @describe_instances = @cached_descriptions = nil
+      end      
+    end
+        
+  end
+end

data/lib/poolparty/net/remoter_bases/ec2/ec2_remote_instance.rb

@@ -0,0 +1,53 @@
+module PoolParty  
+  module Remote
+    class Ec2RemoteInstance < RemoteInstance
+      include Dslify
+      
+      attr_reader :my_cloud, :uniquely_identifiable_by, :found_at
+
+      
+      # A new instance will be created from the passed in hash.  
+      # This hash of passed in values will be converted to methods on this instance.
+      # The parent clouds describe_instances list will be searched for the first one matching any of this instance's provided unique identifiers.
+      # If an instance is found, this instance's properties will be set to the properties provided
+      # If the found instance has properties of the same key as the provided options, the found instance's values will override the passed in options
+      def initialize(opts={}, prnt=Ec2.new)
+        @uniquely_identifiable_by = [:ip, :name, :dns_name, :instance_id]
+        @original_options = opts
+        @my_cloud = prnt
+        super(opts, prnt)
+        find_myself(@uniquely_identifiable_by && opts.keys) if prnt.respond_to?(:describe_instances)
+      end
+   
+      # Search the clouds describe_instances list for the first match on one of this nodes unique identifiers
+      def find_myself(unique_identifiers = @original_options.keys)
+        description_hash = @my_cloud.describe_instances.detect do|node|
+          unique_identifiers.detect{|identifier_key| node[identifier_key] == options[identifier_key]
+            }
+        end
+        return nil if description_hash.blank?
+        @found_at = Time.now
+        self.set_vars_from_options(description_hash)
+        self
+      end
+      
+      # Is this instance running?
+      def running?
+        !(status =~ /running/).nil?
+      end
+      # Is this instance pending?
+      def pending?
+        !(status =~ /pending/).nil?
+      end
+      # Is this instance terminating?
+      def terminating?
+        !(status =~ /shutting/).nil?
+      end
+      # Has this instance been terminated?
+      def terminated?
+        !(status =~ /terminated/).nil?
+      end
+      
+    end
+  end
+end

data/lib/poolparty/net/{remote_bases → remoter_bases}/ec2/ec2_response_object.rb

@@ -1,3 +1,5 @@
+# Convenience class to convert standard amazon-ec2 responses from their camel cased style to a hash using underscore style.
+# For example: instanceId to instance_id
 class EC2ResponseObject
   def self.get_descriptions(resp)      
     rs = get_instance_from_response(resp)
@@ -48,7 +50,6 @@ class EC2ResponseObject
     group
   end
   def self.get_hash_from_response(resp, group = 'default')
-    begin
       {
         :instance_id => resp.instanceId,
         :name => resp.instanceId, 
@@ -59,8 +60,5 @@ class EC2ResponseObject
         :keypair => resp.keyName,
         :security_group => group
       }        
-    rescue Exception => e
-      nil
-    end      
   end
 end

data/lib/poolparty/net/remoter_bases/vmrun/utilities/vm_disk.rb

@@ -0,0 +1,12 @@
+class VmxDisk
+  include Dslify
+  
+  def initialize(o={})
+    dsl_options o
+  end
+  
+  def compile
+    %x[qemu-img create -f vmdk #{base_directory}/#{name}/#{name}.vmdk #{options[:vmx_disk][:image_size]} ]
+    "#{base_directory}/#{name}/#{name}.vmdk"
+  end
+end

data/lib/poolparty/net/remoter_bases/vmrun/utilities/vmx.rb

@@ -0,0 +1,33 @@
+=begin rdoc
+  Vmx file creator
+  
+  Usage:
+    Vmx.new({
+      :name => "MyPP",
+     :vmx_disk => {
+       :image_size => "2G"
+     },
+     :base_directory => "/tmp/vmx"
+    }).compile
+=end
+class Vmx
+  include Dslify
+  
+  default_options(
+    :name => "PoolParty",
+    :base_directory => "~/Documents/Virtual\ Machines.localized"
+  )
+  
+  def initialize(o={})
+    dsl_options o
+    @vmx_file = VmxFile.new o
+    @vmx_disk = VmxDisk.new o    
+  end
+  
+  def compile
+    ::FileUtils.mkdir_p "#{base_directory}/#{options[:name]}"
+    vmdk = @vmx_disk.compile
+    @vmx_file.set("ide0:0.fileName", vmdk)
+    @vmx_file.compile    
+  end
+end