fairchild-poolparty 1.1.4 → 1.1.5

data/lib/poolparty/net/remoter_base.rb

@@ -23,19 +23,25 @@ module PoolParty
     # This class is the base class for all remote types, such as ec2
     # Everything remoting-wise is derived from this class
     class RemoterBase
-      include  Remote
+      include               Dslify
+      include  ::PoolParty::Remote
       
-      def initialize(prnt = nil)
-        @parent = prnt
-      end
+      attr_reader :cloud
       
-      def method_missing(meth, *args, &block)
-        if @parent
-          @parent.send meth, *args, &block rescue super
-        else
-          super
-        end
+      def initialize(prnt, opts={}, &block)
+        dsl_options prnt.options.merge(opts) if prnt && prnt.respond_to?(:options)
+        instance_eval &block if block
+        @cloud = prnt
       end
+
+      
+      # def method_missing(meth, *args, &block)
+      #   if @cloud
+      #     @cloud.send meth, *args, &block rescue super
+      #   else
+      #     super
+      #   end
+      # end
       
       # Required methods
       # The next methods are required on all RemoteInstance types
@@ -43,38 +49,105 @@ module PoolParty
       # An exception will be raised and poolparty will explode into tiny little 
       # pieces. Don't forget to overwrite these methods
       # Launch a new instance
-      def self.launch_new_instance!(o={})
-        raise RemoteException.new(:method_not_defined, "launch_new_instance!")
+      def self.launch_new_instance!(cld, o={})
+        new(cld, o).launch_new_instance!(o)
       end
-      def self.expand(o={});launch_new_instance!(o);end
       def launch_new_instance!(o={})
-        self.class.launch_new_instance!( options.merge(o) )
+        raise RemoteException.new(:method_not_defined, "launch_new_instance!")        
       end
       
       # Terminate an instance by id
-      def self.terminate_instance!(o={})
-        raise RemoteException.new(:method_not_defined, "terminate_instance!")
+      def self.terminate_instance!(cld, o={})
+        new(cld, o).terminate_instance!(o)
       end
-      def self.contract(o={});terminate_instance!(o);end      
-      def terminate_instance!(o={})
-        self.class.terminate_instance!(o ? options.merge(o) : options)
+      def terminate_instance!(o={})        
+        raise RemoteException.new(:method_not_defined, "terminate_instance!")
       end
       
       # Describe an instance's status
-      def self.describe_instance(o={})
-        raise RemoteException.new(:method_not_defined, "describe_instance")
+      def self.describe_instance(cld, o={})
+        new(cld, o).describe_instance(o) 
       end
       def describe_instance(o={})
-        self.class.describe_instance(o ? options.merge(o) : options)
+        raise RemoteException.new(:method_not_defined, "describe_instance")
       end
       
       # Get instances
       # The instances must have a status associated with them on the hash
-      def self.describe_instances(o={})
+      def self.describe_instances(cld, o={})
+        new(cld, o).describe_instances(o)
+      end
+      def describe_instances(o={})        
         raise RemoteException.new(:method_not_defined, "describe_instances")
       end
-      def describe_instances(o={})
-        self.class.describe_instances(o ? options.merge(o) : options)
+      
+      # TODO: Rename and modularize the @inst.status =~ /pending/ so that it works on all 
+      # remoter_bases
+      def launch_instance!(o={}, &block)
+        @inst = launch_new_instance!( o )
+        sleep(2)
+        
+        cloud.dputs "#{cloud.name} launched instance checking for ip..."
+        
+        # Wait for 10 minutes for the instance to gain an ip if it doesn't already have one
+        500.times do |i|
+          break if @inst[:ip] =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/
+          sleep(2)
+          @inst = describe_instance(@inst)
+          cloud.dprint "."
+        end        
+        cloud.dputs "Found an ip"
+        cloud.dputs "#{@cloud.name} Launched instance #{@inst[:ip]}"
+        cloud.dputs "   waiting for it to respond"
+        
+        # Try for 10 minutes to pint port 22 
+        500.times do |i|
+          cloud.dprint "."
+          if ping_port(@inst[:ip], 22)
+            cloud.dputs ""
+            cloud.started_instance = @inst
+            
+            cloud.call_after_launch_instance_callbacks(@inst)
+            block.call(@inst) if block
+            # after_launch_instance(@inst)
+            cloud.call_after_launch_instance_callbacks(@inst)
+            return @inst
+          end
+          sleep(2)
+        end
+        raise "Instance not responding at #{inst.ip}"
+      end
+      
+      def self.launch_instance!(cld, o={}, &block)
+        new(cld, o, &block).launch_instance!
+      end
+
+      # Called after an instance is launched
+      def after_launch_instance(instance=nil)
+        puts "after_launch_instance in remoter_base"
+      end
+
+      #TODO: Remove
+      # def self.when_instance_is_responding(inst, &block)
+      #   if ping_port(inst.ip, 22)
+      #     block.call if block
+      #   else
+      #     raise "Instance not responding at #{inst.ip}"
+      #   end
+      # end
+      # def when_instance_is_responding(inst, &block);self.class.when_instance_is_responding;end
+      
+      # TODO: BAD FORM, already defined in connections.rb. Fix this, ASAP
+      def self.ping_port(host, port=22, retry_times=400)
+        connected = false
+        retry_times.times do |i|
+          begin
+            break if connected = TCPSocket.new(host, port).is_a?(TCPSocket)
+          rescue Exception => e
+            sleep(2)
+          end
+        end
+        connected
       end
       
       # After launch callback

data/lib/poolparty/net/remoter_bases/ec2/ec2.rb

@@ -1,8 +1,7 @@
 =begin rdoc
   EC2 Remoter Base
   
-  This serves as the basis for running PoolParty on Amazon's ec2 cloud
-  cluster. 
+  This serves as the basis for running PoolParty on Amazon's ec2 cloud.
 =end
 require "date"
 require "#{::File.dirname(__FILE__)}/ec2_response_object"
@@ -36,19 +35,27 @@ end
 module PoolParty    
   module Remote
     class Ec2 < Remote::RemoterBase
-
-      def self.launch_new_instance!(o = options)
-        raise "You must pass a keypair to launch an instance, or else you wont be able to login. options = #{o.inspect}" if !o[:keypair]
-        instance = ec2(o).run_instances(
-          :image_id => o[:ami],
-          :user_data => o[:user_data],
-          :minCount => 1,
-          :maxCount => o[:num],
-          :key_name => o[:keypair],
-          :availability_zone => o[:availabilty_zone],
-          :instance_type => o[:size],
-          :group_id => o[:security_group])
-
+      
+      default_options({
+        :image_id => 'ami-bf5eb9d6',
+        # :key_name => ::File.basename(keypair.is_a?(String) ? keypair : keypair.full_filepath),
+        :instance_type => 'm1.small', # or 'm1.large', 'm1.xlarge', 'c1.medium', or 'c1.xlarge'
+        :addressing_type => "public",
+        :availabilty_zone => "us-east-1a",
+        :security_group => ["default"]
+        })
+      
+      # Requires a hash of options
+      def self.launch_new_instance!(parent_cloud, o)
+        new(parent_cloud, o).launch_new_instance!
+      end
+      
+      # TODO: Fix the key_name issue
+      # Start a new instance with the given options
+      def launch_new_instance!(o={})
+        raise "You must pass a keypair to launch an instance, or else you will not be able to login. options = #{o.inspect}" if !cloud.keypair
+        o.merge!( options ).merge!(:key_name=>keypair.basename)
+        instance = ec2(o).run_instances(o)
         begin
           h = EC2ResponseObject.get_hash_from_response(instance.instancesSet.item.first)
           #h = instance.instancesSet.item.first
@@ -59,17 +66,17 @@ module PoolParty
         h
       end
       # Terminate an instance by id
-      def self.terminate_instance!(o={})  #NOTE: maybe we should not allow this command wihtout an instance_idˇ
+      def terminate_instance!(o={})
         ec2(o).terminate_instances(:instance_id => o[:instance_id])
       end
       # Describe an instance's status
-      def self.describe_instance(o={})
+      def describe_instance(o={})
         return describe_instances.first if o[:instance_id].nil?
         describe_instances.detect {|a| a[:name] == o[:instance_id] || a[:ip] == o[:instance_id] || a[:instance_id] == o[:instance_id] }
       end
-      def self.describe_instances(o={})
+      def describe_instances(o={})
         id = 0
-        get_instances_description(o).each_with_index do |h,i|
+        get_instances_description(options.merge(o)).each_with_index do |h,i|          
           if h[:status] == "running"
             inst_name = id == 0 ? "master" : "node#{id}"
             id += 1
@@ -83,24 +90,39 @@ module PoolParty
             :index => i,  #TODO MF get the instance id from the aws result instead
             :launching_time => (h[:launching_time])
           })
-        end.sort {|a,b| a[:index] <=> b[:index] }
+        end.compact.sort {|a,b| a[:index] <=> b[:index] }
       end
       
-      def self.ec2(o={})
+      # ===================================
+      # = Ec2 Specific methods below here =
+      # ===================================
+      
+      # return or create a new base EC2 connection object that will actually connect to ec2
+      def ec2(o={})
         @ec2 ||= EC2::Base.new( :access_key_id => o[:access_key], 
                                 :secret_access_key => o[:secret_access_key]
                               )
       end
+      def self.ec2(o)
+        @ec2 ||= self.class.ec2(o)
+      end
+      
       # Get the ec2 description for the response in a hash format
-      def self.get_instances_description(o={})
-        EC2ResponseObject.get_descriptions(ec2(o).describe_instances)
+      def get_instances_description(o={})
+        #TODO: only use keypair.full_filepath
+        key_hash = {:keypair => ::File.basename(keypair.is_a?(String) ? keypair : keypair.full_filepath)}
+        EC2ResponseObject.get_descriptions(ec2(o).describe_instances).select_with_hash(key_hash)
       end
       def get_descriptions(o={})
         self.class.get_descriptions(o)
       end
       
+      def keypair
+        cloud.keypair
+      end
+      
       # Class method helpers
-      def self.aws_keys
+      def aws_keys
         unless @access_key && @secret_access_key          
           aws_keys = {}
           aws_keys = YAML::load( File.open('/etc/poolparty/aws_keys.yml') ) rescue 'No aws_keys.yml file.   Will try to use enviornment variables'
@@ -109,25 +131,11 @@ module PoolParty
         end
         [@access_key, @secret_access_key]
       end
-
-      def after_launch_master(inst=nil)
-        instance = master
-        vputs "Running tasks after launching the master"
-        begin
-          # when_no_pending_instances do
-            if instance
-              attach_volume(instance)
-              # Let's associate the address LAST so that we can still connect to the instance
-              # for the other tasks here
-              associate_address(instance)
-              reset_remoter_base!
-            end
-          # end
-        rescue Exception => e        
-          vputs "Error in after_launch_master: #{e}"
+      
+      def after_launch_instance(inst)
+        if inst
+          associate_address(inst)
         end
-        reset_remoter_base!
-        when_all_assigned_ips {wait "5.seconds"}
       end
     
       # Attach a volume to the instance
@@ -139,13 +147,41 @@ module PoolParty
           ec2.attach_volume(:volume_id => ebs_volume_id, :instance_id => instance.instance_id, :device => ebs_volume_device) if ebs_volume_id && ebs_volume_mount_point
         end
       end
+      
       # Associate an address with the instance using ec2
-      # DEPRECATE relies on master
+      # Get the next_unused_elastic_ip
+      # and if there is one, associate the instance to the 
+      # public ip
       def associate_address(instance=nil)
-        if set_master_ip_to
-          dputs "Associating master with #{set_master_ip_to}"
-          instance = master
-          ec2.associate_address(:instance_id => instance.instance_id, :public_ip => set_master_ip_to) if set_master_ip_to
+        if ip = next_unused_elastic_ip
+          vputs "Associating #{instance.instance_id} with #{ip}"
+          ec2.associate_address(:instance_id => instance.instance_id, :public_ip => ip)
+        end
+      end
+      
+      # Get the next usable elastic ip
+      # First, get the list of addresses from ec2 that the client
+      # has access to, then select only the ones that are not associated
+      # with an instance.
+      # If the cloud has set elastic_ips to use, then, using the 
+      # intersection of the unused ips and those, find the first one available
+      # and return that, otherwise, return the first elastic ip available
+      def next_unused_elastic_ip
+        # [{"instanceId"=>nil, "publicIp"=>"174.129.212.93"}, {"instanceId"=>nil, "publicIp"=>"174.129.212.94"}]
+        if addressesSet = ec2(options).describe_addresses["addressesSet"]
+          begin
+            empty_addresses = addressesSet["item"].select {|i| i["instanceId"].nil? }
+            ips = empty_addresses.map {|addr| addr["publicIp"]}
+            if cloud.elastic_ips?
+              ips_to_use = cloud.elastic_ips & ips
+              ips_to_use.first
+            else
+              ips.first
+            end
+          rescue Exception => e
+            puts "Error: #{e}"
+            nil
+          end          
         end
       end
 
@@ -166,25 +202,6 @@ module PoolParty
         ec2.create_snapshot(:volume_id => ebs_volume_id)
       end
     
-      # EC2 connections
-      def ec2(o={})
-        @ec2 ||= self.class.ec2(o)
-      end
-    
-      # These are tasks that run before the configuration runs
-      def before_configuration_tasks
-        if set_master_ip_to && master.ip && master.ip.to_s != set_master_ip_to.to_s
-          associate_address(master)
-          reset_remoter_base!
-      
-          when_no_pending_instances do
-            when_all_assigned_ips do
-              vputs "Associated master with #{set_master_ip_to}"
-            end
-          end
-        end
-      
-      end
       def has_cert_and_key?
         pub_key && private_key
       end

data/lib/poolparty/net/remoter_bases/vmrun/utilities/vm_disk.rb

@@ -0,0 +1,12 @@
+class VmxDisk
+  include Dslify
+  
+  def initialize(o={})
+    dsl_options o
+  end
+  
+  def compile
+    %x[qemu-img create -f vmdk #{base_directory}/#{name}/#{name}.vmdk #{options[:vmx_disk][:image_size]} ]
+    "#{base_directory}/#{name}/#{name}.vmdk"
+  end
+end

data/lib/poolparty/net/remoter_bases/vmrun/utilities/vmx.rb

@@ -0,0 +1,33 @@
+=begin rdoc
+  Vmx file creator
+  
+  Usage:
+    Vmx.new({
+      :name => "MyPP",
+     :vmx_disk => {
+       :image_size => "2G"
+     },
+     :base_directory => "/tmp/vmx"
+    }).compile
+=end
+class Vmx
+  include Dslify
+  
+  default_options(
+    :name => "PoolParty",
+    :base_directory => "~/Documents/Virtual\ Machines.localized"
+  )
+  
+  def initialize(o={})
+    dsl_options o
+    @vmx_file = VmxFile.new o
+    @vmx_disk = VmxDisk.new o    
+  end
+  
+  def compile
+    ::FileUtils.mkdir_p "#{base_directory}/#{options[:name]}"
+    vmdk = @vmx_disk.compile
+    @vmx_file.set("ide0:0.fileName", vmdk)
+    @vmx_file.compile    
+  end
+end

data/lib/poolparty/net/remoter_bases/vmrun/utilities/vmx_file.rb

@@ -0,0 +1,117 @@
+class VmxFile
+  include Dslify
+  
+  default_options(
+    ".encoding" => "UTF-8",
+    "config.version" => "8",
+    "virtualHW.version" => "6",
+    "guestOS" => "other26xlinux",
+    "displayName" => "PoolParty vmx",
+    "vumvcpus" => "1",
+    "memsize" => "256",
+    "MemAllowAutoScaleDown" => "TRUE",
+    "MemTrimRate" => "-1",
+    "gui.powerOnAtStartup" => "FALSE",
+    "gui.fullScreenAtPowerOn" => "FALSE",
+    "gui.exitAtPowerOff" => "FALSE",
+    "uuid.action" => "create",
+    # Settings for VMware Tools
+    "tools.remindInstall" => "FALSE",
+    "tools.upgrade.policy" => "upgradeAtPowerCycle",
+    # Startup hints interfers with automatic startup of a virtual machine
+    # This setting has no effect in VMware Player
+    "hints.hideAll" => "TRUE",
+
+    # Enable time synchronization between computer
+    # and virtual machine
+    "tools.syncTime" => "TRUE",
+
+    # USB settings
+    # This config activates USB
+    "usb.present" => "TRUE",
+    "usb.generic.autoconnect" => "FALSE",
+
+    # First serial port, physical COM1 is available
+    "serial0.present" => "FALSE",
+    "serial0.fileName" => "Auto Detect",
+    "serial0.autodetect" => "TRUE",
+    "serial0.hardwareFlowControl" => "TRUE",
+
+    # Optional second serial port, physical COM2 is not available
+    "serial1.present" => "FALSE",
+
+    # First parallel port, physical LPT1 is available
+    "parallel0.present" => "FALSE",
+    "parallel0.fileName" => "Auto Detect",
+    "parallel0.autodetect" => "TRUE",
+    "parallel0.bidirectional" => "TRUE",
+
+    # Sound settings
+    "sound.present" => "TRUE",
+    "sound.fileName" => "-1",
+    "sound.autodetect" => "TRUE",
+
+    # Logging
+    # This config activates logging, and keeps last log
+    "logging" => "TRUE",
+    "log.fileName" => "PoolParty.log",
+    "log.append" => "TRUE",
+    "log.keepOld" => "3",
+
+    # These settings decides interaction between your
+    # computer and the virtual machine
+    "isolation.tools.hgfs.disable" => "FALSE",
+    "isolation.tools.dnd.disable" => "FALSE",
+    "isolation.tools.copy.enable" => "TRUE",
+    "isolation.tools.paste.enabled" => "TRUE",
+
+    # Other default settings
+    "svga.autodetect" => "TRUE",
+    "mks.keyboardFilter" => "allow",
+    "snapshot.action" => "autoCommit",
+
+    # First network interface card
+    "ethernet0.present" => "TRUE",
+    "ethernet0.virtualDev" => "vlance",
+    "ethernet0.connectionType" => "nat",
+    "ethernet0.addressType" => "generated",
+    "ethernet0.generatedAddressOffset" => "0",
+
+    # Settings for physical floppy drive
+    "floppy0.present" => "FALSE",
+
+    # First IDE disk, size 4800Mb
+    "ide0:0.present" => "TRUE",
+    "ide0:0.fileName" => "PoolParty.vmdk",
+    "ide0:0.mode" => "persistent",
+    "ide0:0.startConnected" => "TRUE",
+    "ide0:0.writeThrough" => "TRUE",
+    
+    "checkpoint.vmState" => "",
+    "ethernet0.generatedAddress" => "00:0c:29:73:79:fb",
+    "virtualHW.productCompatibility" => "hosted",
+    "ide0:0.redo" => "",
+    "vmotion.checkpointFBSize" => "65536000",
+    "parallel0.startConnected" => "FALSE",
+    "serial0.startConnected" => "FALSE"
+  )
+  
+  def initialize(o={})
+    dsl_options({:vmx_file => default_dsl_options}.merge(o))
+  end
+  
+  def set(k,v)
+    options[:vmx_file]["#{k}"] = v
+  end
+  
+  def compile
+    ::File.open("#{base_directory}/#{name}/#{name}.vmx", "w") {|f| f << to_vmx }
+  end
+  def to_vmx
+    out = []
+    options[:vmx_file].each do |k,v|
+      out << "#{k}=\"#{v}\""
+    end
+    out.join("\n")
+  end
+end