poolparty 1.4.7 → 1.4.8

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
+:patch: 8
 :major: 1
-:minor: 4
 :build: 
-:patch: 7
+:minor: 4

data/bin/cloud

@@ -31,7 +31,12 @@ EOS
     $DEBUGGING      = true if command[:debug]
     $VERY_DEBUGGING = true if command[:very_debug]
     
-    require command[:clouds_dot_rb]
+    begin
+      require command[:clouds_dot_rb]
+    rescue LoadError => e
+      puts "Failed loading #{command[:clouds_dot_rb]}, try using -c to specify the location of your clouds.rb"
+      exit
+    end
     
     @loaded_pool = pool
     @loaded_clouds = command[:name] ? [pool.clouds[command[:name]]] : pool.clouds.map {|name,cld|cld}    
@@ -47,4 +52,4 @@ EOS
 
     See 'cloud help COMMAND' for more information on a specific command"
   end
-end
+end

data/bin/cloud-console

@@ -16,18 +16,31 @@ EOS
 
   run do |command|    
     
-    irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
-
-    libs =  " -r irb/completion"
-    # Perhaps use a console_lib to store any extra methods I may want available in the cosole
-    # libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
-    libs <<  " -r #{File.dirname(__FILE__) + '/../lib/poolparty.rb'}"
-    f = command[:clouds_dot_rb]
-    File.open("/tmp/console.rb", "w") {|f| f << "require '#{command[:clouds_dot_rb]}'\npool.auto_execute = false" }
-    libs <<  " -r /tmp/console.rb"
-    
     puts "Loading PoolParty console..."
-    exec "#{irb} #{libs} --simple-prompt"
+    #exec "#{irb} #{libs} --simple-prompt"
+    require 'irb/completion'
+    require command[:clouds_dot_rb]
+    require "#{File.dirname(__FILE__) + '/../lib/poolparty.rb'}"
+    require 'irb'
+    pool.auto_execute = false
     
+    IRB.setup(nil)
+    IRB.conf[:IRB_NAME]="cloud"
+    IRB.conf[:PROMPT_MODE]=:CLOUD
+    IRB.conf[:PROMPT][:CLOUD]={
+      :PROMPT_I => "%N> ",
+      :PROMPT_S => "%l> ",
+      :PROMPT_C => "* ",
+      :PROMPT_N => ">> ",
+      :RETURN => "=>%s\n" }
+    irb=IRB::Irb.new
+    IRB.conf[:IRB_RC].call(irb.context) if IRB.conf[:IRB_RC]
+    IRB.conf[:MAIN_CONTEXT] = irb.context
+    trap("SIGINT") do
+      irb.signal_handle
+    end
+    catch(:IRB_EXIT) do
+      irb.eval_input
+    end
   end
-end
+end

data/lib/cloud_providers/cloud_provider.rb

@@ -7,7 +7,8 @@ module CloudProviders
     
     default_options(
       :image_id    => 'ami-ed46a784',
-      :user        => "root"
+      :user        => "root",
+      :bootstrap_gems => []
     )
     
     attr_reader :name, :init_opts
@@ -61,4 +62,4 @@ module CloudProviders
     end
     
   end
-end
+end

data/lib/cloud_providers/ec2/ec2.rb

@@ -62,12 +62,12 @@ module CloudProviders
       return {:access_key => @access_key, :secret_access_key => @secret_access_key} if @access_key and @secret_access_key
       return {} if filename.nil? or not File.exists?(filename)
       puts("Reading keys from file: #{filename}")
-      File.open(filename).each_line {|line|
-	if line =~ /AWSAccessKeyId=([a-zA-Z0-9]+)$/
-	  @access_key=$1.chomp
-	elsif line =~ /AWSSecretKey=([^ 	]+)$/
-	  @secret_access_key=$1.chomp
-	end
+      File.open(filename).each_line { |line|
+        if line =~ /AWSAccessKeyId=([a-zA-Z0-9]+)$/
+          @access_key=$1.chomp
+        elsif line =~ /AWSSecretKey=([^   ]+)$/
+          @secret_access_key=$1.chomp
+        end
       }
       return {:access_key => @access_key, :secret_access_key => @secret_access_key}
     end
@@ -92,8 +92,7 @@ module CloudProviders
       :addressing_type        => nil,
       :kernel_id              => nil,
       :ramdisk_id             => nil,
-      :block_device_mappings  => nil,
-      :ebs_volumes            => []   # The volume id of an ebs volume # TODO: ensure this is consistent with :block_device_mappings
+      :block_device_mapping  => [{}]
     )
 
     # Called when the create command is called on the cloud
@@ -174,6 +173,8 @@ module CloudProviders
       end
       
       assign_elastic_ips
+      puts "Attaching EBS volumes"
+      assign_ebs_volumes # Assign EBS volumes
     end
     
     def teardown
@@ -194,7 +195,8 @@ module CloudProviders
         :instance_type => instance_type,
         :availability_zone => availability_zones.first,
         :base64_encoded => true,
-        :cloud => cloud
+        :cloud => cloud,
+        :block_device_mapping => block_device_mapping
       })
       progress_bar_until("Waiting for node to launch...") do
         wait_for_node(e)
@@ -283,8 +285,8 @@ module CloudProviders
     # Describe instances
     # Describe the instances that are available on this cloud
     # @params id (optional) if present, details about the instance
-    #   with the id given will be returned
-    #   if not given, details for all instances will be returned
+    # with the id given will be returned
+    # if not given, details for all instances will be returned
     def describe_instances(id=nil)
       begin
         @describe_instances = ec2.describe_instances.reservationSet.item.map do |r|
@@ -303,6 +305,10 @@ module CloudProviders
     
     # Extras!
     
+    def block_device_mapping(o=[], given_name=cloud.proper_name )
+      @mappings ||= o
+    end
+
     def load_balancer(given_name=cloud.proper_name, o={}, &block)
       load_balancers << ElasticLoadBalancer.new(given_name, sub_opts.merge(o || {}), &block)
     end
@@ -362,6 +368,25 @@ module CloudProviders
       @elastic_ips ||= []
     end
 
+    def ebs_volume_groups
+      @ebs_volume_groups ||= []
+    end
+
+    # dsl method for EBS volumes. E.G.: 
+    #   ebs_volumes do
+    #     volumes "vol-001248ff", "vol-01ff4b85" # use existing volumes, not mandatory
+    #     device "/dev/sdf"
+    #     snapshot_id "snap-602030dd"
+    #     size 200
+    #   end
+    def ebs_volumes(name=nil, &block)
+      ebs_volume_groups << ElasticBlockStoreGroup.new(sub_opts,&block) 
+    end
+
+    def assign_ebs_volumes
+      ebs_volume_groups.each{|ebs_volume_group| ebs_volume_group.attach(nodes)}
+    end
+    
     def rds_instances
       @rds_instances ||= []
     end
@@ -371,11 +396,29 @@ module CloudProviders
       @nodes = @describe_instances = nil
     end
 
+    # Get existing volumes on EC2. filters is a hash of filters, either single valued or multivalued (value is an array of possible values).
+    # The function will return volumes matching *all* filters. A volume is a filter match if *any* one of the filter values equals the volume parameter value.
+    def list_ec2_volumes(filters=nil)
+      @volumes_on_ec2=ec2.describe_volumes.volumeSet.item unless @volumes_on_ec2
+      return @volumes_on_ec2 if filters.nil? # no filter to check, so return at once
+      @volumes_on_ec2.select{|vol| # select volumes for which no filter failed
+        not filters.map {|filter_key, filter_val|
+          filter_key=filter_key.to_s if filter_key.is_a?(Symbol) # filter_key may be given as a symbol
+          raise ArgumentError, "Filter key #{filter_key} is invalid" unless vol.has_key?(filter_key)
+          if filter_val.is_a?(Array) # Deal with multiple filter values
+            filter_val.map{|val| val.is_a?(String) ? val : val.to_s}.member?(vol[filter_key]) # make sure fiter_val array values are Strings before checking for match
+          else
+            filter_val.is_a?(String) ? filter_val : filter_val.to_s==vol[filter_key] # make sure fiter_val is a String before comparing
+          end
+          }.member?(false) # Check if a filter failed, the 'not' statement at the beginning of the map block negates this so 'select' will choose only when no filter failed
+      }.compact # remove nil results from volume set.
+    end
+
     # Read credentials from credential_file if one exists
     def credential_file(file=nil)
       unless file.nil?
-	dsl_options[:credential_file]=file 
-	dsl_options.merge((Ec2.load_keys_from_credential_file(file)))
+	      dsl_options[:credential_file]=file 
+	      dsl_options.merge((Ec2.load_keys_from_credential_file(file)))
       else
         fetch(:credential_file)
       end
@@ -410,7 +453,9 @@ require "#{File.dirname(__FILE__)}/helpers/ec2_helper"
 %w( security_group
     authorize
     elastic_auto_scaler
+    elastic_block_device_mapping
     elastic_block_store
+    elastic_block_store_group
     elastic_load_balancer
     elastic_ip
     rds_instance

data/lib/cloud_providers/ec2/ec2_instance.rb

@@ -9,7 +9,8 @@ module CloudProviders
       :public_ip => nil,
       :key_name => nil,
       :launch_time => nil,
-      :availability_zones => []
+      :availability_zones => [],
+      :block_device_mapping => [{}]
     )
     
     def initialize(raw_response={})
@@ -25,6 +26,7 @@ module CloudProviders
       self.launch_time = raw_response["launchTime"] rescue nil
       self.availability_zones = raw_response["placement"]["availabilityZone"] rescue nil
       self.status = raw_response["instanceState"]["name"] rescue nil
+      self.block_device_mapping = raw_response["blockDeviceMapping"] rescue nil
       super
     end
     
@@ -53,6 +55,7 @@ module CloudProviders
       :user_data => user_data,
       :instance_type => instance_type,
       :availability_zone => availability_zone,
+      :block_device_mapping => block_device_mapping,
       :base64_encoded => true)
       r.instancesSet.item.map do |i|
         inst_options = i.merge(r.merge(:cloud => cloud)).merge(cloud.cloud_provider.dsl_options)
@@ -130,4 +133,4 @@ module CloudProviders
     end
         
   end
-end
+end

data/lib/cloud_providers/ec2/helpers/ec2_helper.rb

@@ -44,4 +44,4 @@ module CloudProviders
     end
 
   end
-end
+end

data/lib/cloud_providers/ec2/helpers/elastic_block_device_mapping.rb

@@ -0,0 +1,7 @@
+module CloudProviders
+  class ElasticBlockDeviceMapping < Ec2Helper
+    def initialize(name, init_opts={}, &block)
+      super
+    end
+  end
+end

data/lib/cloud_providers/ec2/helpers/elastic_block_store.rb

@@ -1,9 +1,66 @@
 module CloudProviders
-    class ElasticBlockStore < Ec2Helper
-  
-    def run
-      warn "ElasticBlockStore unimplemented as of now"
+  class ElasticBlockStore < Ec2Helper
+
+    # instance methods
+    attr_accessor :volumeId, :size, :snapshotId, :status, :attachments, :device, :availabilityZone, :instance_id
+    attr_reader :createTime
+
+    alias :volume_id :volumeId
+    alias :snapshot_id :snapshotId
+    alias :availability_zone :availabilityZone 
+    alias :create_time :createTime
+
+    def createTime(create_time)
+      unless create_time.class==DateTime
+        @create_time=(DateTime.new(create_time) rescue nil) 
+      else
+        @createTime=create_time
+      end
+    end
+    def initialize(raw_response,init_opts={},&block)
+      parse_raw_response(raw_response)
+      super(volumeId,init_opts,&block)
+    end
+
+    def parse_raw_response(raw_response)
+      @raw_respons = raw_response
+      raw_response.each{|k,v| send k+"=", v if respond_to?(k+"=") }
+      if raw_response.attachmentSet.respond_to?(:item)
+        @attachments=raw_response.attachmentSet.item 
+        @attachments.each{|attch| instance_id=attch.instanceId if attch.status=="attached"}
+      end
+    end
+
+    def attached?(fn_instance_id=nil)
+      return false unless @status=="in-use" or @status=="attaching"
+      return true if fn_instance_id.nil?
+      return true if fn_instance_id == instance_id
+      return false
+    end
+
+    def available?
+       @status=="available"
+    end
+
+    def attach(ec2_instance,device)
+      ec2.attach_volume(:volume_id => volume_id, :instance_id => ec2_instance.instance_id, :device => device).return=="true"
+    end
+
+    def detach
+      ec2.detach_volume(:volume_id => volume_id).return=="true"
+    end
+
+    def detach!
+      ec2.detach_volume(:volume_id => volume_id, :force => true).return=="true"
+    end
+
+    def delete!
+      ec2.delete(:volume_id => volume_id).return == "true"
+    end
+
+    def update!
+      parse_raw_response ec2.describe_volumes(:volume_id => volume_id)
     end
   
   end
-end
+end

data/lib/cloud_providers/ec2/helpers/elastic_block_store_group.rb

@@ -0,0 +1,73 @@
+
+module CloudProviders
+  # ElasticBlockStoreGroup class allows easy manipulation of EBS volumes matching defined criterias.
+  # Existing volumes in *cloud*'s availability zones that match the criterias will be selected for the group. When cloud instances need to attach EBS volumes from the group, the attach method should called.
+  # When attaching volumes the ElasticBlockStoreGroup will select existing (unattached) volumes until there are non, afterwhich the group will create new volumes according to the criterias given as needed.
+  #
+  # Currently, EBS volumes will not be deleted when tearing down a cloud. This is because poolparty is stateless and thus deleting drives from it will probably result in catastroph (deletions will be too general and delete stuff you don't want deleted).
+  # Hopefully, we will come up with a scheme for a deletion flag of some sort to solve this situation.
+  class ElasticBlockStoreGroup < Ec2Helper
+
+    default_options(:device => nil, :size => 0, :snapshot_id => nil)
+    alias :snapshotId :snapshot_id
+    
+    def initialize(name=cloud.proper_name, init_opts={}, &block)
+      @volumes=[]
+      super
+    end
+    def after_initialized
+      unless @volumes.size > 0 
+        filters={:size => size, :availabilityZone => availability_zones}
+        filters[:snapshotId]=snapshot_id if snapshot_id
+        @volumes=cloud.list_ec2_volumes filters
+      end
+    end
+    def volumes(*volume_ids)
+      return @volumes if volume_ids.size==0
+      volume_ids.each{|volume_id| @volumes << cloud.list_ec2_volumes(:volumeId => volume_id)}
+    end
+    def volumes_attached_to(instanceId)
+      @volumes.select {|vol| vol.attached?(instanceId)}
+    end
+
+    # get volumes that are not attached
+    def free_volumes(availability_zone)
+      @volumes.select{|vol| vol.available? and vol.availability_zone == availability_zone}
+    end
+    # Get a free volume from existing volumes in group or create a new one
+    def get_free_volume(availability_zone)
+      free=free_volumes(availability_zone)
+      if free.size>=1
+        return free[0]
+      end
+      create(availability_zone)
+    end
+
+    # Create new volume on availability_zone
+    def create(availability_zone)
+      options={:availability_zone => availability_zone, :size => size.to_s}
+      options[:snapshot_id]=snapshot_id if snapshot_id
+      vol=ElasticBlockStore.new(ec2.create_volume(options),:cloud => cloud)
+      @volumes<<vol
+      vol
+    end
+    
+    def attach(nodes)
+      nodes.each{|node|
+        # Check no volumes are attached to node on device
+        skip_node=false
+        cloud.list_ec2_volumes.each{|vol| 
+          if vol.attached?(node.instance_id)and vol.device == device
+            warn "A volume is allready attached to device #{device} of instance #{node.instance_id}" 
+            skip_node = true
+          end
+        }
+        unless skip_node
+          vol=get_free_volume(node.zone)
+          vol.device=device
+          vol.attach(node,device) 
+        end
+      }
+    end
+  end
+end

data/lib/cloud_providers/remote_instance.rb

@@ -47,8 +47,8 @@ module CloudProviders
           'apt-get autoremove -y',
           'apt-get install -y ruby ruby-dev rubygems git-core',
           'gem sources -a http://gems.opscode.com',
-          'gem install chef ohai --no-rdoc --no-ri'
-        ])
+          'gem install chef ohai --no-rdoc --no-ri' ] + 
+          bootstrap_gems.collect { |gem| "gem install #{gem} --no-rdoc --no-ri" } )
       end
     end
     
@@ -138,4 +138,4 @@ module CloudProviders
     end
     
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: poolparty
 version: !ruby/object:Gem::Version 
-  version: 1.4.7
+  version: 1.4.8
 platform: ruby
 authors: 
 - Ari Lerner
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-13 00:00:00 -08:00
+date: 2010-01-27 00:00:00 -08:00
 default_executable: 
 dependencies: []
 
@@ -250,7 +250,9 @@ files:
 - lib/cloud_providers/ec2/helpers/authorize.rb
 - lib/cloud_providers/ec2/helpers/ec2_helper.rb
 - lib/cloud_providers/ec2/helpers/elastic_auto_scaler.rb
+- lib/cloud_providers/ec2/helpers/elastic_block_device_mapping.rb
 - lib/cloud_providers/ec2/helpers/elastic_block_store.rb
+- lib/cloud_providers/ec2/helpers/elastic_block_store_group.rb
 - lib/cloud_providers/ec2/helpers/elastic_ip.rb
 - lib/cloud_providers/ec2/helpers/elastic_load_balancer.rb
 - lib/cloud_providers/ec2/helpers/rds_instance.rb