poolparty 1.3.15 → 1.4.0

data/lib/cloud_providers/ec2/helpers/authorize.rb

@@ -0,0 +1,33 @@
+module CloudProviders
+  class Authorize < Ec2Helper
+    default_options({
+                :protocol   => "tcp",
+                :from_port  => "22",
+                :to_port    => "22",
+                :network    => "0.0.0.0/0",
+                :group_name => nil,
+                :owner_id   => nil})
+
+    def run
+      puts "Authorizing: #{cloud.proper_name} for #{protocol} to #{from_port}:#{to_port} #{network}"
+      options = 
+      if group_name
+        {:authorize_security_group_ingress => group_name, :source_security_group_owner_id => owner_id}
+      else
+        to_hash
+      end
+      ec2.authorize_security_group_ingress(options) rescue nil
+    end
+    
+    def to_hash
+      {
+        :group_name   => cloud.proper_name,
+        :ip_protocol  => protocol,
+        :from_port    => from_port,
+        :to_port      => to_port,
+        :cidr_ip      => network
+      }
+    end
+    
+  end
+end

data/lib/cloud_providers/ec2/helpers/ec2_helper.rb

@@ -0,0 +1,30 @@
+module CloudProviders
+  class Ec2Helper < CloudProvider
+    
+    def initialize(name=cloud.proper_name, init_opts={}, &block)
+      @name = name
+      if name.is_a?(Hash)
+        @name = name[:name]
+        @init_opts = name if init_opts.empty?
+      else
+        @init_opts = init_opts
+      end
+      set_vars_from_options(init_opts)
+      instance_eval &block if block
+      after_initialized
+    end
+    
+    def elb
+      cloud.elb
+    end
+    
+    def ec2
+      cloud.ec2
+    end
+    
+    def as
+      cloud.as
+    end
+    
+  end
+end

data/lib/cloud_providers/ec2/helpers/elastic_auto_scaler.rb

@@ -0,0 +1,300 @@
+module CloudProviders
+  class ElasticAutoScaler < Ec2Helper
+    default_options(
+      :cooldown => nil,
+      :desired_capacity => nil
+    )
+    def run      
+      if should_create_autoscaling_group?
+        puts "Creating autoscaling group"
+        create_launch_configuration!
+        create_autoscaling_group!
+      else
+        puts "-----> Checking for launch configuration named: #{old_launch_configuration_name}"
+        if should_create_launch_configuration?
+          create_launch_configuration!
+        elsif should_update_launch_configuration? || should_update_autoscaling_group?
+          update_launch_configuration!
+          
+          puts "Updating autoscaling group"
+          update_autoscaling_group!
+          puts "Deleting old launch configuration: #{old_launch_configuration_name}"
+          as.delete_launch_configuration(:launch_configuration_name => old_launch_configuration_name)
+        end
+        
+      end
+    end
+    
+    # First, change the min_count to 
+    def teardown
+      if autoscaling_groups.select {|n| n.name == name }.empty?
+        puts "Cloud #{cloud.name} autoscaling group does not exist"
+      else
+        self.minimum_instances = 0
+        self.maximum_instances = 0
+        @new_launch_configuration_name = old_launch_configuration_name
+        puts "Updating autoscaling group: #{@new_launch_configuration_name}"
+        update_autoscaling_group!
+        puts "Terminating nodes in autoscaling group: #{name}"
+        reset!
+        # cloud.nodes.each {|n| n.terminate! }
+        delete_autoscaling_group!
+        delete_launch_configuration!
+        puts ""
+      end
+    end
+    
+    private
+    def delete_autoscaling_group!
+      ensure_no_scaling_activities
+      reset!
+      begin
+        as.delete_autoscaling_group(:autoscaling_group_name => name)
+      rescue AWS::Error => e
+        if e.message =~ /You cannot delete an AutoScalingGroup while there are scaling activities in progress/
+          delete_autoscaling_group!
+        end
+      rescue Exception => e
+        p e.inspect
+      end
+    end
+    def delete_launch_configuration!(n=new_launch_configuration_name)
+      ensure_no_scaling_activities
+      as.delete_launch_configuration(:launch_configuration_name => n)
+    end
+    def ensure_no_scaling_activities
+      # loop do
+      #   reset!
+      #   activities = scaling_activities.select {|a| !a[:complete] }
+      #   running_nodes = cloud.nodes.select {|n| n.running? }
+      #   if activities.empty? && running_nodes.empty?
+      #     break
+      #   else
+      #     $stdout.print "."
+      #     $stdout.flush
+      #     sleep 1
+      #   end
+      # end
+      progress_bar_until do
+        reset!
+        activities = scaling_activities.select {|a| !a[:complete] }
+        running_nodes = cloud.nodes.select {|n| n.running? }
+        activities.empty? && running_nodes.empty?
+      end
+      reset!
+    end
+    public
+    def should_create_autoscaling_group?
+      known = autoscaling_groups.select {|ag| ag.name == cloud.proper_name }
+      if known.empty?
+        true
+      else
+        puts "Autoscaling group already defined...: #{name}"
+        false
+      end
+    end
+    def should_create_launch_configuration?
+      known = launch_configurations.select {|lc| lc.name =~ /#{name}/ }
+      if known.empty?
+        true
+      else
+        false
+      end
+    end
+    def should_update_launch_configuration?
+      known = launch_configurations.select {|lc| lc.name =~ /#{name}/ }
+      if known.empty?
+        true
+      else
+        differences = known.map do |k|
+          t = k.diff({
+            :image_id => image_id,
+            :instance_type => instance_type,
+            :security_groups => parent.security_group_names.flatten,
+            :key_name => keypair.to_s,
+            :user_data => user_data,
+            }, :user_data, :image_id, :instance_type, :security_groups, :key_name)
+          t.empty? ? nil : t
+        end.reject {|a| a.nil? }
+        if differences.empty?
+          false
+        else
+          true
+        end
+      end
+    end
+    def update_launch_configuration!
+      create_launch_configuration!
+    end
+    def create_launch_configuration!(lc=new_launch_configuration_name)
+      puts "-----> Creating launch configuration: #{new_launch_configuration_name} for #{proper_name}"
+      begin
+        @launch_configuration_name = lc
+        as.create_launch_configuration({
+          :launch_configuration_name => new_launch_configuration_name,
+          :image_id => image_id,
+          :instance_type => instance_type,
+          :security_groups => parent.security_group_names,
+          :key_name => keypair.to_s,
+          :user_data => user_data,
+          :kernel_id => kernel_id,
+          :ramdisk_id => ramdisk_id,
+          :block_device_mappings => block_device_mappings
+        })
+      rescue Exception => e
+        puts <<-EOE
+-----> There was an error: #{e.inspect} when creating the launch_configurations
+        EOE
+      ensure
+        reset!
+      end
+    end
+    def launch_configurations
+      @launch_configurations ||= as.describe_launch_configurations.DescribeLaunchConfigurationsResult.LaunchConfigurations.member.map do |a|
+        {
+          :name => a["LaunchConfigurationName"],
+          :ramdisk_id => a["RamdiskId"],
+          :image_id => a["ImageId"],
+          :security_groups => (a["SecurityGroups"]["member"] rescue ["default"]),
+          :created_time => a["CreatedTime"],
+          :user_data => a["UserData"] || "",
+          :key_name => a["KeyName"],
+          :instance_type => a["InstanceType"]
+        }
+      end
+    end
+    def create_autoscaling_group!
+      as.create_autoscaling_group({
+        :autoscaling_group_name => name,
+        :availability_zones => availability_zones,
+        :launch_configuration_name => new_launch_configuration_name,
+        :min_size => minimum_instances.to_s,
+        :max_size => maximum_instances.to_s,
+        :load_balancer_names => load_balancers.map {|lb| lb.name }
+      })
+      reset!
+    end
+    
+    def should_update_autoscaling_group?
+      known = autoscaling_groups.select {|lc| lc.name =~ /#{name}/ }
+      if known.empty?
+        true
+      else
+        differences = known.map do |k|
+          hsh = {
+            :min_size => minimum_instances.to_s, :max_size => maximum_instances.to_s,
+            :availability_zones => availability_zones,
+            :launch_configuration_name => old_launch_configuration_name
+          }
+          hsh.merge!(:cooldown => cooldown.to_s) if cooldown
+          
+          t = hsh.diff(k, :cooldown,
+                          :min_size,
+                          :max_size, 
+                          :availability_zones, 
+                          :launch_configuration_name)
+          t.empty? ? nil : t
+        end.reject {|a| a.nil? }
+        if differences.empty?
+          false
+        else
+          puts "-----> Recreating the autoscaling group as details have changed: #{differences.inspect}"
+          true
+        end
+      end
+    end
+    
+    def update_autoscaling_group!
+      as.update_autoscaling_group(
+        :autoscaling_group_name => name,
+        :availability_zones => availability_zones.first, # TODO: Figure out how to support multiple availability_zones
+        :launch_configuration_name => new_launch_configuration_name,
+        :min_size => minimum_instances.to_s,
+        :max_size => maximum_instances.to_s,
+        :cooldown => cooldown.to_s,
+        :load_balancer_names => load_balancers.map {|lb| lb.name }
+      )
+    end
+    
+    def autoscaling_groups
+      @autoscaling_groups ||= as.describe_autoscaling_groups.DescribeAutoScalingGroupsResult.AutoScalingGroups.member.map do |g|
+        {
+          :cooldown => g["Cooldown"],
+          :desired_capacity => g["DesiredCapacity"],
+          :created_time => g["CreatedTime"],
+          :min_size => g["MinSize"],
+          :max_size => g["MaxSize"],
+          :load_balancer_names => (g["LoadBalancerNames"]["member"] rescue []),
+          :availability_zones => (g["AvailabilityZones"]["member"] rescue []),
+          :launch_configuration_name => g["LaunchConfigurationName"],
+          :name => g["AutoScalingGroupName"],
+          :instances => (g["Instances"]["member"] rescue []).map {|i|
+            {:instance_id => i["InstanceId"],
+            :state => i["LifecycleState"],
+            :availability_zone => i["AvailabilityZone"]
+          }}
+        }
+      end rescue []
+    end
+    def scaling_activities
+      @scaling_activities ||= as.describe_scaling_activities(:autoscaling_group_name => name).DescribeScalingActivitiesResult.Activities.member.map do |action|
+        {
+          :cause        => action["Cause"],
+          :progress     => action["Progress"].to_i,
+          :activity_id  => action["ActivityId"],
+          :description  => action["Description"],
+          :status_code  => action["StatusCode"],
+          :complete     => action["StatusCode"] == "Pending" ? false : true,
+          :start_time   => action["StartTime"]
+        }
+      end rescue []
+    end
+    # Temporary names so we can create and recreate launch_configurations
+    def new_launch_configuration_name
+      @new_launch_configuration_name ||= "#{name}#{used_launched_config_id.zero? ? 1 : 0}"
+    end
+    def old_launch_configuration_name
+      @old_launch_configuration_name ||= "#{name}#{used_launched_config_id.zero? ? 0 : 1}"
+    end
+    # Compute the next configuration launch id. We'll be cycling through the usage of 0 and 1
+    # Here we are just looking for which one that is, either zero or 1
+    def used_launched_config_id
+      return @used_launched_config_id if @used_launched_config_id
+      used_configuration_names = launch_configurations.map {|hsh| hsh[:name] =~ /#{name}/ ? hsh[:name] : nil }.reject {|a| a.nil?}
+      used_launched_config_id = used_configuration_names.map {|a| a.gsub(/#{name}/, '').to_i }.reject {|a| a.zero? }.first
+      used_launched_config_id = 0 if used_launched_config_id.nil?
+      @used_launched_config_id = used_launched_config_id
+    end
+    
+    def new_auto_scaling_group_name
+      @new_auto_scaling_group_name ||= "#{name}#{used_autoscaling_group_id.zero? ? 1 : 0}"
+    end
+    
+    def old_auto_scaling_group_name
+      @old_auto_scaling_group_name ||= "#{name}#{used_autoscaling_group_id.zero? ? 0 : 1}"
+    end
+    
+    def used_autoscaling_group_id
+      return @used_autoscaling_group_id if @used_autoscaling_group_id
+      used_autoscaling_groups = launch_configurations.map {|hsh| hsh[:name] =~ /#{name}/ ? hsh[:name] : nil }.reject {|a| a.nil?}
+      used_autoscaling_group_id = used_autoscaling_groups.map {|a| a.gsub(/#{name}/, '').to_i }.reject {|a| a.zero? }.first
+      used_autoscaling_group_id = 0 if used_autoscaling_group_id.nil?
+      @used_autoscaling_group_id ||= used_autoscaling_group_id
+    end
+    
+    # Convenience methods
+    # Return an array of hashes describing the autoscaling groups
+    def list
+      describe_autoscaling_groups.DescribeAutoScalingGroupsResult.AutoScalingGroups.member
+    end
+    
+    private
+    def reset!
+      @old_auto_scaling_group_name = 
+        @new_auto_scaling_group_name = 
+          @autoscaling_groups = @scaling_activities =
+            @launch_configurations = nil
+      cloud.reset!
+    end
+  end
+end

data/lib/cloud_providers/ec2/helpers/elastic_block_store.rb

@@ -0,0 +1,9 @@
+module CloudProviders
+    class ElasticBlockStore < Ec2Helper
+  
+    def run
+      warn "ElasticBlockStore unimplemented as of now"
+    end
+  
+  end
+end

data/lib/cloud_providers/ec2/helpers/elastic_ip.rb

@@ -0,0 +1,33 @@
+module CloudProviders
+
+  class ElasticIp < Ec2Helper
+    
+    default_options(:instance_id => nil, :public_ip => nil)
+    
+    def initialize(raw_response={})
+      @raw_response = raw_response
+      self.instance_id = raw_response["instanceId"]
+      self.public_ip = raw_response["publicIp"]
+    end
+    
+    def assign_to(node)
+      puts "-----> Assigning #{public_ip} to node: #{node.instance_id}"
+    end
+    
+    def self.unused_elastic_ips(ec2_instance)
+      @unused_elastic_ips ||= elastic_ips(ec2_instance).select {|ip| ip.instance_id.nil? }
+    end
+    
+    def self.elastic_ips(ec2_instance)
+      @elastic_ips ||= ec2_instance.ec2.describe_addresses.addressesSet.item.map do |i|
+        new(i)
+      end
+    end
+    
+    def self.reset!
+      @elastic_ips = @unused_elastic_ips = nil
+    end
+    
+  end
+
+end

data/lib/cloud_providers/ec2/helpers/elastic_load_balancer.rb

@@ -0,0 +1,200 @@
+module CloudProviders
+  class ElasticLoadBalancer < Ec2Helper
+    default_options(
+      :listeners => []
+    )
+    def create!
+      if should_create_load_balancer?
+        puts "-----> Creating ElasticLoadBalancer: #{name}"
+        create_load_balancer!
+      end
+    end
+    
+    def run
+      create! # Just for now, while we migrate to 2 commands
+      if should_update_load_balancer?
+        create_load_balancer!
+      end
+      _health_checks.each do |ck|
+        configure_health_check!(ck)
+      end
+      # Remove old nodes that are no longer alive
+      detach_instances_if_necessary
+      # Try to unregister and reregister nodes that are out of service, perhaps it was just a setup bug that the setup took too long
+      out_of_service_node_listing = instance_healths.select {|a| a[:state] == "OutOfService" }.map {|a| {:instance_id => a[:instance_id]}}
+      reset!
+      out_of_service_nodes = nodes.select {|n| out_of_service_node_listing.include?(n.instance_id)}
+      unless out_of_service_nodes.empty?
+        puts "Uh oh. Out of service instance!: #{out_of_service_nodes.inspect}"
+        elb.deregister_instances_from_load_balancer(:load_balancer_name => name, :instances => out_of_service_nodes)
+        elb.register_instances_with_load_balancer(:load_balancer_name => name, :instances => out_of_service_nodes)
+      end
+      # Attach new nodes if there are any new nodes
+      attach_instances_if_necessary
+    end
+    
+    def teardown
+      puts "-----> Tearing down load balancer: #{name}"
+      elb.delete_load_balancer(:load_balancer_name => name)
+    end
+    
+    def listener(*listener_hashes)
+      listener_hashes.each do |hsh|
+        _listeners << ElasticListener.new(hsh)
+      end
+    end
+    
+    def health_check(*health_check_hashes)
+      health_check_hashes.each do |hsh|
+        _health_checks << HealthCheck.new(hsh)
+      end
+    end
+        
+    private
+    def _listeners
+      @_listeners ||= []
+    end
+    def _health_checks
+      @_health_checks ||= []
+    end
+    def real_name
+      name
+    end
+    public 
+    def attach_instances_if_necessary
+      parent.reset!
+      instances = parent.nodes.map {|a| {:instance_id => a.instance_id} }
+      elb.register_instances_with_load_balancer(:instances => instances, :load_balancer_name => "#{name}") unless instances.empty?
+    end
+    def detach_instances_if_necessary
+      parent.reset!
+      begin
+        instances = parent.all_nodes.select {|a| !a.running? }.map {|a| {:instance_id => a.instance_id} }
+        elb.deregister_instances_from_load_balancer(:instances => instances, :load_balancer_name => "#{name}") unless instances.empty?
+      rescue Exception => e
+      end
+    end
+    def should_create_load_balancer?
+      elastic_load_balancers.select {|lb| lb.name == name }.empty?
+    end
+    def create_load_balancer!
+      elb.delete_load_balancer(:load_balancer_name => name)
+      elb.create_load_balancer(
+        :availability_zones => parent.availability_zones,
+        :load_balancer_name => real_name,
+        :listeners => _listeners.map {|l| l.to_hash }
+      )
+    end
+    def configure_health_check!(hc)
+      # puts "Configuring health_check: #{hc.to_hash.inspect}"
+      elb.configure_health_check(:health_check => hc.to_hash, :load_balancer_name => name)
+    end
+    def should_update_load_balancer?
+      known = elastic_load_balancers.select {|lc| lc.name =~ /#{name}/ }.flatten
+      if known.empty?
+        true
+      else
+        known_listeners = known.map {|a| a[:listeners]}.flatten
+        # Take the known listeners (that are defined on the cloud_provider)
+        # and compare their describable values to those that are defined in
+        # the clouds.rb. Select only those that are different.
+        differences = _listeners.reject do |listener|
+          known_listeners.reject {|kl| listener.diff(kl).empty? }.empty?
+        end.flatten
+        if differences.empty?
+          false
+        else
+          true
+        end
+      end
+    end
+    
+    def running_load_balancers
+      elastic_load_balancers.select {|lc| lc.name =~ /#{name}/ }.flatten
+    end
+    def elastic_load_balancers
+      @elastic_load_balancers ||= elb.describe_load_balancers.DescribeLoadBalancersResult.LoadBalancerDescriptions.member.map do |lb|
+        {
+          :created_time => lb["CreatedTime"],
+          :availability_zones => (lb["AvailabilityZones"]["member"] rescue []),
+          :dns_name => lb["DNSName"],
+          :name => lb["LoadBalancerName"],
+          :instances => (g["Instances"]["member"] rescue []).map {|i| {:instance_id => i["InstanceId"]}},
+          :health_check => ([lb["HealthCheck"]] rescue []).map do |hc|
+            {
+              :healthy_threshold => hc["HealthyThreshold"],
+              :timeout => hc["Timeout"],
+              :unhealthy_threshold => hc["UnhealthyThreshold"],
+              :interval => hc["Interval"],
+              :target => hc["Target"]
+            }
+          end,
+          :listeners => (lb["Listeners"]["member"] rescue []).map do |listener|
+            {
+              :instance_port => listener["InstancePort"],
+              :protocol => listener["Protocol"],
+              :load_balancer_port => listener["LoadBalancerPort"]
+            }
+          end
+        }
+      end
+    end
+    def instance_healths
+      @instance_healths ||= 
+      begin
+        elb.describe_instance_health(:load_balancer_name => name).DescribeInstanceHealthResult.InstanceStates.member.map do |i|
+          {
+            :instance_id => i["InstanceId"],
+            :reason_code => i["ReasonCode"],
+            :state => i["State"],
+            :description => i["Description"]
+          }
+        end
+      rescue Exception => e
+        []
+      end
+    end
+  end
+  class HealthCheck < Ec2Helper
+    default_options(
+      :target => "HTTP:80/",
+      :interval => 5,
+      :timeout => 3,
+      :unhealthy_threshold => 2,
+      :healthy_threshold => 2
+    )
+    def initialize(name, init_opts={}, &block)
+      set_vars_from_options(name)
+      super
+    end
+    
+    def to_hash
+      {   :target => target, 
+          :interval => interval.to_s, 
+          :timeout => timeout.to_s, 
+          :unhealthy_threshold => unhealthy_threshold.to_s, 
+          :healthy_threshold => healthy_threshold.to_s}
+    end
+  end
+  class ElasticListener < Ec2Helper
+    default_options(
+      :instance_port => 80,
+      :load_balancer_port => 80,
+      :protocol => "http"
+    )
+    def initialize(name, init_opts={}, &block)
+      set_vars_from_options(name)
+      super
+    end
+    
+    def to_hash
+      {:protocol => protocol, :load_balancer_port => load_balancer_port.to_s, :instance_port => instance_port.to_s}
+    end
+    
+    def diff(hsh={})
+      [:protocol, :load_balancer_port, :instance_port].reject do |k|
+        hsh[k].to_s.capitalize == self.send(k).to_s.capitalize
+      end
+    end    
+  end
+end

data/lib/cloud_providers/ec2/helpers/revoke.rb

@@ -0,0 +1,24 @@
+module CloudProviders
+
+  class Revoke < Ec2Helper
+    default_options({
+                :protocol => "tcp",
+                :from_port => "22",
+                :to_port => "22",
+                :cidr_ip => "0.0.0.0/0"})
+    def run
+      puts "Revoking: #{cloud.proper_name} for #{protocol} to #{from_port}:#{to_port} #{network}"
+      options = { :group_name => cloud.proper_name,
+                  :ip_protocol => protocol,
+                  :from_port => from_port,
+                  :to_port => to_port,
+                  :cidr_ip => network}
+
+      ec2.revoke_security_group_ingress(options) rescue nil
+    end
+    
+    alias :network :cidr_ip
+    
+  end
+
+end