poolparty 1.4.6 → 1.4.7

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
-:minor: 4
-:patch: 6
 :major: 1
+:minor: 4
 :build: 
+:patch: 7

data/bin/cloud-contract

@@ -19,6 +19,11 @@ EOS
       node_to_terminate = cld.nodes.last
       # node_to_terminate.cloud = cld
       
+      puts cld.nodes.size
+      if cld.nodes.size - 1 < cld.minimum_instances
+	puts "Contracting instances by 1 in cloud #{cld.name} will lower the number of instances below specified minimum (#{cld.minimum_instances})"
+	next
+      end
       msg = [
         "Contracting cloud #{cld.name} (#{cld.keypair}) by 1",
         "#{cld.nodes.size} running instances (#{cld.minimum_instances} - #{cld.maximum_instances})",
@@ -32,4 +37,4 @@ EOS
       puts "#{node_to_terminate.public_ip} has been terminated"
     end    
   end
-end
+end

data/bin/cloud-list

@@ -17,13 +17,14 @@ EOS
   short_desc "show a list of the current instances on the clouds"
 
   run do |command|
+    puts "#{pool.name}"
+    puts "==="
     @loaded_clouds.each do |cld|
       if command[:instance_id]
         require 'pp'
         pp result = cld.nodes(:instance_id=>command[:instance_id])
       else
-        line = "-----------------------------"
-        msg = ["Listing cloud #{cld.name}"]
+        msg = ["* #{cld.name} cloud,  #{cld.image_id}, #{cld.instance_type}"]
 
         hsh = {}
         
@@ -32,14 +33,12 @@ EOS
         end
         
         hsh.each do |k,v|
-          msg << "#{k.capitalize} nodes"
-          msg << line
+          msg << "  - #{k} nodes"
           v.each do |a|
-            msg << "  #{a.instance_id}\t#{a.status}\t#{a.public_ip}\t#{a.private_ip}\t#{a.dns_name}"
+            msg << "      - #{a.instance_id}\t#{a.status}\t#{a.public_ip}\t#{a.private_ip}\t#{a.zone}"
           end
           msg << ""
         end
-        
         puts msg
       end
       

data/bin/cloud-vnc

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+$:.unshift(File.dirname(__FILE__) + "/../lib")
+require "poolparty"
+
+require 'git-style-binary/command'
+
+GitStyleBinary.command do
+  banner <<-EOS
+Usage: #{$0} #{all_options_string}
+
+  start a vnc server and create an ssh tunnel
+EOS
+
+  short_desc "ssh into an instance of the cloud"
+  
+  opt :instance_number, "Start VNC server on this instance number", :type => :integer, :default => 0
+  opt :local_port, "The local port to tunnel to", :type => :integer, :default => 5900
+  opt :display, "The remote display number to start the VNC Server on", :type => :integer, :default => 0
+  
+  run do |command|
+    cloud = @loaded_clouds.first
+    
+    if !cloud.nodes.empty?
+      inst = cloud.nodes[command[:instance_number]]
+      if inst
+        inst.ssh(["x11vnc -localhost -nopw -display :#{command[:display]}"],
+                 {'-X' => nil,
+                  '-C' => nil,
+                  ' -o' => 'UserKnownHostsFile=/dev/null',
+                  '-L' => "#{command[:local_port]}:localhost:5900"})
+      else
+        puts("Error: The instance number (#{command[:instance_number]}) is too high for the current number of instances")
+      end
+    else
+      puts "No running instances can be found"
+    end
+    
+  end
+end

data/examples/rds_cloud.rb

@@ -0,0 +1,44 @@
+# example pool with RDS enabled
+
+pool :poolparty do
+  cloud :app do
+    using :ec2
+
+    # this block will create an RDS DB instance
+    #   by default, the instance id will match the containing cloud
+    #   ("poolparty-app" in this example)
+    rds do
+      username "admin"                 # required
+      password "secret"                # required
+
+      # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
+      # these properties have the following overridable defaults:
+      #
+      # storage 5
+      # instance_class "db.m1.small"
+      # engine "db.m1.small"
+
+
+
+      # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
+      # by default, a user DB will be created with a name that matches the containing cloud...
+      # ...except RDS does not allow hyphens in DB names, so it will be pool name *underscore* cloud name
+      # ("poolparty_app" in this example).
+      #
+      # to override this, use the database method, like so:
+      # database :db1
+      #   - or -
+      # databases :production, :staging   # :databases is aliased to :database for nice DSL reading.  :)
+
+
+
+      # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
+      # You are required to explicitly open access to the DB server to IP addresses (for outside EC2) or security groups (for inside EC2)
+      # access will be granted to your containing cloud's security groups unless you explicitly list security group authorization.
+      #
+      # to override, use the :authorize method, like so:
+      # authorize :networks => "1.2.3.4/24"  # <--- will authorize access to this CIDR block *in addition to* your EC2 security group
+      # authorize :networks => ["1.2.3.4/32", "10.10.10.10/24"], :security_groups => "my_ec2_group"
+    end
+  end
+end

data/lib/cloud_providers/connections.rb

@@ -31,7 +31,6 @@ module CloudProviders
     def ssh( commands=[], extra_ssh_ops={})
       commands = commands.compact.join(' && ') if commands.is_a?(Array)
       cmd_string = "ssh #{user}@#{host} #{ssh_options(extra_ssh_ops)}"
-      p commands
       if commands.empty?
         #TODO: replace this with a IO.popen call with read_nonblocking to show progress, and accept input
         Kernel.system(cmd_string)

data/lib/cloud_providers/ec2/ec2.rb

@@ -14,11 +14,11 @@ module CloudProviders
   class Ec2 < CloudProvider
     # Set the aws keys from the environment, or load from /etc/poolparty/env.yml if the environment variable is not set
     def self.default_access_key
-      ENV['EC2_ACCESS_KEY'] || load_keys_from_file[:access_key]
+      ENV['EC2_ACCESS_KEY'] || load_keys_from_file[:access_key] || load_keys_from_credential_file[:access_key]
     end
     
     def self.default_secret_access_key
-      ENV['EC2_SECRET_KEY'] || load_keys_from_file[:secret_access_key]
+      ENV['EC2_SECRET_KEY'] || load_keys_from_file[:secret_access_key] || load_keys_from_credential_file[:secret_access_key]
     end
     
     def self.default_private_key
@@ -42,7 +42,11 @@ module CloudProviders
     end
     
     def self.default_cloud_cert
-     ENV['CLOUD_CERT'] || ENV['EUCALYPTUS_CERT'] || load_keys_from_file[:cloud_cert]
+      ENV['CLOUD_CERT'] || ENV['EUCALYPTUS_CERT'] || load_keys_from_file[:cloud_cert]
+    end
+
+    def self.default_credential_file
+      ENV['AWS_CREDENTIAL_FILE'] || load_keys_from_file[:credential_file]
     end
     
     # Load the yaml file containing keys.  If the file does not exist, return an empty hash
@@ -52,6 +56,22 @@ module CloudProviders
       puts("Reading keys from file: #{filename}")
       @aws_yml = YAML::load( open(filename).read ) || {}
     end
+
+    # Load credentials from file
+    def self.load_keys_from_credential_file(filename=default_credential_file, caching=true)
+      return {:access_key => @access_key, :secret_access_key => @secret_access_key} if @access_key and @secret_access_key
+      return {} if filename.nil? or not File.exists?(filename)
+      puts("Reading keys from file: #{filename}")
+      File.open(filename).each_line {|line|
+	if line =~ /AWSAccessKeyId=([a-zA-Z0-9]+)$/
+	  @access_key=$1.chomp
+	elsif line =~ /AWSSecretKey=([^ 	]+)$/
+	  @secret_access_key=$1.chomp
+	end
+      }
+      return {:access_key => @access_key, :secret_access_key => @secret_access_key}
+    end
+      
     
     default_options(
       :instance_type          => 'm1.small',
@@ -65,6 +85,7 @@ module CloudProviders
       :secret_access_key      => default_secret_access_key,
       :ec2_url                => default_ec2_url,
       :s3_url                 => default_s3_url,
+      :credential_file	      => default_credential_file,
       :min_count              => 1,
       :max_count              => 1,
       :user_data              => '',
@@ -74,33 +95,40 @@ module CloudProviders
       :block_device_mappings  => nil,
       :ebs_volumes            => []   # The volume id of an ebs volume # TODO: ensure this is consistent with :block_device_mappings
     )
-    
+
     # Called when the create command is called on the cloud
     def create!
-      [:security_groups, :load_balancers].each do |type|
+      [:security_groups, :load_balancers, :rds_instances].each do |type|
         self.send(type).each {|ele| ele.create! }
       end
     end
-        
-    def run      
+
+    def run
       puts "  for cloud: #{cloud.name}"
       puts "  minimum_instances: #{minimum_instances}"
       puts "  maximum_instances: #{maximum_instances}"
       puts "  security_groups: #{security_group_names.join(", ")}"
       puts "  using keypair: #{keypair}"
       puts "  user: #{user}\n"
-            
+
       security_groups.each do |sg|
         sg.run
       end
-      
+
       unless load_balancers.empty?
         load_balancers.each do |lb|
           puts "    load balancer: #{lb.name}"
           lb.run
         end
       end
-      
+
+      unless rds_instances.empty?
+        rds_instances.each do |rdsi|
+          puts "    rds instance: #{rdsi.name}"
+          rdsi.run
+        end
+      end
+
       if autoscalers.empty? # not using autoscaling
         puts "---- live, running instances (#{nodes.size}) ----"
         if nodes.size < minimum_instances
@@ -191,6 +219,7 @@ module CloudProviders
     end
     
     def contract_by(num=1)
+      raise RuntimeError, "Contracting instances by #{num} will lower the number of instances below specified minimum" unless nodes.size - num > minimum_instances
       num.times do |i|
         id = nodes[-num].instance_id
         Ec2Instance.terminate!(:instance_id => id, :cloud => cloud)
@@ -248,9 +277,7 @@ module CloudProviders
     end
     
     def all_nodes
-      #TODO: need to sort by launch time
-      # 
-      @nodes ||= describe_instances.select {|i| security_group_names.include?(i.security_groups) }
+      @nodes ||= describe_instances.select {|i| security_group_names.include?(i.security_groups) }.sort {|a,b| DateTime.parse(a.launchTime) <=> DateTime.parse(b.launchTime)}
     end
     
     # Describe instances
@@ -266,6 +293,9 @@ module CloudProviders
             Ec2Instance.new(inst_options)
           end
         end.flatten
+      rescue AWS::InvalidClientTokenId => e # AWS credentials invalid
+	puts "Error contacting AWS: #{e}"
+	raise e
       rescue Exception => e
         []
       end
@@ -285,21 +315,37 @@ module CloudProviders
     def elastic_ip(*ips)
       ips.each {|ip| elastic_ips << ip}
     end
-        
+
+    def rds(given_name=cloud.proper_name, o={}, &block)
+      rds_instances << RdsInstance.new(given_name, sub_opts.merge(o || {}), &block)
+    end
+
     # Proxy to the raw Grempe amazon-aws @ec2 instance
     def ec2
-      @ec2 ||= AWS::EC2::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
+      @ec2 ||= begin
+       AWS::EC2::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
+      rescue AWS::ArgumentError => e # AWS credentials missing?
+	puts "Error contacting AWS: #{e}"
+	raise e
+      rescue Exception => e
+	puts "Generic error #{e.class}: #{e}"
+      end
     end
-    
+
     # Proxy to the raw Grempe amazon-aws autoscaling instance
     def as
       @as = AWS::Autoscaling::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
     end
-    
+
     # Proxy to the raw Grempe amazon-aws elastic_load_balancing instance
     def elb
       @elb ||= AWS::ELB::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
     end
+
+    def awsrds
+      @awsrds ||= AWS::RDS::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
+    end
+
     def security_group_names
       security_groups.map {|a| a.to_s }
     end
@@ -315,11 +361,25 @@ module CloudProviders
     def elastic_ips
       @elastic_ips ||= []
     end
-    
+
+    def rds_instances
+      @rds_instances ||= []
+    end
+
     # Clear the cache
     def reset!
       @nodes = @describe_instances = nil
     end
+
+    # Read credentials from credential_file if one exists
+    def credential_file(file=nil)
+      unless file.nil?
+	dsl_options[:credential_file]=file 
+	dsl_options.merge((Ec2.load_keys_from_credential_file(file)))
+      else
+        fetch(:credential_file)
+      end
+    end
     
     private
     # Helper to get the options with self as parent
@@ -341,18 +401,19 @@ module CloudProviders
       end
       keypair n
     end
-    
+
   end
 end
 
 require "#{File.dirname(__FILE__)}/ec2_instance"
 require "#{File.dirname(__FILE__)}/helpers/ec2_helper"
-%w( security_group 
-    authorize 
-    elastic_auto_scaler 
-    elastic_block_store 
-    elastic_load_balancer 
+%w( security_group
+    authorize
+    elastic_auto_scaler
+    elastic_block_store
+    elastic_load_balancer
     elastic_ip
+    rds_instance
     revoke).each do |lib|
   require "#{File.dirname(__FILE__)}/helpers/#{lib}"
-end
+end

data/lib/cloud_providers/ec2/ec2_instance.rb

@@ -12,7 +12,7 @@ module CloudProviders
       :availability_zones => []
     )
     
-    def initialize(raw_response={})      
+    def initialize(raw_response={})
       @raw_response = raw_response
       self.instance_id = raw_response["instanceId"] rescue nil
       self.security_groups = raw_response.groupSet.item[0].groupId rescue nil
@@ -32,6 +32,10 @@ module CloudProviders
       @keypair ||= Keypair.new(self.key_name)
     end
     
+    def zone
+      availability_zones.first
+    end
+    
     def reachable?
       ping_port self.public_ip, 22
     end
@@ -45,7 +49,7 @@ module CloudProviders
       :min_count => min_count,
       :max_count => max_count,
       :key_name => keypair.basename,
-      :group_id => cloud.security_group_names,
+      :security_group => cloud.security_group_names,
       :user_data => user_data,
       :instance_type => instance_type,
       :availability_zone => availability_zone,

data/lib/cloud_providers/ec2/helpers/ec2_helper.rb

@@ -1,6 +1,6 @@
 module CloudProviders
   class Ec2Helper < CloudProvider
-    
+
     def initialize(name=cloud.proper_name, init_opts={}, &block)
       @name = name
       if name.is_a?(Hash)
@@ -13,22 +13,35 @@ module CloudProviders
       instance_eval &block if block
       after_initialized
     end
-    
+
     def elb
       cloud.elb
     end
-    
+
     def ec2
       cloud.ec2
     end
-    
+
     def as
       cloud.as
     end
-    
+
+    def rds
+      cloud.awsrds
+    end
+
     def pool
       cloud.parent
     end
-    
+
+    def self.property(*names)
+      names.each do |name|
+        define_method name do |*args|
+          instance_variable_set("@#{name}", args.first) unless args.empty?
+          instance_variable_get("@#{name}")
+        end
+      end
+    end
+
   end
 end

data/lib/cloud_providers/ec2/helpers/elastic_load_balancer.rb

@@ -21,7 +21,7 @@ module CloudProviders
       # Remove old nodes that are no longer alive
       detach_instances_if_necessary
       # Try to unregister and reregister nodes that are out of service, perhaps it was just a setup bug that the setup took too long
-      out_of_service_node_listing = instance_healths.select {|a| a[:state] == "OutOfService" }.map {|a| {:instance_id => a[:instance_id]}}
+      out_of_service_node_listing = instance_healths.select {|a| a[:state] == "OutOfService" }.map {|a| a[:instance_id] }
       reset!
       out_of_service_nodes = nodes.select {|n| out_of_service_node_listing.include?(n.instance_id)}
       unless out_of_service_nodes.empty?
@@ -63,13 +63,13 @@ module CloudProviders
     public 
     def attach_instances_if_necessary
       parent.reset!
-      instances = parent.nodes.map {|a| {:instance_id => a.instance_id} }
+      instances = parent.nodes.map {|a| a.instance_id } # ec2 gem requires this be an array of names
       elb.register_instances_with_load_balancer(:instances => instances, :load_balancer_name => "#{name}") unless instances.empty?
     end
     def detach_instances_if_necessary
       parent.reset!
       begin
-        instances = parent.all_nodes.select {|a| !a.running? }.map {|a| {:instance_id => a.instance_id} }
+        instances = parent.all_nodes.select {|a| !a.running? }.map {|a| a.instance_id }
         elb.deregister_instances_from_load_balancer(:instances => instances, :load_balancer_name => "#{name}") unless instances.empty?
       rescue Exception => e
       end

data/lib/cloud_providers/ec2/helpers/rds_instance.rb

@@ -0,0 +1,110 @@
+module CloudProviders
+  class RdsInstance < Ec2Helper
+    property :username, :password, :storage, :instance_class, :engine
+
+    def authorize(options)
+      options.each do |key, value|
+        authorizations[key.to_s] = value
+      end
+    end
+
+    def database(*db_names)
+      @databases ||= []
+      @databases = @databases + db_names unless db_names.empty?
+      @databases
+    end
+    alias_method :databases, :database
+
+    def create!
+      if should_create_rds_instance?
+        puts "-----> Creating RDS Instance: #{instance_id}"
+        create_rds_instance!
+      end
+    end
+
+    def run
+      create! # Just for now, while we migrate to 2 commands
+      authorize_access
+      # TODO : wait until accessible?
+    end
+
+    def teardown
+      puts "-----> Tearing down RDS Instance: #{instance_id}"
+      delete_rds_instance!
+    end
+
+  private
+
+    def authorizations
+      @authorizations ||= {}
+    end
+
+    def after_initialized
+      raise "username must be specified" if self.username.nil?
+      raise "password must be specified" if self.password.nil?
+      raise "invalid password format (letters and digits only)" unless self.password =~ /[a-z][a-z0-9]*/i
+      raise "EC2 user id must be defined in ENV or config" if Ec2.default_user_id.nil?
+    end
+
+    def rds_instances
+      @rds_instances ||= (rds.describe_db_instances.DescribeDBInstancesResult.DBInstances || {})['DBInstance'] || []
+      @rds_instances = [@rds_instances] unless @rds_instances.is_a?(Array)
+      @rds_instances
+    end
+
+    def should_create_rds_instance?
+      rds_instances.select{|i| i.DBInstanceIdentifier == instance_id }.empty?
+    end
+
+    def instance_id
+      name.to_s
+    end
+
+    def create_rds_instance!
+      db_name = (databases.shift || instance_id).to_s.gsub(/\-/, '_')
+      params = {
+        :db_instance_identifier => instance_id,
+        :allocated_storage      => storage || 5,
+        :db_instance_class      => instance_class || "db.m1.small",
+        :engine                 => engine || "MySQL5.1",
+        :master_username        => username,
+        :master_user_password   => password,
+        :db_name                => db_name
+      }
+
+      # TODO : optional params : :port, :db_parameter_group, :db_security_groups, :availability_zone, :preferred_backup_window, :backend_retention_period
+      rds.create_db_instance(params)
+
+      # TODO : create additional databases
+    end
+
+    def delete_rds_instance!
+      rds.delete_db_instance(:db_instance_identifier => instance_id, :skip_final_snapshot => "true")
+    end
+
+    def authorize_access
+      authorizations[:security_groups] = cloud.security_groups.map{|sg| sg.name } if authorizations[:security_groups].nil?
+      authorizations.each do |type, values|
+        [*values].each do |value|
+          begin
+            # TODO : allow customization of db sec group name
+            params = {:db_security_group_name => "default"}
+
+            if type.to_s =~ /network/
+              puts "authorizing NET access for #{value}..."
+              params[:cidrip] = value
+            else
+              puts "authorizing SECGRP access for #{value}/#{Ec2.default_user_id}..."
+              params[:ec2_security_group_name] = value
+              params[:ec2_security_group_owner_id] = Ec2.default_user_id
+            end
+
+            rds.authorize_db_security_group(params)
+          rescue AWS::Error => e
+            raise e unless e.message =~ /Authorization already exists/
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cloud_providers/remote_instance.rb

@@ -53,9 +53,11 @@ module CloudProviders
     end
     
     def run_chef!
-      ssh([
-        "chef-solo -j /etc/chef/dna.json -c /etc/chef/solo.rb"
-      ])
+      chef_solo_cmd = <<-CMD
+        GEM_BIN=$(gem env | grep "EXECUTABLE DIRECTORY" | awk "{print \\$4}") \
+        && $GEM_BIN/chef-solo -j /etc/chef/dna.json -c /etc/chef/solo.rb
+      CMD
+      ssh([chef_solo_cmd.strip.squeeze(' ')])
     end
         
     def run

data/lib/poolparty/cloud.rb

@@ -180,6 +180,11 @@ log_level         :info
         puts "-----> Tearing down load balancer: #{lb.name}"
         lb.teardown
       end
+
+      rds_instances.each do |rds|
+        puts "-----> Tearing down RDS Instance: #{rds.name}"
+        rds.teardown
+      end
       # instances belonging to an auto_scaling group must be deleted before the auto_scaling group
       #THIS SCARES ME! nodes.each{|n| n.terminate_instance!}
       # loop {nodes.size>0 ? sleep(4) : break }
@@ -272,11 +277,11 @@ No autoscalers defined
     # Run command/s on all nodes in the cloud.
     # Returns a hash of instance_id=>result pairs
     def cmd(commands, opts={})
-      opts[:key_by]= :instance_id unless opts[:key_by]
+      key_by = opts.delete(:key_by) || :instance_id
       results = {}
       threads = nodes.collect do |n|
-        puts "result for #{n.instance_id} ==> #{n.ssh(commands, opts)}"
-         Thread.new{ results[ n.send(opts[:key_by]) ] = n.ssh(commands, opts) }
+        puts "result for #{n.instance_id} ==> n.ssh(#{commands.inspect}, #{opts.inspect})"
+        Thread.new{ results[ n.send(key_by) ] = n.ssh(commands, opts) }
       end
       threads.each{ |aThread| aThread.join }
       results

data/lib/poolparty/pool.rb

@@ -6,9 +6,21 @@ module PoolParty
     def cloud(name, &block)
       clouds[name.to_s] = Cloud.new(name.to_s, {:parent => self}, &block)
     end
+    
     def clouds
       @clouds ||= {}
     end
+    
+    # Run command/s on all nodes in the pool.
+    # Returns a hash in the form of {cloud => [{instance_id=>result}]}
+    def cmd(commands, opts={})
+      results = {}
+      threads = clouds.collect do |name, c|
+        Thread.new{ results[ name ] = c.cmd(commands, opts) }
+      end
+      threads.each{ |aThread| aThread.join }
+      results
+    end
         
     at_exit do
       if pool.auto_execute

data/test/fixtures/clouds/rds_cloud.rb

@@ -0,0 +1,14 @@
+# test RDS pool
+
+pool :poolparty do
+  cloud :fake_cloud do
+    keypair File.dirname(__FILE__)+"/../keys/test_key"
+    using :ec2
+
+    rds :db1 do
+      username "admin"
+      password "secret"
+      storage 5
+    end
+  end
+end