poolparty 1.6.8 → 1.6.9

data/Rakefile

@@ -14,7 +14,17 @@ require 'config/jeweler' # setup gem configuration
 
 task :default  => [:test, :cleanup_test]
 desc "Update vendor directory and run tests"
-task :ci => ["poolparty:vendor:setup", "poolparty:vendor:update", :spec, :test]
+
+namespace :poolparty do
+    namespace :vendor do
+        desc "Fetch all the submodules"
+        task :submodules do
+            `git submodule update`
+        end
+    end
+end
+
+task :vendor => ["poolparty:vendor:submodules"]
  
 task :cleanup_test do
   ::FileUtils.rm_rf "/tmp/poolparty"
@@ -83,4 +93,4 @@ Rake::RDocTask.new do |rd|
   rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
   rd.rdoc_dir = "rdoc"
   # rd.template = "hanaa"
-end
+end

data/VERSION.yml

@@ -1,4 +1,5 @@
 --- 
-:patch: 8
 :major: 1
 :minor: 6
+:patch: 9
+:build: 

data/bin/cloud

@@ -23,6 +23,7 @@ EOS
   opt :debug, "Debug the output", :type => :boolean, :default => false
   opt :very_debug, "Set very debug mode on", :type => :boolean, :default => false
   opt :name, "Name of the working cloud", :type => String, :default => nil
+  opt :chef_task, "Name of chef task to execute", :type => String, :default => 'default'
 
   before_run do |command|
     # Setup testing/debugging
@@ -38,9 +39,11 @@ EOS
       exit
     end
     
+    pool.chef_step command[:chef_task].to_sym
+
     @loaded_pool = pool
     @loaded_clouds = command[:name] ? [pool.clouds[command[:name]]] : pool.clouds.map {|name,cld|cld}    
-    if @loaded_clouds.count == 0
+    if @loaded_clouds.size == 0
       puts "No clouds loaded. Check your clouds.rb or -n option"
       exit
     end

data/bin/cloud-run

@@ -24,8 +24,8 @@ EOS
         n.ssh(command[:command])
       end
       
-      p output
+      #p output
       
     end
   end
-end
+end

data/bin/cloud-show

@@ -6,8 +6,8 @@ require 'git-style-binary/command'
 
 GitStyleBinary.command do
   @theme = :short
-  
-  version "PoolParty #{$0} command"  
+
+  version "PoolParty #{$0} command"
   banner <<-EOS
 Usage: #{$0} #{all_options_string}
 
@@ -20,7 +20,7 @@ EOS
   run do |command|
 
     @loaded_clouds.each do |cld|
-        
+
       msg = [
         "Cloud: #{cld.name}",
         "----------------------------",
@@ -29,11 +29,12 @@ EOS
         "Maximum instances:   #{cld.maximum_instances}",
         "Running on:          #{cld.cloud_provider.name}",
         "Keypair:             #{cld.keypair.basename}",
-        "Security group:      #{cld.cloud_provider.security_group_names}",
-        "Availability zones:  #{cld.cloud_provider.availability_zones}",
-        "User:  #{cld.user}"
+        "Security group:      #{cld.cloud_provider.security_group_names.join(', ')}",
+        "Availability zones:  #{cld.cloud_provider.availability_zones.join(', ')}",
+        "User:                #{cld.user}",
+        "Active recipes:      #{cld.chef._recipes(cld.pool.chef_step).join ", " }"
       ]
-      
+
       if cld.load_balancers.size > 0
         load_balancers = cld.cloud_provider.load_balancers.first.running_load_balancers.map {|a| a[:dns_name]}
         msg << "Load balancers:      #{load_balancers.join("\n\t\t\t")}"
@@ -45,10 +46,10 @@ EOS
 
         msg << available.join("\n\t\t\t")
       end
-            
+
       puts msg.flatten
-      
+
     end
-    
+
   end
 end

data/lib/cloud_providers/connections.rb

@@ -108,7 +108,7 @@ module CloudProviders
       rsync_opts += %q% --rsync-path="sudo rsync"% unless user=="root"
       rsync_opts += %q% --exclude=.svn --exclude=.git --exclude=.cvs %
       cmd_string =  "rsync -L  -e 'ssh #{ssh_options}' #{rsync_opts} #{opts[:source]}  #{user}@#{host}:#{destination_path}"
-      out = system_run(cmd_string)
+      out = system_run(cmd_string, :quiet => true)
       out
     end
     
@@ -137,7 +137,9 @@ module CloudProviders
           while (chunk = stdin.readpartial(opts[:sysread]))
             buf << chunk
             unless chunk.nil? || chunk.empty?
-              $stdout.write(chunk) #if debugging? || verbose?
+              if not opts[:quiet]
+                $stdout.write(chunk) #if debugging? || verbose?
+              end
             end
           end
           err = stderr.readlines

data/lib/cloud_providers/ec2/ec2.rb

@@ -10,37 +10,39 @@ rescue LoadError
 EOM
 end
 
+require 'pp'
+
 module CloudProviders
   class Ec2 < CloudProvider
     # Set the aws keys from the environment, or load from /etc/poolparty/env.yml if the environment variable is not set
     def self.default_access_key
       ENV['EC2_ACCESS_KEY'] || load_keys_from_file[:access_key] || load_keys_from_credential_file[:access_key]
     end
-    
+
     def self.default_secret_access_key
       ENV['EC2_SECRET_KEY'] || load_keys_from_file[:secret_access_key] || load_keys_from_credential_file[:secret_access_key]
     end
-    
+
     def self.default_private_key
       ENV['EC2_PRIVATE_KEY'] || load_keys_from_file[:private_key]
     end
-    
+
     def self.default_cert
       ENV['EC2_CERT'] || load_keys_from_file[:cert]
     end
-    
+
     def self.default_user_id
       ENV['EC2_USER_ID'] || load_keys_from_file[:user_id]
     end
-    
+
     def self.default_ec2_url
       ENV['EC2_URL'] || load_keys_from_file[:ec2_url]
     end
-    
+
     def self.default_s3_url
       ENV['S3_URL'] || load_keys_from_file[:s3_url]
     end
-    
+
     def self.default_cloud_cert
       ENV['CLOUD_CERT'] || ENV['EUCALYPTUS_CERT'] || load_keys_from_file[:cloud_cert]
     end
@@ -48,7 +50,7 @@ module CloudProviders
     def self.default_credential_file
       ENV['AWS_CREDENTIAL_FILE'] || load_keys_from_file[:credential_file]
     end
-    
+
     # Load the yaml file containing keys.  If the file does not exist, return an empty hash
     def self.load_keys_from_file(filename="#{ENV["HOME"]}/.poolparty/aws", caching=true)
       return @aws_yml if @aws_yml && caching==true
@@ -71,8 +73,8 @@ module CloudProviders
       }
       return {:access_key => @access_key, :secret_access_key => @secret_access_key}
     end
-      
-    
+
+
     default_options(
       :instance_type          => 'm1.small',
       :availability_zones     => ["us-east-1a"],
@@ -84,7 +86,7 @@ module CloudProviders
       :secret_access_key      => default_secret_access_key,
       :ec2_url                => default_ec2_url,
       :s3_url                 => default_s3_url,
-      :credential_file	      => default_credential_file,
+      :credential_file        => default_credential_file,
       :min_count              => 1,
       :max_count              => 1,
       :user_data              => '',
@@ -109,6 +111,7 @@ module CloudProviders
       puts "  maximum_instances: #{maximum_instances}"
       puts "  security_groups: #{security_group_names.join(", ")}"
       puts "  using keypair: #{keypair}"
+      puts "  with user_data #{user_data.to_s[0..100]}"
       puts "  user: #{user}\n"
 
       security_groups.each do |sg|
@@ -152,7 +155,7 @@ module CloudProviders
           puts "    autoscaler: #{a.name}"
           puts "-----> The autoscaling groups will launch the instances"
           a.run
-          
+
           progress_bar_until("Waiting for autoscaler to launch instances") do
             reset!
             running_nodes = nodes.select {|n| n.running? }
@@ -161,8 +164,8 @@ module CloudProviders
           reset!
         end
       end
-      
-      from_ports = security_groups.map {|a| a.authorizes.map {|t| t.from_port.to_i }.flatten }.flatten      
+
+      from_ports = security_groups.map {|a| a.authorizes.map {|t| t.from_port.to_i }.flatten }.flatten
       if from_ports.include?(22)
         progress_bar_until("Waiting for the instances to be accessible by ssh") do
           running_nodes = nodes.select {|n| n.running? }
@@ -172,20 +175,20 @@ module CloudProviders
           accessible_count == running_nodes.size
         end
       end
-      
+
       assign_elastic_ips
       cleanup_ssh_known_hosts!
       puts "Attaching EBS volumes"
       assign_ebs_volumes # Assign EBS volumes
     end
-    
+
     def teardown
       puts "------ Tearing down and cleaning up #{cloud.name} cloud"
       unless autoscalers.empty?
         puts "Tearing down autoscalers"
       end
     end
-    
+
     def expand_by(num=1)
       e = Ec2Instance.run!({
         :image_id => image_id,
@@ -208,7 +211,7 @@ module CloudProviders
       end
       all_nodes.detect {|n| n.instance_id == e.instance_id }
     end
-    
+
     def decoded_user_data
       if user_data
         if File.file?(user_data)
@@ -218,13 +221,13 @@ module CloudProviders
         end
       end
     end
-    
+
     def wait_for_node(instance)
       reset!
       inst = all_nodes.detect {|n| n.instance_id == instance.instance_id }
       inst.running? if inst
     end
-    
+
     def contract_by(num=1)
       raise RuntimeError, "Contracting instances by #{num} will lower the number of instances below specified minimum" unless nodes.size - num > minimum_instances
       num.times do |i|
@@ -235,9 +238,9 @@ module CloudProviders
       end
       reset!
     end
-    
+
     def bootstrap_nodes!(tmp_path=nil)
-      unless security_groups.map {|a| a.authorizes.map {|t| t.from_port.to_i }.flatten }.flatten.include?(22)      
+      unless security_groups.map {|a| a.authorizes.map {|t| t.from_port.to_i }.flatten }.flatten.include?(22)
         warn "Cloud security_groups are not authorized for ssh. Cannot bootstrap."
         return
       end
@@ -250,18 +253,18 @@ module CloudProviders
         node.run_chef!
       end
     end
-    
+
     def configure_nodes!(tmp_path=nil)
       # removed duplicated code (now configure_nodes! invokes
       # node.bootstrap_chef!, while old version did not, but I believe
       # this is harmless)
-      bootstrap_nodes!(tmp_path) 
+      bootstrap_nodes!(tmp_path)
 
       ebs_volume_groups.each do |vol_grp|
         vol_grp.verify_attachments nodes
       end
     end
-    
+
     def assign_elastic_ips
       unless elastic_ips.empty?
         unused_elastic_ip_addresses = ElasticIp.unused_elastic_ips(self).map {|i| i.public_ip }
@@ -299,11 +302,27 @@ module CloudProviders
     def nodes
       all_nodes.select {|i| i.in_service? }#describe_instances.select {|i| i.in_service? && security_groups.include?(i.security_groups) }
     end
-    
+
+    # === Description
+    #
+    # Return all the security groups of the instance that are prefixed with #poolparty.
+    #
+    # These are special security groups used only for tagging
+    #
+    # === Parameters
+    #   instance    -   An ec2 instance as returned from describe_instances
+    def tags instance
+      instance.groupSet.item.collect{|g| g.groupId }.select {|s| s.start_with? "#poolparty"}
+    end
+
     def all_nodes
-      @nodes ||= describe_instances.select {|i| security_group_names.include?(i.security_groups) }.sort {|a,b| DateTime.parse(a.launchTime) <=> DateTime.parse(b.launchTime)}
+      @nodes ||= describe_instances.select { |i| 
+        !(security_group_names & tags(i)).empty? 
+      }.sort {|a,b| 
+        DateTime.parse(a.launchTime) <=> DateTime.parse(b.launchTime)
+      }
     end
-    
+
     # Describe instances
     # Describe the instances that are available on this cloud
     # @params id (optional) if present, details about the instance
@@ -318,15 +337,15 @@ module CloudProviders
           end
         end.flatten
       rescue AWS::InvalidClientTokenId => e # AWS credentials invalid
-	puts "Error contacting AWS: #{e}"
-	raise e
+        puts "Error contacting AWS: #{e}"
+        raise e
       rescue Exception => e
         []
       end
     end
-    
+
     # Extras!
-    
+
     def block_device_mapping(o=[], given_name=cloud.proper_name )
       @mappings ||= o
     end
@@ -357,10 +376,10 @@ module CloudProviders
       @ec2 ||= begin
        AWS::EC2::Base.new( :access_key_id => access_key, :secret_access_key => secret_access_key )
       rescue AWS::ArgumentError => e # AWS credentials missing?
-	puts "Error contacting AWS: #{e}"
-	raise e
+        puts "Error contacting AWS: #{e}"
+        raise e
       rescue Exception => e
-	puts "Generic error #{e.class}: #{e}"
+        puts "Generic error #{e.class}: #{e}"
       end
     end
 
@@ -398,7 +417,7 @@ module CloudProviders
       @ebs_volume_groups ||= []
     end
 
-    # dsl method for EBS volumes. E.G.: 
+    # dsl method for EBS volumes. E.G.:
     #   ebs_volumes do
     #     volumes "vol-001248ff", "vol-01ff4b85" # use existing volumes, not mandatory
     #     device "/dev/sdf"
@@ -412,7 +431,7 @@ module CloudProviders
     def assign_ebs_volumes
       ebs_volume_groups.each{|ebs_volume_group| ebs_volume_group.attach(nodes)}
     end
-    
+
     def rds_instances
       @rds_instances ||= []
     end
@@ -447,13 +466,13 @@ module CloudProviders
     # Read credentials from credential_file if one exists
     def credential_file(file=nil)
       unless file.nil?
-        dsl_options[:credential_file]=file 
+        dsl_options[:credential_file]=file
         dsl_options.merge!(Ec2.load_keys_from_credential_file(file))
       else
         fetch(:credential_file)
       end
     end
-    
+
     private
     # Helper to get the options with self as parent
     def sub_opts

data/lib/cloud_providers/ec2/ec2_instance.rb

@@ -1,6 +1,6 @@
 module CloudProviders
   class Ec2Instance < RemoteInstance
-    
+
     default_options(
       :security_groups => [],
       :private_ip => nil,
@@ -15,35 +15,41 @@ module CloudProviders
       :instance_initiated_shutdown_behavior => nil,
       :subnet_id => nil
     )
-    
+
     def initialize(raw_response={})
       @raw_response = raw_response
-      self.instance_id = raw_response["instanceId"] rescue nil
-      self.security_groups = raw_response.groupSet.item[0].groupId rescue nil
-      self.image_id = raw_response["imageId"] rescue nil
-      self.private_ip = raw_response["privateIpAddress"] rescue nil
-      self.dns_name = raw_response["dnsName"] rescue nil
-      self.instance_type = raw_response["instanceType"] rescue nil
-      self.public_ip = raw_response["ipAddress"] rescue nil
-      self.key_name = raw_response["keyName"] rescue nil
-      self.launch_time = raw_response["launchTime"] rescue nil
-      self.availability_zones = raw_response["placement"]["availabilityZone"] rescue nil
-      self.status = raw_response["instanceState"]["name"] rescue nil
-      self.block_device_mapping = raw_response["blockDeviceMapping"] rescue nil
-      self.disable_api_termination = raw_response["disableApiTermination"] rescue nil
-      self.instance_initiated_shutdown_behavior = raw_response["instance_initiated_shutdown_behavior"] rescue nil
-      self.subnet_id = raw_response["subnetId"] rescue nil
+      self.instance_id                          = raw_response["instanceId"] rescue nil
+      self.security_groups                      = raw_response.groupSet.item.map{|sg| sg.groupId }.sort rescue nil
+      self.image_id                             = raw_response["imageId"] rescue nil
+      self.private_ip                           = raw_response["privateIpAddress"] rescue nil
+      self.dns_name                             = raw_response["dnsName"] rescue nil
+      self.instance_type                        = raw_response["instanceType"] rescue nil
+      self.public_ip                            = raw_response["ipAddress"] rescue nil
+      self.key_name                             = raw_response["keyName"] rescue nil
+      self.launch_time                          = raw_response["launchTime"] rescue nil
+      self.availability_zones                   = raw_response["placement"]["availabilityZone"] rescue nil
+      self.status                               = raw_response["instanceState"]["name"] rescue nil
+      self.block_device_mapping                 = raw_response["blockDeviceMapping"] rescue nil
+      self.subnet_id                            = raw_response["subnetId"] rescue nil
+      # disable_api_termination and instance_initiated_shutdown_behavior don't currently get returned in the request -- you'd need to later call describe_instance_attribute
+      self.disable_api_termination              = raw_response["disableApiTermination"] rescue nil
+      self.instance_initiated_shutdown_behavior = raw_response["instanceInitiatedShutdownBehavior"] rescue nil
       super
     end
-    
+
     def keypair(n=nil)
-      @keypair ||= Keypair.new(self.key_name)
+      return @keypair if @keypair
+      @keypair = (cloud.keypair.basename == self.key_name) ? cloud.keypair : Keypair.new(self.key_name, cloud.keypair.extra_paths)
     end
-    
+
+    def security_group_names
+      security_groups.map{|a| a.to_s }
+    end
+
     def zone
       availability_zones.first
     end
-    
+
     def reachable?
       ping_port self.public_ip, 22
     end
@@ -56,38 +62,42 @@ module CloudProviders
         in_service? and
         keypair and keypair.exists?
     end
-    
+
     def in_service?
       running?
     end
-    
+
     def run!
-      r = cloud_provider.ec2.run_instances(:image_id => image_id,
-      :min_count => min_count,
-      :max_count => max_count,
-      :key_name => keypair.basename,
-      :security_group => cloud.security_group_names,
-      :user_data => user_data,
-      :instance_type => instance_type,
-      :availability_zone => availability_zone,
-      :block_device_mapping => block_device_mapping,
-      :base64_encoded => true)
+      r = cloud_provider.ec2.run_instances(
+        :image_id             => image_id,
+        :min_count            => min_count,
+        :max_count            => max_count,
+        :key_name             => keypair.basename,
+        :security_group       => cloud.security_group_names,
+        :user_data            => user_data,
+        :instance_type        => instance_type,
+        :availability_zone    => availability_zone,
+        :block_device_mapping => block_device_mapping,
+        :disable_api_termination => disable_api_termination,
+        :instance_initiated_shutdown_behavior => instance_initiated_shutdown_behavior,
+        :base64_encoded       => true)
       r.instancesSet.item.map do |i|
         inst_options = i.merge(r.merge(:cloud => cloud)).merge(cloud.cloud_provider.dsl_options)
         Ec2Instance.new(inst_options)
       end.first
     end
     def self.run!(hsh); new(hsh).run!; end
-    
+
     def terminate!
       cloud_provider.ec2.terminate_instances(:instance_id => [self.instance_id])
       cloud_provider.reset!
     end
     def self.terminate!(hsh={}); new(hsh).terminate!; end
-    
+
     # list of directories and files to exclude when bundling an instance
     def rsync_excludes(array_of_abs_paths_to_exclude=nil)
-      array_of_abs_paths_to_exclude ||= %w( /sys
+      array_of_abs_paths_to_exclude ||= %w[
+            /sys
             /proc
             /dev/pts
             /dev
@@ -99,8 +109,8 @@ module CloudProviders
             /etc/ssh/moduli
             /etc/udev/rules.d/70-persistent-net.rules
             /etc/udev/rules.d/z25_persistent-net.rules
-            )
-      array_of_abs_paths_to_exclude.inject(''){|str, path| str<<"--exclude=#{path}"; str}
+            ]
+      array_of_abs_paths_to_exclude.inject(''){|str, path| str << "--exclude=#{path}" ; str}
     end
 
     # create an image file and copy this instance to the image file.
@@ -138,7 +148,7 @@ module CloudProviders
       scp ec2cert, "/mnt/bundle/"
       scp cert, "/mnt/bundle/"
       arch = self[:instanceType].match(/m1\.small|c1\.medium/) ? 'i386' : 'x86_64'
-      image = img ? img : make_image(opts) 
+      image = img ? img : make_image(opts)
       ssh "ec2-bundle-image #{image} -d /mnt/bundle -u #{self[:ownerId]} -k /mnt/bundle/pk-*.pem -c /tmp/cert-*.pem"
       manifest = "/mnt/bundle/#{opts[:prefix]}.manifest.xml"
       ssh "ec2-upload-bundle -a #{access_key} -s #{secret_access_key} -m #{manifest}"
@@ -146,6 +156,6 @@ module CloudProviders
       ami = ami_str.grep(/ami-\w*/).first
       return ami
     end
-        
+
   end
 end