rubber 2.1.2 → 2.2.0

data/lib/rubber/cloud/aws.rb

@@ -28,19 +28,21 @@ module Rubber
       end
       
       def create_image(image_name)
+
+        # validate all needed config set
+        ["key_file", "pk_file", "cert_file", "account", "secret_access_key", "image_bucket"].each do |k|
+          raise "Set #{k} in rubber.yml" unless "#{env[k]}".strip.size > 0
+        end
+        raise "create_image can only be called from a capistrano scope" unless capistrano
+ 
         ec2_key = env.key_file
         ec2_pk = env.pk_file
         ec2_cert = env.cert_file
+
         ec2_key_dest = "/mnt/#{File.basename(ec2_key)}"
         ec2_pk_dest = "/mnt/#{File.basename(ec2_pk)}"
         ec2_cert_dest = "/mnt/#{File.basename(ec2_cert)}"
 
-        # validate all needed config set
-        ["key_file", "pk_file", "cert_file", "account", "secret_access_key", "image_bucket"].each do |k|
-          raise "Set #{k} in rubber.yml" unless "#{env[k]}".strip.size > 0
-        end
-        raise "create_image can only be called from a capistrano scope" unless capistrano
-        
         storage(env.image_bucket).ensure_bucket
         
         capistrano.put(File.read(ec2_key), ec2_key_dest)
@@ -51,7 +53,6 @@ module Rubber
         arch = case arch when /i\d86/ then "i386" else arch end
 
         capistrano.sudo_script "create_bundle", <<-CMD
-          rvm use system
           export RUBYLIB=/usr/lib/site_ruby/
           unset RUBYOPT
           nohup ec2-bundle-vol --batch -d /mnt -k #{ec2_pk_dest} -c #{ec2_cert_dest} -u #{env.account} -p #{image_name} -r #{arch} &> /tmp/ec2-bundle-vol.log &
@@ -66,7 +67,6 @@ module Rubber
         CMD
 
         capistrano.sudo_script "register_bundle", <<-CMD
-          rvm use system
           export RUBYLIB=/usr/lib/site_ruby/
           unset RUBYOPT
           echo "Uploading image to S3..."

data/lib/rubber/commands/cron.rb

@@ -64,7 +64,7 @@ module Rubber
         
         if task?
           log = "#{rootdir}/log/cron-task-#{ident}.log"
-          cmd = ["rubber"] + cmd
+          cmd = [$0] + cmd
         elsif ruby?
           ruby_code = cmd.join(' ')
           ident = ruby_code.gsub(/\W+/, "_").gsub(/(^_+)|(_+$)/, '')[0..19]

data/lib/rubber/commands/util.rb

@@ -310,6 +310,52 @@ module Rubber
       end
       
     end
+
+    class Obfuscation < Clamp::Command
+      
+      def self.subcommand_name
+        "util:obfuscation"
+      end
+
+      def self.subcommand_description
+        "Obfuscates rubber-secret.yml using encryption"
+      end
+      
+      option ["-f", "--secretfile"],
+             "SECRETFILE",
+             "The rubber_secret file\n (default: <Rubber.config.rubber_secret>)"
+      
+      option ["-k", "--secretkey"],
+             "SECRETKEY",
+             "The rubber_secret_key\n (default: <Rubber.config.rubber_secret_key>)"
       
+      option ["-d", "--decrypt"],
+             :flag,
+             "Decrypt and display the current rubber_secret"
+
+      option ["-g", "--generate"],
+             :flag,
+             "Generate a key for rubber_secret_key"
+
+      def execute
+        if generate?
+          puts "Obfuscation key: " + Rubber::Encryption.generate_encrypt_key.inspect
+          exit
+        else
+          signal_usage_error "Need to define a rubber_secret in rubber.yml" unless secretfile
+          signal_usage_error "Need to define a rubber_secret_key in rubber.yml" unless secretkey
+          signal_usage_error "The file pointed to by rubber_secret needs to exist" unless File.exist?(secretfile)
+          data = IO.read(secretfile)
+          
+          if decrypt?
+            puts Rubber::Encryption.decrypt(data, secretkey)
+          else
+            puts Rubber::Encryption.encrypt(data, secretkey)
+          end
+        end
+      end
+
+    end
+    
   end
 end

data/lib/rubber/encryption.rb

@@ -0,0 +1,46 @@
+require 'openssl'
+require 'base64'
+
+module Rubber
+  module Encryption
+    
+    def cipher_algorithm
+      OpenSSL::Cipher.new("AES-256-CBC")
+    end
+    
+    def cipher_digest
+      OpenSSL::Digest.new("SHA256")
+    end
+    
+    def generate_encrypt_key
+      OpenSSL::Digest.hexdigest('md5', rand.to_s)
+    end
+    
+    def encrypt(payload, secret)
+      cipher = cipher_algorithm
+
+      cipher.encrypt
+      cipher.pkcs5_keyivgen(cipher_digest.hexdigest(secret))
+      
+      encrypted_data = cipher.update(payload) + cipher.final
+      encoded_encrypted_data = Base64.encode64(encrypted_data)
+      
+      return encoded_encrypted_data
+    end
+    
+    def decrypt(encoded_encrypted_data, secret)
+      cipher = cipher_algorithm
+
+      cipher.decrypt
+      cipher.pkcs5_keyivgen(cipher_digest.hexdigest(secret))
+      
+      encrypted_data = Base64.decode64(encoded_encrypted_data)
+      payload = cipher.update(encrypted_data) + cipher.final
+      
+      return payload
+    end
+
+    extend self
+    
+  end
+end

data/lib/rubber/environment.rb

@@ -1,6 +1,8 @@
 require 'yaml'
 require 'socket'
 require 'delegate'
+require 'rubber/encryption'
+
 
 module Rubber
   module Configuration
@@ -28,15 +30,17 @@ module Rubber
         
         @items = {}
         @config_files.each { |file| read_config(file) }
-        @config_secret = bind().rubber_secret
-        read_config(@config_secret) if @config_secret
+
+        read_secret_config
       end
       
       def read_config(file)
         Rubber.logger.debug{"Reading rubber configuration from #{file}"}
         if File.exist?(file)
           begin
-            @items = Environment.combine(@items, YAML::load(ERB.new(IO.read(file)).result) || {})
+            data = IO.read(file)
+            data = yield(data) if block_given?
+            @items = Environment.combine(@items, YAML::load(ERB.new(data).result) || {})
           rescue Exception => e
             Rubber.logger.error{"Unable to read rubber configuration from #{file}"}
             raise
@@ -44,6 +48,21 @@ module Rubber
         end
       end
 
+      def read_secret_config
+        bound = bind()
+        @config_secret = bound.rubber_secret
+        if @config_secret
+          obfuscation_key = bound.rubber_secret_key
+          if obfuscation_key
+            read_config(@config_secret) do |data|
+              Rubber::Encryption.decrypt(data, obfuscation_key)
+            end
+          else
+            read_config(@config_secret)
+          end
+        end
+      end
+      
       def known_roles
         return @known_roles if @known_roles
         
@@ -89,7 +108,7 @@ module Rubber
         if old.is_a?(Hash) && new.is_a?(Hash)
           value = old.clone
           new.each do |nk, nv|
-            if nk[0] == '^'
+            if nk.to_s[0..0] == '^'
               nk = nk[1..-1]
               value[nk] = combine(nil, nv)
             else
@@ -212,4 +231,4 @@ module Rubber
     end
 
   end
-end
+end

data/lib/rubber/recipes/rubber.rb

@@ -63,7 +63,7 @@ namespace :rubber do
     # NOTE: for some reason Capistrano requires you to have both the public and
     # the private key in the same folder, the public key should have the
     # extension ".pub".
-    ssh_options[:keys] = cloud.env.key_file
+    ssh_options[:keys] = [cloud.env.key_file].flatten
     ssh_options[:timeout] = fetch(:ssh_timeout, 5)
   end
 

data/lib/rubber/recipes/rubber/instances.rb

@@ -79,18 +79,22 @@ namespace :rubber do
     Stop the EC2 instance for the give ALIAS
   DESC
   required_task :stop do
-    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    instance_aliases = get_env('ALIAS', "Instance alias (e.g. web01 or web01~web05,web09)", true)
+    
+    aliases = Rubber::Util::parse_aliases(instance_aliases)
     ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
-    stop_instance(instance_alias)
+    stop_instances(aliases)
   end
 
   desc <<-DESC
     Start the EC2 instance for the give ALIAS
   DESC
   required_task :start do
-    instance_alias = get_env('ALIAS', "Instance alias (e.g. web01)", true)
+    instance_aliases = get_env('ALIAS', "Instance alias (e.g. web01 or web01~web05,web09)", true)
+
+    aliases = Rubber::Util::parse_aliases(instance_aliases)
     ENV.delete('ROLES') # so we don't get an error if people leave ROLES in env from :create CLI
-    start_instance(instance_alias)
+    start_instances(aliases)
   end
 
   desc <<-DESC
@@ -465,50 +469,106 @@ namespace :rubber do
 
     cloud.reboot_instance(instance_item.instance_id)
   end
-
-  # Stops the given ec2 instance.  Note that this operation only works for instances that use an EBS volume for the root
+  
+  # Stops the given ec2 instances.  Note that this operation only works for instances that use an EBS volume for the root
   # device and that are not spot instances.
-  def stop_instance(instance_alias)
-    instance_item = rubber_instances[instance_alias]
-    fatal "Instance does not exist: #{instance_alias}" if ! instance_item
-    fatal "Cannot stop spot instances!" if ! instance_item.spot_instance_request_id.nil?
-    fatal "Cannot stop instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')
-
-    env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
-
-    value = Capistrano::CLI.ui.ask("About to STOP #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
+  def stop_instances(aliases)
+    stop_threads = []
+    
+    instance_items = aliases.collect{|instance_alias| rubber_instances[instance_alias]}
+    instance_items = aliases.collect do |instance_alias|
+      instance_item = rubber_instances[instance_alias]
+      
+      fatal "Instance does not exist: #{instance_alias}" if ! instance_item
+      fatal "Cannot stop spot instances!" if ! instance_item.spot_instance_request_id.nil?
+      fatal "Cannot stop instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')
+      
+      instance_item
+    end
+    
+    # Get user confirmation
+    human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ')
+    value = Capistrano::CLI.ui.ask("About to STOP #{human_instance_list} in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
     fatal("Exiting", 0) if value != "yes"
+    
+    instance_items.each do |instance_item|
+      logger.info "Stopping instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}"
+      
+      stop_threads << Thread.new do
+        env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
 
-    logger.info "Stopping instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"
-
-    cloud.stop_instance(instance_item.instance_id)
+        cloud.stop_instance(instance_item.instance_id)
+        
+        stopped = false
+        while !stopped
+          sleep 1
+          instance = cloud.describe_instances(instance_item.instance_id).first rescue {}
+          stopped = (instance[:state] == "stopped")
+        end
+      end
+    end
+      
+    print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to stop"
+    while true do
+      print "."
+      sleep 2
+      break unless stop_threads.any?(&:alive?)
+    end
+    print "\n"
+      
+    stop_threads.each(&:join)
   end
 
-  # Starts the given ec2 instance.  Note that this operation only works for instances that use an EBS volume for the root
+  # Starts the given ec2 instances.  Note that this operation only works for instances that use an EBS volume for the root
   # device, that are not spot instances, and that are already stopped.
-  def start_instance(instance_alias)
-    instance_item = rubber_instances[instance_alias]
-    fatal "Instance does not exist: #{instance_alias}" if ! instance_item
-    fatal "Cannot start spot instances!" if ! instance_item.spot_instance_request_id.nil?
-    fatal "Cannot start instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')
-
-    env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
-
-    value = Capistrano::CLI.ui.ask("About to START #{instance_alias} (#{instance_item.instance_id}) in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
+  def start_instances(aliases)
+    start_threads = []
+    refresh_threads = []
+    
+    instance_items = aliases.collect do |instance_alias|
+      instance_item = rubber_instances[instance_alias]
+      
+      fatal "Instance does not exist: #{instance_alias}" if ! instance_item
+      fatal "Cannot start spot instances!" if ! instance_item.spot_instance_request_id.nil?
+      fatal "Cannot start instances with instance-store root device!" if (instance_item.root_device_type != 'ebs')
+      
+      instance_item
+    end
+    
+    # Get user confirmation
+    human_instance_list = instance_items.collect{|instance_item| "#{instance_item.name} (#{instance_item.instance_id})"}.join(', ')
+    value = Capistrano::CLI.ui.ask("About to START #{human_instance_list} in mode #{Rubber.env}.  Are you SURE [yes/NO]?: ")
     fatal("Exiting", 0) if value != "yes"
-
-    logger.info "Starting instance alias=#{instance_alias}, instance_id=#{instance_item.instance_id}"
-
-    cloud.start_instance(instance_item.instance_id)
-
-    # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values.
-    print "Waiting for instance to start"
+  
+    instance_items.each do |instance_item|
+      logger.info "Starting instance alias=#{instance_item.name}, instance_id=#{instance_item.instance_id}"
+      
+      start_threads << Thread.new do
+        env = rubber_cfg.environment.bind(instance_item.role_names, instance_item.name)
+        
+        cloud.start_instance(instance_item.instance_id)
+        
+        # Re-starting an instance will almost certainly give it a new set of IPs and DNS entries, so refresh the values.
+        refresh_threads << Thread.new do
+          while ! refresh_instance(instance_item.name)
+            sleep 1
+          end
+        end
+      end
+    end
+    
+    print "Waiting for #{instance_items.size == 1 ? 'instance' : 'instances'} to start"
     while true do
       print "."
       sleep 2
-
-      break if refresh_instance(instance_alias)
+      break unless start_threads.any?(&:alive?)
     end
+
+    start_threads.each(&:join)
+    refresh_threads.each(&:join)
+    
+    # Static IPs, DNS, etc. need to be set up for the started instances
+    post_refresh
   end
 
   # delete from ~/.ssh/known_hosts all lines that begin with ec2- or instance_alias

data/lib/rubber/recipes/rubber/setup.rb

@@ -515,12 +515,17 @@ namespace :rubber do
       if reboot
 
         logger.info "Rebooting ..."
-        run("#{sudo} reboot", :hosts => reboot_hosts)
+        begin
+          run("#{sudo} reboot", :hosts => reboot_hosts)
+          # since we rebooted, teardown the connections to force cap to reconnect
+          teardown_connections_to(sessions.keys)
+        rescue
+          # swallow exception since there is a chance
+          # net:ssh throws an Exception
+        end
+        
         sleep 30
 
-        # since we rebooted, teardown the connections to force cap to reconnect
-        teardown_connections_to(sessions.keys)
-
         reboot_hosts.each do |host|
           direct_connection(host) do
             run "echo"

data/lib/rubber/recipes/rubber/static_ips.rb

@@ -19,8 +19,9 @@ namespace :rubber do
           rubber_instances.save
         end
 
-        # then, associate it if we don't have a record (on instance) of association
-        if ! ic.static_ip
+        # then, associate it if we don't have a record (on instance) of association or it
+        # doesn't match the instance's current external ip
+        if !ic.static_ip || ip != ic.external_ip
           logger.info "Associating static ip #{ip} with #{ic.full_name}"
           associate_static_ip(ip, ic.instance_id)
 

data/lib/rubber/version.rb

@@ -1,3 +1,3 @@
 module Rubber
-  VERSION = "2.1.2"
+  VERSION = "2.2.0"
 end

data/rubber.gemspec

@@ -29,8 +29,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_dependency 'capistrano', '~> 2.12'
-  # TODO: force older version of net-ssh till connectivity issues with 2.5.x get resolved
-  s.add_dependency 'net-ssh', '~> 2.4.0'
+  s.add_dependency 'net-ssh', '~> 2.6'
   s.add_dependency 'thor'
   s.add_dependency 'clamp'
   s.add_dependency 'open4'