rubber 2.3.1 → 2.4.0

data/lib/rubber/recipes/rubber/instances.rb

@@ -263,16 +263,17 @@ namespace :rubber do
     role_names = instance_roles.collect{|x| x.name}
     env = rubber_cfg.environment.bind(role_names, instance_alias)
 
-    # We need to use security_groups during create, so create them up front
     mutex.synchronize do
-      setup_security_groups(instance_alias, role_names)
+      cloud.before_create_instance(instance_alias, role_names)
     end
+
     security_groups = get_assigned_security_groups(instance_alias, role_names)
 
     cloud_env = env.cloud_providers[env.cloud_provider]
     ami = cloud_env.image_id
     ami_type = cloud_env.image_type
-    availability_zone = env.availability_zone
+    availability_zone = cloud_env.availability_zone
+    region = cloud_env.region
 
     create_spot_instance ||= cloud_env.spot_instance
 
@@ -306,8 +307,8 @@ namespace :rubber do
     end
 
     if !create_spot_instance || (create_spot_instance && max_wait_time < 0)
-      logger.info "Creating instance #{ami}/#{ami_type}/#{security_groups.join(',') rescue 'Default'}/#{availability_zone || 'Default'}"
-      instance_id = cloud.create_instance(ami, ami_type, security_groups, availability_zone)
+      logger.info "Creating instance #{ami}/#{ami_type}/#{security_groups.join(',') rescue 'Default'}/#{availability_zone || region || 'Default'}"
+      instance_id = cloud.create_instance(instance_alias, ami, ami_type, security_groups, availability_zone, region)
     end
 
     logger.info "Instance #{instance_alias} created: #{instance_id}"
@@ -317,10 +318,8 @@ namespace :rubber do
     rubber_instances.add(instance_item)
     rubber_instances.save()
 
-    # Sometimes tag creation will fail, indicating that the instance doesn't exist yet even though it does.  It seems to
-    # be a propagation delay on Amazon's end, so the best we can do is wait and try again.
-    Rubber::Util.retry_on_failure(Exception, :retry_sleep => 0.5, :retry_count => 100) do
-      Rubber::Tag::update_instance_tags(instance_alias)
+    mutex.synchronize do
+      cloud.after_create_instance(instance_item)
     end
   end
 
@@ -349,9 +348,13 @@ namespace :rubber do
 
     env = rubber_cfg.environment.bind(instance_item.role_names, instance_alias)
 
-    instance = cloud.describe_instances(instance_item.instance_id).first rescue {}
+    instance = cloud.describe_instances(instance_item.instance_id).first
+
+    mutex.synchronize do
+      cloud.before_refresh_instance(instance_item)
+    end
 
-    if instance[:state] == "running"
+    if instance[:state] == cloud.active_state
       print "\n"
       logger.info "Instance running, fetching hostname/ip data"
       instance_item.external_host = instance[:external_host]
@@ -359,6 +362,7 @@ namespace :rubber do
       instance_item.internal_host = instance[:internal_host]
       instance_item.internal_ip = instance[:internal_ip]
       instance_item.zone = instance[:zone]
+      instance_item.provider = instance[:provider]
       instance_item.platform = instance[:platform]
       instance_item.root_device_type = instance[:root_device_type]
       rubber_instances.save()
@@ -367,6 +371,8 @@ namespace :rubber do
         # weird cap/netssh bug, sometimes just hangs forever on initial connect, so force a timeout
         begin
           Timeout::timeout(30) do
+            puts 'Trying to enable root login'
+
             # turn back on root ssh access if we are using root as the capistrano user for connecting
             enable_root_ssh(instance_item.external_ip, fetch(:initial_ssh_user, 'ubuntu')) if user == 'root'
             # force a connection so if above isn't enabled we still timeout if initial connection hangs
@@ -380,6 +386,10 @@ namespace :rubber do
         end
       end
 
+      mutex.synchronize do
+        cloud.after_refresh_instance(instance_item)
+      end
+
       return true
     end
     return false
@@ -387,9 +397,6 @@ namespace :rubber do
 
   def post_refresh
     env = rubber_cfg.environment.bind(nil, nil)
-
-    # update the remote name/environment tags
-    update_tags
     
     # setup amazon elastic ips if configured to do so
     setup_static_ips

data/lib/rubber/recipes/rubber/security_groups.rb

@@ -6,7 +6,9 @@ namespace :rubber do
     Likewise, rules within a group will get created, and those not will be removed
   DESC
   required_task :setup_security_groups do
-    setup_security_groups()
+    servers = find_servers_for_task(current_task)
+
+    cloud.setup_security_groups(servers.collect(&:host))
   end
 
   desc <<-DESC
@@ -37,192 +39,7 @@ namespace :rubber do
       security_groups += roles
     end
     security_groups = security_groups.uniq.compact.reject {|x| x.empty? }
-    security_groups = security_groups.collect {|x| isolate_group_name(x) }
+    security_groups = security_groups.collect {|x| cloud.isolate_group_name(x) }
     return security_groups
   end
-
-  def setup_security_groups(host=nil, roles=[])
-    env = rubber_cfg.environment.bind(roles, host)
-    security_group_defns = Hash[env.security_groups.to_a]
-    if env.auto_security_groups
-      sghosts = (rubber_instances.collect{|ic| ic.name } + [host]).uniq.compact
-      sgroles = (rubber_instances.all_roles + roles).uniq.compact
-      security_group_defns = inject_auto_security_groups(security_group_defns, sghosts, sgroles)
-      sync_security_groups(security_group_defns)
-    else
-      sync_security_groups(security_group_defns)
-    end
-  end
-
-  def inject_auto_security_groups(groups, hosts, roles)
-    hosts.each do |name|
-      group_name = name
-      groups[group_name] ||= {'description' => "Rubber automatic security group for host: #{name}", 'rules' => []}
-    end
-    roles.each do |name|
-      group_name = name
-      groups[group_name] ||= {'description' => "Rubber automatic security group for role: #{name}", 'rules' => []}
-    end
-    return groups
-  end
-
-  def isolate_prefix
-    return "#{rubber_env.app_name}_#{Rubber.env}_"
-  end
-
-  def isolate_group_name(group_name)
-    if rubber_env.isolate_security_groups
-      group_name =~ /^#{isolate_prefix}/ ? group_name : "#{isolate_prefix}#{group_name}"
-    else
-      group_name
-    end
-  end
-
-  def isolate_groups(groups)
-    renamed = {}
-    groups.each do |name, group|
-      new_name = isolate_group_name(name)
-      new_group =  Marshal.load(Marshal.dump(group))
-      new_group['rules'].each do |rule|
-        old_ref_name = rule['source_group_name']
-        if old_ref_name
-          # don't mangle names if the user specifies this is an external group they are giving access to.
-          # remove the external_group key to allow this to match with groups retrieved from cloud 
-          is_external = rule.delete('external_group')
-          if ! is_external && old_ref_name !~ /^#{isolate_prefix}/
-            rule['source_group_name'] = isolate_group_name(old_ref_name)
-          end
-        end
-      end
-      renamed[new_name] = new_group
-    end
-    return renamed
-  end
-
-  def sync_security_groups(groups)
-    return unless groups
-
-    groups = Rubber::Util::stringify(groups)
-    groups = isolate_groups(groups)
-    group_keys = groups.keys.clone()
-    
-    # For each group that does already exist in cloud
-    cloud_groups = cloud.describe_security_groups()
-    cloud_groups.each do |cloud_group|
-      group_name = cloud_group[:name]
-
-      # skip those groups that don't belong to this project/env
-      next if rubber_env.isolate_security_groups && group_name !~ /^#{isolate_prefix}/
-
-      if group_keys.delete(group_name)
-        # sync rules
-        logger.debug "Security Group already in cloud, syncing rules: #{group_name}"
-        group = groups[group_name]
-        
-        # convert the special case default rule into what it actually looks like when
-        # we query ec2 so that we can match things up when syncing
-        rules = group['rules'].clone
-        group['rules'].each do |rule|
-          if [2, 3].include?(rule.size) && rule['source_group_name'] && rule['source_group_account']
-            rules << rule.merge({'protocol' => 'tcp', 'from_port' => '1', 'to_port' => '65535' })
-            rules << rule.merge({'protocol' => 'udp', 'from_port' => '1', 'to_port' => '65535' })
-            rules << rule.merge({'protocol' => 'icmp', 'from_port' => '-1', 'to_port' => '-1' })
-            rules.delete(rule)
-          end
-        end
-        
-        rule_maps = []
-
-        # first collect the rule maps from the request (group/user pairs are duplicated for tcp/udp/icmp,
-        # so we need to do this up frnot and remove duplicates before checking against the local rubber rules)
-        cloud_group[:permissions].each do |rule|
-          source_groups = rule.delete(:source_groups)
-          if source_groups
-            source_groups.each do |source_group|
-              rule_map = rule.clone
-              rule_map.delete(:source_ips)
-              rule_map[:source_group_name] = source_group[:name]
-              rule_map[:source_group_account] = source_group[:account]
-              rule_map = Rubber::Util::stringify(rule_map)
-              rule_maps << rule_map unless rule_maps.include?(rule_map)
-            end
-          else
-            rule_map = Rubber::Util::stringify(rule)
-            rule_maps << rule_map unless rule_maps.include?(rule_map)
-          end
-        end if cloud_group[:permissions]
-        # For each rule, if it exists, do nothing, otherwise remove it as its no longer defined locally
-        rule_maps.each do |rule_map|
-          if rules.delete(rule_map)
-            # rules match, don't need to do anything
-            # logger.debug "Rule in sync: #{rule_map.inspect}"
-          else
-            # rules don't match, remove them from cloud and re-add below
-            answer = nil
-            msg = "Rule '#{rule_map.inspect}' exists in cloud, but not locally"
-            if rubber_env.prompt_for_security_group_sync
-              answer = Capistrano::CLI.ui.ask("#{msg}, remove from cloud? [y/N]: ")
-            else
-              logger.info(msg)
-            end
-
-            if answer =~ /^y/
-              rule_map = Rubber::Util::symbolize_keys(rule_map)
-              if rule_map[:source_group_name]
-                cloud.remove_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
-              else
-                rule_map[:source_ips].each do |source_ip|
-                  cloud.remove_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
-                end if rule_map[:source_ips]
-              end
-            end
-          end
-        end
-
-        rules.each do |rule_map|
-          # create non-existing rules
-          logger.debug "Missing rule, creating: #{rule_map.inspect}"
-          rule_map = Rubber::Util::symbolize_keys(rule_map)
-          if rule_map[:source_group_name]
-            cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
-          else
-            rule_map[:source_ips].each do |source_ip|
-              cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
-            end if rule_map[:source_ips]
-          end
-        end
-      else
-        # delete group
-        answer = nil
-        msg = "Security group '#{group_name}' exists in cloud but not locally"
-        if rubber_env.prompt_for_security_group_sync
-          answer = Capistrano::CLI.ui.ask("#{msg}, remove from cloud? [y/N]: ")
-        else
-          logger.debug(msg)
-        end
-        cloud.destroy_security_group(group_name) if answer =~ /^y/
-      end
-    end
-
-    # For each group that didnt already exist in cloud
-    group_keys.each do |group_name|
-      group = groups[group_name]
-      logger.debug "Creating new security group: #{group_name}"
-      # create each group
-      cloud.create_security_group(group_name, group['description'])
-      # create rules for group
-      group['rules'].each do |rule_map|
-        logger.debug "Creating new rule: #{rule_map.inspect}"
-        rule_map = Rubber::Util::symbolize_keys(rule_map)
-        if rule_map[:source_group_name]
-          cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], {:name => rule_map[:source_group_name], :account => rule_map[:source_group_account]})
-        else
-          rule_map[:source_ips].each do |source_ip|
-            cloud.add_security_group_rule(group_name, rule_map[:protocol], rule_map[:from_port], rule_map[:to_port], source_ip)
-          end if rule_map[:source_ips]
-        end
-      end
-    end
-  end
-
 end

data/lib/rubber/recipes/rubber/setup.rb

@@ -9,6 +9,7 @@ namespace :rubber do
     link_bash
     set_timezone
     enable_multiverse
+    install_core_packages
     upgrade_packages
     install_packages
     setup_volumes
@@ -19,9 +20,16 @@ namespace :rubber do
 
   # Sets up instance to allow root access (e.g. recent canonical AMIs)
   def enable_root_ssh(ip, initial_ssh_user)
+    # Capistrano uses the :password variable for sudo commands.  Since this setting is generally used for the deploy user,
+    # but we need it this one time for the initial SSH user, we need to swap out and restore the password.
+    #
+    # We special-case the 'ubuntu' user since Amazon doesn't since the Canonical AMIs on EC2 don't set the password for
+    # this account, making any password prompt potentially confusing.
+    orig_password = fetch(:password)
+    set(:password, initial_ssh_user == 'ubuntu' ? nil : Capistrano::CLI.password_prompt("Password for #{initial_ssh_user} @ #{ip}: "))
 
     task :_allow_root_ssh, :hosts => "#{initial_ssh_user}@#{ip}" do
-      rsudo "cp /home/#{initial_ssh_user}/.ssh/authorized_keys /root/.ssh/"
+      rsudo "mkdir -p /root/.ssh && cp /home/#{initial_ssh_user}/.ssh/authorized_keys /root/.ssh/"
     end
 
     begin
@@ -35,6 +43,9 @@ namespace :rubber do
         retry
       end
     end
+
+    # Restore the original deploy password.
+    set(:password, orig_password)
   end
 
   # Forces a direct connection
@@ -285,10 +296,22 @@ namespace :rubber do
     Install extra packages and gems.
   DESC
   task :install do
+    install_core_packages
     install_packages
     install_gems
   end
 
+  desc <<-DESC
+    Install core packages that are needed before the general install_packages phase.
+  DESC
+  task :install_core_packages do
+    core_packages = [
+                      'python-software-properties', # Needed for add-apt-repository, which we use for adding PPAs.
+                       'bc'                         # Needed for comparing version numbers in bash, which we do for various setup functions.
+                    ]
+    rsudo "export DEBIAN_FRONTEND=noninteractive; apt-get -q -o Dpkg::Options::=--force-confold -y --force-yes install #{core_packages.join(' ')}"
+  end
+
   desc <<-DESC
     Install Ubuntu packages. Set 'packages' in rubber.yml to \
     be an array of strings.
@@ -463,6 +486,7 @@ namespace :rubber do
           expanded_pkg_list << pkg_spec
         end
       end
+      expanded_pkg_list << 'ec2-ami-tools' if rubber_env.cloud_provider == 'aws'
       expanded_pkg_list.join(' ')
     end
 

data/lib/rubber/recipes/rubber/utils.rb

@@ -154,24 +154,24 @@ namespace :rubber do
     return local_alias
   end
 
-  def prepare_script(name, contents, stop_on_error_cmd=rubber_env.stop_on_error_cmd)
+  def prepare_script(name, contents, stop_on_error_cmd=rubber_env.stop_on_error_cmd, opts = {})
     script = "/tmp/#{name}"
     # this lets us abort a script if a command in the middle of it errors out
     contents = "#{stop_on_error_cmd}\n#{contents}" if stop_on_error_cmd
-    put(contents, script)
+    put(contents, script, opts)
     return script
   end
 
   def run_script(name, contents, opts = {})
     args = opts.delete(:script_args)
-    script = prepare_script(name, contents)
+    script = prepare_script(name, contents, rubber_env.stop_on_error_cmd, opts)
     run "bash #{script} #{args}", opts
   end
 
   def sudo_script(name, contents, opts = {})
     user = opts.delete(:as)
     args = opts.delete(:script_args)
-    script = prepare_script(name, contents)
+    script = prepare_script(name, contents, rubber_env.stop_on_error_cmd, opts)
 
     sudo_args = user ? "-H -u #{user}" : ""
     run "#{sudo} #{sudo_args} bash -l #{script} #{args}", opts
@@ -228,7 +228,10 @@ namespace :rubber do
   def update_code_for_bootstrap
     unless (fetch(:rubber_code_was_updated, false))
       deploy.setup
+      logger.info "updating code for bootstrap"
+      set :rubber_updating_code_for_bootstrap_db, true
       deploy.update_code
+      set :rubber_updating_code_for_bootstrap_db, false
     end
   end
   

data/lib/rubber/util.rb

@@ -115,6 +115,10 @@ module Rubber
       end
     end
 
+    def camelcase(str)
+      str.split('_').map{ |part| part.capitalize }.join
+    end
+
     extend self
   end
 end

data/lib/rubber/version.rb

@@ -1,3 +1,3 @@
 module Rubber
-  VERSION = "2.3.1"
+  VERSION = "2.4.0"
 end

data/templates/base/config/deploy.rb

@@ -90,11 +90,17 @@ task :cleanup, :except => { :no_release => true } do
 end
 
 if Rubber::Util.has_asset_pipeline?
-  # load asset pipeline tasks, and reorder them to run after
-  # rubber:config so that database.yml/etc has been generated
+  # load asset pipeline task, disable precompile from being triggered
+  # by deploy:update_code during bootstrap_db, and reorder to run after
+  # rubber:config has generated database.yml/etc.
   load 'deploy/assets'
   callbacks[:after].delete_if {|c| c.source == "deploy:assets:precompile"}
-  callbacks[:before].delete_if {|c| c.source == "deploy:assets:symlink"}
-  before "deploy:assets:precompile", "deploy:assets:symlink"
-  after "rubber:config", "deploy:assets:precompile"
+  task :_skip_assets_precompile_if_bootstrapping_db do
+    if fetch(:rubber_updating_code_for_bootstrap_db, false)
+      logger.info "Skipping assets precompilation"
+    else
+      deploy.assets.precompile
+    end
+  end
+  after 'deploy:update_code', '_skip_assets_precompile_if_bootstrapping_db'
 end

data/templates/base/config/rubber/rubber.yml

@@ -98,7 +98,20 @@ cloud_providers:
     # If a spot instance request can't be fulfilled in 3 minutes, fallback to on-demand instance creation.  If not set,
     # the default is infinite.
     # spot_instance_request_timeout: 180
-    
+
+  digital_ocean:
+    # REQUIRED: The Digital Ocean region that you want to use.
+    #
+    # Options include
+    # New York 1
+    # Amsterdam 1
+    #
+    region: New York 1
+
+    # REQUIRED: The image name and type for creating instances.
+    image_id: Ubuntu 12.04 x64 Server
+    image_type: 512MB
+
   # Use an alternate cloud provider supported by fog.  This doesn't fully work
   # yet due to differences in providers within fog, but gives you a starting
   # point for contributing a new provider to rubber.  See rubber/lib/rubber/cloud(.rb)
@@ -221,7 +234,7 @@ prompt_for_security_group_sync: true
 # OPTIONAL: The packages to install on all instances
 # You can install a specific version of a package by using a sub-array of pkg, version
 # For example, packages: [[rake, 0.7.1], irb]
-packages: [postfix, build-essential, git-core, ec2-ami-tools, libxslt-dev, ntp]
+packages: [postfix, build-essential, git-core, libxslt-dev, ntp]
 
 # OPTIONAL: gem sources to setup for rubygems
 # gemsources: ["https://rubygems.org"]