sml-rubber 0.9.13 → 1.5.5

data/lib/rubber/recipes/rubber/security_groups.rb

@@ -36,7 +36,7 @@ namespace :rubber do
       security_groups << host
       security_groups += roles
     end
-    security_groups = security_groups.uniq.compact
+    security_groups = security_groups.uniq.compact.reject {|x| x.empty? }
     security_groups = security_groups.collect {|x| isolate_group_name(x) } if env.isolate_security_groups
     return security_groups
   end

data/lib/rubber/recipes/rubber/setup.rb

@@ -4,8 +4,9 @@ namespace :rubber do
     Bootstraps instances by setting timezone, installing packages and gems
   DESC
   task :bootstrap do
-    set_timezone
     link_bash
+    set_timezone
+    enable_multiverse
     upgrade_packages
     install_packages
     setup_volumes
@@ -14,6 +15,33 @@ namespace :rubber do
     deploy.setup
   end
 
+  # Sets up instance to allow root access (e.g. recent canonical AMIs)
+  def enable_root_ssh(ip, initial_ssh_user)
+    old_user = user
+    begin
+      set :user, initial_ssh_user
+
+      task :_allow_root_ssh, :hosts => ip do
+        rsudo "cp /home/#{initial_ssh_user}/.ssh/authorized_keys /root/.ssh/"
+      end
+
+      begin
+        _allow_root_ssh
+      rescue ConnectionError => e
+        if e.message =~ /Net::SSH::AuthenticationFailed/
+          logger.info "Can't connect as user #{initial_ssh_user} to #{ip}, assuming root allowed"
+        else
+          sleep 2
+          logger.info "Failed to connect to #{ip}, retrying"
+          retry
+        end
+      end
+    ensure
+      set :user, old_user
+    end
+
+  end
+
   desc <<-DESC
     Sets up aliases for instance hostnames based on contents of instance.yml.
     Generates /etc/hosts for local/remote machines and sets hostname on
@@ -61,6 +89,83 @@ namespace :rubber do
     end
   end
 
+  desc <<-DESC
+    Sets up the additional dns records supplied in the dns_records config in rubber.yml
+  DESC
+  required_task :setup_dns_records do
+    records = rubber_env.dns_records
+    if records && rubber_env.dns_provider
+      provider = Rubber::Dns::get_provider(rubber_env.dns_provider, rubber_env)
+
+      # collect the round robin records (those with the same host/domain/type)
+      rr_records = []
+      records.each_with_index do |record, i|
+        m = records.find_all {|r| record['host'] == r['host'] && record['domain'] == r['domain'] && record['type'] == r['type']}
+        m = m.sort {|a,b| a.object_id <=> b.object_id}
+        rr_records << m if m.size > 1 && ! rr_records.include?(m)
+      end
+
+      # simple records are those that aren't round robin ones
+      simple_records = records - rr_records.flatten
+      
+      # for each simple record, create or update as necessary
+      simple_records.each do |record|
+        matching = provider.find_host_records(:host => record['host'], :domain =>record['domain'], :type => record['type'])
+        if matching.size > 1
+          msg =  "Multiple records in dns provider, but not in rubber.yml\n"
+          msg << "Round robin records need to be in both, or neither.\n"
+          msg << "Please fix manually:\n"
+          msg << matching.pretty_inspect
+          fatal(msg)
+        end
+
+        record = provider.setup_opts(record)
+        if matching.size == 1
+          match = matching.first
+          if  provider.host_records_equal?(record, match)
+            logger.info "Simple dns record already up to date: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
+          else
+            logger.info "Updating simple dns record: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
+            provider.update_host_record(match, record)
+          end
+        else
+          logger.info "Creating simple dns record: #{record[:host]}.#{record[:domain]}:#{record[:type]} => #{record[:data]}"
+          provider.create_host_record(record)
+        end
+      end
+
+      # group round robin records
+      rr_records.each do |rr_group|
+        host = rr_group.first['host']
+        domain = rr_group.first['domain']
+        type = rr_group.first['type']
+        matching = provider.find_host_records(:host => host, :domain => domain, :type => type)
+
+        # remove from consideration the local records that are the same as remote ones
+        matching.clone.each do |r|
+          rr_group.delete_if {|rg| provider.host_records_equal?(r, rg) }
+          matching.delete_if {|rg| provider.host_records_equal?(r, rg) }
+        end
+        if rr_group.size == 0 && matching.size == 0
+          logger.info "Round robin dns records already up to date: #{host}.#{domain}:#{type}"
+        end
+
+        # create the local records that don't exist remotely
+        rr_group.each do |r|
+          r = provider.setup_opts(r)
+          logger.info "Creating round robin dns record: #{r[:host]}.#{r[:domain]}:#{r[:type]} => #{r[:data]}"
+          provider.create_host_record(r)
+        end
+        
+        # remove the remote records that don't exist locally
+        matching.each do |r|
+          logger.info "Removing round robin dns record: #{r[:host]}.#{r[:domain]}:#{r[:type]} => #{r[:data]}"
+          provider.destroy_host_record(r)
+        end
+      end
+    end
+  end
+
   desc <<-DESC
     Sets up aliases for instance hostnames based on contents of instance.yml.
     Generates /etc/hosts for remote machines and sets hostname on remote instances
@@ -87,7 +192,9 @@ namespace :rubber do
       put filtered, hosts_file
 
       # Setup hostname on instance so shell, etcs have nice display
-      sudo "echo $CAPISTRANO:HOST$ > /etc/hostname && hostname $CAPISTRANO:HOST$"
+      rsudo "echo $CAPISTRANO:HOST$ > /etc/hostname && hostname $CAPISTRANO:HOST$"
+      # Newer ubuntus ec2-init script always resets hostname, so prevent it
+      rsudo "mkdir -p /etc/ec2-init && echo compat=0 > /etc/ec2-init/is-compat-env"
     end
 
     # TODO
@@ -141,51 +248,78 @@ namespace :rubber do
     gem_helper(false)
   end
 
+  desc <<-DESC
+    Install ruby gems defined in Gemfile
+  DESC
+  after "deploy:update_code", "rubber:install_bundler_gems" if Rubber::Util.is_bundler?
+  task :install_bundler_gems do
+    rsudo "cd #{current_release} && RAILS_ENV=#{RUBBER_ENV} bundle install --without development test"
+  end
+
   desc <<-DESC
     Install ruby gems defined in the rails environment.rb
   DESC
-  after "deploy:symlink", "rubber:install_rails_gems" if Rubber::Util.is_rails?
+  after "rubber:config", "rubber:install_rails_gems" if (Rubber::Util::is_rails2? && !Rubber::Util.is_bundler?)
   task :install_rails_gems do
-    sudo "sh -c 'cd #{current_path} && RAILS_ENV=#{RUBBER_ENV} rake gems:install'"
+    rsudo "cd #{current_release} && RAILS_ENV=#{RUBBER_ENV} rake gems:install"
   end
 
   desc <<-DESC
-    Setup ruby gems sources. Set 'gemsources' in rubber.yml to \
-    be an array of URI strings.
+    Convenience task for installing your defined set of ruby gems locally.
   DESC
-  task :setup_gem_sources do
-    if rubber_env.gemsources
-      script = prepare_script 'gem_sources_helper', <<-'ENDSCRIPT'
-        ruby - $@ <<-'EOF'
-
-        sources = ARGV
+  required_task :install_local_gems do
+    fatal("install_local_gems can only be run in development") if RUBBER_ENV != 'development'
+    env = rubber_cfg.environment.bind(rubber_cfg.environment.known_roles)
+    gems = env['gems']
+    expanded_gem_list = []
+    gems.each do |gem_spec|
+      if gem_spec.is_a?(Array)
+        expanded_gem_list << "#{gem_spec[0]}:#{gem_spec[1]}"
+      else
+        expanded_gem_list << gem_spec
+      end
+    end
+    expanded_gem_list = expanded_gem_list.join(' ')
 
-        installed = []
-        `gem sources -l`.grep(/^[^*]/) do |line|
-            line = line.strip
-            installed << line if line.size > 0
-        end
+    logger.info "Installing gems:#{expanded_gem_list}"
+    open("/tmp/gem_helper", "w") {|f| f.write(gem_helper_script)}
+    system "ruby /tmp/gem_helper install #{expanded_gem_list}"
+  end
 
-        to_install = sources - installed
-        to_remove = installed - sources
+  set :gem_sources_helper_script, <<-'ENDSCRIPT'
+    sources = ARGV
 
-        if to_install.size > 0
-          to_install.each do |source|
-            system "gem sources -a #{source}"
-            fail "Unable to add gem sources" if $?.exitstatus > 0
-          end
-        end
-        if to_remove.size > 0
-          to_remove.each do |source|
-            system "gem sources -r #{source}"
-            fail "Unable to remove gem sources" if $?.exitstatus > 0
-          end
-        end
+    installed = []
+    `gem sources -l`.each_line do |line|
+      line = line.strip
+      installed << line if line.size > 0 && line =~ /^[^*]/
+    end
 
-        'EOF'
-      ENDSCRIPT
+    to_install = sources - installed
+    to_remove = installed - sources
 
-      sudo "sh #{script} #{rubber_env.gemsources.join(' ')}"
+    if to_install.size > 0
+      to_install.each do |source|
+        system "gem sources -a #{source}"
+        fail "Unable to add gem sources" if $?.exitstatus > 0
+      end
+    end
+    if to_remove.size > 0
+      to_remove.each do |source|
+        system "gem sources -r #{source}"
+        fail "Unable to remove gem sources" if $?.exitstatus > 0
+      end
+    end
+  ENDSCRIPT
+  
+  desc <<-DESC
+    Setup ruby gems sources. Set 'gemsources' in rubber.yml to \
+    be an array of URI strings.
+  DESC
+  task :setup_gem_sources do
+    if rubber_env.gemsources
+      script = prepare_script 'gem_sources_helper', gem_sources_helper_script, nil
+      rsudo "ruby #{script} #{rubber_env.gemsources.join(' ')}"
     end
   end
 
@@ -194,7 +328,7 @@ namespace :rubber do
     You can override this task if you don't want this to happen
   DESC
   task :link_bash do
-    sudo("ln -sf /bin/bash /bin/sh")
+    rsudo "ln -sf /bin/bash /bin/sh"
   end
 
   desc <<-DESC
@@ -208,10 +342,28 @@ namespace :rubber do
   DESC
   task :set_timezone do
     opts = get_host_options('timezone')
-    sudo "bash -c 'echo $CAPISTRANO:VAR$ > /etc/timezone'", opts
-    sudo "cp /usr/share/zoneinfo/$CAPISTRANO:VAR$ /etc/localtime", opts
+    rsudo "echo $CAPISTRANO:VAR$ > /etc/timezone", opts
+    rsudo "cp /usr/share/zoneinfo/$CAPISTRANO:VAR$ /etc/localtime", opts
     # restart syslog so that times match timezone
-    sudo "/etc/init.d/sysklogd restart"
+    sudo_script 'restart_syslog', <<-ENDSCRIPT
+      if [[ -x /etc/init.d/sysklogd ]]; then
+        /etc/init.d/sysklogd restart
+      elif [[ -x /etc/init.d/rsyslog ]]; then
+        service rsyslog restart
+     fi
+    ENDSCRIPT
+  end
+  
+  desc <<-DESC
+    Enable the ubuntu multiverse source for getting packages like
+    ec2-ami-tools used for bundling images
+  DESC
+  task :enable_multiverse do
+    sudo_script 'enable_multiverse', <<-ENDSCRIPT
+      if ! grep -qc multiverse /etc/apt/sources.list /etc/apt/sources.list.d/* &> /dev/null; then
+        cat /etc/apt/sources.list | sed 's/main universe/multiverse/' > /etc/apt/sources.list.d/rubber-multiverse-source.list
+      fi
+    ENDSCRIPT
   end
   
   def update_dyndns(instance_item)
@@ -243,16 +395,16 @@ namespace :rubber do
       expanded_pkg_list.join(' ')
     end
 
-    sudo "apt-get -q update"
+    rsudo "apt-get -q update"
     if upgrade
-      sudo "/bin/sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes dist-upgrade'"
+      rsudo "export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes dist-upgrade"
     else
-      sudo "/bin/sh -c 'export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes install $CAPISTRANO:VAR$'", opts
+      rsudo "export DEBIAN_FRONTEND=noninteractive; apt-get -q -y --force-yes install $CAPISTRANO:VAR$", opts
     end
   end
 
   def custom_package(url_base, name, ver, install_test)
-    rubber.run_script "install_#{name}", <<-ENDSCRIPT
+    rubber.sudo_script "install_#{name}", <<-ENDSCRIPT
       if [[ #{install_test} ]]; then
         arch=`uname -m`
         if [ "$arch" = "x86_64" ]; then
@@ -261,7 +413,7 @@ namespace :rubber do
           src="#{url_base}/#{name}_#{ver}_i386.deb"
         fi
         src_file="${src##*/}"
-        wget -qP /tmp ${src}
+        wget -qNP /tmp ${src}
         dpkg -i /tmp/${src_file}
       fi
     ENDSCRIPT
@@ -280,11 +432,57 @@ namespace :rubber do
     end
   end
 
+  # Rubygems always installs even if the gem is already installed
+  # When providing versions, rubygems fails unless versions are provided for all gems
+  # This helper script works around these issues by installing gems only if they
+  # aren't already installed, and separates versioned/unversioned into two separate
+  # calls to rubygems
+  #
+  set :gem_helper_script, <<-'ENDSCRIPT'
+    gem_cmd = ARGV[0]
+    gems = ARGV[1..-1]
+    cmd = "gem #{gem_cmd} --no-rdoc --no-ri"
+
+    to_install = {}
+    to_install_ver = {}
+    # gem list passed in, possibly with versions, as "gem1 gem2:1.2 gem3"
+    gems.each do |gem_spec|
+      parts = gem_spec.split(':')
+      if parts[1]
+        to_install_ver[parts[0]] = parts[1]
+      else
+        to_install[parts[0]] = true
+      end
+    end
+
+    installed = {}
+    `gem list --local`.each_line do |line|
+        parts = line.scan(/(.*) \((.*)\)/).first
+        next unless parts && parts.size == 2
+        installed[parts[0]] = parts[1].split(",")
+    end
+
+    to_install.delete_if {|g, v| installed.has_key?(g) } if gem_cmd == 'install'
+    to_install_ver.delete_if {|g, v| installed.has_key?(g) && installed[g].include?(v) }
+
+    # rubygems can only do asingle versioned gem at a time so we need
+    # to do the two groups separately
+    # install versioned ones first so unversioned don't pull in a newer version
+    to_install_ver.each do |g, v|
+      system "#{cmd} #{g} -v #{v}"
+      fail "Unable to install versioned gem #{g}:#{v}" if $?.exitstatus > 0
+    end
+    if to_install.size > 0
+      gem_list = to_install.keys.join(' ')
+      system "#{cmd} #{gem_list}"
+      fail "Unable to install gems" if $?.exitstatus > 0
+    end
+  ENDSCRIPT
+
   # Helper for installing gems,allows one to respond to prompts
   def gem_helper(update=false)
     cmd = update ? "update" : "install"
 
-
     opts = get_host_options('gems') do |gem_list|
       expanded_gem_list = []
       gem_list.each do |gem_spec|
@@ -298,60 +496,11 @@ namespace :rubber do
     end
     
     if opts.size > 0
-      # Rubygems always installs even if the gem is already installed
-      # When providing versions, rubygems fails unless versions are provided for all gems
-      # This helper script works around these issues by installing gems only if they
-      # aren't already installed, and separates versioned/unversioned into two separate
-      # calls to rubygems
-      script = prepare_script 'gem_helper', <<-'ENDSCRIPT'
-        ruby - $@ <<-'EOF'
-
-        gem_cmd = ARGV[0]
-        gems = ARGV[1..-1]
-        cmd = "gem #{gem_cmd} --no-rdoc --no-ri"
-
-        to_install = {}
-        to_install_ver = {}
-        # gem list passed in, possibly with versions, as "gem1 gem2:1.2 gem3"
-        gems.each do |gem_spec|
-          parts = gem_spec.split(':')
-          if parts[1]
-            to_install_ver[parts[0]] = parts[1]
-          else
-            to_install[parts[0]] = true
-          end
-        end
-
-        installed = {}
-        `gem list --local`.each do |line|
-            parts = line.scan(/(.*) \((.*)\)/).first
-            next unless parts && parts.size == 2
-            installed[parts[0]] = parts[1].split(",")
-        end
-
-        to_install.delete_if {|g, v| installed.has_key?(g) } if gem_cmd == 'install'
-        to_install_ver.delete_if {|g, v| installed.has_key?(g) && installed[g].include?(v) } 
-
-        # rubygems can only do asingle versioned gem at a time so we need
-        # to do the two groups separately
-        # install versioned ones first so unversioned don't pull in a newer version
-        to_install_ver.each do |g, v|
-          system "#{cmd} #{g} -v #{v}"
-          fail "Unable to install versioned gem #{g}:#{v}" if $?.exitstatus > 0
-        end
-        if to_install.size > 0
-          gem_list = to_install.keys.join(' ')
-          system "#{cmd} #{gem_list}"
-          fail "Unable to install gems" if $?.exitstatus > 0
-        end
-
-        'EOF'
-      ENDSCRIPT
-
-      sudo "sh #{script} #{cmd} $CAPISTRANO:VAR$", opts do |ch, str, data|
+      script = prepare_script('gem_helper', gem_helper_script, nil)
+      rsudo "ruby #{script} #{cmd} $CAPISTRANO:VAR$", opts do |ch, str, data|
         handle_gem_prompt(ch, data, str)
       end
     end
   end
 
-end
+end

data/lib/rubber/recipes/rubber/spot_requests.rb

@@ -0,0 +1,23 @@
+namespace :rubber do
+
+  desc "Describes all your spot instance requests"
+  required_task :describe_spot_instance_requests do
+    requests = cloud.describe_spot_instance_requests()
+    requests.each do |request|
+      logger.info "======================"
+      logger.info "ID: #{request[:id]}"
+      logger.info "Created at: #{request[:created_at]}"
+      logger.info "Max. price: $#{request[:spot_price]}"
+      logger.info "State: #{request[:state]}"
+      logger.info "Instance type: #{request[:type]}"
+      logger.info "AMI: #{request[:image_id]}"
+    end
+  end
+
+  desc "Cancel the spot instances request for the given id"
+  required_task :cancel_spot_instances_request do
+    request_id = get_env('SPOT_INSTANCE_REQUEST_ID', 'The id of the spot instances request to cancel', true)
+    cloud.destroy_spot_instance_request(request_id)
+  end
+
+end