rubber 1.15.0 → 2.0.0.pre1

data/lib/rubber/recipes/rubber/security_groups.rb

@@ -67,7 +67,7 @@ namespace :rubber do
   end
 
   def isolate_prefix
-    return "#{rubber_env.app_name}_#{RUBBER_ENV}_"
+    return "#{rubber_env.app_name}_#{Rubber.env}_"
   end
 
   def isolate_group_name(group_name)
@@ -117,7 +117,7 @@ namespace :rubber do
         # so we need to do this up frnot and remove duplicates before checking against the local rubber rules)
         cloud_group[:permissions].each do |rule|
           if rule[:source_groups]
-            rule.source_groups.each do |source_group|
+            rule[:source_groups].each do |source_group|
               rule_map = {:source_group_name => source_group[:name], :source_group_account => source_group[:account]}
               rule_map = Rubber::Util::stringify(rule_map)
               rule_maps << rule_map unless rule_maps.include?(rule_map)

data/lib/rubber/recipes/rubber/setup.rb

@@ -72,13 +72,24 @@ namespace :rubber do
     hosts_file = '/etc/hosts'
 
     # Generate /etc/hosts contents for the local machine from instance config
-    delim = "## rubber config #{rubber_env.domain} #{RUBBER_ENV}"
+    delim = "## rubber config #{rubber_env.domain} #{Rubber.env}"
     local_hosts = delim + "\n"
     rubber_instances.each do |ic|
       # don't add unqualified hostname in local hosts file since user may be
       # managing multiple domains with same aliases
-      hosts_data = [ic.full_name, ic.external_host, ic.internal_host].join(' ')
-      local_hosts << ic.external_ip << ' ' << hosts_data << "\n"
+      hosts_data = [ic.full_name, ic.external_host, ic.internal_host]
+      
+      # add the ip aliases for web tools hosts so we can map internal tools
+      # to their own vhost to make proxying easier (rewriting url paths for
+      # proxy is a real pain, e.g. '/graphite/' externally to '/' on the
+      # graphite web app)
+      if ic.role_names.include?('web_tools')
+        Array(rubber_env.web_tools_proxies).each do |name, settings|
+          hosts_data << "#{name}.#{ic.full_name}"
+        end
+      end
+      
+      local_hosts << ic.external_ip << ' ' << hosts_data.join(' ') << "\n"
     end
     local_hosts << delim << "\n"
 
@@ -91,6 +102,56 @@ namespace :rubber do
     end
   end
 
+
+  desc <<-DESC
+    Sets up aliases for instance hostnames based on contents of instance.yml.
+    Generates /etc/hosts for remote machines and sets hostname on remote instances
+  DESC
+  task :setup_remote_aliases do
+    hosts_file = '/etc/hosts'
+
+    # Generate /etc/hosts contents for the remote instance from instance config
+    delim = "## rubber config #{Rubber.env}"
+    remote_hosts = []
+    
+    rubber_instances.each do |ic|
+      hosts_data = [ic.internal_ip, ic.full_name, ic.name, ic.external_host, ic.internal_host]
+      
+      # add the ip aliases for web tools hosts so we can map internal tools
+      # to their own vhost to make proxying easier (rewriting url paths for
+      # proxy is a real pain, e.g. '/graphite/' externally to '/' on the
+      # graphite web app)
+      if ic.role_names.include?('web_tools')
+        Array(rubber_env.web_tools_proxies).each do |name, settings|
+          hosts_data << "#{name}.#{ic.full_name}"
+        end
+      end
+      
+      remote_hosts << hosts_data.join(' ')
+    end
+    
+    if rubber_instances.size > 0
+      
+      replace="#{delim}\\n#{remote_hosts.join("\\n")}\\n#{delim}"
+      
+      rubber.sudo_script 'setup_remote_aliases', <<-ENDSCRIPT
+        sed -i.bak '/#{delim}/,/#{delim}/c #{replace}' /etc/hosts
+        if ! grep -q "#{delim}" /etc/hosts; then
+          echo -e "#{replace}" >> /etc/hosts
+        fi
+      ENDSCRIPT
+      
+      # Setup hostname on instance so shell, etcs have nice display
+      rsudo "echo $CAPISTRANO:HOST$ > /etc/hostname && hostname $CAPISTRANO:HOST$"
+      # Newer ubuntus ec2-init script always resets hostname, so prevent it
+      rsudo "mkdir -p /etc/ec2-init && echo compat=0 > /etc/ec2-init/is-compat-env"
+    end
+
+    # TODO
+    # /etc/resolv.conf to add search domain
+    # ~/.ssh/options to setup user/host/key aliases
+  end
+  
   desc <<-DESC
     Sets up aliases in dynamic dns provider for instance hostnames based on contents of instance.yml.
   DESC
@@ -177,42 +238,6 @@ namespace :rubber do
     end
   end
 
-  desc <<-DESC
-    Sets up aliases for instance hostnames based on contents of instance.yml.
-    Generates /etc/hosts for remote machines and sets hostname on remote instances
-  DESC
-  task :setup_remote_aliases do
-    hosts_file = '/etc/hosts'
-
-    # Generate /etc/hosts contents for the remote instance from instance config
-    delim = "## rubber config"
-    delim = "#{delim} #{RUBBER_ENV}"
-    remote_hosts = delim + "\n"
-    rubber_instances.each do |ic|
-      hosts_data = [ic.full_name, ic.name, ic.external_host, ic.internal_host].join(' ')
-      remote_hosts << ic.internal_ip << ' ' << hosts_data << "\n"
-    end
-    remote_hosts << delim << "\n"
-    if rubber_instances.size > 0
-      # write out the hosts file for the remote instances
-      # NOTE that we use "capture" to get the existing hosts
-      # file, which only grabs the hosts file from the first host
-      filtered = (capture "cat #{hosts_file}").gsub(/^#{delim}.*^#{delim}\n?/m, '')
-      filtered = filtered + remote_hosts
-      # Put the generated hosts back on remote instance
-      put filtered, hosts_file
-
-      # Setup hostname on instance so shell, etcs have nice display
-      rsudo "echo $CAPISTRANO:HOST$ > /etc/hostname && hostname $CAPISTRANO:HOST$"
-      # Newer ubuntus ec2-init script always resets hostname, so prevent it
-      rsudo "mkdir -p /etc/ec2-init && echo compat=0 > /etc/ec2-init/is-compat-env"
-    end
-
-    # TODO
-    # /etc/resolv.conf to add search domain
-    # ~/.ssh/options to setup user/host/key aliases
-  end
-
   desc <<-DESC
     Update to the newest versions of all packages/gems.
   DESC
@@ -262,16 +287,16 @@ namespace :rubber do
   desc <<-DESC
     Install ruby gems defined in the rails environment.rb
   DESC
-  after "rubber:config", "rubber:install_rails_gems" if (Rubber::Util::is_rails2? && !Rubber::Util.is_bundler?)
+  after "rubber:config", "rubber:install_rails_gems" if (Rubber::Util::is_rails? && !Rubber::Util.is_bundler?)
   task :install_rails_gems do
-    rsudo "cd #{current_release} && RAILS_ENV=#{RUBBER_ENV} rake gems:install"
+    rsudo "cd #{current_release} && RAILS_ENV=#{Rubber.env} #{fetch(:rake, 'rake')} gems:install"
   end
 
   desc <<-DESC
     Convenience task for installing your defined set of ruby gems locally.
   DESC
   required_task :install_local_gems do
-    fatal("install_local_gems can only be run in development") if RUBBER_ENV != 'development'
+    fatal("install_local_gems can only be run in development") if Rubber.env != 'development'
     env = rubber_cfg.environment.bind(rubber_cfg.environment.known_roles)
     gems = env['gems']
     expanded_gem_list = []
@@ -376,6 +401,16 @@ namespace :rubber do
     if env.dns_provider
       provider = Rubber::Dns::get_provider(env.dns_provider, env)
       provider.update(instance_item.name, instance_item.external_ip)
+      
+      # add the ip aliases for web tools hosts so we can map internal tools
+      # to their own vhost to make proxying easier (rewriting url paths for
+      # proxy is a real pain, e.g. '/graphite/' externally to '/' on the
+      # graphite web app)
+      if instance_item.role_names.include?('web_tools')
+        Array(rubber_env.web_tools_proxies).each do |name, settings|
+          provider.update("#{name}.#{ic.name}", instance_item.external_ip)
+        end
+      end
     end
   end
 
@@ -384,6 +419,16 @@ namespace :rubber do
     if env.dns_provider
       provider = Rubber::Dns::get_provider(env.dns_provider, env)
       provider.destroy(instance_item.name)
+      
+      # add the ip aliases for web tools hosts so we can map internal tools
+      # to their own vhost to make proxying easier (rewriting url paths for
+      # proxy is a real pain, e.g. '/graphite/' externally to '/' on the
+      # graphite web app)
+      if instance_item.role_names.include?('web_tools')
+        Array(rubber_env.web_tools_proxies).each do |name, settings|
+          provider.destroy("#{name}.#{ic.name}")
+        end
+      end
     end
   end
 
@@ -406,8 +451,60 @@ namespace :rubber do
     else
       rsudo "export DEBIAN_FRONTEND=noninteractive; apt-get -q -o Dpkg::Options::=--force-confold -y --force-yes install $CAPISTRANO:VAR$", opts
     end
+    
+    maybe_reboot
+  end
+  
+  def multi_capture(cmd, opts={})
+    mutex = Mutex.new
+    host_data = {}
+    run(cmd, opts) do |channel, stream, data|
+      if data
+        host = channel.properties[:host]
+        mutex.synchronize do
+          host_data[host] ||= ""
+          host_data[host] << data
+        end
+      end
+    end
+    return host_data    
   end
+  
+  def maybe_reboot
+    reboot_needed = multi_capture("echo $(ls /var/run/reboot-required 2> /dev/null)")
+    reboot_hosts = reboot_needed.collect {|k, v| v.strip.size > 0 ? k : nil}.compact
+    
+    if reboot_hosts.size > 0
+
+      ENV['REBOOT'] = 'y' if ENV['FORCE'] =~ /^(t|y)/
+      reboot = get_env('REBOOT', "Updates require a reboot on hosts #{reboot_hosts.inspect}, reboot [y/N]?", false)
+      reboot = (reboot =~ /^y/)
+      
+      if reboot
+
+        logger.info "Rebooting ..."
+        run("#{sudo} reboot", :hosts => reboot_hosts)
+        sleep 30
 
+        # since we rebooted, teardown the connections to force cap to reconnect
+        teardown_connections_to(sessions.keys)
+
+        reboot_hosts.each do |host|
+          direct_connection(host) do
+            run "echo"
+          end
+          logger.info "#{host} completed reboot"
+        end
+        
+      end
+      
+      # could take a while to reboot (or get answer from prompt), so
+      # we need to rebuild all capistrano connections in case they timed out
+      teardown_connections_to(sessions.keys)
+      
+    end
+  end
+        
   def custom_package(url_base, name, ver, install_test)
     rubber.sudo_script "install_#{name}", <<-ENDSCRIPT
       if [[ #{install_test} ]]; then

data/lib/rubber/recipes/rubber/utils.rb

@@ -4,21 +4,26 @@ namespace :rubber do
     Convenience task for creating a staging instance for the given RUBBER_ENV/RAILS_ENV.
     By default this task assigns all known roles when creating the instance,
     but you can specify a different default in rubber.yml:staging_roles
-    At the end, the instance will be up and running
+    At the end, the instance will be up and running.  If the staging instance
+    already exists, the user will be warned, and if they chose to proceed,
+    will skip the create and just bootstrap that instance.
     e.g. RUBBER_ENV=matt cap create_staging
   DESC
   required_task :create_staging do
     if rubber_instances.size > 0
-      value = Capistrano::CLI.ui.ask("The #{RUBBER_ENV} environment already has instances, Are you SURE you want to create a staging instance that may interact with them [y/N]?: ")
+      value = Capistrano::CLI.ui.ask("The #{Rubber.env} environment already has instances, Are you SURE you want to create a staging instance that may interact with them [y/N]?: ")
       fatal("Exiting", 0) if value !~ /^y/
     end
-    instance_alias = ENV['ALIAS'] = rubber.get_env("ALIAS", "Hostname to use for staging instance", true, RUBBER_ENV)
-    default_roles = rubber_env.staging_roles || "*"
-    roles = ENV['ROLES'] = rubber.get_env("ROLES", "Roles to use for staging instance", true, default_roles)
+    instance_alias = ENV['ALIAS'] = rubber.get_env("ALIAS", "Hostname to use for staging instance", true, Rubber.env)
 
     if rubber_instances[instance_alias]
       logger.info "Instance already exists, skipping to bootstrap"
     else
+      default_roles = rubber_env.staging_roles
+      # default staging roles to all roles minus slaves (db without primary=true is a slave)
+      default_roles ||= rubber_cfg.environment.known_roles.reject {|r| r =~ /slave/ || r =~ /^db$/ }.join(",")
+      roles = ENV['ROLES'] = rubber.get_env("ROLES", "Roles to use for staging instance", true, default_roles)
+      
       rubber.create
     end
 
@@ -47,7 +52,7 @@ namespace :rubber do
     Destroy the staging instance for the given RUBBER_ENV.
   DESC
   task :destroy_staging do
-    ENV['ALIAS'] = rubber.get_env("ALIAS", "Hostname of staging instance to be destroyed", true, RUBBER_ENV)
+    ENV['ALIAS'] = rubber.get_env("ALIAS", "Hostname of staging instance to be destroyed", true, Rubber.env)
     rubber.destroy
   end
 
@@ -57,7 +62,7 @@ namespace :rubber do
     set FILE=/path/file.*.glob to tails a different set
   DESC
   task :tail_logs, :roles => :app do
-    log_file_glob = rubber.get_env("FILE", "Log files to tail", true, "#{current_path}/log/#{RUBBER_ENV}*.log")
+    log_file_glob = rubber.get_env("FILE", "Log files to tail", true, "#{current_path}/log/#{Rubber.env}*.log")
     run "tail -qf #{log_file_glob}" do |channel, stream, data|
       puts  # for an extra line break before the host name
       puts data

data/lib/rubber/recipes/rubber/volumes.rb

@@ -227,7 +227,7 @@ namespace :rubber do
           fi
           mv /etc/fstab /etc/fstab.bak
           cat /etc/fstab.bak | grep -vE '#{raid_spec['device']}|#{raid_spec['mount']}' > /etc/fstab
-          echo '#{raid_spec['device']} #{raid_spec['mount']} #{raid_spec['filesystem']} #{vol_spec['mount_opts'] ? vol_spec['mount_opts'] : 'noatime'} 0 0 # rubber raid volume' >> /etc/fstab
+          echo '#{raid_spec['device']} #{raid_spec['mount']} #{raid_spec['filesystem']} #{raid_spec['mount_opts'] ? raid_spec['mount_opts'] : 'noatime'} 0 0 # rubber raid volume' >> /etc/fstab
 
           # seems to help devices initialize, otherwise mdadm fails because
           # device not ready even though ec2 says the volume is attached
@@ -292,7 +292,7 @@ namespace :rubber do
 
           mv /etc/fstab /etc/fstab.bak
           cat /etc/fstab.bak | grep -v '#{volume['mount']}\\b' > /etc/fstab
-          echo '#{device_name} #{volume['mount']} #{volume['filesystem']} #{vol_spec['mount_opts'] ? vol_spec['mount_opts'] : 'noatime'} 0 0 # rubber LVM volume' >> /etc/fstab
+          echo '#{device_name} #{volume['mount']} #{volume['filesystem']} #{volume['mount_opts'] ? volume['mount_opts'] : 'noatime'} 0 0 # rubber LVM volume' >> /etc/fstab
         fi
 
         # Check if the logical volume exists or not.

data/lib/rubber/tag.rb

@@ -2,15 +2,10 @@ module Rubber
   module Tag
     # Updates the tags for the given ec2 instance
     def self.update_instance_tags(instance_alias)
-      instance_item = Rubber::Configuration.rubber_instances[instance_alias]
-      fatal "Instance does not exist: #{instance_alias}" if ! instance_item
+      instance_item = Rubber.instances[instance_alias]
+      raise "Instance does not exist: #{instance_alias}" if ! instance_item
 
-      rubber_cfg = Rubber::Configuration.get_configuration(RUBBER_ENV)
-      rubber_env = rubber_cfg.environment.bind()
-
-      cloud = Rubber::Cloud::get_provider(rubber_env.cloud_provider || "aws", Rubber::Configuration.rubber_env, self)
-
-      cloud.create_tags(instance_item.instance_id, :Name => instance_alias, :Environment => RUBBER_ENV)
+      Rubber.cloud.create_tags(instance_item.instance_id, :Name => instance_alias, :Environment => Rubber.env)
     end
   end
 end

data/lib/rubber/util.rb

@@ -3,14 +3,14 @@
 module Rubber
   module Util
 
-    def self.symbolize_keys(map)
+    def symbolize_keys(map)
       map.inject({}) do |options, (key, value)|
         options[key.to_sym || key] = value
         options
       end
     end
     
-    def self.stringify(val)
+    def stringify(val)
       case val
       when String
         val
@@ -24,7 +24,7 @@ module Rubber
       
     end
 
-    def self.parse_aliases(instance_aliases)
+    def parse_aliases(instance_aliases)
       aliases = []
       alias_patterns = instance_aliases.to_s.strip.split(/\s*,\s*/)
       alias_patterns.each do |a|
@@ -41,31 +41,23 @@ module Rubber
     end
 
     # Opens the file for writing by root
-    def self.sudo_open(path, perms, &block)
+    def sudo_open(path, perms, &block)
       open("|sudo tee #{path} > /dev/null", perms, &block)
     end
 
-    def self.is_rails?
-      File.exist?(File.join(RUBBER_ROOT, 'config', 'boot.rb'))
+    def is_rails?
+      File.exist?(File.join(Rubber.root, 'config', 'boot.rb'))
     end
 
-    def self.is_rails2?
-      defined?(Rails) && defined?(Rails::VERSION) && Rails::VERSION::MAJOR == 2
+    def is_bundler?
+      File.exist?(File.join(Rubber.root, 'Gemfile'))
     end
 
-    def self.is_rails3?
-      defined?(Rails) && defined?(Rails::VERSION) && Rails::VERSION::MAJOR == 3
+    def has_asset_pipeline?
+      is_rails? && Dir["#{Rubber.root}/*/assets"].size > 0
     end
 
-    def self.is_bundler?
-      File.exist?(File.join(RUBBER_ROOT, 'Gemfile'))
-    end
-
-    def self.rubber_as_plugin?
-      File.exist?(File.join(RUBBER_ROOT, 'vendor/plugins/rubber'))
-    end
-
-    def self.prompt(name, desc, required=false, default=nil)
+    def prompt(name, desc, required=false, default=nil)
       value = ENV.delete(name)
       msg = "#{desc}"
       msg << " [#{default}]" if default
@@ -75,14 +67,43 @@ module Rubber
         value = gets
       end
       value = value.size == 0 ? default : value
-      self.fatal "#{name} is required, pass using environment or enter at prompt" if required && ! value
+      fatal "#{name} is required, pass using environment or enter at prompt" if required && ! value
       return value
     end
 
-    def self.fatal(msg, code=1)
+    def fatal(msg, code=1)
       puts msg
       exit code
     end
-    
+
+    # remove leading whitespace from "here" strings so they look good in code
+    # skips empty lines
+    def clean_indent(str)
+      counts = str.lines.collect {|l| l.scan(/^\s*/).first.size }
+      m = counts.reject {|x| x <= 1 }.min
+      str.lines.collect {|l| l.size < m ? l : l[m..-1] }.join("")
+    end
+
+    # execute the given block, retrying only when one of the given exceptions is raised
+    def retry_on_failure(*exception_list)
+      opts = exception_list.last.is_a?(Hash) ? exception_list.pop : {}
+      opts = {:retry_count => 3}.merge(opts)
+      retry_count = opts[:retry_count]
+      begin
+        yield
+      rescue *exception_list => e
+        if retry_count > 0
+          retry_count -= 1
+          Rubber.logger.info "Exception, trying again #{retry_count} more times"
+          sleep opts[:retry_sleep].to_i if opts[:retry_sleep] 
+          retry
+        else
+          Rubber.logger.error "Too many exceptions...re-raising"
+          raise
+        end
+      end
+    end
+
+    extend self
   end
 end