rubber 1.2.1 → 1.3.0

data/CHANGELOG

@@ -1,3 +1,11 @@
+1.2.1
+-----
+
+Jeweler will release to gemcutter. <5ab598f> [Kevin Menard]
+Work around issue with capistrano 2.5.16. <ce9617b> [Kevin Menard]
+Bumped to the latest version of redis. <e32f072> [Kevin Menard]
+Bumped to latest version of passenger. <251f2ee> [Kevin Menard]
+
 1.2.0
 -----
 

data/VERSION

@@ -1 +1 @@
-1.2.1
+1.3.0

data/generators/vulcanize/templates/apache/config/rubber/deploy-apache.rb

@@ -15,10 +15,10 @@ namespace :rubber do
     # rubber auto-roles don't get defined till after all tasks are defined
     on :load do
       rubber.serial_task self, :serial_restart, :roles => :apache do
-        run "/etc/init.d/apache2 restart"
+        sudo "/etc/init.d/apache2 restart"
       end
       rubber.serial_task self, :serial_reload, :roles => :apache do
-        run "if ! ps ax | grep -v grep | grep -c apache2 &> /dev/null; then /etc/init.d/apache2 start; else /etc/init.d/apache2 reload; fi"
+        sudo "sh -c 'if ! ps ax | grep -v grep | grep -c apache2 &> /dev/null; then /etc/init.d/apache2 start; else /etc/init.d/apache2 reload; fi'"
       end
     end
     
@@ -28,12 +28,12 @@ namespace :rubber do
     
     desc "Stops the apache web server"
     task :stop, :roles => :apache, :on_error => :continue do
-      run "/etc/init.d/apache2 stop"
+      sudo "/etc/init.d/apache2 stop"
     end
     
     desc "Starts the apache web server"
     task :start, :roles => :apache do
-      run "/etc/init.d/apache2 start"
+      sudo "/etc/init.d/apache2 start"
     end
     
     desc "Restarts the apache web server"

data/generators/vulcanize/templates/base/config/rubber/deploy-setup.rb

@@ -20,7 +20,7 @@ namespace :rubber do
       if ent_ruby_hosts.size > 0
         task :_install_enterprise_ruby, :hosts => ent_ruby_hosts do
           ver = "1.8.7-2010.01"
-          rubber.run_script "install_ruby-enterprise", <<-ENDSCRIPT
+          rubber.sudo_script "install_ruby-enterprise", <<-ENDSCRIPT
             if [[ ! `ruby --version 2> /dev/null` =~ "Ruby Enterprise Edition 2010.01" ]]; then
               arch=`uname -m`
               if [ "$arch" = "x86_64" ]; then
@@ -43,8 +43,8 @@ namespace :rubber do
     #  The ubuntu rubygem package is woefully out of date, so install it manually
     after "rubber:install_packages", "rubber:base:install_rubygems"
     task :install_rubygems do
-      ver = "1.3.5"
-      src_url = "http://rubyforge.org/frs/download.php/60718/rubygems-#{ver}.tgz"
+      ver = "1.3.6"
+      src_url = "http://production.cf.rubygems.org/rubygems/rubygems-#{ver}.tgz"
       rubber.sudo_script 'install_rubygems', <<-ENDSCRIPT
         if [[ `gem --version 2>&1` != "#{ver}" ]]; then
           wget -qNP /tmp #{src_url}
@@ -59,7 +59,7 @@ namespace :rubber do
     
     after "rubber:install_packages", "rubber:base:configure_git" if scm == "git"
     task :configure_git do
-      rubber.run_script 'configure_git', <<-ENDSCRIPT
+      rubber.sudo_script 'configure_git', <<-ENDSCRIPT
         if [[ "#{repository}" =~ "@" ]]; then
           # Get host key for src machine to prevent ssh from failing
           rm -f ~/.ssh/known_hosts

data/generators/vulcanize/templates/base/config/rubber/rubber-dns.yml

@@ -11,7 +11,6 @@ dns_providers:
     type: A
     ttl: 300
   zerigo:
-    customer_id: 1234
     email: foo@bar.com
     token: hexxy
     type: A

data/generators/vulcanize/templates/base/config/rubber/rubber.yml

@@ -75,10 +75,10 @@ cloud_providers:
 
     # REQUIRED: the ami and instance type for creating instances
     # The Ubuntu images at http://alestic.com/ work well
-    # Ubuntu 9.04 Jaunty	base install	32-bit	2.6.21.7-2.fc8xen	ami-ccf615a5
-    # Ubuntu 9.04 Jaunty	base install	64-bit	2.6.21.7-2.fc8xen	ami-eef61587
+    # Ubuntu 9.10 Karmic	base install	32-bit	ami-bb709dd2
+    # Ubuntu 9.10 Karmic	base install	64-bit	ami-55739e3c
     #
-    image_id: ami-ccf615a5
+    image_id: ami-bb709dd2
     # m1.small or m1.large or m1.xlarge
     image_type: m1.small
 
@@ -143,7 +143,7 @@ ruby_prefix: "#{use_enterprise_ruby ? '/usr/local' : '/usr'}"
 packages: [postfix, build-essential, ruby-full, ruby1.8-dev, rake, irb, git-core]
 
 # OPTIONAL: gem sources to setup for rubygems
-gemsources: ["http://gemcutter.org", "http://gems.rubyforge.org/", "http://gems.github.com"]
+# gemsources: ["http://rubygems.org", "http://gems.github.com"]
 
 # OPTIONAL: The gems to install on all instances
 # You can install a specific version of a gem by using a sub-array of gem, version

data/generators/vulcanize/templates/complete_mongrel_mysql/config/rubber/role/haproxy/haproxy-mongrel.conf

@@ -8,14 +8,14 @@ listen mongrel_proxy 0.0.0.0:<%= rubber_env.haproxy_listen_port %>
     # Since we do "fair" load balancing to the mongrels, we don't want to share mongrels
     # between haproxy instances otherwise we still end up with requests getting queued behind
     # slow ones.
-    mongrel_hosts = rubber_instances.for_role('mongrel').collect { |i| i.name }
+    app_hosts = rubber_instances.for_role('app').collect { |i| i.name }
     haproxy_hosts = rubber_instances.for_role('haproxy').collect { |i| i.name }
     ports = (rubber_env.mongrel_base_port.to_i ... rubber_env.mongrel_base_port.to_i + rubber_env.mongrel_count.to_i).to_a
     host_count = haproxy_hosts.size
     host_mod = haproxy_hosts.sort.index(rubber_env.host) rescue 0
     ports = ports.find_all {|x| x % host_count == host_mod}
   %>
-  <% mongrel_hosts.each do |server| %>
+  <% app_hosts.each do |server| %>
     <% ports.each do |port| %>
       server <%= server %>_<%= port %> <%= server %>:<%= port %> maxconn 1 check
     <% end %>

data/generators/vulcanize/templates/complete_passenger_mysql/config/rubber/role/haproxy/haproxy-passenger.conf

@@ -4,7 +4,7 @@
 %>
 
 <%
-  backend_hosts = rubber_instances.for_role('passenger').collect { |i| i.name }
+  backend_hosts = rubber_instances.for_role('app').collect { |i| i.name }
 %>
 
 listen passenger_proxy 0.0.0.0:<%= rubber_env.web_port %>

data/generators/vulcanize/templates/cruise/config/rubber/deploy-cruise.rb

@@ -54,12 +54,12 @@ namespace :rubber do
 
     desc "Start cruise control daemon"
     task :start, :roles => :cruise do
-      run "/etc/init.d/cruise start"
+      sudo "/etc/init.d/cruise start"
     end
     
     desc "Stop cruise control daemon"
     task :stop, :roles => :cruise, :on_error => :continue do
-      run "/etc/init.d/cruise stop"
+      sudo "/etc/init.d/cruise stop"
     end
     
     desc "Restart cruise control daemon"

data/generators/vulcanize/templates/haproxy/config/rubber/deploy-haproxy.rb

@@ -9,10 +9,10 @@ namespace :rubber do
     # rubber auto-roles don't get defined till after all tasks are defined
     on :load do
       rubber.serial_task self, :serial_restart, :roles => :haproxy do
-        run "/etc/init.d/haproxy restart"
+        sudo "/etc/init.d/haproxy restart"
       end
       rubber.serial_task self, :serial_reload, :roles => :haproxy do
-        run "if ! ps ax | grep -v grep | grep -c haproxy &> /dev/null; then /etc/init.d/haproxy start; else /etc/init.d/haproxy reload; fi"
+        sudo "sh -c 'if ! ps ax | grep -v grep | grep -c haproxy &> /dev/null; then /etc/init.d/haproxy start; else /etc/init.d/haproxy reload; fi'"
       end
     end
     
@@ -22,12 +22,12 @@ namespace :rubber do
     
     desc "Stops the haproxy server"
     task :stop, :roles => :haproxy, :on_error => :continue do
-      run "/etc/init.d/haproxy stop"
+      sudo "/etc/init.d/haproxy stop"
     end
     
     desc "Starts the haproxy server"
     task :start, :roles => :haproxy do
-      run "/etc/init.d/haproxy start"
+      sudo "/etc/init.d/haproxy start"
     end
     
     desc "Restarts the haproxy server"

data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslog-haproxy.conf

@@ -1,6 +1,11 @@
 <%
-  @path = "/etc/syslog.conf"
-  @post = "/etc/init.d/sysklogd restart"
-  @additive = ["# rubber-haproxy-start", "# rubber-haproxy-end"]
+  is_rsyslog = File.exist?('/etc/init.d/rsyslog')
+  if is_rsyslog
+    @path = "/etc/rsyslog.d/haproxy.conf"
+    @post = "/etc/init.d/rsyslog restart"
+  else
+    @path = "/etc/syslog.conf"
+    @post = "/etc/init.d/sysklogd restart"
+  end
 %>
 local0.* /var/log/haproxy.log

data/generators/vulcanize/templates/haproxy/config/rubber/role/haproxy/syslogd-default.conf

@@ -1,7 +1,11 @@
 <%
+  is_rsyslog = File.exist?('/etc/init.d/rsyslog')
+  @skip = is_rsyslog
   @path = "/etc/default/syslogd"
   @post = "/etc/init.d/sysklogd restart"
 %>
+# This file goes away once we remove backward compatibility for sysklog
+# in alestic AMIs
 #
 # Top configuration file for syslogd
 #

data/generators/vulcanize/templates/jetty/config/rubber/deploy-jetty.rb

@@ -36,7 +36,7 @@ namespace :rubber do
     after "rubber:setup_app_permissions", "rubber:jetty:setup_jetty_permissions"
 
     task :setup_jetty_permissions, :roles => :jetty do
-      run "chown -R #{rubber_env.app_user}:#{rubber_env.app_user} #{rubber_env.jetty_dir}"
+      sudo "chown -R #{rubber_env.app_user}:#{rubber_env.app_user} #{rubber_env.jetty_dir}"
     end
     
     before "deploy:stop", "rubber:jetty:stop"
@@ -44,15 +44,15 @@ namespace :rubber do
     after "deploy:restart", "rubber:jetty:restart"
     
     task :restart, :roles => :jetty do
-      run "#{rubber_env.jetty_dir}/bin/jetty.sh restart"
+      sudo "#{rubber_env.jetty_dir}/bin/jetty.sh restart"
     end
     
     task :stop, :roles => :jetty do
-      run "#{rubber_env.jetty_dir}/bin/jetty.sh stop"
+      sudo "#{rubber_env.jetty_dir}/bin/jetty.sh stop"
     end
     
     task :start, :roles => :jetty do
-      run "#{rubber_env.jetty_dir}/bin/jetty.sh start"
+      sudo "#{rubber_env.jetty_dir}/bin/jetty.sh start"
     end
     
   end

data/generators/vulcanize/templates/mongrel/config/rubber/deploy-mongrel.rb

@@ -21,16 +21,16 @@ namespace :rubber do
     
     
     def mongrel_stop
-        run "cd #{current_path} && mongrel_rails cluster::stop"
+        sudo "cd #{current_path} && mongrel_rails cluster::stop"
         sleep 5 # Give the graceful stop a chance to complete
-        run "cd #{current_path} && mongrel_rails cluster::stop --force --clean"
+        sudo "cd #{current_path} && mongrel_rails cluster::stop --force --clean"
     end
     
     def mongrel_start
-        run "cd #{current_path} && mongrel_rails cluster::start --clean"
+        sudo "cd #{current_path} && mongrel_rails cluster::start --clean"
         pid_cnt = rubber_env.mongrel_count
         logger.info "Waiting for mongrel pid files to show up"
-        run "while ((`ls #{current_path}/tmp/pids/mongrel.*.pid 2> /dev/null | wc -l` < #{pid_cnt})); do sleep 1; done"
+        sudo "sh -c 'while ((`ls #{current_path}/tmp/pids/mongrel.*.pid 2> /dev/null | wc -l` < #{pid_cnt})); do sleep 1; done'"
     end
     
     # serial_task can only be called after roles defined - not normally a problem, but

data/generators/vulcanize/templates/monit/config/rubber/common/monit.conf

@@ -95,7 +95,6 @@ set alert <%= rubber_env.admin_email %> only on {
       gid
       icmp
       invalid
-      match
       nonexist
       permission
       resource

data/generators/vulcanize/templates/monit/config/rubber/deploy-monit.rb

@@ -1,6 +1,7 @@
 
 namespace :rubber do
-  
  namespace :monit do
+  
+  namespace :monit do
   
     rubber.allow_optional_tasks(self)
     
@@ -13,12 +14,12 @@ namespace :rubber do
 
     desc "Start monit daemon monitoring"
     task :start do
-      run "/etc/init.d/monit start"
+      sudo "/etc/init.d/monit start"
     end
     
     desc "Stop monit daemon monitoring"
     task :stop, :on_error => :continue do
-      run "/etc/init.d/monit stop"
+      sudo "/etc/init.d/monit stop"
     end
     
     desc "Restart monit daemon monitoring"

data/generators/vulcanize/templates/munin/config/rubber/deploy-munin.rb

@@ -14,7 +14,7 @@ namespace :rubber do
       Reconfigures munin
     DESC
     task :custom_install do
-      rubber.run_script 'setup_munin_plugins', <<-ENDSCRIPT
+      rubber.sudo_script 'setup_munin_plugins', <<-ENDSCRIPT
         munin-node-configure --shell --remove-also > /tmp/setup-munin-plugins
         sh /tmp/setup-munin-plugins
       ENDSCRIPT
@@ -27,12 +27,12 @@ namespace :rubber do
 
     desc "Start munin system monitoring"
     task :start do
-      run "/etc/init.d/munin-node start"
+      sudo "/etc/init.d/munin-node start"
     end
     
     desc "Stop munin system monitoring"
     task :stop, :on_error => :continue do
-      run "/etc/init.d/munin-node stop"
+      sudo "/etc/init.d/munin-node stop"
     end
     
     desc "Restart munin system monitoring"

data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-plugins.conf

@@ -22,7 +22,7 @@
     File.open("#{plugin_dir}/#{plugin_name}", 'w') do |p|
       p.chmod(0755)
       p.puts('#!/bin/sh')
-      p.puts("RUBBER_ENV=#{RUBBER_ENV} #{script} $*")
+      p.puts("RUBBER_ENV=#{RUBBER_ENV} #{rubber_env.ruby_prefix}/bin/ruby #{script} $*")
     end
   end
 %>

data/generators/vulcanize/templates/mysql/config/rubber/deploy-mysql.rb

@@ -36,8 +36,7 @@ namespace :rubber do
           exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
           if exists.strip.size == 0
             common_bootstrap("mysql_master")
-            sudo "dpkg-reconfigure --frontend=noninteractive mysql-server-5.0"
-            sleep 5
+            
             pass = "identified by '#{env.db_pass}'" if env.db_pass
             sudo "mysql -u root -e 'create database #{env.db_name};'"
             sudo "mysql -u root -e \"grant all on *.* to '#{env.db_user}'@'%' #{pass};\""
@@ -57,8 +56,6 @@ namespace :rubber do
           exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
           if exists.strip.size == 0
             common_bootstrap("mysql_slave")
-            sudo "dpkg-reconfigure --frontend=noninteractive mysql-server-5.0"
-            sleep 5
 
             master = rubber_instances.for_role("mysql_master").first
 
@@ -80,7 +77,7 @@ namespace :rubber do
             if source == master
               logger.info "Creating slave from a dump of master #{source_host}"
               sudo "mysql -u root -e \"change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}' #{master_pass}\""
-              sudo "mysqldump -u #{env.db_user} #{pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root"
+              sudo "sh -c 'mysqldump -u #{env.db_user} #{pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root'"
             else
               logger.info "Creating slave from a dump of slave #{source_host}"
               sudo "mysql -u #{env.db_user} #{pass} -h #{source_host} -e \"stop slave;\""
@@ -88,11 +85,15 @@ namespace :rubber do
               slave_config = Hash[*slave_status.scan(/([^\s:]+): ([^\s]*)/).flatten]
               log_file = slave_config['Master_Log_File']
               log_pos = slave_config['Read_Master_Log_Pos']
-              sudo "mysqldump -u #{env.db_user} #{pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root"
+              sudo "sh -c 'mysqldump -u #{env.db_user} #{pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root'"
               sudo "mysql -u root -e \"change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}', master_log_file='#{log_file}', master_log_pos=#{log_pos} #{master_pass}\""
               sudo "mysql -u #{env.db_user} #{pass} -h #{source_host} -e \"start slave;\""
             end
 
+            # this doesn't work without agent forwarding which sudo breaks, as well as not having your
+            # ec2 private key ssh-added on workstation
+            # sudo "scp -o \"StrictHostKeyChecking=no\" #{source_host}:/etc/mysql/debian.cnf /etc/mysql"
+
             sudo "mysql -u root -e \"flush privileges;\""
             sudo "mysql -u root -e \"start slave;\""
           end
@@ -113,7 +114,14 @@ namespace :rubber do
       deploy.update_code
       
       # Gen just the conf for the given mysql role
-      rubber.run_config(:RUBBER_ENV => RUBBER_ENV, :FILE => "role/#{role}|role/db/my.cnf", :deploy_path => release_path)
+      rubber.run_config(:RUBBER_ENV => RUBBER_ENV, :FILE => "role/#{role}|role/db/", :FORCE => true, :deploy_path => release_path)
+
+      # reconfigure mysql so that it sets up data dir in /mnt with correct files
+      sudo_script 'reconfigure-mysql', <<-ENDSCRIPT
+        server_package=`dpkg -l | grep mysql-server-[0-9] | awk '{print $2}'`
+        dpkg-reconfigure --frontend=noninteractive $server_package
+      ENDSCRIPT
+      sleep 5
     end
     
     before "rubber:munin:custom_install", "rubber:mysql:custom_install_munin"
@@ -122,7 +130,7 @@ namespace :rubber do
       Installs some extra munin graphs
     DESC
     task :custom_install_munin, :roles => [:mysql_master, :mysql_slave] do
-      rubber.run_script 'install_munin_mysql', <<-ENDSCRIPT
+      rubber.sudo_script 'install_munin_mysql', <<-ENDSCRIPT
         if [ ! -f /usr/share/munin/plugins/mysql_ ]; then
           wget -q -O /usr/share/munin/plugins/mysql_ http://github.com/kjellm/munin-mysql/raw/master/mysql_
           wget -q -O /etc/munin/plugin-conf.d/mysql_.conf http://github.com/kjellm/munin-mysql/raw/master/mysql_.conf
@@ -150,6 +158,96 @@ namespace :rubber do
     task :restart, :roles => [:mysql_master, :mysql_slave] do
       sudo "/etc/init.d/mysql restart"
     end
+
+    desc <<-DESC
+      Backup production database using rake task rubber:backup_db
+    DESC
+    task :backup, :roles => [:mysql_master, :mysql_slave] do
+      master_instances = rubber_instances.for_role("mysql_master")
+      slaves = rubber_instances.for_role("mysql_slave")
+
+      # Select only one instance for backup.  Favor slave database.
+      selected_mysql_instance = (slaves+master_instances).first
+            
+      task_name = "_backup_mysql_slave_#{selected_mysql_instance.full_name}".to_sym()
+      task task_name, :hosts => selected_mysql_instance.full_name do
+        run "cd #{current_path} && RUBBER_ENV=production RAILS_ENV=production RUBYOPT=rubygems BACKUP_DIR=/mnt/db_backups DBUSER=#{rubber_env.db_user} DBPASS=#{rubber_env.db_pass} DBNAME=#{rubber_env.db_name} DBHOST=#{selected_mysql_instance.full_name} rake rubber:backup_db"
+      end
+      send task_name
+    end
+    
+    desc <<-DESC
+      Restore production database from s3 using rake task rubber:restore_db_s3
+    DESC
+    task :restore_s3, :roles => [:mysql_master, :mysql_slave] do
+      master_instances = rubber_instances.for_role("mysql_master")
+      slaves = rubber_instances.for_role("mysql_slave")
+
+      for instance in master_instances+slaves
+        task_name = "_restore_mysql_s3_#{instance.full_name}".to_sym()
+        task task_name, :hosts => instance.full_name do
+          run "cd #{current_path} && RUBBER_ENV=production RAILS_ENV=production RUBYOPT=rubygems BACKUP_DIR=/mnt/db_backups DBUSER=#{rubber_env.db_user} DBPASS=#{rubber_env.db_pass} DBNAME=#{rubber_env.db_name} DBHOST=#{instance.full_name} rake rubber:restore_db_s3"
+        end
+        send task_name
+      end
+    end    
+    
+    desc <<-DESC
+      Overwrite ec2 production database with export from local production database.
+    DESC
+    task :local_to_ec2, :roles => [:mysql_master, :mysql_slave] do
+      require 'yaml'      
+      master_instances = rubber_instances.for_role("mysql_master")
+      slaves = rubber_instances.for_role("mysql_slave")
+
+      # Select only one instance for backup.  Favor slave database.
+      selected_mysql_instance = (slaves+master_instances).first
+            
+      task_name = "_load_local_to_#{selected_mysql_instance.full_name}".to_sym()
+      task task_name, :hosts => selected_mysql_instance.full_name do
+
+        # Dump Local to tmp folder
+        filename = "#{application}.local.#{Time.now.to_i}.sql.gz" 
+        backup_file = "/tmp/#{filename}" 
+        on_rollback { delete file }
+        FileUtils.mkdir_p(File.dirname(backup_file))
+
+        # Use database.yml to get connection params
+        db = YAML::load(ERB.new(IO.read(File.join(File.dirname(__FILE__), '..','database.yml'))).result)['production']
+        user = db['username']
+        pass = db['passsword']
+        pass = nil if pass and pass.strip.size == 0
+        host = db['host']
+        name = db['database']
+        
+        raise "No db_backup_cmd defined in rubber.yml, cannot backup!" unless rubber_env.db_backup_cmd
+        db_backup_cmd = rubber_env.db_backup_cmd.gsub(/%([^%]+)%/, '#{\1}')
+        db_backup_cmd = eval('%Q{' + db_backup_cmd + '}')
+
+        # mysqldump (or backup app) needs to be in your path
+        puts "Backing up database with command:"
+        system(db_backup_cmd)
+        puts "Created backup: #{backup_file}"
+
+        # Upload Local to S3
+        cloud_provider = rubber_env.cloud_providers[rubber_env.cloud_provider]
+        s3_prefix = "db/"
+        backup_bucket = cloud_provider.backup_bucket
+        if backup_bucket
+          AWS::S3::Base.establish_connection!(:access_key_id => cloud_provider.access_key, :secret_access_key => cloud_provider.secret_access_key)
+          unless AWS::S3::Bucket.list.find { |b| b.name == backup_bucket }
+            AWS::S3::Bucket.create(backup_bucket)
+          end
+          dest = "#{s3_prefix}#{File.basename(backup_file)}"
+          puts "Saving db dump to S3: #{backup_bucket}:#{dest}"
+          AWS::S3::S3Object.store(dest, open(backup_file), backup_bucket)
+        end
+        
+        send :restore_s3
+
+      end
+      send task_name
+    end    
   
   end