rubber 2.0.0.pre9 → 2.0.0.pre10

data/lib/rubber/commands/util.rb

@@ -156,6 +156,9 @@ module Rubber
       option ["-d", "--directory"],
              "DIRECTORY",
              "The directory to stage backups into\nRequired"
+      option ["-f", "--filename"],
+             "FILENAME",
+             "The name of the backup file"
       option ["-u", "--dbuser"],
              "DBUSER",
              "The database user to connect with\nRequired"
@@ -177,9 +180,12 @@ module Rubber
       def execute
         signal_usage_error "DIRECTORY, DBUSER, DBHOST, DBNAME are required" unless directory && dbuser && dbhost && dbname
         
-        
         time_stamp = Time.now.strftime("%Y-%m-%d_%H-%M")
-        backup_file = "#{directory}/#{Rubber.env}_dump_#{time_stamp}.sql.gz"
+        if filename
+          backup_file = "#{directory}/#{filename}"
+        else
+          backup_file = "#{directory}/#{Rubber.env}_dump_#{time_stamp}.sql.gz"
+        end
         FileUtils.mkdir_p(File.dirname(backup_file))
         
         # extra variables for command interpolation

data/lib/rubber/dns/fog.rb

@@ -35,7 +35,7 @@ module Rubber
       end
 
       def find_or_create_zone(domain)
-        zone = @client.zones.all.find {|z| z.domain =~ /#{domain}\.?/}
+        zone = @client.zones.all.find {|z| z.domain =~ /^#{domain}\.?/}
         if ! zone
           zone = @client.zones.create(:domain => domain)
         end

data/lib/rubber/version.rb

@@ -1,4 +1,4 @@
 module Rubber
-  VERSION = "2.0.0.pre9"
+  VERSION = "2.0.0.pre10"
 end
 

data/templates/mysql/config/rubber/role/db/apparmor-mysql.conf

@@ -38,6 +38,8 @@
   /{,var/}run/mysqld/mysqld.pid w,
   /{,var/}run/mysqld/mysqld.sock w,
 
+	/usr/lib/mysql/plugin/ r,
+
   /sys/devices/system/cpu/ r,
   /sys/devices/system/cpu/** r,
 

data/templates/mysql/config/rubber/role/db/my.cnf

@@ -42,7 +42,7 @@ nice    = 0
 #
 # * Basic Settings
 #
-default-character-set = utf8
+character-set-server = utf8
 user    = mysql
 pid-file  = /var/run/mysqld/mysqld.pid
 socket    = /var/run/mysqld/mysqld.sock

data/templates/percona/config/rubber/common/database.yml

@@ -0,0 +1,12 @@
+<%
+  @path = "#{Rubber.root}/config/database.yml"
+%>
+
+<%= Rubber.env %>:
+  adapter: <%= YAML::load(File.open("#{Rubber.root}/config/database.yml"))["production"]["adapter"] %>
+  encoding: utf8
+  database: <%= rubber_env.db_name %>
+  username: <%= rubber_env.db_user %>
+  password: <%= rubber_env.db_pass %>
+  host: <%= rubber_instances.for_role('db', 'primary' => true).first.full_name %>
+  pool: 5

data/templates/percona/config/rubber/deploy-percona.rb

@@ -0,0 +1,169 @@
+namespace :rubber do
+  
+  namespace :percona do
+    
+    rubber.allow_optional_tasks(self)
+    
+    before "rubber:install_packages", "rubber:percona:add_repo"
+    
+    task :add_repo, :roles => [:percona_master, :percona_slave] do
+      # Setup apt sources for percona
+      codename = capture('lsb_release -c -s').chomp
+      sources = <<-SOURCES
+        deb http://repo.percona.com/apt #{codename} main
+        deb-src http://repo.percona.com/apt #{codename} main
+      SOURCES
+      sources.gsub!(/^ */, '')
+      put(sources, "/etc/apt/sources.list.d/percona.list") 
+      rsudo "gpg --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A"
+      rsudo "gpg -a --export CD2EFD2A | apt-key add -"
+    end
+    
+    after "rubber:create", "rubber:percona:validate_db_roles"
+
+    task :validate_db_roles do
+      db_instances = rubber_instances.for_role("percona_slave")
+      db_instances.each do |instance|
+        if instance.role_names.find {|n| n == 'percona_master'}
+          fatal "Cannot have a percona slave and master on the same instance, please removing slave role for #{instance.name}"
+        end
+      end
+    end
+
+    after "rubber:bootstrap", "rubber:percona:bootstrap"
+  
+    
+    # Bootstrap the production database config.  Db bootstrap is special - the
+    # user could be requiring the rails env inside some of their config
+    # templates, which creates a catch 22 situation with the db, so we try and
+    # bootstrap the db separate from the rest of the config
+    task :bootstrap, :roles => [:percona_master, :percona_slave] do
+      
+      # Conditionaly bootstrap for each node/role only if that node has not
+      # been boostrapped for that role before
+      
+      master_instances = rubber_instances.for_role("percona_master") & rubber_instances.filtered  
+      master_instances.each do |ic|
+        task_name = "_bootstrap_percona_master_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          env = rubber_cfg.environment.bind("percona_master", ic.name)
+          exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
+          if exists.strip.size == 0
+            common_bootstrap("percona_master")
+            
+            pass = "identified by '#{env.db_pass}'" if env.db_pass
+            rubber.sudo_script "create_master_db", <<-ENDSCRIPT
+              mysql -u root -e "create database #{env.db_name};"
+              mysql -u root -e "grant all on *.* to '#{env.db_user}'@'%' #{pass};"
+              mysql -u root -e "grant select on *.* to '#{env.db_slave_user}'@'%' #{pass};"
+              mysql -u root -e "grant replication slave on *.* to '#{env.db_replicator_user}'@'%' #{pass};"
+              mysql -u root -e "flush privileges;"
+            ENDSCRIPT
+          end
+        end
+        send task_name
+      end
+    
+      slave_instances = rubber_instances.for_role("percona_slave") & rubber_instances.filtered  
+      slave_instances.each do |ic|
+        task_name = "_bootstrap_percona_slave_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          env = rubber_cfg.environment.bind("percona_slave", ic.name)
+          exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
+          if exists.strip.size == 0
+            common_bootstrap("percona_slave")
+
+            master = rubber_instances.for_role("percona_master").first
+
+            # Doing a mysqldump locks the db, so ideally we'd do it against a slave replica thats
+            # not serving traffic (mysql_util role), but if thats not available try a regular
+            # slave (percona_slave role), and finally default dumping from master (percona_master role)
+            # TODO: handle simultaneous creating of multi slaves/utils
+            slaves = rubber_instances.for_role("percona_slave")
+            slaves.delete(ic) # don't want to try and dump from self
+            source = slaves.find {|sc| sc.role_names.include?("mysql_util")}
+            source = slaves.first unless source
+            source = master unless source
+
+            pass = "identified by '#{env.db_pass}'" if env.db_pass
+            master_pass = ", master_password='#{env.db_pass}'" if env.db_pass
+            master_host = master.full_name
+            source_host = source.full_name
+
+            if source == master
+              logger.info "Creating slave from a dump of master #{source_host}"
+              rubber.sudo_script "create_slave_db_from_master", <<-ENDSCRIPT
+                mysql -u root -e "change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}' #{master_pass}"
+                mysqldump -u #{env.db_user} #{env.db_pass.nil? ? '' : '--password=' + env.db_pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root
+              ENDSCRIPT
+            else
+              logger.info "Creating slave from a dump of slave #{source_host}"
+              rsudo "mysql -u #{env.db_user} --password #{env.db_pass} -h #{source_host} -e \"stop slave;\""
+              slave_status = capture("mysql -u #{env.db_user} #{pass} -h #{source_host} -e \"show slave status\\G\"")
+              slave_config = Hash[*slave_status.scan(/([^\s:]+): ([^\s]*)/).flatten]
+              log_file = slave_config['Master_Log_File']
+              log_pos = slave_config['Read_Master_Log_Pos']
+              rubber.sudo_script "create_slave_db_from_slave", <<-ENDSCRIPT
+                mysqldump -u #{env.db_user} #{env.db_pass.nil? ? '' : '--password=' + env.db_pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root
+                mysql -u root -e "change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}', master_log_file='#{log_file}', master_log_pos=#{log_pos} #{master_pass}"
+                mysql -u #{env.db_user} --password #{env.db_pass} -h #{source_host} -e "start slave;"
+              ENDSCRIPT
+            end
+
+            # this doesn't work without agent forwarding which sudo breaks, as well as not having your
+            # ec2 private key ssh-added on workstation
+            # sudo "scp -o \"StrictHostKeyChecking=no\" #{source_host}:/etc/mysql/debian.cnf /etc/mysql"
+
+            rsudo "mysql -u root -e \"flush privileges;\""
+            rsudo "mysql -u root -e \"start slave;\""
+          end
+        end
+        send task_name
+      end
+      
+    end
+  
+    # TODO: Make the setup/update happen just once per host
+    def common_bootstrap(role)
+      # mysql package install starts mysql, so stop it
+      rsudo "service mysql stop" rescue nil
+      
+      # After everything installed on machines, we need the source tree
+      # on hosts in order to run rubber:config for bootstrapping the db
+      rubber.update_code_for_bootstrap
+      
+      # Gen just the conf for the given mysql role
+      rubber.run_config(:file => "role/#{role}|role/db/", :force => true, :deploy_path => release_path)
+
+      # reconfigure mysql so that it sets up data dir in /mnt with correct files
+      sudo_script 'reconfigure-mysql', <<-ENDSCRIPT
+        server_package=`dpkg -l | grep percona-server-server-[0-9] | awk '{print $2}'`
+        dpkg-reconfigure --frontend=noninteractive $server_package
+      ENDSCRIPT
+      sleep 5
+    end
+    
+    desc <<-DESC
+      Starts the mysql daemons
+    DESC
+    task :start, :roles => [:percona_master, :percona_slave] do
+      rsudo "service mysql start"
+    end
+    
+    desc <<-DESC
+      Stops the mysql daemons
+    DESC
+    task :stop, :roles => [:percona_master, :percona_slave] do
+      rsudo "service mysql stop"
+    end
+  
+    desc <<-DESC
+      Restarts the mysql daemons
+    DESC
+    task :restart, :roles => [:percona_master, :percona_slave] do
+      rsudo "service mysql restart"
+    end
+  
+  end
+
+end

data/templates/percona/config/rubber/role/db/apparmor-mysql.conf

@@ -0,0 +1,49 @@
+<%
+  @path = '/etc/apparmor.d/usr.sbin.mysqld'
+  @post = "service apparmor restart"
+  @backup = false
+%>
+
+#include <tunables/global>
+
+/usr/sbin/mysqld {
+  #include <abstractions/base>
+  #include <abstractions/nameservice>
+  #include <abstractions/user-tmp>
+  #include <abstractions/mysql>
+  #include <abstractions/winbind>
+
+  capability dac_override,
+  capability sys_resource,
+  capability setgid,
+  capability setuid,
+
+  network tcp,
+
+  /etc/hosts.allow r,
+  /etc/hosts.deny r,
+
+  /etc/mysql/*.pem r,
+  /etc/mysql/conf.d/ r,
+  /etc/mysql/conf.d/* r,
+  /etc/mysql/my.cnf r,
+  /usr/sbin/mysqld mr,
+  /usr/share/mysql/** r,
+  /var/log/mysql.log rw,
+  /var/log/mysql.err rw,
+  /var/lib/mysql/ r,
+  /var/lib/mysql/** rwk,
+  /var/log/mysql/ r,
+  /var/log/mysql/* rw,
+  /{,var/}run/mysqld/mysqld.pid w,
+  /{,var/}run/mysqld/mysqld.sock w,
+
+  /etc/mtab r,
+  /usr/lib/mysql/plugin/ r,
+
+  /sys/devices/system/cpu/ r,
+  /sys/devices/system/cpu/** r,
+
+  <%= rubber_env.db_root_dir %>/ r,
+  <%= rubber_env.db_root_dir %>/** rwk,
+}

data/templates/percona/config/rubber/role/db/crontab

@@ -0,0 +1,16 @@
+<%
+  @read_cmd = 'crontab -l'
+  @write_cmd = 'crontab -'
+  @additive = ["# rubber-mysql-start", "# rubber-mysql-end"]
+%>
+
+<%
+  # Backup db every 3 hours to secondary db if available, else primary db
+  backup_db_host = rubber_instances.for_role('db', {}).first rescue nil
+  backup_db_host ||= rubber_instances.for_role('db', 'primary' => true).first
+  if backup_db_host.name == rubber_env.host
+%>
+
+0 */3 * * * <%= Rubber.root %>/script/rubber cron --task util:backup_db --directory=/mnt/db_backups --dbuser=<%= rubber_env.db_user %> --dbpass=<%= rubber_env.db_pass %> --dbname=<%= rubber_env.db_name %> --dbhost=<%= backup_db_host.full_name %>
+
+<% end %>

data/templates/percona/config/rubber/role/db/my.cnf

@@ -0,0 +1,163 @@
+<%
+  data_dir = rubber_env.db_data_dir
+  log_dir = rubber_env.db_log_dir
+  server_id = rubber_env.db_server_id 
+  @path = '/etc/mysql/my.cnf'
+  @post = "mkdir -p #{log_dir}; chown mysql:adm #{log_dir}; chmod 2750 #{log_dir}"
+%>
+
+# Generated by rubber for host <%= rubber_instances[rubber_env.host].external_ip %>
+#
+# The MySQL database server configuration file.
+#
+# You can copy this to one of:
+# - "/etc/mysql/my.cnf" to set global options,
+# - "~/.my.cnf" to set user-specific options.
+#
+# One can use all long options that the program supports.
+# Run program with --help to get a list of available options and with
+# --print-defaults to see which it would actually understand and use.
+#
+# For explanations see
+# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
+
+# This will be passed to all mysql clients
+# It has been reported that passwords should be enclosed with ticks/quotes
+# escpecially if they contain "#" chars...
+# Remember to edit /etc/mysql/debian.cnf when changing the socket location.
+[client]
+default-character-set = utf8
+port    = 3306
+socket    = /var/run/mysqld/mysqld.sock
+
+# Here is entries for some specific programs
+# The following values assume you have at least 32M ram
+
+# This was formally known as [safe_mysqld]. Both versions are currently parsed.
+[mysqld_safe]
+socket    = /var/run/mysqld/mysqld.sock
+nice    = 0
+
+[mysqld]
+#
+# * Basic Settings
+#
+character-set-server = utf8
+user    = mysql
+pid-file  = /var/run/mysqld/mysqld.pid
+socket    = /var/run/mysqld/mysqld.sock
+port    = 3306
+basedir   = /usr
+datadir = <%= data_dir %>
+tmpdir    = /tmp
+language  = /usr/share/mysql/english
+skip-external-locking
+
+#
+# Instead of skip-networking the default is now to listen only on
+# localhost which is more compatible and is not less secure.
+# bind-address    = 127.0.0.1
+#
+# * Fine Tuning
+#
+key_buffer    = 16M
+max_allowed_packet  = 32M
+thread_stack    = 128K
+thread_cache_size = 8
+#table_cache=500
+#max_heap_table_size=16M
+#tmp_table_size=32M
+
+#max_connections        = 100
+#table_cache            = 64
+#thread_concurrency     = 10
+#
+# * Query Cache Configuration
+#
+query_cache_limit       = 1M
+query_cache_size        = 16M
+#
+# * Logging and Replication
+#
+# Both location gets rotated by the cronjob.
+# Be aware that this log type is a performance killer.
+#log    = /var/log/mysql/mysql.log
+#
+# Error logging goes to syslog. This is a Debian improvement :)
+#
+# Here you can see queries with especially long duration
+log_slow_queries = /var/log/mysql/mysql-slow.log
+long_query_time = 2
+log-queries-not-using-indexes
+#
+# The following can be used as easy to replay backup logs or for replication.
+# note: if you are setting up a replication slave, see README.Debian about
+#       other settings you may need to change.
+server-id = <%= server_id %>
+log_bin     = <%= log_dir %>/mysql-bin.log
+relay_log   = <%= log_dir %>/mysql-relay-bin.log
+
+# WARNING: Using expire_logs_days without bin_log crashes the server! See README.Debian!
+expire_logs_days  = 10
+max_binlog_size         = 100M
+#binlog_do_db   = include_database_name
+#binlog_ignore_db = include_database_name
+
+
+# * InnoDB
+#
+# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
+# Read the manual for more InnoDB related options. There are many!
+# You might want to disable InnoDB to shrink the mysqld process by circa 100MB.
+#skip-innodb
+
+# Set buffer pool size to 50-80% of your computer's memory
+#innodb_buffer_pool_size=512M
+#innodb_additional_mem_pool_size=64M
+
+# Set the log file size to about 25% of the buffer pool size
+#innodb_log_file_size=128M
+#innodb_log_buffer_size=32M
+#innodb_flush_log_at_trx_commit=2
+
+#
+# * Security Features
+#
+# Read the manual, too, if you want chroot!
+# chroot = /var/lib/mysql/
+#
+# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
+#
+# ssl-ca=/etc/mysql/cacert.pem
+# ssl-cert=/etc/mysql/server-cert.pem
+# ssl-key=/etc/mysql/server-key.pem
+
+
+
+[mysqldump]
+quick
+quote-names
+max_allowed_packet  = 16M
+
+[mysql]
+#no-auto-rehash # faster start of mysql but no tab completition
+
+[isamchk]
+key_buffer    = 16M
+
+#
+# * NDB Cluster
+#
+# See /usr/share/doc/mysql-server-*/README.Debian for more information.
+#
+# The following configuration is read by the NDB Data Nodes (ndbd processes)
+# not from the NDB Management Nodes (ndb_mgmd processes).
+#
+# [MYSQL_CLUSTER]
+# ndb-connectstring=127.0.0.1
+
+
+#
+# * IMPORTANT: Additional settings that can override those from this file!
+#
+!includedir /etc/mysql/conf.d/

data/templates/percona/config/rubber/rubber-percona.yml

@@ -0,0 +1,43 @@
+# REQUIRED: The credentials for creating/accessong your app's database
+#
+db_user: "#{app_name[0,16]}"
+db_pass:
+db_name: "#{app_name}_#{Rubber.env}"
+
+db_slave_user: "#{('slave_' + db_user)[0,16]}"
+db_replicator_user: "#{('replicator_' + db_user)[0,16]}"
+
+# REQUIRED: The command to use to do a full backup of your database using
+#           "rubber util:backup_db"
+db_backup_cmd: "nice mysqldump -h %host% -u %user% --password=%pass% %name% | gzip -c > %backup_file%"
+# REQUIRED: The command to use to restore the database using
+#           "rubber util:restore_db_s3"
+#           This command will receive the data generated by db_backup_cmd via
+#           its standard input
+db_restore_cmd: "zcat - | mysql -h %host% -u %user% --password=%pass% %name%"
+
+# REQUIRED: The directory to store the db data in
+#
+db_root_dir: /mnt/mysql
+db_data_dir: "#{db_root_dir}/data"
+db_log_dir: "#{db_root_dir}/log"
+
+role_dependencies:
+  percona_master: [percona, "db:primary=true"]
+  percona_slave: [percona, db]
+  db:primary=true: [percona, percona_master]
+  db: [percona, percona_slave]
+
+packages: [percona-server-client, libmysqlclient-dev]
+gems: [mysql]
+
+roles:
+  percona_master:
+    db_server_id: 1
+  percona_slave:
+    db_server_id: 2
+  percona:
+    packages: [
+                percona-server-server,
+                maatkit # useful mysql tools for replication
+              ]

data/templates/percona/templates.yml

@@ -0,0 +1 @@
+description: The percona mysql server module

data/templates/xtrabackup/config/rubber/deploy-xtrabackup.rb

@@ -0,0 +1,24 @@
+namespace :rubber do
+  
+  namespace :xtrabackup do
+    
+    rubber.allow_optional_tasks(self)
+
+    before "rubber:install_packages", "rubber:xtrabackup:add_repo"
+
+    task :add_repo, :roles => [:percona, :mysql] do
+      # Setup apt sources for percona
+      codename = capture('lsb_release -c -s').chomp
+      sources = <<-SOURCES
+        deb http://repo.percona.com/apt #{codename} main
+        deb-src http://repo.percona.com/apt #{codename} main
+      SOURCES
+      sources.gsub!(/^ */, '')
+      put(sources, "/etc/apt/sources.list.d/percona.list") 
+      rsudo "gpg --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A"
+      rsudo "gpg -a --export CD2EFD2A | apt-key add -"
+    end
+    
+  end
+  
+end

data/templates/xtrabackup/config/rubber/role/db/rubber-xtrabackup.sh

@@ -0,0 +1,246 @@
+<%
+  @path = "/mnt/db-backup-tools/rubber-xtrabackup.sh"
+	@perms = 0755
+	@backup = false
+%>#!/bin/bash
+# 
+# Run innobackupex for databases, ensure it succeeded, and put file in place for S3 upload by Rubber.
+# First, set up some variables.
+INNOBACKUP="/usr/bin/innobackupex"
+LOGFILE="/tmp/rubber-xtrabackup-log"
+MEMORY="512M"
+MYSQL="/usr/bin/mysql"
+MYSQLADMIN="/usr/bin/mysqladmin"
+# If you use MyISAM tables, comment this out or Bad Things will happen. Leave as-is if you use InnoDB.
+NOLOCK="--no-lock"
+# By default, don't use differential backups unless -d command line argument supplied.
+DIFFERENTIALS=0
+# The number of differential backups to perform before rotating into a new full backup
+# NOTE: Ensure you have enough disk space to keep this amount of diffs on the volume
+#       you supply via the -t (backup directory) option.
+MAXDIFFS=24
+# Some folks have empty passwords and won't supply one.
+PASSWORD=""
+# Have rubber include slave commands if we're on a slave.
+<%  dbm_inst = rubber_instances.for_role('db', 'primary' => true).collect { |i| i.name }
+ if dbm_inst.include?(rubber_env.host)
+%>
+# We're on a primary DB instance
+SLAVECMD=""
+<% else %>
+# We're on a slave DB instance.
+SLAVECMD="--safe-slave-backup --slave-info"
+<% end %>
+
+# Lets get our command line parameters
+while getopts ":u:p:t:db:" opt; do
+  case $opt in
+    u)
+      USERNAME="$OPTARG"
+      ;;
+		p)
+			PASSWORD="$OPTARG"
+			;;
+		t)
+			BACKUPDIR="$OPTARG"
+			;;
+		b)
+			BACKUPFILE="$OPTARG"
+			;;
+		d)
+			DIFFERENTIALS=1
+			;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      exit 1
+      ;;
+    :)
+      echo "Option -$OPTARG requires an argument." >&2
+      exit 1
+      ;;
+  esac
+done
+
+if [[ -z "$USERNAME" || -z "$BACKUPDIR" || -z "$BACKUPFILE" ]]; then
+	echo "Required parameters missing. Please supply -u (database username), -t (backup directory) and -b (backup file)"
+	exit 1
+fi
+
+differential_backup () {
+	# This method will handle differential backups
+	BASEDIR="$BACKUPDIR/diffs/base"
+	
+	if [ ! -d "$BASEDIR" ]; then
+		echo "$BASEDIR does not exist. Running full base backup..."
+		clean_base_backup
+		echo "Success. Exiting."
+		exit 0
+	else
+		if [ -f "$BACKUPDIR/diffs/diff-status" ]; then
+			# Check current diff number
+			DIFFNUM=`cat $BACKUPDIR/diffs/diff-status`
+			# Increase diff number by one
+			let "NEWDIFFNUM = $DIFFNUM + 1"
+			if [ "$NEWDIFFNUM" -gt "$MAXDIFFS" ]; then
+				echo "Exceeded maximum number of differentials. Performing new base backup."
+				clean_base_backup
+				echo "Success. Exiting."
+				exit 0				
+			else
+				if [ "$NEWDIFFNUM" -eq "1" ]; then
+					# First diff, let's set the base for the diff to $BASEDIR
+					DIFFBASE=$BASEDIR
+				else
+					DIFFBASE="$BACKUPDIR/diffs/$DIFFNUM"
+				fi
+				# Run the backup
+				incremental_backup $DIFFBASE "$BACKUPDIR/diffs/$NEWDIFFNUM"
+				# Once this is verified we'll merge this incremental into the base folder
+				prepare_backup $BASEDIR "$BACKUPDIR/diffs/$NEWDIFFNUM"
+				# Then prepare the backup once again
+				prepare_backup $BASEDIR
+				# Finally, tar it all up.
+				tar czf $BACKUPFILE -C $BASEDIR .
+				# Update our diff status.
+				echo $NEWDIFFNUM > "$BACKUPDIR/diffs/diff-status"
+				echo "Success. exiting."
+				exit 0
+			fi
+		else
+			echo "Can't get backup status. Performing new base backup."
+			clean_base_backup
+			echo "Success. Exiting."
+			exit 0
+		fi
+	fi
+}			
+
+full_backup () {
+	# This method will handle full backups for further processing. 
+	# This function requires a single argument, the destination folder of the backup.
+	if [ -z "$1" ]; then
+		echo "ERROR: full_backup() called without destination parameter. Exiting."
+		exit 1
+	fi
+	echo "Running full backup into $1"
+	$INNOBACKUP --no-timestamp --user=$USERNAME $PASSCMD $SLAVECMD $NOLOCK $1 2> $LOGFILE
+	echo "Checking backup result..."
+	check_backup_result
+}
+
+prepare_backup () {
+	# This prepares the backups for immediate restore.
+	# This function supports two arguments:
+	# 1) The base folder (required)
+	# 2) The incremental folder (optional)
+	if [ -z "$1" ]; then
+		echo "ERROR: prepare_backup() called without base folder. Exiting."
+		exit 1
+	fi
+	if [ -z "$2" ]; then
+		$INNOBACKUP --apply-log --redo-only $1 2> $LOGFILE
+	else
+		$INNOBACKUP --apply-log --redo-only $1 --incremental-dir=$2 2> $LOGFILE
+	fi
+	check_backup_result
+}
+
+clean_base_backup () {
+	echo "Removing $BACKUPDIR/diffs to ensure clean start..."
+	rm -rf "$BACKUPDIR/diffs"
+	mkdir -p "$BACKUPDIR/diffs"
+	ensure_destination_exists
+	full_backup $BASEDIR
+	prepare_backup $BASEDIR
+	echo "Backup OK. Compressing backup..."
+	tar czf $BACKUPFILE -C $BASEDIR .
+	echo "Creating backup-status"
+	echo 0 > "$BACKUPDIR/diffs/diff-status"
+}
+
+incremental_backup () {
+	# This function requires two arguments:
+	# 1) The base folder for the incremental
+	# 2) The destination folder for the incremental
+	if [[ -z "$1" || -z "$2" ]]; then
+		echo "ERROR: incremental_backup() called without base or destination parameter. Exiting."
+		exit 1
+	fi
+	echo "Running incremental backup with base at $1 into $2"
+	$INNOBACKUP --no-timestamp --user=$USERNAME $PASSCMD $SLAVECMD $NOLOCK --incremental $2 --incremental-basedir=$1 2> $LOGFILE
+	echo "Checking backup result..."
+	check_backup_result
+}
+
+full_compressed_backup () {
+	# This method will create a full backup and compress it into $BACKUPFILE in one step.
+	# First remove the existing full backup, if any.
+	rm -rf "$BACKUPDIR/full"
+	ensure_destination_exists
+	full_backup "$BACKUPDIR/full"
+	echo "Checking backup result..."
+	check_backup_result
+	echo "Backup OK. Compressing backup..."
+	tar czf $BACKUPFILE -C "$BACKUPDIR/full" .
+	echo "Cleaning up"
+	rm -rf "$BACKUPDIR/full"	
+}
+
+check_backup_result () {
+	if [ -z "`tail -1 $LOGFILE | grep 'completed OK!'`" ] ; then
+	 echo "ERROR: $INNOBACKUPEX failed:"
+	 echo "----------------------------"
+	 cat $LOGFILE
+	 rm -f $LOGFILE
+	 rm -f $BACKUPFILE
+	 exit 1
+	fi
+}
+
+ensure_destination_exists () {
+	mkdir -p `dirname $BACKUPFILE`
+}
+
+echo "----------------------------------"
+echo "Started innobackupex backup script"
+echo "Start: `date`"
+echo "----------------------------------"
+
+if [ ! -x "$INNOBACKUP" ]; then
+ echo "$INNOBACKUP does not exist. Ensure you have bootstrapped this instance and that the xtrabackup package is installed."
+ exit 1
+fi
+
+# Only supply --password if password supplied.
+if [ -z "$PASSWORD" ]; then
+	PASSCMD=""
+else
+	PASSCMD="--password=$PASSWORD"
+fi
+
+if [ -z "`$MYSQLADMIN --user=$USERNAME $PASSCMD status | grep 'Uptime'`" ] ; then
+ echo "ERROR: MySQL not running."
+ exit 1
+fi
+
+if ! `echo 'exit' | $MYSQL -s --user=$USERNAME $PASSCMD` ; then
+ echo "ERROR: Mysql username or password is incorrect"
+ exit 1
+fi
+
+if [ ! -d "$BACKUPDIR" ]; then
+	echo "$BACKUPDIR did not exist. Creating..."
+	mkdir -p $BACKUPDIR
+fi
+
+if [ "$DIFFERENTIALS" -eq "1" ]; then
+	echo "Running differential backup..."
+	differential_backup
+	echo "Success. Exiting."
+	exit 0
+else
+	echo "Running full compressed backup..."
+	full_compressed_backup
+	echo "Success. Exiting."
+	exit 0
+fi

data/templates/xtrabackup/config/rubber/role/db/rubber-xtrarestore.sh

@@ -0,0 +1,45 @@
+<%
+  @path = "/mnt/db-backup-tools/rubber-xtrarestore.sh"
+	@perms = 0755
+	@backup = false
+%>#!/bin/bash
+# Variables
+LOGFILE="/tmp/rubber-xtrarestore-log"
+# Take the file from STDIN, write it into a real file, extract it, service mysql stop,
+# then mkdir /mnt/mysql/data & /mnt/mysql/log (move old ones out of the way)
+# then innobackupex --copy-back . in the extracted folder, then service mysql start
+# Create a temporary folder
+rm -rf /mnt/db_restore
+mkdir -p /mnt/db_restore
+# Write STDIN into file
+cat > /mnt/db_restore/current.tar.gz
+cd /mnt/db_restore
+tar xzvf current.tar.gz
+echo 'Stopping MySQL'
+if [ -z "`service mysql stop | grep 'done'`" ] ; then
+	echo "ERROR: Couldn't stop mysql daemon."
+	exit 1
+fi
+rm -rf /mnt/mysql/old
+mkdir -p /mnt/mysql/old
+echo 'Moving Data/Log Directories to /old'
+mv /mnt/mysql/data /mnt/mysql/log /mnt/mysql/old
+mkdir /mnt/mysql/data /mnt/mysql/log
+echo 'Copying back'
+innobackupex --copy-back . 2> $LOGFILE
+if [ -z "`tail -1 $LOGFILE | grep 'completed OK!'`" ] ; then
+	echo "ERROR: Innobackupex couldn't copy back."
+	exit 1
+fi
+chown -R mysql.mysql /mnt/mysql/data
+chown -R mysql.mysql /mnt/mysql/log
+echo 'Starting MySQL'
+if [ -z "`service mysql start | grep 'done'`" ] ; then
+	echo "ERROR: Couldn't start mysql daemon."
+	exit 1
+fi
+echo 'Cleaning up'
+rm -rf /mnt/mysql/old
+rm -rf /mnt/db_restore
+echo "Success."
+exit 0

data/templates/xtrabackup/config/rubber/rubber-xtrabackup.yml

@@ -0,0 +1,6 @@
+role_dependencies:
+  mysql: [xtrabackup]
+  percona: [xtrabackup]  
+roles:
+  xtrabackup:
+    packages: [xtrabackup]

data/templates/xtrabackup/templates.rb

@@ -0,0 +1,17 @@
+if FileTest.exist?("config/rubber/rubber-percona.yml")
+  gsub_file "config/rubber/rubber-percona.yml", /^db_backup_cmd.*$/, \
+  "# Replaced by xtrabackup vulcanizer.\n# \\0\n" + \
+  "# ** If you'd like to run differential backups, add '-d' to the command line below.\n" + \
+  "db_backup_cmd: \"/mnt/db-backup-tools/rubber-xtrabackup.sh -u %user% -p %pass% -t /mnt/db_backups -b %backup_file%\"\n"
+  gsub_file "config/rubber/rubber-percona.yml", /^db_restore_cmd.*$/, \
+  "# Replaced by xtrabackup vulcanizer.\n# \\0\n" + \
+  "db_restore_cmd: \"/mnt/db-backup-tools/rubber-xtrarestore.sh\"\n"
+elsif FileTest.exist?("config/rubber/rubber-mysql.yml")
+  gsub_file "config/rubber/rubber-mysql.yml", /^db_backup_cmd.*$/, \
+  "# Replaced by xtrabackup vulcanizer.\n# \\0\n" + \
+  "# ** If you'd like to run differential backups, add '-d' to the command line below.\n" + \
+  "db_backup_cmd: \"/mnt/db-backup-tools/rubber-xtrabackup.sh -u %user% -p %pass% -t /mnt/db_backups -b %backup_file%\"\n"
+  gsub_file "config/rubber/rubber-mysql.yml", /^db_restore_cmd.*$/, \
+  "# Replaced by xtrabackup vulcanizer.\n# \\0\n" + \
+  "db_restore_cmd: \"/mnt/db-backup-tools/rubber-xtrarestore.sh\"\n"
+end

data/templates/xtrabackup/templates.yml

@@ -0,0 +1 @@
+description: "The xtrabackup module for MySQL/Percona Servers. When vulcanized, this module will replace itself as the Rubber database backup method. After vulcanization, simply run cap rubber:bootstrap (you can filter for DB roles if you like) and cap deploy."

data/test/dns/fog_test.rb

@@ -34,12 +34,28 @@ class FogTest < Test::Unit::TestCase
         zone0 = @dns.find_or_create_zone("example1.com")
         
         assert_equal 1, @dns.client.zones.all.size
-        zone1 = @dns.client.zones.all.find {|z| z.domain =~ /example1.com/ }
+        zone1 = @dns.client.zones.all.find {|z| z.domain =~ /^example1.com/ }
         assert zone1
         assert_equal zone0.id, zone1.id
         assert_equal zone0.domain, zone1.domain
       end
       
+      should "match the same domain that was passed" do
+        @dns = Rubber::Dns::Fog.new(@env)
+    
+        assert_equal 0, @dns.client.zones.all.size
+        
+        zone0 = @dns.find_or_create_zone("abcfoo.com")
+        zone1 = @dns.find_or_create_zone("foo.com")
+        
+        assert_equal 2, @dns.client.zones.all.size
+        
+        zone2 = @dns.client.zones.all.find {|z| z.domain =~ /^foo.com/ }
+        assert zone2
+        assert_equal zone1.id, zone2.id
+        assert_equal zone1.domain, zone2.domain
+      end
+      
       should "do nothing if domain already exists" do
         @dns = Rubber::Dns::Fog.new(@env)
         
@@ -49,7 +65,7 @@ class FogTest < Test::Unit::TestCase
         zone0 = @dns.find_or_create_zone("example1.com")
         
         assert_equal 1, @dns.client.zones.all.size
-        zone1 = @dns.client.zones.all.find {|z| z.domain =~ /example1.com/ }
+        zone1 = @dns.client.zones.all.find {|z| z.domain =~ /^example1.com/ }
         assert_equal zone0.id, zone1.id
         assert_equal zone0.domain, zone1.domain
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubber
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre9
+  version: 2.0.0.pre10
   prerelease: 6
 platform: ruby
 authors:
@@ -10,11 +10,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-27 00:00:00.000000000 Z
+date: 2012-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capistrano
-  requirement: &70315019511180 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -22,10 +22,15 @@ dependencies:
         version: '2.8'
   type: :runtime
   prerelease: false
-  version_requirements: *70315019511180
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &70315019491420 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +38,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70315019491420
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: clamp
-  requirement: &70315019486920 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +54,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70315019486920
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: open4
-  requirement: &70315019481600 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +70,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70315019481600
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fog
-  requirement: &70315019450520 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -66,10 +86,15 @@ dependencies:
         version: '1.2'
   type: :runtime
   prerelease: false
-  version_requirements: *70315019450520
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70315019443320 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -77,10 +102,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70315019443320
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
-  requirement: &70315019434780 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -88,10 +118,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70315019434780
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: shoulda-context
-  requirement: &70315019432700 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -99,10 +134,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70315019432700
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mocha
-  requirement: &70315019431720 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -110,10 +150,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70315019431720
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-debug19
-  requirement: &70315019428140 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -121,7 +166,12 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70315019428140
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ! "    The rubber plugin enables relatively complex multi-instance deployments
   of RubyOnRails applications to\n    Amazon's Elastic Compute Cloud (EC2).  Like
   capistrano, rubber is role based, so you can define a set\n    of configuration
@@ -421,6 +471,13 @@ files:
 - templates/passenger_nginx/config/rubber/role/web_tools/tools-nginx.auth
 - templates/passenger_nginx/config/rubber/rubber-passenger_nginx.yml
 - templates/passenger_nginx/templates.yml
+- templates/percona/config/rubber/common/database.yml
+- templates/percona/config/rubber/deploy-percona.rb
+- templates/percona/config/rubber/role/db/apparmor-mysql.conf
+- templates/percona/config/rubber/role/db/crontab
+- templates/percona/config/rubber/role/db/my.cnf
+- templates/percona/config/rubber/rubber-percona.yml
+- templates/percona/templates.yml
 - templates/postgresql/config/rubber/common/database.yml
 - templates/postgresql/config/rubber/deploy-postgresql.rb
 - templates/postgresql/config/rubber/role/db/crontab
@@ -458,6 +515,12 @@ files:
 - templates/unicorn/config/rubber/role/unicorn/unicorn.rb
 - templates/unicorn/config/rubber/rubber-unicorn.yml
 - templates/unicorn/templates.yml
+- templates/xtrabackup/config/rubber/deploy-xtrabackup.rb
+- templates/xtrabackup/config/rubber/role/db/rubber-xtrabackup.sh
+- templates/xtrabackup/config/rubber/role/db/rubber-xtrarestore.sh
+- templates/xtrabackup/config/rubber/rubber-xtrabackup.yml
+- templates/xtrabackup/templates.rb
+- templates/xtrabackup/templates.yml
 - test/cloud/aws_table_store_test.rb
 - test/cloud/aws_test.rb
 - test/cloud/fog_storage_test.rb
@@ -506,7 +569,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1969407401613214093
+      hash: 4223981313086308948
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -515,43 +578,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: rubber
-rubygems_version: 1.8.15
+rubygems_version: 1.8.19
 signing_key: 
 specification_version: 3
 summary: A capistrano plugin for managing multi-instance deployments to the cloud
   (ec2)
-test_files:
-- test/cloud/aws_table_store_test.rb
-- test/cloud/aws_test.rb
-- test/cloud/fog_storage_test.rb
-- test/cloud/fog_test.rb
-- test/command_test.rb
-- test/dns/fog_test.rb
-- test/environment_test.rb
-- test/fixtures/basic/common/bar.conf
-- test/fixtures/basic/common/foo.conf
-- test/fixtures/basic/host/host1/foo.conf
-- test/fixtures/basic/host/host2/foo.conf
-- test/fixtures/basic/role/role1/foo.conf
-- test/fixtures/basic/role/role2/foo.conf
-- test/fixtures/basic/rubber-extra.yml
-- test/fixtures/basic/rubber.yml
-- test/fixtures/expansion/rubber.yml
-- test/fixtures/generator_order/common/a_first.conf
-- test/fixtures/generator_order/common/z_last.conf
-- test/fixtures/generator_order/host/host1/a_first.conf
-- test/fixtures/generator_order/host/host1/z_last.conf
-- test/fixtures/generator_order/role/role1/a_first.conf
-- test/fixtures/generator_order/role/role1/z_last.conf
-- test/fixtures/generator_order/role/role2/a_first.conf
-- test/fixtures/generator_order/role/role2/z_last.conf
-- test/fixtures/instance_expansion/instance-test.yml
-- test/fixtures/instance_expansion/rubber.yml
-- test/fixtures/nested/rubber.yml
-- test/fixtures/secret/rubber.yml
-- test/fixtures/secret/secret.yml
-- test/generator_test.rb
-- test/instance_test.rb
-- test/test-rails-template.rb
-- test/test_helper.rb
-- test/util_test.rb
+test_files: []