nirvdrum-rubber 1.1.7 → 2.0.0.rails3.beta6

data/lib/generators/vulcanize/templates/munin/script/munin/example_simple.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+#
+# Example munin plugin.
+# See munin plugin docs http://munin.projects.linpro.no/wiki/Documentation#Plugins
+#
+# Scripts added to RUBBER_ROOT/script/munin automatically
+# get installed as munin plugins by rubber
+
+# remove this line to enable
+exit 0
+
+# Print config info need by munin for generating graphs
+if ARGV[0] == "config"
+  puts 'graph_title Example Stats'
+  puts 'graph_vlabel random 1-100'
+  puts 'graph_category Examples'
+  puts 'graph_scale no'
+  puts 'rand.label random'
+
+  exit 0
+end
+
+# print graph data values
+puts "rand.value #{rand(100)}"

data/lib/generators/vulcanize/templates/munin/templates.yml

@@ -0,0 +1 @@
+description: The munin module

data/lib/generators/vulcanize/templates/mysql/config/rubber/common/database.yml

@@ -0,0 +1,12 @@
+<%
+  @path = "#{RUBBER_ROOT}/config/database.yml"
+%>
+
+<%= RUBBER_ENV %>:
+  adapter: mysql
+  encoding: utf8
+  database: <%= rubber_env.db_name %>
+  username: <%= rubber_env.db_user %>
+  password: <%= rubber_env.db_pass %>
+  host: <%= rubber_instances.for_role('db', 'primary' => true).first.full_name %>
+  pool: 5

data/lib/generators/vulcanize/templates/mysql/config/rubber/deploy-mysql.rb

@@ -0,0 +1,169 @@
+
+namespace :rubber do
+  
+  namespace :mysql do
+    
+    rubber.allow_optional_tasks(self)
+    
+    after "rubber:create", "rubber:mysql:validate_db_roles"
+
+    task :validate_db_roles do
+      db_instances = rubber_instances.for_role("mysql_slave")
+      db_instances.each do |instance|
+        if instance.role_names.find {|n| n == 'mysql_master'}
+          fatal "Cannot have a mysql slave and master on the same instance, please removing slave role for #{instance.name}"
+        end
+      end
+    end
+
+    after "rubber:bootstrap", "rubber:mysql:bootstrap"
+  
+    
+    # Bootstrap the production database config.  Db bootstrap is special - the
+    # user could be requiring the rails env inside some of their config
+    # templates, which creates a catch 22 situation with the db, so we try and
+    # bootstrap the db separate from the rest of the config
+    task :bootstrap, :roles => [:mysql_master, :mysql_slave] do
+      
+      # Conditionaly bootstrap for each node/role only if that node has not
+      # been boostrapped for that role before
+      
+      master_instances = rubber_instances.for_role("mysql_master") & rubber_instances.filtered  
+      master_instances.each do |ic|
+        task_name = "_bootstrap_mysql_master_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          env = rubber_cfg.environment.bind("mysql_master", ic.name)
+          exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
+          if exists.strip.size == 0
+            common_bootstrap("mysql_master")
+            
+            pass = "identified by '#{env.db_pass}'" if env.db_pass
+            rubber.sudo_script "create_master_db", <<-ENDSCRIPT
+              mysql -u root -e "create database #{env.db_name};"
+              mysql -u root -e "grant all on *.* to '#{env.db_user}'@'%' #{pass};"
+              mysql -u root -e "grant select on *.* to '#{env.db_slave_user}'@'%' #{pass};"
+              mysql -u root -e "grant replication slave on *.* to '#{env.db_replicator_user}'@'%' #{pass};"
+              mysql -u root -e "flush privileges;"
+            ENDSCRIPT
+          end
+        end
+        send task_name
+      end
+    
+      slave_instances = rubber_instances.for_role("mysql_slave") & rubber_instances.filtered  
+      slave_instances.each do |ic|
+        task_name = "_bootstrap_mysql_slave_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          env = rubber_cfg.environment.bind("mysql_slave", ic.name)
+          exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
+          if exists.strip.size == 0
+            common_bootstrap("mysql_slave")
+
+            master = rubber_instances.for_role("mysql_master").first
+
+            # Doing a mysqldump locks the db, so ideally we'd do it against a slave replica thats
+            # not serving traffic (mysql_util role), but if thats not available try a regular
+            # slave (mysql_slave role), and finally default dumping from master (mysql_master role)
+            # TODO: handle simultaneous creating of multi slaves/utils
+            slaves = rubber_instances.for_role("mysql_slave")
+            slaves.delete(ic) # don't want to try and dump from self
+            source = slaves.find {|sc| sc.role_names.include?("mysql_util")}
+            source = slaves.first unless source
+            source = master unless source
+
+            pass = "identified by '#{env.db_pass}'" if env.db_pass
+            master_pass = ", master_password='#{env.db_pass}'" if env.db_pass
+            master_host = master.full_name
+            source_host = source.full_name
+
+            if source == master
+              logger.info "Creating slave from a dump of master #{source_host}"
+              rubber.sudo_script "create_slave_db_from_master", <<-ENDSCRIPT
+                mysql -u root -e "change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}' #{master_pass}"
+                mysqldump -u #{env.db_user} --password #{env.db_pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root
+              ENDSCRIPT
+            else
+              logger.info "Creating slave from a dump of slave #{source_host}"
+              rsudo "mysql -u #{env.db_user} --password #{env.db_pass} -h #{source_host} -e \"stop slave;\""
+              slave_status = capture("mysql -u #{env.db_user} #{pass} -h #{source_host} -e \"show slave status\\G\"")
+              slave_config = Hash[*slave_status.scan(/([^\s:]+): ([^\s]*)/).flatten]
+              log_file = slave_config['Master_Log_File']
+              log_pos = slave_config['Read_Master_Log_Pos']
+              rubber.sudo_script "create_slave_db_from_slave", <<-ENDSCRIPT
+                mysqldump -u #{env.db_user} --password #{env.db_pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root
+                mysql -u root -e "change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}', master_log_file='#{log_file}', master_log_pos=#{log_pos} #{master_pass}"
+                mysql -u #{env.db_user} --password #{env.db_pass} -h #{source_host} -e "start slave;"
+              ENDSCRIPT
+            end
+
+            # this doesn't work without agent forwarding which sudo breaks, as well as not having your
+            # ec2 private key ssh-added on workstation
+            # sudo "scp -o \"StrictHostKeyChecking=no\" #{source_host}:/etc/mysql/debian.cnf /etc/mysql"
+
+            rsudo "mysql -u root -e \"flush privileges;\""
+            rsudo "mysql -u root -e \"start slave;\""
+          end
+        end
+        send task_name
+      end
+      
+    end
+  
+    # TODO: Make the setup/update happen just once per host
+    def common_bootstrap(role)
+      # mysql package install starts mysql, so stop it
+      rsudo "service mysql stop" rescue nil
+      
+      # After everything installed on machines, we need the source tree
+      # on hosts in order to run rubber:config for bootstrapping the db
+      rubber.update_code_for_bootstrap
+      
+      # Gen just the conf for the given mysql role
+      rubber.run_config(:RUBBER_ENV => RUBBER_ENV, :FILE => "role/#{role}|role/db/", :FORCE => true, :deploy_path => release_path)
+
+      # reconfigure mysql so that it sets up data dir in /mnt with correct files
+      sudo_script 'reconfigure-mysql', <<-ENDSCRIPT
+        server_package=`dpkg -l | grep mysql-server-[0-9] | awk '{print $2}'`
+        dpkg-reconfigure --frontend=noninteractive $server_package
+      ENDSCRIPT
+      sleep 5
+    end
+    
+    before "rubber:munin:custom_install", "rubber:mysql:custom_install_munin"
+
+    desc <<-DESC
+      Installs some extra munin graphs
+    DESC
+    task :custom_install_munin, :roles => [:mysql_master, :mysql_slave] do
+      rubber.sudo_script 'install_munin_mysql', <<-ENDSCRIPT
+        if [ ! -f /usr/share/munin/plugins/mysql_ ]; then
+          wget -qN -O /usr/share/munin/plugins/mysql_ http://github.com/kjellm/munin-mysql/raw/master/mysql_
+          wget -qN -O /etc/munin/plugin-conf.d/mysql_.conf http://github.com/kjellm/munin-mysql/raw/master/mysql_.conf
+        fi
+      ENDSCRIPT
+    end
+
+    desc <<-DESC
+      Starts the mysql daemons
+    DESC
+    task :start, :roles => [:mysql_master, :mysql_slave] do
+      rsudo "service mysql start"
+    end
+    
+    desc <<-DESC
+      Stops the mysql daemons
+    DESC
+    task :stop, :roles => [:mysql_master, :mysql_slave] do
+      rsudo "service mysql stop"
+    end
+  
+    desc <<-DESC
+      Restarts the mysql daemons
+    DESC
+    task :restart, :roles => [:mysql_master, :mysql_slave] do
+      rsudo "service mysql restart"
+    end
+  
+  end
+
+end

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/db/apparmor-mysql.conf

@@ -0,0 +1,46 @@
+<%
+  @path = '/etc/apparmor.d/usr.sbin.mysqld'
+  @post = "service apparmor restart"
+  @backup = false
+%>
+
+#include <tunables/global>
+
+/usr/sbin/mysqld {
+  #include <abstractions/base>
+  #include <abstractions/nameservice>
+  #include <abstractions/user-tmp>
+  #include <abstractions/mysql>
+  #include <abstractions/winbind>
+
+  capability dac_override,
+  capability sys_resource,
+  capability setgid,
+  capability setuid,
+
+  network tcp,
+
+  /etc/hosts.allow r,
+  /etc/hosts.deny r,
+
+  /etc/mysql/*.pem r,
+  /etc/mysql/conf.d/ r,
+  /etc/mysql/conf.d/* r,
+  /etc/mysql/my.cnf r,
+  /usr/sbin/mysqld mr,
+  /usr/share/mysql/** r,
+  /var/log/mysql.log rw,
+  /var/log/mysql.err rw,
+  /var/lib/mysql/ r,
+  /var/lib/mysql/** rwk,
+  /var/log/mysql/ r,
+  /var/log/mysql/* rw,
+  /var/run/mysqld/mysqld.pid w,
+  /var/run/mysqld/mysqld.sock w,
+
+  /sys/devices/system/cpu/ r,
+  /sys/devices/system/cpu/** r,
+
+  <%= rubber_env.db_root_dir %>/ r,
+  <%= rubber_env.db_root_dir %>/** rwk,
+}

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/db/crontab

@@ -0,0 +1,14 @@
+<%
+  @read_cmd = 'crontab -l'
+  @write_cmd = 'crontab -'
+  @additive = ["# rubber-mysql-start", "# rubber-mysql-end"]
+%>
+
+<%
+  # Backup db every 3 hours to secondary db if available, else primary db
+  backup_db_host = rubber_instances.for_role('db', {}).first rescue nil
+  backup_db_host ||= rubber_instances.for_role('db', 'primary' => true).first
+  if backup_db_host.name == rubber_env.host
+%>
+0 */3 * * * BACKUP_DIR=/mnt/db_backups DBUSER=<%= rubber_env.db_user %> DBPASS=<%= rubber_env.db_pass %> DBNAME=<%= rubber_env.db_name %> DBHOST=<%= backup_db_host.full_name %> <%= RUBBER_ROOT %>/script/cron-rake rubber:backup_db
+<% end %>

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/db/monit-mysql.cnf

@@ -0,0 +1,10 @@
+<%
+  @path = '/etc/monit/monit.d/monit-mysql.conf'
+%>
+
+check process mysql with pidfile /var/run/mysqld/mysqld.pid
+   group mysql-<%= RUBBER_ENV %>
+   start program = "/usr/sbin/service mysql restart"
+   stop program = "/usr/sbin/service mysql stop"
+   if failed host 127.0.0.1 port 3306 protocol mysql then alert
+   if 5 restarts within 5 cycles then timeout

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/db/my.cnf

@@ -0,0 +1,163 @@
+<%
+  data_dir = rubber_env.db_data_dir
+  log_dir = rubber_env.db_log_dir
+  server_id = rubber_env.db_server_id 
+  @path = '/etc/mysql/my.cnf'
+  @post = "mkdir -p #{log_dir}; chown mysql:adm #{log_dir}; chmod 2750 #{log_dir}"
+%>
+
+# Generated by rubber for host <%= rubber_instances[rubber_env.host].external_ip %>
+#
+# The MySQL database server configuration file.
+#
+# You can copy this to one of:
+# - "/etc/mysql/my.cnf" to set global options,
+# - "~/.my.cnf" to set user-specific options.
+#
+# One can use all long options that the program supports.
+# Run program with --help to get a list of available options and with
+# --print-defaults to see which it would actually understand and use.
+#
+# For explanations see
+# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
+
+# This will be passed to all mysql clients
+# It has been reported that passwords should be enclosed with ticks/quotes
+# escpecially if they contain "#" chars...
+# Remember to edit /etc/mysql/debian.cnf when changing the socket location.
+[client]
+default-character-set = utf8
+port    = 3306
+socket    = /var/run/mysqld/mysqld.sock
+
+# Here is entries for some specific programs
+# The following values assume you have at least 32M ram
+
+# This was formally known as [safe_mysqld]. Both versions are currently parsed.
+[mysqld_safe]
+socket    = /var/run/mysqld/mysqld.sock
+nice    = 0
+
+[mysqld]
+#
+# * Basic Settings
+#
+default-character-set = utf8
+user    = mysql
+pid-file  = /var/run/mysqld/mysqld.pid
+socket    = /var/run/mysqld/mysqld.sock
+port    = 3306
+basedir   = /usr
+datadir = <%= data_dir %>
+tmpdir    = /tmp
+language  = /usr/share/mysql/english
+skip-external-locking
+
+#
+# Instead of skip-networking the default is now to listen only on
+# localhost which is more compatible and is not less secure.
+# bind-address    = 127.0.0.1
+#
+# * Fine Tuning
+#
+key_buffer    = 16M
+max_allowed_packet  = 32M
+thread_stack    = 128K
+thread_cache_size = 8
+#table_cache=500
+#max_heap_table_size=16M
+#tmp_table_size=32M
+
+#max_connections        = 100
+#table_cache            = 64
+#thread_concurrency     = 10
+#
+# * Query Cache Configuration
+#
+query_cache_limit       = 1M
+query_cache_size        = 16M
+#
+# * Logging and Replication
+#
+# Both location gets rotated by the cronjob.
+# Be aware that this log type is a performance killer.
+#log    = /var/log/mysql/mysql.log
+#
+# Error logging goes to syslog. This is a Debian improvement :)
+#
+# Here you can see queries with especially long duration
+log_slow_queries = /var/log/mysql/mysql-slow.log
+long_query_time = 2
+log-queries-not-using-indexes
+#
+# The following can be used as easy to replay backup logs or for replication.
+# note: if you are setting up a replication slave, see README.Debian about
+#       other settings you may need to change.
+server-id = <%= server_id %>
+log_bin     = <%= log_dir %>/mysql-bin.log
+relay_log   = <%= log_dir %>/mysql-relay-bin.log
+
+# WARNING: Using expire_logs_days without bin_log crashes the server! See README.Debian!
+expire_logs_days  = 10
+max_binlog_size         = 100M
+#binlog_do_db   = include_database_name
+#binlog_ignore_db = include_database_name
+
+
+# * InnoDB
+#
+# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
+# Read the manual for more InnoDB related options. There are many!
+# You might want to disable InnoDB to shrink the mysqld process by circa 100MB.
+#skip-innodb
+
+# Set buffer pool size to 50-80% of your computer's memory
+#innodb_buffer_pool_size=512M
+#innodb_additional_mem_pool_size=64M
+
+# Set the log file size to about 25% of the buffer pool size
+#innodb_log_file_size=128M
+#innodb_log_buffer_size=32M
+#innodb_flush_log_at_trx_commit=2
+
+#
+# * Security Features
+#
+# Read the manual, too, if you want chroot!
+# chroot = /var/lib/mysql/
+#
+# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
+#
+# ssl-ca=/etc/mysql/cacert.pem
+# ssl-cert=/etc/mysql/server-cert.pem
+# ssl-key=/etc/mysql/server-key.pem
+
+
+
+[mysqldump]
+quick
+quote-names
+max_allowed_packet  = 16M
+
+[mysql]
+#no-auto-rehash # faster start of mysql but no tab completition
+
+[isamchk]
+key_buffer    = 16M
+
+#
+# * NDB Cluster
+#
+# See /usr/share/doc/mysql-server-*/README.Debian for more information.
+#
+# The following configuration is read by the NDB Data Nodes (ndbd processes)
+# not from the NDB Management Nodes (ndb_mgmd processes).
+#
+# [MYSQL_CLUSTER]
+# ndb-connectstring=127.0.0.1
+
+
+#
+# * IMPORTANT: Additional settings that can override those from this file!
+#
+!includedir /etc/mysql/conf.d/

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/mysql_slave/mysql_slave_munin_plugin

@@ -0,0 +1,51 @@
+<%
+  @path = "/etc/munin/plugins/mysql_slave"
+  @perms = 0755
+%>#!/bin/sh
+#
+# Plugin to monitor the slave delay
+#
+# Parameters:
+#   
+#   config
+#   autoconf
+#
+# Configuration variables
+#
+# mysqlopts    - Options to pass to mysql
+#
+#%# family=auto
+#%# capabilities=autoconf
+
+MYSQLOPTS="$mysqlopts"
+MYSQLADMIN=${mysqladmin:-mysqladmin}
+
+if [ "$1" = "autoconf" ]; then
+        $MYSQLADMIN --version 2>/dev/null >/dev/null
+        if [ $? -eq 0 ]
+        then
+                $MYSQLADMIN $MYSQLOPTS status 2>/dev/null >/dev/null
+                if [ $? -eq 0 ]
+                then
+                        echo yes
+                        exit 0
+                else
+                        echo "no (could not connect to mysql)"
+                fi
+        else
+                echo "no (mysqladmin not found)"
+        fi
+        exit 1
+fi
+
+if [ "$1" = "config" ]; then
+  echo 'graph_title MySQL Slave Delay'
+  echo 'graph_vlabel seconds'
+  echo 'graph_category mysql'
+  echo 'slave.label slave delay'
+  echo 'graph_args --base 1000'
+  exit 0
+fi
+
+/usr/bin/printf "slave.value "
+echo "show slave status\G"| mysql $MYSQLOPTS | grep "Seconds_Behind_Master" | awk '{print $2}'

data/lib/generators/vulcanize/templates/mysql/config/rubber/rubber-mysql.yml

@@ -0,0 +1,47 @@
+
+# REQUIRED: The credentials for creating/accessong your app's database
+#
+db_user: "#{app_name[0,16]}"
+db_pass:
+db_name: "#{app_name}_#{RUBBER_ENV}"
+
+db_slave_user: "#{('slave_' + db_user)[0,16]}"
+db_replicator_user: "#{('replicator_' + db_user)[0,16]}"
+
+# REQUIRED: The command to use to do a full backup of your database using
+#           "rake rubber:backup_db"
+db_backup_cmd: "nice mysqldump -h %host% -u %user% --password=%pass% %name% | gzip -c > %backup_file%"
+# REQUIRED: The command to use to restore the database using
+#           "rake rubber:restore_db_s3"
+#           This command will receive the data generated by db_backup_cmd via
+#           its standard input
+db_restore_cmd: "zcat - | mysql -h %host% -u %user% --password=%pass% %name%"
+
+# REQUIRED: The directory to store the db data in
+#
+db_root_dir: /mnt/mysql
+db_data_dir: "#{db_root_dir}/data"
+db_log_dir: "#{db_root_dir}/log"
+
+# Capistrano needs db:primary role for migrate to work,
+# so we might as well just make consistent across the board
+role_dependencies:
+  mysql_master: [db:primary=true]
+  mysql_slave: [db]
+  db:primary=true: [mysql_master]
+  db: [mysql_slave]
+
+packages: [mysql-client, libmysqlclient15-dev]
+gems: [mysql]
+
+roles:
+  mysql_master:
+    db_server_id: 1
+  mysql_slave:
+    db_server_id: 2
+  db:
+    packages: [
+                mysql-server,
+                maatkit, # useful mysql tools for replication
+                libdbi-perl, libdbd-mysql, libcache-cache-perl # for extended munin plugins
+              ]

data/lib/generators/vulcanize/templates/mysql/templates.yml

@@ -0,0 +1 @@
+description: The mysql module

data/lib/generators/vulcanize/templates/mysql_cluster/config/rubber/common/mysql_cluster_migrations.rb

@@ -0,0 +1,13 @@
+<%
+  @path = "#{RUBBER_ROOT}/config/initializers/mysql_cluster_migrations.rb"
+%>
+
+# mysql adapter in rails hardcodes engine to be innodb, so if we want all
+# rails tables to be clustered, we need to override this behavior
+#
+class ActiveRecord::ConnectionAdapters::MysqlAdapter
+  def create_table(table_name, options = {}) #:nodoc:
+    super(table_name, options.reverse_merge(:options => "ENGINE=ndbcluster"))
+  end
+end
+