rubber 1.3.2 → 1.5.0

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/db/apparmor-mysql.conf

@@ -0,0 +1,46 @@
+<%
+  @path = '/etc/apparmor.d/usr.sbin.mysqld'
+  @post = "service apparmor restart"
+  @backup = false
+%>
+
+#include <tunables/global>
+
+/usr/sbin/mysqld {
+  #include <abstractions/base>
+  #include <abstractions/nameservice>
+  #include <abstractions/user-tmp>
+  #include <abstractions/mysql>
+  #include <abstractions/winbind>
+
+  capability dac_override,
+  capability sys_resource,
+  capability setgid,
+  capability setuid,
+
+  network tcp,
+
+  /etc/hosts.allow r,
+  /etc/hosts.deny r,
+
+  /etc/mysql/*.pem r,
+  /etc/mysql/conf.d/ r,
+  /etc/mysql/conf.d/* r,
+  /etc/mysql/my.cnf r,
+  /usr/sbin/mysqld mr,
+  /usr/share/mysql/** r,
+  /var/log/mysql.log rw,
+  /var/log/mysql.err rw,
+  /var/lib/mysql/ r,
+  /var/lib/mysql/** rwk,
+  /var/log/mysql/ r,
+  /var/log/mysql/* rw,
+  /var/run/mysqld/mysqld.pid w,
+  /var/run/mysqld/mysqld.sock w,
+
+  /sys/devices/system/cpu/ r,
+  /sys/devices/system/cpu/** r,
+
+  <%= rubber_env.db_root_dir %>/ r,
+  <%= rubber_env.db_root_dir %>/** rwk,
+}

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/db/crontab

@@ -0,0 +1,14 @@
+<%
+  @read_cmd = 'crontab -l'
+  @write_cmd = 'crontab -'
+  @additive = ["# rubber-mysql-start", "# rubber-mysql-end"]
+%>
+
+<%
+  # Backup db every 3 hours to secondary db if available, else primary db
+  backup_db_host = rubber_instances.for_role('db', {}).first rescue nil
+  backup_db_host ||= rubber_instances.for_role('db', 'primary' => true).first
+  if backup_db_host.name == rubber_env.host
+%>
+0 */3 * * * BACKUP_DIR=/mnt/db_backups DBUSER=<%= rubber_env.db_user %> DBPASS=<%= rubber_env.db_pass %> DBNAME=<%= rubber_env.db_name %> DBHOST=<%= backup_db_host.full_name %> <%= RUBBER_ROOT %>/script/cron-rake rubber:backup_db
+<% end %>

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/db/monit-mysql.cnf

@@ -0,0 +1,10 @@
+<%
+  @path = '/etc/monit/monit.d/monit-mysql.conf'
+%>
+
+check process mysql with pidfile /var/run/mysqld/mysqld.pid
+   group mysql-<%= RUBBER_ENV %>
+   start program = "/usr/sbin/service mysql restart"
+   stop program = "/usr/sbin/service mysql stop"
+   if failed host 127.0.0.1 port 3306 protocol mysql then alert
+   if 5 restarts within 5 cycles then timeout

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/db/my.cnf

@@ -0,0 +1,163 @@
+<%
+  data_dir = rubber_env.db_data_dir
+  log_dir = rubber_env.db_log_dir
+  server_id = rubber_env.db_server_id 
+  @path = '/etc/mysql/my.cnf'
+  @post = "mkdir -p #{log_dir}; chown mysql:adm #{log_dir}; chmod 2750 #{log_dir}"
+%>
+
+# Generated by rubber for host <%= rubber_instances[rubber_env.host].external_ip %>
+#
+# The MySQL database server configuration file.
+#
+# You can copy this to one of:
+# - "/etc/mysql/my.cnf" to set global options,
+# - "~/.my.cnf" to set user-specific options.
+#
+# One can use all long options that the program supports.
+# Run program with --help to get a list of available options and with
+# --print-defaults to see which it would actually understand and use.
+#
+# For explanations see
+# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
+
+# This will be passed to all mysql clients
+# It has been reported that passwords should be enclosed with ticks/quotes
+# escpecially if they contain "#" chars...
+# Remember to edit /etc/mysql/debian.cnf when changing the socket location.
+[client]
+default-character-set = utf8
+port    = 3306
+socket    = /var/run/mysqld/mysqld.sock
+
+# Here is entries for some specific programs
+# The following values assume you have at least 32M ram
+
+# This was formally known as [safe_mysqld]. Both versions are currently parsed.
+[mysqld_safe]
+socket    = /var/run/mysqld/mysqld.sock
+nice    = 0
+
+[mysqld]
+#
+# * Basic Settings
+#
+default-character-set = utf8
+user    = mysql
+pid-file  = /var/run/mysqld/mysqld.pid
+socket    = /var/run/mysqld/mysqld.sock
+port    = 3306
+basedir   = /usr
+datadir = <%= data_dir %>
+tmpdir    = /tmp
+language  = /usr/share/mysql/english
+skip-external-locking
+
+#
+# Instead of skip-networking the default is now to listen only on
+# localhost which is more compatible and is not less secure.
+# bind-address    = 127.0.0.1
+#
+# * Fine Tuning
+#
+key_buffer    = 16M
+max_allowed_packet  = 32M
+thread_stack    = 128K
+thread_cache_size = 8
+#table_cache=500
+#max_heap_table_size=16M
+#tmp_table_size=32M
+
+#max_connections        = 100
+#table_cache            = 64
+#thread_concurrency     = 10
+#
+# * Query Cache Configuration
+#
+query_cache_limit       = 1M
+query_cache_size        = 16M
+#
+# * Logging and Replication
+#
+# Both location gets rotated by the cronjob.
+# Be aware that this log type is a performance killer.
+#log    = /var/log/mysql/mysql.log
+#
+# Error logging goes to syslog. This is a Debian improvement :)
+#
+# Here you can see queries with especially long duration
+log_slow_queries = /var/log/mysql/mysql-slow.log
+long_query_time = 2
+log-queries-not-using-indexes
+#
+# The following can be used as easy to replay backup logs or for replication.
+# note: if you are setting up a replication slave, see README.Debian about
+#       other settings you may need to change.
+server-id = <%= server_id %>
+log_bin     = <%= log_dir %>/mysql-bin.log
+relay_log   = <%= log_dir %>/mysql-relay-bin.log
+
+# WARNING: Using expire_logs_days without bin_log crashes the server! See README.Debian!
+expire_logs_days  = 10
+max_binlog_size         = 100M
+#binlog_do_db   = include_database_name
+#binlog_ignore_db = include_database_name
+
+
+# * InnoDB
+#
+# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
+# Read the manual for more InnoDB related options. There are many!
+# You might want to disable InnoDB to shrink the mysqld process by circa 100MB.
+#skip-innodb
+
+# Set buffer pool size to 50-80% of your computer's memory
+#innodb_buffer_pool_size=512M
+#innodb_additional_mem_pool_size=64M
+
+# Set the log file size to about 25% of the buffer pool size
+#innodb_log_file_size=128M
+#innodb_log_buffer_size=32M
+#innodb_flush_log_at_trx_commit=2
+
+#
+# * Security Features
+#
+# Read the manual, too, if you want chroot!
+# chroot = /var/lib/mysql/
+#
+# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
+#
+# ssl-ca=/etc/mysql/cacert.pem
+# ssl-cert=/etc/mysql/server-cert.pem
+# ssl-key=/etc/mysql/server-key.pem
+
+
+
+[mysqldump]
+quick
+quote-names
+max_allowed_packet  = 16M
+
+[mysql]
+#no-auto-rehash # faster start of mysql but no tab completition
+
+[isamchk]
+key_buffer    = 16M
+
+#
+# * NDB Cluster
+#
+# See /usr/share/doc/mysql-server-*/README.Debian for more information.
+#
+# The following configuration is read by the NDB Data Nodes (ndbd processes)
+# not from the NDB Management Nodes (ndb_mgmd processes).
+#
+# [MYSQL_CLUSTER]
+# ndb-connectstring=127.0.0.1
+
+
+#
+# * IMPORTANT: Additional settings that can override those from this file!
+#
+!includedir /etc/mysql/conf.d/

data/lib/generators/vulcanize/templates/mysql/config/rubber/role/mysql_slave/mysql_slave_munin_plugin

@@ -0,0 +1,51 @@
+<%
+  @path = "/etc/munin/plugins/mysql_slave"
+  @perms = 0755
+%>#!/bin/sh
+#
+# Plugin to monitor the slave delay
+#
+# Parameters:
+#   
+#   config
+#   autoconf
+#
+# Configuration variables
+#
+# mysqlopts    - Options to pass to mysql
+#
+#%# family=auto
+#%# capabilities=autoconf
+
+MYSQLOPTS="$mysqlopts"
+MYSQLADMIN=${mysqladmin:-mysqladmin}
+
+if [ "$1" = "autoconf" ]; then
+        $MYSQLADMIN --version 2>/dev/null >/dev/null
+        if [ $? -eq 0 ]
+        then
+                $MYSQLADMIN $MYSQLOPTS status 2>/dev/null >/dev/null
+                if [ $? -eq 0 ]
+                then
+                        echo yes
+                        exit 0
+                else
+                        echo "no (could not connect to mysql)"
+                fi
+        else
+                echo "no (mysqladmin not found)"
+        fi
+        exit 1
+fi
+
+if [ "$1" = "config" ]; then
+  echo 'graph_title MySQL Slave Delay'
+  echo 'graph_vlabel seconds'
+  echo 'graph_category mysql'
+  echo 'slave.label slave delay'
+  echo 'graph_args --base 1000'
+  exit 0
+fi
+
+/usr/bin/printf "slave.value "
+echo "show slave status\G"| mysql $MYSQLOPTS | grep "Seconds_Behind_Master" | awk '{print $2}'

data/lib/generators/vulcanize/templates/mysql/config/rubber/rubber-mysql.yml

@@ -0,0 +1,47 @@
+
+# REQUIRED: The credentials for creating/accessong your app's database
+#
+db_user: "#{app_name[0,16]}"
+db_pass:
+db_name: "#{app_name}_#{RUBBER_ENV}"
+
+db_slave_user: "#{('slave_' + db_user)[0,16]}"
+db_replicator_user: "#{('replicator_' + db_user)[0,16]}"
+
+# REQUIRED: The command to use to do a full backup of your database using
+#           "rake rubber:backup_db"
+db_backup_cmd: "nice mysqldump -h %host% -u %user% --password=%pass% %name% | gzip -c > %backup_file%"
+# REQUIRED: The command to use to restore the database using
+#           "rake rubber:restore_db_s3"
+#           This command will receive the data generated by db_backup_cmd via
+#           its standard input
+db_restore_cmd: "zcat - | mysql -h %host% -u %user% --password=%pass% %name%"
+
+# REQUIRED: The directory to store the db data in
+#
+db_root_dir: /mnt/mysql
+db_data_dir: "#{db_root_dir}/data"
+db_log_dir: "#{db_root_dir}/log"
+
+# Capistrano needs db:primary role for migrate to work,
+# so we might as well just make consistent across the board
+role_dependencies:
+  mysql_master: [db:primary=true]
+  mysql_slave: [db]
+  db:primary=true: [mysql_master]
+  db: [mysql_slave]
+
+packages: [mysql-client, libmysqlclient15-dev]
+gems: [mysql]
+
+roles:
+  mysql_master:
+    db_server_id: 1
+  mysql_slave:
+    db_server_id: 2
+  db:
+    packages: [
+                mysql-server,
+                maatkit, # useful mysql tools for replication
+                libdbi-perl, libdbd-mysql, libcache-cache-perl # for extended munin plugins
+              ]

data/lib/generators/vulcanize/templates/mysql/templates.yml

@@ -0,0 +1 @@
+description: The mysql module

data/lib/generators/vulcanize/templates/mysql_cluster/config/rubber/common/mysql_cluster_migrations.rb

@@ -0,0 +1,13 @@
+<%
+  @path = "#{RUBBER_ROOT}/config/initializers/mysql_cluster_migrations.rb"
+%>
+
+# mysql adapter in rails hardcodes engine to be innodb, so if we want all
+# rails tables to be clustered, we need to override this behavior
+#
+class ActiveRecord::ConnectionAdapters::MysqlAdapter
+  def create_table(table_name, options = {}) #:nodoc:
+    super(table_name, options.reverse_merge(:options => "ENGINE=ndbcluster"))
+  end
+end
+

data/lib/generators/vulcanize/templates/mysql_cluster/config/rubber/deploy-mysql_cluster.rb

@@ -0,0 +1,177 @@
+
+namespace :rubber do
+  
+  namespace :mysql_cluster do
+    
+    rubber.allow_optional_tasks(self)
+  
+    after "rubber:create", "rubber:mysql_cluster:set_db_role"
+      
+    # Capistrano needs db:primary role for migrate to work
+    task :set_db_role do
+      sql_instances = rubber_instances.for_role("mysql_sql")
+      sql_instances.each do |instance|
+        if ! instance.role_names.find {|n| n == 'db'}
+          role = Rubber::Configuration::RoleItem.new('db')
+          primary_exists = rubber_instances.for_role("db", "primary" => true).size > 0
+          role.options["primary"] = true  unless primary_exists
+          instance.roles << role
+        end
+      end
+      rubber_instances.save()
+      load_roles() unless rubber_env.disable_auto_roles
+    end
+    
+    before "rubber:install_packages", "rubber:mysql_cluster:install"
+  
+    task :install, :roles => [:mysql_mgm, :mysql_data, :mysql_sql] do
+      # Setup apt sources to get a newer version of mysql cluster
+      # https://launchpad.net/~mysql-cge-testing/+archive
+      #
+      
+      sources = <<-SOURCES
+         # for mysql cluster 6.2
+         # deb http://ppa.launchpad.net/mysql-cge-testing/ubuntu hardy main
+         # deb-src http://ppa.launchpad.net/mysql-cge-testing/ubuntu hardy main
+         
+         # for mysql cluster 6.3
+         deb http://ppa.launchpad.net/ndb-bindings/ubuntu hardy main
+         deb-src http://ppa.launchpad.net/ndb-bindings/ubuntu hardy main
+      SOURCES
+      sources.gsub!(/^ */, '')
+      put(sources, "/etc/apt/sources.list.d/mysql_cluster.list")
+    end
+      
+    after "rubber:bootstrap", "rubber:mysql_cluster:bootstrap"
+  
+    task :bootstrap, :roles => [:mysql_mgm, :mysql_data, :mysql_sql] do
+      # mysql package install starts mysql, so stop it
+      rsudo "service mysql stop" rescue nil
+      
+      # After everything installed on machines, we need the source tree
+      # on hosts in order to run rubber:config for bootstrapping the db
+      rubber.update_code_for_bootstrap
+  
+      # Conditionaly bootstrap for each node/role only if that node has not
+      # been boostrapped for that role before
+      
+      rubber_instances.for_role("mysql_mgm").each do |ic|
+        task_name = "_bootstrap_mysql_mgm_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          exists = capture("if grep -c rubber.*mysql_mgm /etc/mysql/ndb_mgmd.cnf &> /dev/null; then echo exists; fi")
+          if exists.strip.size == 0
+            rubber.run_config(:RUBBER_ENV => RUBBER_ENV, :FILE => "role/mysql_mgm", :deploy_path => release_path)
+            rsudo "service mysql-ndb-mgm start"
+          end
+        end
+        send task_name
+      end
+    
+      rubber_instances.for_role("mysql_data").each do |ic|
+        task_name = "_bootstrap_mysql_data_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          exists = capture("if grep -c rubber.*mysql_data /etc/mysql/my.cnf &> /dev/null; then echo exists; fi")
+          if exists.strip.size == 0
+            rubber.run_config(:RUBBER_ENV => RUBBER_ENV, :FILE => "role/mysql_data", :deploy_path => release_path)
+            rsudo "service mysql-ndb start-initial"
+          end
+        end
+        send task_name
+      end
+      
+      rubber_instances.for_role("mysql_sql").each do |ic|
+        task_name = "_bootstrap_mysql_sql_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          exists = capture("if grep -c rubber.*mysql_sql /etc/mysql/my.cnf &> /dev/null; then echo exists; fi")
+          if exists.strip.size == 0
+            rubber.run_config(:RUBBER_ENV => RUBBER_ENV, :FILE => "role/mysql_sql", :deploy_path => release_path)
+            rsudo "service mysql start"
+            env = rubber_cfg.environment.bind()
+            # For mysql 5.0 cluster, need to create users and database for EVERY sql node
+            pass = "identified by '#{env.db_pass}'" if env.db_pass
+            rubber.sudo_script "create_mysql_sql_db", <<-ENDSCRIPT
+              mysql -u root -e "create database #{env.db_name};"
+              mysql -u root -e "grant all on *.* to '#{env.db_user}'@'%' #{pass};"
+              mysql -u root -e "grant select on *.* to '#{env.db_slave_user}'@'%' #{pass};"
+              mysql -u root -e "grant replication slave on *.* to '#{env.db_replicator_user}'@'%' #{pass};"
+              mysql -u root -e "flush privileges;"
+            ENDSCRIPT
+            
+          end
+        end
+        send task_name
+      end
+      
+    end
+  
+    desc <<-DESC
+      Starts the mysql cluster management daemon on the management node
+    DESC
+    task :start_mgm, :roles => :mysql_mgm do
+      rsudo "service mysql-ndb-mgm start"
+    end
+    
+    desc <<-DESC
+      Starts the mysql cluster storage daemon on the data nodes
+    DESC
+    task :start_data, :roles => :mysql_data do
+      rsudo "service mysql-ndb start"
+    end
+    
+    desc <<-DESC
+      Starts the mysql cluster sql daemon on the sql nodes
+    DESC
+    task :start_sql, :roles => :mysql_sql do
+      rsudo "service mysql start"
+    end
+    
+    desc <<-DESC
+      Stops the mysql cluster management daemon on the management node
+    DESC
+    task :stop_mgm, :roles => :mysql_mgm do
+      rsudo "service mysql-ndb-mgm stop"
+    end
+    
+    desc <<-DESC
+      Stops the mysql cluster storage daemon on the data nodes
+    DESC
+    task :stop_data, :roles => :mysql_data do
+      rsudo "service mysql-ndb stop"
+    end
+    
+    desc <<-DESC
+      Stops the mysql cluster sql daemon on the sql nodes
+    DESC
+    task :stop_sql, :roles => :mysql_sql do
+      rsudo "service mysql stop"
+    end
+  
+    desc <<-DESC
+      Stops all the mysql cluster daemons
+    DESC
+    task :stop do
+      stop_sql
+      stop_data
+      stop_mgm
+    end
+    
+    desc <<-DESC
+      Starts all the mysql cluster daemons
+    DESC
+    task :start do
+      start_mgm
+      start_data
+      start_sql
+    end
+  
+    desc <<-DESC
+      Restarts all the mysql cluster daemons
+    DESC
+    task :restart do
+      stop
+      start
+    end
+    
+  end
+
+end