rubber-jcarnegie 0.0.1

data/generators/vulcanize/templates/monit/config/rubber/role/web_tools/monit-admin-nginx.conf

@@ -0,0 +1,10 @@
+<%
+  @path = "/etc/nginx/rubber/tools/monit.conf"
+%>
+
+<% rubber_instances.each do |ic| %>
+  location /monit_<%= ic.name %>
+  {
+    proxy_pass http://<%= ic.full_name %>:<%= rubber_env.monit_admin_port %>;
+  }
+<% end %>

data/generators/vulcanize/templates/monit/config/rubber/rubber-monit.yml

@@ -0,0 +1,6 @@
+# Uses admin_email from base
+
+monit_admin_port: 2812
+
+packages: [monit]
+

data/generators/vulcanize/templates/monit/templates.yml

@@ -0,0 +1 @@
+description: The monit module

data/generators/vulcanize/templates/munin/config/rubber/common/monit-munin.conf

@@ -0,0 +1,8 @@
+<%
+  @path = '/etc/monit/monit.d/monit-munin.conf'
+%>
+
+check process munin-node with pidfile /var/run/munin/munin-node.pid
+  start program  "/usr/bin/env service munin-node start"
+  stop program  "/usr/bin/env service munin-node stop"
+  if 5 restarts within 5 cycles then timeout

data/generators/vulcanize/templates/munin/config/rubber/common/munin-node.conf

@@ -0,0 +1,48 @@
+<%
+  @path = '/etc/munin/munin-node.conf'
+  @post = 'service munin-node restart'
+%>
+
+#
+# Example config-file for munin-node
+#
+
+log_level 4
+log_file /var/log/munin/munin-node.log
+port 4949
+pid_file /var/run/munin/munin-node.pid
+background 1
+setseid 1
+
+# Which port to bind to;
+host *
+user root
+group root
+setsid yes
+
+# Regexps for files to ignore
+
+ignore_file ~$
+ignore_file \.bak$
+ignore_file %$
+ignore_file \.dpkg-(tmp|new|old|dist)$
+ignore_file \.rpm(save|new)$
+
+# Set this if the client doesn't report the correct hostname when
+# telnetting to localhost, port 4949
+#
+#host_name localhost.localdomain
+
+# A list of addresses that are allowed to connect.  This must be a
+# regular expression, due to brain damage in Net::Server, which
+# doesn't understand CIDR-style network notation.  You may repeat
+# the allow line as many times as you'd like
+
+allow ^127\.0\.0\.1$
+
+<%
+  munin_server = rubber_instances.for_role('web_tools').first.internal_ip rescue "127.0.0.1"
+  munin_server = munin_server.gsub(/\./, "\\.")
+%>
+allow ^<%= munin_server %>$
+

data/generators/vulcanize/templates/munin/config/rubber/common/munin-plugins.conf

@@ -0,0 +1,11 @@
+<%
+  @path = '/etc/munin/plugin-conf.d/rubber'
+%>
+
+# munin-node clears out enviroment variables, so we need to add these in
+# to make sure we are running the correct ruby (rvm env vars, plus paths)
+[*]
+<%- ENV.select {|k, v| k =~ /rvm|ruby|bundler|gem|path/i }.each do |k, v| -%>
+env.<%= k %> <%= v %>
+<%- end -%>
+env.RUBYOPT rubygems

data/generators/vulcanize/templates/munin/config/rubber/deploy-munin.rb

@@ -0,0 +1,46 @@
+
+namespace :rubber do
+  
+  namespace :munin do
+  
+    rubber.allow_optional_tasks(self)
+
+    after "rubber:install_packages", "rubber:munin:custom_install"
+
+    # sometimes apt-get install of munin doesn't end up configuring
+    # plugins (e.g. installing postfix at same time, postfix plugin
+    # configure barfs as postfix not configured yet)
+    desc <<-DESC
+      Reconfigures munin
+    DESC
+    task :custom_install do
+      rubber.sudo_script 'setup_munin_plugins', <<-ENDSCRIPT
+        munin-node-configure --shell --remove-also > /tmp/setup-munin-plugins 2> /dev/null || true
+        sh /tmp/setup-munin-plugins
+      ENDSCRIPT
+      restart
+    end
+
+    # after "deploy:stop", "rubber:munin:stop"
+    # before "deploy:start", "rubber:munin:start"
+    # after "deploy:restart", "rubber:munin:restart"
+
+    desc "Start munin system monitoring"
+    task :start do
+      rsudo "service munin-node start"
+    end
+    
+    desc "Stop munin system monitoring"
+    task :stop do
+      rsudo "service munin-node stop; exit 0"
+    end
+    
+    desc "Restart munin system monitoring"
+    task :restart do
+      stop
+      start
+    end
+  
+  end
+
+end

data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-nginx.conf

@@ -0,0 +1,8 @@
+<%
+  @path = "/etc/nginx/rubber/tools/munin.conf"
+%>
+
+location /munin
+{
+  alias /var/www/munin;
+}

data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-plugins.conf

@@ -0,0 +1,31 @@
+<%
+  @path = "/etc/munin/plugin-conf.d/munin-node"
+  @additive = ["# rubber-munin-plugins-start", "# rubber-munin-plugins-end"]
+  @post = <<-POST
+    service munin-node restart
+  POST
+
+  root = RUBBER_ENV == 'development' ? "#{RUBBER_ROOT}/tmp/rubber" : ""
+  script_dir = "#{RUBBER_ROOT}/script/munin"
+  plugin_dir = "#{root}/etc/munin/plugins"
+  plugin_prefix = "rubber_"
+%>
+
+<%
+  # cleanup old plugins
+  File.delete(*Dir["#{plugin_dir}/#{plugin_prefix}*"])
+
+  # for each script in our source tree, create a wrapper in
+  # munin plgin dir so that we can set rails env
+  Dir["#{script_dir}/*"].each do |script|
+    plugin_name = plugin_prefix + File.basename(script)
+    File.open("#{plugin_dir}/#{plugin_name}", 'w') do |p|
+      p.chmod(0755)
+      p.puts('#!/bin/sh')
+      p.puts("RUBBER_ENV=#{RUBBER_ENV} #{script} $*")
+    end
+  end
+%>
+
+# Active rubber plugins: <%= Dir["#{plugin_dir}/#{plugin_prefix}*"].collect {|f| File.basename(f) }.join(", ") %>
+

data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin.conf

@@ -0,0 +1,115 @@
+<%
+  @path = '/etc/munin/munin.conf'
+%>
+
+# Example configuration file for Munin, generated by 'make build'
+
+# The next three variables specifies where the location of the RRD
+# databases, the HTML output, logs and the lock/pid files.  They all
+# must be writable by the user running munin-cron.  They are all
+# defaulted to the values you see here.
+#
+dbdir	/var/lib/munin
+htmldir /var/cache/munin/www
+logdir /var/log/munin
+rundir  /var/run/munin
+
+# Where to look for the HTML templates
+tmpldir /etc/munin/templates
+
+# (Exactly one) directory to include all files from.
+#
+includedir /etc/munin/munin-conf.d
+
+# Make graphs show values per minute instead of per second
+#graph_period minute
+
+# Graphics files are normaly generated by munin-graph, no matter if
+# the graphs are used or not.  You can change this to
+# on-demand-graphing by following the instructions in
+# http://munin.projects.linpro.no/wiki/CgiHowto
+#
+#graph_strategy cgi
+
+# munin-cgi-graph is invoked by the web server up to very many times at the
+# same time.  This is not optimal since it results in high CPU and memory
+# consumption to the degree that the system can thrash.  Again the default is
+# 6.  Most likely the optimal number for max_cgi_graph_jobs is the same as
+# max_graph_jobs.
+#
+#munin_cgi_graph_jobs 6
+
+# If the automatic CGI url is wrong for your system override it here:
+#
+#cgiurl_graph /cgi-bin/munin-cgi-graph
+
+# munin-graph runs in parallel, the number of concurrent processes is
+# 6.  If you want munin-graph to not be parallel set to 0.  If set too
+# high it will slow down munin-graph.  Some experiments are needed to
+# determine how many are optimal on your system.  On a multi-core
+# system with good SCSI disks the number can probably be quite high.
+#
+#max_graph_jobs 6
+
+# Drop somejuser@fnord.comm and anotheruser@blibb.comm an email everytime
+# something changes (OK -> WARNING, CRITICAL -> OK, etc)
+#contact.someuser.command mail -s "Munin notification" somejuser@fnord.comm
+#contact.anotheruser.command mail -s "Munin notification" anotheruser@blibb.comm
+#
+
+contacts dev
+contact.dev.command mail -s "Munin notification ${var:host} : ${var:group}" <%= rubber_env.admin_email %>
+contact.dev.always_send warning critical
+
+# For those with Nagios, the following might come in handy. In addition,
+# the services must be defined in the Nagios server as well.
+#contact.nagios.command /usr/bin/send_nsca nagios.host.comm -c /etc/nsca.conf
+
+# The host tree generated by rubber
+#
+<% rubber_instances.each do |i| %>
+[<%= i.full_name %>]
+  address <%= i.internal_ip %>
+  use_node_name yes
+<% end %>
+
+#
+# A more complex example of a host tree
+#
+## First our "normal" host.
+# [fii.foo.com]
+#       address foo
+#
+## Then our other host...
+# [fay.foo.com]
+#       address fay
+#
+## Then we want totals...
+# [foo.com;Totals] #Force it into the "foo.com"-domain...
+#       update no   # Turn off data-fetching for this "host".
+#
+#   # The graph "load1". We want to see the loads of both machines...
+#   # "fii=fii.foo.com:load.load" means "label=machine:graph.field"
+#       load1.graph_title Loads side by side
+#       load1.graph_order fii=fii.foo.com:load.load fay=fay.foo.com:load.load
+#
+#   # The graph "load2". Now we want them stacked on top of each other.
+#       load2.graph_title Loads on top of each other
+#       load2.dummy_field.stack fii=fii.foo.com:load.load fay=fay.foo.com:load.load
+#       load2.dummy_field.draw AREA # We want area instead the default LINE2.
+#       load2.dummy_field.label dummy # This is needed. Silly, really.
+#
+#   # The graph "load3". Now we want them summarised into one field
+#       load3.graph_title Loads summarised
+#       load3.combined_loads.sum fii.foo.com:load.load fay.foo.com:load.load
+#       load3.combined_loads.label Combined loads # Must be set, as this is
+#                                                 # not a dummy field!
+#
+## ...and on a side note, I want them listen in another order (default is
+## alphabetically)
+#
+# # Since [foo.com] would be interpreted as a host in the domain "com", we
+# # specify that this is a domain by adding a semicolon.
+# [foo.com;]
+#       node_order Totals fii.foo.com fay.foo.com
+#

data/generators/vulcanize/templates/munin/config/rubber/rubber-munin.yml

@@ -0,0 +1,8 @@
+# Uses admin_email from base
+
+packages: [munin-node, munin-plugins-extra, heirloom-mailx]
+
+roles:
+  web_tools:
+    packages: [munin]
+

data/generators/vulcanize/templates/munin/script/munin/example_mysql_query.rb

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+#
+# Example munin plugin for querying stats out of db
+# See munin plugin docs http://munin.projects.linpro.no/wiki/Documentation#Plugins
+#
+# Scripts added to RUBBER_ROOT/script/munin automatically
+# get installed as munin plugins by rubber
+
+# remove this line to enable
+exit 0
+
+# load just rubber config without loading rails (you can load rails
+# if you want, but may be slow if your app loads a lot of plugins)
+
+require "rubber"
+env = ENV["RUBBER_ENV"] ||= "development"
+root = File.join(File.dirname(__FILE__), '../..')
+Rubber::initialize(root, env)
+
+# Print config info need by munin for generating graphs
+if ARGV[0] == "config"
+  puts 'graph_title User Count'
+  puts 'graph_vlabel Users'
+  puts 'graph_category Examples'
+  puts 'graph_scale no'
+  puts 'tusers.label total users'
+  puts 'tusers.type GAUGE'
+  puts 'nusers.label user delta'
+  puts 'nusers.type COUNTER'
+  exit 0
+end
+
+
+query = <<-EOF
+  select count(id) from users;
+EOF
+
+# pick a non-critical db if possible
+source = Rubber.instances.for_role("mysql_util").first
+source ||= Rubber.instances.for_role("mysql_slave").first
+source ||= Rubber.instances.for_role("mysql_master").first
+db_host = source ? source.full_name : 'localhost'
+
+command = "mysql -u #{Rubber.config.db_slave_user}"
+command << " --password=#{Rubber.config.db_pass}"
+command << " -h #{db_host}"
+command << " #{Rubber.config.db_name} --skip-column-names"
+
+# execute a sql query to get some data
+data = `echo "#{query}" | #{command}`
+fail "Couldn't execute command" if $?.exitstatus > 0
+
+# print graph data values
+puts "tusers.value #{data}"
+puts "nusers.value #{data}"

data/generators/vulcanize/templates/munin/script/munin/example_simple.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+#
+# Example munin plugin.
+# See munin plugin docs http://munin.projects.linpro.no/wiki/Documentation#Plugins
+#
+# Scripts added to RUBBER_ROOT/script/munin automatically
+# get installed as munin plugins by rubber
+
+# remove this line to enable
+exit 0
+
+# Print config info need by munin for generating graphs
+if ARGV[0] == "config"
+  puts 'graph_title Example Stats'
+  puts 'graph_vlabel random 1-100'
+  puts 'graph_category Examples'
+  puts 'graph_scale no'
+  puts 'rand.label random'
+
+  exit 0
+end
+
+# print graph data values
+puts "rand.value #{rand(100)}"

data/generators/vulcanize/templates/munin/templates.yml

@@ -0,0 +1 @@
+description: The munin module

data/generators/vulcanize/templates/mysql/config/rubber/common/database.yml

@@ -0,0 +1,12 @@
+<%
+  @path = "#{RUBBER_ROOT}/config/database.yml"
+%>
+
+<%= RUBBER_ENV %>:
+  adapter: mysql
+  encoding: utf8
+  database: <%= rubber_env.db_name %>
+  username: <%= rubber_env.db_user %>
+  password: <%= rubber_env.db_pass %>
+  host: <%= rubber_instances.for_role('db', 'primary' => true).first.full_name %>
+  pool: 5

data/generators/vulcanize/templates/mysql/config/rubber/deploy-mysql.rb

@@ -0,0 +1,169 @@
+
+namespace :rubber do
+  
+  namespace :mysql do
+    
+    rubber.allow_optional_tasks(self)
+    
+    after "rubber:create", "rubber:mysql:validate_db_roles"
+
+    task :validate_db_roles do
+      db_instances = rubber_instances.for_role("mysql_slave")
+      db_instances.each do |instance|
+        if instance.role_names.find {|n| n == 'mysql_master'}
+          fatal "Cannot have a mysql slave and master on the same instance, please removing slave role for #{instance.name}"
+        end
+      end
+    end
+
+    after "rubber:bootstrap", "rubber:mysql:bootstrap"
+  
+    
+    # Bootstrap the production database config.  Db bootstrap is special - the
+    # user could be requiring the rails env inside some of their config
+    # templates, which creates a catch 22 situation with the db, so we try and
+    # bootstrap the db separate from the rest of the config
+    task :bootstrap, :roles => [:mysql_master, :mysql_slave] do
+      
+      # Conditionaly bootstrap for each node/role only if that node has not
+      # been boostrapped for that role before
+      
+      master_instances = rubber_instances.for_role("mysql_master") & rubber_instances.filtered  
+      master_instances.each do |ic|
+        task_name = "_bootstrap_mysql_master_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          env = rubber_cfg.environment.bind("mysql_master", ic.name)
+          exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
+          if exists.strip.size == 0
+            common_bootstrap("mysql_master")
+            
+            pass = "identified by '#{env.db_pass}'" if env.db_pass
+            rubber.sudo_script "create_master_db", <<-ENDSCRIPT
+              mysql -u root -e "create database #{env.db_name};"
+              mysql -u root -e "grant all on *.* to '#{env.db_user}'@'%' #{pass};"
+              mysql -u root -e "grant select on *.* to '#{env.db_slave_user}'@'%' #{pass};"
+              mysql -u root -e "grant replication slave on *.* to '#{env.db_replicator_user}'@'%' #{pass};"
+              mysql -u root -e "flush privileges;"
+            ENDSCRIPT
+          end
+        end
+        send task_name
+      end
+    
+      slave_instances = rubber_instances.for_role("mysql_slave") & rubber_instances.filtered  
+      slave_instances.each do |ic|
+        task_name = "_bootstrap_mysql_slave_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          env = rubber_cfg.environment.bind("mysql_slave", ic.name)
+          exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
+          if exists.strip.size == 0
+            common_bootstrap("mysql_slave")
+
+            master = rubber_instances.for_role("mysql_master").first
+
+            # Doing a mysqldump locks the db, so ideally we'd do it against a slave replica thats
+            # not serving traffic (mysql_util role), but if thats not available try a regular
+            # slave (mysql_slave role), and finally default dumping from master (mysql_master role)
+            # TODO: handle simultaneous creating of multi slaves/utils
+            slaves = rubber_instances.for_role("mysql_slave")
+            slaves.delete(ic) # don't want to try and dump from self
+            source = slaves.find {|sc| sc.role_names.include?("mysql_util")}
+            source = slaves.first unless source
+            source = master unless source
+
+            pass = "identified by '#{env.db_pass}'" if env.db_pass
+            master_pass = ", master_password='#{env.db_pass}'" if env.db_pass
+            master_host = master.full_name
+            source_host = source.full_name
+
+            if source == master
+              logger.info "Creating slave from a dump of master #{source_host}"
+              rubber.sudo_script "create_slave_db_from_master", <<-ENDSCRIPT
+                mysql -u root -e "change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}' #{master_pass}"
+                mysqldump -u #{env.db_user} #{env.db_pass.nil? ? '' : '--password ' + env.db_pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root
+              ENDSCRIPT
+            else
+              logger.info "Creating slave from a dump of slave #{source_host}"
+              rsudo "mysql -u #{env.db_user} --password #{env.db_pass} -h #{source_host} -e \"stop slave;\""
+              slave_status = capture("mysql -u #{env.db_user} #{pass} -h #{source_host} -e \"show slave status\\G\"")
+              slave_config = Hash[*slave_status.scan(/([^\s:]+): ([^\s]*)/).flatten]
+              log_file = slave_config['Master_Log_File']
+              log_pos = slave_config['Read_Master_Log_Pos']
+              rubber.sudo_script "create_slave_db_from_slave", <<-ENDSCRIPT
+                mysqldump -u #{env.db_user} --password #{env.db_pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root
+                mysql -u root -e "change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}', master_log_file='#{log_file}', master_log_pos=#{log_pos} #{master_pass}"
+                mysql -u #{env.db_user} --password #{env.db_pass} -h #{source_host} -e "start slave;"
+              ENDSCRIPT
+            end
+
+            # this doesn't work without agent forwarding which sudo breaks, as well as not having your
+            # ec2 private key ssh-added on workstation
+            # sudo "scp -o \"StrictHostKeyChecking=no\" #{source_host}:/etc/mysql/debian.cnf /etc/mysql"
+
+            rsudo "mysql -u root -e \"flush privileges;\""
+            rsudo "mysql -u root -e \"start slave;\""
+          end
+        end
+        send task_name
+      end
+      
+    end
+  
+    # TODO: Make the setup/update happen just once per host
+    def common_bootstrap(role)
+      # mysql package install starts mysql, so stop it
+      rsudo "service mysql stop" rescue nil
+      
+      # After everything installed on machines, we need the source tree
+      # on hosts in order to run rubber:config for bootstrapping the db
+      rubber.update_code_for_bootstrap
+      
+      # Gen just the conf for the given mysql role
+      rubber.run_config(:RUBBER_ENV => RUBBER_ENV, :FILE => "role/#{role}|role/db/", :FORCE => true, :deploy_path => release_path)
+
+      # reconfigure mysql so that it sets up data dir in /mnt with correct files
+      sudo_script 'reconfigure-mysql', <<-ENDSCRIPT
+        server_package=`dpkg -l | grep mysql-server-[0-9] | awk '{print $2}'`
+        dpkg-reconfigure --frontend=noninteractive $server_package
+      ENDSCRIPT
+      sleep 5
+    end
+    
+    before "rubber:munin:custom_install", "rubber:mysql:custom_install_munin"
+
+    desc <<-DESC
+      Installs some extra munin graphs
+    DESC
+    task :custom_install_munin, :roles => [:mysql_master, :mysql_slave] do
+      rubber.sudo_script 'install_munin_mysql', <<-ENDSCRIPT
+        if [ ! -f /usr/share/munin/plugins/mysql_ ]; then
+          wget --no-check-certificate -qN -O /usr/share/munin/plugins/mysql_ https://github.com/kjellm/munin-mysql/raw/master/mysql_
+          wget --no-check-certificate -qN -O /etc/munin/plugin-conf.d/mysql_.conf https://github.com/kjellm/munin-mysql/raw/master/mysql_.conf
+        fi
+      ENDSCRIPT
+    end
+
+    desc <<-DESC
+      Starts the mysql daemons
+    DESC
+    task :start, :roles => [:mysql_master, :mysql_slave] do
+      rsudo "service mysql start"
+    end
+    
+    desc <<-DESC
+      Stops the mysql daemons
+    DESC
+    task :stop, :roles => [:mysql_master, :mysql_slave] do
+      rsudo "service mysql stop"
+    end
+  
+    desc <<-DESC
+      Restarts the mysql daemons
+    DESC
+    task :restart, :roles => [:mysql_master, :mysql_slave] do
+      rsudo "service mysql restart"
+    end
+  
+  end
+
+end