sml-rubber 0.9.1

data/generators/vulcanize/templates/monit/config/rubber/role/web_tools/monit-admin-nginx.conf

@@ -0,0 +1,10 @@
+<%
+  @path = "/etc/nginx/rubber/monit.conf"
+%>
+
+<% rubber_instances.each do |ic| %>
+  location /monit_<%= ic.name %>
+  {
+    proxy_pass http://<%= ic.full_name %>:<%= rubber_env.monit_admin_port %>;
+  }
+<% end %>

data/generators/vulcanize/templates/monit/config/rubber/rubber-monit.yml

@@ -0,0 +1,6 @@
+# Uses admin_email from base
+
+monit_admin_port: 2812
+
+packages: [monit]
+

data/generators/vulcanize/templates/monit/templates.yml

@@ -0,0 +1 @@
+description: The monit module

data/generators/vulcanize/templates/munin/config/rubber/common/monit-munin.conf

@@ -0,0 +1,8 @@
+<%
+  @path = '/etc/monit/monit.d/monit-munin.conf'
+%>
+
+check process munin-node with pidfile /var/run/munin/munin-node.pid
+  start program  "/etc/init.d/munin-node start"
+  stop program  "/etc/init.d/munin-node stop"
+  if 5 restarts within 5 cycles then timeout

data/generators/vulcanize/templates/munin/config/rubber/common/munin-node.conf

@@ -0,0 +1,48 @@
+<%
+  @path = '/etc/munin/munin-node.conf'
+  @post = '/etc/init.d/munin-node restart'
+%>
+
+#
+# Example config-file for munin-node
+#
+
+log_level 4
+log_file /var/log/munin/munin-node.log
+port 4949
+pid_file /var/run/munin/munin-node.pid
+background 1
+setseid 1
+
+# Which port to bind to;
+host *
+user root
+group root
+setsid yes
+
+# Regexps for files to ignore
+
+ignore_file ~$
+ignore_file \.bak$
+ignore_file %$
+ignore_file \.dpkg-(tmp|new|old|dist)$
+ignore_file \.rpm(save|new)$
+
+# Set this if the client doesn't report the correct hostname when
+# telnetting to localhost, port 4949
+#
+#host_name localhost.localdomain
+
+# A list of addresses that are allowed to connect.  This must be a
+# regular expression, due to brain damage in Net::Server, which
+# doesn't understand CIDR-style network notation.  You may repeat
+# the allow line as many times as you'd like
+
+allow ^127\.0\.0\.1$
+
+<%
+  munin_server = rubber_instances.for_role('web_tools').first.internal_ip rescue "127.0.0.1"
+  munin_server = munin_server.gsub(/\./, "\\.")
+%>
+allow ^<%= munin_server %>$
+

data/generators/vulcanize/templates/munin/config/rubber/deploy-munin.rb

@@ -0,0 +1,30 @@
+
+namespace :rubber do
+  
+  namespace :munin do
+  
+    rubber.allow_optional_tasks(self)
+    
+    # after "deploy:stop", "rubber:munin:stop"
+    # before "deploy:start", "rubber:munin:start"
+    # after "deploy:restart", "rubber:munin:restart"
+
+    desc "Start munin system monitoring"
+    task :start do
+      run "/etc/init.d/munin-node start"
+    end
+    
+    desc "Stop munin system monitoring"
+    task :stop, :on_error => :continue do
+      run "/etc/init.d/munin-node stop"
+    end
+    
+    desc "Restart munin system monitoring"
+    task :restart do
+      stop
+      start
+    end
+  
+  end
+
+end

data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-nginx.conf

@@ -0,0 +1,8 @@
+<%
+  @path = "/etc/nginx/rubber/munin.conf"
+%>
+
+location /munin
+{
+  alias /var/www/munin;
+}

data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin-plugins.conf

@@ -0,0 +1,31 @@
+<%
+  @path = "/etc/munin/plugin-conf.d/munin-node"
+  @additive = ["# rubber-munin-plugins-start", "# rubber-munin-plugins-end"]
+  @post = <<-POST
+    /etc/init.d/munin-node restart
+  POST
+
+  root = RUBBER_ENV == 'development' ? "#{RUBBER_ROOT}/tmp/rubber" : ""
+  script_dir = "#{RUBBER_ROOT}/script/munin"
+  plugin_dir = "#{root}/etc/munin/plugins"
+  plugin_prefix = "rubber_"
+%>
+
+<%
+  # cleanup old plugins
+  File.delete(*Dir["#{plugin_dir}/#{plugin_prefix}*"])
+
+  # for each script in our source tree, create a wrapper in
+  # munin plgin dir so that we can set rails env
+  Dir["#{script_dir}/*"].each do |script|
+    plugin_name = plugin_prefix + File.basename(script)
+    File.open("#{plugin_dir}/#{plugin_name}", 'w') do |p|
+      p.chmod(0755)
+      p.puts('#!/bin/sh')
+      p.puts("RUBBER_ENV=#{RUBBER_ENV} #{script} $*")
+    end
+  end
+%>
+
+# Active rubber plugins: <%= Dir["#{plugin_dir}/#{plugin_prefix}*"].collect {|f| File.basename(f) }.join(", ") %>
+

data/generators/vulcanize/templates/munin/config/rubber/role/web_tools/munin.conf

@@ -0,0 +1,80 @@
+<%
+  @path = '/etc/munin/munin.conf'
+%>
+
+# Example configuration file for Munin, generated by 'make build'
+
+# The next three variables specifies where the location of the RRD
+# databases, the HTML output, and the logs, severally.  They all
+# must be writable by the user running munin-cron.
+dbdir /var/lib/munin
+htmldir /var/www/munin
+logdir  /var/log/munin
+rundir  /var/run/munin
+
+# Where to look for the HTML templates
+tmpldir /etc/munin/templates
+
+# Make graphs show values per minute instead of per second
+#graph_period minute
+
+# Drop somejuser@fnord.comm and anotheruser@blibb.comm an email everytime
+# something changes (OK -> WARNING, CRITICAL -> OK, etc)
+#contact.someuser.command mail -s "Munin notification" somejuser@fnord.comm
+#contact.anotheruser.command mail -s "Munin notification" anotheruser@blibb.comm
+#
+# For those with Nagios, the following might come in handy. In addition,
+# the services must be defined in the Nagios server as well.
+#contact.nagios.command /usr/sbin/send_nsca -H nagios.host.com -c /etc/send_nsca.cfg
+
+contacts dev
+contact.dev.command mail -s "Munin notification ${var:host} : ${var:group}" <%= rubber_env.admin_email %>
+contact.dev.always_send warning critical
+
+
+<% rubber_instances.each do |i| %>
+[<%= i.full_name %>]
+  address <%= i.internal_ip %>
+  use_node_name yes
+<% end %>
+
+#
+# A more complex example of a host tree
+#
+## First our "normal" host.
+# [fii.foo.com]
+#       address foo
+#
+## Then our other host...
+# [fay.foo.com]
+#       address fay
+#
+## Then we want totals...
+# [foo.com;Totals] #Force it into the "foo.com"-domain...
+#       update no   # Turn off data-fetching for this "host".
+#
+#   # The graph "load1". We want to see the loads of both machines...
+#   # "fii=fii.foo.com:load.load" means "label=machine:graph.field"
+#       load1.graph_title Loads side by side
+#       load1.graph_order fii=fii.foo.com:load.load fay=fay.foo.com:load.load
+#
+#   # The graph "load2". Now we want them stacked on top of each other.
+#       load2.graph_title Loads on top of each other
+#       load2.dummy_field.stack fii=fii.foo.com:load.load fay=fay.foo.com:load.load
+#       load2.dummy_field.draw AREA # We want area instead the default LINE2.
+#       load2.dummy_field.label dummy # This is needed. Silly, really.
+#
+#   # The graph "load3". Now we want them summarised into one field
+#       load3.graph_title Loads summarised
+#       load3.combined_loads.sum fii.foo.com:load.load fay.foo.com:load.load
+#       load3.combined_loads.label Combined loads # Must be set, as this is
+#                                                 # not a dummy field!
+#
+## ...and on a side note, I want them listen in another order (default is
+## alphabetically)
+#
+# # Since [foo.com] would be interpreted as a host in the domain "com", we
+# # specify that this is a domain by adding a semicolon.
+# [foo.com;]
+#       node_order Totals fii.foo.com fay.foo.com
+#

data/generators/vulcanize/templates/munin/config/rubber/rubber-munin.yml

@@ -0,0 +1,8 @@
+# Uses admin_email from base
+
+packages: [munin-node]
+
+roles:
+  web_tools:
+    packages: [munin]
+

data/generators/vulcanize/templates/munin/script/munin/example_mysql_query.rb

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+#
+# Example munin plugin for querying stats out of db
+# See munin plugin docs http://munin.projects.linpro.no/wiki/Documentation#Plugins
+#
+# Scripts added to RUBBER_ROOT/script/munin automatically
+# get installed as munin plugins by rubber
+
+# remove this line to enable
+exit 0
+
+# load just rubber config without loading rails (you can load rails
+# if you want, but may be slow if your app loads a lot of plugins)
+
+require "rubber"
+env = ENV["RUBBER_ENV"] ||= "development"
+root = File.join(File.dirname(__FILE__), '../..')
+Rubber::initialize(root, env)
+RUBBER_CONFIG = Rubber::Configuration.rubber_env
+RUBBER_INSTANCES = Rubber::Configuration.rubber_instances
+
+# Print config info need by munin for generating graphs
+if ARGV[0] == "config"
+  puts 'graph_title User Count'
+  puts 'graph_vlabel Users'
+  puts 'graph_category Examples'
+  puts 'graph_scale no'
+  puts 'tusers.label total users'
+  puts 'tusers.type GAUGE'
+  puts 'nusers.label user delta'
+  puts 'nusers.type COUNTER'
+  exit 0
+end
+
+
+query = <<-EOF
+  select count(id) from users;
+EOF
+
+# pick a non-critical db if possible
+source = RUBBER_INSTANCES.for_role("mysql_util").first
+source ||= RUBBER_INSTANCES.for_role("mysql_slave").first
+source ||= RUBBER_INSTANCES.for_role("mysql_master").first
+db_host = source ? source.full_name : 'localhost'
+
+command = "mysql -u #{RUBBER_ENV.db_slave_user}"
+command << " --password=#{RUBBER_ENV.db_pass}"
+command << " -h #{db_host}"
+command << " #{RUBBER_ENV.db_name} --skip-column-names"
+
+# execute a sql query to get some data
+data = `echo "#{query}" | #{command}`
+fail "Couldn't execute command" if $?.exitstatus > 0
+
+# print graph data values
+puts "tusers.value #{data}"
+puts "nusers.value #{data}"

data/generators/vulcanize/templates/munin/script/munin/example_simple.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+#
+# Example munin plugin.
+# See munin plugin docs http://munin.projects.linpro.no/wiki/Documentation#Plugins
+#
+# Scripts added to RUBBER_ROOT/script/munin automatically
+# get installed as munin plugins by rubber
+
+# remove this line to enable
+exit 0
+
+# Print config info need by munin for generating graphs
+if ARGV[0] == "config"
+  puts 'graph_title Example Stats'
+  puts 'graph_vlabel random 1-100'
+  puts 'graph_category Examples'
+  puts 'graph_scale no'
+  puts 'rand.label random'
+
+  exit 0
+end
+
+# print graph data values
+puts "rand.value #{rand(100)}"

data/generators/vulcanize/templates/munin/templates.yml

@@ -0,0 +1 @@
+description: The munin module

data/generators/vulcanize/templates/mysql/config/rubber/common/database.yml

@@ -0,0 +1,11 @@
+<%
+  @path = "#{RUBBER_ROOT}/config/database.yml"
+%>
+
+<%= RUBBER_ENV %>:
+  adapter: mysql
+  encoding: utf8
+  database: <%= rubber_env.db_name %>
+  username: <%= rubber_env.db_user %>
+  password: <%= rubber_env.db_pass %>
+  host: <%= rubber_instances.for_role('db', 'primary' => true).first.full_name %>

data/generators/vulcanize/templates/mysql/config/rubber/deploy-mysql.rb

@@ -0,0 +1,178 @@
+
+namespace :rubber do
+  
+  namespace :mysql do
+    
+    rubber.allow_optional_tasks(self)
+    
+    after "rubber:create", "rubber:mysql:set_db_role"
+      
+    # Capistrano needs db:primary role for migrate to work
+    task :set_db_role do
+      db_instances = rubber_cfg.instance.for_role("mysql_master")
+      db_instances.each do |instance|
+        if ! instance.role_names.find {|n| n == 'db'}
+          role = Rubber::Configuration::RoleItem.new('db')
+          primary_exists = rubber_cfg.instance.for_role("db", "primary" => true).size > 0
+          role.options["primary"] = true  unless primary_exists
+          instance.roles << role
+        end
+      end
+      db_instances = rubber_cfg.instance.for_role("mysql_slave")
+      db_instances.each do |instance|
+        if instance.role_names.find {|n| n == 'mysql_master'}
+          logger.info "Cannot have a mysql slave and master on the same instance, removing slave role"
+          instance.roles.delete_if {|r| r.name == 'mysql_slave'}
+          next
+        end
+        if ! instance.role_names.find {|n| n == 'db'}
+          role = Rubber::Configuration::RoleItem.new('db')
+          instance.roles << role
+        end
+      end
+      rubber_cfg.instance.save()
+      load_roles() unless rubber_cfg.environment.bind().disable_auto_roles
+    end
+  
+    after "rubber:bootstrap", "rubber:mysql:bootstrap"
+  
+    
+    # Bootstrap the production database config.  Db bootstrap is special - the
+    # user could be requiring the rails env inside some of their config
+    # templates, which creates a catch 22 situation with the db, so we try and
+    # bootstrap the db separate from the rest of the config
+    task :bootstrap, :roles => [:mysql_master, :mysql_slave] do
+      
+      # Conditionaly bootstrap for each node/role only if that node has not
+      # been boostrapped for that role before
+      
+      master_instances = rubber_cfg.instance.for_role("mysql_master") & rubber_cfg.instance.filtered  
+      master_instances.each do |ic|
+        task_name = "_bootstrap_mysql_master_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          env = rubber_cfg.environment.bind("mysql_master", ic.name)
+          exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
+          if exists.strip.size == 0
+            common_bootstrap("mysql_master")
+            sudo "dpkg-reconfigure --frontend=noninteractive mysql-server-5.0"
+            sleep 5
+            pass = "identified by '#{env.db_pass}'" if env.db_pass
+            sudo "mysql -u root -e 'create database #{env.db_name};'"
+            sudo "mysql -u root -e \"grant all on *.* to '#{env.db_user}'@'%' #{pass};\""
+            sudo "mysql -u root -e \"grant select on *.* to '#{env.db_slave_user}'@'%' #{pass};\""
+            sudo "mysql -u root -e \"grant replication slave on *.* to '#{env.db_replicator_user}'@'%' #{pass};\""
+            sudo "mysql -u root -e \"flush privileges;\""
+          end
+        end
+        send task_name
+      end
+    
+      slave_instances = rubber_cfg.instance.for_role("mysql_slave") & rubber_cfg.instance.filtered  
+      slave_instances.each do |ic|
+        task_name = "_bootstrap_mysql_slave_#{ic.full_name}".to_sym()
+        task task_name, :hosts => ic.full_name do
+          env = rubber_cfg.environment.bind("mysql_slave", ic.name)
+          exists = capture("echo $(ls #{env.db_data_dir}/ 2> /dev/null)")
+          if exists.strip.size == 0
+            common_bootstrap("mysql_slave")
+            sudo "dpkg-reconfigure --frontend=noninteractive mysql-server-5.0"
+            sleep 5
+
+            master = rubber_cfg.instance.for_role("mysql_master").first
+
+            # Doing a mysqldump locks the db, so ideally we'd do it against a slave replica thats
+            # not serving traffic (mysql_util role), but if thats not available try a regular
+            # slave (mysql_slave role), and finally default dumping from master (mysql_master role)
+            # TODO: handle simultaneous creating of multi slaves/utils
+            slaves = rubber_cfg.instance.for_role("mysql_slave")
+            slaves.delete(ic) # don't want to try and dump from self
+            source = slaves.find {|sc| sc.role_names.include?("mysql_util")}
+            source = slaves.first unless source
+            source = master unless source
+
+            pass = "identified by '#{env.db_pass}'" if env.db_pass
+            master_pass = ", master_password='#{env.db_pass}'" if env.db_pass
+            master_host = master.full_name
+            source_host = source.full_name
+
+            if source == master
+              logger.info "Creating slave from a dump of master #{source_host}"
+              sudo "mysql -u root -e \"change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}' #{master_pass}\""
+              sudo "mysqldump -u #{env.db_user} #{pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root"
+            else
+              logger.info "Creating slave from a dump of slave #{source_host}"
+              sudo "mysql -u #{env.db_user} #{pass} -h #{source_host} -e \"stop slave;\""
+              slave_status = capture("mysql -u #{env.db_user} #{pass} -h #{source_host} -e \"show slave status\\G\"")
+              slave_config = Hash[*slave_status.scan(/([^\s:]+): ([^\s]*)/).flatten]
+              log_file = slave_config['Master_Log_File']
+              log_pos = slave_config['Read_Master_Log_Pos']
+              sudo "mysqldump -u #{env.db_user} #{pass} -h #{source_host} --all-databases --master-data=1 | mysql -u root"
+              sudo "mysql -u root -e \"change master to master_host='#{master_host}', master_user='#{env.db_replicator_user}', master_log_file='#{log_file}', master_log_pos=#{log_pos} #{master_pass}\""
+              sudo "mysql -u #{env.db_user} #{pass} -h #{source_host} -e \"start slave;\""
+            end
+
+            sudo "mysql -u root -e \"flush privileges;\""
+            sudo "mysql -u root -e \"start slave;\""
+          end
+        end
+        send task_name
+      end
+      
+    end
+  
+    # TODO: Make the setup/update happen just once per host
+    def common_bootstrap(role)
+      # mysql package install starts mysql, so stop it
+      sudo "/etc/init.d/mysql stop" rescue nil
+      
+      # After everything installed on machines, we need the source tree
+      # on hosts in order to run rubber:config for bootstrapping the db
+      deploy.setup
+      deploy.update_code
+      
+      # Gen just the conf for the given mysql role
+      rubber.run_config(:RUBBER_ENV => rails_env, :FILE => "role/#{role}|role/db/my.cnf", :deploy_path => release_path)
+    end
+    
+    after "rubber:install_packages", "rubber:mysql:custom_install"
+
+    desc <<-DESC
+      Installs some extra munin graphs
+    DESC
+    task :custom_install, :roles => [:mysql_master, :mysql_slave] do
+      rubber.run_script 'install_munin_mysql', <<-ENDSCRIPT
+        if [ ! -f /etc/munin/plugins/mysql_ ]; then
+          wget -q -O /etc/munin/plugins/mysql_ http://github.com/kjellm/munin-mysql/raw/master/mysql_
+          wget -q -O /etc/munin/plugin-conf.d/mysql_.conf http://github.com/kjellm/munin-mysql/raw/master/mysql_.conf
+
+          cd /etc/munin/plugins
+          chmod +x mysql_
+          ./mysql_ suggest | while read x; do ln -sf mysql_ $x; done
+        fi
+      ENDSCRIPT
+    end
+
+    desc <<-DESC
+      Starts the mysql daemons
+    DESC
+    task :start, :roles => [:mysql_master, :mysql_slave] do
+      sudo "/etc/init.d/mysql start"
+    end
+    
+    desc <<-DESC
+      Stops the mysql daemons
+    DESC
+    task :stop, :roles => [:mysql_master, :mysql_slave] do
+      sudo "/etc/init.d/mysql stop"
+    end
+  
+    desc <<-DESC
+      Restarts the mysql daemons
+    DESC
+    task :restart, :roles => [:mysql_master, :mysql_slave] do
+      sudo "/etc/init.d/mysql restart"
+    end
+  
+  end
+
+end