rubber 1.15.0 → 2.0.0.pre1

data/{lib/generators/vulcanize/templates/passenger/config/rubber/role/passenger/munin-passenger-sudoers.conf → templates/collectd/config/rubber/role/passenger/passenger-status-sudoers.conf}

@@ -1,6 +1,6 @@
 <%
   @path = '/etc/sudoers'
-  @additive = ["# munin passenger start", "# munin passenger end"]
+  @additive = ["# passenger-status start", "# passenger-status end"]
 %>
 Defaults env_keep = GEM_PATH
 

data/templates/collectd/config/rubber/role/postgresql/collectd-postgresql.conf

@@ -0,0 +1,22 @@
+<%
+  @path = "/etc/collectd/conf.d/postgresql.conf"
+%>
+
+LoadPlugin postgresql
+
+<Plugin postgresql>
+
+  <Database <%= rubber_env.db_name %>>
+    User "<%= rubber_env.db_user %>"
+
+    # Predefined, see /usr/share/collectd/postgresql_default.conf
+    Query backends
+    Query transactions
+    Query queries
+    Query query_plans
+    Query table_states
+    Query disk_io
+    Query disk_usage
+
+  </Database>
+</Plugin>

data/templates/collectd/config/rubber/role/postgresql_slave/collectd-postgresql-threshold.conf

@@ -0,0 +1,23 @@
+<%
+  @path = '/etc/collectd/thresholds.d/postgresql_slave_thresholds.conf'
+%>
+
+<Threshold>
+
+  <Plugin "postgresql">
+    <Type "gauge">
+      Instance "replication_receive_delay"
+      DataSource "value"
+      WarningMax 500000
+      FailureMax 1000000
+    </Type>
+
+    <Type "gauge">
+      Instance "replication_replay_delay"
+      DataSource "value"
+      WarningMax 20000000
+      FailureMax 30000000
+    </Type>
+  </Plugin>
+
+</Threshold>

data/templates/collectd/config/rubber/rubber-collectd.yml

@@ -0,0 +1,8 @@
+
+rubber_collectd_runner: "script/collectd/collectd-runner.rb"
+collectd_graphite_plugin_version: 6107730c56b4ede27631
+packages: [collectd libperl5.10]
+collectd_syslog: daemon
+
+role_dependencies:
+  common: [collectd]

data/templates/collectd/script/collectd/collectd-runner.rb

@@ -0,0 +1,107 @@
+#!/usr/bin/env ruby
+
+STDOUT.sync = STDERR.sync = true
+
+action = ARGV[0]
+run_script = ARGV[1]
+
+if ARGV.size > 2 || (action == 'run' && run_script.nil?)
+  puts "usage: #{File.basename(__FILE__)} [config | run script]"
+  puts "\tWith no arguments run all scripts in plugin dir in an infinite loop"
+  puts "\tWith config action, lists scripts that would be run"
+  puts "\tWith run action, runs the given script"
+  exit 1
+end
+
+HOSTNAME = ENV["COLLECTD_HOSTNAME"] || `hostname -f`.chomp
+INTERVAL = (ENV["COLLECTD_INTERVAL"] || 60).to_i
+
+env = ENV["RUBBER_ENV"] ||= "development"
+root = File.expand_path(File.dirname(__FILE__) + '/../..')
+
+STDERR.reopen("#{root}/log/collectd-runner.log", "a") unless action == 'run'
+
+require "rubber"
+Rubber::initialize(root, env)
+
+# Gives us a way to run a single script
+if action == 'run'
+  load ARGV[1]
+  exit
+end
+
+# for each script in our source tree, call that script to generate collectd data
+#
+# scripts directly in script_dir get run for each host
+# scripts in script_dir/role/role_name only get run if current host is a member of that role
+# scripts in script_dir/host/host_name only get run if current host is that host_name
+
+scripts = []
+this_script = File.expand_path(__FILE__)
+script_dir = "#{Rubber.root}/script/collectd"
+
+Dir["#{script_dir}/**/*"].each do |script|
+  next if File.directory?(script)
+  next if script == this_script
+
+  # skip scripts not specific to the current host or roles
+  #
+  relative = script.gsub("#{script_dir}/", "")
+  segments = relative.split("/")
+  if segments[0] == 'host'
+    next unless segments[1] == Rubber.config.host
+  end
+  if segments[0] == 'role'
+    next unless Array(Rubber.config.roles).include?(segments[1])
+  end
+
+  scripts << script
+end
+
+# Allows collectd conf to see if it needs to setup an Exec Plugin block
+if ARGV[0] == 'config'
+  puts scripts.join("\n")
+  exit
+end
+
+# Collectd scripts can run indefinately, outputting values
+# every INTERVAL.  We do this for all the rubber/collectd
+# scripts so that we keep the ruby vm loaded for faster
+# execution time at the expense of  the ruby vm memory usage.
+#
+# Foreach rubber/collectd plugin script, load it in a fork
+# to run so that our master loop doesn't get compromised by
+# a bad script, segfault, etc.
+#
+STDERR.puts "#{Time.now}: Starting rubber-collectd execution loop"
+loop do
+  start_time = Time.now.to_i
+
+  scripts.each do |script|
+    fork do
+      begin
+        load script
+      rescue Exception => e
+        STDERR.puts("#{script}: #{e}")
+      end
+    end
+  end
+  Process.waitall
+
+  run_time = Time.now.to_i - start_time
+  begin
+    puts "PUTVAL #{HOSTNAME}/rubber/gauge-collectd_runner interval=#{INTERVAL} N:#{run_time}"
+  rescue Errno::EPIPE
+    STDERR.puts "#{Time.now}: Exiting rubber-collectd execution loop"
+    exit
+  end
+
+  sleep_time = INTERVAL - run_time
+  if sleep_time < 0
+    sleep_time = INTERVAL
+    STDERR.puts("Plugins taking longer (#{run_time}s) than #{INTERVAL}s")
+  end
+
+  sleep sleep_time
+  
+end

data/templates/collectd/script/collectd/role/examples/example_db_query.rb

@@ -0,0 +1,19 @@
+query = <<-EOF
+  select count(id) from users;
+EOF
+
+# pick a non-critical db if possible
+source = Rubber.instances.for_role("db").first
+source ||= Rubber.instances.for_role("db", "primary" => true).first
+db_host = source ? source.full_name : 'localhost'
+
+command = "psql -U#{Rubber.config.db_slave_user}"
+command << " -h #{db_host}"
+command << " #{Rubber.config.db_name} --tuples-only --no-align"
+
+# execute a sql query to get some data
+data = `echo "#{query}" | #{command}`
+fail "Couldn't execute command" if $?.exitstatus > 0
+
+# print graph data values
+puts "PUTVAL #{HOSTNAME}/users/total interval=#{INTERVAL} N:#{data}"

data/templates/collectd/script/collectd/role/examples/example_simple.rb

@@ -0,0 +1 @@
+puts "PUTVAL #{HOSTNAME}/example/rand_value interval=#{INTERVAL} N:#{rand(100)}"

data/templates/collectd/script/collectd/role/postgresql_slave/replication_status.rb

@@ -0,0 +1,31 @@
+require 'pg'
+
+master = Rubber.instances.for_role('postgresql_master').first.full_name
+slave = "localhost"
+opts = { :dbname => Rubber.config.db_name,
+         :user => Rubber.config.db_user,
+         :password => Rubber.config.db_pass }
+mconn = PGconn.open(opts.merge({:host => master}))
+sconn = PGconn.open(opts.merge({:host => slave}))
+
+mval = mconn.exec("select pg_current_xlog_location()")[0]["pg_current_xlog_location"]
+sresult = sconn.exec("select pg_last_xlog_receive_location(), pg_last_xlog_replay_location()")[0]
+sval_receive = sresult["pg_last_xlog_receive_location"]
+sval_replay = sresult["pg_last_xlog_replay_location"]
+
+
+def numeric(val)
+  # First part is logid, second part is record offset
+  parts = val.split("/")
+  raise "Invalid location" if parts.size != 2 && parts.any {|p| p.to_s.strip.size == 0}
+  result = (0xFFFFFFFF * parts[0].to_i) + parts[1].to_i(16)
+  return result
+end
+
+
+master_offset = numeric(mval)
+receive_offset = numeric(sval_receive)
+replay_offset = numeric(sval_replay)
+
+puts "PUTVAL #{HOSTNAME}/postgresql/gauge-replication_receive_delay interval=#{INTERVAL} N:#{master_offset - receive_offset}"
+puts "PUTVAL #{HOSTNAME}/postgresql/gauge-replication_replay_delay interval=#{INTERVAL} N:#{master_offset - replay_offset}"

data/templates/collectd/script/collectd/role/redis/redis_info.rb

@@ -0,0 +1,28 @@
+require 'redis'
+
+redis = Redis.new(:host => 'localhost', :port => 6379, :timeout => INTERVAL - 1)
+info = redis.info
+
+items = {
+  'connected_clients' => 'gauge',
+  'client_longest_output_list' => 'gauge',
+  'client_biggest_input_buf' => 'gauge',
+  'blocked_clients' => 'gauge',
+  'total_commands_processed' => 'derive',
+  'total_connections_received' => 'derive',
+  'connected_clients' => 'gauge',
+  'used_memory' => 'gauge',
+  'changes_since_last_save' => 'gauge'
+}
+
+items.each do |item, ctype|
+  puts "PUTVAL #{HOSTNAME}/redis/#{ctype}-#{item} interval=#{INTERVAL} N:#{info[item]}"
+end
+
+info.keys.grep(/^db[0-9]+/).each do |key|
+  data = info[key]
+  if data =~ /keys=(\d+),expires=(\d+)/
+    puts "PUTVAL #{HOSTNAME}/redis/gauge-keys_#{key} interval=#{INTERVAL} N:#{$1}"
+    puts "PUTVAL #{HOSTNAME}/redis/gauge-expires_#{key} interval=#{INTERVAL} N:#{$2}"
+  end
+end

data/templates/collectd/script/collectd/role/resque_worker/global_worker_stats.rb

@@ -0,0 +1,12 @@
+require "resque"
+
+redis_server = Rubber.instances.for_role('redis_slave').first.full_name rescue nil
+redis_server ||= 'localhost' if Rubber.env == 'development'
+Resque.redis = "#{redis_server}:6379"
+
+queue_sizes = {}
+Resque.redis.smembers("queues").each do |queue_name|
+  queue_size = Resque.redis.llen("queue:#{queue_name}")
+
+  puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-#{queue_name}_queued interval=#{INTERVAL} N:#{queue_size}"
+end

data/templates/collectd/script/collectd/role/resque_worker/host_worker_stats.rb

@@ -0,0 +1,70 @@
+
+WorkerItem = Struct.new(:pid, :process_type, :target, :start_time)
+
+idle = []
+starting = []
+paused = []
+parents = []
+children = []
+
+data = `ps ax | grep '[r]esque.*:'`
+data.lines do |line|
+  line.chomp!
+  cols = line.split
+
+  # 12887 ?        Sl     0:01 resque-1.13.0: Forked 31395 at 1300564431
+  # 13054 ?        Sl     0:23 resque-1.13.0: Processing facebook since 1300565337
+  # 28561 ?        Sl     0:03 resque-1.13.0: Waiting for *index*
+
+  item = WorkerItem.new
+  item.pid = cols[0]
+  item.process_type = case cols[5]
+    when 'Processing' then :child
+    when 'Forked' then :parent
+    when 'Waiting' then (cols.delete_at(6); :idle)
+    when 'Starting' then :starting
+    when 'Paused' then :paused
+    else :unknown
+  end
+  item.target = cols[6]
+  item.start_time = cols[8].to_i
+
+  idle << item if item.process_type == :idle
+  starting << item if item.process_type == :starting
+  paused << item if item.process_type == :paused
+  parents << item if item.process_type == :parent
+  children << item if item.process_type == :child
+end
+
+pairs = {}
+[*parents, *children].each {|item| pairs[item.pid] ||= [];  pairs[item.pid] << item}
+stuck_parents = pairs.select{|item| item.size == 1 && item.first.type == :parent}
+stuck_children = pairs.select{|item| item.size == 1 && item.first.type == :child}
+
+counts = children.group_by {|item| item.target || 'unknown' }
+
+working = counts.values.collect(&:size).inject(0) { |sum, p| sum + p }
+utilization = (working.to_f / (working + idle.size + starting.size + paused.size) * 100).to_i rescue 0
+puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-working interval=#{INTERVAL} N:#{working}"
+puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-utilization interval=#{INTERVAL} N:#{utilization}"
+puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-idle interval=#{INTERVAL} N:#{idle.size}"
+puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-starting interval=#{INTERVAL} N:#{starting.size}"
+puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-paused interval=#{INTERVAL} N:#{paused.size}"
+puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-stuck_parents interval=#{INTERVAL} N:#{stuck_parents.size}"
+puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-stuck_children interval=#{INTERVAL} N:#{stuck_children.size}"
+
+counts.each do |target, items|
+  puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-#{target}_working interval=#{INTERVAL} N:#{items.size}"
+end
+
+times = [1, 5, 15, 30, 60, 120, 180, 540]
+ages = children.group_by do |item|
+  runtime = Time.now.to_i - item.start_time
+  runtime = 0 if item.start_time == 0
+  times.find {|t| runtime < (t * 60)}
+end
+
+times.each do |t|
+  items = ages[t] || []
+  puts "PUTVAL #{HOSTNAME}/resque_worker/gauge-runtime_#{t} interval=#{INTERVAL} N:#{items.size}"
+end

data/templates/collectd/script/collectd/role/resque_worker/resque_jobs_processed.rb

@@ -0,0 +1,8 @@
+require "resque"
+
+redis_server = Rubber.instances.for_role('redis_master').first.full_name rescue nil
+redis_server ||= 'localhost' if Rubber.env == 'development'
+Resque.redis = "#{redis_server}:6379"
+
+puts "PUTVAL #{HOSTNAME}/resque/derive-jobs_processed interval=#{INTERVAL} N:#{Resque.redis.get("stat:processed").to_i}"
+puts "PUTVAL #{HOSTNAME}/resque/derive-jobs_failed interval=#{INTERVAL} N:#{Resque.redis.get("stat:failed").to_i}"

data/templates/collectd/templates.yml

@@ -0,0 +1,2 @@
+description: The collectd monitoring module
+skip_unknown_roles: true

data/{generators/vulcanize/templates → templates}/complete_mongrel_mysql/config/rubber/role/nginx/nginx-mongrel.conf

@@ -20,8 +20,8 @@ upstream haproxy_mongrel
       ssl                  on;
       ssl_certificate    /etc/ssl/certs/ssl-cert-snakeoil.pem;
       ssl_certificate_key /etc/ssl/private/ssl-cert-snakeoil.key;
-      # ssl_certificate      <%= RUBBER_ROOT %>/config/snapmylife.com.crt;
-      # ssl_certificate_key  <%= RUBBER_ROOT %>/config/snapmylife.com.key;
+      # ssl_certificate      <%= Rubber.root %>/config/snapmylife.com.crt;
+      # ssl_certificate_key  <%= Rubber.root %>/config/snapmylife.com.key;
     <% else %>
       listen <%= rubber_env.web_port %>;
     <% end %>
@@ -35,7 +35,7 @@ upstream haproxy_mongrel
       rewrite ^\/(.*)$ http://<%= rubber_env.domain %>/$1 permanent;
     }
 
-    root <%= RUBBER_ROOT + "/public" %>;
+    root <%= Rubber.root + "/public" %>;
 
     # this rewrites all the requests to the maintenance.html
     # page if it exists in the doc root. This is for capistrano's
@@ -101,12 +101,12 @@ upstream haproxy_mongrel
     error_page   500 502 503 504  /500.html;
     location = /500.html
     {
-      root <%= RUBBER_ROOT + "/public" %>;
+      root <%= Rubber.root + "/public" %>;
     }
     error_page 404  /404.html;
     location = /404.html
     {
-      root <%= RUBBER_ROOT + "/public" %>;
+      root <%= Rubber.root + "/public" %>;
     }
   }
 

data/{lib/generators/vulcanize/templates → templates}/complete_mongrel_mysql/config/rubber/rubber-complete.yml

@@ -2,7 +2,7 @@
 role_dependencies:
   web: [nginx, haproxy]
   app: [mongrel]
-  web_tools: [nginx]
+  web_tools: [nginx, graphite_server, graphite_web]
 
 haproxy_listen_port: 9000
 web_port: 80

data/{lib/generators/vulcanize/templates → templates}/complete_mongrel_mysql/templates.yml

@@ -2,5 +2,5 @@ description: A fairly complete and scalable mysql deployment setup
 dependent_templates:
   - minimal_mysql
   - monit
-  - munin
-  - memcached
+  - collectd
+  - graphite

data/{lib/generators/vulcanize/templates → templates}/complete_passenger/config/rubber/rubber-complete.yml

@@ -2,7 +2,7 @@
 role_dependencies:
   web: [haproxy]
   app: [passenger]
-  web_tools: [apache]
+  web_tools: [apache, graphite_server, graphite_web]
 
 web_port: 80
 web_ssl_port: 443

data/{generators/vulcanize/templates → templates}/complete_passenger/templates.yml

@@ -5,5 +5,7 @@ dependent_templates:
   - apache
   - passenger
   - monit
-  - munin
-  - memcached
+  - collectd
+  - graphite
+  - graylog
+