munin_manager 1.2.1

data/lib/munin_manager/plugins/haproxy_app_response_time.rb

@@ -0,0 +1,104 @@
+module MuninManager
+  class Plugins::HaproxyAppResponseTime < LogReader
+    include ActsAsMuninPlugin
+
+    def data
+      @data ||= Hash.new {|h, k| h[k] = Hash.new{|d,v| d[v] = Array.new}}
+    end
+
+    def scan(log_file)
+      loop do
+        line = log_file.readline
+        chunks = line.split(/\s+/)
+        server, port = chunks[8].split(":") rescue []
+        server_name = server.split("/")[1] rescue nil
+        next if server_name.nil?
+        timers = chunks[9].split("/") rescue []
+        data[server_name][:client_connect] << timers[0].to_f
+        data[server_name][:waiting_in_queue] << timers[1].to_f
+        data[server_name][:server_connect] << timers[2].to_f
+        data[server_name][:server_response] << timers[3].to_f
+        data[server_name][:rails_action] << line.match(/\{([0-9.]+)\}/).captures[0].to_f rescue 0
+        data[server_name][:total] << timers[4].to_f
+      end
+    end
+
+    def process!
+      data.each do |server, values|
+        values.each do |k, v|
+          values[k] = values[k].inject(0) {|sum, i| sum + i} / values[k].length rescue 0
+          values[k] = formatted(values[k] / 1000)
+        end
+      end
+    end
+    
+    def config
+      log_file = ENV['log_file'] || "/var/log/haproxy.log"
+      f = File.open(log_file)
+      count = 0
+      server_hash = {}
+      while(!f.eof? && count < 100)
+        count += 1
+        line = f.readline
+        chunks = line.split(/\s+/)
+        server, port = chunks[8].split(":") rescue []
+        server_name = server.split("/")[1] rescue nil
+        server_hash[server_name] = '' unless server_name.nil?
+      end
+      default = [:client_connect,:waiting_in_queue, :server_connect, :server_response, :rails_action, :total]
+            
+      config_text = <<-LABEL                     
+graph_title HAProxy Response Breakdown
+graph_vlabel time (secs)
+graph_category Haproxy
+      LABEL
+      server_hash.keys.sort.each do |server|
+        config_text << default.map{|k| "#{server}_#{k}.label #{server}_#{k}"}.join("\n")
+        config_text << "\n"
+      end
+      config_text
+    end
+
+    def values
+      data.inject([]){|datas, (server, values)| (datas + values.map{|k,v| "#{server}_#{k}.value #{v}"})}.join("\n")
+    end
+
+    def self.run
+      log_file = ENV['log_file'] || "/var/log/haproxy.log"
+      allowed_commands = ['config']
+
+      haproxy = new(log_file)
+
+      if cmd = ARGV[0] and allowed_commands.include? cmd then
+        puts haproxy.send(cmd.to_sym)
+      else
+        haproxy.collect!
+        puts haproxy.values
+      end
+    end
+
+    def self.help_text
+      %Q{
+#{plugin_name.capitalize} Munin Plugin
+===========================
+
+Please remember to add something like the lines below to /etc/munin/plugin-conf.d/munin-node
+if the haproxy log file is not at /var/log/haproxy.log
+
+[#{plugin_name}]
+env.log_file /var/log/custom/haproxy.log
+
+Also, make sure that the '/var/lib/munin/plugin-state' is writable by munin.
+
+$ sudo chmod 777 /var/lib/munin/plugin-state
+
+}
+    end
+    
+    private
+
+    def formatted(num)
+      "%.10f" % num
+    end
+  end
+end

data/lib/munin_manager/plugins/haproxy_response_time.rb

@@ -0,0 +1,87 @@
+module MuninManager
+  class Plugins::HaproxyResponseTime < LogReader
+    include ActsAsMuninPlugin
+
+    def data
+      @data ||= Hash.new {|h, k| h[k] = Array.new}
+    end
+
+    def scan(log_file)
+      loop do
+        line = log_file.readline
+        chunks = line.split(/\s+/)
+
+        timers = chunks[9].split("/") rescue []
+        data[:client_connect] << timers[0].to_f
+        data[:waiting_in_queue] << timers[1].to_f
+        data[:server_connect] << timers[2].to_f
+        data[:server_response] << timers[3].to_f
+        data[:rails_action] << line.match(/\{([0-9.]+)\}/).captures[0].to_f * 1000 rescue 0
+        data[:total] << timers[4].to_f
+      end
+    end
+
+    def process!
+      data.each do |k, v|
+        data[k] = data[k].inject(0) {|sum, i| sum + i} / data[k].length rescue 0
+        data[k] = formatted(data[k] / 1000)
+      end
+    end
+    
+    def config
+      <<-LABEL                     
+graph_title HAProxy Response Breakdown
+graph_vlabel time (secs)
+graph_category Haproxy                
+client_connect.label Client Connect
+waiting_in_queue.label  Waiting In Queue
+server_connect.label  Server Connect
+server_response.label  Server Response
+rails_action.label  Rails Controller Action
+total.label Total Response Time
+      LABEL
+    end
+
+    def values
+      data.map {|k, v| "#{k}.value #{v}"}.join("\n")
+    end
+
+    def self.run
+      log_file = ENV['log_file'] || "/var/log/haproxy.log"
+      allowed_commands = ['config']
+
+      haproxy = new(log_file)
+
+      if cmd = ARGV[0] and allowed_commands.include? cmd then
+        puts haproxy.send(cmd.to_sym)
+      else
+        haproxy.collect!
+        puts haproxy.values
+      end
+    end
+
+    def self.help_text
+      %Q{
+#{plugin_name.capitalize} Munin Plugin
+===========================
+
+Please remember to add something like the lines below to /etc/munin/plugin-conf.d/munin-node
+if the haproxy log file is not at /var/log/haproxy.log
+
+[#{plugin_name}]
+env.log_file /var/log/custom/haproxy.log
+
+Also, make sure that the '/var/lib/munin/plugin-state' is writable by munin.
+
+$ sudo chmod 777 /var/lib/munin/plugin-state
+
+}
+    end
+    
+    private
+
+    def formatted(num)
+      "%.10f" % num
+    end
+  end
+end

data/lib/munin_manager/plugins/haproxy_rps.rb

@@ -0,0 +1,95 @@
+#! /usr/bin/env ruby
+# Munin plugin for starling.
+require 'open-uri'
+require 'base64'
+
+module MuninManager
+  class Plugins::HaproxyRps
+    include ActsAsMuninPlugin
+
+    def initialize(url, user, pass)
+      @url = url
+      @user = user
+      @pass = pass
+      @data = Hash.new{|h,k|h[k] = 0}
+      @category = 'Haproxy'
+      parse_csv
+    end
+
+    def parse_csv
+      base64 = Base64.encode64("#{@user}:#{@pass}")
+      file = open(@url, {"Authorization" => "Basic #{base64}"})
+      file.readline #skip first line
+      file.each_line do |line|
+        data_ary = line.split(",")
+        host,port = data_ary[1].split(":")
+        @data[host] += data_ary[7].to_i
+      end
+    end
+
+    def ops_stats
+      defaults = {
+      'min' => 0,
+      'type' => 'DERIVE',
+      'draw' => 'LINE2',
+      }
+
+      stats = {
+        
+      }
+      
+      @data.each_key do |k|
+        stats[k] = defaults.merge({:label => k})
+      end
+      return stats
+    end
+
+    def config
+      graph_config = <<-END.gsub(/  +/, '')
+      graph_title Haproxy RPS / App Server
+      graph_args --base 1000
+      graph_vlabel ops/${graph_period}
+      graph_category #{@category}
+      graph_order #{@data.keys.sort.join(" ")}
+      END
+            
+      stat_config = []
+      ops_stats.each do |stat,config|
+        config.each do |var,value|
+          stat_config << "#{stat}.#{var} #{value}\n"
+        end
+      end
+      return graph_config + stat_config.sort.join
+    end
+
+    def values
+      ret = ''
+      @data.each do |key, stat|
+        ret << "#{key}.value #{stat}\n"
+      end
+      return ret
+    end
+
+    def self.run
+      url = ENV['URL'] || 'http://lb01.ffs.seriousops.com/haproxy?stats;csv'
+      user = ENV['USERNAME'] || 'admin'
+      pass = ENV['PASSWORD'] || 'pass'
+      haproxy = new(url, user, pass)
+
+      allowed_commands = ['config']
+
+      if cmd = ARGV[0] and allowed_commands.include? cmd then
+        puts haproxy.send(cmd.to_sym)
+      else
+        puts haproxy.values
+      end
+    end
+
+  private
+
+    def format_for_munin(str)
+      str.to_s.gsub(/[^A-Za-z0-9_]/, "_")
+    end
+
+  end
+end

data/lib/munin_manager/plugins/network_latency.rb

@@ -0,0 +1,53 @@
+module MuninManager
+  class Plugins::NetworkLatency < LogReader
+    include ActsAsMuninPlugin
+    
+    def self.run
+      hostnames = (ENV['hostnames'] || 'localhost').split(",")
+      count = (ENV["count"] || 1).to_i
+      if ARGV[0] == "config"
+        puts config(hostnames)
+      else
+        values = Hash.new {|h,k| h[k] = []}
+        threads = []
+        count.times do |i|
+          threads << Thread.new do
+            output = %x{/usr/sbin/fping -c 10 -p 100 -q #{hostnames.join(" ")} 2>&1}
+            Thread.current[:output] = output
+          end
+        end
+        threads.each do |t|
+          t.join
+          t[:output].to_s.split("\n").each do |line|
+            if line =~ /^([^\s]+)\s*:\s*xmt\/rcv\/%loss = [0-9]+\/[0-9]+\/[0-9]+%, min\/avg\/max = [0-9.]+\/([0-9.]+)\/[0-9.]+/
+              values[$1] << $2.to_f
+            end
+          end
+        end
+        
+        values.each do |host, results|
+          avg = results.size > 0 ? results.inject(0){|a,b| a + b}.to_f / results.size : -1
+          puts "#{sanitize(host)}.value #{avg}"
+        end
+      end
+    end
+    
+    def self.sanitize(hostname)
+      hostname.to_s.gsub(/[^\w]/, '_')
+    end
+    
+    def self.config(hostnames)
+      configs = {
+        "graph_title" => "Network Latency",
+        "graph_vlabel" => "time (ms)",
+        "graph_category" => "Network",
+        "graph_order" => ""
+      }
+      hostnames.each do |hostname|
+        configs["#{sanitize(hostname)}.label"] = hostname
+        configs["graph_order"] += "#{sanitize(hostname)} "
+      end
+      configs.map {|key_value_pair| key_value_pair.join(" ")}.join("\n")
+    end
+  end
+end

data/lib/munin_manager/plugins/notification_classification.rb

@@ -0,0 +1,27 @@
+require 'rubygems'
+require 'active_support'
+
+module MuninManager
+  class Plugins::NotificationClassification
+    include ActsAsMuninPlugin
+    
+    def config
+      "graph_title #{ENV['app']} Notification Classification Creation
+      graph_vlabel new classifications / hour
+      notification_rate.label new classifications / hour
+      notification_rate.type derive
+      notification_rate.warning 10
+      notification_rate.critical 100"
+    end
+    
+    def self.run
+      allowed_commands = ['config']
+      if cmd = ARGV[0] and allowed_commands.include? cmd
+        puts new.config
+      else
+        cmd = "mysql -u #{ENV['mysql_user']} --password=#{ENV['mysql_password']} -h #{ENV['host']} -e 'use #{ENV['database']}; select count(*) from notification_classifications where created_at >= \"#{1.hour.ago.to_s(:db)}\";' --skip-column-names --silent"
+        puts "notification_rate.value %s" % `#{cmd}`
+      end
+    end
+  end
+end

data/lib/munin_manager/plugins/packet_loss.rb

@@ -0,0 +1,53 @@
+module MuninManager
+  class Plugins::PacketLoss < LogReader
+    include ActsAsMuninPlugin
+    
+    def self.run
+      hostnames = (ENV['hostnames'] || 'localhost').split(",")
+      count = (ENV["count"] || 1).to_i
+      if ARGV[0] == "config"
+        puts config(hostnames)
+      else
+        values = Hash.new {|h,k| h[k] = []}
+        threads = []
+        count.times do |i|
+          threads << Thread.new do
+            output = %x{/usr/sbin/fping -c 10 -p 100 -q #{hostnames.join(" ")} 2>&1}
+            Thread.current[:output] = output
+          end
+        end
+        threads.each do |t|
+          t.join
+          t[:output].to_s.split("\n").each do |line|
+            if line =~ /^([^\s]+)\s*:\s*xmt\/rcv\/%loss = [0-9]+\/[0-9]+\/([0-9]+)%/
+              values[$1] << $2.to_f
+            end
+          end
+        end
+
+        values.each do |host, results|
+          avg = results.size > 0 ? results.inject(0){|a,b| a + b}.to_f / results.size : -1
+          puts "#{sanitize(host)}.value #{avg}"
+        end
+      end
+    end
+
+    def self.sanitize(hostname)
+      hostname.to_s.gsub(/[^\w]/, '_')
+    end
+
+    def self.config(hostnames)
+      configs = {
+        "graph_title" => "Packet Loss",
+        "graph_vlabel" => "percentage",
+        "graph_category" => "Network",
+        "graph_order" => ""
+      }
+      hostnames.each do |hostname|
+        configs["#{sanitize(hostname)}.label"] = hostname
+        configs["graph_order"] += "#{sanitize(hostname)} "
+      end
+      configs.map {|key_value_pair| key_value_pair.join(" ")}.join("\n")
+    end
+  end
+end

data/lib/munin_manager/plugins/rails_rendering.rb

@@ -0,0 +1,82 @@
+module MuninManager
+  class Plugins::RailsRendering < LogReader
+    include ActsAsMuninPlugin
+
+    def data
+      @data ||= Hash.new {|h, k| h[k] = Array.new}
+    end
+
+    def scan(log_file)
+      loop do
+        line = log_file.readline
+        next unless line.match(/^Completed in/)
+
+        chunks = line.split(/\s/)
+        data[:total] << chunks[2].to_f
+        data[:rendering] << chunks[7].to_f
+        data[:memcache] << chunks[11].to_f
+        data[:db] << chunks[14].to_f
+      end
+    end
+
+    def process!
+      data.each_pair do |component, response_times|
+        data[component] = response_times.inject(0) {|sum, i| sum + i} / data[component].length rescue 0
+      end
+    end
+
+    def config
+      <<-LABEL
+graph_title Rails Response Breakdown
+graph_vlabel response time          
+graph_category Rails                
+total.label total                   
+rendering.label rendering           
+db.label db                         
+memcache.label memcache             
+LABEL
+    end
+
+    def values
+      data.map {|k, v| "#{format_for_munin(k)}.value #{"%.10f" % v}"}.join("\n")
+    end
+
+    def self.run
+      log_file = ENV['log_file'] || "/var/log/rails.log"
+      allowed_commands = ['config']
+
+      rails = new(log_file)
+
+      if cmd = ARGV[0] and allowed_commands.include? cmd then
+        puts rails.send(cmd.to_sym)
+      else
+        rails.collect!
+        puts rails.values
+      end
+    end
+
+    def self.help_text(options = {})
+      %Q{
+#{plugin_name.capitalize} Munin Plugin
+===========================
+
+Please remember to add something like the lines below to /etc/munin/plugin-conf.d/munin-node
+if the rails log file is not at /var/log/rails.log
+
+[#{options[:symlink] || plugin_name}]
+env.log_file /var/log/custom/rails.log
+
+Also, make sure that the '/var/lib/munin/plugin-state' is writable by munin.
+
+$ sudo chmod 777 /var/lib/munin/plugin-state
+
+}
+    end
+
+    private
+
+    def format_for_munin(str)
+      str.to_s.gsub(/[^A-Za-z0-9_]/, "_")
+    end
+  end
+end