munin_manager 1.2.1

data/Manifest

@@ -0,0 +1,30 @@
+bin/munin_manager
+bin/runner
+ext/string.rb
+lib/munin_manager/acts_as_munin_plugin.rb
+lib/munin_manager/log_reader.rb
+lib/munin_manager/plugins/fbproxy_latency.rb
+lib/munin_manager/plugins/haproxy_app_response_time.rb
+lib/munin_manager/plugins/haproxy_response_time.rb
+lib/munin_manager/plugins/haproxy_rps.rb
+lib/munin_manager/plugins/network_latency.rb
+lib/munin_manager/plugins/notification_classification.rb
+lib/munin_manager/plugins/packet_loss.rb
+lib/munin_manager/plugins/rails_rendering.rb
+lib/munin_manager/plugins/rails_response_time.rb
+lib/munin_manager/plugins/scribe_net.rb
+lib/munin_manager/plugins/starling_age.rb
+lib/munin_manager/plugins/starling_net.rb
+lib/munin_manager/plugins/starling_ops.rb
+lib/munin_manager/plugins/starling_queue.rb
+lib/munin_manager/plugins.rb
+lib/munin_manager/starling/starling_stats.rb
+lib/munin_manager.rb
+Manifest
+munin_manager.gemspec
+Rakefile
+README.markdown
+test/haproxy_response_time_test.rb
+test/log_reader_test.rb
+test/rails_response_time_test.rb
+test/test_helper.rb

data/Rakefile

@@ -0,0 +1,29 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "munin_manager"
+    s.executables = "munin_manager"
+    s.summary = "Tool to maintain and install munin plugins written in Ruby"
+    s.email = "entombedvirus@gmail.com"
+    s.homepage = "http://github.com/entombedvirus/jeweler"
+    s.authors = ["Rohith Ravi"]
+    s.files =  FileList["[A-Z]*", "{bin,generators,lib,test}/**/*", 'lib/jeweler/templates/.gitignore']
+  end
+rescue LoadError
+  puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+#spec = Gem::Specification.new do |s| 
+#  s.name = "MuninManager"
+#  s.version = "0.0.1"
+#  s.platform = Gem::Platform::RUBY
+#  s.files = FileList["{bin,lib,ext}/**/*"].to_a
+#  s.require_path = "lib"
+#  s.autorequire = "munin_manager"
+#  s.test_files = FileList["{test}/**/*test.rb"].to_a
+#  s.has_rdoc = false
+#  s.extra_rdoc_files = ["README.markdown"]
+#end
+# 
+#Rake::GemPackageTask.new(spec) do |pkg| 
+#  pkg.need_tar = true 
+#end

data/VERSION

@@ -0,0 +1 @@
+1.2.1

data/bin/munin_manager

@@ -0,0 +1,95 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'ruby-debug'
+
+require 'optparse'
+require 'ostruct'
+require "#{File.dirname(__FILE__)}/../lib/munin_manager"
+
+options = OpenStruct.new
+options.plugin_dir = "/etc/munin/plugins"
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: munin_manager <command>  [options]"
+  opts.separator ""
+  
+  opts.separator "Commands:"
+  opts.separator "PLUGIN_NAME is of the form <plugin>[:<symlink>]"
+  opts.separator ""
+  
+  opts.on("-l", "--list", "List available plugins to install") do
+    buffer = []
+    buffer << "Available Plugins"
+    buffer << "================="
+    buffer << ""
+    
+    MuninManager::Plugins.each do |plugin|
+      buffer << plugin.plugin_name
+    end
+    
+    buffer << ""
+    puts buffer.join("\n")
+  end
+  
+  opts.on("-s", "--show PLUGIN_NAME", "Shows details about a plugin") do |name|
+    begin
+      puts MuninManager::Plugins[name].help_text
+    rescue NoMethodError
+      STDERR.puts "'%s' plugin was not found" % name
+    rescue
+      STDERR.puts "No additional information is available about this plugin."
+    end
+  end
+  
+  opts.on("-i", "--install PLUGIN_NAME[,PLUGIN_NAME]", Array,
+    "Installs a plugin by creating a symlink in '%s'" % options.plugin_dir) do |names|
+      options.action = :install
+      options.plugin_names = names
+  end
+  
+  opts.on("-u", "--uninstall PLUGIN_NAME[,PLUGIN_NAME]", Array,
+    "Removes plugins from '%s'" % options.plugin_dir,
+    "if it is a symlink") do |names|
+      options.action = :uninstall      
+      options.plugin_names = names
+  end
+
+  opts.separator ""
+  opts.separator "Options:"
+
+  opts.on("--plugin-dir DIR", "Directory where symlinks will be created when a plugin is installed",
+         "(default is '%s'" % options.plugin_dir) do |dir|
+    if File.directory?(dir) && File.writable?(dir)
+      options.plugin_dir = dir
+    else
+      STDERR.puts "'%s' does not exist or is not writable" % options.plugin_dir
+    end
+  end
+
+  opts.on("--force", "Forces the installation or uninstallation of plugins") do
+    options.force = true
+  end
+  
+  
+end
+
+parser.parse!(ARGV)
+options.freeze!
+
+case options.action
+when :install, :uninstall
+  options.plugin_names.each do |plugin_name|
+    plugin_klass = MuninManager::Plugins.search(plugin_name)
+    
+    unless plugin_klass
+      STDERR.puts "ERROR: Matching plugin not found for '%s'" % plugin_name
+      exit(1)
+    end
+
+    new_options = options.dup
+    new_options.install_name = plugin_name
+
+    plugin_klass.send(options.action, new_options)
+  end
+end

data/bin/runner

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+# This file will serve as the target for the symlinks in /etc/munin/plugins.
+# It's the responsibility of this script to run the plugin indicated by the
+# symlink name.
+
+this_file = File.symlink?(__FILE__) ? File.readlink(__FILE__) : __FILE__
+require "#{File.dirname(this_file)}/../lib/munin_manager"
+
+plugin_name = File.basename($0)
+
+if plugin = MuninManager::Plugins[plugin_name.split('.')[0]]
+  plugin.run
+  exit(0)
+else
+  STDERR.puts "'%s' plugin was not found" % plugin_name
+  exit(1)
+end

data/lib/munin_manager.rb

@@ -0,0 +1,9 @@
+munin_manager = File.join(File.dirname(__FILE__), "munin_manager")
+
+%w(../../ext/string log_reader plugins acts_as_munin_plugin).each do |f|
+  require File.join(munin_manager, f)
+end
+
+Dir[File.join(munin_manager, "plugins", "*")].each do |file|
+  require file
+end

data/lib/munin_manager/acts_as_munin_plugin.rb

@@ -0,0 +1,82 @@
+require 'fileutils'
+
+module MuninManager
+  module ActsAsMuninPlugin
+    def self.included(klass)
+      klass.send(:include, InstanceMethods)
+      klass.extend(ClassMethods)
+    end
+    
+    module ClassMethods
+      def run
+        raise "This is the entry point of the plugin when invoked by munin. Needs implementation by included class"
+      end
+
+      def plugin_name
+        # Name of the plugin. Must not contain spaces or special chars
+        
+        # Default is underscorized version of the class name
+        self.name.split('::').last.
+          gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+          gsub(/([a-z\d])([A-Z])/,'\1_\2').
+          tr("-", "_").
+          downcase
+      end
+
+      def help_text(options = {})
+        # Any general info concerning the plugin. Should be overriden by included class
+      end
+
+      def install(options)
+        install_as = options.install_name.split(":").last
+        symlink = File.join(options.plugin_dir, install_as)
+        runner = File.join(File.dirname(__FILE__), "..", "..", "bin", "runner")
+        runner = File.expand_path(runner)
+
+        if File.exists?(symlink)
+          if options.force
+            File.unlink(symlink)
+          else
+            STDERR.puts "'%s' already exists. Please specify --force option to overwrite" % symlink
+            return
+          end
+        end
+
+        STDOUT.puts "Installing '%s' at '%s'" % [plugin_name, symlink]
+        FileUtils.ln_sf(runner, symlink)
+
+        STDOUT.puts help_text(:symlink => install_as)
+
+      rescue Errno::EACCES
+        STDERR.puts "ERROR: Permission denied while attempting to install to '%s'" % symlink
+      end
+
+      # Default uninstaller. Override in included classes if the default is not sufficient
+      def uninstall(options)
+        install_as = options.install_name.split(":").last
+        symlink = File.join(options.plugin_dir, install_as)
+
+        unless File.exists?(symlink)
+          STDERR.puts "'%s' does not seem to exist. Aborting..." % symlink
+          return
+        end
+
+        unless File.symlink?(symlink) || options.force
+          STDERR.puts "'%s' does not appear to be a symlink. Please specify --force option to remove" % symlink
+          return
+        end
+
+        STDOUT.puts "Removing '%s'..." % symlink
+        File.unlink(symlink)
+      rescue Errno::EACCES
+        STDERR.puts "ERROR: Permission denied while attempting to uninstall '%s'" % symlink
+      end
+    end
+
+    module InstanceMethods
+      def config
+        raise "This is invoked by munin to get graph details. Needs implementation by included class"
+      end
+    end
+  end
+end

data/lib/munin_manager/log_reader.rb

@@ -0,0 +1,85 @@
+module MuninManager
+  # A LogReader that continues from where it left off.
+  # Ex:
+  # class RailsLogReader < LogReader
+  #     def scan(log_file)
+  #       @req_counter = 0
+  #       loop do
+  #         line = log_file.readline
+  #         @req_counter += 1 if line =~ /Completed in/
+  #       end
+  #     end
+  #     
+  #     def process!
+  #       # Do nothing
+  #     end
+  #     
+  #     def print_data
+  #       "num_requests.value #{@req_counter}"
+  #     end
+  #   end
+  #   
+  #   Usage:
+  #   
+  #   reader = RailsLogReader.new("log/development.log")
+  #   reader.collect!
+  #   reader.print_data
+  #   
+  class LogReader
+    attr_accessor :file_name, :me, :state_dir, :state_file
+
+    def initialize(file_name)
+      @file_name = file_name
+      @me = File.basename($0)
+      @state_dir = ENV['MUNIN_PLUGSTATE'] || '/var/lib/munin/plugin-state'
+    end
+
+    def state_file
+      @state_file ||= File.join(@state_dir, @me)
+    end
+
+    def collect!(options = {})
+      options = {
+        :save_state => true
+      }.merge(options)
+
+      File.open(@file_name, "r") do |f|
+        load_saved_state(f)
+        
+        begin
+          scan(f)
+        rescue EOFError
+        end
+        
+        process!
+        save_state(f) if options[:save_state]
+      end
+    end
+
+    def load_saved_state(log_file)
+      return unless File.exists?(state_file) && !(state = File.read(state_file)).nil?
+      pos, last_file_size = Marshal.load(state)
+
+      # Check for log rotation
+      return if File.size(@file_name) < last_file_size
+
+      log_file.pos = pos
+    end
+
+    def scan(log_file)
+      # Only subclasses know how to process each type of logfile
+      raise "Needs to be implemented by subclasses"
+    end
+
+    def process!
+      raise "Needs to be implemented by subclasses"
+    end
+    
+    def save_state(log_file)
+      File.open(state_file, "w") do |f|
+        f.write(Marshal.dump([log_file.pos, File.size(log_file)]))
+        f.flush
+      end
+    end
+  end
+end

data/lib/munin_manager/plugins.rb

@@ -0,0 +1,29 @@
+module MuninManager
+  module Plugins
+    extend self
+    extend Enumerable
+    
+    def each(&block)
+      registered_plugins.each(&block)
+    end    
+    
+    def registered_plugins
+      @registered_plugins ||= constants.
+        map {|c| const_get(c) }.
+        select {|const| const.is_a?(Class) && const < MuninManager::ActsAsMuninPlugin}
+    end
+    
+    def [](*names)
+      if names.length == 1
+        return detect {|plugin_klass| plugin_klass.plugin_name == names.first}
+      end
+      
+      names.map {|name| self[name]}
+    end
+
+    def search(name)
+      str = name.to_s.split(':', 2).first
+      detect {|plugin_klass| plugin_klass.plugin_name.starts_with?(str)}
+    end
+  end
+end

data/lib/munin_manager/plugins/fbproxy_latency.rb

@@ -0,0 +1,78 @@
+module MuninManager
+  class Plugins::FbProxyLatency < LogReader
+    include ActsAsMuninPlugin
+
+    def data
+      @data ||= Hash.new {|h, k| h[k] = Array.new}
+    end
+
+    def scan(log_file)
+      loop do
+        line = log_file.readline
+        next unless line.match(/^Benchmark /)
+        chunks = line.split(/-/).map{ |x| x.strip }
+        data_type = chunks[1] =~ /Queue/ ? 'queue' : 'fb_api'
+        next if chunks[2].nil?
+        data[data_type] << chunks[2].match('\((.*)\)')[1].to_f
+      end
+    end
+
+    def process!
+      data.each_pair do |component, response_times|
+        data[component] = response_times.inject(0.0) {|sum, i| sum + i} / data[component].length rescue 0
+      end
+    end
+
+    def config
+      <<-LABEL
+graph_title Facebook Proxy Latency
+graph_vlabel latency          
+graph_category Facebook Proxy                
+queue.label queue_latency                   
+fb_api.label fb_api_latency
+LABEL
+    end
+
+    def values
+      data.map {|k, v| "#{format_for_munin(k)}.value #{"%.10f" % v}"}.join("\n")
+    end
+
+    def self.run
+      log_file = ENV['log_file'] || "/var/log/rails.log"
+      allowed_commands = ['config']
+
+      rails = new(log_file)
+
+      if cmd = ARGV[0] and allowed_commands.include? cmd then
+        puts rails.send(cmd.to_sym)
+      else
+        rails.collect!
+        puts rails.values
+      end
+    end
+
+    def self.help_text(options = {})
+      %Q{
+#{plugin_name.capitalize} Munin Plugin
+===========================
+
+Please remember to add something like the lines below to /etc/munin/plugin-conf.d/munin-node
+if the rails log file is not at /var/log/rails.log
+
+[#{options[:symlink] || plugin_name}]
+env.log_file /var/log/custom/rails.log
+
+Also, make sure that the '/var/lib/munin/plugin-state' is writable by munin.
+
+$ sudo chmod 777 /var/lib/munin/plugin-state
+
+}
+    end
+
+    private
+
+    def format_for_munin(str)
+      str.to_s.gsub(/[^A-Za-z0-9_]/, "_")
+    end
+  end
+end