munin2graphite 0.1.2 → 0.1.6

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.1.6

data/bin/munin2gdash

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__),"..","lib")))
+require 'rubygems'
+require 'munin2graphite'
+require 'optparse'
+require 'erb'
+
+options = {} 
+optparse = OptionParser.new do |opts|
+  opts.banner = "Usage: #{__FILE__} [options]"
+
+  opts.on( '-t', '--template-dir TEMPLATE_DIR', 'The directory you want the template files to be placed' ) do |td|
+    options[:template_dir] = td
+  end
+
+  options[:config_file] = "/etc/munin2graphite/munin2graphite.conf"
+  opts.on( '-c', '--config CONFIG_FILE', 'The munin2graphite config file' ) do |td|
+    options[:config_file] = td
+  end  
+
+  opts.on( '-n', '--node NODE', "The munin node you want the data to be extracted from") do |n|
+    options[:node] = n
+  end
+end
+
+optparse.parse!
+
+if !options[:template_dir]
+  puts "Error, no template dir specified"
+  puts optparse.help
+  exit 1
+end
+  
+Munin2Graphite::Config.config_file = options[:config_file]
+scheduler = Munin2Graphite::Scheduler.new(Munin2Graphite::Config)
+
+munin_config = scheduler.munin_config
+munin_config[:workers].each do |worker|
+  time = Time.now
+  config = Munin2Graphite::Config.config_for_worker worker
+  munin_config[worker][:nodes].keys.each do |node|
+    config.log.info("Graphs for #{node}")
+    munin_config[worker][:nodes][node][:metrics].each do |metric,value|
+      munin_graph = MuninGraph.graph_for value[:raw_config]
+      munin_graph.config = config.merge("metric" => "#{metric}","hostname" => node.split(".").first)
+      File.open(File.join(options[:template_dir],metric.to_s + ".graph"), "w+") do |file|
+        file.write munin_graph.to_gdash 
+      end
+    end
+  end
+  config.log.info("End : Sending Graph Information to Graphite for worker #{worker}, elapsed time (#{Time.now - time}s)")
+end
+#puts scheduler.munin_config
+scheduler

data/conf/munin2graphite.conf.example

@@ -1,8 +1,12 @@
 # Log config
 # log: The logfile, STDOUT if stdout is needed
 # log_level: Either DEBUG, INFO or WARN
+# Rotation (shift_age and shift_size)
+# 
 log=/var/log/munin2graphite.log
 log_level=INFO
+log_shift_age=1        # 0 if for no rotation
+log_shift_size=100000
 
 # Carbon backend
 # This has to point to the carbon backend to submit metrics

data/lib/ast_node.rb

@@ -96,6 +96,22 @@ class ASTNode
     url = "#{properties[:endpoint]}/render/?width=586&height=308&#{properties_to_url}&target=" + URI.escape(targets.map{|i| i.compile}.compact.join("&target="))
   end
 
+  def to_gdash
+    output = ""
+    self.compile
+    self.graph_properties.each do |k,v| 
+      output += k.to_s + "\t\t" + '"' + v.to_s + '"' + "\n" unless k == :colorList || k == :yMin || k== :yMax
+    end
+    count = 0
+    targets.each do |tg|
+      metric_alias = tg.properties.delete(:alias)
+      tg.children.delete_if { |i| i.class == LabelFieldPropertyNode}
+      output += "field :#{tg.metric.split(".").last.to_sym},:alias => '#{metric_alias}', :data => \"#{tg.compile}\"\n"
+      count += 1
+    end
+    return output
+  end
+
 end
 
 
@@ -248,8 +264,9 @@ class TypeFieldPropertyNode < FieldPropertyNode
   
   def apply_function(operand)
     if @value == "DERIVE" || @value == "COUNTER"
-      # The scaling is because of the minutes/seconds"
-      return "scale(nonNegativeDerivative(#{operand}),0.0166666666666667)"
+      # The scaling is because of the minutes/(60*seconds)"
+      #return "scale(nonNegativeDerivative(#{operand}),0.0166666666666667)"
+      return "scale(nonNegativeDerivative(#{operand}),0.00333333333333333)"
     end
     operand
   end

data/lib/munin2graphite/config.rb

@@ -89,11 +89,13 @@ module Munin2Graphite
         super
       end
 
-      def log        
+      def log
+        shift_age  = self["log_shift_age"] ? self["log_shift_age"].to_i : 1
+        shift_size = self["log_shift_size"].to_i || 100000
         @log ||= if self["log"] == "STDOUT"
-                   Logger.new(STDOUT)
+                   Logger.new(STDOUT, shift_age, shift_size)
                  else
-                   Logger.new(self["log"])
+                   Logger.new(self["log"], shift_age, shift_size)
                  end
         @log.level = self["log_level"] == "DEBUG" ? Logger::DEBUG : Logger::INFO
         @log

data/lib/munin2graphite/scheduler.rb

@@ -26,43 +26,50 @@ module Munin2Graphite
     def initialize(config)
       @config = config
     end
-    
+
     def category_from_config(config)
       config.each_line do |configline|
-        if configline =~ /^graph_category ([0-9A-Za-z_-]+)$/
+        if configline =~ /^graph_category ([\w\-_\.]+)$/
           return configline.split[1]
         end
-      end      
-      return "other"
+      end
       raise "CategoryNotFound in #{config}"
     end
 
     def munin_config
-      return @munin_config if @munin_config 
+      return @munin_config if @munin_config
       @munin_config = {}
-
+      @config.log.info("Obtaining metrics configuration")
+      @munin_config[:workers] = []
       workers.each do |worker|
-        @munin_config[worker] = {}
+        current_config = {}
         config = @config.config_for_worker(worker)
-        begin
-          munin  = Munin::Node.new(config["munin_hostname"],config["munin_port"])
-          nodes = config["munin_nodes"] ? config["munin_nodes"].split(",") : munin.nodes
-          @munin_config[worker][:nodes] = {}
-          nodes.each do |node|
-            @munin_config[worker][:nodes][node] = {:metrics => munin.list(node)}
-            @munin_config[worker][:nodes][node][:categories] = {}
-            @munin_config[worker][:nodes][node][:metrics].each do |metric|
-              @munin_config[worker][:nodes][node][:config] = munin.config(metric)[metric]
-              @munin_config[worker][:nodes][node][:raw_config] = munin.config(metric,true)[metric]
-              @munin_config[worker][:nodes][node][:categories][metric] = category_from_config(@munin_config[worker][:nodes][node][:raw_config])
+        munin  = Munin::Node.new(config["munin_hostname"],config["munin_port"])
+        nodes = config["munin_nodes"] ? config["munin_nodes"].split(",") : munin.nodes
+        current_config[:nodes] = {}
+        nodes.each do |node|
+          metrics = munin.list(node)
+          current_config[:nodes][node] = { :metrics => {} }
+          metrics.each do |metric|
+            begin
+              raw_config = munin.config(metric,true)[metric]
+              category = category_from_config(raw_config)
+              current_config[:nodes][node][:metrics][metric] = {
+                :config => munin.config(metric)[metric],
+                :raw_config => raw_config,
+                :category => category
+              }
+            rescue Exception
+              config.log.error("Error when trying to obtain graph conf. Ignored (config was #{raw_config})")
             end
           end
-          munin.disconnect
-        rescue Exception
-          config.log.error("There was an error trying to obtain info from node #{config["munin_hostname"]}")
-          config.log.error $!
         end
+        #       @config.log.debug(current_config.inspect)
+        @munin_config[worker] = current_config
+        @munin_config[:workers] << worker
+        munin.disconnect
       end
+#      @config.log.debug(@munin_config.inspect)
       @munin_config
     end
 
@@ -72,112 +79,107 @@ module Munin2Graphite
 
     #
     # This is the loop of the metrics scheduling
-    def obtain_metrics
+    def obtain_metrics(worker = "global")
       config = @config.config_for_worker("global")
-      config.log.info("Obtaining metrics configuration")
-      munin_config
-      config.log.info("Getting metrics")
       time = Time.now
-      workers.each do |worker|        
-        config = @config.config_for_worker(worker)
-        config.log.info("Worker #{worker}")
-
-        metric_base = config["graphite_metric_prefix"]
-
-        threads = []
-        munin_config[worker][:nodes].keys.each do |node|
-          threads << Thread.new do 
-            node_name = metric_base + "." + node.split(".").first
-            config.log.debug("Doing #{node_name}")
-            values = {}                       
-            config.log.debug("Asking for: #{node}")		
-            metric_time = Time.now
-            metrics = munin_config[worker][:nodes][node][:metrics]
-	    config.log.debug("Metrics " + metrics.join(","))
-            metrics_threads = []
-            categories = {}
-            metrics.each do |metric|
-              metrics_threads << Thread.new do
-                begin
-                  local_munin  = Munin::Node.new(config["munin_hostname"],config["munin_port"])
-                  values[metric] =  local_munin.fetch metric
-                  local_munin.disconnect
-                rescue Exception
-                  config.log.error("Error when trying to obtain values for #{metric}. Ignored")
-                  config.log.error $!
-                end
-              end            
-            end 
-            metrics_threads.each {|i| i.join;i.kill}
-            config.log.info("Done with: #{node} (#{Time.now - metric_time} s)")	
-            carbon = Carbon.new(config["carbon_hostname"],config["carbon_port"])
-            string_to_send = ""
-            values.each do |metric,results|          
-              category = munin_config[worker][:nodes][node][:categories][metric] 
-              results.each do |k,v|
-                v.each do |c_metric,c_value|
-                  string_to_send += "#{node_name}.#{category}.#{metric}.#{c_metric} #{c_value} #{Time.now.to_i}\n".gsub("-","_")  if c_value != "U"
-                end
-              end
+      config = @config.config_for_worker(worker)
+      config.log.info("Worker #{worker}")
+      metric_base = config["graphite_metric_prefix"]
+      
+      munin_config[worker][:nodes].each do |node,node_info|
+        node_name = metric_base + "." + node.split(".").first
+        config.log.debug("Doing #{node_name}")
+        values = {}
+        config.log.debug("Asking for: #{node}")
+        metric_time = Time.now
+        metrics = node_info[:metrics].keys
+        config.log.debug("Metrics " + metrics.join(","))
+        metrics_threads = []
+        categories = {}
+        metrics.each do |metric|
+          metrics_threads << Thread.new do
+            begin
+              local_munin  = Munin::Node.new(config["munin_hostname"],config["munin_port"])
+              values[metric] =  local_munin.fetch metric
+              local_munin.disconnect
+            rescue
+              @config.log.error("There was a problem when getting the metric #{metric} for #{node} , Ignored")
             end
-            send_time = Time.now
-            carbon.send(string_to_send)
-            carbon.flush
-            carbon.close
-           end
-        end if munin_config[worker][:nodes]
-        threads.each { |i| i.join }
-      end
-      @config.log.info("End   getting metrics, elapsed time (#{Time.now - time}s)")
+          end
+        end
+        metrics_threads.each {|i| i.join;i.kill}
+        config.log.debug(values.inspect)
+        config.log.info("Done with: #{node} (#{Time.now - metric_time} s)")
+        carbon = Carbon.new(config["carbon_hostname"],config["carbon_port"])
+        string_to_send = ""
+        values.each do |metric,results|          
+          category = node_info[:metrics][metric][:category] 
+          results.each do |k,v|
+            v.each do |c_metric,c_value|
+              string_to_send += "#{node_name}.#{category}.#{metric}.#{c_metric} #{c_value} #{Time.now.to_i}\n".gsub("-","_")  if c_value != "U"
+            end
+          end
+        end
+        @config.log.debug(string_to_send)
+        send_time = Time.now
+        carbon.send(string_to_send)
+        carbon.flush
+        carbon.close
+      end if munin_config[worker]
+      @config.log.info("End getting metrics for worker #{worker}, elapsed time (#{Time.now - time}s)")
     end
 
-    ##
-    # The loop of the graphics creation
     def obtain_graphs
-
-      workers = @config.workers
-      workers = ["global"] if workers.empty?
-
-      workers.each do |worker|
-        time = Time.now 
+      munin_config
+      munin_config[:workers].each do |worker|
+        time = Time.now
         config = @config.config_for_worker worker
-        config.log.info("Begin : Sending Graph Information to Graphite for worker #{worker}")
-	begin
-          munin  = Munin::Node.new(config["munin_hostname"],config["munin_port"])
-          Graphite::Base.set_connection(config["graphite_endpoint"])
-          Graphite::Base.authenticate(config["graphite_user"],config["graphite_password"])
-
-          nodes = config["munin_nodes"] ? config["munin_nodes"].split(",") : munin.nodes
-          nodes.each do |node|
-            config.log.info("Graphs for #{node}")
-            munin.list(node).each do |metric|
-              config.log.info("Configuring #{metric}")
-	      munin_graph = MuninGraph.graph_for munin.config(metric,true)[metric]
-              munin_graph.config = config.merge("metric" => "#{metric}","hostname" => node.split(".").first)
-              config.log.debug("Saving graph #{metric}")
-              munin_graph.to_graphite.save!
-            end
+        @config.log.info("Begin : Sending Graph Information to Graphite for worker #{worker}")
+        Graphite::Base.set_connection(config["graphite_endpoint"])
+        Graphite::Base.authenticate(config["graphite_user"],config["graphite_password"])
+        munin_config[worker][:nodes].keys.each do |node|
+          @config.log.info("Graphs for #{node}")
+          munin_config[worker][:nodes][node][:metrics].each do |metric,value|
+            @config.log.info("Configuring #{metric}")
+            munin_graph = MuninGraph.graph_for value[:raw_config]
+            munin_graph.config = config.merge("metric" => "#{metric}","hostname" => node.split(".").first)
+            @config.log.debug("Saving graph #{metric}")
+            munin_graph.to_graphite.save!
           end
-          config.log.info("End   : Sending Graph Information to Graphite for worker #{worker}, elapsed time (#{Time.now - time}s)")
-  	  munin_graph.to_graphite.save!
-
-          munin.disconnect
-        rescue Exception
-          config.log.error("Error when trying to obtain graph conf. Ignored")
-          config.log.error $!
         end
+        config.log.info("End : Sending Graph Information to Graphite for worker #{worker}, elapsed time (#{Time.now - time}s)")
       end
+    end
 
+    def metric_loop(worker)
+      config = @config.config_for_worker worker        
+      retries = 3
+      begin
+        obtain_metrics(worker)
+      rescue => e
+        config.log.error("Exception found: (#{e.to_s})")
+        e.backtrace.each { |line| config.log.error(line) }
+        sleep 1
+        retries -= 1
+        config.log.error("Retrying")
+        retry unless retries < 0
+        config.log.error("Exitting, exception not solved")
+        exit(1)
+      end    
     end
 
     def start
       @config.log.info("Scheduler started")
-      @scheduler = Rufus::Scheduler.start_new
-      obtain_metrics
-      @scheduler.every @config["scheduler_metrics_period"] do
-        obtain_metrics
-      end
       obtain_graphs
+      @scheduler = Rufus::Scheduler.start_new
+      workers.each do |worker|        
+        config = @config.config_for_worker worker        
+        config.log.info("Scheduling worker #{worker} every  #{config["scheduler_metrics_period"]} ")
+        metric_loop(worker)
+        @scheduler.every config["scheduler_metrics_period"] do
+          metric_loop(worker)
+        end
+      end      
     end
   end
 end

data/lib/munin_graph.rb

@@ -52,6 +52,11 @@ class MuninGraph
     return graph
   end
 
+  def to_gdash
+    self.root.compile
+    self.root.to_gdash
+  end
+
   attr_reader :root
   
   # This array of hashes will be used to match what kind of line we are dealing with and 

data/munin2graphite.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = "munin2graphite"
-  s.version = "0.1.2"
+  s.version = "0.1.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Fernandez (magec)"]
-  s.date = "2011-12-14"
+  s.date = "2012-02-17"
   s.description = "This gem will install as a daemon and can be used to connect to a graphite and a carbon backend. It will not only post the data for the metrics but also create graphs into graphite, by means of a translation from munin-node."
   s.email = "jfernandezperez@gmail.com"
-  s.executables = ["munin2graphite", "munin2graphite-daemon"]
+  s.executables = ["munin2gdash", "munin2graphite", "munin2graphite-daemon"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.markdown"
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
     "README.markdown",
     "Rakefile",
     "VERSION",
+    "bin/munin2gdash",
     "bin/munin2graphite",
     "bin/munin2graphite-daemon",
     "conf/munin2graphite.conf.example",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: munin2graphite
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-14 00:00:00.000000000Z
+date: 2012-02-17 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
-  requirement: &23592960 !ruby/object:Gem::Requirement
+  requirement: &13760080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -21,10 +21,10 @@ dependencies:
         version: 2.0.10
   type: :runtime
   prerelease: false
-  version_requirements: *23592960
+  version_requirements: *13760080
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &23592440 !ruby/object:Gem::Requirement
+  requirement: &13744200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -32,10 +32,10 @@ dependencies:
         version: 1.1.4
   type: :runtime
   prerelease: false
-  version_requirements: *23592440
+  version_requirements: *13744200
 - !ruby/object:Gem::Dependency
   name: parseconfig
-  requirement: &23591880 !ruby/object:Gem::Requirement
+  requirement: &13743300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *23591880
+  version_requirements: *13743300
 - !ruby/object:Gem::Dependency
   name: munin-ruby
-  requirement: &23591340 !ruby/object:Gem::Requirement
+  requirement: &13742340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.2.1
   type: :runtime
   prerelease: false
-  version_requirements: *23591340
+  version_requirements: *13742340
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &23590860 !ruby/object:Gem::Requirement
+  requirement: &13741460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 1.6.3
   type: :runtime
   prerelease: false
-  version_requirements: *23590860
+  version_requirements: *13741460
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &23590320 !ruby/object:Gem::Requirement
+  requirement: &13740580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *23590320
+  version_requirements: *13740580
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &23589820 !ruby/object:Gem::Requirement
+  requirement: &13740060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +87,10 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *23589820
+  version_requirements: *13740060
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &23589320 !ruby/object:Gem::Requirement
+  requirement: &13739540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,12 +98,13 @@ dependencies:
         version: 0.6.0
   type: :development
   prerelease: false
-  version_requirements: *23589320
+  version_requirements: *13739540
 description: This gem will install as a daemon and can be used to connect to a graphite
   and a carbon backend. It will not only post the data for the metrics but also create
   graphs into graphite, by means of a translation from munin-node.
 email: jfernandezperez@gmail.com
 executables:
+- munin2gdash
 - munin2graphite
 - munin2graphite-daemon
 extensions: []
@@ -116,6 +117,7 @@ files:
 - README.markdown
 - Rakefile
 - VERSION
+- bin/munin2gdash
 - bin/munin2graphite
 - bin/munin2graphite-daemon
 - conf/munin2graphite.conf.example
@@ -155,7 +157,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3397207126724814962
+      hash: -732552108967367199
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: