fluent-plugin-munin 0.3.1 → 0.3.2

data/README.md

@@ -44,17 +44,22 @@ gem install fluent-plugin-munin
 `````
 
 ### Output Sample
-record_hostname: no, nest_result: no
+record_hostname: no, nest_result: no  #DEFAULT
 `````
 input.munin.cpu: {"service":"cpu","user":"113183","nice":"340","system":"26584","idle":"74205345","iowait":"26134","irq":"1","softirq":"506","steal":"0","guest":"0"}
 `````
 
+tag_prefix: input.${hostname}-munin, record_hostname: yes, nest_result: no
+`````
+input.myhost.example.com-munin.cpu: {"hostname":"myhost.example.com","service":"cpu","user":"113183","nice":"340","system":"26584","idle":"74205345","iowait":"26134","irq":"1","softirq":"506","steal":"0","guest":"0"}
+`````
+
 record_hostname: yes, nest_result: no
 `````
 input.munin.cpu: {"hostname":"myhost.example.com","service":"cpu","user":"113183","nice":"340","system":"26584","idle":"74205345","iowait":"26134","irq":"1","softirq":"506","steal":"0","guest":"0"}
 `````
 
-record_hostname: yes, nest_result: no, convert_type: yes #RECOMMEND
+record_hostname: yes, nest_result: no, convert_type: yes  #RECOMMEND
 `````
 input.munin.cpu: {"hostname":"myhost.example.com","service":"cpu","user":113183,"nice":340,"system":26584,"idle":74205345,"iowait":26134,"irq":1,"softirq":506,"steal":0,"guest":0}
 `````
@@ -71,8 +76,17 @@ record_hostname: yes, nest_result: yes, convert_type: yes
 > db.cpu.find({ "data.iowait" : { $gt : 200000 } })
 `````
 
+### Example
+
+* Example1: how to send munin metrics to treasuredata.<br>
+https://github.com/y-ken/fluent-plugin-munin/blob/master/example.conf
+
+* Example2: how to send munin metrics to mongoDB.<br>
+https://github.com/y-ken/fluent-plugin-munin/blob/master/example2.conf
+
+
 ## TODO
-patches welcome!
+Pull requests are very welcome!!
 
 ## Copyright
 

data/example.conf

@@ -0,0 +1,43 @@
+It is a sample to send munin metrics to treasuredata.
+
+# appearing plugins:
+#  munin: http://rubygems.org/gems/fluent-plugin-munin
+#  rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
+#  tdlog: http://rubygems.org/gems/fluent-plugin-td
+
+# recieve all metrics from munin.
+<source>
+  type             munin
+  interval         60s
+  tag_prefix       rewrite.munin
+  record_hostname  yes
+  convert_type     yes
+</source>
+
+# rewrite tag for "df" to pass the treasuredata table name length limitation.
+# It could avoid the error below.
+# "Name must be 3 to 256 characters, got 2 characters."
+<match rewrite.munin.*>
+  type rewrite_tag_filter
+  rewriterule1 service ^df$ td.munin.diskfree
+  rewriterule2 service ^(.+)$ td.munin.$1
+</match>
+
+<match td.*.*>
+  type tdlog
+  apikey YOUR_API_KEY
+  auto_create_table
+  buffer_type file
+  buffer_path /var/log/td-agent/buffer/td
+  use_ssl true
+</match>
+
+# treasure data stored sample.
+# $ td tables
+# +----------+---------------------+------+-------+--------+
+# | Database | Table               | Type | Count | Schema |
+# +----------+---------------------+------+-------+--------+
+# | munin    | cpu                 | log  | 5     |        |
+# | munin    | diskfree            | log  | 5     |        |
+# +----------+---------------------+------+-------+--------+
+# 2 rows in set

data/example2.conf

@@ -0,0 +1,32 @@
+It is a sample to send munin metrics to mongoDB.
+
+# appearing plugins:
+#  munin: http://rubygems.org/gems/fluent-plugin-munin
+#  mongo: http://rubygems.org/gems/fluent-plugin-mongo
+
+# recieve all metrics from munin.
+<source>
+  type             munin
+  interval         60s
+  tag_prefix       mongo.munin
+  record_hostname  yes
+  convert_type     yes
+</source>
+
+<match mongo.*.*>
+  type mongo
+  host localhost
+  port 27017
+  database mongo
+  tag_mapped
+</match>
+
+# mongoDB stored sample.
+# $ mongo
+# MongoDB shell version: 2.2.0
+# connecting to: test
+# > use munin
+# switched to db munin
+# > show collections
+# cpu
+# df

data/fluent-plugin-munin.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-munin"
-  s.version     = "0.3.1"
+  s.version     = "0.3.2"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-munin"
-  s.summary     = %q{Fluentd Input plugin to fetch munin-node metrics data with custom intervals.}
+  s.summary     = %q{Fluentd Input plugin to fetch munin-node metrics data with custom intervals. It supports all of munin plugins.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fluent/plugin/in_munin.rb

@@ -19,15 +19,31 @@ module Fluent
 
     def configure(conf)
       super
-      @hostname = get_munin_hostname
+
       @interval = Config.time_value(@interval)
-      service_list = get_service_list
-      @services = @service == 'all' ? service_list : @service.split(',')
       @nest_result = Config.bool_value(@nest_result) || false
       @convert_type = Config.bool_value(@convert_type) || false
       @record_hostname = Config.bool_value(@record_hostname) || false
-      $log.info "munin-node connected: #{@hostname} #{service_list}"
-      $log.info "following munin-node service: #{@service}"
+
+      self
+    end
+
+    def configure_munin
+      retry_interval = 30
+      max_retry_interval = retry_interval * 2 ** (8 - 1)
+      begin
+        @munin = get_connection
+        @hostname = @munin.nodes.join(',')
+        service_list = get_service_list
+        @services = @service == 'all' ? service_list : @service.split(',')
+        $log.info "munin: munin-node ready ", :hostname=>@hostname, :service_list=>service_list
+        $log.info "munin: activating service ", :service=>@services
+      rescue => e
+        $log.warn "munin: connect failed ",  :error_class=>e.class, :error=>e.message, :retry_interval=>retry_interval
+        sleep retry_interval
+        retry_interval *= 2 if retry_interval < max_retry_interval
+        retry
+      end
     end
 
     def start
@@ -41,20 +57,9 @@ module Fluent
     end
 
     def run
-      loop do 
-        @services.each do |key|
-          tag = "#{@tag_prefix}.#{key}".gsub('__HOSTNAME__', @hostname).gsub('${hostname}', @hostname)
-          record = Hash.new
-          record.store('hostname', @hostname) if @record_hostname
-          record.store('service', key)
-          if (@nest_result)
-            record.store(@nest_key, fetch(key))
-          else
-            record.merge!(fetch(key).to_hash)
-          end
-          Engine.emit(tag, Engine.now, record)
-        end
-        disconnect
+      configure_munin
+      loop do
+        emit_collected_service
         sleep @interval
       end
     end
@@ -68,36 +73,34 @@ module Fluent
       @munin.connection.close
     end
 
-    def get_munin_hostname
-      @munin ||= get_connection
-      begin
-        return @munin.nodes.join(',')
-      rescue Munin::ConnectionError
-        @munin = get_connection
-        retry
-      end
+    def get_service_list
+      @munin = get_connection
+      return @munin.list
     end
 
-    def get_service_list
-      @munin ||= get_connection
-      begin
-        return @munin.list 
-      rescue Munin::ConnectionError
-        @munin = get_connection
-        retry
+    def emit_collected_service
+      @services.each do |key|
+        tag = "#{@tag_prefix}.#{key}".gsub(/(\${[a-z]+}|__[A-Z]+__)/, get_placeholder)
+        record = Hash.new
+        record.store('hostname', @hostname) if @record_hostname
+        record.store('service', key)
+        if (@nest_result)
+          record.store(@nest_key, fetch(key))
+        else
+          record.merge!(fetch(key).to_hash)
+        end
+        Engine.emit(tag, Engine.now, record)
       end
+      disconnect
+      rescue => e
+      $log.warn "munin: fetch failed ",  :error_class=>e.class, :error=>e.message
     end
 
     def fetch(key)
-      @munin ||= get_connection
-      begin
-        values = @munin.fetch(key)
-        return convert_type(values[key]) if @convert_type
-        return values[key]
-      rescue Munin::ConnectionError
-        @munin = get_connection
-        retry
-      end
+      @munin = get_connection
+      values = @munin.fetch(key)
+      return convert_type(values[key]) if @convert_type
+      return values[key]
     end
 
     def convert_type(ary)
@@ -113,5 +116,12 @@ module Fluent
       end
       return data
     end
+    
+    def get_placeholder
+      return {
+        '__HOSTNAME__' => @hostname,
+        '${hostname}' => @hostname,
+      }
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-munin
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-20 00:00:00.000000000 Z
+date: 2013-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &15089900 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *15089900
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &15089460 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +37,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *15089460
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: munin-ruby
-  requirement: &15089040 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +53,12 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *15089040
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - y.ken.studio@gmail.com
@@ -55,6 +70,8 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- example.conf
+- example2.conf
 - fluent-plugin-munin.gemspec
 - lib/fluent/plugin/in_munin.rb
 - test/helper.rb
@@ -79,10 +96,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Fluentd Input plugin to fetch munin-node metrics data with custom intervals.
+  It supports all of munin plugins.
 test_files:
 - test/helper.rb
 - test/plugin/test_in_munin.rb