fluent-plugin-collectd-unroll 0.1.0

data/ChangeLog

@@ -0,0 +1,3 @@
+Release 0.1.0 - 2014/10/10
+
+  * First release

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,52 @@
+# Output filter plugin to rewrite Collectd JSON output to unroll into a flat json
+
+Rewrites the message coming from Collectd to store as a flat json. Can be used in Elasticsearch to display metrics.
+
+## Installation
+
+Use RubyGems:
+
+    gem install fluent-plugin-collectd-unroll
+
+## Configuration
+
+    <match pattern>
+      type collectd_unroll
+    </match>
+
+If following record is passed:
+
+```js
+[{"time" => 1000, "host" => 'host_v', "interval" => 5, "plugin" => 'plugin_v', "plugin_instance" => 'plugin_instance_v', "type" => 'type_v', "type_instance" => 'type_instance_v', "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']}]
+```
+
+then you got new record like below:
+
+```js
+[{"n1"=>"v1", "n2"=>"v2"}]
+```
+
+and the record tag will be changed to
+
+```js
+host_v.plugin_v.plugin_instance_v.type_v.type_instance_v
+```
+
+Empty values in "plugin", "plugin_instance", "type" or "type_instance" will not be copied into the new tag name
+
+
+## WARNING
+
+* This plugin was written to deal with a specific use-case, might not be the best fit for everyone. If you need more configurability/features, create a PR
+
+
+## Copyright
+
+<table>
+  <tr>
+    <td>Author</td><td>Manoj Sharma <vigyanik@gmail.com></td>
+  </tr>
+  <tr>
+    <td>License</td><td>MIT License</td>
+  </tr>
+</table>

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/fluent-plugin-collectd-unroll.gemspec

@@ -0,0 +1,22 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-collectd-unroll"
+  gem.description = "Output filter plugin to rewrite Collectd JSON output to flat json"
+  gem.homepage    = "https://github.com/giannello/fluent-plugin-collectd-unroll"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Manoj Sharma"]
+  gem.email       = "vigyanik@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.license     = 'MIT'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.17"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/out_collectd_unroll.rb

@@ -0,0 +1,57 @@
+module Fluent
+  class CollectdUnrollOutput < Output
+    Fluent::Plugin.register_output('collectd_unroll', self)
+
+    config_param :remove_tag_prefix,  :string, :default => nil
+    config_param :add_tag_prefix,     :string, :default => nil
+
+
+    def emit(tag, es, chain)
+      tag = update_tag(tag)
+      es.each { |time, record|
+        Engine.emit(tag, time, normalize_record(record))
+      }
+
+      chain.next
+    end
+
+
+    def update_tag(tag)
+      if remove_tag_prefix
+        if remove_tag_prefix == tag
+          tag = ''
+        elsif tag.to_s.start_with?(remove_tag_prefix+'.')
+          tag = tag[remove_tag_prefix.length+1 .. -1]
+        end
+      end
+      if add_tag_prefix
+        tag = tag && tag.length > 0 ? "#{add_tag_prefix}.#{tag}" : add_tag_prefix
+      end
+      return tag
+    end
+
+    private
+
+    def normalize_record(record)
+      if record.nil?
+        return record
+      end
+      if !(record.has_key?('values')) || !(record.has_key?('dsnames')) || !(record.has_key?('dstypes')) || !(record.has_key?('host')) || !(record.has_key?('plugin')) || !(record.has_key?('plugin_instance')) || !(record.has_key?('type')) || !(record.has_key?('type_instance'))
+        return record
+      end
+      
+      record['values'].each_with_index { |value, index|
+        @tags = [record['host'].gsub(".","/"), record['plugin'], record['plugin_instance'], record['type'], record['type_instance'], record['dsnames'][index]]
+        tag = @tags.join(".").squeeze(".").gsub(/\.$/, '')
+        record[tag] = value
+        record[record['dsnames'][index]] = value
+        record['dstype_' + record['dsnames'][index]] = record['dstypes'][index]
+        record['dstype'] = record['dstypes'][index]
+      }
+      record.delete('dstypes')
+      record.delete('dsnames')
+      record.delete('values')
+      record
+    end
+  end
+end

data/sample-zeus-config/td-agent.conf

@@ -0,0 +1,69 @@
+## match tag=debug.** and dump to console
+<match debug.**>
+  type stdout
+</match>
+
+####
+## Source descriptions:
+##
+
+## built-in TCP input
+## @see http://docs.fluentd.org/articles/in_forward
+<source>
+  type forward
+</source>
+
+
+# HTTP input
+# POST http://localhost:8888/<tag>?json=<json>
+# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
+# @see http://docs.fluentd.org/articles/in_http
+# receives collectd input
+<source>
+  type http
+  port 8888
+</source>
+
+## live debugging agent
+<source>
+  type debug_agent
+  bind 127.0.0.1
+  port 24230
+</source>
+<match collectd>
+  type collectd_unroll
+  add_tag_prefix metrics_unrolled
+</match>
+
+<match metrics_unrolled.**>
+  type record_reformer
+  tag logs.collectd.zeushack085-2f586ba2
+  <record>
+    @timestamp ${time}
+  </record>
+</match>
+
+<source>
+  type syslog
+  port 42185
+  tag  syslog
+</source>
+
+<match syslog.**>
+  type record_reformer
+  tag logs.${tag}.zeushack085-2f586ba2
+  <record>
+    @timestamp ${time}
+  </record>
+</match>
+
+<match logs.**>
+  type secure_forward
+  shared_key cisco_zeus_log_metric_pipline
+  self_hostname fluentd-client1.ciscozeus.io
+  secure false
+  keepalive 10
+  <server>
+     host data01.ciscozeus.io
+  </server>
+</match>

data/test/out_collectd_unroll.rb

@@ -0,0 +1,100 @@
+require 'fluent/test'
+require 'fluent/plugin/out_collectd_unroll'
+
+
+class CollectdUnrollOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    type collectd_unroll
+    tag foo.filtered
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::OutputTestDriver.new(Fluent::CollectdUnrollOutput, tag='test_tag').configure(conf)
+  end
+
+  def test_rewrite_tag
+    d = create_driver %[
+      type collectd_unroll
+    ]
+
+    d.run do
+      d.emit([{
+        "time" => 1000, "host" => 'host', "interval" => 5,
+        "plugin" => 'plugin', "plugin_instance" => 'plugin_instance',
+        "type" => 'type', "type_instance" => 'type_instance',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      }])
+      d.emit([{
+        "time" => 1000, "host" => 'host', "interval" => 5,
+        "plugin" => 'plugin', "plugin_instance" => '',
+        "type" => 'type', "type_instance" => 'type_instance',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      }])
+      d.emit([{
+        "time" => 1000, "host" => 'host', "interval" => 5,
+        "plugin" => 'plugin', "plugin_instance" => 'plugin_instance',
+        "type" => '', "type_instance" => 'type_instance',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      }])
+      d.emit([{
+        "time" => 1000, "host" => 'host', "interval" => 5,
+        "plugin" => 'plugin', "plugin_instance" => 'plugin_instance',
+        "type" => 'type', "type_instance" => '',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      }])
+    end
+
+    assert_equal 4, d.emits.length
+    assert_equal "test_tag.host.plugin.plugin_instance.type.type_instance", d.emits[0][0]
+    assert_equal "test_tag.host.plugin.type.type_instance", d.emits[1][0]
+    assert_equal "test_tag.host.plugin.plugin_instance.type_instance", d.emits[2][0]
+    assert_equal "test_tag.host.plugin.plugin_instance.type", d.emits[3][0]
+  end
+
+  def test_use_timestamp
+    d = create_driver %[
+      type collectd_unroll
+    ]
+    
+    d.run do
+      d.emit([{
+        "time" => 1000, "host" => 'host', "interval" => 5,
+        "plugin" => 'plugin', "plugin_instance" => 'plugin_instance',
+        "type" => 'type', "type_instance" => 'type_instance',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      }])
+      d.emit([{
+        "time" => 9999, "host" => 'host', "interval" => 5,
+        "plugin" => 'plugin', "plugin_instance" => '',
+        "type" => 'type', "type_instance" => 'type_instance',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      }])
+    end
+    
+    assert_equal 2, d.emits.length
+    assert_equal 1000, d.emits[0][1]
+    assert_equal 9999, d.emits[1][1]
+  end
+
+  def test_normalize_record
+    d = create_driver %[
+      type collectd_unroll
+    ]
+
+    d.run do
+      d.emit([{
+        "time" => 1000, "host" => 'v', "interval" => 5,
+        "plugin" => 'v', "plugin_instance" => 'v',
+        "type" => 'v', "type_instance" => 'v',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      }])
+    end
+
+    assert_equal ({'n1'=>'v1','n2'=>'v2'} - d.records[0]).empty?, true
+  end
+
+end

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-collectd-unroll
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Manoj Sharma
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Output filter plugin to rewrite Collectd JSON output to flat json
+email: vigyanik@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ChangeLog
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-collectd-unroll.gemspec
+- lib/fluent/plugin/out_collectd_unroll.rb
+- sample-zeus-config/td-agent.conf
+- test/out_collectd_unroll.rb
+homepage: https://github.com/giannello/fluent-plugin-collectd-unroll
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Output filter plugin to rewrite Collectd JSON output to flat json
+test_files: []