fluent-plugin-collectd-nest 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0430bec9b4a9a1efbfd7354c85b236be1a0d3c38
+  data.tar.gz: 80336a440baf3cc336c63dbe40bd5182354383f9
+SHA512:
+  metadata.gz: 762eefef377a4da04743d5638077573e0c5813ac2fc91e7354cd802551ab8e4f2d367bc4f642e1a90d00c986c4cb215457563d3afbab0a52d998194b3afabb81
+  data.tar.gz: 454dc13c0f0f8b4c4dd372c12acaaec198dd35dd64ba3ce16df545288227180797140132aff6f50541662b9735f832d2ade92b2ced867b499013941f4bda02db

data/ChangeLog

@@ -0,0 +1,3 @@
+Release 0.1.0 - 2014/10/10
+
+  * First release

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,54 @@
+# Output filter plugin to rewrite Collectd JSON output to nest into a nested json
+
+Rewrites the message coming from Collectd to store as a nested json. Can be used in Elasticsearch to display metrics.
+
+## Installation
+
+Use RubyGems:
+If you use td-agent
+
+    td-agent-gem install fluent-plugin-collectd-nest
+
+If you use fluentd
+
+    gem install fluent-plugin-collectd-nest
+
+## Configuration
+
+    <match pattern>
+      type collectd_nest
+    </match>
+
+If the following record is passed:
+
+```js
+[{"time" => 1000, "host" => 'host_v', "interval" => 5, "plugin" => 'plugin_v', "plugin_instance" => 'plugin_instance_v', "type" => 'type_v', "type_instance" => 'type_instance_v', "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']}]
+```
+
+then you get new record:
+
+```js
+[{"host" => "host_v", "collectd": {"time" => 1000, "interval" => 5,
+"plugin" => "plugin_v", "plugin_instace" => "plugin_instance_v",
+"type" => "type_v", "type_instance" => "type_instance_v", "dstypes" => "t1",
+"plugin_v" => {"type_v" => {"n1" => "v1", "n2" => "v2"}}}}]
+```
+
+Empty values in "plugin", "plugin_instance", "type" or "type_instance" will not be copied into the new tag name
+
+
+## WARNING
+
+* This plugin was written to deal with a specific use-case, might not be the best fit for everyone. If you need more configurability/features, create a PR
+
+
+## Copyright
+
+<table>
+  <tr>
+    <td>Author</td><td> Viaq <TBD></td>
+  </tr>
+  <tr>
+    <td>License</td><td>MIT License</td>
+  </tr>
+</table>

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/fluent-plugin-collectd-nest.gemspec

@@ -0,0 +1,22 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-collectd-nest"
+  gem.description = "Output filter plugin to rewrite Collectd JSON output to nested json"
+  gem.homepage    = "https://github.com/viaq/fluent-plugin-collectd-nest"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Anton Sherkhonov"]
+  gem.email       = "sherkhonov@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.license     = 'MIT'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.17"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/out_collectd_nest.rb

@@ -0,0 +1,60 @@
+module Fluent
+  class CollectdNestOutput < Output
+    Fluent::Plugin.register_output('collectd_nest', self)
+
+    config_param :remove_tag_prefix,  :string, :default => nil
+    config_param :add_tag_prefix,     :string, :default => nil
+
+
+    def emit(tag, es, chain)
+      tag = update_tag(tag)
+      es.each { |time, record|
+        Engine.emit(tag, time, normalize_record(record))
+      }
+
+      chain.next
+    end
+
+
+    def update_tag(tag)
+      if remove_tag_prefix
+        if remove_tag_prefix == tag
+          tag = ''
+        elsif tag.to_s.start_with?(remove_tag_prefix+'.')
+          tag = tag[remove_tag_prefix.length+1 .. -1]
+        end
+      end
+      if add_tag_prefix
+        tag = tag && tag.length > 0 ? "#{add_tag_prefix}.#{tag}" : add_tag_prefix
+      end
+      return tag
+    end
+
+    private
+
+    def normalize_record(record)
+      if record.nil?
+        return record
+      end
+      if !(record.has_key?('values')) || !(record.has_key?('dsnames')) || !(record.has_key?('dstypes')) || !(record.has_key?('host')) || !(record.has_key?('plugin')) || !(record.has_key?('type'))
+        return record
+      end
+      new_rec = {}
+      new_rec['hostname']= record['host']
+      rec_plugin = record['plugin']
+      rec_type = record['type']
+      record[rec_plugin] = {rec_type => {}}
+
+      record['values'].each_with_index { |value, index|
+        record[rec_plugin][rec_type][record['dsnames'][index]] = value
+      }
+      record['dstypes'] = record['dstypes'].uniq
+      record.delete('host')
+      record.delete('dsnames')
+      record.delete('values')
+      new_rec['collectd']= record
+      new_rec
+    end
+  end
+end
+

data/sample-zeus-config/td-agent.conf

@@ -0,0 +1,72 @@
+## match tag=debug.** and dump to console
+<match debug.**>
+  type stdout
+</match>
+
+####
+## Source descriptions:
+##
+
+## built-in TCP input
+## @see http://docs.fluentd.org/articles/in_forward
+<source>
+  type forward
+</source>
+
+
+# HTTP input
+# POST http://localhost:8888/<tag>?json=<json>
+# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
+# @see http://docs.fluentd.org/articles/in_http
+# receives collectd input
+<source>
+  type http
+  port 8888
+</source>
+
+## live debugging agent
+<source>
+  type debug_agent
+  bind 127.0.0.1
+  port 24230
+</source>
+<match collectd>
+  type collectd_nest
+  add_tag_prefix metrics_nested
+</match>
+
+#<match metrics_nested.**>
+#  type record_reformer
+#  tag logs.collectd.<USERNAME>-<TOKEN>
+#  <record>
+#    @timestamp ${time}
+#  </record>
+#</match>
+<match metrics_nested.**>
+  type stdout
+</match>
+
+<source>
+  type syslog
+  port 42185
+  tag  syslog
+</source>
+
+<match syslog.**>
+  type record_reformer
+  tag logs.${tag}.<USERNAME>-<TOKEN>
+  <record>
+    @timestamp ${time}
+  </record>
+</match>
+
+#<match logs.**>
+#  type secure_forward
+#  shared_key cisco_zeus_log_metric_pipline
+#  self_hostname fluentd-client1.ciscozeus.io
+#  secure false
+#  keepalive 10
+#  <server>
+#     host data01.ciscozeus.io
+#  </server>
+#</match>

data/test/out_collectd_nest.rb

@@ -0,0 +1,77 @@
+require 'fluent/test'
+require 'fluent/plugin/out_collectd_nest'
+
+
+class CollectdNestOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    type collectd_nest
+    tag foo.filtered
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::OutputTestDriver.new(Fluent::CollectdNestOutput, tag='test_tag').configure(conf)
+  end
+
+  def test_rewrite_tag
+    d = create_driver %[
+      type collectd_nest
+      add_tag_prefix test_tag
+    ]
+
+    d.run do
+      d.emit({
+        "time" => 1000, "host" => 'host', "interval" => 5,
+        "plugin" => 'plugin1', "plugin_instance" => 'plugin_instance',
+        "type" => 'type1', "type_instance" => 'type_instance',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      })
+      d.emit({
+        "time" => 1000, "host" => 'host', "interval" => 5,
+        "plugin" => 'plugin2', "plugin_instance" => '',
+        "type" => 'type2', "type_instance" => 'type_instance',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      })
+      d.emit({
+        "time" => 1000, "host" => 'host', "interval" => 5,
+        "plugin" => 'plugin3', "plugin_instance" => 'plugin_instance',
+        "type" => 'type3', "type_instance" => 'type_instance',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      })
+    end
+
+    assert_equal 3, d.emits.length
+    assert_equal "test_tag.plugin1.type1", d.emits[0][0]
+    assert_equal "test_tag.plugin2.type2", d.emits[1][0]
+    assert_equal "test_tag.plugin3.type3", d.emits[2][0]
+  end
+
+  def test_normalize_record
+    d = create_driver %[
+      type collectd_nest
+    ]
+
+    d.run do
+      d.emit({
+        "time" => 1000, "host" => 'host_v', "interval" => 5,
+        "plugin" => 'plugin_v', "plugin_instance" => 'plugin_instance_v',
+        "type" => 'type_v', "type_instance" => 'type_instance_v',
+        "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
+      })
+    end
+
+    assert_equal d.records[0][:collectd]['plugin_v']['type_v']['n1'], 'v1'
+    assert_equal d.records[0][:collectd]['plugin_v']['type_v']['n2'], 'v2'
+    assert_equal d.records[0][:collectd].has_key?('host'), false
+    assert_equal d.records[0][:collectd]['plugin'], 'plugin_v'
+    assert_equal d.records[0][:collectd]['type'], 'type_v'
+    assert_equal d.records[0][:collectd]['plugin_instance'], 'plugin_instance_v'
+    assert_equal d.records[0][:collectd]['type_instance'], 'type_instance_v'
+    assert_equal d.records[0][:collectd]['dstypes'], 't1'
+
+  end
+
+end

metadata

@@ -0,0 +1,81 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-collectd-nest
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Anton Sherkhonov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Output filter plugin to rewrite Collectd JSON output to nested json
+email: sherkhonov@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ChangeLog
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-collectd-nest.gemspec
+- lib/fluent/plugin/out_collectd_nest.rb
+- sample-zeus-config/td-agent.conf
+- test/out_collectd_nest.rb
+homepage: https://github.com/viaq/fluent-plugin-collectd-nest
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Output filter plugin to rewrite Collectd JSON output to nested json
+test_files:
+- test/out_collectd_nest.rb