fluent-plugin-collectd-nest 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 0430bec9b4a9a1efbfd7354c85b236be1a0d3c38
4
+ data.tar.gz: 80336a440baf3cc336c63dbe40bd5182354383f9
5
+ SHA512:
6
+ metadata.gz: 762eefef377a4da04743d5638077573e0c5813ac2fc91e7354cd802551ab8e4f2d367bc4f642e1a90d00c986c4cb215457563d3afbab0a52d998194b3afabb81
7
+ data.tar.gz: 454dc13c0f0f8b4c4dd372c12acaaec198dd35dd64ba3ce16df545288227180797140132aff6f50541662b9735f832d2ade92b2ced867b499013941f4bda02db
@@ -0,0 +1,3 @@
1
+ Release 0.1.0 - 2014/10/10
2
+
3
+ * First release
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source "http://rubygems.org"
2
+
3
+ gemspec
@@ -0,0 +1,54 @@
1
+ # Output filter plugin to rewrite Collectd JSON output to nest into a nested json
2
+
3
+ Rewrites the message coming from Collectd to store as a nested json. Can be used in Elasticsearch to display metrics.
4
+
5
+ ## Installation
6
+
7
+ Use RubyGems:
8
+ If you use td-agent
9
+
10
+ td-agent-gem install fluent-plugin-collectd-nest
11
+
12
+ If you use fluentd
13
+
14
+ gem install fluent-plugin-collectd-nest
15
+
16
+ ## Configuration
17
+
18
+ <match pattern>
19
+ type collectd_nest
20
+ </match>
21
+
22
+ If the following record is passed:
23
+
24
+ ```js
25
+ [{"time" => 1000, "host" => 'host_v', "interval" => 5, "plugin" => 'plugin_v', "plugin_instance" => 'plugin_instance_v', "type" => 'type_v', "type_instance" => 'type_instance_v', "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']}]
26
+ ```
27
+
28
+ then you get new record:
29
+
30
+ ```js
31
+ [{"host" => "host_v", "collectd": {"time" => 1000, "interval" => 5,
32
+ "plugin" => "plugin_v", "plugin_instace" => "plugin_instance_v",
33
+ "type" => "type_v", "type_instance" => "type_instance_v", "dstypes" => "t1",
34
+ "plugin_v" => {"type_v" => {"n1" => "v1", "n2" => "v2"}}}}]
35
+ ```
36
+
37
+ Empty values in "plugin", "plugin_instance", "type" or "type_instance" will not be copied into the new tag name
38
+
39
+
40
+ ## WARNING
41
+
42
+ * This plugin was written to deal with a specific use-case, might not be the best fit for everyone. If you need more configurability/features, create a PR
43
+
44
+
45
+ ## Copyright
46
+
47
+ <table>
48
+ <tr>
49
+ <td>Author</td><td> Viaq <TBD></td>
50
+ </tr>
51
+ <tr>
52
+ <td>License</td><td>MIT License</td>
53
+ </tr>
54
+ </table>
@@ -0,0 +1,12 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ require 'rake/testtask'
5
+
6
+ Rake::TestTask.new(:test) do |test|
7
+ test.libs << 'lib' << 'test'
8
+ test.test_files = FileList['test/*.rb']
9
+ test.verbose = true
10
+ end
11
+
12
+ task :default => [:build]
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.1
@@ -0,0 +1,22 @@
1
+ # encoding: utf-8
2
+ $:.push File.expand_path('../lib', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = "fluent-plugin-collectd-nest"
6
+ gem.description = "Output filter plugin to rewrite Collectd JSON output to nested json"
7
+ gem.homepage = "https://github.com/viaq/fluent-plugin-collectd-nest"
8
+ gem.summary = gem.description
9
+ gem.version = File.read("VERSION").strip
10
+ gem.authors = ["Anton Sherkhonov"]
11
+ gem.email = "sherkhonov@gmail.com"
12
+ gem.has_rdoc = false
13
+ #gem.platform = Gem::Platform::RUBY
14
+ gem.license = 'MIT'
15
+ gem.files = `git ls-files`.split("\n")
16
+ gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ gem.require_paths = ['lib']
19
+
20
+ gem.add_dependency "fluentd", "~> 0.10.17"
21
+ gem.add_development_dependency "rake", ">= 0.9.2"
22
+ end
@@ -0,0 +1,60 @@
1
+ module Fluent
2
+ class CollectdNestOutput < Output
3
+ Fluent::Plugin.register_output('collectd_nest', self)
4
+
5
+ config_param :remove_tag_prefix, :string, :default => nil
6
+ config_param :add_tag_prefix, :string, :default => nil
7
+
8
+
9
+ def emit(tag, es, chain)
10
+ tag = update_tag(tag)
11
+ es.each { |time, record|
12
+ Engine.emit(tag, time, normalize_record(record))
13
+ }
14
+
15
+ chain.next
16
+ end
17
+
18
+
19
+ def update_tag(tag)
20
+ if remove_tag_prefix
21
+ if remove_tag_prefix == tag
22
+ tag = ''
23
+ elsif tag.to_s.start_with?(remove_tag_prefix+'.')
24
+ tag = tag[remove_tag_prefix.length+1 .. -1]
25
+ end
26
+ end
27
+ if add_tag_prefix
28
+ tag = tag && tag.length > 0 ? "#{add_tag_prefix}.#{tag}" : add_tag_prefix
29
+ end
30
+ return tag
31
+ end
32
+
33
+ private
34
+
35
+ def normalize_record(record)
36
+ if record.nil?
37
+ return record
38
+ end
39
+ if !(record.has_key?('values')) || !(record.has_key?('dsnames')) || !(record.has_key?('dstypes')) || !(record.has_key?('host')) || !(record.has_key?('plugin')) || !(record.has_key?('type'))
40
+ return record
41
+ end
42
+ new_rec = {}
43
+ new_rec['hostname']= record['host']
44
+ rec_plugin = record['plugin']
45
+ rec_type = record['type']
46
+ record[rec_plugin] = {rec_type => {}}
47
+
48
+ record['values'].each_with_index { |value, index|
49
+ record[rec_plugin][rec_type][record['dsnames'][index]] = value
50
+ }
51
+ record['dstypes'] = record['dstypes'].uniq
52
+ record.delete('host')
53
+ record.delete('dsnames')
54
+ record.delete('values')
55
+ new_rec['collectd']= record
56
+ new_rec
57
+ end
58
+ end
59
+ end
60
+
@@ -0,0 +1,72 @@
1
+ ## match tag=debug.** and dump to console
2
+ <match debug.**>
3
+ type stdout
4
+ </match>
5
+
6
+ ####
7
+ ## Source descriptions:
8
+ ##
9
+
10
+ ## built-in TCP input
11
+ ## @see http://docs.fluentd.org/articles/in_forward
12
+ <source>
13
+ type forward
14
+ </source>
15
+
16
+
17
+ # HTTP input
18
+ # POST http://localhost:8888/<tag>?json=<json>
19
+ # POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"}
20
+ # @see http://docs.fluentd.org/articles/in_http
21
+ # receives collectd input
22
+ <source>
23
+ type http
24
+ port 8888
25
+ </source>
26
+
27
+ ## live debugging agent
28
+ <source>
29
+ type debug_agent
30
+ bind 127.0.0.1
31
+ port 24230
32
+ </source>
33
+ <match collectd>
34
+ type collectd_nest
35
+ add_tag_prefix metrics_nested
36
+ </match>
37
+
38
+ #<match metrics_nested.**>
39
+ # type record_reformer
40
+ # tag logs.collectd.<USERNAME>-<TOKEN>
41
+ # <record>
42
+ # @timestamp ${time}
43
+ # </record>
44
+ #</match>
45
+ <match metrics_nested.**>
46
+ type stdout
47
+ </match>
48
+
49
+ <source>
50
+ type syslog
51
+ port 42185
52
+ tag syslog
53
+ </source>
54
+
55
+ <match syslog.**>
56
+ type record_reformer
57
+ tag logs.${tag}.<USERNAME>-<TOKEN>
58
+ <record>
59
+ @timestamp ${time}
60
+ </record>
61
+ </match>
62
+
63
+ #<match logs.**>
64
+ # type secure_forward
65
+ # shared_key cisco_zeus_log_metric_pipline
66
+ # self_hostname fluentd-client1.ciscozeus.io
67
+ # secure false
68
+ # keepalive 10
69
+ # <server>
70
+ # host data01.ciscozeus.io
71
+ # </server>
72
+ #</match>
@@ -0,0 +1,77 @@
1
+ require 'fluent/test'
2
+ require 'fluent/plugin/out_collectd_nest'
3
+
4
+
5
+ class CollectdNestOutputTest < Test::Unit::TestCase
6
+ def setup
7
+ Fluent::Test.setup
8
+ end
9
+
10
+ CONFIG = %[
11
+ type collectd_nest
12
+ tag foo.filtered
13
+ ]
14
+
15
+ def create_driver(conf = CONFIG)
16
+ Fluent::Test::OutputTestDriver.new(Fluent::CollectdNestOutput, tag='test_tag').configure(conf)
17
+ end
18
+
19
+ def test_rewrite_tag
20
+ d = create_driver %[
21
+ type collectd_nest
22
+ add_tag_prefix test_tag
23
+ ]
24
+
25
+ d.run do
26
+ d.emit({
27
+ "time" => 1000, "host" => 'host', "interval" => 5,
28
+ "plugin" => 'plugin1', "plugin_instance" => 'plugin_instance',
29
+ "type" => 'type1', "type_instance" => 'type_instance',
30
+ "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
31
+ })
32
+ d.emit({
33
+ "time" => 1000, "host" => 'host', "interval" => 5,
34
+ "plugin" => 'plugin2', "plugin_instance" => '',
35
+ "type" => 'type2', "type_instance" => 'type_instance',
36
+ "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
37
+ })
38
+ d.emit({
39
+ "time" => 1000, "host" => 'host', "interval" => 5,
40
+ "plugin" => 'plugin3', "plugin_instance" => 'plugin_instance',
41
+ "type" => 'type3', "type_instance" => 'type_instance',
42
+ "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
43
+ })
44
+ end
45
+
46
+ assert_equal 3, d.emits.length
47
+ assert_equal "test_tag.plugin1.type1", d.emits[0][0]
48
+ assert_equal "test_tag.plugin2.type2", d.emits[1][0]
49
+ assert_equal "test_tag.plugin3.type3", d.emits[2][0]
50
+ end
51
+
52
+ def test_normalize_record
53
+ d = create_driver %[
54
+ type collectd_nest
55
+ ]
56
+
57
+ d.run do
58
+ d.emit({
59
+ "time" => 1000, "host" => 'host_v', "interval" => 5,
60
+ "plugin" => 'plugin_v', "plugin_instance" => 'plugin_instance_v',
61
+ "type" => 'type_v', "type_instance" => 'type_instance_v',
62
+ "values" => ['v1', 'v2'], "dsnames" => ['n1', 'n2'], "dstypes" => ['t1', 't2']
63
+ })
64
+ end
65
+
66
+ assert_equal d.records[0][:collectd]['plugin_v']['type_v']['n1'], 'v1'
67
+ assert_equal d.records[0][:collectd]['plugin_v']['type_v']['n2'], 'v2'
68
+ assert_equal d.records[0][:collectd].has_key?('host'), false
69
+ assert_equal d.records[0][:collectd]['plugin'], 'plugin_v'
70
+ assert_equal d.records[0][:collectd]['type'], 'type_v'
71
+ assert_equal d.records[0][:collectd]['plugin_instance'], 'plugin_instance_v'
72
+ assert_equal d.records[0][:collectd]['type_instance'], 'type_instance_v'
73
+ assert_equal d.records[0][:collectd]['dstypes'], 't1'
74
+
75
+ end
76
+
77
+ end
metadata ADDED
@@ -0,0 +1,81 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-collectd-nest
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Anton Sherkhonov
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-02-10 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 0.10.17
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 0.10.17
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 0.9.2
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 0.9.2
41
+ description: Output filter plugin to rewrite Collectd JSON output to nested json
42
+ email: sherkhonov@gmail.com
43
+ executables: []
44
+ extensions: []
45
+ extra_rdoc_files: []
46
+ files:
47
+ - ChangeLog
48
+ - Gemfile
49
+ - README.md
50
+ - Rakefile
51
+ - VERSION
52
+ - fluent-plugin-collectd-nest.gemspec
53
+ - lib/fluent/plugin/out_collectd_nest.rb
54
+ - sample-zeus-config/td-agent.conf
55
+ - test/out_collectd_nest.rb
56
+ homepage: https://github.com/viaq/fluent-plugin-collectd-nest
57
+ licenses:
58
+ - MIT
59
+ metadata: {}
60
+ post_install_message:
61
+ rdoc_options: []
62
+ require_paths:
63
+ - lib
64
+ required_ruby_version: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ required_rubygems_version: !ruby/object:Gem::Requirement
70
+ requirements:
71
+ - - ">="
72
+ - !ruby/object:Gem::Version
73
+ version: '0'
74
+ requirements: []
75
+ rubyforge_project:
76
+ rubygems_version: 2.5.2
77
+ signing_key:
78
+ specification_version: 4
79
+ summary: Output filter plugin to rewrite Collectd JSON output to nested json
80
+ test_files:
81
+ - test/out_collectd_nest.rb