fluent-plugin-dogstatsd-mediba 0.0.7

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a297b61bd0812d8e8131266bea4dffa56beb19ac
4
+ data.tar.gz: 324fccae8ed2f25f452b19e04bb9c6a193f929f6
5
+ SHA512:
6
+ metadata.gz: 8951f7726ebfa04b3a5d0267b8ace4359d49f386553a9dee7dfbbda304a4f27a5b6ad750fae93ffd327f145d31be995556b37f43adf435ddb0431db58a53f0b1
7
+ data.tar.gz: 8ddd66288081dcb0382fb58327c1b697d8a07e662bdb8a8e34f95efe5d42952960dd5cd28d07a1db8d1a56bc238b7506ceae2efa2a4d56daac3245bc735f40cc
data/.gitignore ADDED
@@ -0,0 +1,22 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ *.bundle
19
+ *.so
20
+ *.o
21
+ *.a
22
+ mkmf.log
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-dogstatsd.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Ryota Arai
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,129 @@
1
+ # fluent-plugin-dogstatsd
2
+
3
+ Fluend plugin for Dogstatsd, that is statsd server for Datadog.
4
+
5
+ ## Installation
6
+
7
+ $ gem install fluent-plugin-dogstatsd
8
+
9
+ ## Usage
10
+
11
+ ```
12
+ $ echo '{"type": "increment", "key": "apache.requests", "tags": {"url": "/"}}' | fluent-cat dogstatsd.hello
13
+ $ echo '{"type": "histogram", "key": "apache.response_time", "value": 10.5, "tags": {"url": "/hello"}}' | fluent-cat dogstatsd.hello
14
+ $ echo '{"type": "event", "title": "Deploy", "text": "New revision"}' | fluent-cat dogstatsd.hello
15
+ ```
16
+
17
+ Supported types are `increment`, `decrement`, `count`, `gauge`, `histogram`, `timing`, `set` and `event`.
18
+
19
+ ## Configuration
20
+
21
+ ```
22
+ <match dogstatsd.*>
23
+ type dogstatsd
24
+
25
+ # Dogstatsd host
26
+ host localhost
27
+
28
+ # Dogstatsd port
29
+ port 8125
30
+
31
+ # Use tag of fluentd record as key sent to Dogstatsd
32
+ use_tag_as_key false
33
+
34
+ # (Treat fields in a record as tags)
35
+ # flat_tags true
36
+
37
+ # (Metric type in Datadog.)
38
+ # metric_type increment
39
+
40
+ # Default: "value"
41
+ # value_key Value
42
+ </match>
43
+ ```
44
+
45
+ ## Example
46
+
47
+ ### Count log lines
48
+
49
+ ```apache
50
+ <source>
51
+ type tail
52
+ path /tmp/sample.log
53
+ tag datadog.increment.sample
54
+ format ...
55
+ </source>
56
+
57
+ <match datadog.increment.*>
58
+ type dogstatsd
59
+ metric_type increment
60
+ flat_tags true
61
+ use_tag_as_key true
62
+ </match>
63
+ ```
64
+
65
+ ### Histogram
66
+
67
+ ```apache
68
+ <source>
69
+ type tail
70
+ path /tmp/sample.log
71
+ tag datadog.histogram.sample
72
+ format /^(?<value>[^ ]*) (?<host>[^ ]*)$/
73
+ </source>
74
+
75
+ <match datadog.histogram.*>
76
+ type dogstatsd
77
+ metric_type histogram
78
+ flat_tags true
79
+ use_tag_as_key true
80
+ </match>
81
+ ```
82
+
83
+ ### Event
84
+
85
+ ```apache
86
+ <source>
87
+ type tail
88
+ path /var/log/nginx/error.log
89
+ pos_file /var/log/td-agent/error.log.pos
90
+ format multiline
91
+ format_firstline /^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} \[\w+\] (?<pid>\d+).(?<tid>\d+): /
92
+ format1 /^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<log_level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)/
93
+ multiline_flush_interval 3s
94
+ tag datadog.nginx.errorlog
95
+ </source>
96
+
97
+ <match datadog.nginx.errorlog>
98
+ type dogstatsd
99
+ metric_type event
100
+ use_tag_as_title true
101
+ text_key message
102
+ </match>
103
+ ```
104
+
105
+ ### MySQL threads
106
+
107
+ ```apache
108
+ <source>
109
+ type mysql_query
110
+ tag datadog.histogram.mysql_threads
111
+ query SHOW VARIABLES LIKE 'Thread_%'
112
+ </source>
113
+
114
+ <match datadog.histogram.mysql_threads>
115
+ type dogstatsd
116
+ metric_type histogram
117
+ value_key Value
118
+ flat_tags true
119
+ use_tag_as_key true
120
+ </match>
121
+ ```
122
+
123
+ ## Contributing
124
+
125
+ 1. Fork it ( https://github.com/ryotarai/fluent-plugin-dogstatsd/fork )
126
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
127
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
128
+ 4. Push to the branch (`git push origin my-new-feature`)
129
+ 5. Create a new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ require 'rake/testtask'
4
+
5
+ Rake::TestTask.new(:test) do |test|
6
+ test.libs << 'test'
7
+ test.test_files = FileList['test/plugin/*.rb']
8
+ end
9
+
@@ -0,0 +1,26 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'fluent/plugin/dogstatsd/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "fluent-plugin-dogstatsd-mediba"
8
+ spec.version = Fluent::Plugin::Dogstatsd::VERSION
9
+ spec.authors = ["Hiroki Yoshida"]
10
+ spec.email = ["h-yoshida@mediba.jp"]
11
+ spec.summary = %q{Fluent plugin for Dogstatsd, that is statsd server for Datadog.}
12
+ spec.homepage = "https://github.com/yoshida-mediba/fluent-plugin-dogstatsd"
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_dependency "fluentd"
21
+ spec.add_dependency "dogstatsd-ruby", "~> 1.4.1"
22
+
23
+ spec.add_development_dependency "bundler", "~> 1.6"
24
+ spec.add_development_dependency "rake"
25
+ spec.add_development_dependency "test-unit"
26
+ end
@@ -0,0 +1,9 @@
1
+ require "fluent/plugin/dogstatsd/version"
2
+
3
+ module Fluent
4
+ module Plugin
5
+ module Dogstatsd
6
+ # Your code goes here...
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,7 @@
1
+ module Fluent
2
+ module Plugin
3
+ module Dogstatsd
4
+ VERSION = "0.0.7"
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,108 @@
1
+ module Fluent
2
+ class DogstatsdOutput < BufferedOutput
3
+ Plugin.register_output('dogstatsd', self)
4
+
5
+ config_param :host, :string, :default => nil
6
+ config_param :port, :integer, :default => nil
7
+ config_param :use_tag_as_key, :bool, :default => false
8
+ config_param :use_tag_as_key_if_missing, :bool, :default => false
9
+ config_param :flat_tags, :bool, :default => false
10
+ config_param :flat_tag, :bool, :default => false # obsolete
11
+ config_param :metric_type, :string, :default => nil
12
+ config_param :value_key, :string, :default => nil
13
+ config_param :text_key, :string, :default => nil
14
+ config_param :sample_rate, :float, :default => nil
15
+
16
+ unless method_defined?(:log)
17
+ define_method(:log) { $log }
18
+ end
19
+
20
+ attr_accessor :statsd
21
+
22
+ def initialize
23
+ super
24
+
25
+ require 'statsd' # dogstatsd-ruby
26
+ end
27
+
28
+ def start
29
+ super
30
+
31
+ host = @host || Statsd::DEFAULT_HOST
32
+ port = @port || Statsd::DEFAULT_PORT
33
+
34
+ @statsd ||= Statsd.new(host, port)
35
+ end
36
+
37
+ def format(tag, time, record)
38
+ [tag, time, record].to_msgpack
39
+ end
40
+
41
+ def write(chunk)
42
+ @statsd.batch do |s|
43
+ chunk.msgpack_each do |tag, time, record|
44
+ key = if @use_tag_as_key
45
+ tag
46
+ else
47
+ record.delete('key')
48
+ end
49
+
50
+ if !key && @use_tag_as_key_if_missing
51
+ key = tag
52
+ end
53
+
54
+ unless key
55
+ log.warn "'key' is not specified. skip this record:", tag: tag
56
+ next
57
+ end
58
+
59
+ value = record.delete(@value_key || 'value')
60
+
61
+ options = {}
62
+ text = record.delete(@text_key || 'text')
63
+ type = @metric_type || record.delete('type')
64
+ sample_rate = @sample_rate || record.delete('sample_rate')
65
+
66
+ if sample_rate
67
+ options[:sample_rate] = sample_rate
68
+ end
69
+
70
+ tags = if @flat_tags || @flat_tag
71
+ record
72
+ else
73
+ record['tags']
74
+ end
75
+ if tags
76
+ options[:tags] = tags.map do |k, v|
77
+ "#{k}:#{v}"
78
+ end
79
+ end
80
+
81
+ case type
82
+ when 'increment'
83
+ s.increment(key, options)
84
+ when 'decrement'
85
+ s.decrement(key, options)
86
+ when 'count'
87
+ s.count(key, value, options)
88
+ when 'gauge'
89
+ s.gauge(key, value, options)
90
+ when 'histogram'
91
+ s.histogram(key, value, options)
92
+ when 'timing'
93
+ s.timing(key, value, options)
94
+ when 'set'
95
+ s.set(key, value, options)
96
+ when 'event'
97
+ options[:alert_type] = record['alert_type']
98
+ s.event(key, text, options)
99
+ when nil
100
+ log.warn "type is not provided (You can provide type via `metric_type` in config or `type` field in a record."
101
+ else
102
+ log.warn "Type '#{type}' is unknown."
103
+ end
104
+ end
105
+ end
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,171 @@
1
+ require 'test_helper'
2
+
3
+ class DummyStatsd
4
+ attr_reader :messages
5
+
6
+ def initialize
7
+ @messages = []
8
+ end
9
+
10
+ def batch
11
+ yield(self)
12
+ end
13
+
14
+ %i!increment decrement count gauge histogram timing set event!.each do |name|
15
+ define_method(name) do |*args|
16
+ @messages << [name, args].flatten
17
+ end
18
+ end
19
+ end
20
+
21
+ class DogstatsdOutputTest < Test::Unit::TestCase
22
+ def setup
23
+ Fluent::Test.setup
24
+ require 'fluent/plugin/out_dogstatsd'
25
+ end
26
+
27
+ def teardown
28
+ end
29
+
30
+ def test_configure
31
+ d = create_driver(<<-EOC)
32
+ type dogstatsd
33
+ host HOST
34
+ port 12345
35
+ EOC
36
+
37
+ assert_equal('HOST', d.instance.host)
38
+ assert_equal(12345, d.instance.port)
39
+ end
40
+
41
+ def test_write
42
+ d = create_driver
43
+
44
+ d.emit({'type' => 'increment', 'key' => 'hello.world1'}, Time.now.to_i)
45
+ d.emit({'type' => 'increment', 'key' => 'hello.world2'}, Time.now.to_i)
46
+ d.emit({'type' => 'decrement', 'key' => 'hello.world'}, Time.now.to_i)
47
+ d.emit({'type' => 'count', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
48
+ d.emit({'type' => 'gauge', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
49
+ d.emit({'type' => 'histogram', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
50
+ d.emit({'type' => 'timing', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
51
+ d.emit({'type' => 'set', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
52
+ d.emit({'type' => 'event', 'title' => 'Deploy', 'text' => 'Revision', 'key' => 'hello.world'}, Time.now.to_i)
53
+ d.run
54
+
55
+ assert_equal(d.instance.statsd.messages, [
56
+ [:increment, 'hello.world1', {}],
57
+ [:increment, 'hello.world2', {}],
58
+ [:decrement, 'hello.world', {}],
59
+ [:count, 'hello.world', 10, {}],
60
+ [:gauge, 'hello.world', 10, {}],
61
+ [:histogram, 'hello.world', 10, {}],
62
+ [:timing, 'hello.world', 10, {}],
63
+ [:set, 'hello.world', 10, {}],
64
+ [:event, 'Deploy', 'Revision', {}],
65
+ ])
66
+ end
67
+
68
+ def test_flat_tag
69
+ d = create_driver(<<-EOC)
70
+ #{default_config}
71
+ flat_tag true
72
+ EOC
73
+
74
+ d.emit({'type' => 'increment', 'key' => 'hello.world', 'tagKey' => 'tagValue'}, Time.now.to_i)
75
+ d.run
76
+
77
+ assert_equal(d.instance.statsd.messages, [
78
+ [:increment, 'hello.world', {tags: ["tagKey:tagValue"]}],
79
+ ])
80
+ end
81
+
82
+ def test_metric_type
83
+ d = create_driver(<<-EOC)
84
+ #{default_config}
85
+ metric_type decrement
86
+ EOC
87
+
88
+ d.emit({'key' => 'hello.world', 'tags' => {'tagKey' => 'tagValue'}}, Time.now.to_i)
89
+ d.run
90
+
91
+ assert_equal(d.instance.statsd.messages, [
92
+ [:decrement, 'hello.world', {tags: ["tagKey:tagValue"]}],
93
+ ])
94
+ end
95
+
96
+ def test_use_tag_as_key
97
+ d = create_driver(<<-EOC)
98
+ #{default_config}
99
+ use_tag_as_key true
100
+ EOC
101
+
102
+ d.emit({'type' => 'increment'}, Time.now.to_i)
103
+ d.run
104
+
105
+ assert_equal(d.instance.statsd.messages, [
106
+ [:increment, 'dogstatsd.tag', {}],
107
+ ])
108
+ end
109
+
110
+ def test_use_tag_as_key_fallback
111
+ d = create_driver(<<-EOC)
112
+ #{default_config}
113
+ use_tag_as_key_if_missing true
114
+ EOC
115
+
116
+ d.emit({'type' => 'increment'}, Time.now.to_i)
117
+ d.run
118
+
119
+ assert_equal(d.instance.statsd.messages, [
120
+ [:increment, 'dogstatsd.tag', {}],
121
+ ])
122
+ end
123
+
124
+ def test_tags
125
+ d = create_driver
126
+ d.emit({'type' => 'increment', 'key' => 'hello.world', 'tags' => {'key' => 'value'}}, Time.now.to_i)
127
+ d.run
128
+
129
+ assert_equal(d.instance.statsd.messages, [
130
+ [:increment, 'hello.world', {tags: ["key:value"]}],
131
+ ])
132
+ end
133
+
134
+ def test_sample_rate_config
135
+ d = create_driver(<<-EOC)
136
+ #{default_config}
137
+ sample_rate .5
138
+ EOC
139
+
140
+ d.emit({'type' => 'increment', 'key' => 'tag'}, Time.now.to_i)
141
+ d.run
142
+
143
+ assert_equal(d.instance.statsd.messages, [
144
+ [:increment, 'tag', {sample_rate: 0.5}],
145
+ ])
146
+ end
147
+
148
+ def test_sample_rate
149
+ d = create_driver
150
+ d.emit({'type' => 'increment', 'sample_rate' => 0.5, 'key' => 'tag'}, Time.now.to_i)
151
+ d.run
152
+
153
+ assert_equal(d.instance.statsd.messages, [
154
+ [:increment, 'tag', {sample_rate: 0.5}],
155
+ ])
156
+ end
157
+
158
+ private
159
+ def default_config
160
+ <<-EOC
161
+ type dogstatsd
162
+ EOC
163
+ end
164
+
165
+ def create_driver(conf = default_config)
166
+ Fluent::Test::BufferedOutputTestDriver.new(Fluent::DogstatsdOutput, 'dogstatsd.tag').configure(conf).tap do |d|
167
+ d.instance.statsd = DummyStatsd.new
168
+ end
169
+ end
170
+ end
171
+
@@ -0,0 +1,3 @@
1
+ require 'test/unit'
2
+ require 'fluent/test'
3
+
metadata ADDED
@@ -0,0 +1,127 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-dogstatsd-mediba
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.7
5
+ platform: ruby
6
+ authors:
7
+ - Hiroki Yoshida
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-11-24 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: dogstatsd-ruby
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: 1.4.1
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 1.4.1
41
+ - !ruby/object:Gem::Dependency
42
+ name: bundler
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.6'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.6'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: test-unit
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ description:
84
+ email:
85
+ - h-yoshida@mediba.jp
86
+ executables: []
87
+ extensions: []
88
+ extra_rdoc_files: []
89
+ files:
90
+ - ".gitignore"
91
+ - Gemfile
92
+ - LICENSE.txt
93
+ - README.md
94
+ - Rakefile
95
+ - fluent-plugin-dogstatsd-mediba.gemspec
96
+ - lib/fluent/plugin/dogstatsd.rb
97
+ - lib/fluent/plugin/dogstatsd/version.rb
98
+ - lib/fluent/plugin/out_dogstatsd.rb
99
+ - test/plugin/test_out_dogstatsd.rb
100
+ - test/test_helper.rb
101
+ homepage: https://github.com/yoshida-mediba/fluent-plugin-dogstatsd
102
+ licenses:
103
+ - MIT
104
+ metadata: {}
105
+ post_install_message:
106
+ rdoc_options: []
107
+ require_paths:
108
+ - lib
109
+ required_ruby_version: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ - - ">="
112
+ - !ruby/object:Gem::Version
113
+ version: '0'
114
+ required_rubygems_version: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - ">="
117
+ - !ruby/object:Gem::Version
118
+ version: '0'
119
+ requirements: []
120
+ rubyforge_project:
121
+ rubygems_version: 2.5.2
122
+ signing_key:
123
+ specification_version: 4
124
+ summary: Fluent plugin for Dogstatsd, that is statsd server for Datadog.
125
+ test_files:
126
+ - test/plugin/test_out_dogstatsd.rb
127
+ - test/test_helper.rb