fluent-plugin-prometheus-thread 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,22 @@
1
+ log_format ltsv 'time:$time_iso8601\t'
2
+ 'remote_addr:$remote_addr\t'
3
+ 'request_method:$request_method\t'
4
+ 'request_length:$request_length\t'
5
+ 'request_uri:$request_uri\t'
6
+ 'uri:$uri\t'
7
+ 'status:$status\t'
8
+ 'bytes_sent:$bytes_sent\t'
9
+ 'body_bytes_sent:$body_bytes_sent\t'
10
+ 'referer:$http_referer\t'
11
+ 'useragent:$http_user_agent\t'
12
+ 'request_time:$request_time\t'
13
+ 'upstream_response_time:$upstream_response_time';
14
+
15
+ server {
16
+ access_log /var/log/nginx/access_proxy.log ltsv;
17
+ listen 9999;
18
+ location / {
19
+ proxy_pass https://www.google.com;
20
+ }
21
+ }
22
+
@@ -0,0 +1,10 @@
1
+ # A job to scrape an endpoint of Fluentd running on localhost.
2
+ job: {
3
+ name: "fluentd"
4
+ scrape_interval: "5s"
5
+
6
+ target_group: {
7
+ target: "http://localhost:24231/metrics"
8
+ }
9
+ }
10
+
@@ -0,0 +1,8 @@
1
+ # A job to scrape an endpoint of Fluentd running on localhost.
2
+ scrape_configs:
3
+ - job_name: fluentd
4
+ scrape_interval: 5s
5
+ target_groups:
6
+ - targets:
7
+ - 'localhost:24231'
8
+ metrics_path: /metrics
@@ -0,0 +1,34 @@
1
+ require 'spec_helper'
2
+ require 'fluent/plugin/filter_prometheus'
3
+ require_relative 'shared'
4
+
5
+ describe Fluent::PrometheusFilter do
6
+ let(:tag) { 'prometheus.test' }
7
+ let(:driver) { Fluent::Test::FilterTestDriver.new(Fluent::PrometheusFilter, tag).configure(config, true) }
8
+ let(:registry) { ::Prometheus::Client.registry }
9
+
10
+ describe '#configure' do
11
+ it_behaves_like 'output configuration'
12
+ end
13
+
14
+ describe '#run' do
15
+ let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100} }
16
+ let(:es) { driver.run { driver.emit(message, Time.now) }.filtered }
17
+
18
+ context 'simple config' do
19
+ include_context 'simple_config'
20
+
21
+ it 'adds a new counter metric' do
22
+ expect(registry.metrics.map(&:name)).not_to include(name)
23
+ es
24
+ expect(registry.metrics.map(&:name)).to include(name)
25
+ end
26
+
27
+ it 'should keep original message' do
28
+ expect(es.first[1]).to eq(message)
29
+ end
30
+ end
31
+
32
+ it_behaves_like 'instruments record'
33
+ end
34
+ end if defined?(Fluent::PrometheusFilter)
@@ -0,0 +1,30 @@
1
+ require 'spec_helper'
2
+ require 'fluent/plugin/out_prometheus'
3
+ require_relative 'shared'
4
+
5
+ describe Fluent::PrometheusOutput do
6
+ let(:tag) { 'prometheus.test' }
7
+ let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::PrometheusOutput, tag).configure(config) }
8
+ let(:registry) { ::Prometheus::Client.registry }
9
+
10
+ describe '#configure' do
11
+ it_behaves_like 'output configuration'
12
+ end
13
+
14
+ describe '#run' do
15
+ let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100} }
16
+ let(:es) { driver.run { driver.emit(message, Time.now) } }
17
+
18
+ context 'simple config' do
19
+ include_context 'simple_config'
20
+
21
+ it 'adds a new counter metric' do
22
+ expect(registry.metrics.map(&:name)).not_to include(name)
23
+ es
24
+ expect(registry.metrics.map(&:name)).to include(name)
25
+ end
26
+ end
27
+
28
+ it_behaves_like 'instruments record'
29
+ end
30
+ end
@@ -0,0 +1,76 @@
1
+ require 'spec_helper'
2
+ require 'fluent/plugin/in_prometheus'
3
+
4
+ require 'net/http'
5
+
6
+ describe Fluent::PrometheusInput do
7
+ CONFIG = %[
8
+ type prometheus
9
+ ]
10
+
11
+ LOCAL_CONFIG = %[
12
+ type prometheus
13
+ bind 127.0.0.1
14
+ ]
15
+
16
+ let(:config) { CONFIG }
17
+ let(:port) { 24231 }
18
+ let(:driver) { Fluent::Test::InputTestDriver.new(Fluent::PrometheusInput).configure(config) }
19
+
20
+ describe '#configure' do
21
+ describe 'bind' do
22
+ let(:config) { CONFIG + %[
23
+ bind 127.0.0.1
24
+ ] }
25
+ it 'should be configurable' do
26
+ expect(driver.instance.bind).to eq('127.0.0.1')
27
+ end
28
+ end
29
+
30
+ describe 'port' do
31
+ let(:config) { CONFIG + %[
32
+ port 8888
33
+ ] }
34
+ it 'should be configurable' do
35
+ expect(driver.instance.port).to eq(8888)
36
+ end
37
+ end
38
+
39
+ describe 'metrics_path' do
40
+ let(:config) { CONFIG + %[
41
+ metrics_path /_test
42
+ ] }
43
+ it 'should be configurable' do
44
+ expect(driver.instance.metrics_path).to eq('/_test')
45
+ end
46
+ end
47
+ end
48
+
49
+ describe '#run' do
50
+ context '/metrics' do
51
+ let(:config) { LOCAL_CONFIG }
52
+ it 'returns 200' do
53
+ driver.run do
54
+ Net::HTTP.start("127.0.0.1", port) do |http|
55
+ req = Net::HTTP::Get.new("/metrics")
56
+ res = http.request(req)
57
+ expect(res.code).to eq('200')
58
+ end
59
+ end
60
+ end
61
+ end
62
+
63
+ context '/foo' do
64
+ let(:config) { LOCAL_CONFIG }
65
+ it 'does not return 200' do
66
+ driver.run do
67
+ Net::HTTP.start("127.0.0.1", port) do |http|
68
+ req = Net::HTTP::Get.new("/foo")
69
+ res = http.request(req)
70
+ expect(res.code).not_to eq('200')
71
+ end
72
+ end
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,210 @@
1
+
2
+ BASE_CONFIG = %[
3
+ type prometheus
4
+ ]
5
+
6
+
7
+ SIMPLE_CONFIG = BASE_CONFIG + %[
8
+ type prometheus
9
+ <metric>
10
+ name simple_foo
11
+ type counter
12
+ desc Something foo.
13
+ key foo
14
+ </metric>
15
+ ]
16
+
17
+ FULL_CONFIG = BASE_CONFIG + %[
18
+ <metric>
19
+ name full_foo
20
+ type counter
21
+ desc Something foo.
22
+ key foo
23
+ <labels>
24
+ key foo1
25
+ </labels>
26
+ </metric>
27
+ <metric>
28
+ name full_bar
29
+ type gauge
30
+ desc Something bar.
31
+ key bar
32
+ <labels>
33
+ key foo2
34
+ </labels>
35
+ </metric>
36
+ <metric>
37
+ name full_baz
38
+ type summary
39
+ desc Something baz.
40
+ key baz
41
+ <labels>
42
+ key foo3
43
+ </labels>
44
+ </metric>
45
+ <labels>
46
+ test_key test_value
47
+ </labels>
48
+ ]
49
+
50
+ PLACEHOLDER_CONFIG = BASE_CONFIG + %[
51
+ <metric>
52
+ name placeholder_foo
53
+ type counter
54
+ desc Something foo.
55
+ key foo
56
+ <labels>
57
+ foo ${foo}
58
+ </labels>
59
+ </metric>
60
+ <labels>
61
+ tag ${tag}
62
+ hostname ${hostname}
63
+ </labels>
64
+ ]
65
+
66
+ COUNTER_WITHOUT_KEY_CONFIG = BASE_CONFIG + %[
67
+ <metric>
68
+ name without_key_foo
69
+ type counter
70
+ desc Something foo.
71
+ </metric>
72
+ ]
73
+
74
+ shared_context 'simple_config' do
75
+ let(:orig_name) { 'simple_foo' }
76
+ let(:config) { SIMPLE_CONFIG.gsub(orig_name, name.to_s) }
77
+ let(:name) { "#{orig_name}_#{Time.now.to_f}".to_sym }
78
+ let(:counter) { registry.get(name) }
79
+ end
80
+
81
+ shared_context 'full_config' do
82
+ let(:config) { FULL_CONFIG }
83
+ let(:counter) { registry.get(:full_foo) }
84
+ let(:gauge) { registry.get(:full_bar) }
85
+ let(:summary) { registry.get(:full_baz) }
86
+ end
87
+
88
+ shared_context 'placeholder_config' do
89
+ let(:orig_name) { 'placeholder_foo' }
90
+ let(:config) { PLACEHOLDER_CONFIG.gsub(orig_name, name.to_s) }
91
+ let(:name) { "#{orig_name}_#{Time.now.to_f}".to_sym }
92
+ let(:counter) { registry.get(name) }
93
+ end
94
+
95
+ shared_context 'counter_without_key_config' do
96
+ let(:orig_name) { 'without_key_foo' }
97
+ let(:config) { COUNTER_WITHOUT_KEY_CONFIG.gsub(orig_name, name.to_s) }
98
+ let(:name) { "#{orig_name}_#{Time.now.to_f}".to_sym }
99
+ let(:counter) { registry.get(name) }
100
+ end
101
+
102
+ shared_examples_for 'output configuration' do
103
+ context 'base config' do
104
+ let(:config) { BASE_CONFIG }
105
+ it 'does not raise error' do
106
+ expect{driver}.not_to raise_error
107
+ end
108
+ end
109
+
110
+ describe 'configure simple configuration' do
111
+ include_context 'simple_config'
112
+ it { expect{driver}.not_to raise_error }
113
+ end
114
+
115
+ describe 'configure full configuration' do
116
+ include_context 'full_config'
117
+ it { expect{driver}.not_to raise_error }
118
+ end
119
+
120
+ describe 'configure placeholder configuration' do
121
+ include_context 'placeholder_config'
122
+ it { expect{driver}.not_to raise_error }
123
+ end
124
+
125
+ describe 'configure counter without key configuration' do
126
+ include_context 'counter_without_key_config'
127
+ it { expect{driver}.not_to raise_error }
128
+ end
129
+
130
+ context 'unknown type' do
131
+ let(:config) { BASE_CONFIG + %[
132
+ <metric>
133
+ type foo
134
+ </metric>
135
+ ] }
136
+ it 'raises ConfigError' do
137
+ expect{driver}.to raise_error Fluent::ConfigError
138
+ end
139
+ end
140
+ end
141
+
142
+ shared_examples_for 'instruments record' do
143
+ context 'full config' do
144
+ include_context 'full_config'
145
+
146
+ before :each do
147
+ es
148
+ end
149
+
150
+ it 'adds all metrics' do
151
+ expect(registry.metrics.map(&:name)).to include(:full_foo)
152
+ expect(registry.metrics.map(&:name)).to include(:full_bar)
153
+ expect(registry.metrics.map(&:name)).to include(:full_baz)
154
+ expect(counter).to be_kind_of(::Prometheus::Client::Metric)
155
+ expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
156
+ expect(summary).to be_kind_of(::Prometheus::Client::Metric)
157
+ end
158
+
159
+ it 'instruments counter metric' do
160
+ expect(counter.type).to eq(:counter)
161
+ expect(counter.get({test_key: 'test_value', key: 'foo1'})).to be_kind_of(Integer)
162
+ end
163
+
164
+ it 'instruments gauge metric' do
165
+ expect(gauge.type).to eq(:gauge)
166
+ expect(gauge.get({test_key: 'test_value', key: 'foo2'})).to eq(100)
167
+ end
168
+
169
+ it 'instruments summary metric' do
170
+ expect(summary.type).to eq(:summary)
171
+ expect(summary.get({test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
172
+ expect(summary.get({test_key: 'test_value', key: 'foo3'})[0.99]).to eq(100)
173
+ end
174
+ end
175
+
176
+ context 'placeholder config' do
177
+ include_context 'placeholder_config'
178
+
179
+ before :each do
180
+ es
181
+ end
182
+
183
+ it 'expands placeholders with record values' do
184
+ expect(registry.metrics.map(&:name)).to include(name)
185
+ expect(counter).to be_kind_of(::Prometheus::Client::Metric)
186
+ key, _ = counter.values.find {|k,v| v == 100 }
187
+ expect(key).to be_kind_of(Hash)
188
+ expect(key[:tag]).to eq(tag)
189
+ expect(key[:hostname]).to be_kind_of(String)
190
+ expect(key[:hostname]).not_to eq("${hostname}")
191
+ expect(key[:hostname]).not_to be_empty
192
+ expect(key[:foo]).to eq("100")
193
+ end
194
+ end
195
+
196
+ context 'counter_without config' do
197
+ include_context 'counter_without_key_config'
198
+
199
+ before :each do
200
+ es
201
+ end
202
+
203
+ it 'just increments by 1' do
204
+ expect(registry.metrics.map(&:name)).to include(name)
205
+ expect(counter).to be_kind_of(::Prometheus::Client::Metric)
206
+ _, value = counter.values.find {|k,v| k == {} }
207
+ expect(value).to eq(1)
208
+ end
209
+ end
210
+ end
@@ -0,0 +1,8 @@
1
+ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
2
+ require 'fluent/test'
3
+ require 'fluent/plugin/prometheus'
4
+
5
+ # Disable Test::Unit
6
+ Test::Unit::AutoRunner.need_auto_run = false
7
+
8
+ Fluent::Test.setup
metadata ADDED
@@ -0,0 +1,159 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-prometheus-thread
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.3
5
+ platform: ruby
6
+ authors:
7
+ - Masahiro Sano
8
+ - James Harlow
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2016-07-18 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: fluentd
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - '>='
19
+ - !ruby/object:Gem::Version
20
+ version: '0'
21
+ type: :runtime
22
+ prerelease: false
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - '>='
26
+ - !ruby/object:Gem::Version
27
+ version: '0'
28
+ - !ruby/object:Gem::Dependency
29
+ name: prometheus-client
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - '>='
33
+ - !ruby/object:Gem::Version
34
+ version: '0'
35
+ type: :runtime
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - '>='
40
+ - !ruby/object:Gem::Version
41
+ version: '0'
42
+ - !ruby/object:Gem::Dependency
43
+ name: bundler
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - '>='
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
49
+ type: :development
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - '>='
54
+ - !ruby/object:Gem::Version
55
+ version: '0'
56
+ - !ruby/object:Gem::Dependency
57
+ name: rake
58
+ requirement: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - '>='
61
+ - !ruby/object:Gem::Version
62
+ version: '0'
63
+ type: :development
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - '>='
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ - !ruby/object:Gem::Dependency
71
+ name: rspec
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - '>='
75
+ - !ruby/object:Gem::Version
76
+ version: '0'
77
+ type: :development
78
+ prerelease: false
79
+ version_requirements: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - '>='
82
+ - !ruby/object:Gem::Version
83
+ version: '0'
84
+ - !ruby/object:Gem::Dependency
85
+ name: test-unit
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - '>='
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ type: :development
92
+ prerelease: false
93
+ version_requirements: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - '>='
96
+ - !ruby/object:Gem::Version
97
+ version: '0'
98
+ description: A td-agent plugin that collects metrics and exposes for Prometheus.
99
+ email:
100
+ - sabottenda@gmail.com
101
+ - j+ruby-gems@thread.com
102
+ executables: []
103
+ extensions: []
104
+ extra_rdoc_files: []
105
+ files:
106
+ - .gitignore
107
+ - .rspec
108
+ - .travis.yml
109
+ - Gemfile
110
+ - Gemfile.fluentd.0.10
111
+ - Gemfile.fluentd.0.12
112
+ - LICENSE.txt
113
+ - README.md
114
+ - Rakefile
115
+ - fluent-plugin-prometheus.gemspec
116
+ - lib/fluent/plugin/filter_prometheus.rb
117
+ - lib/fluent/plugin/in_prometheus.rb
118
+ - lib/fluent/plugin/in_prometheus_monitor.rb
119
+ - lib/fluent/plugin/out_prometheus.rb
120
+ - lib/fluent/plugin/prometheus.rb
121
+ - misc/fluentd_sample.conf
122
+ - misc/nginx_proxy.conf
123
+ - misc/prometheus-old-format.conf
124
+ - misc/prometheus.yaml
125
+ - spec/fluent/plugin/filter_prometheus_spec.rb
126
+ - spec/fluent/plugin/out_prometheus_spec.rb
127
+ - spec/fluent/plugin/prometheus_spec.rb
128
+ - spec/fluent/plugin/shared.rb
129
+ - spec/spec_helper.rb
130
+ homepage: https://github.com/hythloday/fluent-plugin-prometheus
131
+ licenses:
132
+ - MIT
133
+ metadata: {}
134
+ post_install_message:
135
+ rdoc_options: []
136
+ require_paths:
137
+ - lib
138
+ required_ruby_version: !ruby/object:Gem::Requirement
139
+ requirements:
140
+ - - '>='
141
+ - !ruby/object:Gem::Version
142
+ version: '0'
143
+ required_rubygems_version: !ruby/object:Gem::Requirement
144
+ requirements:
145
+ - - '>='
146
+ - !ruby/object:Gem::Version
147
+ version: '0'
148
+ requirements: []
149
+ rubyforge_project:
150
+ rubygems_version: 2.0.14.1
151
+ signing_key:
152
+ specification_version: 4
153
+ summary: A td-agent plugin that collects metrics and exposes for Prometheus.
154
+ test_files:
155
+ - spec/fluent/plugin/filter_prometheus_spec.rb
156
+ - spec/fluent/plugin/out_prometheus_spec.rb
157
+ - spec/fluent/plugin/prometheus_spec.rb
158
+ - spec/fluent/plugin/shared.rb
159
+ - spec/spec_helper.rb