fluent-plugin-prometheus-test 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,77 @@
1
+ module Fluent::Plugin
2
+
3
+ ##
4
+ # PromMetricsAggregator aggregates multiples metrics exposed using Prometheus text-based format
5
+ # see https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
6
+
7
+
8
+ class PrometheusMetrics
9
+ def initialize
10
+ @comments = []
11
+ @metrics = []
12
+ end
13
+
14
+ def to_string
15
+ (@comments + @metrics).join("\n")
16
+ end
17
+
18
+ def add_comment(comment)
19
+ @comments << comment
20
+ end
21
+
22
+ def add_metric_value(value)
23
+ @metrics << value
24
+ end
25
+
26
+ attr_writer :comments, :metrics
27
+ end
28
+
29
+ class PromMetricsAggregator
30
+ def initialize
31
+ @metrics = {}
32
+ end
33
+
34
+ def get_metric_name_from_comment(line)
35
+ tokens = line.split(' ')
36
+ if ['HELP', 'TYPE'].include?(tokens[1])
37
+ tokens[2]
38
+ else
39
+ ''
40
+ end
41
+ end
42
+
43
+ def add_metrics(metrics)
44
+ current_metric = ''
45
+ new_metric = false
46
+ lines = metrics.split("\n")
47
+ for line in lines
48
+ if line[0] == '#'
49
+ # Metric comment (# TYPE, # HELP)
50
+ parsed_metric = get_metric_name_from_comment(line)
51
+ if parsed_metric != ''
52
+ if parsed_metric != current_metric
53
+ # Starting a new metric comment block
54
+ new_metric = !@metrics.key?(parsed_metric)
55
+ if new_metric
56
+ @metrics[parsed_metric] = PrometheusMetrics.new()
57
+ end
58
+ current_metric = parsed_metric
59
+ end
60
+
61
+ if new_metric && parsed_metric == current_metric
62
+ # New metric, inject comments (# TYPE, # HELP)
63
+ @metrics[parsed_metric].add_comment(line)
64
+ end
65
+ end
66
+ else
67
+ # Metric value, simply append line
68
+ @metrics[current_metric].add_metric_value(line)
69
+ end
70
+ end
71
+ end
72
+
73
+ def get_metrics
74
+ @metrics.map{|k,v| v.to_string()}.join("\n") + (@metrics.length ? "\n" : "")
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,170 @@
1
+ ## Prometheus Input Plugin Configuration
2
+
3
+ # input plugin that exports metrics
4
+ <source>
5
+ @type prometheus
6
+ </source>
7
+
8
+ <source>
9
+ @type monitor_agent
10
+ </source>
11
+
12
+ <source>
13
+ @type forward
14
+ </source>
15
+
16
+ # input plugin that collects metrics from MonitorAgent
17
+ <source>
18
+ @type prometheus_monitor
19
+ <labels>
20
+ host ${hostname}
21
+ </labels>
22
+ </source>
23
+
24
+ # input plugin that collects metrics for output plugin
25
+ <source>
26
+ @type prometheus_output_monitor
27
+ <labels>
28
+ host ${hostname}
29
+ </labels>
30
+ </source>
31
+
32
+ # input plugin that collects metrics for in_tail plugin
33
+ <source>
34
+ @type prometheus_tail_monitor
35
+ <labels>
36
+ host ${hostname}
37
+ </labels>
38
+ </source>
39
+
40
+ ## Nginx Access Log Configuration
41
+
42
+ <source>
43
+ @type tail
44
+ format nginx
45
+ tag nginx
46
+ path /var/log/nginx/access.log
47
+ pos_file /tmp/fluent_nginx.pos
48
+ types size:integer
49
+ </source>
50
+
51
+ <filter nginx>
52
+ @type prometheus
53
+
54
+ # You can use counter type with specifying a key,
55
+ # and increments counter by the value
56
+ <metric>
57
+ name nginx_size_counter_bytes
58
+ type counter
59
+ desc nginx bytes sent
60
+ key size
61
+ <labels>
62
+ host ${hostname}
63
+ foo bar
64
+ </labels>
65
+ </metric>
66
+
67
+ # You can use counter type without specifying a key
68
+ # This just increments counter by 1
69
+ <metric>
70
+ name nginx_record_counts
71
+ type counter
72
+ desc the number of emited records
73
+ <labels>
74
+ host ${hostname}
75
+ </labels>
76
+ </metric>
77
+ </filter>
78
+
79
+ <match nginx>
80
+ @type copy
81
+ # for MonitorAgent sample
82
+ <store>
83
+ @id test_forward
84
+ @type forward
85
+ buffer_type memory
86
+ flush_interval 1s
87
+ max_retry_wait 2s
88
+ <buffer>
89
+ # max_retry_wait 10s
90
+ flush_interval 1s
91
+ # retry_type periodic
92
+ disable_retry_limit
93
+ </buffer>
94
+ # retry_limit 3
95
+ disable_retry_limit
96
+ <server>
97
+ host 127.0.0.1
98
+ port 20000
99
+ </server>
100
+ </store>
101
+ <store>
102
+ @type stdout
103
+ </store>
104
+ </match>
105
+
106
+ ## Nginx Proxy Log Configuration
107
+
108
+ <source>
109
+ @type tail
110
+ format ltsv
111
+ tag nginx_proxy
112
+ path /var/log/nginx/access_proxy.log
113
+ pos_file /tmp/fluent_nginx_proxy.pos
114
+ types size:integer,request_length:integer,bytes_sent:integer,body_bytes_sent:integer,request_time:float,upstream_response_time:float
115
+ </source>
116
+
117
+ <filter nginx_proxy>
118
+ @type prometheus
119
+
120
+ # common labels for all metrics
121
+ <labels>
122
+ host ${hostname}
123
+ method ${request_method}
124
+ status ${status}
125
+ </labels>
126
+
127
+ <metric>
128
+ name nginx_proxy_request_length_total_bytes
129
+ type counter
130
+ desc nginx proxy request length bytes
131
+ key request_length
132
+ </metric>
133
+ <metric>
134
+ name nginx_proxy_bytes_sent_total_bytes
135
+ type counter
136
+ desc nginx proxy bytes sent
137
+ key bytes_sent
138
+ </metric>
139
+ <metric>
140
+ name nginx_proxy_request_duration_total_milliseconds
141
+ type counter
142
+ desc nginx proxy request time
143
+ key request_time
144
+ </metric>
145
+ <metric>
146
+ name nginx_proxy_upstream_response_duration_total_milliseconds
147
+ type counter
148
+ desc nginx proxy upstream response time
149
+ key upstream_response_time
150
+ </metric>
151
+ <metric>
152
+ name nginx_proxy_request_duration_milliseconds
153
+ type summary
154
+ desc nginx proxy request duration summary
155
+ key request_time
156
+ </metric>
157
+ <metric>
158
+ name nginx_proxy_upstream_duration_milliseconds
159
+ type summary
160
+ desc nginx proxy upstream response duration summary
161
+ key upstream_response_time
162
+ </metric>
163
+ </filter>
164
+
165
+ <match nginx_proxy>
166
+ @type copy
167
+ <store>
168
+ @type stdout
169
+ </store>
170
+ </match>
@@ -0,0 +1,22 @@
1
+ log_format ltsv 'time:$time_iso8601\t'
2
+ 'remote_addr:$remote_addr\t'
3
+ 'request_method:$request_method\t'
4
+ 'request_length:$request_length\t'
5
+ 'request_uri:$request_uri\t'
6
+ 'uri:$uri\t'
7
+ 'status:$status\t'
8
+ 'bytes_sent:$bytes_sent\t'
9
+ 'body_bytes_sent:$body_bytes_sent\t'
10
+ 'referer:$http_referer\t'
11
+ 'useragent:$http_user_agent\t'
12
+ 'request_time:$request_time\t'
13
+ 'upstream_response_time:$upstream_response_time';
14
+
15
+ server {
16
+ access_log /var/log/nginx/access_proxy.log ltsv;
17
+ listen 9999;
18
+ location / {
19
+ proxy_pass https://www.google.com;
20
+ }
21
+ }
22
+
@@ -0,0 +1,13 @@
1
+ # A job to scrape an endpoint of Fluentd running on localhost.
2
+ scrape_configs:
3
+ - job_name: 'prometheus'
4
+ scrape_interval: 5s
5
+ static_configs:
6
+ - targets:
7
+ - 'localhost:9090'
8
+ - job_name: fluentd
9
+ scrape_interval: 5s
10
+ static_configs:
11
+ - targets:
12
+ - 'localhost:24231'
13
+ metrics_path: /metrics
@@ -0,0 +1,59 @@
1
+ ALERT FluentdNodeDown
2
+ IF up{job="fluentd"} == 0
3
+ FOR 10m
4
+ LABELS {
5
+ service = "fluentd",
6
+ severity = "warning"
7
+ }
8
+ ANNOTATIONS {
9
+ summary = "fluentd cannot be scraped",
10
+ description = "Prometheus could not scrape {{ $labels.job }} for more than 10 minutes",
11
+ }
12
+
13
+ ALERT FluentdNodeDown
14
+ IF up{job="fluentd"} == 0
15
+ FOR 30m
16
+ LABELS {
17
+ service = "fluentd",
18
+ severity = "critical"
19
+ }
20
+ ANNOTATIONS {
21
+ summary = "fluentd cannot be scraped",
22
+ description = "Prometheus could not scrape {{ $labels.job }} for more than 30 minutes",
23
+ }
24
+
25
+ ALERT FluentdQueueLength
26
+ IF rate(fluentd_status_buffer_queue_length[5m]) > 0.3
27
+ FOR 1m
28
+ LABELS {
29
+ service = "fluentd",
30
+ severity = "warning"
31
+ }
32
+ ANNOTATIONS {
33
+ summary = "fluentd node are failing",
34
+ description = "In the last 5 minutes, fluentd queues increased 30%. Current value is {{ $value }} ",
35
+ }
36
+
37
+ ALERT FluentdQueueLength
38
+ IF rate(fluentd_status_buffer_queue_length[5m]) > 0.5
39
+ FOR 1m
40
+ LABELS {
41
+ service = "fluentd",
42
+ severity = "critical"
43
+ }
44
+ ANNOTATIONS {
45
+ summary = "fluentd node are critical",
46
+ description = "In the last 5 minutes, fluentd queues increased 50%. Current value is {{ $value }} ",
47
+ }
48
+
49
+ ALERT FluentdRecordsCountsHigh
50
+ IF sum(rate(fluentd_output_status_emit_records{job="fluentd"}[5m])) BY (instance) > (3 * sum(rate(fluentd_output_status_emit_records{job="fluentd"}[15m])) BY (instance))
51
+ FOR 1m
52
+ LABELS {
53
+ service = "fluentd",
54
+ severity = "critical"
55
+ }
56
+ ANNOTATIONS {
57
+ summary = "fluentd records count are critical",
58
+ description = "In the last 5m, records counts increased 3 times, comparing to the latest 15 min.",
59
+ }
@@ -0,0 +1,48 @@
1
+ require 'spec_helper'
2
+ require 'fluent/test/driver/filter'
3
+ require 'fluent/plugin/filter_prometheus'
4
+ require_relative 'shared'
5
+
6
+ describe Fluent::Plugin::PrometheusFilter do
7
+ let(:tag) { 'prometheus.test' }
8
+ let(:driver) { Fluent::Test::Driver::Filter.new(Fluent::Plugin::PrometheusFilter).configure(config) }
9
+ let(:registry) { ::Prometheus::Client::Registry.new }
10
+
11
+ before do
12
+ allow(Prometheus::Client).to receive(:registry).and_return(registry)
13
+ end
14
+
15
+ describe '#configure' do
16
+ it_behaves_like 'output configuration'
17
+ end
18
+
19
+ describe '#run' do
20
+ let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
21
+
22
+ context 'simple config' do
23
+ let(:config) {
24
+ BASE_CONFIG + %(
25
+ <metric>
26
+ name simple
27
+ type counter
28
+ desc Something foo.
29
+ key foo
30
+ </metric>
31
+ )
32
+ }
33
+
34
+ it 'adds a new counter metric' do
35
+ expect(registry.metrics.map(&:name)).not_to eq([:simple])
36
+ driver.run(default_tag: tag) { driver.feed(event_time, message) }
37
+ expect(registry.metrics.map(&:name)).to eq([:simple])
38
+ end
39
+
40
+ it 'should keep original message' do
41
+ driver.run(default_tag: tag) { driver.feed(event_time, message) }
42
+ expect(driver.filtered_records.first).to eq(message)
43
+ end
44
+ end
45
+
46
+ it_behaves_like 'instruments record'
47
+ end
48
+ end
@@ -0,0 +1,42 @@
1
+ require 'spec_helper'
2
+ require 'fluent/plugin/in_prometheus_monitor'
3
+ require 'fluent/test/driver/input'
4
+
5
+ describe Fluent::Plugin::PrometheusMonitorInput do
6
+ MONITOR_CONFIG = %[
7
+ @type prometheus_monitor
8
+ <labels>
9
+ host ${hostname}
10
+ foo bar
11
+ </labels>
12
+ ]
13
+
14
+ INVALID_MONITOR_CONFIG = %[
15
+ @type prometheus_monitor
16
+
17
+ <labels>
18
+ host ${hostname}
19
+ foo bar
20
+ invalid_use1 $.foo.bar
21
+ invalid_use2 $[0][1]
22
+ </labels>
23
+ ]
24
+
25
+ let(:config) { MONITOR_CONFIG }
26
+ let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusMonitorInput).configure(config) }
27
+
28
+ describe '#configure' do
29
+ describe 'valid' do
30
+ it 'does not raise error' do
31
+ expect{driver}.not_to raise_error
32
+ end
33
+ end
34
+
35
+ describe 'invalid' do
36
+ let(:config) { INVALID_MONITOR_CONFIG }
37
+ it 'expect raise error' do
38
+ expect{driver}.to raise_error
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,225 @@
1
+ require 'spec_helper'
2
+ require 'fluent/plugin/in_prometheus'
3
+ require 'fluent/test/driver/input'
4
+
5
+ require 'net/http'
6
+
7
+ describe Fluent::Plugin::PrometheusInput do
8
+ CONFIG = %[
9
+ @type prometheus
10
+ ]
11
+
12
+ LOCAL_CONFIG = %[
13
+ @type prometheus
14
+ bind 127.0.0.1
15
+ ]
16
+
17
+ let(:config) { CONFIG }
18
+ let(:port) { 24231 }
19
+ let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusInput).configure(config) }
20
+
21
+ describe '#configure' do
22
+ describe 'bind' do
23
+ let(:config) { CONFIG + %[
24
+ bind 127.0.0.1
25
+ ] }
26
+ it 'should be configurable' do
27
+ expect(driver.instance.bind).to eq('127.0.0.1')
28
+ end
29
+ end
30
+
31
+ describe 'port' do
32
+ let(:config) { CONFIG + %[
33
+ port 8888
34
+ ] }
35
+ it 'should be configurable' do
36
+ expect(driver.instance.port).to eq(8888)
37
+ end
38
+ end
39
+
40
+ describe 'metrics_path' do
41
+ let(:config) { CONFIG + %[
42
+ metrics_path /_test
43
+ ] }
44
+ it 'should be configurable' do
45
+ expect(driver.instance.metrics_path).to eq('/_test')
46
+ end
47
+ end
48
+ end
49
+
50
+ describe '#start' do
51
+ context 'with transport section' do
52
+ let(:config) do
53
+ %[
54
+ @type prometheus
55
+ bind 127.0.0.1
56
+ <transport tls>
57
+ insecure true
58
+ </transport>
59
+ ]
60
+ end
61
+
62
+ it 'returns 200' do
63
+ driver.run(timeout: 1) do
64
+ Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
65
+ req = Net::HTTP::Get.new('/metrics')
66
+ res = http.request(req)
67
+ expect(res.code).to eq('200')
68
+ end
69
+ end
70
+ end
71
+ end
72
+
73
+ context 'old parameters are given' do
74
+ context 'when extra_conf is used' do
75
+ let(:config) do
76
+ %[
77
+ @type prometheus
78
+ bind 127.0.0.1
79
+ <ssl>
80
+ enable true
81
+ extra_conf { "SSLCertName": [["CN", "nobody"], ["DC", "example"]] }
82
+ </ssl>
83
+ ]
84
+ end
85
+
86
+ it 'uses webrick' do
87
+ expect(driver.instance).to receive(:start_webrick).once
88
+ driver.run(timeout: 1)
89
+ end
90
+
91
+ it 'returns 200' do
92
+ driver.run(timeout: 1) do
93
+ Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
94
+ req = Net::HTTP::Get.new('/metrics')
95
+ res = http.request(req)
96
+ expect(res.code).to eq('200')
97
+ end
98
+ end
99
+ end
100
+ end
101
+
102
+ context 'cert_path and private_key_path combination' do
103
+ let(:config) do
104
+ %[
105
+ @type prometheus
106
+ bind 127.0.0.1
107
+ <ssl>
108
+ enable true
109
+ certificate_path path
110
+ private_key_path path1
111
+ </ssl>
112
+ ]
113
+ end
114
+
115
+ it 'converts them into new transport section' do
116
+ expect(driver.instance).to receive(:http_server_create_http_server).with(
117
+ :in_prometheus_server,
118
+ addr: anything,
119
+ logger: anything,
120
+ port: anything,
121
+ proto: :tls,
122
+ tls_opts: { 'cert_path' => 'path', 'private_key_path' => 'path1' }
123
+ ).once
124
+
125
+ driver.run(timeout: 1)
126
+ end
127
+ end
128
+
129
+ context 'insecure and ca_path' do
130
+ let(:config) do
131
+ %[
132
+ @type prometheus
133
+ bind 127.0.0.1
134
+ <ssl>
135
+ enable true
136
+ ca_path path
137
+ </ssl>
138
+ ]
139
+ end
140
+
141
+ it 'converts them into new transport section' do
142
+ expect(driver.instance).to receive(:http_server_create_http_server).with(
143
+ :in_prometheus_server,
144
+ addr: anything,
145
+ logger: anything,
146
+ port: anything,
147
+ proto: :tls,
148
+ tls_opts: { 'ca_path' => 'path', 'insecure' => true }
149
+ ).once
150
+
151
+ driver.run(timeout: 1)
152
+ end
153
+ end
154
+
155
+ context 'when only private_key_path is geven' do
156
+ let(:config) do
157
+ %[
158
+ @type prometheus
159
+ bind 127.0.0.1
160
+ <ssl>
161
+ enable true
162
+ private_key_path path
163
+ </ssl>
164
+ ]
165
+ end
166
+
167
+ it 'raises ConfigError' do
168
+ expect { driver.run(timeout: 1) }.to raise_error(Fluent::ConfigError, 'both certificate_path and private_key_path must be defined')
169
+ end
170
+ end
171
+ end
172
+ end
173
+
174
+ describe '#run' do
175
+ context '/metrics' do
176
+ let(:config) { LOCAL_CONFIG }
177
+ it 'returns 200' do
178
+ driver.run(timeout: 1) do
179
+ Net::HTTP.start("127.0.0.1", port) do |http|
180
+ req = Net::HTTP::Get.new("/metrics")
181
+ res = http.request(req)
182
+ expect(res.code).to eq('200')
183
+ end
184
+ end
185
+ end
186
+ end
187
+
188
+ context '/foo' do
189
+ let(:config) { LOCAL_CONFIG }
190
+ it 'does not return 200' do
191
+ driver.run(timeout: 1) do
192
+ Net::HTTP.start("127.0.0.1", port) do |http|
193
+ req = Net::HTTP::Get.new("/foo")
194
+ res = http.request(req)
195
+ expect(res.code).not_to eq('200')
196
+ end
197
+ end
198
+ end
199
+ end
200
+ end
201
+
202
+ describe '#run_multi_workers' do
203
+ context '/metrics' do
204
+ Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
205
+ let(:config) { CONFIG + %[
206
+ port #{port - 2}
207
+ ] }
208
+
209
+ it 'should configure port using sequential number' do
210
+ driver = Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusInput)
211
+ driver.instance.instance_eval{ @_fluentd_worker_id = 2 }
212
+ driver.configure(config)
213
+ expect(driver.instance.port).to eq(port)
214
+ driver.run(timeout: 1) do
215
+ Net::HTTP.start("127.0.0.1", port) do |http|
216
+ req = Net::HTTP::Get.new("/metrics")
217
+ res = http.request(req)
218
+ expect(res.code).to eq('200')
219
+ end
220
+ end
221
+ end
222
+ end
223
+ end
224
+ end
225
+ end