fluent-plugin-prometheus 1.7.3 → 1.8.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1f14f2b2162bace2a18a80f885df28261ed260a7b8e865a12e7828d46af9b02f
4
- data.tar.gz: 20749cd6ee743ac3726719e654e1a4f046aa2f953245604e6aa75cd87ed00e62
3
+ metadata.gz: c268ed901a0c48ea1885ce64b621c3abc79983a95696180236156b7844029a4d
4
+ data.tar.gz: ac9aea40b3eab787f14ba1e402732898ab2cf355a3836d252b85867100914a93
5
5
  SHA512:
6
- metadata.gz: 28ca37140a4b634665a9ba184c1615380bc7c458ed745d5548cb75b92d75d3b1069cd342d3119785a11d1beb369e7b41b5fa0319021bb0f69400ffe327dea83e
7
- data.tar.gz: b04770fec96856fc37b6943c7129c5e8ef16ce8b1415b425932e4419594e58a3255406de54e3aad1c28af1e3d62074391bd2b40e4aded8b5f598fbece53cab82
6
+ metadata.gz: 8bead7bf6d8534398ef24706348ae55d6437e3b898d89e6edb9981c6311d92be2e6d25f841a6bbf6be973a870966ff1296f3f4fb5314be62d06b900668f51b85
7
+ data.tar.gz: 8e9d8a48bdb3a3ba73626c01acda706a8af9f6dc309a1c005a240eb3cf051ef50ffe3b0012e966765ba8ffec415b2d2fd4a24852938c7fad84882ee842a96899
@@ -0,0 +1,23 @@
1
+ Release 1.8.4 - 2020/09/24
2
+
3
+ * in_prometheus_output_monitor: Add gauge_all parameter
4
+
5
+ Release 1.8.3 - 2020/08/24
6
+
7
+ * Fix resourcr leak in async-http based server
8
+
9
+ Release 1.8.2 - 2020/07/17
10
+
11
+ * in_prometheus_output_monitor/in_prometheus_tail_monitor: Support USR2 reload
12
+
13
+ Release 1.8.1 - 2020/07/06
14
+
15
+ * Fix aggregate bug with async-http
16
+
17
+ Release 1.8.0 - 2020/04/17
18
+
19
+ * Use http_server helper
20
+ * Require fluentd v1.9.1 or later
21
+
22
+
23
+ For older releases, see commits on github repository.
data/README.md CHANGED
@@ -8,9 +8,13 @@ A fluent plugin that instruments metrics from records and exposes them via web i
8
8
 
9
9
  | fluent-plugin-prometheus | fluentd | ruby |
10
10
  |--------------------------|------------|--------|
11
- | 1.x.y | >= v0.14.8 | >= 2.1 |
11
+ | 1.x.y | >= v1.9.1 | >= 2.4 |
12
+ | 1.[0-7].y | >= v0.14.8 | >= 2.1 |
12
13
  | 0.x.y | >= v0.12.0 | >= 1.9 |
13
14
 
15
+ Since v1.8.0, fluent-plugin-prometheus uses [http_server helper](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-http_server) to launch HTTP server.
16
+ If you want to handle lots of connections, install `async-http` gem.
17
+
14
18
  ## Installation
15
19
 
16
20
  Add this line to your application's Gemfile:
@@ -63,6 +67,19 @@ More configuration parameters:
63
67
  When using multiple workers, each worker binds to port + `fluent_worker_id`.
64
68
  To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
65
69
 
70
+ #### TLS setting
71
+
72
+ Use `<trasnport tls>`. See [transport config article](https://docs.fluentd.org/configuration/transport-section) for more details.
73
+
74
+ ```
75
+ <source>
76
+ @type prometheus
77
+ <transport tls>
78
+ # TLS parameters...
79
+ </transport
80
+ </source>
81
+ ```
82
+
66
83
  ### prometheus_monitor input plugin
67
84
 
68
85
  This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](https://docs.fluentd.org/input/monitor_agent).
@@ -71,7 +88,7 @@ This plugin collects internal metrics in Fluentd. The metrics are similar to/par
71
88
  #### Exposed metrics
72
89
 
73
90
  - `fluentd_status_buffer_queue_length`
74
- - `fluentd_status_buffer_total_queued_size`
91
+ - `fluentd_status_buffer_total_bytes`
75
92
  - `fluentd_status_retry_count`
76
93
  - `fluentd_status_buffer_newest_timekey` from fluentd v1.4.2
77
94
  - `fluentd_status_buffer_oldest_timekey` from fluentd v1.4.2
@@ -107,7 +124,7 @@ Metrics for output
107
124
  - `fluentd_output_status_emit_records`
108
125
  - `fluentd_output_status_write_count`
109
126
  - `fluentd_output_status_rollback_count`
110
- - `fluentd_output_status_flush_time_count` from fluentd v1.6.0
127
+ - `fluentd_output_status_flush_time_count` in milliseconds from fluentd v1.6.0
111
128
  - `fluentd_output_status_slow_flush_count` from fluentd v1.6.0
112
129
 
113
130
  Metrics for buffer
@@ -135,6 +152,7 @@ More configuration parameters:
135
152
 
136
153
  - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
137
154
  - `interval`: interval to update monitor_agent information in seconds (default: 5)
155
+ - `gauge_all`: Specify metric type. If `true`, use `gauge` type. If `false`, use `counter` type. Since v2, this parameter will be removed and use `counter` type.
138
156
 
139
157
  ### prometheus_tail_monitor input plugin
140
158
 
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "fluent-plugin-prometheus"
3
- spec.version = "1.7.3"
3
+ spec.version = "1.8.4"
4
4
  spec.authors = ["Masahiro Sano"]
5
5
  spec.email = ["sabottenda@gmail.com"]
6
6
  spec.summary = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
13
13
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
14
14
  spec.require_paths = ["lib"]
15
15
 
16
- spec.add_dependency "fluentd", ">= 0.14.20", "< 2"
16
+ spec.add_dependency "fluentd", ">= 1.9.1", "< 2"
17
17
  spec.add_dependency "prometheus-client", "< 0.10"
18
18
  spec.add_development_dependency "bundler"
19
19
  spec.add_development_dependency "rake"
@@ -2,13 +2,13 @@ require 'fluent/plugin/input'
2
2
  require 'fluent/plugin/prometheus'
3
3
  require 'fluent/plugin/prometheus_metrics'
4
4
  require 'net/http'
5
- require 'webrick'
5
+ require 'openssl'
6
6
 
7
7
  module Fluent::Plugin
8
8
  class PrometheusInput < Fluent::Plugin::Input
9
9
  Fluent::Plugin.register_input('prometheus', self)
10
10
 
11
- helpers :thread
11
+ helpers :thread, :http_server
12
12
 
13
13
  config_param :bind, :string, default: '0.0.0.0'
14
14
  config_param :port, :integer, default: 24231
@@ -17,30 +17,25 @@ module Fluent::Plugin
17
17
 
18
18
  desc 'Enable ssl configuration for the server'
19
19
  config_section :ssl, required: false, multi: false do
20
- config_param :enable, :bool, default: false
20
+ config_param :enable, :bool, default: false, deprecated: 'Use <transport tls> section'
21
21
 
22
22
  desc 'Path to the ssl certificate in PEM format. Read from file and added to conf as "SSLCertificate"'
23
- config_param :certificate_path, :string, default: nil
23
+ config_param :certificate_path, :string, default: nil, deprecated: 'Use cert_path in <transport tls> section'
24
24
 
25
25
  desc 'Path to the ssl private key in PEM format. Read from file and added to conf as "SSLPrivateKey"'
26
- config_param :private_key_path, :string, default: nil
26
+ config_param :private_key_path, :string, default: nil, deprecated: 'Use private_key_path in <transport tls> section'
27
27
 
28
28
  desc 'Path to CA in PEM format. Read from file and added to conf as "SSLCACertificateFile"'
29
- config_param :ca_path, :string, default: nil
29
+ config_param :ca_path, :string, default: nil, deprecated: 'Use ca_path in <transport tls> section'
30
30
 
31
31
  desc 'Additional ssl conf for the server. Ref: https://github.com/ruby/webrick/blob/master/lib/webrick/ssl.rb'
32
- config_param :extra_conf, :hash, default: {:SSLCertName => [['CN','nobody'],['DC','example']]}, symbolize_keys: true
32
+ config_param :extra_conf, :hash, default: nil, symbolize_keys: true, deprecated: 'See http helper config'
33
33
  end
34
34
 
35
- attr_reader :registry
36
-
37
- attr_reader :num_workers
38
- attr_reader :base_port
39
- attr_reader :metrics_path
40
-
41
35
  def initialize
42
36
  super
43
37
  @registry = ::Prometheus::Client.registry
38
+ @secure = nil
44
39
  end
45
40
 
46
41
  def configure(conf)
@@ -55,6 +50,7 @@ module Fluent::Plugin
55
50
  nil
56
51
  end
57
52
  @num_workers = sysconf && sysconf.workers ? sysconf.workers : 1
53
+ @secure = @transport_config.protocol == :tls || (@ssl && @ssl['enable'])
58
54
 
59
55
  @base_port = @port
60
56
  @port += fluentd_worker_id
@@ -66,7 +62,71 @@ module Fluent::Plugin
66
62
 
67
63
  def start
68
64
  super
69
- log.debug "listening prometheus http server on http://#{@bind}:#{@port}/#{@metrics_path} for worker#{fluentd_worker_id}"
65
+
66
+ scheme = @secure ? 'https' : 'http'
67
+ log.debug "listening prometheus http server on #{scheme}:://#{@bind}:#{@port}/#{@metrics_path} for worker#{fluentd_worker_id}"
68
+
69
+ proto = @secure ? :tls : :tcp
70
+
71
+ if @ssl && @ssl['enable'] && @ssl['extra_conf']
72
+ start_webrick
73
+ return
74
+ end
75
+
76
+ begin
77
+ require 'async'
78
+ require 'fluent/plugin/in_prometheus/async_wrapper'
79
+ extend AsyncWrapper
80
+ rescue LoadError => _
81
+ # ignore
82
+ end
83
+
84
+ tls_opt = if @ssl && @ssl['enable']
85
+ ssl_config = {}
86
+
87
+ if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
88
+ raise Fluent::ConfigError.new('both certificate_path and private_key_path must be defined')
89
+ end
90
+
91
+ if @ssl['certificate_path']
92
+ ssl_config['cert_path'] = @ssl['certificate_path']
93
+ end
94
+
95
+ if @ssl['private_key_path']
96
+ ssl_config['private_key_path'] = @ssl['private_key_path']
97
+ end
98
+
99
+ if @ssl['ca_path']
100
+ ssl_config['ca_path'] = @ssl['ca_path']
101
+ # Only ca_path is insecure in fluentd
102
+ # https://github.com/fluent/fluentd/blob/2236ad45197ba336fd9faf56f442252c8b226f25/lib/fluent/plugin_helper/cert_option.rb#L68
103
+ ssl_config['insecure'] = true
104
+ end
105
+
106
+ ssl_config
107
+ end
108
+
109
+ http_server_create_http_server(:in_prometheus_server, addr: @bind, port: @port, logger: log, proto: proto, tls_opts: tls_opt) do |server|
110
+ server.get(@metrics_path) { |_req| all_metrics }
111
+ server.get(@aggregated_metrics_path) { |_req| all_workers_metrics }
112
+ end
113
+ end
114
+
115
+ def shutdown
116
+ if @webrick_server
117
+ @webrick_server.shutdown
118
+ @webrick_server = nil
119
+ end
120
+ super
121
+ end
122
+
123
+ private
124
+
125
+ # For compatiblity because http helper can't support extra_conf option
126
+ def start_webrick
127
+ require 'webrick/https'
128
+ require 'webrick'
129
+
70
130
  config = {
71
131
  BindAddress: @bind,
72
132
  Port: @port,
@@ -74,90 +134,96 @@ module Fluent::Plugin
74
134
  Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
75
135
  AccessLog: [],
76
136
  }
77
- unless @ssl.nil? || !@ssl['enable']
78
- require 'webrick/https'
79
- require 'openssl'
80
- if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
81
- raise RuntimeError.new("certificate_path and private_key_path most both be defined")
82
- end
83
- ssl_config = {
84
- SSLEnable: true
85
- }
86
- if @ssl['certificate_path']
87
- cert = OpenSSL::X509::Certificate.new(File.read(@ssl['certificate_path']))
88
- ssl_config[:SSLCertificate] = cert
89
- end
90
- if @ssl['private_key_path']
91
- key = OpenSSL::PKey::RSA.new(File.read(@ssl['private_key_path']))
92
- ssl_config[:SSLPrivateKey] = key
93
- end
94
- ssl_config[:SSLCACertificateFile] = @ssl['ca_path'] if @ssl['ca_path']
95
- ssl_config = ssl_config.merge(@ssl['extra_conf'])
96
- config = ssl_config.merge(config)
137
+ if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
138
+ raise RuntimeError.new("certificate_path and private_key_path most both be defined")
139
+ end
140
+
141
+ ssl_config = {
142
+ SSLEnable: true,
143
+ SSLCertName: [['CN', 'nobody'], ['DC', 'example']]
144
+ }
145
+
146
+ if @ssl['certificate_path']
147
+ cert = OpenSSL::X509::Certificate.new(File.read(@ssl['certificate_path']))
148
+ ssl_config[:SSLCertificate] = cert
97
149
  end
150
+
151
+ if @ssl['private_key_path']
152
+ key = OpenSSL::PKey.read(@ssl['private_key_path'])
153
+ ssl_config[:SSLPrivateKey] = key
154
+ end
155
+
156
+ ssl_config[:SSLCACertificateFile] = @ssl['ca_path'] if @ssl['ca_path']
157
+ ssl_config = ssl_config.merge(@ssl['extra_conf']) if @ssl['extra_conf']
158
+ config = ssl_config.merge(config)
159
+
98
160
  @log.on_debug do
99
161
  @log.debug("WEBrick conf: #{config}")
100
162
  end
101
163
 
102
- @server = WEBrick::HTTPServer.new(config)
103
- @server.mount(@metrics_path, MonitorServlet, self)
104
- @server.mount(@aggregated_metrics_path, MonitorServletAll, self)
105
- thread_create(:in_prometheus) do
106
- @server.start
164
+ @webrick_server = WEBrick::HTTPServer.new(config)
165
+ @webrick_server.mount_proc(@metrics_path) do |_req, res|
166
+ status, header, body = all_metrics
167
+ res.status = status
168
+ res['Content-Type'] = header['Content-Type']
169
+ res.body = body
170
+ res
107
171
  end
108
- end
109
172
 
110
- def shutdown
111
- if @server
112
- @server.shutdown
113
- @server = nil
173
+ @webrick_server.mount_proc(@aggregated_metrics_path) do |_req, res|
174
+ status, header, body = all_workers_metrics
175
+ res.status = status
176
+ res['Content-Type'] = header['Content-Type']
177
+ res.body = body
178
+ res
179
+ end
180
+
181
+ thread_create(:in_prometheus_webrick) do
182
+ @webrick_server.start
114
183
  end
115
- super
116
184
  end
117
185
 
118
- class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
119
- def initialize(server, prometheus)
120
- @prometheus = prometheus
186
+ def all_metrics
187
+ [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, ::Prometheus::Client::Formats::Text.marshal(@registry)]
188
+ rescue => e
189
+ [500, { 'Content-Type' => 'text/plain' }, e.to_s]
190
+ end
191
+
192
+ def all_workers_metrics
193
+ full_result = PromMetricsAggregator.new
194
+
195
+ send_request_to_each_worker do |resp|
196
+ if resp.code.to_s == '200'
197
+ full_result.add_metrics(resp.body)
198
+ end
121
199
  end
122
200
 
123
- def do_GET(req, res)
124
- res.status = 200
125
- res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
126
- res.body = ::Prometheus::Client::Formats::Text.marshal(@prometheus.registry)
127
- rescue
128
- res.status = 500
129
- res['Content-Type'] = 'text/plain'
130
- res.body = $!.to_s
201
+ [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, full_result.get_metrics]
202
+ rescue => e
203
+ [500, { 'Content-Type' => 'text/plain' }, e.to_s]
204
+ end
205
+
206
+ def send_request_to_each_worker
207
+ bind = (@bind == '0.0.0.0') ? '127.0.0.1' : @bind
208
+ [*(@base_port...(@base_port + @num_workers))].each do |worker_port|
209
+ do_request(host: bind, port: worker_port, secure: @secure) do |http|
210
+ yield(http.get(@metrics_path))
211
+ end
131
212
  end
132
213
  end
133
214
 
134
- class MonitorServletAll < WEBrick::HTTPServlet::AbstractServlet
135
- def initialize(server, prometheus)
136
- @prometheus = prometheus
215
+ # might be replaced by AsyncWrapper if async gem is installed
216
+ def do_request(host:, port:, secure:)
217
+ http = Net::HTTP.new(host, port)
218
+
219
+ if secure
220
+ http.use_ssl = true
221
+ # target is our child process. so it's secure.
222
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
137
223
  end
138
224
 
139
- def do_GET(req, res)
140
- res.status = 200
141
- res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
142
-
143
- full_result = PromMetricsAggregator.new
144
- fluent_server_ip = @prometheus.bind == '0.0.0.0' ? '127.0.0.1' : @prometheus.bind
145
- current_worker = 0
146
- while current_worker < @prometheus.num_workers
147
- Net::HTTP.start(fluent_server_ip, @prometheus.base_port + current_worker) do |http|
148
- req = Net::HTTP::Get.new(@prometheus.metrics_path)
149
- result = http.request(req)
150
- if result.is_a?(Net::HTTPSuccess)
151
- full_result.add_metrics(result.body)
152
- end
153
- end
154
- current_worker += 1
155
- end
156
- res.body = full_result.get_metrics
157
- rescue
158
- res.status = 500
159
- res['Content-Type'] = 'text/plain'
160
- res.body = $!.to_s
225
+ http.start do
226
+ yield(http)
161
227
  end
162
228
  end
163
229
  end
@@ -0,0 +1,47 @@
1
+ require 'async'
2
+
3
+ module Fluent::Plugin
4
+ class PrometheusInput
5
+ module AsyncWrapper
6
+ def do_request(host:, port:, secure:)
7
+ endpoint =
8
+ if secure
9
+ context = OpenSSL::SSL::SSLContext.new
10
+ context.verify_mode = OpenSSL::SSL::VERIFY_NONE
11
+ Async::HTTP::Endpoint.parse("https://#{host}:#{port}", ssl_context: context)
12
+ else
13
+ Async::HTTP::Endpoint.parse("http://#{host}:#{port}")
14
+ end
15
+
16
+ Async::HTTP::Client.open(endpoint) do |client|
17
+ yield(AsyncHttpWrapper.new(client))
18
+ end
19
+ end
20
+
21
+ Response = Struct.new(:code, :body, :headers)
22
+
23
+ class AsyncHttpWrapper
24
+ def initialize(http)
25
+ @http = http
26
+ end
27
+
28
+ def get(path)
29
+ error = nil
30
+ response = Async::Task.current.async {
31
+ begin
32
+ @http.get(path)
33
+ rescue => e # Async::Reactor rescue all error. handle it by itself
34
+ error = e
35
+ end
36
+ }.wait
37
+
38
+ if error
39
+ raise error
40
+ end
41
+
42
+ Response.new(response.status.to_s, response.read || '', response.headers)
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -1,15 +1,16 @@
1
- require 'fluent/input'
1
+ require 'fluent/plugin/input'
2
2
  require 'fluent/plugin/in_monitor_agent'
3
3
  require 'fluent/plugin/prometheus'
4
4
 
5
5
  module Fluent::Plugin
6
- class PrometheusOutputMonitorInput < Fluent::Input
6
+ class PrometheusOutputMonitorInput < Fluent::Plugin::Input
7
7
  Fluent::Plugin.register_input('prometheus_output_monitor', self)
8
8
  include Fluent::Plugin::PrometheusLabelParser
9
9
 
10
10
  helpers :timer
11
11
 
12
12
  config_param :interval, :time, default: 5
13
+ config_param :gauge_all, :bool, default: true
13
14
  attr_reader :registry
14
15
 
15
16
  MONITOR_IVARS = [
@@ -53,12 +54,9 @@ module Fluent::Plugin
53
54
  @base_labels[key] = expander.expand(value)
54
55
  end
55
56
 
56
- if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
57
- # from v0.14.6
58
- @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
59
- else
60
- @monitor_agent = Fluent::MonitorAgentInput.new
61
- end
57
+ @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
58
+
59
+ @gauge_or_counter = @gauge_all ? :gauge : :counter
62
60
  end
63
61
 
64
62
  def start
@@ -66,57 +64,57 @@ module Fluent::Plugin
66
64
 
67
65
  @metrics = {
68
66
  # Buffer metrics
69
- buffer_total_queued_size: @registry.gauge(
67
+ buffer_total_queued_size: get_gauge(
70
68
  :fluentd_output_status_buffer_total_bytes,
71
69
  'Current total size of stage and queue buffers.'),
72
- buffer_stage_length: @registry.gauge(
70
+ buffer_stage_length: get_gauge(
73
71
  :fluentd_output_status_buffer_stage_length,
74
72
  'Current length of stage buffers.'),
75
- buffer_stage_byte_size: @registry.gauge(
73
+ buffer_stage_byte_size: get_gauge(
76
74
  :fluentd_output_status_buffer_stage_byte_size,
77
75
  'Current total size of stage buffers.'),
78
- buffer_queue_length: @registry.gauge(
76
+ buffer_queue_length: get_gauge(
79
77
  :fluentd_output_status_buffer_queue_length,
80
78
  'Current length of queue buffers.'),
81
- buffer_queue_byte_size: @registry.gauge(
79
+ buffer_queue_byte_size: get_gauge(
82
80
  :fluentd_output_status_queue_byte_size,
83
81
  'Current total size of queue buffers.'),
84
- buffer_available_buffer_space_ratios: @registry.gauge(
82
+ buffer_available_buffer_space_ratios: get_gauge(
85
83
  :fluentd_output_status_buffer_available_space_ratio,
86
84
  'Ratio of available space in buffer.'),
87
- buffer_newest_timekey: @registry.gauge(
85
+ buffer_newest_timekey: get_gauge(
88
86
  :fluentd_output_status_buffer_newest_timekey,
89
87
  'Newest timekey in buffer.'),
90
- buffer_oldest_timekey: @registry.gauge(
88
+ buffer_oldest_timekey: get_gauge(
91
89
  :fluentd_output_status_buffer_oldest_timekey,
92
90
  'Oldest timekey in buffer.'),
93
91
 
94
92
  # Output metrics
95
- retry_counts: @registry.gauge(
93
+ retry_counts: get_gauge_or_counter(
96
94
  :fluentd_output_status_retry_count,
97
95
  'Current retry counts.'),
98
- num_errors: @registry.gauge(
96
+ num_errors: get_gauge_or_counter(
99
97
  :fluentd_output_status_num_errors,
100
98
  'Current number of errors.'),
101
- emit_count: @registry.gauge(
99
+ emit_count: get_gauge_or_counter(
102
100
  :fluentd_output_status_emit_count,
103
101
  'Current emit counts.'),
104
- emit_records: @registry.gauge(
102
+ emit_records: get_gauge_or_counter(
105
103
  :fluentd_output_status_emit_records,
106
104
  'Current emit records.'),
107
- write_count: @registry.gauge(
105
+ write_count: get_gauge_or_counter(
108
106
  :fluentd_output_status_write_count,
109
107
  'Current write counts.'),
110
- rollback_count: @registry.gauge(
108
+ rollback_count: get_gauge(
111
109
  :fluentd_output_status_rollback_count,
112
110
  'Current rollback counts.'),
113
- flush_time_count: @registry.gauge(
111
+ flush_time_count: get_gauge_or_counter(
114
112
  :fluentd_output_status_flush_time_count,
115
113
  'Total flush time.'),
116
- slow_flush_count: @registry.gauge(
114
+ slow_flush_count: get_gauge_or_counter(
117
115
  :fluentd_output_status_slow_flush_count,
118
116
  'Current slow flush counts.'),
119
- retry_wait: @registry.gauge(
117
+ retry_wait: get_gauge(
120
118
  :fluentd_output_status_retry_wait,
121
119
  'Current retry wait'),
122
120
  }
@@ -162,14 +160,22 @@ module Fluent::Plugin
162
160
 
163
161
  monitor_info.each do |name, metric|
164
162
  if info[name]
165
- metric.set(label, info[name])
163
+ if metric.is_a?(::Prometheus::Client::Gauge)
164
+ metric.set(label, info[name])
165
+ elsif metric.is_a?(::Prometheus::Client::Counter)
166
+ metric.increment(label, info[name] - metric.get(label))
167
+ end
166
168
  end
167
169
  end
168
170
 
169
171
  if info['instance_variables']
170
172
  instance_vars_info.each do |name, metric|
171
173
  if info['instance_variables'][name]
172
- metric.set(label, info['instance_variables'][name])
174
+ if metric.is_a?(::Prometheus::Client::Gauge)
175
+ metric.set(label, info['instance_variables'][name])
176
+ elsif metric.is_a?(::Prometheus::Client::Counter)
177
+ metric.increment(label, info['instance_variables'][name] - metric.get(label))
178
+ end
173
179
  end
174
180
  end
175
181
  end
@@ -198,5 +204,21 @@ module Fluent::Plugin
198
204
  type: plugin_info["type"],
199
205
  )
200
206
  end
207
+
208
+ def get_gauge(name, docstring)
209
+ if @registry.exist?(name)
210
+ @registry.get(name)
211
+ else
212
+ @registry.gauge(name, docstring)
213
+ end
214
+ end
215
+
216
+ def get_gauge_or_counter(name, docstring)
217
+ if @registry.exist?(name)
218
+ @registry.get(name)
219
+ else
220
+ @registry.public_send(@gauge_or_counter, name, docstring)
221
+ end
222
+ end
201
223
  end
202
224
  end
@@ -38,22 +38,17 @@ module Fluent::Plugin
38
38
  @base_labels[key] = expander.expand(value)
39
39
  end
40
40
 
41
- if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
42
- # from v0.14.6
43
- @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
44
- else
45
- @monitor_agent = Fluent::MonitorAgentInput.new
46
- end
41
+ @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
47
42
  end
48
43
 
49
44
  def start
50
45
  super
51
46
 
52
47
  @metrics = {
53
- position: @registry.gauge(
48
+ position: get_gauge(
54
49
  :fluentd_tail_file_position,
55
50
  'Current position of file.'),
56
- inode: @registry.gauge(
51
+ inode: get_gauge(
57
52
  :fluentd_tail_file_inode,
58
53
  'Current inode of file.'),
59
54
  }
@@ -91,5 +86,13 @@ module Fluent::Plugin
91
86
  path: path,
92
87
  )
93
88
  end
89
+
90
+ def get_gauge(name, docstring)
91
+ if @registry.exist?(name)
92
+ @registry.get(name)
93
+ else
94
+ @registry.gauge(name, docstring)
95
+ end
96
+ end
94
97
  end
95
98
  end
@@ -6,11 +6,11 @@ require 'net/http'
6
6
 
7
7
  describe Fluent::Plugin::PrometheusInput do
8
8
  CONFIG = %[
9
- type prometheus
9
+ @type prometheus
10
10
  ]
11
11
 
12
12
  LOCAL_CONFIG = %[
13
- type prometheus
13
+ @type prometheus
14
14
  bind 127.0.0.1
15
15
  ]
16
16
 
@@ -47,6 +47,130 @@ describe Fluent::Plugin::PrometheusInput do
47
47
  end
48
48
  end
49
49
 
50
+ describe '#start' do
51
+ context 'with transport section' do
52
+ let(:config) do
53
+ %[
54
+ @type prometheus
55
+ bind 127.0.0.1
56
+ <transport tls>
57
+ insecure true
58
+ </transport>
59
+ ]
60
+ end
61
+
62
+ it 'returns 200' do
63
+ driver.run(timeout: 1) do
64
+ Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
65
+ req = Net::HTTP::Get.new('/metrics')
66
+ res = http.request(req)
67
+ expect(res.code).to eq('200')
68
+ end
69
+ end
70
+ end
71
+ end
72
+
73
+ context 'old parameters are given' do
74
+ context 'when extra_conf is used' do
75
+ let(:config) do
76
+ %[
77
+ @type prometheus
78
+ bind 127.0.0.1
79
+ <ssl>
80
+ enable true
81
+ extra_conf { "SSLCertName": [["CN", "nobody"], ["DC", "example"]] }
82
+ </ssl>
83
+ ]
84
+ end
85
+
86
+ it 'uses webrick' do
87
+ expect(driver.instance).to receive(:start_webrick).once
88
+ driver.run(timeout: 1)
89
+ end
90
+
91
+ it 'returns 200' do
92
+ driver.run(timeout: 1) do
93
+ Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
94
+ req = Net::HTTP::Get.new('/metrics')
95
+ res = http.request(req)
96
+ expect(res.code).to eq('200')
97
+ end
98
+ end
99
+ end
100
+ end
101
+
102
+ context 'cert_path and private_key_path combination' do
103
+ let(:config) do
104
+ %[
105
+ @type prometheus
106
+ bind 127.0.0.1
107
+ <ssl>
108
+ enable true
109
+ certificate_path path
110
+ private_key_path path1
111
+ </ssl>
112
+ ]
113
+ end
114
+
115
+ it 'converts them into new transport section' do
116
+ expect(driver.instance).to receive(:http_server_create_http_server).with(
117
+ :in_prometheus_server,
118
+ addr: anything,
119
+ logger: anything,
120
+ port: anything,
121
+ proto: :tls,
122
+ tls_opts: { 'cert_path' => 'path', 'private_key_path' => 'path1' }
123
+ ).once
124
+
125
+ driver.run(timeout: 1)
126
+ end
127
+ end
128
+
129
+ context 'insecure and ca_path' do
130
+ let(:config) do
131
+ %[
132
+ @type prometheus
133
+ bind 127.0.0.1
134
+ <ssl>
135
+ enable true
136
+ ca_path path
137
+ </ssl>
138
+ ]
139
+ end
140
+
141
+ it 'converts them into new transport section' do
142
+ expect(driver.instance).to receive(:http_server_create_http_server).with(
143
+ :in_prometheus_server,
144
+ addr: anything,
145
+ logger: anything,
146
+ port: anything,
147
+ proto: :tls,
148
+ tls_opts: { 'ca_path' => 'path', 'insecure' => true }
149
+ ).once
150
+
151
+ driver.run(timeout: 1)
152
+ end
153
+ end
154
+
155
+ context 'when only private_key_path is geven' do
156
+ let(:config) do
157
+ %[
158
+ @type prometheus
159
+ bind 127.0.0.1
160
+ <ssl>
161
+ enable true
162
+ private_key_path path
163
+ </ssl>
164
+ ]
165
+ end
166
+
167
+ it 'raises ConfigError' do
168
+ expect { driver.run(timeout: 1) }.to raise_error(Fluent::ConfigError, 'both certificate_path and private_key_path must be defined')
169
+ end
170
+ end
171
+ end
172
+ end
173
+
50
174
  describe '#run' do
51
175
  context '/metrics' do
52
176
  let(:config) { LOCAL_CONFIG }
@@ -1,6 +1,6 @@
1
1
 
2
2
  BASE_CONFIG = %[
3
- type prometheus
3
+ @type prometheus
4
4
  ]
5
5
 
6
6
  SIMPLE_CONFIG = BASE_CONFIG + %[
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-prometheus
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.7.3
4
+ version: 1.8.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro Sano
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-02-14 00:00:00.000000000 Z
11
+ date: 2020-09-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.14.20
19
+ version: 1.9.1
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: '2'
@@ -26,7 +26,7 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 0.14.20
29
+ version: 1.9.1
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'
@@ -110,6 +110,7 @@ files:
110
110
  - ".gitignore"
111
111
  - ".rspec"
112
112
  - ".travis.yml"
113
+ - ChangeLog
113
114
  - Gemfile
114
115
  - LICENSE
115
116
  - README.md
@@ -117,6 +118,7 @@ files:
117
118
  - fluent-plugin-prometheus.gemspec
118
119
  - lib/fluent/plugin/filter_prometheus.rb
119
120
  - lib/fluent/plugin/in_prometheus.rb
121
+ - lib/fluent/plugin/in_prometheus/async_wrapper.rb
120
122
  - lib/fluent/plugin/in_prometheus_monitor.rb
121
123
  - lib/fluent/plugin/in_prometheus_output_monitor.rb
122
124
  - lib/fluent/plugin/in_prometheus_tail_monitor.rb