fluent-plugin-prometheus 2.0.3 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/linux.yml +2 -7
- data/ChangeLog +9 -0
- data/README.md +62 -5
- data/fluent-plugin-prometheus.gemspec +1 -1
- data/lib/fluent/plugin/in_prometheus.rb +19 -3
- data/lib/fluent/plugin/in_prometheus_output_monitor.rb +1 -1
- data/lib/fluent/plugin/in_prometheus_tail_monitor.rb +15 -0
- data/lib/fluent/plugin/prometheus.rb +98 -1
- data/spec/fluent/plugin/in_prometheus_spec.rb +57 -1
- data/spec/fluent/plugin/out_prometheus_spec.rb +4 -0
- data/spec/fluent/plugin/shared.rb +203 -2
- metadata +3 -3
- data/.travis.yml +0 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c42edc59b51fa5a911f51e5c174f71dc573c99068379d3ee8ba7d2d710dbcbb5
|
4
|
+
data.tar.gz: 32206e06e3c6a6a9b19f6b93583af0c416ae76dc707a1a2dbefe0b54975a936b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 178009b03c28bf7dde6207cbf7f99c1bcdd8e787669630646f937eed5249b8d8fc17443ec08777620caafbc2498ceb13d753db45aea0a86d9b309a396078bf44
|
7
|
+
data.tar.gz: 3051330dbb989c965054f6171f9db9c00775d5642d7d4210da909c2501cca7c66a68b5e319385e81d579a0cdf389d7362873e95b6d5f4bc3fe84ed1004d70071
|
data/.github/workflows/linux.yml
CHANGED
@@ -11,17 +11,13 @@ jobs:
|
|
11
11
|
strategy:
|
12
12
|
fail-fast: false
|
13
13
|
matrix:
|
14
|
-
ruby: [ '3.
|
14
|
+
ruby: [ '3.3', '3.2', '3.1', '3.0', '2.7' ]
|
15
15
|
os:
|
16
16
|
- ubuntu-latest
|
17
17
|
experimental: [false]
|
18
|
-
include:
|
19
|
-
- ruby: head
|
20
|
-
os: ubuntu-latest
|
21
|
-
experimental: true
|
22
18
|
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
23
19
|
steps:
|
24
|
-
- uses: actions/checkout@
|
20
|
+
- uses: actions/checkout@v4
|
25
21
|
- uses: ruby/setup-ruby@v1
|
26
22
|
with:
|
27
23
|
ruby-version: ${{ matrix.ruby }}
|
@@ -29,6 +25,5 @@ jobs:
|
|
29
25
|
env:
|
30
26
|
CI: true
|
31
27
|
run: |
|
32
|
-
gem install bundler rake
|
33
28
|
bundle install --jobs 4 --retry 3
|
34
29
|
bundle exec rake spec
|
data/ChangeLog
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
Release 2.2.0 - 2024/08/02
|
2
|
+
|
3
|
+
* in_prometheus: Add gzip support (Add a new parameter `content_encoding gzip`)
|
4
|
+
|
5
|
+
Release 2.1.0 - 2023/06/15
|
6
|
+
|
7
|
+
* Add `initialized` and `initlabels` parameters to `<metric>` element
|
8
|
+
* in_prometheus_tail_monitor: Add file open/closed/rotation metrics
|
9
|
+
|
1
10
|
Release 2.0.3 - 2022/05/06
|
2
11
|
|
3
12
|
* in_prometheus_output_monitor: Fix a bug where output_status_num_errors metric is missing
|
data/README.md
CHANGED
@@ -63,6 +63,7 @@ More configuration parameters:
|
|
63
63
|
- `port`: listen port (default: 24231)
|
64
64
|
- `metrics_path`: metrics HTTP endpoint (default: /metrics)
|
65
65
|
- `aggregated_metrics_path`: metrics HTTP endpoint (default: /aggregated_metrics)
|
66
|
+
- `content_encoding`: encoding format for the exposed metrics (default: identity). Supported formats are {identity, gzip}
|
66
67
|
|
67
68
|
When using multiple workers, each worker binds to port + `fluent_worker_id`.
|
68
69
|
To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
|
@@ -162,10 +163,11 @@ This plugin uses internal class of Fluentd, so it's easy to break.
|
|
162
163
|
|
163
164
|
#### Exposed metrics
|
164
165
|
|
165
|
-
- `fluentd_tail_file_position
|
166
|
-
|
167
|
-
- `
|
168
|
-
|
166
|
+
- `fluentd_tail_file_position`: Current bytes which plugin reads from the file
|
167
|
+
- `fluentd_tail_file_inode`: inode of the file
|
168
|
+
- `fluentd_tail_file_closed`: Number of closed files
|
169
|
+
- `fluentd_tail_file_opened`: Number of opened files
|
170
|
+
- `fluentd_tail_file_rotated`: Number of rotated files
|
169
171
|
|
170
172
|
Default labels:
|
171
173
|
|
@@ -286,7 +288,9 @@ For details of each metric type, see [Prometheus documentation](http://prometheu
|
|
286
288
|
- `type`: metric type (required)
|
287
289
|
- `desc`: description of this metric (required)
|
288
290
|
- `key`: key name of record for instrumentation (**optional**)
|
291
|
+
- `initialized`: boolean controlling initilization of metric (**optional**). See [Metric initialization](#metric-initialization)
|
289
292
|
- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
|
293
|
+
- `<initlabels>`: labels to use for initialization of ReccordAccessors/Placeholder labels (**optional**). See [Metric initialization](#metric-initialization)
|
290
294
|
|
291
295
|
If key is empty, the metric values is treated as 1, so the counter increments by 1 on each record regardless of contents of the record.
|
292
296
|
|
@@ -310,7 +314,9 @@ If key is empty, the metric values is treated as 1, so the counter increments by
|
|
310
314
|
- `type`: metric type (required)
|
311
315
|
- `desc`: description of metric (required)
|
312
316
|
- `key`: key name of record for instrumentation (required)
|
317
|
+
- `initialized`: boolean controlling initilization of metric (**optional**). See [Metric initialization](#metric-initialization)
|
313
318
|
- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
|
319
|
+
- `<initlabels>`: labels to use for initialization of ReccordAccessors/Placeholder labels (**optional**). See [Metric initialization](#metric-initialization)
|
314
320
|
|
315
321
|
### summary type
|
316
322
|
|
@@ -332,7 +338,9 @@ If key is empty, the metric values is treated as 1, so the counter increments by
|
|
332
338
|
- `type`: metric type (required)
|
333
339
|
- `desc`: description of metric (required)
|
334
340
|
- `key`: key name of record for instrumentation (required)
|
341
|
+
- `initialized`: boolean controlling initilization of metric (**optional**). See [Metric initialization](#metric-initialization)
|
335
342
|
- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
|
343
|
+
- `<initlabels>`: labels to use for initialization of ReccordAccessors/Placeholder labels (**optional**). See [Metric initialization](#metric-initialization)
|
336
344
|
|
337
345
|
### histogram type
|
338
346
|
|
@@ -355,8 +363,10 @@ If key is empty, the metric values is treated as 1, so the counter increments by
|
|
355
363
|
- `type`: metric type (required)
|
356
364
|
- `desc`: description of metric (required)
|
357
365
|
- `key`: key name of record for instrumentation (required)
|
366
|
+
- `initialized`: boolean controlling initilization of metric (**optional**). See [Metric initialization](#metric-initialization)
|
358
367
|
- `buckets`: buckets of record for instrumentation (optional)
|
359
368
|
- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
|
369
|
+
- `<initlabels>`: labels to use for initialization of ReccordAccessors/Placeholder labels (**optional**). See [Metric initialization](#metric-initialization)
|
360
370
|
|
361
371
|
## Labels
|
362
372
|
|
@@ -395,6 +405,53 @@ Reserved placeholders are:
|
|
395
405
|
- where `tagsize` is the size of tag which is splitted with `.` (when tag is `1.2.3`, then `tagsize` is 3)
|
396
406
|
- only available in Prometheus output/filter plugin
|
397
407
|
|
408
|
+
### Metric initialization
|
409
|
+
|
410
|
+
You can configure if a metric should be initialized to its zero value before receiving any event. To do so you just need to specify `initialized true`.
|
411
|
+
|
412
|
+
```
|
413
|
+
<metric>
|
414
|
+
name message_bar_counter
|
415
|
+
type counter
|
416
|
+
desc The total number of bar in message.
|
417
|
+
key bar
|
418
|
+
initialized true
|
419
|
+
<labels>
|
420
|
+
foo bar
|
421
|
+
</labels>
|
422
|
+
</metric>
|
423
|
+
```
|
424
|
+
|
425
|
+
If your labels contains ReccordAccessors or Placeholders, you must use `<initlabels>` to specify the values your ReccordAccessors/Placeholders will take. This feature is useful only if your Placeholders/ReccordAccessors contain deterministic values. Initialization will create as many zero value metrics as `<initlabels>` blocks you defined.
|
426
|
+
Potential reserved placeholders `${hostname}` and `${worker_id}`, as well as static labels, are automatically added and should not be specified in `<initlabels>` configuration.
|
427
|
+
|
428
|
+
```
|
429
|
+
<metric>
|
430
|
+
name message_bar_counter
|
431
|
+
type counter
|
432
|
+
desc The total number of bar in message.
|
433
|
+
key bar
|
434
|
+
initialized true
|
435
|
+
<labels>
|
436
|
+
key $.foo
|
437
|
+
tag ${tag}
|
438
|
+
foo bar
|
439
|
+
worker_id ${worker_id}
|
440
|
+
</labels>
|
441
|
+
<initlabels>
|
442
|
+
key foo1
|
443
|
+
tag tag1
|
444
|
+
</initlabels>
|
445
|
+
<initlabels>
|
446
|
+
key foo2
|
447
|
+
tag tag2
|
448
|
+
</initlabels>
|
449
|
+
</metric>
|
450
|
+
<labels>
|
451
|
+
hostname ${hostname}
|
452
|
+
</labels>
|
453
|
+
```
|
454
|
+
|
398
455
|
### top-level labels and labels inside metric
|
399
456
|
|
400
457
|
Prometheus output/filter plugin can have multiple metric section. Top-level labels section specifies labels for all metrics. Labels section inside metric section specifies labels for the metric. Both are specified, labels are merged.
|
@@ -436,7 +493,7 @@ In this case, `message_foo_counter` has `tag`, `hostname`, `key` and `data_type`
|
|
436
493
|
Checkout repository and setup.
|
437
494
|
|
438
495
|
```
|
439
|
-
$ git clone git://github.com/fluent/fluent-plugin-prometheus
|
496
|
+
$ git clone git://github.com/fluent/fluent-plugin-prometheus.git
|
440
497
|
$ cd fluent-plugin-prometheus
|
441
498
|
$ bundle install --path vendor/bundle
|
442
499
|
```
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "fluent-plugin-prometheus"
|
3
|
-
spec.version = "2.0
|
3
|
+
spec.version = "2.2.0"
|
4
4
|
spec.authors = ["Masahiro Sano"]
|
5
5
|
spec.email = ["sabottenda@gmail.com"]
|
6
6
|
spec.summary = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
|
@@ -3,6 +3,7 @@ require 'fluent/plugin/prometheus'
|
|
3
3
|
require 'fluent/plugin/prometheus_metrics'
|
4
4
|
require 'net/http'
|
5
5
|
require 'openssl'
|
6
|
+
require 'zlib'
|
6
7
|
|
7
8
|
module Fluent::Plugin
|
8
9
|
class PrometheusInput < Fluent::Plugin::Input
|
@@ -32,6 +33,9 @@ module Fluent::Plugin
|
|
32
33
|
config_param :extra_conf, :hash, default: nil, symbolize_keys: true, deprecated: 'See http helper config'
|
33
34
|
end
|
34
35
|
|
36
|
+
desc 'Content encoding of the exposed metrics, Currently supported encoding is identity, gzip. Ref: https://prometheus.io/docs/instrumenting/exposition_formats/#basic-info'
|
37
|
+
config_param :content_encoding, :enum, list: [:identity, :gzip], default: :identity
|
38
|
+
|
35
39
|
def initialize
|
36
40
|
super
|
37
41
|
@registry = ::Prometheus::Client.registry
|
@@ -184,7 +188,7 @@ module Fluent::Plugin
|
|
184
188
|
end
|
185
189
|
|
186
190
|
def all_metrics
|
187
|
-
|
191
|
+
response(::Prometheus::Client::Formats::Text.marshal(@registry))
|
188
192
|
rescue => e
|
189
193
|
[500, { 'Content-Type' => 'text/plain' }, e.to_s]
|
190
194
|
end
|
@@ -197,8 +201,7 @@ module Fluent::Plugin
|
|
197
201
|
full_result.add_metrics(resp.body)
|
198
202
|
end
|
199
203
|
end
|
200
|
-
|
201
|
-
[200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, full_result.get_metrics]
|
204
|
+
response(full_result.get_metrics)
|
202
205
|
rescue => e
|
203
206
|
[500, { 'Content-Type' => 'text/plain' }, e.to_s]
|
204
207
|
end
|
@@ -226,5 +229,18 @@ module Fluent::Plugin
|
|
226
229
|
yield(http)
|
227
230
|
end
|
228
231
|
end
|
232
|
+
|
233
|
+
def response(metrics)
|
234
|
+
body = nil
|
235
|
+
case @content_encoding
|
236
|
+
when :gzip
|
237
|
+
gzip = Zlib::GzipWriter.new(StringIO.new)
|
238
|
+
gzip << metrics
|
239
|
+
body = gzip.close.string
|
240
|
+
when :identity
|
241
|
+
body = metrics
|
242
|
+
end
|
243
|
+
[200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE, 'Content-Encoding' => @content_encoding.to_s }, body]
|
244
|
+
end
|
229
245
|
end
|
230
246
|
end
|
@@ -77,7 +77,7 @@ module Fluent::Plugin
|
|
77
77
|
:fluentd_output_status_buffer_queue_length,
|
78
78
|
'Current length of queue buffers.'),
|
79
79
|
buffer_queue_byte_size: get_gauge(
|
80
|
-
:
|
80
|
+
:fluentd_output_status_buffer_queue_byte_size,
|
81
81
|
'Current total size of queue buffers.'),
|
82
82
|
buffer_available_buffer_space_ratios: get_gauge(
|
83
83
|
:fluentd_output_status_buffer_available_space_ratio,
|
@@ -51,6 +51,15 @@ module Fluent::Plugin
|
|
51
51
|
inode: get_gauge(
|
52
52
|
:fluentd_tail_file_inode,
|
53
53
|
'Current inode of file.'),
|
54
|
+
closed_file_metrics: get_gauge(
|
55
|
+
:fluentd_tail_file_closed,
|
56
|
+
'Number of files closed.'),
|
57
|
+
opened_file_metrics: get_gauge(
|
58
|
+
:fluentd_tail_file_opened,
|
59
|
+
'Number of files opened.'),
|
60
|
+
rotated_file_metrics: get_gauge(
|
61
|
+
:fluentd_tail_file_rotated,
|
62
|
+
'Number of files rotated.'),
|
54
63
|
}
|
55
64
|
timer_execute(:in_prometheus_tail_monitor, @interval, &method(:update_monitor_info))
|
56
65
|
end
|
@@ -72,9 +81,15 @@ module Fluent::Plugin
|
|
72
81
|
# Access to internal variable of internal class...
|
73
82
|
# Very fragile implementation
|
74
83
|
pe = watcher.instance_variable_get(:@pe)
|
84
|
+
monitor_info = watcher.instance_variable_get(:@metrics)
|
75
85
|
label = labels(info, watcher.path)
|
76
86
|
@metrics[:inode].set(pe.read_inode, labels: label)
|
77
87
|
@metrics[:position].set(pe.read_pos, labels: label)
|
88
|
+
unless monitor_info.nil?
|
89
|
+
@metrics[:closed_file_metrics].set(monitor_info.closed.get, labels: label)
|
90
|
+
@metrics[:opened_file_metrics].set(monitor_info.opened.get, labels: label)
|
91
|
+
@metrics[:rotated_file_metrics].set(monitor_info.rotated.get, labels: label)
|
92
|
+
end
|
78
93
|
end
|
79
94
|
end
|
80
95
|
end
|
@@ -56,6 +56,69 @@ module Fluent
|
|
56
56
|
base_labels
|
57
57
|
end
|
58
58
|
|
59
|
+
def self.parse_initlabels_elements(conf, base_labels)
|
60
|
+
base_initlabels = []
|
61
|
+
|
62
|
+
# We first treat the special case of RecordAccessors and Placeholders labels if any declared
|
63
|
+
conf.elements.select { |e| e.name == 'initlabels' }.each { |block|
|
64
|
+
initlabels = {}
|
65
|
+
|
66
|
+
block.each do |key, value|
|
67
|
+
if not base_labels.has_key? key.to_sym
|
68
|
+
raise ConfigError, "Key #{key} in <initlabels> is non existent in <labels> for metric #{conf['name']}"
|
69
|
+
end
|
70
|
+
|
71
|
+
if value.start_with?('$.') || value.start_with?('$[') || value.start_with?('${')
|
72
|
+
raise ConfigError, "Cannot use RecordAccessor or placeholder #{value} (key #{key}) in a <initlabels> in metric #{conf['name']}"
|
73
|
+
end
|
74
|
+
|
75
|
+
base_label_value = base_labels[key.to_sym]
|
76
|
+
|
77
|
+
if !(base_label_value.class == Fluent::PluginHelper::RecordAccessor::Accessor) && ! (base_label_value.start_with?('${') )
|
78
|
+
raise ConfigError, "Cannot set <initlabels> on non RecordAccessor/Placeholder key #{key} (value #{value}) in metric #{conf['name']}"
|
79
|
+
end
|
80
|
+
|
81
|
+
if base_label_value == '${worker_id}' || base_label_value == '${hostname}'
|
82
|
+
raise ConfigError, "Cannot set <initlabels> on reserved placeholder #{base_label_value} for key #{key} in metric #{conf['name']}"
|
83
|
+
end
|
84
|
+
|
85
|
+
initlabels[key.to_sym] = value
|
86
|
+
end
|
87
|
+
|
88
|
+
# Now adding all labels that are not RecordAccessor nor Placeholder labels as is
|
89
|
+
base_labels.each do |key, value|
|
90
|
+
if base_labels[key.to_sym].class != Fluent::PluginHelper::RecordAccessor::Accessor
|
91
|
+
if value == '${worker_id}'
|
92
|
+
# We retrieve fluentd_worker_id this way to not overcomplicate the code
|
93
|
+
initlabels[key.to_sym] = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
|
94
|
+
elsif value == '${hostname}'
|
95
|
+
initlabels[key.to_sym] = Socket.gethostname
|
96
|
+
elsif !(value.start_with?('${'))
|
97
|
+
initlabels[key.to_sym] = value
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
base_initlabels << initlabels
|
103
|
+
}
|
104
|
+
|
105
|
+
# Testing for RecordAccessor/Placeholder labels missing a declaration in <initlabels> blocks
|
106
|
+
base_labels.each do |key, value|
|
107
|
+
if value.class == Fluent::PluginHelper::RecordAccessor::Accessor || value.start_with?('${')
|
108
|
+
if not base_initlabels.map(&:keys).flatten.include? (key.to_sym)
|
109
|
+
raise ConfigError, "RecordAccessor/Placeholder key #{key} with value #{value} has not been set in a <initlabels> for initialized metric #{conf['name']}"
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
if base_initlabels.length == 0
|
115
|
+
# There were no RecordAccessor nor Placeholder labels, we blunty retrieve the static base_labels
|
116
|
+
base_initlabels << base_labels
|
117
|
+
end
|
118
|
+
|
119
|
+
base_initlabels
|
120
|
+
end
|
121
|
+
|
59
122
|
def self.parse_metrics_elements(conf, registry, labels = {})
|
60
123
|
metrics = []
|
61
124
|
conf.elements.select { |element|
|
@@ -162,9 +225,27 @@ module Fluent
|
|
162
225
|
@name = element['name']
|
163
226
|
@key = element['key']
|
164
227
|
@desc = element['desc']
|
165
|
-
|
228
|
+
element['initialized'].nil? ? @initialized = false : @initialized = element['initialized'] == 'true'
|
229
|
+
|
166
230
|
@base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(element)
|
167
231
|
@base_labels = labels.merge(@base_labels)
|
232
|
+
|
233
|
+
if @initialized
|
234
|
+
@base_initlabels = Fluent::Plugin::Prometheus.parse_initlabels_elements(element, @base_labels)
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
def self.init_label_set(metric, base_initlabels, base_labels)
|
239
|
+
base_initlabels.each { |initlabels|
|
240
|
+
# Should never happen, but handy test should code evolution break current implementation
|
241
|
+
if initlabels.keys.sort != base_labels.keys.sort
|
242
|
+
raise ConfigError, "initlabels for metric #{metric.name} must have the same signature than labels " \
|
243
|
+
"(initlabels given: #{initlabels.keys} vs." \
|
244
|
+
" expected from labels: #{base_labels.keys})"
|
245
|
+
end
|
246
|
+
|
247
|
+
metric.init_label_set(initlabels)
|
248
|
+
}
|
168
249
|
end
|
169
250
|
|
170
251
|
def labels(record, expander)
|
@@ -206,6 +287,10 @@ module Fluent
|
|
206
287
|
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
207
288
|
@gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
|
208
289
|
end
|
290
|
+
|
291
|
+
if @initialized
|
292
|
+
Fluent::Plugin::Prometheus::Metric.init_label_set(@gauge, @base_initlabels, @base_labels)
|
293
|
+
end
|
209
294
|
end
|
210
295
|
|
211
296
|
def instrument(record, expander)
|
@@ -228,6 +313,10 @@ module Fluent
|
|
228
313
|
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
229
314
|
@counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
|
230
315
|
end
|
316
|
+
|
317
|
+
if @initialized
|
318
|
+
Fluent::Plugin::Prometheus::Metric.init_label_set(@counter, @base_initlabels, @base_labels)
|
319
|
+
end
|
231
320
|
end
|
232
321
|
|
233
322
|
def instrument(record, expander)
|
@@ -259,6 +348,10 @@ module Fluent
|
|
259
348
|
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
260
349
|
@summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
|
261
350
|
end
|
351
|
+
|
352
|
+
if @initialized
|
353
|
+
Fluent::Plugin::Prometheus::Metric.init_label_set(@summary, @base_initlabels, @base_labels)
|
354
|
+
end
|
262
355
|
end
|
263
356
|
|
264
357
|
def instrument(record, expander)
|
@@ -292,6 +385,10 @@ module Fluent
|
|
292
385
|
rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
|
293
386
|
@histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
|
294
387
|
end
|
388
|
+
|
389
|
+
if @initialized
|
390
|
+
Fluent::Plugin::Prometheus::Metric.init_label_set(@histogram, @base_initlabels, @base_labels)
|
391
|
+
end
|
295
392
|
end
|
296
393
|
|
297
394
|
def instrument(record, expander)
|
@@ -3,6 +3,7 @@ require 'fluent/plugin/in_prometheus'
|
|
3
3
|
require 'fluent/test/driver/input'
|
4
4
|
|
5
5
|
require 'net/http'
|
6
|
+
require 'zlib'
|
6
7
|
|
7
8
|
describe Fluent::Plugin::PrometheusInput do
|
8
9
|
CONFIG = %[
|
@@ -45,6 +46,24 @@ describe Fluent::Plugin::PrometheusInput do
|
|
45
46
|
expect(driver.instance.metrics_path).to eq('/_test')
|
46
47
|
end
|
47
48
|
end
|
49
|
+
|
50
|
+
describe 'content_encoding_identity' do
|
51
|
+
let(:config) { CONFIG + %[
|
52
|
+
content_encoding identity
|
53
|
+
] }
|
54
|
+
it 'should be configurable' do
|
55
|
+
expect(driver.instance.content_encoding).to eq(:identity)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe 'content_encoding_gzip' do
|
60
|
+
let(:config) { CONFIG + %[
|
61
|
+
content_encoding gzip
|
62
|
+
] }
|
63
|
+
it 'should be configurable' do
|
64
|
+
expect(driver.instance.content_encoding).to eq(:gzip)
|
65
|
+
end
|
66
|
+
end
|
48
67
|
end
|
49
68
|
|
50
69
|
describe '#start' do
|
@@ -197,12 +216,49 @@ describe Fluent::Plugin::PrometheusInput do
|
|
197
216
|
end
|
198
217
|
end
|
199
218
|
end
|
219
|
+
|
220
|
+
context 'response content_encoding identity' do
|
221
|
+
let(:config) { LOCAL_CONFIG + %[
|
222
|
+
content_encoding identity
|
223
|
+
] }
|
224
|
+
it 'exposes metric' do
|
225
|
+
driver.run(timeout: 1) do
|
226
|
+
registry = driver.instance.instance_variable_get(:@registry)
|
227
|
+
registry.counter(:test,docstring: "Testing metrics") unless registry.exist?(:test)
|
228
|
+
Net::HTTP.start("127.0.0.1", port) do |http|
|
229
|
+
req = Net::HTTP::Get.new("/metrics")
|
230
|
+
req['accept-encoding'] = nil
|
231
|
+
res = http.request(req)
|
232
|
+
expect(res.body).to include("test Testing metrics")
|
233
|
+
end
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
context 'response content_encoding gzip' do
|
239
|
+
let(:config) { LOCAL_CONFIG + %[
|
240
|
+
content_encoding gzip
|
241
|
+
] }
|
242
|
+
it 'exposes metric' do
|
243
|
+
driver.run(timeout: 1) do
|
244
|
+
registry = driver.instance.instance_variable_get(:@registry)
|
245
|
+
registry.counter(:test,docstring: "Testing metrics") unless registry.exist?(:test)
|
246
|
+
Net::HTTP.start("127.0.0.1", port) do |http|
|
247
|
+
req = Net::HTTP::Get.new("/metrics")
|
248
|
+
req['accept-encoding'] = nil
|
249
|
+
res = http.request(req)
|
250
|
+
gzip = Zlib::GzipReader.new(StringIO.new(res.body.to_s))
|
251
|
+
expect(gzip.read).to include("test Testing metrics")
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|
255
|
+
end
|
200
256
|
end
|
201
257
|
|
202
258
|
describe '#run_multi_workers' do
|
203
259
|
context '/metrics' do
|
204
260
|
Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
|
205
|
-
let(:config) {
|
261
|
+
let(:config) { FULL_CONFIG + %[
|
206
262
|
port #{port - 2}
|
207
263
|
] }
|
208
264
|
|
@@ -16,6 +16,10 @@ describe Fluent::Plugin::PrometheusOutput do
|
|
16
16
|
it_behaves_like 'output configuration'
|
17
17
|
end
|
18
18
|
|
19
|
+
describe '#testinitlabels' do
|
20
|
+
it_behaves_like 'initalized metrics'
|
21
|
+
end
|
22
|
+
|
19
23
|
describe '#run' do
|
20
24
|
let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
|
21
25
|
|
@@ -27,6 +27,7 @@ FULL_CONFIG = BASE_CONFIG + %[
|
|
27
27
|
type gauge
|
28
28
|
desc Something bar.
|
29
29
|
key bar
|
30
|
+
initialized true
|
30
31
|
<labels>
|
31
32
|
key foo2
|
32
33
|
</labels>
|
@@ -36,6 +37,7 @@ FULL_CONFIG = BASE_CONFIG + %[
|
|
36
37
|
type summary
|
37
38
|
desc Something baz.
|
38
39
|
key baz
|
40
|
+
initialized true
|
39
41
|
<labels>
|
40
42
|
key foo3
|
41
43
|
</labels>
|
@@ -46,6 +48,7 @@ FULL_CONFIG = BASE_CONFIG + %[
|
|
46
48
|
desc Something qux.
|
47
49
|
key qux
|
48
50
|
buckets 0.1, 1, 5, 10
|
51
|
+
initialized true
|
49
52
|
<labels>
|
50
53
|
key foo4
|
51
54
|
</labels>
|
@@ -62,12 +65,32 @@ FULL_CONFIG = BASE_CONFIG + %[
|
|
62
65
|
<metric>
|
63
66
|
name full_accessor2
|
64
67
|
type counter
|
65
|
-
desc Something with accessor
|
68
|
+
desc Something with accessor
|
66
69
|
key $.foo
|
70
|
+
initialized true
|
67
71
|
<labels>
|
68
72
|
key foo6
|
69
73
|
</labels>
|
70
74
|
</metric>
|
75
|
+
<metric>
|
76
|
+
name full_accessor3
|
77
|
+
type counter
|
78
|
+
desc Something with accessor and several initialized metrics
|
79
|
+
initialized true
|
80
|
+
<labels>
|
81
|
+
key $.foo
|
82
|
+
key2 $.foo2
|
83
|
+
key3 footix
|
84
|
+
</labels>
|
85
|
+
<initlabels>
|
86
|
+
key foo6
|
87
|
+
key2 foo7
|
88
|
+
</initlabels>
|
89
|
+
<initlabels>
|
90
|
+
key foo8
|
91
|
+
key2 foo9
|
92
|
+
</initlabels>
|
93
|
+
</metric>
|
71
94
|
<labels>
|
72
95
|
test_key test_value
|
73
96
|
</labels>
|
@@ -79,13 +102,20 @@ PLACEHOLDER_CONFIG = BASE_CONFIG + %[
|
|
79
102
|
type counter
|
80
103
|
desc Something foo.
|
81
104
|
key foo
|
105
|
+
initialized true
|
82
106
|
<labels>
|
83
107
|
foo ${foo}
|
108
|
+
foo2 foo2
|
84
109
|
</labels>
|
110
|
+
<initlabels>
|
111
|
+
tag tag
|
112
|
+
foo foo
|
113
|
+
</initlabels>
|
85
114
|
</metric>
|
86
115
|
<labels>
|
87
116
|
tag ${tag}
|
88
117
|
hostname ${hostname}
|
118
|
+
workerid ${worker_id}
|
89
119
|
</labels>
|
90
120
|
]
|
91
121
|
|
@@ -150,6 +180,111 @@ shared_examples_for 'output configuration' do
|
|
150
180
|
end
|
151
181
|
it { expect { driver }.to raise_error(Fluent::ConfigError) }
|
152
182
|
end
|
183
|
+
|
184
|
+
|
185
|
+
context 'with missing <initlabels>' do
|
186
|
+
let(:config) do
|
187
|
+
BASE_CONFIG + %[
|
188
|
+
<metric>
|
189
|
+
name simple_foo
|
190
|
+
type counter
|
191
|
+
desc Something foo but incorrect
|
192
|
+
key foo
|
193
|
+
initialized true
|
194
|
+
<labels>
|
195
|
+
key $.accessor
|
196
|
+
</labels>
|
197
|
+
</metric>
|
198
|
+
]
|
199
|
+
end
|
200
|
+
it { expect { driver }.to raise_error(Fluent::ConfigError) }
|
201
|
+
end
|
202
|
+
|
203
|
+
context 'with RecordAccessor set in <initlabels>' do
|
204
|
+
let(:config) do
|
205
|
+
BASE_CONFIG + %[
|
206
|
+
<metric>
|
207
|
+
name simple_foo
|
208
|
+
type counter
|
209
|
+
desc Something foo but incorrect
|
210
|
+
key foo
|
211
|
+
initialized true
|
212
|
+
<labels>
|
213
|
+
key $.accessor
|
214
|
+
</labels>
|
215
|
+
<initlabels>
|
216
|
+
key $.accessor2
|
217
|
+
<initlabels>
|
218
|
+
</metric>
|
219
|
+
]
|
220
|
+
end
|
221
|
+
it { expect { driver }.to raise_error(Fluent::ConfigError) }
|
222
|
+
end
|
223
|
+
|
224
|
+
context 'with PlaceHolder set in <initlabels>' do
|
225
|
+
let(:config) do
|
226
|
+
BASE_CONFIG + %[
|
227
|
+
<metric>
|
228
|
+
name simple_foo
|
229
|
+
type counter
|
230
|
+
desc Something foo but incorrect
|
231
|
+
key foo
|
232
|
+
initialized true
|
233
|
+
<labels>
|
234
|
+
key ${foo}
|
235
|
+
</labels>
|
236
|
+
<initlabels>
|
237
|
+
key ${foo}
|
238
|
+
<initlabels>
|
239
|
+
</metric>
|
240
|
+
]
|
241
|
+
end
|
242
|
+
it { expect { driver }.to raise_error(Fluent::ConfigError) }
|
243
|
+
end
|
244
|
+
|
245
|
+
context 'with non RecordAccessor label set in <initlabels>' do
|
246
|
+
let(:config) do
|
247
|
+
BASE_CONFIG + %[
|
248
|
+
<metric>
|
249
|
+
name simple_foo
|
250
|
+
type counter
|
251
|
+
desc Something foo but incorrect
|
252
|
+
key foo
|
253
|
+
initialized true
|
254
|
+
<labels>
|
255
|
+
key $.accessor
|
256
|
+
key2 foo2
|
257
|
+
</labels>
|
258
|
+
<initlabels>
|
259
|
+
key foo
|
260
|
+
key2 foo2
|
261
|
+
<initlabels>
|
262
|
+
</metric>
|
263
|
+
]
|
264
|
+
end
|
265
|
+
it { expect { driver }.to raise_error(Fluent::ConfigError) }
|
266
|
+
end
|
267
|
+
|
268
|
+
context 'with non-matching label keys set in <initlabels>' do
|
269
|
+
let(:config) do
|
270
|
+
BASE_CONFIG + %[
|
271
|
+
<metric>
|
272
|
+
name simple_foo
|
273
|
+
type counter
|
274
|
+
desc Something foo but incorrect
|
275
|
+
key foo
|
276
|
+
initialized true
|
277
|
+
<labels>
|
278
|
+
key $.accessor
|
279
|
+
</labels>
|
280
|
+
<initlabels>
|
281
|
+
key2 foo
|
282
|
+
<initlabels>
|
283
|
+
</metric>
|
284
|
+
]
|
285
|
+
end
|
286
|
+
it { expect { driver }.to raise_error(Fluent::ConfigError) }
|
287
|
+
end
|
153
288
|
end
|
154
289
|
|
155
290
|
shared_examples_for 'instruments record' do
|
@@ -165,14 +300,16 @@ shared_examples_for 'instruments record' do
|
|
165
300
|
let(:histogram) { registry.get(:full_qux) }
|
166
301
|
let(:summary_with_accessor) { registry.get(:full_accessor1) }
|
167
302
|
let(:counter_with_accessor) { registry.get(:full_accessor2) }
|
303
|
+
let(:counter_with_two_accessors) { registry.get(:full_accessor3) }
|
168
304
|
|
169
305
|
it 'adds all metrics' do
|
170
|
-
expect(registry.metrics.map(&:name)).to eq(%i[full_foo full_bar full_baz full_qux full_accessor1 full_accessor2])
|
306
|
+
expect(registry.metrics.map(&:name)).to eq(%i[full_foo full_bar full_baz full_qux full_accessor1 full_accessor2 full_accessor3])
|
171
307
|
expect(counter).to be_kind_of(::Prometheus::Client::Metric)
|
172
308
|
expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
|
173
309
|
expect(summary).to be_kind_of(::Prometheus::Client::Metric)
|
174
310
|
expect(summary_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
|
175
311
|
expect(counter_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
|
312
|
+
expect(counter_with_two_accessors).to be_kind_of(::Prometheus::Client::Metric)
|
176
313
|
expect(histogram).to be_kind_of(::Prometheus::Client::Metric)
|
177
314
|
end
|
178
315
|
|
@@ -180,6 +317,7 @@ shared_examples_for 'instruments record' do
|
|
180
317
|
expect(counter.type).to eq(:counter)
|
181
318
|
expect(counter.get(labels: {test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
|
182
319
|
expect(counter_with_accessor.get(labels: {test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
|
320
|
+
expect(counter_with_two_accessors.get(labels: {test_key: 'test_value', key: 'foo6', key2: 'foo7', key3: 'footix'})).to be_kind_of(Numeric)
|
183
321
|
end
|
184
322
|
|
185
323
|
it 'instruments gauge metric' do
|
@@ -246,3 +384,66 @@ shared_examples_for 'instruments record' do
|
|
246
384
|
end
|
247
385
|
end
|
248
386
|
end
|
387
|
+
|
388
|
+
shared_examples_for 'initalized metrics' do
|
389
|
+
before do
|
390
|
+
driver.run(default_tag: tag)
|
391
|
+
end
|
392
|
+
|
393
|
+
context 'full config' do
|
394
|
+
let(:config) { FULL_CONFIG }
|
395
|
+
let(:counter) { registry.get(:full_foo) }
|
396
|
+
let(:gauge) { registry.get(:full_bar) }
|
397
|
+
let(:summary) { registry.get(:full_baz) }
|
398
|
+
let(:histogram) { registry.get(:full_qux) }
|
399
|
+
let(:summary_with_accessor) { registry.get(:full_accessor1) }
|
400
|
+
let(:counter_with_accessor) { registry.get(:full_accessor2) }
|
401
|
+
let(:counter_with_two_accessors) { registry.get(:full_accessor3) }
|
402
|
+
|
403
|
+
it 'adds all metrics' do
|
404
|
+
expect(registry.metrics.map(&:name)).to eq(%i[full_foo full_bar full_baz full_qux full_accessor1 full_accessor2 full_accessor3])
|
405
|
+
expect(counter).to be_kind_of(::Prometheus::Client::Metric)
|
406
|
+
expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
|
407
|
+
expect(summary).to be_kind_of(::Prometheus::Client::Metric)
|
408
|
+
expect(summary_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
|
409
|
+
expect(counter_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
|
410
|
+
expect(counter_with_two_accessors).to be_kind_of(::Prometheus::Client::Metric)
|
411
|
+
expect(histogram).to be_kind_of(::Prometheus::Client::Metric)
|
412
|
+
end
|
413
|
+
|
414
|
+
it 'tests uninitialized metrics' do
|
415
|
+
expect(counter.values).to eq({})
|
416
|
+
expect(summary_with_accessor.values).to eq({})
|
417
|
+
end
|
418
|
+
|
419
|
+
it 'tests initialized metrics' do
|
420
|
+
expect(gauge.values).to eq({{:key=>"foo2", :test_key=>"test_value"}=>0.0})
|
421
|
+
expect(summary.values).to eq({:key=>"foo3", :test_key=>"test_value"}=>{"count"=>0.0, "sum"=>0.0})
|
422
|
+
expect(histogram.values).to eq({:key=>"foo4", :test_key=>"test_value"} => {"+Inf"=>0.0, "0.1"=>0.0, "1"=>0.0, "10"=>0.0, "5"=>0.0, "sum"=>0.0})
|
423
|
+
expect(counter_with_accessor.values).to eq({{:key=>"foo6", :test_key=>"test_value"}=>0.0})
|
424
|
+
expect(counter_with_two_accessors.values).to eq({{:key=>"foo6", :key2=>"foo7", :key3=>"footix", :test_key=>"test_value"}=>0.0, {:key=>"foo8", :key2=>"foo9", :key3=>"footix", :test_key=>"test_value"}=>0.0})
|
425
|
+
end
|
426
|
+
end
|
427
|
+
|
428
|
+
context 'placeholder config' do
|
429
|
+
let(:config) { PLACEHOLDER_CONFIG }
|
430
|
+
let(:counter) { registry.get(:placeholder_foo) }
|
431
|
+
|
432
|
+
it 'expands placeholders with record values' do
|
433
|
+
expect(registry.metrics.map(&:name)).to eq([:placeholder_foo])
|
434
|
+
expect(counter).to be_kind_of(::Prometheus::Client::Metric)
|
435
|
+
|
436
|
+
key, _ = counter.values.find {|k,v| v == 0.0 }
|
437
|
+
expect(key).to be_kind_of(Hash)
|
438
|
+
expect(key[:foo]).to eq("foo")
|
439
|
+
expect(key[:foo2]).to eq("foo2")
|
440
|
+
expect(key[:hostname]).to be_kind_of(String)
|
441
|
+
expect(key[:hostname]).not_to eq("${hostname}")
|
442
|
+
expect(key[:hostname]).not_to be_empty
|
443
|
+
expect(key[:workerid]).to be_kind_of(String)
|
444
|
+
expect(key[:workerid]).not_to eq("${worker_id}")
|
445
|
+
expect(key[:workerid]).not_to be_empty
|
446
|
+
expect(key[:tag]).to eq("tag")
|
447
|
+
end
|
448
|
+
end
|
449
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-prometheus
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Sano
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-08-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -107,10 +107,10 @@ executables: []
|
|
107
107
|
extensions: []
|
108
108
|
extra_rdoc_files: []
|
109
109
|
files:
|
110
|
+
- ".github/dependabot.yml"
|
110
111
|
- ".github/workflows/linux.yml"
|
111
112
|
- ".gitignore"
|
112
113
|
- ".rspec"
|
113
|
-
- ".travis.yml"
|
114
114
|
- ChangeLog
|
115
115
|
- Gemfile
|
116
116
|
- LICENSE
|