fluent-plugin-prometheus 2.2.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +72 -0
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
- data/.github/dependabot.yml +18 -1
- data/.github/workflows/add-to-project.yml +24 -0
- data/.github/workflows/linux.yml +13 -3
- data/.gitignore +1 -0
- data/ChangeLog +17 -0
- data/README.md +129 -2
- data/fluent-plugin-prometheus.gemspec +1 -1
- data/lib/fluent/plugin/filter_prometheus.rb +1 -1
- data/lib/fluent/plugin/in_prometheus/async_wrapper.rb +9 -2
- data/lib/fluent/plugin/in_prometheus.rb +73 -24
- data/lib/fluent/plugin/out_prometheus.rb +3 -1
- data/lib/fluent/plugin/prometheus/log_throttle.rb +45 -0
- data/lib/fluent/plugin/prometheus/placeholder_expander.rb +15 -1
- data/lib/fluent/plugin/prometheus.rb +299 -15
- data/spec/fluent/plugin/filter_prometheus_spec.rb +141 -0
- data/spec/fluent/plugin/in_prometheus_spec.rb +257 -1
- data/spec/fluent/plugin/out_prometheus_spec.rb +29 -1
- data/spec/fluent/plugin/prometheus/log_throttle_spec.rb +127 -0
- data/spec/fluent/plugin/prometheus/placeholder_expander_spec.rb +96 -0
- data/spec/fluent/plugin/prometheus/series_limit_spec.rb +469 -0
- data/spec/fluent/plugin/shared.rb +113 -1
- data/spec/spec_helper.rb +18 -0
- metadata +12 -6
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c10a2e202128626380ce6fd1003ca82baa6f1ae55bd41642b8f6c0bc44670abc
|
|
4
|
+
data.tar.gz: 34a77887e5da8f4bb4dc08018478a1592d7dec51411e338b4773ab41ec281e06
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 131d844af8a45deb00d2ee53dba0928534a104193aec9a26f5d0760600ffefa2c11cb284fa6e49ae81f1cb4e17520db1b62a2a1acc6af295b964bdfa4c35aa5c
|
|
7
|
+
data.tar.gz: b9da9de5e2019b290ccd45bfca81ad59cdf7942109e961f8919d1121596e00bc63661cf76825251e4f0a9b796929830a6a7a7dade75ddcc00785f0c901317476
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
name: Bug Report
|
|
2
|
+
description: Create a report with a procedure for reproducing the bug
|
|
3
|
+
labels: "waiting-for-triage"
|
|
4
|
+
body:
|
|
5
|
+
- type: markdown
|
|
6
|
+
attributes:
|
|
7
|
+
value: |
|
|
8
|
+
Please check the [README](https://github.com/fluent/fluent-plugin-prometheus/blob/master/README.md) first. To help us investigate the problem, please provide the following information.
|
|
9
|
+
- type: textarea
|
|
10
|
+
id: description
|
|
11
|
+
attributes:
|
|
12
|
+
label: Describe the bug
|
|
13
|
+
description: A clear and concise description of what the bug is
|
|
14
|
+
validations:
|
|
15
|
+
required: true
|
|
16
|
+
- type: textarea
|
|
17
|
+
id: reproduce
|
|
18
|
+
attributes:
|
|
19
|
+
label: To Reproduce
|
|
20
|
+
description: Steps to reproduce the behavior
|
|
21
|
+
validations:
|
|
22
|
+
required: true
|
|
23
|
+
- type: textarea
|
|
24
|
+
id: expected
|
|
25
|
+
attributes:
|
|
26
|
+
label: Expected behavior
|
|
27
|
+
description: A clear and concise description of what you expected to happen
|
|
28
|
+
validations:
|
|
29
|
+
required: true
|
|
30
|
+
- type: textarea
|
|
31
|
+
id: environment
|
|
32
|
+
attributes:
|
|
33
|
+
label: Your Environment
|
|
34
|
+
description: |
|
|
35
|
+
- Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
|
|
36
|
+
- Operating system: `cat /etc/os-release`
|
|
37
|
+
- Kernel version: `uname -r`
|
|
38
|
+
|
|
39
|
+
Tip: If you hit the problem with older fluentd version, try latest version first.
|
|
40
|
+
value: |
|
|
41
|
+
- Fluentd version:
|
|
42
|
+
- TD Agent/Fluent Package version:
|
|
43
|
+
- fluent-plugin-prometheus version:
|
|
44
|
+
- prometheus-client version:
|
|
45
|
+
- Operating system:
|
|
46
|
+
- Kernel version:
|
|
47
|
+
render: markdown
|
|
48
|
+
validations:
|
|
49
|
+
required: true
|
|
50
|
+
- type: textarea
|
|
51
|
+
id: configuration
|
|
52
|
+
attributes:
|
|
53
|
+
label: Your Configuration
|
|
54
|
+
description: |
|
|
55
|
+
Write your configuration here. Minimum reproducible fluentd.conf is recommended.
|
|
56
|
+
validations:
|
|
57
|
+
required: true
|
|
58
|
+
- type: textarea
|
|
59
|
+
id: logs
|
|
60
|
+
attributes:
|
|
61
|
+
label: Your Error Log
|
|
62
|
+
description: Write your ALL error log here
|
|
63
|
+
render: shell
|
|
64
|
+
validations:
|
|
65
|
+
required: true
|
|
66
|
+
- type: textarea
|
|
67
|
+
id: addtional-context
|
|
68
|
+
attributes:
|
|
69
|
+
label: Additional context
|
|
70
|
+
description: Add any other context about the problem here.
|
|
71
|
+
validations:
|
|
72
|
+
required: false
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: Feature request
|
|
2
|
+
description: Suggest an idea for this project
|
|
3
|
+
labels: "waiting-for-triage"
|
|
4
|
+
body:
|
|
5
|
+
- type: markdown
|
|
6
|
+
attributes:
|
|
7
|
+
value: |
|
|
8
|
+
Please check the [README](https://github.com/fluent/fluent-plugin-prometheus/blob/master/README.md) first. To help us understand the request, please provide the following information.
|
|
9
|
+
- type: textarea
|
|
10
|
+
id: description
|
|
11
|
+
attributes:
|
|
12
|
+
label: Is your feature request related to a problem? Please describe.
|
|
13
|
+
description: |
|
|
14
|
+
A clear and concise description of what the problem is.
|
|
15
|
+
Ex. I'm always frustrated when [...]
|
|
16
|
+
validations:
|
|
17
|
+
required: true
|
|
18
|
+
- type: textarea
|
|
19
|
+
id: solution
|
|
20
|
+
attributes:
|
|
21
|
+
label: Describe the solution you'd like
|
|
22
|
+
description: A clear and concise description of what you want to happen.
|
|
23
|
+
validations:
|
|
24
|
+
required: true
|
|
25
|
+
- type: textarea
|
|
26
|
+
id: alternative
|
|
27
|
+
attributes:
|
|
28
|
+
label: Describe alternatives you've considered
|
|
29
|
+
description: A clear and concise description of any alternative solutions or features you've considered.
|
|
30
|
+
validations:
|
|
31
|
+
required: true
|
|
32
|
+
- type: textarea
|
|
33
|
+
id: addtional-context
|
|
34
|
+
attributes:
|
|
35
|
+
label: Additional context
|
|
36
|
+
description: Add any other context or screenshots about the feature request here.
|
|
37
|
+
validations:
|
|
38
|
+
required: false
|
data/.github/dependabot.yml
CHANGED
|
@@ -3,4 +3,21 @@ updates:
|
|
|
3
3
|
- package-ecosystem: 'github-actions'
|
|
4
4
|
directory: '/'
|
|
5
5
|
schedule:
|
|
6
|
-
interval: '
|
|
6
|
+
interval: 'monthly'
|
|
7
|
+
groups:
|
|
8
|
+
# PR: "Security update [package] from [old] to [new]"
|
|
9
|
+
# This PR should be merged in hurry
|
|
10
|
+
security-updates:
|
|
11
|
+
applies-to: security-updates
|
|
12
|
+
patterns:
|
|
13
|
+
- '*'
|
|
14
|
+
|
|
15
|
+
# PR: "Bump [package] from [old] to [new]"
|
|
16
|
+
# No need to be merged this PR in hurry. It is enough to merge
|
|
17
|
+
# once in a month.
|
|
18
|
+
monthly-updates:
|
|
19
|
+
applies-to: version-updates
|
|
20
|
+
patterns:
|
|
21
|
+
- '*'
|
|
22
|
+
# Allow to create PR both of security and normal updates.
|
|
23
|
+
open-pull-requests-limit: 1
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: Add bugs to fluent project
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
issues:
|
|
5
|
+
types:
|
|
6
|
+
- opened
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
add-to-project:
|
|
10
|
+
name: Add issue to project
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
steps:
|
|
13
|
+
- name: Generate token
|
|
14
|
+
id: generate_token
|
|
15
|
+
uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
|
|
16
|
+
with:
|
|
17
|
+
app-id: ${{ secrets.PROJECT_APP_ID }}
|
|
18
|
+
private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
|
|
19
|
+
owner: fluent
|
|
20
|
+
- uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
|
|
21
|
+
with:
|
|
22
|
+
project-url: https://github.com/orgs/fluent/projects/4
|
|
23
|
+
github-token: ${{ steps.generate_token.outputs.token }}
|
|
24
|
+
labeled: waiting-for-triage
|
data/.github/workflows/linux.yml
CHANGED
|
@@ -4,21 +4,31 @@ on:
|
|
|
4
4
|
branches: [master]
|
|
5
5
|
pull_request:
|
|
6
6
|
branches: [master]
|
|
7
|
+
schedule:
|
|
8
|
+
- cron: '0 0 1 * *'
|
|
7
9
|
jobs:
|
|
10
|
+
ruby-versions:
|
|
11
|
+
uses: ruby/actions/.github/workflows/ruby_versions.yml@master
|
|
12
|
+
with:
|
|
13
|
+
engine: cruby
|
|
14
|
+
min_version: 2.7
|
|
8
15
|
build:
|
|
16
|
+
needs: ruby-versions
|
|
9
17
|
runs-on: ${{ matrix.os }}
|
|
10
18
|
continue-on-error: ${{ matrix.experimental }}
|
|
11
19
|
strategy:
|
|
12
20
|
fail-fast: false
|
|
13
21
|
matrix:
|
|
14
|
-
ruby:
|
|
22
|
+
ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
|
|
23
|
+
exclude:
|
|
24
|
+
- ruby: head
|
|
15
25
|
os:
|
|
16
26
|
- ubuntu-latest
|
|
17
27
|
experimental: [false]
|
|
18
28
|
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
|
19
29
|
steps:
|
|
20
|
-
- uses: actions/checkout@
|
|
21
|
-
- uses: ruby/setup-ruby@v1
|
|
30
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
31
|
+
- uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
|
|
22
32
|
with:
|
|
23
33
|
ruby-version: ${{ matrix.ruby }}
|
|
24
34
|
- name: unit testing
|
data/.gitignore
CHANGED
data/ChangeLog
CHANGED
|
@@ -1,3 +1,20 @@
|
|
|
1
|
+
Release 2.3.0 - 2026/09/02
|
|
2
|
+
|
|
3
|
+
* in_prometheus: Change default bind address to 127.0.0.1. (#258)
|
|
4
|
+
This is incompatible behavior change.
|
|
5
|
+
* in_prometheus: Do not disclosure error details for client (#259)
|
|
6
|
+
* in_prometheus: Add `ignore_error_log_interval` parameter to suppress repeated error logs (#259,#264)
|
|
7
|
+
This introduces the log throttling feature.
|
|
8
|
+
* filter_prometheus: Do not let placeholders built from records override the tag ones (#262)
|
|
9
|
+
* out_prometheus: Fix a bug that a failing record never reached to @ERROR (#263)
|
|
10
|
+
* Support `max_series_per_metric` to limit label sets cardinarity. (#265)
|
|
11
|
+
* Refuse wrong <initlabels> which cannot fit into `max_series_per_metric` (#266,#267)
|
|
12
|
+
This blocks `max_series_per_metric` mis-configuration on startup.
|
|
13
|
+
|
|
14
|
+
Release 2.2.2 - 2026/03/27
|
|
15
|
+
|
|
16
|
+
* in_prometheus: fix: Add IPv6 support for bind parameter (#240)
|
|
17
|
+
|
|
1
18
|
Release 2.2.1 - 2025/03/24
|
|
2
19
|
|
|
3
20
|
* in_prometheus_tail_monitor: Add throttling metrics as `fluentd_tail_file_throttled`. (GitHub#227)
|
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# fluent-plugin-prometheus, a plugin for [Fluentd](https://www.fluentd.org)
|
|
2
2
|
|
|
3
|
-
[](https://github.com/fluent/fluent-plugin-prometheus/actions/workflows/linux.yml)
|
|
4
4
|
|
|
5
5
|
A fluent plugin that instruments metrics from records and exposes them via web interface. Intended to be used together with a [Prometheus server](https://github.com/prometheus/prometheus).
|
|
6
6
|
|
|
@@ -59,11 +59,12 @@ With following configuration, you can access http://localhost:24231/metrics on a
|
|
|
59
59
|
|
|
60
60
|
More configuration parameters:
|
|
61
61
|
|
|
62
|
-
- `bind`: binding interface (default: '
|
|
62
|
+
- `bind`: binding interface (default: '127.0.0.1')
|
|
63
63
|
- `port`: listen port (default: 24231)
|
|
64
64
|
- `metrics_path`: metrics HTTP endpoint (default: /metrics)
|
|
65
65
|
- `aggregated_metrics_path`: metrics HTTP endpoint (default: /aggregated_metrics)
|
|
66
66
|
- `content_encoding`: encoding format for the exposed metrics (default: identity). Supported formats are {identity, gzip}
|
|
67
|
+
- `ignore_error_log_interval`: Suppress repeated error logs in a certain period of time or until message was changed (default: 1h)
|
|
67
68
|
|
|
68
69
|
When using multiple workers, each worker binds to port + `fluent_worker_id`.
|
|
69
70
|
To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
|
|
@@ -265,6 +266,132 @@ You can access nested keys in records via dot or bracket notation (https://docs.
|
|
|
265
266
|
|
|
266
267
|
See Supported Metric Type and Labels for more configuration parameters.
|
|
267
268
|
|
|
269
|
+
#### Limiting label expansion
|
|
270
|
+
|
|
271
|
+
Label values come from records, so a metric grows unboundedly when a label is
|
|
272
|
+
bound to a field with many distinct values. Both plugins can bound it with
|
|
273
|
+
`max_series_per_metric`: once a metric holds that many label sets, a record
|
|
274
|
+
which brings a new one is dropped, while the label sets already known keep
|
|
275
|
+
being instrumented.
|
|
276
|
+
|
|
277
|
+
|parameter|description|default|
|
|
278
|
+
|---|---|---|
|
|
279
|
+
|max_series_per_metric|The maximum number of label sets a metric can hold. `0` means unlimited.|0|
|
|
280
|
+
|ignore_error_log_interval|The interval in seconds to suppress the repeated warning about the drops. `0` logs every occurrence.|3600|
|
|
281
|
+
|
|
282
|
+
**The limit is disabled by default and must be enabled explicitly**, since a
|
|
283
|
+
dropped record is lost and cannot be recovered. A `<metric>` section overrides
|
|
284
|
+
the value given to the plugin, so that a metric which expands faster than the
|
|
285
|
+
others is bound on its own, while one whose labels are known to be bounded
|
|
286
|
+
stays unlimited with `0`:
|
|
287
|
+
|
|
288
|
+
```
|
|
289
|
+
<filter message>
|
|
290
|
+
@type prometheus
|
|
291
|
+
max_series_per_metric 1000
|
|
292
|
+
<metric>
|
|
293
|
+
name message_foo_counter
|
|
294
|
+
type counter
|
|
295
|
+
desc The total number of foo in message.
|
|
296
|
+
key foo
|
|
297
|
+
max_series_per_metric 10
|
|
298
|
+
<labels>
|
|
299
|
+
path $.kubernetes.pod_name
|
|
300
|
+
</labels>
|
|
301
|
+
</metric>
|
|
302
|
+
</filter>
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
The label sets are counted per metric name, not per `<metric>` section: sections
|
|
306
|
+
with the same `name`, in one plugin or in two, share one count. Each of them
|
|
307
|
+
refuses a new label set once that shared count reaches its own limit, so a
|
|
308
|
+
section which stays at `0` adds label sets without counting them. Its
|
|
309
|
+
`<initlabels>` are counted anyway. They come from the configuration and their
|
|
310
|
+
number is fixed. The metric holds them in every section.
|
|
311
|
+
|
|
312
|
+
The count is per worker process as well, since a worker has its own registry and
|
|
313
|
+
exposes the metrics it holds itself. With `workers N`, a metric can hold up to N
|
|
314
|
+
times `max_series_per_metric` label sets in total, so divide the number of label
|
|
315
|
+
sets the metric may reach by the number of workers.
|
|
316
|
+
|
|
317
|
+
A record which fails to be instrumented, for example when the value of `key` is
|
|
318
|
+
not a number, does not consume the limit. A pre-initialized label set
|
|
319
|
+
(`initialized` and `<initlabels>`) consumes it from the start, since the metric
|
|
320
|
+
holds it before any record arrives.
|
|
321
|
+
|
|
322
|
+
A `<metric>` section is refused at startup with a configuration error when the
|
|
323
|
+
`<initlabels>` label sets already fill its limit. The limit is exceeded before
|
|
324
|
+
any record arrives, and the section can never take a new label set.
|
|
325
|
+
|
|
326
|
+
These label sets are shared by every section with the same `name`. A section can
|
|
327
|
+
be refused because of another section:
|
|
328
|
+
|
|
329
|
+
```
|
|
330
|
+
<metric>
|
|
331
|
+
name shared
|
|
332
|
+
type counter
|
|
333
|
+
desc Something foo.
|
|
334
|
+
max_series_per_metric 2
|
|
335
|
+
initialized true
|
|
336
|
+
<labels>
|
|
337
|
+
path $.path
|
|
338
|
+
</labels>
|
|
339
|
+
<initlabels>
|
|
340
|
+
path /a
|
|
341
|
+
</initlabels>
|
|
342
|
+
<initlabels>
|
|
343
|
+
path /b
|
|
344
|
+
</initlabels>
|
|
345
|
+
</metric>
|
|
346
|
+
<metric>
|
|
347
|
+
name shared # the same name, so the 2 label sets count here too
|
|
348
|
+
type counter
|
|
349
|
+
desc Something foo.
|
|
350
|
+
max_series_per_metric 1 # refused: 2 label sets do not fit into 1
|
|
351
|
+
<labels>
|
|
352
|
+
path $.path
|
|
353
|
+
</labels>
|
|
354
|
+
</metric>
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
The second section is refused even though it declares no `<initlabels>` of its
|
|
358
|
+
own. The same happens when the first section sets `max_series_per_metric 0`.
|
|
359
|
+
The metric holds its label sets in both cases. A limit equal to their number is
|
|
360
|
+
fine because all label sets of the metric are known in advance.
|
|
361
|
+
|
|
362
|
+
The check runs after every `<metric>` section of a plugin is read. It does not
|
|
363
|
+
depend on the order of the sections. Sections that share a `name` across two
|
|
364
|
+
plugins are only checked against the sections read before them.
|
|
365
|
+
|
|
366
|
+
The check runs again when Fluentd reloads the configuration. It counts only the
|
|
367
|
+
label sets from `<initlabels>`. It does not count the label sets that records
|
|
368
|
+
brought, so the metric does not make the reload fail with the label sets it took
|
|
369
|
+
while it was running.
|
|
370
|
+
|
|
371
|
+
A reload does not clear the registry. The metric still holds the label sets from
|
|
372
|
+
the `<initlabels>` of the old configuration, and the check counts them too. So
|
|
373
|
+
it can refuse a new configuration which has fewer `<initlabels>` than the old
|
|
374
|
+
one. Restart the worker to drop them.
|
|
375
|
+
|
|
376
|
+
##### Observing what the limit leaves out
|
|
377
|
+
|
|
378
|
+
A dropped label set is not routed to `@ERROR`, because it is what the
|
|
379
|
+
configuration asks for. It is reported in two ways instead:
|
|
380
|
+
|
|
381
|
+
* a warning in the Fluentd log, throttled per metric: it is suppressed for
|
|
382
|
+
`ignore_error_log_interval` seconds and reports how many warnings were
|
|
383
|
+
suppressed in the meantime.
|
|
384
|
+
* `fluentd_prometheus_dropped_label_sets_total{name}`, which counts the records
|
|
385
|
+
the metric `name` did not instrument. It is registered on the first drop, so
|
|
386
|
+
it does not show up as long as nothing is dropped, and its label comes from
|
|
387
|
+
the configuration and not from a record, so it cannot expand on its own.
|
|
388
|
+
|
|
389
|
+
Alert on the counter to notice that a metric is losing records:
|
|
390
|
+
|
|
391
|
+
```
|
|
392
|
+
rate(fluentd_prometheus_dropped_label_sets_total[5m]) > 0
|
|
393
|
+
```
|
|
394
|
+
|
|
268
395
|
## Supported Metric Types
|
|
269
396
|
|
|
270
397
|
For details of each metric type, see [Prometheus documentation](http://prometheus.io/docs/concepts/metric_types/). Also see [metric name guide](http://prometheus.io/docs/practices/naming/).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |spec|
|
|
2
2
|
spec.name = "fluent-plugin-prometheus"
|
|
3
|
-
spec.version = "2.
|
|
3
|
+
spec.version = "2.3.0"
|
|
4
4
|
spec.authors = ["Masahiro Sano"]
|
|
5
5
|
spec.email = ["sabottenda@gmail.com"]
|
|
6
6
|
spec.summary = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
|
|
@@ -19,7 +19,7 @@ module Fluent::Plugin
|
|
|
19
19
|
def configure(conf)
|
|
20
20
|
super
|
|
21
21
|
labels = parse_labels_elements(conf)
|
|
22
|
-
@metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
|
|
22
|
+
@metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels, metric_options)
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
def filter(tag, time, record)
|
|
@@ -4,13 +4,20 @@ module Fluent::Plugin
|
|
|
4
4
|
class PrometheusInput
|
|
5
5
|
module AsyncWrapper
|
|
6
6
|
def do_request(host:, port:, secure:)
|
|
7
|
+
# Format host for URI - bracket IPv6 addresses if not already bracketed
|
|
8
|
+
uri_host = if host.include?(':') && !host.start_with?('[')
|
|
9
|
+
"[#{host}]"
|
|
10
|
+
else
|
|
11
|
+
host
|
|
12
|
+
end
|
|
13
|
+
|
|
7
14
|
endpoint =
|
|
8
15
|
if secure
|
|
9
16
|
context = OpenSSL::SSL::SSLContext.new
|
|
10
17
|
context.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
11
|
-
Async::HTTP::Endpoint.parse("https://#{
|
|
18
|
+
Async::HTTP::Endpoint.parse("https://#{uri_host}:#{port}", ssl_context: context)
|
|
12
19
|
else
|
|
13
|
-
Async::HTTP::Endpoint.parse("http://#{
|
|
20
|
+
Async::HTTP::Endpoint.parse("http://#{uri_host}:#{port}")
|
|
14
21
|
end
|
|
15
22
|
|
|
16
23
|
Async::HTTP::Client.open(endpoint) do |client|
|
|
@@ -11,7 +11,7 @@ module Fluent::Plugin
|
|
|
11
11
|
|
|
12
12
|
helpers :thread, :http_server
|
|
13
13
|
|
|
14
|
-
config_param :bind, :string, default: '
|
|
14
|
+
config_param :bind, :string, default: '127.0.0.1'
|
|
15
15
|
config_param :port, :integer, default: 24231
|
|
16
16
|
config_param :metrics_path, :string, default: '/metrics'
|
|
17
17
|
config_param :aggregated_metrics_path, :string, default: '/aggregated_metrics'
|
|
@@ -36,10 +36,14 @@ module Fluent::Plugin
|
|
|
36
36
|
desc 'Content encoding of the exposed metrics, Currently supported encoding is identity, gzip. Ref: https://prometheus.io/docs/instrumenting/exposition_formats/#basic-info'
|
|
37
37
|
config_param :content_encoding, :enum, list: [:identity, :gzip], default: :identity
|
|
38
38
|
|
|
39
|
+
desc 'Suppress repeated error logs in a certain period of time (1h) or until message was changed'
|
|
40
|
+
config_param :ignore_error_log_interval, :time, default: 3600
|
|
41
|
+
|
|
39
42
|
def initialize
|
|
40
43
|
super
|
|
41
44
|
@registry = ::Prometheus::Client.registry
|
|
42
45
|
@secure = nil
|
|
46
|
+
@error_log_throttle = nil
|
|
43
47
|
end
|
|
44
48
|
|
|
45
49
|
def configure(conf)
|
|
@@ -58,6 +62,8 @@ module Fluent::Plugin
|
|
|
58
62
|
|
|
59
63
|
@base_port = @port
|
|
60
64
|
@port += fluentd_worker_id
|
|
65
|
+
|
|
66
|
+
@error_log_throttle = Fluent::Plugin::Prometheus::LogThrottle.new(@ignore_error_log_interval)
|
|
61
67
|
end
|
|
62
68
|
|
|
63
69
|
def multi_workers_ready?
|
|
@@ -67,12 +73,25 @@ module Fluent::Plugin
|
|
|
67
73
|
def start
|
|
68
74
|
super
|
|
69
75
|
|
|
76
|
+
# Normalize bind address: strip brackets if present (for consistency)
|
|
77
|
+
# Brackets are only for URI formatting, not for socket binding
|
|
78
|
+
@bind = @bind[1..-2] if @bind.start_with?('[') && @bind.end_with?(']')
|
|
79
|
+
|
|
70
80
|
scheme = @secure ? 'https' : 'http'
|
|
71
|
-
|
|
81
|
+
# Format bind address properly for URLs (add brackets for IPv6)
|
|
82
|
+
bind_display = @bind.include?(':') ? "[#{@bind}]" : @bind
|
|
83
|
+
log.debug "listening prometheus http server on #{scheme}://#{bind_display}:#{@port}/#{@metrics_path} for worker#{fluentd_worker_id}"
|
|
72
84
|
|
|
73
85
|
proto = @secure ? :tls : :tcp
|
|
74
86
|
|
|
75
|
-
|
|
87
|
+
# IPv6 + TLS combination is not currently supported
|
|
88
|
+
if @bind.include?(':') && @secure
|
|
89
|
+
raise Fluent::ConfigError, 'IPv6 with <transport tls> is not currently supported. Use bind 0.0.0.0 with TLS, or bind ::1 without TLS.'
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Use webrick for IPv6 or SSL extra_conf
|
|
93
|
+
# The http_server helper has issues with IPv6 URI construction
|
|
94
|
+
if (@ssl && @ssl['enable'] && @ssl['extra_conf']) || @bind.include?(':')
|
|
76
95
|
start_webrick
|
|
77
96
|
return
|
|
78
97
|
end
|
|
@@ -110,6 +129,8 @@ module Fluent::Plugin
|
|
|
110
129
|
ssl_config
|
|
111
130
|
end
|
|
112
131
|
|
|
132
|
+
# Use raw bind address for socket binding (no brackets)
|
|
133
|
+
# Brackets are only for URL/URI formatting, not for bind()
|
|
113
134
|
http_server_create_http_server(:in_prometheus_server, addr: @bind, port: @port, logger: log, proto: proto, tls_opts: tls_opt) do |server|
|
|
114
135
|
server.get(@metrics_path) { |_req| all_metrics }
|
|
115
136
|
server.get(@aggregated_metrics_path) { |_req| all_workers_metrics }
|
|
@@ -127,6 +148,7 @@ module Fluent::Plugin
|
|
|
127
148
|
private
|
|
128
149
|
|
|
129
150
|
# For compatiblity because http helper can't support extra_conf option
|
|
151
|
+
# Also used for IPv6 addresses since http helper has IPv6 URI issues
|
|
130
152
|
def start_webrick
|
|
131
153
|
require 'webrick/https'
|
|
132
154
|
require 'webrick'
|
|
@@ -138,28 +160,32 @@ module Fluent::Plugin
|
|
|
138
160
|
Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
|
|
139
161
|
AccessLog: [],
|
|
140
162
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
163
|
+
|
|
164
|
+
# Configure SSL if enabled
|
|
165
|
+
if @ssl && @ssl['enable']
|
|
166
|
+
if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
|
|
167
|
+
raise RuntimeError.new("certificate_path and private_key_path most both be defined")
|
|
168
|
+
end
|
|
144
169
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
170
|
+
ssl_config = {
|
|
171
|
+
SSLEnable: true,
|
|
172
|
+
SSLCertName: [['CN', 'nobody'], ['DC', 'example']]
|
|
173
|
+
}
|
|
149
174
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
175
|
+
if @ssl['certificate_path']
|
|
176
|
+
cert = OpenSSL::X509::Certificate.new(File.read(@ssl['certificate_path']))
|
|
177
|
+
ssl_config[:SSLCertificate] = cert
|
|
178
|
+
end
|
|
154
179
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
180
|
+
if @ssl['private_key_path']
|
|
181
|
+
key = OpenSSL::PKey.read(@ssl['private_key_path'])
|
|
182
|
+
ssl_config[:SSLPrivateKey] = key
|
|
183
|
+
end
|
|
159
184
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
185
|
+
ssl_config[:SSLCACertificateFile] = @ssl['ca_path'] if @ssl['ca_path']
|
|
186
|
+
ssl_config = ssl_config.merge(@ssl['extra_conf']) if @ssl['extra_conf']
|
|
187
|
+
config = ssl_config.merge(config)
|
|
188
|
+
end
|
|
163
189
|
|
|
164
190
|
@log.on_debug do
|
|
165
191
|
@log.debug("WEBrick conf: #{config}")
|
|
@@ -190,7 +216,8 @@ module Fluent::Plugin
|
|
|
190
216
|
def all_metrics
|
|
191
217
|
response(::Prometheus::Client::Formats::Text.marshal(@registry))
|
|
192
218
|
rescue => e
|
|
193
|
-
|
|
219
|
+
log_error_throttled(:metrics, "in_prometheus: failed to render metrics", error: e)
|
|
220
|
+
[500, { 'Content-Type' => 'text/plain' }, "in_prometheus server error: <#{e.class}>"]
|
|
194
221
|
end
|
|
195
222
|
|
|
196
223
|
def all_workers_metrics
|
|
@@ -203,11 +230,21 @@ module Fluent::Plugin
|
|
|
203
230
|
end
|
|
204
231
|
response(full_result.get_metrics)
|
|
205
232
|
rescue => e
|
|
206
|
-
|
|
233
|
+
log_error_throttled(:workers_metrics, "in_prometheus: failed to render workers metrics", error: e)
|
|
234
|
+
[500, { 'Content-Type' => 'text/plain' }, "in_prometheus server error: <#{e.class}>"]
|
|
207
235
|
end
|
|
208
236
|
|
|
209
237
|
def send_request_to_each_worker
|
|
210
|
-
|
|
238
|
+
# Convert bind address to localhost for inter-worker communication
|
|
239
|
+
# 0.0.0.0 and :: are not connectable, use localhost instead
|
|
240
|
+
bind = case @bind
|
|
241
|
+
when '0.0.0.0'
|
|
242
|
+
'127.0.0.1'
|
|
243
|
+
when '::'
|
|
244
|
+
'::1' # IPv6 localhost
|
|
245
|
+
else
|
|
246
|
+
@bind
|
|
247
|
+
end
|
|
211
248
|
[*(@base_port...(@base_port + @num_workers))].each do |worker_port|
|
|
212
249
|
do_request(host: bind, port: worker_port, secure: @secure) do |http|
|
|
213
250
|
yield(http.get(@metrics_path))
|
|
@@ -242,5 +279,17 @@ module Fluent::Plugin
|
|
|
242
279
|
end
|
|
243
280
|
[200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE, 'Content-Encoding' => @content_encoding.to_s }, body]
|
|
244
281
|
end
|
|
282
|
+
|
|
283
|
+
def log_error_throttled(scope, message, error:)
|
|
284
|
+
fingerprint = [error.class, error.message]
|
|
285
|
+
emit, suppressed = @error_log_throttle.check(scope, fingerprint)
|
|
286
|
+
return unless emit
|
|
287
|
+
|
|
288
|
+
if suppressed > 0
|
|
289
|
+
log.error message, error_class: error.class, error: error, suppressed_log_count: suppressed
|
|
290
|
+
else
|
|
291
|
+
log.error message, error_class: error.class, error: error
|
|
292
|
+
end
|
|
293
|
+
end
|
|
245
294
|
end
|
|
246
295
|
end
|
|
@@ -7,6 +7,8 @@ module Fluent::Plugin
|
|
|
7
7
|
include Fluent::Plugin::PrometheusLabelParser
|
|
8
8
|
include Fluent::Plugin::Prometheus
|
|
9
9
|
|
|
10
|
+
helpers :event_emitter
|
|
11
|
+
|
|
10
12
|
def initialize
|
|
11
13
|
super
|
|
12
14
|
@registry = ::Prometheus::Client.registry
|
|
@@ -19,7 +21,7 @@ module Fluent::Plugin
|
|
|
19
21
|
def configure(conf)
|
|
20
22
|
super
|
|
21
23
|
labels = parse_labels_elements(conf)
|
|
22
|
-
@metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
|
|
24
|
+
@metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels, metric_options)
|
|
23
25
|
end
|
|
24
26
|
|
|
25
27
|
def process(tag, es)
|