fluent-plugin-prometheus 2.2.2 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2c3701c496726e677bf8182e4b549d493648b1f639930a7508ead0a07611ed6
-  data.tar.gz: c7f62e9dfc1a37d8089e07b41a6704e92fc365c0811b960bb628ffafe01134eb
+  metadata.gz: c10a2e202128626380ce6fd1003ca82baa6f1ae55bd41642b8f6c0bc44670abc
+  data.tar.gz: 34a77887e5da8f4bb4dc08018478a1592d7dec51411e338b4773ab41ec281e06
 SHA512:
-  metadata.gz: 53f721520289f3ca5f8b7b47fe4c5daf7829fae4e22f4bd6145dffaa1d4d173fc81909576067136299b0e256a55d8b961d432ac15aa16074f73dfb6003b9fbfb
-  data.tar.gz: 5cdf65cb05bf988df0cfc255a2af2e0f5b8223e5b54c56d4bbabe081827c111e7e210d45d2256d6f3af9cf4dab9154d2b6aba85c7226a87ebb96b420e9230402
+  metadata.gz: 131d844af8a45deb00d2ee53dba0928534a104193aec9a26f5d0760600ffefa2c11cb284fa6e49ae81f1cb4e17520db1b62a2a1acc6af295b964bdfa4c35aa5c
+  data.tar.gz: b9da9de5e2019b290ccd45bfca81ad59cdf7942109e961f8919d1121596e00bc63661cf76825251e4f0a9b796929830a6a7a7dade75ddcc00785f0c901317476

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,72 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please check the [README](https://github.com/fluent/fluent-plugin-prometheus/blob/master/README.md) first. To help us investigate the problem, please provide the following information.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent/Fluent Package version:
+        - fluent-plugin-prometheus version:
+        - prometheus-client version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://discuss.fluentd.org/
+    about: I have questions about fluent-plugin-prometheus. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: "waiting-for-triage"
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Please check the [README](https://github.com/fluent/fluent-plugin-prometheus/blob/master/README.md) first. To help us understand the request, please provide the following information.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false

data/.github/dependabot.yml

@@ -3,4 +3,21 @@ updates:
   - package-ecosystem: 'github-actions'
     directory: '/'
     schedule:
-      interval: 'weekly'
+      interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/add-to-project.yml

@@ -0,0 +1,24 @@
+name: Add bugs to fluent project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ secrets.PROJECT_APP_ID }}
+          private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
+          owner: fluent
+      - uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
+        with:
+          project-url: https://github.com/orgs/fluent/projects/4
+          github-token: ${{ steps.generate_token.outputs.token }}
+          labeled: waiting-for-triage

data/.github/workflows/linux.yml

@@ -27,8 +27,8 @@ jobs:
         experimental: [false]
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v6
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: unit testing

data/ChangeLog

@@ -1,3 +1,16 @@
+Release 2.3.0 - 2026/09/02
+
+	* in_prometheus: Change default bind address to 127.0.0.1. (#258)
+          This is incompatible behavior change.
+	* in_prometheus: Do not disclosure error details for client (#259)
+	* in_prometheus: Add `ignore_error_log_interval` parameter to suppress repeated error logs (#259,#264)
+          This introduces the log throttling feature.
+	* filter_prometheus: Do not let placeholders built from records override the tag ones (#262)
+	* out_prometheus: Fix a bug that a failing record never reached to @ERROR (#263)
+	* Support `max_series_per_metric` to limit label sets cardinarity. (#265)
+	* Refuse wrong <initlabels> which cannot fit into `max_series_per_metric` (#266,#267)
+          This blocks `max_series_per_metric` mis-configuration on startup.
+
 Release 2.2.2 - 2026/03/27
 
 	* in_prometheus: fix: Add IPv6 support for bind parameter (#240)

data/README.md

@@ -59,11 +59,12 @@ With following configuration, you can access http://localhost:24231/metrics on a
 
 More configuration parameters:
 
-- `bind`: binding interface (default: '0.0.0.0')
+- `bind`: binding interface (default: '127.0.0.1')
 - `port`: listen port (default: 24231)
 - `metrics_path`: metrics HTTP endpoint (default: /metrics)
 - `aggregated_metrics_path`: metrics HTTP endpoint (default: /aggregated_metrics)
 - `content_encoding`: encoding format for the exposed metrics (default: identity). Supported formats are {identity, gzip}
+- `ignore_error_log_interval`: Suppress repeated error logs in a certain period of time or until message was changed (default: 1h)
 
 When using multiple workers, each worker binds to port + `fluent_worker_id`.
 To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
@@ -265,6 +266,132 @@ You can access nested keys in records via dot or bracket notation (https://docs.
 
 See Supported Metric Type and Labels for more configuration parameters.
 
+#### Limiting label expansion
+
+Label values come from records, so a metric grows unboundedly when a label is
+bound to a field with many distinct values. Both plugins can bound it with
+`max_series_per_metric`: once a metric holds that many label sets, a record
+which brings a new one is dropped, while the label sets already known keep
+being instrumented.
+
+|parameter|description|default|
+|---|---|---|
+|max_series_per_metric|The maximum number of label sets a metric can hold. `0` means unlimited.|0|
+|ignore_error_log_interval|The interval in seconds to suppress the repeated warning about the drops. `0` logs every occurrence.|3600|
+
+**The limit is disabled by default and must be enabled explicitly**, since a
+dropped record is lost and cannot be recovered. A `<metric>` section overrides
+the value given to the plugin, so that a metric which expands faster than the
+others is bound on its own, while one whose labels are known to be bounded
+stays unlimited with `0`:
+
+```
+<filter message>
+  @type prometheus
+  max_series_per_metric 1000
+  <metric>
+    name message_foo_counter
+    type counter
+    desc The total number of foo in message.
+    key foo
+    max_series_per_metric 10
+    <labels>
+      path $.kubernetes.pod_name
+    </labels>
+  </metric>
+</filter>
+```
+
+The label sets are counted per metric name, not per `<metric>` section: sections
+with the same `name`, in one plugin or in two, share one count. Each of them
+refuses a new label set once that shared count reaches its own limit, so a
+section which stays at `0` adds label sets without counting them. Its
+`<initlabels>` are counted anyway. They come from the configuration and their
+number is fixed. The metric holds them in every section.
+
+The count is per worker process as well, since a worker has its own registry and
+exposes the metrics it holds itself. With `workers N`, a metric can hold up to N
+times `max_series_per_metric` label sets in total, so divide the number of label
+sets the metric may reach by the number of workers.
+
+A record which fails to be instrumented, for example when the value of `key` is
+not a number, does not consume the limit. A pre-initialized label set
+(`initialized` and `<initlabels>`) consumes it from the start, since the metric
+holds it before any record arrives.
+
+A `<metric>` section is refused at startup with a configuration error when the
+`<initlabels>` label sets already fill its limit. The limit is exceeded before
+any record arrives, and the section can never take a new label set.
+
+These label sets are shared by every section with the same `name`. A section can
+be refused because of another section:
+
+```
+<metric>
+  name shared
+  type counter
+  desc Something foo.
+  max_series_per_metric 2
+  initialized true
+  <labels>
+    path $.path
+  </labels>
+  <initlabels>
+    path /a
+  </initlabels>
+  <initlabels>
+    path /b
+  </initlabels>
+</metric>
+<metric>
+  name shared               # the same name, so the 2 label sets count here too
+  type counter
+  desc Something foo.
+  max_series_per_metric 1   # refused: 2 label sets do not fit into 1
+  <labels>
+    path $.path
+  </labels>
+</metric>
+```
+
+The second section is refused even though it declares no `<initlabels>` of its
+own. The same happens when the first section sets `max_series_per_metric 0`.
+The metric holds its label sets in both cases. A limit equal to their number is
+fine because all label sets of the metric are known in advance.
+
+The check runs after every `<metric>` section of a plugin is read. It does not
+depend on the order of the sections. Sections that share a `name` across two
+plugins are only checked against the sections read before them.
+
+The check runs again when Fluentd reloads the configuration. It counts only the
+label sets from `<initlabels>`. It does not count the label sets that records
+brought, so the metric does not make the reload fail with the label sets it took
+while it was running.
+
+A reload does not clear the registry. The metric still holds the label sets from
+the `<initlabels>` of the old configuration, and the check counts them too. So
+it can refuse a new configuration which has fewer `<initlabels>` than the old
+one. Restart the worker to drop them.
+
+##### Observing what the limit leaves out
+
+A dropped label set is not routed to `@ERROR`, because it is what the
+configuration asks for. It is reported in two ways instead:
+
+* a warning in the Fluentd log, throttled per metric: it is suppressed for
+  `ignore_error_log_interval` seconds and reports how many warnings were
+  suppressed in the meantime.
+* `fluentd_prometheus_dropped_label_sets_total{name}`, which counts the records
+  the metric `name` did not instrument. It is registered on the first drop, so
+  it does not show up as long as nothing is dropped, and its label comes from
+  the configuration and not from a record, so it cannot expand on its own.
+
+Alert on the counter to notice that a metric is losing records:
+
+```
+rate(fluentd_prometheus_dropped_label_sets_total[5m]) > 0
+```
+
 ## Supported Metric Types
 
 For details of each metric type, see [Prometheus documentation](http://prometheus.io/docs/concepts/metric_types/). Also see [metric name guide](http://prometheus.io/docs/practices/naming/).

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "2.2.2"
+  spec.version       = "2.3.0"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}

data/lib/fluent/plugin/filter_prometheus.rb

@@ -19,7 +19,7 @@ module Fluent::Plugin
     def configure(conf)
       super
       labels = parse_labels_elements(conf)
-      @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
+      @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels, metric_options)
     end
 
     def filter(tag, time, record)

data/lib/fluent/plugin/in_prometheus.rb

@@ -11,7 +11,7 @@ module Fluent::Plugin
 
     helpers :thread, :http_server
 
-    config_param :bind, :string, default: '0.0.0.0'
+    config_param :bind, :string, default: '127.0.0.1'
     config_param :port, :integer, default: 24231
     config_param :metrics_path, :string, default: '/metrics'
     config_param :aggregated_metrics_path, :string, default: '/aggregated_metrics'
@@ -36,10 +36,14 @@ module Fluent::Plugin
     desc 'Content encoding of the exposed metrics, Currently supported encoding is identity, gzip. Ref: https://prometheus.io/docs/instrumenting/exposition_formats/#basic-info'
     config_param :content_encoding, :enum, list: [:identity, :gzip], default: :identity
 
+    desc 'Suppress repeated error logs in a certain period of time (1h) or until message was changed'
+    config_param :ignore_error_log_interval, :time, default: 3600
+
     def initialize
       super
       @registry = ::Prometheus::Client.registry
       @secure = nil
+      @error_log_throttle = nil
     end
 
     def configure(conf)
@@ -58,6 +62,8 @@ module Fluent::Plugin
 
       @base_port = @port
       @port += fluentd_worker_id
+
+      @error_log_throttle = Fluent::Plugin::Prometheus::LogThrottle.new(@ignore_error_log_interval)
     end
 
     def multi_workers_ready?
@@ -210,7 +216,8 @@ module Fluent::Plugin
     def all_metrics
       response(::Prometheus::Client::Formats::Text.marshal(@registry))
     rescue => e
-      [500, { 'Content-Type' => 'text/plain' }, e.to_s]
+      log_error_throttled(:metrics, "in_prometheus: failed to render metrics", error: e)
+      [500, { 'Content-Type' => 'text/plain' }, "in_prometheus server error: <#{e.class}>"]
     end
 
     def all_workers_metrics
@@ -223,7 +230,8 @@ module Fluent::Plugin
       end
       response(full_result.get_metrics)
     rescue => e
-      [500, { 'Content-Type' => 'text/plain' }, e.to_s]
+      log_error_throttled(:workers_metrics, "in_prometheus: failed to render workers metrics", error: e)
+      [500, { 'Content-Type' => 'text/plain' }, "in_prometheus server error: <#{e.class}>"]
     end
 
     def send_request_to_each_worker
@@ -271,5 +279,17 @@ module Fluent::Plugin
       end
       [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE, 'Content-Encoding' => @content_encoding.to_s }, body]
     end
+
+    def log_error_throttled(scope, message, error:)
+      fingerprint = [error.class, error.message]
+      emit, suppressed = @error_log_throttle.check(scope, fingerprint)
+      return unless emit
+
+      if suppressed > 0
+        log.error message, error_class: error.class, error: error, suppressed_log_count: suppressed
+      else
+        log.error message, error_class: error.class, error: error
+      end
+    end
   end
 end

data/lib/fluent/plugin/out_prometheus.rb

@@ -7,6 +7,8 @@ module Fluent::Plugin
     include Fluent::Plugin::PrometheusLabelParser
     include Fluent::Plugin::Prometheus
 
+    helpers :event_emitter
+
     def initialize
       super
       @registry = ::Prometheus::Client.registry
@@ -19,7 +21,7 @@ module Fluent::Plugin
     def configure(conf)
       super
       labels = parse_labels_elements(conf)
-      @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
+      @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels, metric_options)
     end
 
     def process(tag, es)

data/lib/fluent/plugin/prometheus/log_throttle.rb

@@ -0,0 +1,45 @@
+require 'fluent/clock'
+
+module Fluent
+  module Plugin
+    module Prometheus
+      # Suppresses the repeated log for the same key within the interval.
+      # in_prometheus and filter/out_prometheus use it, each with its own
+      # instance. The key decides what is throttled: an error scope or a
+      # metric. When a fingerprint is given, a log whose fingerprint differs
+      # from the last one is not suppressed.
+      class LogThrottle
+        Entry = Struct.new(:time, :fingerprint, :suppressed)
+
+        def initialize(interval)
+          @interval = interval
+          @mutex = Mutex.new
+          # one entry per key, so this does not grow without a limit
+          @entries = {}
+        end
+
+        # Returns [emit, suppressed_count]. emit is true for the first log of a
+        # key, for a new fingerprint, and after the interval has passed.
+        # suppressed_count is how many logs were suppressed since the last one
+        # was emitted. Without a fingerprint, a key is throttled by the
+        # interval alone.
+        def check(key, fingerprint = nil)
+          return [true, 0] if @interval <= 0
+
+          @mutex.synchronize do
+            now = Fluent::Clock.now
+            last = @entries[key]
+            if last.nil? || last.fingerprint != fingerprint || (now - last.time) >= @interval
+              suppressed = (last && last.fingerprint == fingerprint) ? last.suppressed : 0
+              @entries[key] = Entry.new(now, fingerprint, 0)
+              [true, suppressed]
+            else
+              last.suppressed += 1
+              [false, 0]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/prometheus/placeholder_expander.rb

@@ -2,6 +2,10 @@ module Fluent
   module Plugin
     module Prometheus
       class ExpandBuilder
+        # ${tag}, ${tag_parts[...]}, ${tag_prefix[...]} and ${tag_suffix[...]}
+        # must be built from the tag only.
+        TAG_DERIVED_PLACEHOLDER = /\A\$\{tag(_parts|_prefix|_suffix)?(\[[^\]]*\])?\}\z/.freeze
+
         def self.build(placeholder, log:)
           new(log: log).build(placeholder)
         end
@@ -12,6 +16,7 @@ module Fluent
 
         def build(placeholder_values)
           placeholders = {}
+          tag_placeholders = {}
           placeholder_values.each do |key, value|
             case value
             when Array
@@ -26,13 +31,22 @@ module Fluent
               end
             else
               if key == 'tag'
-                placeholders.merge!(build_tag(value))
+                tag_placeholders = build_tag(value)
               else
                 placeholders["${#{key}}"] = value
               end
             end
           end
 
+          # A record may have an array named "tag_parts", or a key named
+          # "tag_parts[0]". Both make the same placeholder as the tag does.
+          # merge! is not enough here, because a record can also use an index
+          # which build_tag does not make, such as ${tag_parts[3]} for a tag
+          # of 3 parts, or ${tag_prefix[-1]}. So remove them first, then such
+          # a placeholder stays unknown.
+          placeholders.delete_if { |k, _| TAG_DERIVED_PLACEHOLDER.match?(k) }
+          placeholders.merge!(tag_placeholders)
+
           Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander.new(@log, placeholders)
         end