fluent-plugin-prometheus 2.2.1 → 2.3.0

data/lib/fluent/plugin/prometheus/log_throttle.rb

@@ -0,0 +1,45 @@
+require 'fluent/clock'
+
+module Fluent
+  module Plugin
+    module Prometheus
+      # Suppresses the repeated log for the same key within the interval.
+      # in_prometheus and filter/out_prometheus use it, each with its own
+      # instance. The key decides what is throttled: an error scope or a
+      # metric. When a fingerprint is given, a log whose fingerprint differs
+      # from the last one is not suppressed.
+      class LogThrottle
+        Entry = Struct.new(:time, :fingerprint, :suppressed)
+
+        def initialize(interval)
+          @interval = interval
+          @mutex = Mutex.new
+          # one entry per key, so this does not grow without a limit
+          @entries = {}
+        end
+
+        # Returns [emit, suppressed_count]. emit is true for the first log of a
+        # key, for a new fingerprint, and after the interval has passed.
+        # suppressed_count is how many logs were suppressed since the last one
+        # was emitted. Without a fingerprint, a key is throttled by the
+        # interval alone.
+        def check(key, fingerprint = nil)
+          return [true, 0] if @interval <= 0
+
+          @mutex.synchronize do
+            now = Fluent::Clock.now
+            last = @entries[key]
+            if last.nil? || last.fingerprint != fingerprint || (now - last.time) >= @interval
+              suppressed = (last && last.fingerprint == fingerprint) ? last.suppressed : 0
+              @entries[key] = Entry.new(now, fingerprint, 0)
+              [true, suppressed]
+            else
+              last.suppressed += 1
+              [false, 0]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/prometheus/placeholder_expander.rb

@@ -2,6 +2,10 @@ module Fluent
   module Plugin
     module Prometheus
       class ExpandBuilder
+        # ${tag}, ${tag_parts[...]}, ${tag_prefix[...]} and ${tag_suffix[...]}
+        # must be built from the tag only.
+        TAG_DERIVED_PLACEHOLDER = /\A\$\{tag(_parts|_prefix|_suffix)?(\[[^\]]*\])?\}\z/.freeze
+
         def self.build(placeholder, log:)
           new(log: log).build(placeholder)
         end
@@ -12,6 +16,7 @@ module Fluent
 
         def build(placeholder_values)
           placeholders = {}
+          tag_placeholders = {}
           placeholder_values.each do |key, value|
             case value
             when Array
@@ -26,13 +31,22 @@ module Fluent
               end
             else
               if key == 'tag'
-                placeholders.merge!(build_tag(value))
+                tag_placeholders = build_tag(value)
               else
                 placeholders["${#{key}}"] = value
               end
             end
           end
 
+          # A record may have an array named "tag_parts", or a key named
+          # "tag_parts[0]". Both make the same placeholder as the tag does.
+          # merge! is not enough here, because a record can also use an index
+          # which build_tag does not make, such as ${tag_parts[3]} for a tag
+          # of 3 parts, or ${tag_prefix[-1]}. So remove them first, then such
+          # a placeholder stays unknown.
+          placeholders.delete_if { |k, _| TAG_DERIVED_PLACEHOLDER.match?(k) }
+          placeholders.merge!(tag_placeholders)
+
           Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander.new(@log, placeholders)
         end
 

data/lib/fluent/plugin/prometheus.rb

@@ -1,5 +1,6 @@
 require 'prometheus/client'
 require 'prometheus/client/formats/text'
+require 'fluent/plugin/prometheus/log_throttle'
 require 'fluent/plugin/prometheus/placeholder_expander'
 
 module Fluent
@@ -31,6 +32,101 @@ module Fluent
 
     module Prometheus
       class AlreadyRegisteredError < StandardError; end
+      class LabelSetLimitError < StandardError; end
+
+      # 0 or less means unlimited. The limit is unlimited by default, because
+      # enabling it changes the existing metrics silently: dropping a label set
+      # loses the record without any way to recover it. An operator who needs
+      # to bound the cardinality has to opt in explicitly.
+      DEFAULT_MAX_SERIES_PER_METRIC = 0
+      DEFAULT_IGNORE_ERROR_LOG_INTERVAL = 3600
+
+      # the drops are visible in Prometheus, not only in the Fluentd log
+      DROPPED_LABEL_SETS_METRIC_NAME = :fluentd_prometheus_dropped_label_sets_total
+      DROPPED_LABEL_SETS_METRIC_DESC = 'The total number of records dropped because the metric reached max_series_per_metric.'
+
+      def self.included(klass)
+        klass.class_eval do
+          desc 'The maximum number of label sets a metric can hold. Exceeding label sets are dropped. 0 (default) means unlimited.'
+          config_param :max_series_per_metric, :integer, default: DEFAULT_MAX_SERIES_PER_METRIC
+          desc 'The interval to suppress the repeated warning about the drops.'
+          config_param :ignore_error_log_interval, :time, default: DEFAULT_IGNORE_ERROR_LOG_INTERVAL
+        end
+      end
+
+      # The label sets a client metric holds. The client registry keys its
+      # metrics by name alone, so every <metric> section with the same name
+      # instruments the same client metric and shares this set, instead of
+      # holding max_series_per_metric label sets of its own.
+      class SeriesSet
+        # The set is kept on the client metric, so that it is found again by
+        # every section. The registry does not drop its metrics, so the set is
+        # still there after a reload.
+        IVAR = :@fluent_plugin_prometheus_series_set
+
+        # Sections are built at configuration time, which is single threaded,
+        # so this needs no lock.
+        def self.of(client_metric)
+          client_metric.instance_variable_get(IVAR) ||
+            client_metric.instance_variable_set(IVAR, new)
+        end
+
+        def initialize
+          @series = {}
+          @mutex = Mutex.new
+        end
+
+        # Only <initlabels> come from the configuration. Counting a label set
+        # a record brought would refuse a good configuration after a reload.
+        def initial_size
+          @mutex.synchronize { @series.count { |_, state| state == :initial } }
+        end
+
+        # Checking the limit and taking the slot happen under the same lock, so
+        # that concurrent calls cannot both take the last one. The slot stays
+        # :reserved until the instrumentation confirms it, so that a failing
+        # call can tell an in-flight reservation from a series the client holds.
+        def reserve(label, limit, name)
+          @mutex.synchronize do
+            next false if @series.key?(label)
+
+            if @series.size >= limit
+              raise LabelSetLimitError, "#{name} reached max_series_per_metric (#{limit})"
+            end
+
+            @series[label] = :reserved
+            next true
+          end
+        end
+
+        # Marks a label set as established, once the client actually holds it.
+        # The limit is not checked on purpose: the series exists on the client
+        # side already, so it has to be accounted for even when a concurrent
+        # failure gave the reservation back in the meantime.
+        def confirm(label)
+          @mutex.synchronize do
+            # #initial_size has to keep counting it, so a record on it does not
+            # change where it came from
+            next if @series[label] == :initial
+
+            @series[label] = :confirmed
+          end
+        end
+
+        # The limit is not checked here either: a section which cannot hold
+        # these label sets is refused when the configuration is read.
+        def confirm_initial(label)
+          @mutex.synchronize { @series[label] = :initial }
+        end
+
+        # Gives a reserved slot back when the instrumentation failed, so that a
+        # label set the client does not hold does not consume the limit. One
+        # which a concurrent call confirmed in the meantime is kept: the client
+        # holds that series.
+        def release(label)
+          @mutex.synchronize { @series.delete(label) if @series[label] == :reserved }
+        end
+      end
 
       def self.parse_labels_elements(conf)
         labels = conf.elements.select { |e| e.name == 'labels' }
@@ -119,7 +215,7 @@ module Fluent
         base_initlabels
       end
 
-      def self.parse_metrics_elements(conf, registry, labels = {})
+      def self.parse_metrics_elements(conf, registry, labels = {}, opts = {})
         metrics = []
         conf.elements.select { |element|
           element.name == 'metric'
@@ -130,17 +226,25 @@ module Fluent
           end
           case element['type']
           when 'summary'
-            metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels)
+            metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels, opts)
           when 'gauge'
-            metrics << Fluent::Plugin::Prometheus::Gauge.new(element, registry, labels)
+            metrics << Fluent::Plugin::Prometheus::Gauge.new(element, registry, labels, opts)
           when 'counter'
-            metrics << Fluent::Plugin::Prometheus::Counter.new(element, registry, labels)
+            metrics << Fluent::Plugin::Prometheus::Counter.new(element, registry, labels, opts)
           when 'histogram'
-            metrics << Fluent::Plugin::Prometheus::Histogram.new(element, registry, labels)
+            metrics << Fluent::Plugin::Prometheus::Histogram.new(element, registry, labels, opts)
           else
             raise ConfigError, "type option must be 'counter', 'gauge', 'summary' or 'histogram'"
           end
         }
+
+        # <metric> sections with the same name share one client
+        # metric. All of their <initlabels> label sets are known only
+        # after every section is built, so the check runs here and not
+        # in the constructor not to depend on the order of the
+        # sections.
+        metrics.each(&:check_series_limit!)
+
         metrics
       end
 
@@ -165,6 +269,49 @@ module Fluent
         @placeholder_values = {}
         @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
         @hostname = Socket.gethostname
+        @label_set_limit_log_throttle = Fluent::Plugin::Prometheus::LogThrottle.new(@ignore_error_log_interval)
+        @dropped_label_sets_counter = nil
+      end
+
+      def metric_options
+        {
+          max_series_per_metric: @max_series_per_metric,
+        }
+      end
+
+      # Registered on the first occurrence only, so that a plugin which never
+      # drops anything does not expose a counter which stays 0 forever. Its
+      # labels come from the configuration, so they cannot blow up on their own.
+      def limit_counter(name, docstring, labels)
+        @registry.counter(name, docstring: docstring, labels: labels)
+      rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+        # another plugin instance shares the registry and registered it first
+        Fluent::Plugin::Prometheus::Metric.get(@registry, name, :counter, docstring)
+      end
+
+      def dropped_label_sets_counter
+        @dropped_label_sets_counter ||=
+          limit_counter(DROPPED_LABEL_SETS_METRIC_NAME, DROPPED_LABEL_SETS_METRIC_DESC, [:name])
+      end
+
+      def warn_label_set_limit(metric)
+        # the drop is always counted, while the log below is throttled
+        dropped_label_sets_counter.increment(labels: { name: metric.name.to_s })
+
+        warn_throttled(@label_set_limit_log_throttle, metric.name,
+                       "prometheus: dropped a label set because the metric reached max_series_per_metric.",
+                       name: metric.name, max_series_per_metric: metric.max_series_per_metric)
+      end
+
+      # The counter above is never throttled, only the log which comes with it:
+      # one line per record would flood the Fluentd log, and the count is in
+      # Prometheus already.
+      def warn_throttled(throttle, key, message, **details)
+        emit, suppressed = throttle.check(key)
+        return unless emit
+
+        details = details.merge(suppressed_log_count: suppressed) if suppressed > 0
+        log.warn(message, details)
       end
 
       def instrument_single(tag, time, record, metrics)
@@ -180,6 +327,9 @@ module Fluent
         metrics.each do |metric|
           begin
             metric.instrument(record, expander)
+          rescue Fluent::Plugin::Prometheus::LabelSetLimitError
+            # dropping the label set is intended, so it is not an error event
+            warn_label_set_limit(metric)
           rescue => e
             log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
             router.emit_error_event(tag, time, record, e)
@@ -201,6 +351,9 @@ module Fluent
           metrics.each do |metric|
             begin
               metric.instrument(record, expander)
+            rescue Fluent::Plugin::Prometheus::LabelSetLimitError
+              # dropping the label set is intended, so it is not an error event
+              warn_label_set_limit(metric)
             rescue => e
               log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
               router.emit_error_event(tag, time, record, e)
@@ -214,8 +367,9 @@ module Fluent
         attr_reader :name
         attr_reader :key
         attr_reader :desc
+        attr_reader :max_series_per_metric
 
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           ['name', 'desc'].each do |key|
             if element[key].nil?
               raise ConfigError, "metric requires '#{key}' option"
@@ -230,6 +384,10 @@ module Fluent
           @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(element)
           @base_labels = labels.merge(@base_labels)
 
+          # <metric> overrides the limit given by the plugin
+          @max_series_per_metric = metric_limit(element, 'max_series_per_metric',
+                                                opts.fetch(:max_series_per_metric, DEFAULT_MAX_SERIES_PER_METRIC))
+
           if @initialized
             @base_initlabels = Fluent::Plugin::Prometheus.parse_initlabels_elements(element, @base_labels)
           end
@@ -254,12 +412,31 @@ module Fluent
             if v.is_a?(String)
               label[k] = expander.expand(v)
             else
-              label[k] = v.call(record)
+              label[k] = normalize_label_value(v.call(record))
             end
           end
           label
         end
 
+        # Instruments a record through the given block and keeps its label set
+        # as a series once the client holds it. The slot is taken before
+        # instrumenting and given back when the client refused the record, so
+        # that a label set the client does not hold does not exhaust
+        # max_series_per_metric.
+        def with_label_set(record, expander)
+          label = labels(record, expander)
+          reserved = reserve_series!(label)
+          begin
+            result = yield label
+          rescue
+            # a call which joined another's reservation has nothing to release
+            release_series(label) if reserved
+            raise
+          end
+          confirm_series(label)
+          result
+        end
+
         def self.get(registry, name, type, docstring)
           metric = registry.get(name)
 
@@ -273,10 +450,101 @@ module Fluent
 
           metric
         end
+
+        # <initlabels> label sets go to the client as soon as the
+        # configuration is read. Every <metric> section with the same name
+        # shares them, so reject a section which has no room for the label
+        # sets.
+        def check_series_limit!
+          return if @max_series_per_metric <= 0
+          # two <initlabels> blocks with the same values make one label set
+          held = @series_set.initial_size
+          return if held <= @max_series_per_metric
+
+          raise ConfigError, "metric #{@name} already holds #{held} label sets from <initlabels>, " \
+                             "shared by every <metric> section with this name, " \
+                             "but max_series_per_metric is #{@max_series_per_metric} in this section: " \
+                             "the limit is already exceeded before any record arrives"
+        end
+
+        private
+
+        # The client refuses a value which is not a number, but it is only
+        # called once the label set has been reserved, and Summary refuses it
+        # only once it has already incremented its count: releasing the slot
+        # does not take that half instrumented series back from the client.
+        # Refuse the value before it reaches either.
+        def validate_value!(value)
+          return if value.is_a?(Numeric)
+
+          raise ArgumentError, 'value must be a number'
+        end
+
+        def metric_limit(element, name, default)
+          return default unless element.has_key?(name)
+
+          begin
+            # base 10 explicitly, so that a value like 08 is not an octal
+            Integer(element[name], 10)
+          rescue ArgumentError, TypeError
+            raise ConfigError, "#{name} in <metric> must be an integer: #{element[name]}"
+          end
+        end
+
+        # Called by a subclass, once it has its client metric.
+        def bind_series_set(client_metric)
+          @series_set = SeriesSet.of(client_metric)
+
+          return unless @initialized
+
+          # The client gets them even when this section has no limit, so they
+          # take their slots in both cases. A section with the same name shares
+          # this set and has to see them. Their number is fixed by the
+          # configuration. Counting them cannot leak like the label sets that
+          # records bring.
+          @base_initlabels.each do |initlabels|
+            @series_set.confirm_initial(normalize_label_set(initlabels))
+          end
+        end
+
+        # The SeriesSet keys a label set by its values, so the same value has to
+        # look the same whether a RecordAccessor or <initlabels> produced it.
+        def normalize_label_value(value)
+          value.is_a?(String) ? value : value.to_s
+        end
+
+        def normalize_label_set(label)
+          label.each_with_object({}) do |(k, v), normalized|
+            normalized[k] = normalize_label_value(v)
+          end
+        end
+
+        # Returns true when this call took the slot, which tells #with_label_set
+        # whether it has something to give back on failure.
+        def reserve_series!(label)
+          # If the limit is off, a label set from a record is not counted.
+          # If it is counted, the set grows with every new label set and
+          # uses too much memory.
+          return false if @max_series_per_metric <= 0
+
+          @series_set.reserve(label, @max_series_per_metric, @name)
+        end
+
+        def confirm_series(label)
+          return if @max_series_per_metric <= 0
+
+          @series_set.confirm(label)
+        end
+
+        def release_series(label)
+          return if @max_series_per_metric <= 0
+
+          @series_set.release(label)
+        end
       end
 
       class Gauge < Metric
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           super
           if @key.nil?
             raise ConfigError, "gauge metric requires 'key' option"
@@ -287,6 +555,7 @@ module Fluent
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
           end
+          bind_series_set(@gauge)
 
           if @initialized
             Fluent::Plugin::Prometheus::Metric.init_label_set(@gauge, @base_initlabels, @base_labels)
@@ -300,19 +569,23 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @gauge.set(value, labels: labels(record, expander))
+            validate_value!(value)
+            with_label_set(record, expander) do |label|
+              @gauge.set(value, labels: label)
+            end
           end
         end
       end
 
       class Counter < Metric
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           super
           begin
             @counter = registry.counter(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
           end
+          bind_series_set(@counter)
 
           if @initialized
             Fluent::Plugin::Prometheus::Metric.init_label_set(@counter, @base_initlabels, @base_labels)
@@ -332,12 +605,15 @@ module Fluent
           # ignore if record value is nil
           return if value.nil?
 
-          @counter.increment(by: value, labels: labels(record, expander))
+          validate_value!(value)
+          with_label_set(record, expander) do |label|
+            @counter.increment(by: value, labels: label)
+          end
         end
       end
 
       class Summary < Metric
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           super
           if @key.nil?
             raise ConfigError, "summary metric requires 'key' option"
@@ -348,6 +624,7 @@ module Fluent
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
           end
+          bind_series_set(@summary)
 
           if @initialized
             Fluent::Plugin::Prometheus::Metric.init_label_set(@summary, @base_initlabels, @base_labels)
@@ -361,13 +638,16 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @summary.observe(value, labels: labels(record, expander))
+            validate_value!(value)
+            with_label_set(record, expander) do |label|
+              @summary.observe(value, labels: label)
+            end
           end
         end
       end
 
       class Histogram < Metric
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           super
           if @key.nil?
             raise ConfigError, "histogram metric requires 'key' option"
@@ -385,6 +665,7 @@ module Fluent
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
           end
+          bind_series_set(@histogram)
 
           if @initialized
             Fluent::Plugin::Prometheus::Metric.init_label_set(@histogram, @base_initlabels, @base_labels)
@@ -398,7 +679,10 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @histogram.observe(value, labels: labels(record, expander))
+            validate_value!(value)
+            with_label_set(record, expander) do |label|
+              @histogram.observe(value, labels: label)
+            end
           end
         end
       end