fluent-plugin-prometheus 2.2.2 → 2.3.0

data/lib/fluent/plugin/prometheus.rb

@@ -1,5 +1,6 @@
 require 'prometheus/client'
 require 'prometheus/client/formats/text'
+require 'fluent/plugin/prometheus/log_throttle'
 require 'fluent/plugin/prometheus/placeholder_expander'
 
 module Fluent
@@ -31,6 +32,101 @@ module Fluent
 
     module Prometheus
       class AlreadyRegisteredError < StandardError; end
+      class LabelSetLimitError < StandardError; end
+
+      # 0 or less means unlimited. The limit is unlimited by default, because
+      # enabling it changes the existing metrics silently: dropping a label set
+      # loses the record without any way to recover it. An operator who needs
+      # to bound the cardinality has to opt in explicitly.
+      DEFAULT_MAX_SERIES_PER_METRIC = 0
+      DEFAULT_IGNORE_ERROR_LOG_INTERVAL = 3600
+
+      # the drops are visible in Prometheus, not only in the Fluentd log
+      DROPPED_LABEL_SETS_METRIC_NAME = :fluentd_prometheus_dropped_label_sets_total
+      DROPPED_LABEL_SETS_METRIC_DESC = 'The total number of records dropped because the metric reached max_series_per_metric.'
+
+      def self.included(klass)
+        klass.class_eval do
+          desc 'The maximum number of label sets a metric can hold. Exceeding label sets are dropped. 0 (default) means unlimited.'
+          config_param :max_series_per_metric, :integer, default: DEFAULT_MAX_SERIES_PER_METRIC
+          desc 'The interval to suppress the repeated warning about the drops.'
+          config_param :ignore_error_log_interval, :time, default: DEFAULT_IGNORE_ERROR_LOG_INTERVAL
+        end
+      end
+
+      # The label sets a client metric holds. The client registry keys its
+      # metrics by name alone, so every <metric> section with the same name
+      # instruments the same client metric and shares this set, instead of
+      # holding max_series_per_metric label sets of its own.
+      class SeriesSet
+        # The set is kept on the client metric, so that it is found again by
+        # every section. The registry does not drop its metrics, so the set is
+        # still there after a reload.
+        IVAR = :@fluent_plugin_prometheus_series_set
+
+        # Sections are built at configuration time, which is single threaded,
+        # so this needs no lock.
+        def self.of(client_metric)
+          client_metric.instance_variable_get(IVAR) ||
+            client_metric.instance_variable_set(IVAR, new)
+        end
+
+        def initialize
+          @series = {}
+          @mutex = Mutex.new
+        end
+
+        # Only <initlabels> come from the configuration. Counting a label set
+        # a record brought would refuse a good configuration after a reload.
+        def initial_size
+          @mutex.synchronize { @series.count { |_, state| state == :initial } }
+        end
+
+        # Checking the limit and taking the slot happen under the same lock, so
+        # that concurrent calls cannot both take the last one. The slot stays
+        # :reserved until the instrumentation confirms it, so that a failing
+        # call can tell an in-flight reservation from a series the client holds.
+        def reserve(label, limit, name)
+          @mutex.synchronize do
+            next false if @series.key?(label)
+
+            if @series.size >= limit
+              raise LabelSetLimitError, "#{name} reached max_series_per_metric (#{limit})"
+            end
+
+            @series[label] = :reserved
+            next true
+          end
+        end
+
+        # Marks a label set as established, once the client actually holds it.
+        # The limit is not checked on purpose: the series exists on the client
+        # side already, so it has to be accounted for even when a concurrent
+        # failure gave the reservation back in the meantime.
+        def confirm(label)
+          @mutex.synchronize do
+            # #initial_size has to keep counting it, so a record on it does not
+            # change where it came from
+            next if @series[label] == :initial
+
+            @series[label] = :confirmed
+          end
+        end
+
+        # The limit is not checked here either: a section which cannot hold
+        # these label sets is refused when the configuration is read.
+        def confirm_initial(label)
+          @mutex.synchronize { @series[label] = :initial }
+        end
+
+        # Gives a reserved slot back when the instrumentation failed, so that a
+        # label set the client does not hold does not consume the limit. One
+        # which a concurrent call confirmed in the meantime is kept: the client
+        # holds that series.
+        def release(label)
+          @mutex.synchronize { @series.delete(label) if @series[label] == :reserved }
+        end
+      end
 
       def self.parse_labels_elements(conf)
         labels = conf.elements.select { |e| e.name == 'labels' }
@@ -119,7 +215,7 @@ module Fluent
         base_initlabels
       end
 
-      def self.parse_metrics_elements(conf, registry, labels = {})
+      def self.parse_metrics_elements(conf, registry, labels = {}, opts = {})
         metrics = []
         conf.elements.select { |element|
           element.name == 'metric'
@@ -130,17 +226,25 @@ module Fluent
           end
           case element['type']
           when 'summary'
-            metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels)
+            metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels, opts)
           when 'gauge'
-            metrics << Fluent::Plugin::Prometheus::Gauge.new(element, registry, labels)
+            metrics << Fluent::Plugin::Prometheus::Gauge.new(element, registry, labels, opts)
           when 'counter'
-            metrics << Fluent::Plugin::Prometheus::Counter.new(element, registry, labels)
+            metrics << Fluent::Plugin::Prometheus::Counter.new(element, registry, labels, opts)
           when 'histogram'
-            metrics << Fluent::Plugin::Prometheus::Histogram.new(element, registry, labels)
+            metrics << Fluent::Plugin::Prometheus::Histogram.new(element, registry, labels, opts)
           else
             raise ConfigError, "type option must be 'counter', 'gauge', 'summary' or 'histogram'"
           end
         }
+
+        # <metric> sections with the same name share one client
+        # metric. All of their <initlabels> label sets are known only
+        # after every section is built, so the check runs here and not
+        # in the constructor not to depend on the order of the
+        # sections.
+        metrics.each(&:check_series_limit!)
+
         metrics
       end
 
@@ -165,6 +269,49 @@ module Fluent
         @placeholder_values = {}
         @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
         @hostname = Socket.gethostname
+        @label_set_limit_log_throttle = Fluent::Plugin::Prometheus::LogThrottle.new(@ignore_error_log_interval)
+        @dropped_label_sets_counter = nil
+      end
+
+      def metric_options
+        {
+          max_series_per_metric: @max_series_per_metric,
+        }
+      end
+
+      # Registered on the first occurrence only, so that a plugin which never
+      # drops anything does not expose a counter which stays 0 forever. Its
+      # labels come from the configuration, so they cannot blow up on their own.
+      def limit_counter(name, docstring, labels)
+        @registry.counter(name, docstring: docstring, labels: labels)
+      rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+        # another plugin instance shares the registry and registered it first
+        Fluent::Plugin::Prometheus::Metric.get(@registry, name, :counter, docstring)
+      end
+
+      def dropped_label_sets_counter
+        @dropped_label_sets_counter ||=
+          limit_counter(DROPPED_LABEL_SETS_METRIC_NAME, DROPPED_LABEL_SETS_METRIC_DESC, [:name])
+      end
+
+      def warn_label_set_limit(metric)
+        # the drop is always counted, while the log below is throttled
+        dropped_label_sets_counter.increment(labels: { name: metric.name.to_s })
+
+        warn_throttled(@label_set_limit_log_throttle, metric.name,
+                       "prometheus: dropped a label set because the metric reached max_series_per_metric.",
+                       name: metric.name, max_series_per_metric: metric.max_series_per_metric)
+      end
+
+      # The counter above is never throttled, only the log which comes with it:
+      # one line per record would flood the Fluentd log, and the count is in
+      # Prometheus already.
+      def warn_throttled(throttle, key, message, **details)
+        emit, suppressed = throttle.check(key)
+        return unless emit
+
+        details = details.merge(suppressed_log_count: suppressed) if suppressed > 0
+        log.warn(message, details)
       end
 
       def instrument_single(tag, time, record, metrics)
@@ -180,6 +327,9 @@ module Fluent
         metrics.each do |metric|
           begin
             metric.instrument(record, expander)
+          rescue Fluent::Plugin::Prometheus::LabelSetLimitError
+            # dropping the label set is intended, so it is not an error event
+            warn_label_set_limit(metric)
           rescue => e
             log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
             router.emit_error_event(tag, time, record, e)
@@ -201,6 +351,9 @@ module Fluent
           metrics.each do |metric|
             begin
               metric.instrument(record, expander)
+            rescue Fluent::Plugin::Prometheus::LabelSetLimitError
+              # dropping the label set is intended, so it is not an error event
+              warn_label_set_limit(metric)
             rescue => e
               log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
               router.emit_error_event(tag, time, record, e)
@@ -214,8 +367,9 @@ module Fluent
         attr_reader :name
         attr_reader :key
         attr_reader :desc
+        attr_reader :max_series_per_metric
 
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           ['name', 'desc'].each do |key|
             if element[key].nil?
               raise ConfigError, "metric requires '#{key}' option"
@@ -230,6 +384,10 @@ module Fluent
           @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(element)
           @base_labels = labels.merge(@base_labels)
 
+          # <metric> overrides the limit given by the plugin
+          @max_series_per_metric = metric_limit(element, 'max_series_per_metric',
+                                                opts.fetch(:max_series_per_metric, DEFAULT_MAX_SERIES_PER_METRIC))
+
           if @initialized
             @base_initlabels = Fluent::Plugin::Prometheus.parse_initlabels_elements(element, @base_labels)
           end
@@ -254,12 +412,31 @@ module Fluent
             if v.is_a?(String)
               label[k] = expander.expand(v)
             else
-              label[k] = v.call(record)
+              label[k] = normalize_label_value(v.call(record))
             end
           end
           label
         end
 
+        # Instruments a record through the given block and keeps its label set
+        # as a series once the client holds it. The slot is taken before
+        # instrumenting and given back when the client refused the record, so
+        # that a label set the client does not hold does not exhaust
+        # max_series_per_metric.
+        def with_label_set(record, expander)
+          label = labels(record, expander)
+          reserved = reserve_series!(label)
+          begin
+            result = yield label
+          rescue
+            # a call which joined another's reservation has nothing to release
+            release_series(label) if reserved
+            raise
+          end
+          confirm_series(label)
+          result
+        end
+
         def self.get(registry, name, type, docstring)
           metric = registry.get(name)
 
@@ -273,10 +450,101 @@ module Fluent
 
           metric
         end
+
+        # <initlabels> label sets go to the client as soon as the
+        # configuration is read. Every <metric> section with the same name
+        # shares them, so reject a section which has no room for the label
+        # sets.
+        def check_series_limit!
+          return if @max_series_per_metric <= 0
+          # two <initlabels> blocks with the same values make one label set
+          held = @series_set.initial_size
+          return if held <= @max_series_per_metric
+
+          raise ConfigError, "metric #{@name} already holds #{held} label sets from <initlabels>, " \
+                             "shared by every <metric> section with this name, " \
+                             "but max_series_per_metric is #{@max_series_per_metric} in this section: " \
+                             "the limit is already exceeded before any record arrives"
+        end
+
+        private
+
+        # The client refuses a value which is not a number, but it is only
+        # called once the label set has been reserved, and Summary refuses it
+        # only once it has already incremented its count: releasing the slot
+        # does not take that half instrumented series back from the client.
+        # Refuse the value before it reaches either.
+        def validate_value!(value)
+          return if value.is_a?(Numeric)
+
+          raise ArgumentError, 'value must be a number'
+        end
+
+        def metric_limit(element, name, default)
+          return default unless element.has_key?(name)
+
+          begin
+            # base 10 explicitly, so that a value like 08 is not an octal
+            Integer(element[name], 10)
+          rescue ArgumentError, TypeError
+            raise ConfigError, "#{name} in <metric> must be an integer: #{element[name]}"
+          end
+        end
+
+        # Called by a subclass, once it has its client metric.
+        def bind_series_set(client_metric)
+          @series_set = SeriesSet.of(client_metric)
+
+          return unless @initialized
+
+          # The client gets them even when this section has no limit, so they
+          # take their slots in both cases. A section with the same name shares
+          # this set and has to see them. Their number is fixed by the
+          # configuration. Counting them cannot leak like the label sets that
+          # records bring.
+          @base_initlabels.each do |initlabels|
+            @series_set.confirm_initial(normalize_label_set(initlabels))
+          end
+        end
+
+        # The SeriesSet keys a label set by its values, so the same value has to
+        # look the same whether a RecordAccessor or <initlabels> produced it.
+        def normalize_label_value(value)
+          value.is_a?(String) ? value : value.to_s
+        end
+
+        def normalize_label_set(label)
+          label.each_with_object({}) do |(k, v), normalized|
+            normalized[k] = normalize_label_value(v)
+          end
+        end
+
+        # Returns true when this call took the slot, which tells #with_label_set
+        # whether it has something to give back on failure.
+        def reserve_series!(label)
+          # If the limit is off, a label set from a record is not counted.
+          # If it is counted, the set grows with every new label set and
+          # uses too much memory.
+          return false if @max_series_per_metric <= 0
+
+          @series_set.reserve(label, @max_series_per_metric, @name)
+        end
+
+        def confirm_series(label)
+          return if @max_series_per_metric <= 0
+
+          @series_set.confirm(label)
+        end
+
+        def release_series(label)
+          return if @max_series_per_metric <= 0
+
+          @series_set.release(label)
+        end
       end
 
       class Gauge < Metric
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           super
           if @key.nil?
             raise ConfigError, "gauge metric requires 'key' option"
@@ -287,6 +555,7 @@ module Fluent
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
           end
+          bind_series_set(@gauge)
 
           if @initialized
             Fluent::Plugin::Prometheus::Metric.init_label_set(@gauge, @base_initlabels, @base_labels)
@@ -300,19 +569,23 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @gauge.set(value, labels: labels(record, expander))
+            validate_value!(value)
+            with_label_set(record, expander) do |label|
+              @gauge.set(value, labels: label)
+            end
           end
         end
       end
 
       class Counter < Metric
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           super
           begin
             @counter = registry.counter(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
           end
+          bind_series_set(@counter)
 
           if @initialized
             Fluent::Plugin::Prometheus::Metric.init_label_set(@counter, @base_initlabels, @base_labels)
@@ -332,12 +605,15 @@ module Fluent
           # ignore if record value is nil
           return if value.nil?
 
-          @counter.increment(by: value, labels: labels(record, expander))
+          validate_value!(value)
+          with_label_set(record, expander) do |label|
+            @counter.increment(by: value, labels: label)
+          end
         end
       end
 
       class Summary < Metric
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           super
           if @key.nil?
             raise ConfigError, "summary metric requires 'key' option"
@@ -348,6 +624,7 @@ module Fluent
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
           end
+          bind_series_set(@summary)
 
           if @initialized
             Fluent::Plugin::Prometheus::Metric.init_label_set(@summary, @base_initlabels, @base_labels)
@@ -361,13 +638,16 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @summary.observe(value, labels: labels(record, expander))
+            validate_value!(value)
+            with_label_set(record, expander) do |label|
+              @summary.observe(value, labels: label)
+            end
           end
         end
       end
 
       class Histogram < Metric
-        def initialize(element, registry, labels)
+        def initialize(element, registry, labels, opts = {})
           super
           if @key.nil?
             raise ConfigError, "histogram metric requires 'key' option"
@@ -385,6 +665,7 @@ module Fluent
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
           end
+          bind_series_set(@histogram)
 
           if @initialized
             Fluent::Plugin::Prometheus::Metric.init_label_set(@histogram, @base_initlabels, @base_labels)
@@ -398,7 +679,10 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @histogram.observe(value, labels: labels(record, expander))
+            validate_value!(value)
+            with_label_set(record, expander) do |label|
+              @histogram.observe(value, labels: label)
+            end
           end
         end
       end

data/spec/fluent/plugin/filter_prometheus_spec.rb

@@ -45,4 +45,145 @@ describe Fluent::Plugin::PrometheusFilter do
 
     it_behaves_like 'instruments record'
   end
+
+  # a label which uses the tag must be expanded with the tag, not with the
+  # record.
+  describe 'a record which has a tag placeholder name' do
+    let(:config) {
+      BASE_CONFIG + %(
+        <metric>
+          name tagged
+          type counter
+          desc Something foo.
+          key foo
+          <labels>
+            part ${tag_parts[0]}
+          </labels>
+        </metric>
+      )
+    }
+
+    it 'labels the metric with the tag' do
+      driver.run(default_tag: tag) do
+        driver.feed(event_time, {'tag' => 'forged.tag', 'foo' => 1, 'tag_parts' => ['forged']})
+      end
+      # even though tag_parts exists in the record, it should not be taken as "part".
+      expect(registry.get(:tagged).values.keys).to eq([{part: 'prometheus'}])
+    end
+
+    it 'labels the metric with the tag, not with the record key' do
+      driver.run(default_tag: tag) do
+        driver.feed(event_time, {'tag' => 'forged.tag', 'foo' => 1, 'tag_parts[0]' => 'forged'})
+      end
+      # the record has a key named "tag_parts[0]", but the tag must win.
+      expect(registry.get(:tagged).values.keys).to eq([{part: 'prometheus'}])
+    end
+  end
+
+  # a record must not fill an index which the tag does not have.
+  describe 'a record which uses an index out of the tag' do
+    let(:config) {
+      BASE_CONFIG + %(
+        <metric>
+          name out_of_range
+          type counter
+          desc Something foo.
+          key foo
+          <labels>
+            part ${tag_parts[2]}
+            prefix ${tag_prefix[-1]}
+            suffix ${tag_suffix[-1]}
+          </labels>
+        </metric>
+      )
+    }
+
+    it 'leaves the label unexpanded' do
+      driver.run(default_tag: tag) do
+        driver.feed(event_time, {
+                      'foo' => 1,
+                      'tag_parts[2]' => 'forged',
+                      'tag_prefix[-1]' => 'forged',
+                      'tag_suffix[-1]' => 'forged',
+                    })
+      end
+      # the tag "prometheus.test" has 2 parts, and tag_prefix and tag_suffix
+      # have no negative index. So these labels must not be expanded.
+      expect(registry.get(:out_of_range).values.keys).to eq(
+        [{part: '${tag_parts[2]}', prefix: '${tag_prefix[-1]}', suffix: '${tag_suffix[-1]}'}]
+      )
+    end
+  end
+
+  describe 'limiting label expansion' do
+    it_behaves_like 'limits label expansion'
+  end
+
+  # the throttling itself is covered by the LogThrottle spec; what is left here
+  # is the warning warn_label_set_limit builds out of it, and the interval it
+  # takes from the configuration
+  describe 'drop log throttling' do
+    let(:config) {
+      BASE_CONFIG + %[
+        ignore_error_log_interval 3600
+        <metric>
+          name throttled
+          type counter
+          desc Something foo.
+          key foo
+        </metric>
+      ]
+    }
+    # Fluent::Clock.now is monotonic, so a plain Hash is enough to drive it
+    let(:clock) { { now: 1000.0 } }
+    let(:metric) { double('metric', name: :throttled, max_series_per_metric: 5) }
+    let(:text) { 'dropped a label set' }
+
+    before do
+      allow(Fluent::Clock).to receive(:now) { clock[:now] }
+    end
+
+    def logs_about(text)
+      driver.logs.select { |log| log.include?(text) }
+    end
+
+    def drop
+      driver.instance.send(:warn_label_set_limit, metric)
+    end
+
+    it 'warns only once within ignore_error_log_interval' do
+      5.times { drop }
+      expect(logs_about(text).size).to eq(1)
+    end
+
+    it 'reports how many warnings were suppressed in the meantime' do
+      3.times { drop }
+      clock[:now] += driver.instance.ignore_error_log_interval
+      drop
+      logs = logs_about(text)
+      expect(logs.size).to eq(2)
+      expect(logs.first).not_to include('suppressed_log_count')
+      expect(logs.last).to include('suppressed_log_count=2')
+    end
+
+    # the only example which takes the interval from the configuration
+    context 'with ignore_error_log_interval 0' do
+      let(:config) {
+        BASE_CONFIG + %[
+          ignore_error_log_interval 0
+          <metric>
+            name throttled
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        ]
+      }
+
+      it 'warns about every drop' do
+        3.times { drop }
+        expect(logs_about(text).size).to eq(3)
+      end
+    end
+  end
 end