fluent-plugin-prometheus 1.4.0 → 1.7.1

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusTailMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_tail_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -27,14 +28,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.prepare_placeholders({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)

data/lib/fluent/plugin/out_prometheus.rb

@@ -4,6 +4,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusOutput < Fluent::Plugin::Output
     Fluent::Plugin.register_output('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
     include Fluent::Plugin::Prometheus
 
     def initialize
@@ -17,7 +18,7 @@ module Fluent::Plugin
 
     def configure(conf)
       super
-      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      labels = parse_labels_elements(conf)
       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 

data/lib/fluent/plugin/prometheus.rb

@@ -1,9 +1,34 @@
 require 'prometheus/client'
 require 'prometheus/client/formats/text'
-require 'fluent/plugin/filter_record_transformer'
+require 'fluent/plugin/prometheus/placeholder_expander'
 
 module Fluent
   module Plugin
+    module PrometheusLabelParser
+      def configure(conf)
+        super
+        # Check if running with multiple workers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+          owner.system_config
+        elsif self.respond_to?(:system_config)
+          self.system_config
+        else
+          nil
+        end
+        @multi_worker = sysconf && sysconf.workers ? (sysconf.workers > 1) : false
+      end
+
+      def parse_labels_elements(conf)
+        base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+
+        if @multi_worker
+          base_labels[:worker_id] = fluentd_worker_id.to_s
+        end
+
+        base_labels
+      end
+    end
+
     module Prometheus
       class AlreadyRegisteredError < StandardError; end
 
@@ -57,16 +82,35 @@ module Fluent
       end
 
       def self.placeholder_expander(log)
-        # Use internal class in order to expand placeholder
-        Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander.new(log: log)
+        Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
       end
 
       def configure(conf)
         super
-        @placeholder_expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
+        @placeholder_values = {}
+        @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
         @hostname = Socket.gethostname
       end
 
+      def instrument_single(tag, time, record, metrics)
+        @placeholder_values[tag] ||= {
+          'tag' => tag,
+          'hostname' => @hostname,
+          'worker_id' => fluentd_worker_id,
+        }
+
+        placeholders = record.merge(@placeholder_values[tag])
+        expander = @placeholder_expander_builder.build(placeholders)
+        metrics.each do |metric|
+          begin
+            metric.instrument(record, expander)
+          rescue => e
+            log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
+            router.emit_error_event(tag, time, record, e)
+          end
+        end
+      end
+
       def instrument(tag, es, metrics)
         placeholder_values = {
           'tag' => tag,
@@ -76,10 +120,10 @@ module Fluent
 
         es.each do |time, record|
           placeholders = record.merge(placeholder_values)
-          placeholders = @placeholder_expander.prepare_placeholders(placeholders)
+          expander = @placeholder_expander_builder.build(placeholders)
           metrics.each do |metric|
             begin
-              metric.instrument(record, @placeholder_expander, placeholders)
+              metric.instrument(record, expander)
             rescue => e
               log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
               router.emit_error_event(tag, time, record, e)
@@ -109,11 +153,11 @@ module Fluent
           @base_labels = labels.merge(@base_labels)
         end
 
-        def labels(record, expander, placeholders)
+        def labels(record, expander)
           label = {}
           @base_labels.each do |k, v|
             if v.is_a?(String)
-              label[k] = expander.expand(v, placeholders)
+              label[k] = expander.expand(v)
             else
               label[k] = v.call(record)
             end
@@ -150,14 +194,14 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @gauge.set(labels(record, expander, placeholders), value)
+            @gauge.set(labels(record, expander), value)
           end
         end
       end
@@ -172,7 +216,7 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           # use record value of the key if key is specified, otherwise just increment
           if @key.nil?
             value = 1
@@ -185,7 +229,7 @@ module Fluent
           # ignore if record value is nil
           return if value.nil?
 
-          @counter.increment(labels(record, expander, placeholders), value)
+          @counter.increment(labels(record, expander), value)
         end
       end
 
@@ -203,14 +247,14 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @summary.observe(labels(record, expander, placeholders), value)
+            @summary.observe(labels(record, expander), value)
           end
         end
       end
@@ -236,14 +280,14 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @histogram.observe(labels(record, expander, placeholders), value)
+            @histogram.observe(labels(record, expander), value)
           end
         end
       end

data/lib/fluent/plugin/prometheus/placeholder_expander.rb

@@ -0,0 +1,132 @@
+module Fluent
+  module Plugin
+    module Prometheus
+      class ExpandBuilder
+        def self.build(placeholder, log:)
+          new(log: log).build(placeholder)
+        end
+
+        def initialize(log:)
+          @log = log
+        end
+
+        def build(placeholder_values)
+          placeholders = {}
+          placeholder_values.each do |key, value|
+            case value
+            when Array
+              size = value.size
+              value.each_with_index do |v, i|
+                placeholders["${#{key}[#{i}]}"] = v
+                placeholders["${#{key}[#{i - size}]}"] = v
+              end
+            when Hash
+              value.each do |k, v|
+                placeholders[%(${#{key}["#{k}"]})] = v
+              end
+            else
+              if key == 'tag'
+                placeholders.merge!(build_tag(value))
+              else
+                placeholders["${#{key}}"] = value
+              end
+            end
+          end
+
+          Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander.new(@log, placeholders)
+        end
+
+        private
+
+        def build_tag(tag)
+          tags = tag.split('.')
+
+          placeholders = { '${tag}' => tag }
+
+          size = tags.size
+
+          tags.each_with_index do |v, i|
+            placeholders["${tag_parts[#{i}]}"] = v
+            placeholders["${tag_parts[#{i - size}]}"] = v
+          end
+
+          tag_prefix(tags).each_with_index do |v, i|
+            placeholders["${tag_prefix[#{i}]}"] = v
+          end
+
+          tag_suffix(tags).each_with_index do |v, i|
+            placeholders["${tag_suffix[#{i}]}"] = v
+          end
+
+          placeholders
+        end
+
+        def tag_prefix(tags)
+          tags = tags.dup
+          return [] if tags.empty?
+
+          ret = [tags.shift]
+          tags.each.with_index(1) do |tag, i|
+            ret[i] = "#{ret[i-1]}.#{tag}"
+          end
+          ret
+        end
+
+        def tag_suffix(tags)
+          return [] if tags.empty?
+
+          tags = tags.dup.reverse
+          ret = [tags.shift]
+          tags.each.with_index(1) do |tag, i|
+            ret[i] = "#{tag}.#{ret[i-1]}"
+          end
+          ret
+        end
+
+        class PlaceholderExpander
+          PLACEHOLDER_REGEX = /(\${[^\[}]+(\[[^\]]+\])?})/.freeze
+
+          attr_reader :placeholder
+
+          def initialize(log, placeholder)
+            @placeholder = placeholder
+            @log = log
+            @expander_cache = {}
+          end
+
+          def merge_placeholder(placeholder)
+            @placeholder.merge!(placeholder)
+          end
+
+          def expand(str, dynamic_placeholders: nil)
+            expander = if dynamic_placeholders
+                         if @expander_cache[dynamic_placeholders]
+                           @expander_cache[dynamic_placeholders]
+                         else
+                           e = ExpandBuilder.build(dynamic_placeholders, log: @log)
+                           e.merge_placeholder(@placeholder)
+                           @expander_cache[dynamic_placeholders] = e
+                           e
+                         end
+                       else
+                         self
+                       end
+
+            expander.expand!(str)
+          end
+
+          protected
+
+          def expand!(str)
+            str.gsub(PLACEHOLDER_REGEX) { |value|
+              @placeholder.fetch(value) do
+                @log.warn("unknown placeholder `#{value}` found")
+                value # return as it is
+              end
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/prometheus_metrics.rb

@@ -0,0 +1,77 @@
+module Fluent::Plugin
+
+  ##
+  # PromMetricsAggregator aggregates multiples metrics exposed using Prometheus text-based format
+  # see https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
+
+
+  class PrometheusMetrics
+    def initialize
+      @comments = []
+      @metrics = []
+    end
+
+    def to_string
+      (@comments + @metrics).join("\n")
+    end
+
+    def add_comment(comment)
+      @comments << comment
+    end
+
+    def add_metric_value(value)
+      @metrics << value
+    end
+
+    attr_writer :comments, :metrics
+  end
+
+  class PromMetricsAggregator
+    def initialize
+      @metrics = {}
+    end
+
+    def get_metric_name_from_comment(line)
+      tokens = line.split(' ')
+      if ['HELP', 'TYPE'].include?(tokens[1])
+        tokens[2]
+      else
+        ''
+      end
+    end
+
+    def add_metrics(metrics)
+      current_metric = ''
+      new_metric = false
+      lines = metrics.split("\n")
+      for line in lines
+        if line[0] == '#'
+          # Metric comment (# TYPE, # HELP)
+          parsed_metric = get_metric_name_from_comment(line)
+          if parsed_metric != ''
+            if parsed_metric != current_metric
+              # Starting a new metric comment block
+              new_metric = !@metrics.key?(parsed_metric)
+              if new_metric
+                @metrics[parsed_metric] = PrometheusMetrics.new()
+              end
+              current_metric = parsed_metric
+            end
+
+            if new_metric && parsed_metric == current_metric
+              # New metric, inject comments (# TYPE, # HELP)
+              @metrics[parsed_metric].add_comment(line)
+            end
+          end
+        else
+          # Metric value, simply append line
+          @metrics[current_metric].add_metric_value(line)
+        end
+      end
+    end
+
+    def get_metrics
+      @metrics.map{|k,v| v.to_string()}.join("\n") + (@metrics.length ? "\n" : "")
+    end
+  end
+end

data/misc/prometheus_alerts.yaml

@@ -47,7 +47,7 @@ ALERT FluentdQueueLength
   }
 
 ALERT FluentdRecordsCountsHigh
-  IF sum(rate(fluentd_record_counts{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_record_counts{job="fluentd"}[15m])) BY (instance))
+  IF sum(rate(fluentd_output_status_emit_records{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_output_status_emit_records{job="fluentd"}[15m])) BY (instance))
   FOR 1m
   LABELS {
     service = "fluentd",

data/spec/fluent/plugin/filter_prometheus_spec.rb

@@ -6,7 +6,11 @@ require_relative 'shared'
 describe Fluent::Plugin::PrometheusFilter do
   let(:tag) { 'prometheus.test' }
   let(:driver) { Fluent::Test::Driver::Filter.new(Fluent::Plugin::PrometheusFilter).configure(config) }
-  let(:registry) { ::Prometheus::Client.registry }
+  let(:registry) { ::Prometheus::Client::Registry.new }
+
+  before do
+    allow(Prometheus::Client).to receive(:registry).and_return(registry)
+  end
 
   describe '#configure' do
     it_behaves_like 'output configuration'
@@ -14,22 +18,28 @@ describe Fluent::Plugin::PrometheusFilter do
 
   describe '#run' do
     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
-    let(:es) {
-      driver.run(default_tag: tag) { driver.feed(event_time, message) }
-      driver.filtered_records
-    }
 
     context 'simple config' do
-      include_context 'simple_config'
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
 
       it 'adds a new counter metric' do
-        expect(registry.metrics.map(&:name)).not_to include(name)
-        es
-        expect(registry.metrics.map(&:name)).to include(name)
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
       end
 
       it 'should keep original message' do
-        expect(es.first).to eq(message)
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(driver.filtered_records.first).to eq(message)
       end
     end