fluent-plugin-prometheus 1.7.0 → 2.0.2

data/lib/fluent/plugin/in_prometheus/async_wrapper.rb

@@ -0,0 +1,47 @@
+require 'async'
+
+module Fluent::Plugin
+  class PrometheusInput
+    module AsyncWrapper
+      def do_request(host:, port:, secure:)
+        endpoint =
+          if secure
+            context = OpenSSL::SSL::SSLContext.new
+            context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            Async::HTTP::Endpoint.parse("https://#{host}:#{port}", ssl_context: context)
+          else
+            Async::HTTP::Endpoint.parse("http://#{host}:#{port}")
+          end
+
+        Async::HTTP::Client.open(endpoint) do |client|
+          yield(AsyncHttpWrapper.new(client))
+        end
+      end
+
+      Response = Struct.new(:code, :body, :headers)
+
+      class AsyncHttpWrapper
+        def initialize(http)
+          @http = http
+        end
+
+        def get(path)
+          error = nil
+          response = Async::Task.current.async {
+            begin
+              @http.get(path)
+            rescue => e               # Async::Reactor rescue all error. handle it by itself
+              error = e
+            end
+          }.wait
+
+          if error
+            raise error
+          end
+
+          Response.new(response.status.to_s, response.read || '', response.headers)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -24,14 +24,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
       @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
@@ -46,19 +46,19 @@ module Fluent::Plugin
     def start
       super
 
-      @buffer_newest_timekey = @registry.gauge(
+      @buffer_newest_timekey = get_gauge(
         :fluentd_status_buffer_newest_timekey,
         'Newest timekey in buffer.')
-      @buffer_oldest_timekey = @registry.gauge(
+      @buffer_oldest_timekey = get_gauge(
         :fluentd_status_buffer_oldest_timekey,
         'Oldest timekey in buffer.')
-      buffer_queue_length = @registry.gauge(
+      buffer_queue_length = get_gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
-      buffer_total_queued_size = @registry.gauge(
+      buffer_total_queued_size = get_gauge(
         :fluentd_status_buffer_total_bytes,
         'Current total size of queued buffers.')
-      retry_counts = @registry.gauge(
+      retry_counts = get_gauge(
         :fluentd_status_retry_count,
         'Current retry counts.')
 
@@ -76,14 +76,14 @@ module Fluent::Plugin
 
         @monitor_info.each do |name, metric|
           if info[name]
-            metric.set(label, info[name])
+            metric.set(info[name], labels: label)
           end
         end
 
         timekeys = info["buffer_timekeys"]
         if timekeys && !timekeys.empty?
-          @buffer_newest_timekey.set(label, timekeys.max)
-          @buffer_oldest_timekey.set(label, timekeys.min)
+          @buffer_newest_timekey.set(timekeys.max, labels: label)
+          @buffer_oldest_timekey.set(timekeys.min, labels: label)
         end
       end
     end
@@ -95,5 +95,13 @@ module Fluent::Plugin
         type: plugin_info["type"],
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :plugin_category, :type])
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -1,15 +1,16 @@
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/plugin/in_monitor_agent'
 require 'fluent/plugin/prometheus'
 
 module Fluent::Plugin
-  class PrometheusOutputMonitorInput < Fluent::Input
+  class PrometheusOutputMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_output_monitor', self)
     include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
     config_param :interval, :time, default: 5
+    config_param :gauge_all, :bool, default: true
     attr_reader :registry
 
     MONITOR_IVARS = [
@@ -43,22 +44,19 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
       @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
-      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
-        # from v0.14.6
-        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
-      else
-        @monitor_agent = Fluent::MonitorAgentInput.new
-      end
+      @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+
+      @gauge_or_counter = @gauge_all ? :gauge : :counter
     end
 
     def start
@@ -66,57 +64,57 @@ module Fluent::Plugin
 
       @metrics = {
         # Buffer metrics
-        buffer_total_queued_size: @registry.gauge(
+        buffer_total_queued_size: get_gauge(
           :fluentd_output_status_buffer_total_bytes,
           'Current total size of stage and queue buffers.'),
-        buffer_stage_length: @registry.gauge(
+        buffer_stage_length: get_gauge(
           :fluentd_output_status_buffer_stage_length,
           'Current length of stage buffers.'),
-        buffer_stage_byte_size: @registry.gauge(
+        buffer_stage_byte_size: get_gauge(
           :fluentd_output_status_buffer_stage_byte_size,
           'Current total size of stage buffers.'),
-        buffer_queue_length: @registry.gauge(
+        buffer_queue_length: get_gauge(
           :fluentd_output_status_buffer_queue_length,
           'Current length of queue buffers.'),
-        buffer_queue_byte_size: @registry.gauge(
+        buffer_queue_byte_size: get_gauge(
           :fluentd_output_status_queue_byte_size,
           'Current total size of queue buffers.'),
-        buffer_available_buffer_space_ratios: @registry.gauge(
+        buffer_available_buffer_space_ratios: get_gauge(
           :fluentd_output_status_buffer_available_space_ratio,
           'Ratio of available space in buffer.'),
-        buffer_newest_timekey: @registry.gauge(
+        buffer_newest_timekey: get_gauge(
           :fluentd_output_status_buffer_newest_timekey,
           'Newest timekey in buffer.'),
-        buffer_oldest_timekey: @registry.gauge(
+        buffer_oldest_timekey: get_gauge(
           :fluentd_output_status_buffer_oldest_timekey,
           'Oldest timekey in buffer.'),
 
         # Output metrics
-        retry_counts: @registry.gauge(
+        retry_counts: get_gauge_or_counter(
           :fluentd_output_status_retry_count,
           'Current retry counts.'),
-        num_errors: @registry.gauge(
+        num_errors: get_gauge_or_counter(
           :fluentd_output_status_num_errors,
           'Current number of errors.'),
-        emit_count: @registry.gauge(
+        emit_count: get_gauge_or_counter(
           :fluentd_output_status_emit_count,
           'Current emit counts.'),
-        emit_records: @registry.gauge(
+        emit_records: get_gauge_or_counter(
           :fluentd_output_status_emit_records,
           'Current emit records.'),
-        write_count: @registry.gauge(
+        write_count: get_gauge_or_counter(
           :fluentd_output_status_write_count,
           'Current write counts.'),
-        rollback_count: @registry.gauge(
+        rollback_count: get_gauge(
           :fluentd_output_status_rollback_count,
           'Current rollback counts.'),
-        flush_time_count: @registry.gauge(
+        flush_time_count: get_gauge_or_counter(
           :fluentd_output_status_flush_time_count,
           'Total flush time.'),
-        slow_flush_count: @registry.gauge(
+        slow_flush_count: get_gauge_or_counter(
           :fluentd_output_status_slow_flush_count,
           'Current slow flush counts.'),
-        retry_wait: @registry.gauge(
+        retry_wait: get_gauge(
           :fluentd_output_status_retry_wait,
           'Current retry wait'),
       }
@@ -146,7 +144,16 @@ module Fluent::Plugin
 
         # output metrics
         'retry_count' => @metrics[:retry_counts],
+        # Needed since Fluentd v1.14 due to metrics extensions.
+        'num_errors' => @metrics[:num_errors],
+        'write_count' => @metrics[:write_count],
+        'emit_count' => @metrics[:emit_count],
+        'emit_records' => @metrics[:emit_records],
+        'rollback_count' => @metrics[:rollback_count],
+        'flush_time_count' => @metrics[:flush_time_count],
+        'slow_flush_count' => @metrics[:slow_flush_count],
       }
+      # No needed for Fluentd v1.14 but leave as-is for backward compatibility.
       instance_vars_info = {
         num_errors: @metrics[:num_errors],
         write_count: @metrics[:write_count],
@@ -162,14 +169,22 @@ module Fluent::Plugin
 
         monitor_info.each do |name, metric|
           if info[name]
-            metric.set(label, info[name])
+            if metric.is_a?(::Prometheus::Client::Gauge)
+              metric.set(info[name], labels: label)
+            elsif metric.is_a?(::Prometheus::Client::Counter)
+              metric.increment(by: info[name] - metric.get(labels: label), labels: label)
+            end
           end
         end
 
         if info['instance_variables']
           instance_vars_info.each do |name, metric|
             if info['instance_variables'][name]
-              metric.set(label, info['instance_variables'][name])
+              if metric.is_a?(::Prometheus::Client::Gauge)
+                metric.set(info['instance_variables'][name], labels: label)
+              elsif metric.is_a?(::Prometheus::Client::Counter)
+                metric.increment(by: info['instance_variables'][name] - metric.get(labels: label), labels: label)
+              end
             end
           end
         end
@@ -187,7 +202,7 @@ module Fluent::Plugin
           if next_time && start_time
             wait = next_time - start_time
           end
-          @metrics[:retry_wait].set(label, wait.to_f)
+          @metrics[:retry_wait].set(wait.to_f, labels: label)
         end
       end
     end
@@ -198,5 +213,21 @@ module Fluent::Plugin
         type: plugin_info["type"],
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
+      end
+    end
+
+    def get_gauge_or_counter(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.public_send(@gauge_or_counter, name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -28,32 +28,27 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
       @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
-      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
-        # from v0.14.6
-        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
-      else
-        @monitor_agent = Fluent::MonitorAgentInput.new
-      end
+      @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
     end
 
     def start
       super
 
       @metrics = {
-        position: @registry.gauge(
+        position: get_gauge(
           :fluentd_tail_file_position,
           'Current position of file.'),
-        inode: @registry.gauge(
+        inode: get_gauge(
           :fluentd_tail_file_inode,
           'Current inode of file.'),
       }
@@ -78,8 +73,8 @@ module Fluent::Plugin
           # Very fragile implementation
           pe = watcher.instance_variable_get(:@pe)
           label = labels(info, watcher.path)
-          @metrics[:inode].set(label, pe.read_inode)
-          @metrics[:position].set(label, pe.read_pos)
+          @metrics[:inode].set(pe.read_inode, labels: label)
+          @metrics[:position].set(pe.read_pos, labels: label)
         end
       end
     end
@@ -91,5 +86,13 @@ module Fluent::Plugin
         path: path,
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type, :path])
+      end
+    end
   end
 end

data/lib/fluent/plugin/prometheus.rb

@@ -1,6 +1,6 @@
 require 'prometheus/client'
 require 'prometheus/client/formats/text'
-require 'fluent/plugin/filter_record_transformer'
+require 'fluent/plugin/prometheus/placeholder_expander'
 
 module Fluent
   module Plugin
@@ -82,14 +82,25 @@ module Fluent
       end
 
       def self.placeholder_expander(log)
-        # Use internal class in order to expand placeholder
-        Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander.new(log: log)
+        Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
+      end
+
+      def stringify_keys(hash_to_stringify)
+        # Adapted from: https://www.jvt.me/posts/2019/09/07/ruby-hash-keys-string-symbol/
+        hash_to_stringify.map do |k,v|
+          value_or_hash = if v.instance_of? Hash
+                            stringify_keys(v)
+                          else
+                            v
+                          end
+          [k.to_s, value_or_hash]
+        end.to_h
       end
 
       def configure(conf)
         super
         @placeholder_values = {}
-        @placeholder_expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
+        @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
         @hostname = Socket.gethostname
       end
 
@@ -100,11 +111,12 @@ module Fluent
           'worker_id' => fluentd_worker_id,
         }
 
+        record = stringify_keys(record)
         placeholders = record.merge(@placeholder_values[tag])
-        placeholders = @placeholder_expander.prepare_placeholders(placeholders)
+        expander = @placeholder_expander_builder.build(placeholders)
         metrics.each do |metric|
           begin
-            metric.instrument(record, @placeholder_expander, placeholders)
+            metric.instrument(record, expander)
           rescue => e
             log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
             router.emit_error_event(tag, time, record, e)
@@ -120,11 +132,12 @@ module Fluent
         }
 
         es.each do |time, record|
+          record = stringify_keys(record)
           placeholders = record.merge(placeholder_values)
-          placeholders = @placeholder_expander.prepare_placeholders(placeholders)
+          expander = @placeholder_expander_builder.build(placeholders)
           metrics.each do |metric|
             begin
-              metric.instrument(record, @placeholder_expander, placeholders)
+              metric.instrument(record, expander)
             rescue => e
               log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
               router.emit_error_event(tag, time, record, e)
@@ -154,11 +167,11 @@ module Fluent
           @base_labels = labels.merge(@base_labels)
         end
 
-        def labels(record, expander, placeholders)
+        def labels(record, expander)
           label = {}
           @base_labels.each do |k, v|
             if v.is_a?(String)
-              label[k] = expander.expand(v, placeholders)
+              label[k] = expander.expand(v)
             else
               label[k] = v.call(record)
             end
@@ -189,20 +202,20 @@ module Fluent
           end
 
           begin
-            @gauge = registry.gauge(element['name'].to_sym, element['desc'])
+            @gauge = registry.gauge(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @gauge.set(labels(record, expander, placeholders), value)
+            @gauge.set(value, labels: labels(record, expander))
           end
         end
       end
@@ -211,13 +224,13 @@ module Fluent
         def initialize(element, registry, labels)
           super
           begin
-            @counter = registry.counter(element['name'].to_sym, element['desc'])
+            @counter = registry.counter(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           # use record value of the key if key is specified, otherwise just increment
           if @key.nil?
             value = 1
@@ -230,7 +243,7 @@ module Fluent
           # ignore if record value is nil
           return if value.nil?
 
-          @counter.increment(labels(record, expander, placeholders), value)
+          @counter.increment(by: value, labels: labels(record, expander))
         end
       end
 
@@ -242,20 +255,20 @@ module Fluent
           end
 
           begin
-            @summary = registry.summary(element['name'].to_sym, element['desc'])
+            @summary = registry.summary(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @summary.observe(labels(record, expander, placeholders), value)
+            @summary.observe(value, labels: labels(record, expander))
           end
         end
       end
@@ -272,23 +285,23 @@ module Fluent
               buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
                 e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
               end
-              @histogram = registry.histogram(element['name'].to_sym, element['desc'], {}, buckets)
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys, buckets: buckets)
             else
-              @histogram = registry.histogram(element['name'].to_sym, element['desc'])
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
             end
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @histogram.observe(labels(record, expander, placeholders), value)
+            @histogram.observe(value, labels: labels(record, expander))
           end
         end
       end