fluent-plugin-prometheus-smarter 1.8.4

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -0,0 +1,95 @@
+require 'fluent/plugin/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/plugin/prometheus'
+
+module Fluent::Plugin
+  class PrometheusTailMonitorInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus_tail_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
+
+    helpers :timer
+
+    config_param :interval, :time, default: 5
+    attr_reader :registry
+
+    MONITOR_IVARS = [
+      :tails,
+    ]
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def configure(conf)
+      super
+      hostname = Socket.gethostname
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
+      @base_labels = parse_labels_elements(conf)
+      @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"
+        end
+        @base_labels[key] = expander.expand(value)
+      end
+
+      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
+        # from v0.14.6
+        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+      else
+        @monitor_agent = Fluent::MonitorAgentInput.new
+      end
+    end
+
+    def start
+      super
+
+      @metrics = {
+        position: @registry.gauge(
+          :fluentd_tail_file_position,
+          'Current position of file.'),
+        inode: @registry.gauge(
+          :fluentd_tail_file_inode,
+          'Current inode of file.'),
+      }
+      timer_execute(:in_prometheus_tail_monitor, @interval, &method(:update_monitor_info))
+    end
+
+    def update_monitor_info
+      opts = {
+        ivars: MONITOR_IVARS,
+      }
+
+      agent_info = @monitor_agent.plugins_info_all(opts).select {|info|
+        info['type'] == 'tail'.freeze
+      }
+
+      agent_info.each do |info|
+        tails = info['instance_variables'][:tails]
+        next if tails.nil?
+
+        tails.clone.each do |_, watcher|
+          # Access to internal variable of internal class...
+          # Very fragile implementation
+          pe = watcher.instance_variable_get(:@pe)
+          label = labels(info, watcher.path)
+          @metrics[:inode].set(label, pe.read_inode)
+          @metrics[:position].set(label, pe.read_pos)
+        end
+      end
+    end
+
+    def labels(plugin_info, path)
+      @base_labels.merge(
+        plugin_id: plugin_info["plugin_id"],
+        type: plugin_info["type"],
+        path: path,
+      )
+    end
+  end
+end

data/lib/fluent/plugin/metric_prometheus.rb

@@ -0,0 +1,71 @@
+require 'fluent/plugin/output'
+require 'fluent/plugin/prometheus'
+
+module Fluent::Plugin
+  class PrometheusOutput < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('prometheus_metric', self)
+    include Fluent::Plugin::PrometheusLabelParser
+    include Fluent::Plugin::Prometheus
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    config_param :key, :string
+
+    def labels(record, expander)
+      label = {}
+      labels.each do |k, v|
+        if v.is_a?(String)
+          label[k] = expander.expand(v)
+        else
+          label[k] = v.call(record)
+        end
+      end
+      label
+    end
+
+    def configure(conf)
+      super
+      @labels = parse_labels_elements(conf)
+      @placeholder_values = {}
+      @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      @hostname = Socket.gethostname
+    end
+
+    def process(tag, es)
+      placeholder_values = {
+        'tag' => tag,
+        'hostname' => @hostname,
+        'worker_id' => fluentd_worker_id,
+      }
+      
+      # Create metric if not exists
+      begin
+        gauge = registry.gauge(tag.to_sym)
+      rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+        gauge = registry.get(tag.to_sym)
+      end
+
+      # Write out values in event stream to Registry
+      es.each do |time, record|
+        placeholders = record.merge(placeholder_values)
+        expander = @placeholder_expander_builder.build(placeholders)
+        if @key.is_a?(String)
+          value = record[@key]
+        else
+          value = @key.call(record)
+        end
+        if value
+          gauge.set(labels(record, expander), value)
+        end
+      end
+    end
+
+  end
+end

data/lib/fluent/plugin/out_prometheus.rb

@@ -0,0 +1,29 @@
+require 'fluent/plugin/output'
+require 'fluent/plugin/prometheus'
+
+module Fluent::Plugin
+  class PrometheusOutput < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
+    include Fluent::Plugin::Prometheus
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def configure(conf)
+      super
+      labels = parse_labels_elements(conf)
+      @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
+    end
+
+    def process(tag, es)
+      instrument(tag, es, @metrics)
+    end
+  end
+end

data/lib/fluent/plugin/prometheus.rb

@@ -0,0 +1,296 @@
+require 'prometheus/client'
+require 'prometheus/client/formats/text'
+require 'fluent/plugin/prometheus/placeholder_expander'
+
+module Fluent
+  module Plugin
+    module PrometheusLabelParser
+      def configure(conf)
+        super
+        # Check if running with multiple workers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+          owner.system_config
+        elsif self.respond_to?(:system_config)
+          self.system_config
+        else
+          nil
+        end
+        @multi_worker = sysconf && sysconf.workers ? (sysconf.workers > 1) : false
+      end
+
+      def parse_labels_elements(conf)
+        base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+
+        if @multi_worker
+          base_labels[:worker_id] = fluentd_worker_id.to_s
+        end
+
+        base_labels
+      end
+    end
+
+    module Prometheus
+      class AlreadyRegisteredError < StandardError; end
+
+      def self.parse_labels_elements(conf)
+        labels = conf.elements.select { |e| e.name == 'labels' }
+        if labels.size > 1
+          raise ConfigError, "labels section must have at most 1"
+        end
+
+        base_labels = {}
+        unless labels.empty?
+          labels.first.each do |key, value|
+            labels.first.has_key?(key)
+
+            # use RecordAccessor only for $. and $[ syntax
+            # otherwise use the value as is or expand the value by RecordTransformer for ${} syntax
+            if value.start_with?('$.') || value.start_with?('$[')
+              base_labels[key.to_sym] = PluginHelper::RecordAccessor::Accessor.new(value)
+            else
+              base_labels[key.to_sym] = value
+            end
+          end
+        end
+
+        base_labels
+      end
+
+      def self.parse_metrics_elements(conf, registry, labels = {})
+        metrics = []
+        conf.elements.select { |element|
+          element.name == 'metric'
+        }.each { |element|
+          if element.has_key?('key') && (element['key'].start_with?('$.') || element['key'].start_with?('$['))
+            value = element['key']
+            element['key'] = PluginHelper::RecordAccessor::Accessor.new(value)
+          end
+          case element['type']
+          when 'summary'
+            metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels)
+          when 'gauge'
+            metrics << Fluent::Plugin::Prometheus::Gauge.new(element, registry, labels)
+          when 'counter'
+            metrics << Fluent::Plugin::Prometheus::Counter.new(element, registry, labels)
+          when 'histogram'
+            metrics << Fluent::Plugin::Prometheus::Histogram.new(element, registry, labels)
+          else
+            raise ConfigError, "type option must be 'counter', 'gauge', 'summary' or 'histogram'"
+          end
+        }
+        metrics
+      end
+
+      def self.placeholder_expander(log)
+        Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
+      end
+
+      def configure(conf)
+        super
+        @placeholder_values = {}
+        @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+        @hostname = Socket.gethostname
+      end
+
+      def instrument_single(tag, time, record, metrics)
+        @placeholder_values[tag] ||= {
+          'tag' => tag,
+          'hostname' => @hostname,
+          'worker_id' => fluentd_worker_id,
+        }
+
+        placeholders = record.merge(@placeholder_values[tag])
+        expander = @placeholder_expander_builder.build(placeholders)
+        metrics.each do |metric|
+          begin
+            metric.instrument(record, expander)
+          rescue => e
+            log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
+            router.emit_error_event(tag, time, record, e)
+          end
+        end
+      end
+
+      def instrument(tag, es, metrics)
+        placeholder_values = {
+          'tag' => tag,
+          'hostname' => @hostname,
+          'worker_id' => fluentd_worker_id,
+        }
+
+        es.each do |time, record|
+          placeholders = record.merge(placeholder_values)
+          expander = @placeholder_expander_builder.build(placeholders)
+          metrics.each do |metric|
+            begin
+              metric.instrument(record, expander)
+            rescue => e
+              log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
+              router.emit_error_event(tag, time, record, e)
+            end
+          end
+        end
+      end
+
+      class Metric
+        attr_reader :type
+        attr_reader :name
+        attr_reader :key
+        attr_reader :desc
+
+        def initialize(element, registry, labels)
+          ['name', 'desc'].each do |key|
+            if element[key].nil?
+              raise ConfigError, "metric requires '#{key}' option"
+            end
+          end
+          @type = element['type']
+          @name = element['name']
+          @key = element['key']
+          @desc = element['desc']
+
+          @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(element)
+          @base_labels = labels.merge(@base_labels)
+        end
+
+        def labels(record, expander)
+          label = {}
+          @base_labels.each do |k, v|
+            if v.is_a?(String)
+              label[k] = expander.expand(v)
+            else
+              label[k] = v.call(record)
+            end
+          end
+          label
+        end
+
+        def self.get(registry, name, type, docstring)
+          metric = registry.get(name)
+
+          # should have same type, docstring
+          if metric.type != type
+            raise AlreadyRegisteredError, "#{name} has already been registered as #{type} type"
+          end
+          if metric.docstring != docstring
+            raise AlreadyRegisteredError, "#{name} has already been registered with different docstring"
+          end
+
+          metric
+        end
+      end
+
+      class Gauge < Metric
+        def initialize(element, registry, labels)
+          super
+          if @key.nil?
+            raise ConfigError, "gauge metric requires 'key' option"
+          end
+
+          begin
+            @gauge = registry.gauge(element['name'].to_sym, element['desc'])
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
+          end
+        end
+
+        def instrument(record, expander)
+          if @key.is_a?(String)
+            value = record[@key]
+          else
+            value = @key.call(record)
+          end
+          if value
+            @gauge.set(labels(record, expander), value)
+          end
+        end
+      end
+
+      class Counter < Metric
+        def initialize(element, registry, labels)
+          super
+          begin
+            @counter = registry.counter(element['name'].to_sym, element['desc'])
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
+          end
+        end
+
+        def instrument(record, expander)
+          # use record value of the key if key is specified, otherwise just increment
+          if @key.nil?
+            value = 1
+          elsif @key.is_a?(String)
+            value = record[@key]
+          else
+            value = @key.call(record)
+          end
+
+          # ignore if record value is nil
+          return if value.nil?
+
+          @counter.increment(labels(record, expander), value)
+        end
+      end
+
+      class Summary < Metric
+        def initialize(element, registry, labels)
+          super
+          if @key.nil?
+            raise ConfigError, "summary metric requires 'key' option"
+          end
+
+          begin
+            @summary = registry.summary(element['name'].to_sym, element['desc'])
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
+          end
+        end
+
+        def instrument(record, expander)
+          if @key.is_a?(String)
+            value = record[@key]
+          else
+            value = @key.call(record)
+          end
+          if value
+            @summary.observe(labels(record, expander), value)
+          end
+        end
+      end
+
+      class Histogram < Metric
+        def initialize(element, registry, labels)
+          super
+          if @key.nil?
+            raise ConfigError, "histogram metric requires 'key' option"
+          end
+
+          begin
+            if element['buckets']
+              buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
+                e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
+              end
+              @histogram = registry.histogram(element['name'].to_sym, element['desc'], {}, buckets)
+            else
+              @histogram = registry.histogram(element['name'].to_sym, element['desc'])
+            end
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
+          end
+        end
+
+        def instrument(record, expander)
+          if @key.is_a?(String)
+            value = record[@key]
+          else
+            value = @key.call(record)
+          end
+          if value
+            @histogram.observe(labels(record, expander), value)
+          end
+        end
+      end
+    end
+  end
+end