phihos-fluent-plugin-prometheus 2.0.3.pre.1

data/lib/fluent/plugin/prometheus.rb

@@ -0,0 +1,445 @@
+require 'prometheus/client'
+require 'prometheus/client/formats/text'
+require 'fluent/plugin/prometheus/placeholder_expander'
+require 'fluent/plugin/prometheus/data_store'
+
+module Fluent
+  module Plugin
+    module PrometheusLabelParser
+      def configure(conf)
+        super
+        # Check if running with multiple workers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+          owner.system_config
+        elsif self.respond_to?(:system_config)
+          self.system_config
+        else
+          nil
+        end
+        @multi_worker = sysconf && sysconf.workers ? (sysconf.workers > 1) : false
+      end
+
+      def parse_labels_elements(conf)
+        base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+
+        if @multi_worker
+          base_labels[:worker_id] = fluentd_worker_id.to_s
+        end
+
+        base_labels
+      end
+    end
+
+    module Prometheus
+      class AlreadyRegisteredError < StandardError; end
+
+      def self.parse_labels_elements(conf)
+        labels = conf.elements.select { |e| e.name == 'labels' }
+        if labels.size > 1
+          raise ConfigError, "labels section must have at most 1"
+        end
+
+        base_labels = {}
+        unless labels.empty?
+          labels.first.each do |key, value|
+            labels.first.has_key?(key)
+
+            # use RecordAccessor only for $. and $[ syntax
+            # otherwise use the value as is or expand the value by RecordTransformer for ${} syntax
+            if value.start_with?('$.') || value.start_with?('$[')
+              base_labels[key.to_sym] = PluginHelper::RecordAccessor::Accessor.new(value)
+            else
+              base_labels[key.to_sym] = value
+            end
+          end
+        end
+
+        base_labels
+      end
+
+      def self.parse_metrics_elements(conf, registry, labels = {})
+        metrics = []
+        conf.elements.select { |element|
+          element.name == 'metric'
+        }.each { |element|
+          if element.has_key?('key') && (element['key'].start_with?('$.') || element['key'].start_with?('$['))
+            value = element['key']
+            element['key'] = PluginHelper::RecordAccessor::Accessor.new(value)
+          end
+          case element['type']
+          when 'summary'
+            metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels)
+          when 'gauge'
+            metrics << Fluent::Plugin::Prometheus::Gauge.new(element, registry, labels)
+          when 'counter'
+            metrics << Fluent::Plugin::Prometheus::Counter.new(element, registry, labels)
+          when 'histogram'
+            metrics << Fluent::Plugin::Prometheus::Histogram.new(element, registry, labels)
+          else
+            raise ConfigError, "type option must be 'counter', 'gauge', 'summary' or 'histogram'"
+          end
+        }
+        metrics
+      end
+
+      def self.start_retention_threads(metrics, registry, thread_create, thread_running, log)
+        metrics.select { |metric| metric.has_retention? }.each do |metric|
+          thread_create.call("prometheus_retention_#{metric.name}".to_sym) do
+            while thread_running.call()
+              metric.remove_expired_metrics(registry, log)
+              sleep(metric.retention_check_interval)
+            end
+          end
+        end
+      end
+
+      def self.start_reset_threads(metrics, registry, thread_create, thread_running, log)
+        metrics.select { |metric| metric.has_reset? }.each do |metric|
+          thread_create.call("prometheus_reset_#{metric.name}".to_sym) do
+            while thread_running.call()
+              sleep(metric.reset_after)
+              metric.reset_values(registry, log)
+            end
+          end
+        end
+      end
+
+      def self.placeholder_expander(log)
+        Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
+      end
+
+      def stringify_keys(hash_to_stringify)
+        # Adapted from: https://www.jvt.me/posts/2019/09/07/ruby-hash-keys-string-symbol/
+        hash_to_stringify.map do |k,v|
+          value_or_hash = if v.instance_of? Hash
+                            stringify_keys(v)
+                          else
+                            v
+                          end
+          [k.to_s, value_or_hash]
+        end.to_h
+      end
+
+      def initialize
+        super
+        ::Prometheus::Client.config.data_store = Fluent::Plugin::Prometheus::DataStore.new
+      end
+
+      def configure(conf)
+        super
+        @placeholder_values = {}
+        @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+        @hostname = Socket.gethostname
+      end
+
+      def instrument_single(tag, time, record, metrics)
+        @placeholder_values[tag] ||= {
+          'tag' => tag,
+          'hostname' => @hostname,
+          'worker_id' => fluentd_worker_id,
+        }
+
+        record = stringify_keys(record)
+        placeholders = record.merge(@placeholder_values[tag])
+        expander = @placeholder_expander_builder.build(placeholders)
+        metrics.each do |metric|
+          begin
+            metric.instrument(record, expander)
+          rescue => e
+            log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
+            router.emit_error_event(tag, time, record, e)
+          end
+        end
+      end
+
+      def instrument(tag, es, metrics)
+        placeholder_values = {
+          'tag' => tag,
+          'hostname' => @hostname,
+          'worker_id' => fluentd_worker_id,
+        }
+
+        es.each do |time, record|
+          record = stringify_keys(record)
+          placeholders = record.merge(placeholder_values)
+          expander = @placeholder_expander_builder.build(placeholders)
+          metrics.each do |metric|
+            begin
+              metric.instrument(record, expander)
+            rescue => e
+              log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
+              router.emit_error_event(tag, time, record, e)
+            end
+          end
+        end
+      end
+
+      class Metric
+        attr_reader :type
+        attr_reader :name
+        attr_reader :key
+        attr_reader :desc
+        attr_reader :retention
+        attr_reader :retention_check_interval
+        attr_reader :reset_after
+
+        def initialize(element, registry, labels)
+          ['name', 'desc'].each do |key|
+            if element[key].nil?
+              raise ConfigError, "metric requires '#{key}' option"
+            end
+          end
+          @type = element['type']
+          @name = element['name']
+          @key = element['key']
+          @desc = element['desc']
+          @retention = element['retention'].to_i
+          @retention_check_interval = element.fetch('retention_check_interval', 60).to_i
+          if has_retention?
+            @last_modified_store = LastModifiedStore.new
+          end
+          @topk = element['topk'].to_i
+          @reset_after = element['reset_after'].to_i
+
+          @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(element)
+          @base_labels = labels.merge(@base_labels)
+        end
+
+        def labels(record, expander)
+          label = {}
+          @base_labels.each do |k, v|
+            if v.is_a?(String)
+              label[k] = expander.expand(v)
+            else
+              label[k] = v.call(record)
+            end
+          end
+          label
+        end
+
+        def self.get(registry, name, type, docstring)
+          metric = registry.get(name)
+
+          # should have same type, docstring
+          if metric.type != type
+            raise AlreadyRegisteredError, "#{name} has already been registered as #{type} type"
+          end
+          if metric.docstring != docstring
+            raise AlreadyRegisteredError, "#{name} has already been registered with different docstring"
+          end
+
+          metric
+        end
+
+        def set_value?(value)
+          if value
+            return true
+          end
+          false
+        end
+
+        def instrument(record, expander)
+          value = self.value(record)
+          if self.set_value?(value)
+            labels = labels(record, expander)
+            set_value(value, labels)
+            if has_retention?
+              @last_modified_store.set_last_updated(labels)
+            end
+          end
+        end
+
+        def has_retention?
+          @retention > 0
+        end
+
+        def has_reset?
+          reset_after > 0
+        end
+
+        def remove_expired_metrics(registry, log)
+          if has_retention?
+            metric = registry.get(@name)
+
+            expiration_time = Time.now - @retention
+            expired_label_sets = @last_modified_store.get_labels_not_modified_since(expiration_time)
+
+            expired_label_sets.each { |expired_label_set|
+              log.debug "Metric #{@name} with labels #{expired_label_set} expired. Removing..."
+              metric.remove(expired_label_set) # this method is supplied by the require at the top of this method
+              @last_modified_store.remove(expired_label_set)
+            }
+          else
+            log.warn('remove_expired_metrics should not be called when retention is not set for this metric!')
+          end
+        end
+
+        def reset_values(registry, log)
+          if has_reset?
+            metric = registry.get(@name)
+            log.debug "Resetting values nof metric #{@name}..."
+            metric.reset_values
+          else
+            log.warn('reset_store should not be called when reset_after is not set for this metric!')
+          end
+        end
+
+        class LastModifiedStore
+          def initialize
+            @internal_store = Hash.new
+            @lock = Monitor.new
+          end
+
+          def synchronize
+            @lock.synchronize { yield }
+          end
+
+          def set_last_updated(labels)
+            synchronize do
+              @internal_store[labels] = Time.now
+            end
+          end
+
+          def remove(labels)
+            synchronize do
+              @internal_store.delete(labels)
+            end
+          end
+
+          def get_labels_not_modified_since(time)
+            synchronize do
+              @internal_store.select { |k, v| v < time }.keys
+            end
+          end
+        end
+      end
+
+      class Gauge < Metric
+        def initialize(element, registry, labels)
+          super
+          if @key.nil?
+            raise ConfigError, "gauge metric requires 'key' option"
+          end
+
+          begin
+            @gauge = registry.gauge(
+              element['name'].to_sym,
+              docstring: element['desc'],
+              labels: @base_labels.keys,
+              store_settings: { topk: @topk }
+            )
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
+          end
+        end
+
+        def value(record)
+          if @key.is_a?(String)
+            record[@key]
+          else
+            @key.call(record)
+          end
+        end
+
+        def set_value(value, labels)
+          @gauge.set(value, labels: labels)
+        end
+      end
+
+      class Counter < Metric
+        def initialize(element, registry, labels)
+          super
+          begin
+            @counter = registry.counter(
+              element['name'].to_sym,
+              docstring: element['desc'],
+              labels: @base_labels.keys,
+              store_settings: { topk: @topk }
+            )
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
+          end
+        end
+
+        def value(record)
+          if @key.nil?
+            1
+          elsif @key.is_a?(String)
+            record[@key]
+          else
+            @key.call(record)
+          end
+        end
+
+        def set_value?(value)
+          !value.nil?
+        end
+
+        def set_value(value, labels)
+          @counter.increment(by: value, labels: labels)
+        end
+      end
+
+      class Summary < Metric
+        def initialize(element, registry, labels)
+          super
+          if @key.nil?
+            raise ConfigError, "summary metric requires 'key' option"
+          end
+
+          begin
+            @summary = registry.summary(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
+          end
+        end
+
+        def value(record)
+          if @key.is_a?(String)
+            record[@key]
+          else
+            @key.call(record)
+          end
+        end
+
+        def set_value(value, labels)
+          @summary.observe(value, labels: labels)
+        end
+      end
+
+      class Histogram < Metric
+        def initialize(element, registry, labels)
+          super
+          if @key.nil?
+            raise ConfigError, "histogram metric requires 'key' option"
+          end
+
+          begin
+            if element['buckets']
+              buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
+                e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
+              end
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys, buckets: buckets)
+            else
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
+            end
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
+          end
+        end
+
+        def value(record)
+          if @key.is_a?(String)
+            record[@key]
+          else
+            @key.call(record)
+          end
+        end
+
+        def set_value(value, labels)
+          @histogram.observe(value, labels: labels)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/prometheus_metrics.rb

@@ -0,0 +1,77 @@
+module Fluent::Plugin
+
+  ##
+  # PromMetricsAggregator aggregates multiples metrics exposed using Prometheus text-based format
+  # see https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
+
+
+  class PrometheusMetrics
+    def initialize
+      @comments = []
+      @metrics = []
+    end
+
+    def to_string
+      (@comments + @metrics).join("\n")
+    end
+
+    def add_comment(comment)
+      @comments << comment
+    end
+
+    def add_metric_value(value)
+      @metrics << value
+    end
+
+    attr_writer :comments, :metrics
+  end
+
+  class PromMetricsAggregator
+    def initialize
+      @metrics = {}
+    end
+
+    def get_metric_name_from_comment(line)
+      tokens = line.split(' ')
+      if ['HELP', 'TYPE'].include?(tokens[1])
+        tokens[2]
+      else
+        ''
+      end
+    end
+
+    def add_metrics(metrics)
+      current_metric = ''
+      new_metric = false
+      lines = metrics.split("\n")
+      for line in lines
+        if line[0] == '#'
+          # Metric comment (# TYPE, # HELP)
+          parsed_metric = get_metric_name_from_comment(line)
+          if parsed_metric != ''
+            if parsed_metric != current_metric
+              # Starting a new metric comment block
+              new_metric = !@metrics.key?(parsed_metric)
+              if new_metric
+                @metrics[parsed_metric] = PrometheusMetrics.new()
+              end
+              current_metric = parsed_metric
+            end
+
+            if new_metric && parsed_metric == current_metric
+              # New metric, inject comments (# TYPE, # HELP)
+              @metrics[parsed_metric].add_comment(line)
+            end
+          end
+        else
+          # Metric value, simply append line
+          @metrics[current_metric].add_metric_value(line)
+        end
+      end
+    end
+
+    def get_metrics
+      @metrics.map{|k,v| v.to_string()}.join("\n") + (@metrics.length ? "\n" : "")
+    end
+  end
+end

data/misc/fluentd_sample.conf

@@ -0,0 +1,170 @@
+## Prometheus Input Plugin Configuration
+
+# input plugin that exports metrics
+<source>
+  @type prometheus
+</source>
+
+<source>
+  @type monitor_agent
+</source>
+
+<source>
+  @type forward
+</source>
+
+# input plugin that collects metrics from MonitorAgent
+<source>
+  @type prometheus_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+# input plugin that collects metrics for output plugin
+<source>
+  @type prometheus_output_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+# input plugin that collects metrics for in_tail plugin
+<source>
+  @type prometheus_tail_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+## Nginx Access Log Configuration
+
+<source>
+  @type tail
+  format nginx
+  tag nginx
+  path /var/log/nginx/access.log
+  pos_file /tmp/fluent_nginx.pos
+  types size:integer
+</source>
+
+<filter nginx>
+  @type prometheus
+
+  # You can use counter type with specifying a key,
+  # and increments counter by the value
+  <metric>
+    name nginx_size_counter_bytes
+    type counter
+    desc nginx bytes sent
+    key size
+    <labels>
+       host ${hostname}
+       foo bar
+    </labels>
+  </metric>
+
+  # You can use counter type without specifying a key
+  # This just increments counter by 1
+  <metric>
+    name nginx_record_counts
+    type counter
+    desc the number of emited records
+    <labels>
+       host ${hostname}
+    </labels>
+  </metric>
+</filter>
+
+<match nginx>
+  @type copy
+  # for MonitorAgent sample
+  <store>
+    @id test_forward
+    @type forward
+    buffer_type memory
+    flush_interval 1s
+    max_retry_wait 2s
+    <buffer>
+    # max_retry_wait 10s
+    flush_interval 1s
+    # retry_type periodic
+    disable_retry_limit
+</buffer>
+    # retry_limit 3
+    disable_retry_limit
+   <server>
+     host 127.0.0.1
+     port 20000
+   </server>
+  </store>
+  <store>
+    @type stdout
+  </store>
+</match>
+
+## Nginx Proxy Log Configuration
+
+<source>
+  @type tail
+  format ltsv
+  tag nginx_proxy
+  path /var/log/nginx/access_proxy.log
+  pos_file /tmp/fluent_nginx_proxy.pos
+  types size:integer,request_length:integer,bytes_sent:integer,body_bytes_sent:integer,request_time:float,upstream_response_time:float
+</source>
+
+<filter nginx_proxy>
+  @type prometheus
+
+  # common labels for all metrics
+  <labels>
+     host ${hostname}
+     method ${request_method}
+     status ${status}
+  </labels>
+
+  <metric>
+    name nginx_proxy_request_length_total_bytes
+    type counter
+    desc nginx proxy request length bytes
+    key request_length
+  </metric>
+  <metric>
+    name nginx_proxy_bytes_sent_total_bytes
+    type counter
+    desc nginx proxy bytes sent
+    key bytes_sent
+  </metric>
+  <metric>
+    name nginx_proxy_request_duration_total_milliseconds
+    type counter
+    desc nginx proxy request time
+    key request_time
+  </metric>
+  <metric>
+    name nginx_proxy_upstream_response_duration_total_milliseconds
+    type counter
+    desc nginx proxy upstream response time
+    key upstream_response_time
+  </metric>
+  <metric>
+    name nginx_proxy_request_duration_milliseconds
+    type summary
+    desc nginx proxy request duration summary
+    key request_time
+  </metric>
+  <metric>
+    name nginx_proxy_upstream_duration_milliseconds
+    type summary
+    desc nginx proxy upstream response duration summary
+    key upstream_response_time
+  </metric>
+</filter>
+
+<match nginx_proxy>
+  @type copy
+  <store>
+    @type stdout
+  </store>
+</match>

data/misc/nginx_proxy.conf

@@ -0,0 +1,22 @@
+log_format ltsv 'time:$time_iso8601\t'
+                'remote_addr:$remote_addr\t'
+                'request_method:$request_method\t'
+                'request_length:$request_length\t'
+                'request_uri:$request_uri\t'
+                'uri:$uri\t'
+                'status:$status\t'
+                'bytes_sent:$bytes_sent\t'
+                'body_bytes_sent:$body_bytes_sent\t'
+                'referer:$http_referer\t'
+                'useragent:$http_user_agent\t'
+                'request_time:$request_time\t'
+                'upstream_response_time:$upstream_response_time';
+
+server {
+  access_log /var/log/nginx/access_proxy.log ltsv;
+  listen 9999;
+  location / {
+    proxy_pass https://www.google.com;
+  }
+}
+