fluent-plugin-prometheus-thread 0.1.3

data/lib/fluent/plugin/filter_prometheus.rb

@@ -0,0 +1,24 @@
+require 'fluent/plugin/prometheus'
+
+module Fluent
+  class PrometheusFilter < Filter
+    Plugin.register_filter('prometheus', self)
+    include Fluent::Prometheus
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def configure(conf)
+      super
+      labels = Fluent::Prometheus.parse_labels_elements(conf)
+      @metrics = Fluent::Prometheus.parse_metrics_elements(conf, @registry, labels)
+    end
+
+    def filter_stream(tag, es)
+      instrument(tag, es, @metrics)
+      es
+    end
+  end if defined?(Filter)
+end

data/lib/fluent/plugin/in_prometheus.rb

@@ -0,0 +1,62 @@
+require 'fluent/input'
+require 'fluent/plugin/prometheus'
+require 'webrick'
+
+module Fluent
+  class PrometheusInput < Input
+    Plugin.register_input('prometheus', self)
+
+    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :port, :integer, :default => 24231
+    config_param :metrics_path, :string, :default => '/metrics'
+
+    attr_reader :registry
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def configure(conf)
+      super
+    end
+
+    def start
+      @server = WEBrick::HTTPServer.new(
+        BindAddress: @bind,
+        Port: @port,
+        Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+        AccessLog: [],
+      )
+      @server.mount(@metrics_path, MonitorServlet, self)
+      @thread = Thread.new { @server.start }
+    end
+
+    def shutdown
+      if @server
+        @server.shutdown
+        @server = nil
+      end
+      if @thread
+        @thread.join
+        @thread = nil
+      end
+    end
+
+    class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
+      def initialize(server, prometheus)
+        @prometheus = prometheus
+      end
+
+      def do_GET(req, res)
+        res.status = 200
+        res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
+        res.body = ::Prometheus::Client::Formats::Text.marshal(@prometheus.registry)
+      rescue
+        res.status = 500
+        res['Content-Type'] = 'text/plain'
+        res.body = $!.to_s
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -0,0 +1,101 @@
+require 'fluent/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/plugin/prometheus'
+require 'webrick'
+
+module Fluent
+  class PrometheusMonitorInput < Input
+    Plugin.register_input('prometheus_monitor', self)
+
+    config_param :interval, :time, :default => 5
+    attr_reader :registry
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def configure(conf)
+      super
+      hostname = Socket.gethostname
+      expander = Fluent::Prometheus.placeholder_expander(log)
+      placeholders = expander.prepare_placeholders(
+        Time.now, {'hostname' => hostname}, [])
+      @base_labels = Fluent::Prometheus.parse_labels_elements(conf)
+      @base_labels.each do |key, value|
+        @base_labels[key] = expander.expand(value, placeholders)
+      end
+
+      @monitor_agent = Fluent::MonitorAgentInput.new
+
+      buffer_queue_length = @registry.gauge(
+        :fluentd_status_buffer_queue_length,
+        'Current buffer queue length.')
+      buffer_total_queued_size = @registry.gauge(
+        :fluentd_status_buffer_total_bytes,
+        'Current total size of queued buffers.')
+      retry_counts = @registry.gauge(
+        :fluentd_status_retry_count,
+        'Current retry counts.')
+
+      @monitor_info = {
+        'buffer_queue_length' => buffer_queue_length,
+        'buffer_total_queued_size' => buffer_total_queued_size,
+        'retry_count' => retry_counts,
+      }
+    end
+
+    class TimerWatcher < Coolio::TimerWatcher
+      def initialize(interval, repeat, log, &callback)
+        @callback = callback
+        @log = log
+        super(interval, repeat)
+      end
+
+      def on_timer
+        @callback.call
+      rescue
+        @log.error $!.to_s
+        @log.error_backtrace
+      end
+    end
+
+    def start
+      @loop = Coolio::Loop.new
+      @timer = TimerWatcher.new(@interval, true, log, &method(:update_monitor_info))
+      @loop.attach(@timer)
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @loop.watchers.each {|w| w.detach }
+      @loop.stop
+      @thread.join
+    end
+
+    def run
+      @loop.run
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
+    end
+
+    def update_monitor_info
+      @monitor_agent.plugins_info_all.each do |info|
+        @monitor_info.each do |name, metric|
+          if info[name]
+            metric.set(labels(info), info[name])
+          end
+        end
+      end
+    end
+
+    def labels(plugin_info)
+      @base_labels.merge(
+        plugin_id: plugin_info["plugin_id"],
+        plugin_category: plugin_info["plugin_category"],
+        type: plugin_info["type"],
+      )
+    end
+  end
+end

data/lib/fluent/plugin/out_prometheus.rb

@@ -0,0 +1,25 @@
+require 'fluent/output'
+require 'fluent/plugin/prometheus'
+
+module Fluent
+  class PrometheusOutput < Output
+    Plugin.register_output('prometheus', self)
+    include Fluent::Prometheus
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def configure(conf)
+      super
+      labels = Fluent::Prometheus.parse_labels_elements(conf)
+      @metrics = Fluent::Prometheus.parse_metrics_elements(conf, @registry, labels)
+    end
+
+    def emit(tag, es, chain)
+      instrument(tag, es, @metrics)
+      chain.next
+    end
+  end
+end

data/lib/fluent/plugin/prometheus.rb

@@ -0,0 +1,196 @@
+require 'prometheus/client'
+require 'prometheus/client/formats/text'
+
+module Fluent
+  module Prometheus
+    class AlreadyRegisteredError < StandardError; end
+
+    def self.parse_labels_elements(conf)
+      labels = conf.elements.select { |e| e.name == 'labels' }
+      if labels.size > 1
+        raise ConfigError, "labels section must have at most 1"
+      end
+
+      base_labels = {}
+      unless labels.empty?
+        labels.first.each do |key, value|
+          labels.first.has_key?(key)
+          base_labels[key.to_sym] = value
+        end
+      end
+
+      base_labels
+    end
+
+    def self.parse_metrics_elements(conf, registry, labels = {})
+      metrics = []
+      conf.elements.select { |element|
+        element.name == 'metric'
+      }.each { |element|
+        case element['type']
+        when 'summary'
+          metrics << Fluent::Prometheus::Summary.new(element, registry, labels)
+        when 'gauge'
+          metrics << Fluent::Prometheus::Gauge.new(element, registry, labels)
+        when 'counter'
+          metrics << Fluent::Prometheus::Counter.new(element, registry, labels)
+        else
+          raise ConfigError, "type option must be 'counter', 'gauge' or 'summary'"
+        end
+      }
+      metrics
+    end
+
+    def self.placeholder_expander(log)
+      # Use internal class in order to expand placeholder
+      if defined?(Fluent::Filter) # for v0.12, built-in PlaceholderExpander
+        begin
+          require 'fluent/plugin/filter_record_transformer'
+          return Fluent::RecordTransformerFilter::PlaceholderExpander.new(log: log)
+        rescue LoadError => e
+          raise ConfigError, "cannot find filter_record_transformer plugin: #{e.message}"
+        end
+      else # for v0.10, use PlaceholderExapander in fluent-plugin-record-reformer plugin
+        begin
+          require 'fluent/plugin/out_record_reformer.rb'
+          return Fluent::RecordReformerOutput::PlaceholderExpander.new(log: log)
+        rescue LoadError => e
+          raise ConfigError, "cannot find fluent-plugin-record-reformer: #{e.message}"
+        end
+      end
+    end
+
+    def configure(conf)
+      super
+      @placeholder_expander = Fluent::Prometheus.placeholder_expander(log)
+      @hostname = Socket.gethostname
+    end
+
+    def instrument(tag, es, metrics)
+      placeholder_values = {
+        'tag' => tag,
+        'hostname' => @hostname,
+      }
+
+      es.each do |time, record|
+        placeholders = record.merge(placeholder_values)
+        placeholders = @placeholder_expander.prepare_placeholders(placeholders)
+        metrics.each do |metric|
+          begin
+            metric.instrument(record, @placeholder_expander, placeholders)
+          rescue => e
+            log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
+            router.emit_error_event(tag, time, record, e)
+          end
+        end
+      end
+    end
+
+    class Metric
+      attr_reader :type
+      attr_reader :name
+      attr_reader :key
+      attr_reader :desc
+
+      def initialize(element, registry, labels)
+        ['name', 'desc'].each do |key|
+          if element[key].nil?
+            raise ConfigError, "metric requires '#{key}' option"
+          end
+        end
+        @type = element['type']
+        @name = element['name']
+        @key = element['key']
+        @desc = element['desc']
+
+        @base_labels = Fluent::Prometheus.parse_labels_elements(element)
+        @base_labels = labels.merge(@base_labels)
+      end
+
+      def labels(record, expander, placeholders)
+        label = {}
+        @base_labels.each do |k, v|
+          label[k] = expander.expand(v, placeholders)
+        end
+        label
+      end
+
+      def self.get(registry, name, type, docstring)
+        metric = registry.get(name)
+
+        # should have same type, docstring
+        if metric.type != type
+          raise AlreadyRegisteredError, "#{name} has already been registered as #{type} type"
+        end
+        if metric.docstring != docstring
+          raise AlreadyRegisteredError, "#{name} has already been registered with different docstring"
+        end
+
+        metric
+      end
+    end
+
+    class Gauge < Metric
+      def initialize(element, registry, labels)
+        super
+        if @key.nil?
+          raise ConfigError, "gauge metric requires 'key' option"
+        end
+
+        begin
+          @gauge = registry.gauge(element['name'].to_sym, element['desc'])
+        rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+          @gauge = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
+        end
+      end
+
+      def instrument(record, expander, placeholders)
+        if record[@key]
+          @gauge.set(labels(record, expander, placeholders), record[@key])
+        end
+      end
+    end
+
+    class Counter < Metric
+      def initialize(element, registry, labels)
+        super
+        begin
+          @counter = registry.counter(element['name'].to_sym, element['desc'])
+        rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+          @counter = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
+        end
+      end
+
+      def instrument(record, expander, placeholders)
+        # use record value of the key if key is specified, otherwise just increment
+        value = @key ? record[@key] : 1
+
+        # ignore if record value is nil
+        return if value.nil?
+
+        @counter.increment(labels(record, expander, placeholders), value)
+      end
+    end
+
+    class Summary < Metric
+      def initialize(element, registry, labels)
+        super
+        if @key.nil?
+          raise ConfigError, "summary metric requires 'key' option"
+        end
+
+        begin
+          @summary = registry.summary(element['name'].to_sym, element['desc'])
+        rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+          @summary = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
+        end
+      end
+
+      def instrument(record, expander, placeholders)
+        if record[@key]
+          @summary.add(labels(record, expander, placeholders), record[@key])
+        end
+      end
+    end
+  end
+end

data/misc/fluentd_sample.conf

@@ -0,0 +1,135 @@
+## Prometheus Input Plugin Configuration
+
+# input plugin that exports metrics
+<source>
+  @type prometheus
+</source>
+
+# input plugin that collects metrics from MonitorAgent
+<source>
+  @type prometheus_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+## Nginx Access Log Configuration
+
+<source>
+  @type tail
+  format nginx
+  tag nginx
+  path /var/log/nginx/access.log
+  pos_file /tmp/fluent_nginx.pos
+  types size:integer
+</source>
+
+<filter nginx>
+  @type prometheus
+
+  # You can use counter type with specifying a key,
+  # and increments counter by the value
+  <metric>
+    name nginx_size_counter_bytes
+    type counter
+    desc nginx bytes sent
+    key size
+    <labels>
+       host ${hostname}
+    </labels>
+  </metric>
+
+  # You can use counter type without specifying a key
+  # This just increments counter by 1
+  <metric>
+    name nginx_record_counts
+    type counter
+    desc the number of emited records
+    <labels>
+       host ${hostname}
+    </labels>
+  </metric>
+</filter>
+
+<match nginx>
+  @type copy
+  # for MonitorAgent sample
+  # <store>
+  #   @id test_forward
+  #   @type forward
+  #   buffer_type memory
+  #  <server>
+  #    host 127.0.0.1
+  #    port 20000
+  #  </server>
+  # </store>
+  <store>
+    @type stdout
+  </store>
+</match>
+
+## Nginx Proxy Log Configuration
+
+<source>
+  @type tail
+  format ltsv
+  tag nginx_proxy
+  path /var/log/nginx/access_proxy.log
+  pos_file /tmp/fluent_nginx_proxy.pos
+  types size:integer,request_length:integer,bytes_sent:integer,body_bytes_sent:integer,request_time:float,upstream_response_time:float
+</source>
+
+<filter nginx_proxy>
+  @type prometheus
+
+  # common labels for all metrics
+  <labels>
+     host ${hostname}
+     method ${request_method}
+     status ${status}
+  </labels>
+
+  <metric>
+    name nginx_proxy_request_length_total_bytes
+    type counter
+    desc nginx proxy request length bytes
+    key request_length
+  </metric>
+  <metric>
+    name nginx_proxy_bytes_sent_total_bytes
+    type counter
+    desc nginx proxy bytes sent
+    key bytes_sent
+  </metric>
+  <metric>
+    name nginx_proxy_request_duration_total_milliseconds
+    type counter
+    desc nginx proxy request time
+    key request_time
+  </metric>
+  <metric>
+    name nginx_proxy_upstream_response_duration_total_milliseconds
+    type counter
+    desc nginx proxy upstream response time
+    key upstream_response_time
+  </metric>
+  <metric>
+    name nginx_proxy_request_duration_milliseconds
+    type summary
+    desc nginx proxy request duration summary
+    key request_time
+  </metric>
+  <metric>
+    name nginx_proxy_upstream_duration_milliseconds
+    type summary
+    desc nginx proxy upstream response duration summary
+    key upstream_response_time
+  </metric>
+</filter>
+
+<match nginx_proxy>
+  @type copy
+  <store>
+    @type stdout
+  </store>
+</match>