fluent-plugin-prometheus-smarter 1.8.4

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+

data/fluent-plugin-prometheus.gemspec

@@ -0,0 +1,22 @@
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-prometheus-smarter"
+  spec.version       = "1.8.4"
+  spec.authors       = ["Masahiro Sano", "Josh Minor"]
+  spec.email         = ["sabottenda@gmail.com"]
+  spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
+  spec.description   = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
+  spec.homepage      = "https://github.com/jishminor/fluent-plugin-prometheus"
+  spec.license       = "Apache-2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "fluentd", ">= 1.9.1", "< 2"
+  spec.add_dependency "prometheus-client", "< 0.10"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "test-unit"
+end

data/lib/fluent/plugin/filter_prometheus.rb

@@ -0,0 +1,30 @@
+require 'fluent/plugin/prometheus'
+require 'fluent/plugin/filter'
+
+module Fluent::Plugin
+  class PrometheusFilter < Fluent::Plugin::Filter
+    Fluent::Plugin.register_filter('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
+    include Fluent::Plugin::Prometheus
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def configure(conf)
+      super
+      labels = parse_labels_elements(conf)
+      @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
+    end
+
+    def filter(tag, time, record)
+      instrument_single(tag, time, record, @metrics)
+      record
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus.rb

@@ -0,0 +1,222 @@
+require 'fluent/plugin/input'
+require 'fluent/plugin/prometheus'
+require 'fluent/plugin/prometheus_metrics'
+require 'net/http'
+require 'openssl'
+
+module Fluent::Plugin
+  class PrometheusInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus', self)
+
+    helpers :thread, :http_server
+
+    config_param :bind, :string, default: '0.0.0.0'
+    config_param :port, :integer, default: 24231
+    config_param :metrics_path, :string, default: '/metrics'
+    config_param :aggregated_metrics_path, :string, default: '/aggregated_metrics'
+
+    desc 'Enable ssl configuration for the server'
+    config_section :ssl, required: false, multi: false do
+      config_param :enable, :bool, default: false, deprecated: 'Use <transport tls> section'
+
+      desc 'Path to the ssl certificate in PEM format.  Read from file and added to conf as "SSLCertificate"'
+      config_param :certificate_path, :string, default: nil, deprecated: 'Use cert_path in <transport tls> section'
+
+      desc 'Path to the ssl private key in PEM format.  Read from file and added to conf as "SSLPrivateKey"'
+      config_param :private_key_path, :string, default: nil, deprecated: 'Use private_key_path in <transport tls> section'
+
+      desc 'Path to CA in PEM format.  Read from file and added to conf as "SSLCACertificateFile"'
+      config_param :ca_path, :string, default: nil, deprecated: 'Use ca_path in <transport tls> section'
+
+      desc 'Additional ssl conf for the server.  Ref: https://github.com/ruby/webrick/blob/master/lib/webrick/ssl.rb'
+      config_param :extra_conf, :hash, default: nil, symbolize_keys: true, deprecated: 'See http helper config'
+    end
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+      @secure = nil
+    end
+
+    def configure(conf)
+      super
+
+      # Get how many workers we have
+      sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                  owner.system_config
+                elsif self.respond_to?(:system_config)
+                  self.system_config
+                else
+                  nil
+                end
+      @num_workers = sysconf && sysconf.workers ? sysconf.workers : 1
+      @secure = @transport_config.protocol == :tls || (@ssl && @ssl['enable'])
+
+      @base_port = @port
+      @port += fluentd_worker_id
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      scheme = @secure ? 'https' : 'http'
+      log.debug "listening prometheus http server on #{scheme}:://#{@bind}:#{@port}/#{@metrics_path} for worker#{fluentd_worker_id}"
+
+      proto = @secure ? :tls : :tcp
+
+      if @ssl && @ssl['enable'] && @ssl['extra_conf']
+        start_webrick
+        return
+      end
+
+      tls_opt = if @ssl && @ssl['enable']
+                  ssl_config = {}
+
+                  if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
+                    raise Fluent::ConfigError.new('both certificate_path and private_key_path must be defined')
+                  end
+
+                  if @ssl['certificate_path']
+                    ssl_config['cert_path'] = @ssl['certificate_path']
+                  end
+
+                  if @ssl['private_key_path']
+                    ssl_config['private_key_path'] = @ssl['private_key_path']
+                  end
+
+                  if @ssl['ca_path']
+                    ssl_config['ca_path'] = @ssl['ca_path']
+                    # Only ca_path is insecure in fluentd
+                    # https://github.com/fluent/fluentd/blob/2236ad45197ba336fd9faf56f442252c8b226f25/lib/fluent/plugin_helper/cert_option.rb#L68
+                    ssl_config['insecure'] = true
+                  end
+
+                  ssl_config
+                end
+
+      http_server_create_http_server(:in_prometheus_server, addr: @bind, port: @port, logger: log, proto: proto, tls_opts: tls_opt) do |server|
+        server.get(@metrics_path) { |_req| all_metrics }
+        server.get(@aggregated_metrics_path) { |_req| all_workers_metrics }
+      end
+    end
+
+    def shutdown
+      if @webrick_server
+        @webrick_server.shutdown
+        @webrick_server = nil
+      end
+      super
+    end
+
+    private
+
+    # For compatiblity because http helper can't support extra_conf option
+    def start_webrick
+      require 'webrick/https'
+      require 'webrick'
+
+      config = {
+        BindAddress: @bind,
+        Port: @port,
+        MaxClients: 5,
+        Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+        AccessLog: [],
+      }
+      if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
+        raise RuntimeError.new("certificate_path and private_key_path most both be defined")
+      end
+
+      ssl_config = {
+        SSLEnable: true,
+        SSLCertName: [['CN', 'nobody'], ['DC', 'example']]
+      }
+
+      if @ssl['certificate_path']
+        cert = OpenSSL::X509::Certificate.new(File.read(@ssl['certificate_path']))
+        ssl_config[:SSLCertificate] = cert
+      end
+
+      if @ssl['private_key_path']
+        key = OpenSSL::PKey.read(@ssl['private_key_path'])
+        ssl_config[:SSLPrivateKey] = key
+      end
+
+      ssl_config[:SSLCACertificateFile] = @ssl['ca_path'] if @ssl['ca_path']
+      ssl_config = ssl_config.merge(@ssl['extra_conf']) if @ssl['extra_conf']
+      config = ssl_config.merge(config)
+
+      @log.on_debug do
+        @log.debug("WEBrick conf: #{config}")
+      end
+
+      @webrick_server = WEBrick::HTTPServer.new(config)
+      @webrick_server.mount_proc(@metrics_path) do |_req, res|
+        status, header, body = all_metrics
+        res.status = status
+        res['Content-Type'] = header['Content-Type']
+        res.body = body
+        res
+      end
+
+      @webrick_server.mount_proc(@aggregated_metrics_path) do |_req, res|
+        status, header, body = all_workers_metrics
+        res.status = status
+        res['Content-Type'] = header['Content-Type']
+        res.body = body
+        res
+      end
+
+      thread_create(:in_prometheus_webrick) do
+        @webrick_server.start
+      end
+    end
+
+    def all_metrics
+      [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, ::Prometheus::Client::Formats::Text.marshal(@registry)]
+    rescue => e
+      [500, { 'Content-Type' => 'text/plain' }, e.to_s]
+    end
+
+    def all_workers_metrics
+      full_result = PromMetricsAggregator.new
+
+      send_request_to_each_worker do |resp|
+        if resp.is_a?(Net::HTTPSuccess)
+          full_result.add_metrics(resp.body)
+        end
+      end
+
+      [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, full_result.get_metrics]
+    rescue => e
+      [500, { 'Content-Type' => 'text/plain' }, e.to_s]
+    end
+
+    def send_request_to_each_worker
+      bind = (@bind == '0.0.0.0') ? '127.0.0.1' : @bind
+      req = Net::HTTP::Get.new(@metrics_path)
+      [*(@base_port...(@base_port + @num_workers))].each do |worker_port|
+        do_request(host: bind, port: worker_port, secure: @secure) do |http|
+          yield(http.request(req))
+        end
+      end
+    end
+
+    def do_request(host:, port:, secure:)
+      http = Net::HTTP.new(host, port)
+
+      if secure
+        http.use_ssl = true
+        # target is our child process. so it's secure.
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+       http.start do
+        yield(http)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -0,0 +1,99 @@
+require 'fluent/plugin/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/plugin/prometheus'
+
+module Fluent::Plugin
+  class PrometheusMonitorInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
+
+    helpers :timer
+
+    config_param :interval, :time, default: 5
+    attr_reader :registry
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def configure(conf)
+      super
+      hostname = Socket.gethostname
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
+      @base_labels = parse_labels_elements(conf)
+      @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
+        end
+        @base_labels[key] = expander.expand(value)
+      end
+
+      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
+        # from v0.14.6
+        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+      else
+        @monitor_agent = Fluent::MonitorAgentInput.new
+      end
+
+    end
+
+    def start
+      super
+
+      @buffer_newest_timekey = @registry.gauge(
+        :fluentd_status_buffer_newest_timekey,
+        'Newest timekey in buffer.')
+      @buffer_oldest_timekey = @registry.gauge(
+        :fluentd_status_buffer_oldest_timekey,
+        'Oldest timekey in buffer.')
+      buffer_queue_length = @registry.gauge(
+        :fluentd_status_buffer_queue_length,
+        'Current buffer queue length.')
+      buffer_total_queued_size = @registry.gauge(
+        :fluentd_status_buffer_total_bytes,
+        'Current total size of queued buffers.')
+      retry_counts = @registry.gauge(
+        :fluentd_status_retry_count,
+        'Current retry counts.')
+
+      @monitor_info = {
+        'buffer_queue_length' => buffer_queue_length,
+        'buffer_total_queued_size' => buffer_total_queued_size,
+        'retry_count' => retry_counts,
+      }
+      timer_execute(:in_prometheus_monitor, @interval, &method(:update_monitor_info))
+    end
+
+    def update_monitor_info
+      @monitor_agent.plugins_info_all.each do |info|
+        label = labels(info)
+
+        @monitor_info.each do |name, metric|
+          if info[name]
+            metric.set(label, info[name])
+          end
+        end
+
+        timekeys = info["buffer_timekeys"]
+        if timekeys && !timekeys.empty?
+          @buffer_newest_timekey.set(label, timekeys.max)
+          @buffer_oldest_timekey.set(label, timekeys.min)
+        end
+      end
+    end
+
+    def labels(plugin_info)
+      @base_labels.merge(
+        plugin_id: plugin_info["plugin_id"],
+        plugin_category: plugin_info["plugin_category"],
+        type: plugin_info["type"],
+      )
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -0,0 +1,202 @@
+require 'fluent/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/plugin/prometheus'
+
+module Fluent::Plugin
+  class PrometheusOutputMonitorInput < Fluent::Input
+    Fluent::Plugin.register_input('prometheus_output_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
+
+    helpers :timer
+
+    config_param :interval, :time, default: 5
+    attr_reader :registry
+
+    MONITOR_IVARS = [
+      :retry,
+
+      :num_errors,
+      :emit_count,
+
+      # for v0.12
+      :last_retry_time,
+
+      # from v0.14
+      :emit_records,
+      :write_count,
+      :rollback_count,
+
+      # from v1.6.0
+      :flush_time_count,
+      :slow_flush_count,
+    ]
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def configure(conf)
+      super
+      hostname = Socket.gethostname
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
+      @base_labels = parse_labels_elements(conf)
+      @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
+        end
+        @base_labels[key] = expander.expand(value)
+      end
+
+      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
+        # from v0.14.6
+        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+      else
+        @monitor_agent = Fluent::MonitorAgentInput.new
+      end
+    end
+
+    def start
+      super
+
+      @metrics = {
+        # Buffer metrics
+        buffer_total_queued_size: @registry.gauge(
+          :fluentd_output_status_buffer_total_bytes,
+          'Current total size of stage and queue buffers.'),
+        buffer_stage_length: @registry.gauge(
+          :fluentd_output_status_buffer_stage_length,
+          'Current length of stage buffers.'),
+        buffer_stage_byte_size: @registry.gauge(
+          :fluentd_output_status_buffer_stage_byte_size,
+          'Current total size of stage buffers.'),
+        buffer_queue_length: @registry.gauge(
+          :fluentd_output_status_buffer_queue_length,
+          'Current length of queue buffers.'),
+        buffer_queue_byte_size: @registry.gauge(
+          :fluentd_output_status_queue_byte_size,
+          'Current total size of queue buffers.'),
+        buffer_available_buffer_space_ratios: @registry.gauge(
+          :fluentd_output_status_buffer_available_space_ratio,
+          'Ratio of available space in buffer.'),
+        buffer_newest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_newest_timekey,
+          'Newest timekey in buffer.'),
+        buffer_oldest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_oldest_timekey,
+          'Oldest timekey in buffer.'),
+
+        # Output metrics
+        retry_counts: @registry.gauge(
+          :fluentd_output_status_retry_count,
+          'Current retry counts.'),
+        num_errors: @registry.gauge(
+          :fluentd_output_status_num_errors,
+          'Current number of errors.'),
+        emit_count: @registry.gauge(
+          :fluentd_output_status_emit_count,
+          'Current emit counts.'),
+        emit_records: @registry.gauge(
+          :fluentd_output_status_emit_records,
+          'Current emit records.'),
+        write_count: @registry.gauge(
+          :fluentd_output_status_write_count,
+          'Current write counts.'),
+        rollback_count: @registry.gauge(
+          :fluentd_output_status_rollback_count,
+          'Current rollback counts.'),
+        flush_time_count: @registry.gauge(
+          :fluentd_output_status_flush_time_count,
+          'Total flush time.'),
+        slow_flush_count: @registry.gauge(
+          :fluentd_output_status_slow_flush_count,
+          'Current slow flush counts.'),
+        retry_wait: @registry.gauge(
+          :fluentd_output_status_retry_wait,
+          'Current retry wait'),
+      }
+      timer_execute(:in_prometheus_output_monitor, @interval, &method(:update_monitor_info))
+    end
+
+    def update_monitor_info
+      opts = {
+        ivars: MONITOR_IVARS,
+        with_retry: true,
+      }
+
+      agent_info = @monitor_agent.plugins_info_all(opts).select {|info|
+        info['plugin_category'] == 'output'.freeze
+      }
+
+      monitor_info = {
+        # buffer metrics
+        'buffer_total_queued_size' => @metrics[:buffer_total_queued_size],
+        'buffer_stage_length' => @metrics[:buffer_stage_length],
+        'buffer_stage_byte_size' => @metrics[:buffer_stage_byte_size],
+        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        'buffer_queue_byte_size' => @metrics[:buffer_queue_byte_size],
+        'buffer_available_buffer_space_ratios' => @metrics[:buffer_available_buffer_space_ratios],
+        'buffer_newest_timekey' => @metrics[:buffer_newest_timekey],
+        'buffer_oldest_timekey' => @metrics[:buffer_oldest_timekey],
+
+        # output metrics
+        'retry_count' => @metrics[:retry_counts],
+      }
+      instance_vars_info = {
+        num_errors: @metrics[:num_errors],
+        write_count: @metrics[:write_count],
+        emit_count: @metrics[:emit_count],
+        emit_records: @metrics[:emit_records],
+        rollback_count: @metrics[:rollback_count],
+        flush_time_count: @metrics[:flush_time_count],
+        slow_flush_count: @metrics[:slow_flush_count],
+      }
+
+      agent_info.each do |info|
+        label = labels(info)
+
+        monitor_info.each do |name, metric|
+          if info[name]
+            metric.set(label, info[name])
+          end
+        end
+
+        if info['instance_variables']
+          instance_vars_info.each do |name, metric|
+            if info['instance_variables'][name]
+              metric.set(label, info['instance_variables'][name])
+            end
+          end
+        end
+
+        # compute current retry_wait
+        if info['retry']
+          next_time = info['retry']['next_time']
+          start_time = info['retry']['start']
+          if start_time.nil? && info['instance_variables']
+            # v0.12 does not include start, use last_retry_time instead
+            start_time = info['instance_variables'][:last_retry_time]
+          end
+
+          wait = 0
+          if next_time && start_time
+            wait = next_time - start_time
+          end
+          @metrics[:retry_wait].set(label, wait.to_f)
+        end
+      end
+    end
+
+    def labels(plugin_info)
+      @base_labels.merge(
+        plugin_id: plugin_info["plugin_id"],
+        type: plugin_info["type"],
+      )
+    end
+  end
+end