phihos-fluent-plugin-prometheus 2.0.3.pre.1

data/lib/fluent/plugin/in_prometheus.rb

@@ -0,0 +1,230 @@
+require 'fluent/plugin/input'
+require 'fluent/plugin/prometheus'
+require 'fluent/plugin/prometheus_metrics'
+require 'net/http'
+require 'openssl'
+
+module Fluent::Plugin
+  class PrometheusInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus', self)
+
+    helpers :thread, :http_server
+
+    config_param :bind, :string, default: '0.0.0.0'
+    config_param :port, :integer, default: 24231
+    config_param :metrics_path, :string, default: '/metrics'
+    config_param :aggregated_metrics_path, :string, default: '/aggregated_metrics'
+
+    desc 'Enable ssl configuration for the server'
+    config_section :ssl, required: false, multi: false do
+      config_param :enable, :bool, default: false, deprecated: 'Use <transport tls> section'
+
+      desc 'Path to the ssl certificate in PEM format.  Read from file and added to conf as "SSLCertificate"'
+      config_param :certificate_path, :string, default: nil, deprecated: 'Use cert_path in <transport tls> section'
+
+      desc 'Path to the ssl private key in PEM format.  Read from file and added to conf as "SSLPrivateKey"'
+      config_param :private_key_path, :string, default: nil, deprecated: 'Use private_key_path in <transport tls> section'
+
+      desc 'Path to CA in PEM format.  Read from file and added to conf as "SSLCACertificateFile"'
+      config_param :ca_path, :string, default: nil, deprecated: 'Use ca_path in <transport tls> section'
+
+      desc 'Additional ssl conf for the server.  Ref: https://github.com/ruby/webrick/blob/master/lib/webrick/ssl.rb'
+      config_param :extra_conf, :hash, default: nil, symbolize_keys: true, deprecated: 'See http helper config'
+    end
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+      @secure = nil
+    end
+
+    def configure(conf)
+      super
+
+      # Get how many workers we have
+      sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                  owner.system_config
+                elsif self.respond_to?(:system_config)
+                  self.system_config
+                else
+                  nil
+                end
+      @num_workers = sysconf && sysconf.workers ? sysconf.workers : 1
+      @secure = @transport_config.protocol == :tls || (@ssl && @ssl['enable'])
+
+      @base_port = @port
+      @port += fluentd_worker_id
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      scheme = @secure ? 'https' : 'http'
+      log.debug "listening prometheus http server on #{scheme}:://#{@bind}:#{@port}/#{@metrics_path} for worker#{fluentd_worker_id}"
+
+      proto = @secure ? :tls : :tcp
+
+      if @ssl && @ssl['enable'] && @ssl['extra_conf']
+        start_webrick
+        return
+      end
+
+      begin
+        require 'async'
+        require 'fluent/plugin/in_prometheus/async_wrapper'
+        extend AsyncWrapper
+      rescue LoadError => _
+        # ignore
+      end
+
+      tls_opt = if @ssl && @ssl['enable']
+                  ssl_config = {}
+
+                  if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
+                    raise Fluent::ConfigError.new('both certificate_path and private_key_path must be defined')
+                  end
+
+                  if @ssl['certificate_path']
+                    ssl_config['cert_path'] = @ssl['certificate_path']
+                  end
+
+                  if @ssl['private_key_path']
+                    ssl_config['private_key_path'] = @ssl['private_key_path']
+                  end
+
+                  if @ssl['ca_path']
+                    ssl_config['ca_path'] = @ssl['ca_path']
+                    # Only ca_path is insecure in fluentd
+                    # https://github.com/fluent/fluentd/blob/2236ad45197ba336fd9faf56f442252c8b226f25/lib/fluent/plugin_helper/cert_option.rb#L68
+                    ssl_config['insecure'] = true
+                  end
+
+                  ssl_config
+                end
+
+      http_server_create_http_server(:in_prometheus_server, addr: @bind, port: @port, logger: log, proto: proto, tls_opts: tls_opt) do |server|
+        server.get(@metrics_path) { |_req| all_metrics }
+        server.get(@aggregated_metrics_path) { |_req| all_workers_metrics }
+      end
+    end
+
+    def shutdown
+      if @webrick_server
+        @webrick_server.shutdown
+        @webrick_server = nil
+      end
+      super
+    end
+
+    private
+
+    # For compatiblity because http helper can't support extra_conf option
+    def start_webrick
+      require 'webrick/https'
+      require 'webrick'
+
+      config = {
+        BindAddress: @bind,
+        Port: @port,
+        MaxClients: 5,
+        Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+        AccessLog: [],
+      }
+      if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
+        raise RuntimeError.new("certificate_path and private_key_path most both be defined")
+      end
+
+      ssl_config = {
+        SSLEnable: true,
+        SSLCertName: [['CN', 'nobody'], ['DC', 'example']]
+      }
+
+      if @ssl['certificate_path']
+        cert = OpenSSL::X509::Certificate.new(File.read(@ssl['certificate_path']))
+        ssl_config[:SSLCertificate] = cert
+      end
+
+      if @ssl['private_key_path']
+        key = OpenSSL::PKey.read(@ssl['private_key_path'])
+        ssl_config[:SSLPrivateKey] = key
+      end
+
+      ssl_config[:SSLCACertificateFile] = @ssl['ca_path'] if @ssl['ca_path']
+      ssl_config = ssl_config.merge(@ssl['extra_conf']) if @ssl['extra_conf']
+      config = ssl_config.merge(config)
+
+      @log.on_debug do
+        @log.debug("WEBrick conf: #{config}")
+      end
+
+      @webrick_server = WEBrick::HTTPServer.new(config)
+      @webrick_server.mount_proc(@metrics_path) do |_req, res|
+        status, header, body = all_metrics
+        res.status = status
+        res['Content-Type'] = header['Content-Type']
+        res.body = body
+        res
+      end
+
+      @webrick_server.mount_proc(@aggregated_metrics_path) do |_req, res|
+        status, header, body = all_workers_metrics
+        res.status = status
+        res['Content-Type'] = header['Content-Type']
+        res.body = body
+        res
+      end
+
+      thread_create(:in_prometheus_webrick) do
+        @webrick_server.start
+      end
+    end
+
+    def all_metrics
+      [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, ::Prometheus::Client::Formats::Text.marshal(@registry)]
+    rescue => e
+      [500, { 'Content-Type' => 'text/plain' }, e.to_s]
+    end
+
+    def all_workers_metrics
+      full_result = PromMetricsAggregator.new
+
+      send_request_to_each_worker do |resp|
+        if resp.code.to_s == '200'
+          full_result.add_metrics(resp.body)
+        end
+      end
+
+      [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, full_result.get_metrics]
+    rescue => e
+      [500, { 'Content-Type' => 'text/plain' }, e.to_s]
+    end
+
+    def send_request_to_each_worker
+      bind = (@bind == '0.0.0.0') ? '127.0.0.1' : @bind
+      [*(@base_port...(@base_port + @num_workers))].each do |worker_port|
+        do_request(host: bind, port: worker_port, secure: @secure) do |http|
+          yield(http.get(@metrics_path))
+        end
+      end
+    end
+
+    # might be replaced by AsyncWrapper if async gem is installed
+    def do_request(host:, port:, secure:)
+      http = Net::HTTP.new(host, port)
+
+      if secure
+        http.use_ssl = true
+        # target is our child process. so it's secure.
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+
+      http.start do
+        yield(http)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -0,0 +1,107 @@
+require 'fluent/plugin/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/plugin/prometheus'
+
+module Fluent::Plugin
+  class PrometheusMonitorInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
+
+    helpers :timer
+
+    config_param :interval, :time, default: 5
+    attr_reader :registry
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def configure(conf)
+      super
+      hostname = Socket.gethostname
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
+      @base_labels = parse_labels_elements(conf)
+      @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
+        end
+        @base_labels[key] = expander.expand(value)
+      end
+
+      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
+        # from v0.14.6
+        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+      else
+        @monitor_agent = Fluent::MonitorAgentInput.new
+      end
+
+    end
+
+    def start
+      super
+
+      @buffer_newest_timekey = get_gauge(
+        :fluentd_status_buffer_newest_timekey,
+        'Newest timekey in buffer.')
+      @buffer_oldest_timekey = get_gauge(
+        :fluentd_status_buffer_oldest_timekey,
+        'Oldest timekey in buffer.')
+      buffer_queue_length = get_gauge(
+        :fluentd_status_buffer_queue_length,
+        'Current buffer queue length.')
+      buffer_total_queued_size = get_gauge(
+        :fluentd_status_buffer_total_bytes,
+        'Current total size of queued buffers.')
+      retry_counts = get_gauge(
+        :fluentd_status_retry_count,
+        'Current retry counts.')
+
+      @monitor_info = {
+        'buffer_queue_length' => buffer_queue_length,
+        'buffer_total_queued_size' => buffer_total_queued_size,
+        'retry_count' => retry_counts,
+      }
+      timer_execute(:in_prometheus_monitor, @interval, &method(:update_monitor_info))
+    end
+
+    def update_monitor_info
+      @monitor_agent.plugins_info_all.each do |info|
+        label = labels(info)
+
+        @monitor_info.each do |name, metric|
+          if info[name]
+            metric.set(info[name], labels: label)
+          end
+        end
+
+        timekeys = info["buffer_timekeys"]
+        if timekeys && !timekeys.empty?
+          @buffer_newest_timekey.set(timekeys.max, labels: label)
+          @buffer_oldest_timekey.set(timekeys.min, labels: label)
+        end
+      end
+    end
+
+    def labels(plugin_info)
+      @base_labels.merge(
+        plugin_id: plugin_info["plugin_id"],
+        plugin_category: plugin_info["plugin_category"],
+        type: plugin_info["type"],
+      )
+    end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :plugin_category, :type])
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -0,0 +1,234 @@
+require 'fluent/plugin/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/plugin/prometheus'
+
+module Fluent::Plugin
+  class PrometheusOutputMonitorInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus_output_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
+
+    helpers :timer
+
+    config_param :interval, :time, default: 5
+    config_param :gauge_all, :bool, default: true
+    attr_reader :registry
+
+    MONITOR_IVARS = [
+      :retry,
+
+      :num_errors,
+      :emit_count,
+
+      # for v0.12
+      :last_retry_time,
+
+      # from v0.14
+      :emit_records,
+      :write_count,
+      :rollback_count,
+
+      # from v1.6.0
+      :flush_time_count,
+      :slow_flush_count,
+    ]
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def configure(conf)
+      super
+      hostname = Socket.gethostname
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
+      @base_labels = parse_labels_elements(conf)
+      @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
+        end
+        @base_labels[key] = expander.expand(value)
+      end
+
+      @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+
+      @gauge_or_counter = @gauge_all ? :gauge : :counter
+    end
+
+    def start
+      super
+
+      @metrics = {
+        # Buffer metrics
+        buffer_total_queued_size: get_gauge(
+          :fluentd_output_status_buffer_total_bytes,
+          'Current total size of stage and queue buffers.'),
+        buffer_stage_length: get_gauge(
+          :fluentd_output_status_buffer_stage_length,
+          'Current length of stage buffers.'),
+        buffer_stage_byte_size: get_gauge(
+          :fluentd_output_status_buffer_stage_byte_size,
+          'Current total size of stage buffers.'),
+        buffer_queue_length: get_gauge(
+          :fluentd_output_status_buffer_queue_length,
+          'Current length of queue buffers.'),
+        buffer_queue_byte_size: get_gauge(
+          :fluentd_output_status_queue_byte_size,
+          'Current total size of queue buffers.'),
+        buffer_available_buffer_space_ratios: get_gauge(
+          :fluentd_output_status_buffer_available_space_ratio,
+          'Ratio of available space in buffer.'),
+        buffer_newest_timekey: get_gauge(
+          :fluentd_output_status_buffer_newest_timekey,
+          'Newest timekey in buffer.'),
+        buffer_oldest_timekey: get_gauge(
+          :fluentd_output_status_buffer_oldest_timekey,
+          'Oldest timekey in buffer.'),
+
+        # Output metrics
+        retry_counts: get_gauge_or_counter(
+          :fluentd_output_status_retry_count,
+          'Current retry counts.'),
+        num_errors: get_gauge_or_counter(
+          :fluentd_output_status_num_errors,
+          'Current number of errors.'),
+        emit_count: get_gauge_or_counter(
+          :fluentd_output_status_emit_count,
+          'Current emit counts.'),
+        emit_records: get_gauge_or_counter(
+          :fluentd_output_status_emit_records,
+          'Current emit records.'),
+        write_count: get_gauge_or_counter(
+          :fluentd_output_status_write_count,
+          'Current write counts.'),
+        rollback_count: get_gauge(
+          :fluentd_output_status_rollback_count,
+          'Current rollback counts.'),
+        flush_time_count: get_gauge_or_counter(
+          :fluentd_output_status_flush_time_count,
+          'Total flush time.'),
+        slow_flush_count: get_gauge_or_counter(
+          :fluentd_output_status_slow_flush_count,
+          'Current slow flush counts.'),
+        retry_wait: get_gauge(
+          :fluentd_output_status_retry_wait,
+          'Current retry wait'),
+      }
+      timer_execute(:in_prometheus_output_monitor, @interval, &method(:update_monitor_info))
+    end
+
+    def update_monitor_info
+      opts = {
+        ivars: MONITOR_IVARS,
+        with_retry: true,
+      }
+
+      agent_info = @monitor_agent.plugins_info_all(opts).select {|info|
+        info['plugin_category'] == 'output'.freeze
+      }
+
+      monitor_info = {
+        # buffer metrics
+        'buffer_total_queued_size' => [@metrics[:buffer_total_queued_size]],
+        'buffer_stage_length' => [@metrics[:buffer_stage_length]],
+        'buffer_stage_byte_size' => [@metrics[:buffer_stage_byte_size]],
+        'buffer_queue_length' => [@metrics[:buffer_queue_length]],
+        'buffer_queue_byte_size' => [@metrics[:buffer_queue_byte_size]],
+        'buffer_available_buffer_space_ratios' => [@metrics[:buffer_available_buffer_space_ratios]],
+        'buffer_newest_timekey' => [@metrics[:buffer_newest_timekey]],
+        'buffer_oldest_timekey' => [@metrics[:buffer_oldest_timekey]],
+
+        # output metrics
+        'retry_count' => [@metrics[:retry_counts], @metrics[:num_errors]],
+        # Needed since Fluentd v1.14 due to metrics extensions.
+        'write_count' => [@metrics[:write_count]],
+        'emit_count' => [@metrics[:emit_count]],
+        'emit_records' => [@metrics[:emit_records]],
+        'rollback_count' => [@metrics[:rollback_count]],
+        'flush_time_count' => [@metrics[:flush_time_count]],
+        'slow_flush_count' => [@metrics[:slow_flush_count]],
+      }
+      # No needed for Fluentd v1.14 but leave as-is for backward compatibility.
+      instance_vars_info = {
+        num_errors: @metrics[:num_errors],
+        write_count: @metrics[:write_count],
+        emit_count: @metrics[:emit_count],
+        emit_records: @metrics[:emit_records],
+        rollback_count: @metrics[:rollback_count],
+        flush_time_count: @metrics[:flush_time_count],
+        slow_flush_count: @metrics[:slow_flush_count],
+      }
+
+      agent_info.each do |info|
+        label = labels(info)
+
+        monitor_info.each do |name, metrics|
+          metrics.each do |metric|
+            if info[name]
+              if metric.is_a?(::Prometheus::Client::Gauge)
+                metric.set(info[name], labels: label)
+              elsif metric.is_a?(::Prometheus::Client::Counter)
+                metric.increment(by: info[name] - metric.get(labels: label), labels: label)
+              end
+            end
+          end
+        end
+
+        if info['instance_variables']
+          instance_vars_info.each do |name, metric|
+            if info['instance_variables'][name]
+              if metric.is_a?(::Prometheus::Client::Gauge)
+                metric.set(info['instance_variables'][name], labels: label)
+              elsif metric.is_a?(::Prometheus::Client::Counter)
+                metric.increment(by: info['instance_variables'][name] - metric.get(labels: label), labels: label)
+              end
+            end
+          end
+        end
+
+        # compute current retry_wait
+        if info['retry']
+          next_time = info['retry']['next_time']
+          start_time = info['retry']['start']
+          if start_time.nil? && info['instance_variables']
+            # v0.12 does not include start, use last_retry_time instead
+            start_time = info['instance_variables'][:last_retry_time]
+          end
+
+          wait = 0
+          if next_time && start_time
+            wait = next_time - start_time
+          end
+          @metrics[:retry_wait].set(wait.to_f, labels: label)
+        end
+      end
+    end
+
+    def labels(plugin_info)
+      @base_labels.merge(
+        plugin_id: plugin_info["plugin_id"],
+        type: plugin_info["type"],
+      )
+    end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
+      end
+    end
+
+    def get_gauge_or_counter(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.public_send(@gauge_or_counter, name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -0,0 +1,98 @@
+require 'fluent/plugin/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/plugin/prometheus'
+
+module Fluent::Plugin
+  class PrometheusTailMonitorInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus_tail_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
+
+    helpers :timer
+
+    config_param :interval, :time, default: 5
+    attr_reader :registry
+
+    MONITOR_IVARS = [
+      :tails,
+    ]
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def configure(conf)
+      super
+      hostname = Socket.gethostname
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
+      @base_labels = parse_labels_elements(conf)
+      @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"
+        end
+        @base_labels[key] = expander.expand(value)
+      end
+
+      @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+    end
+
+    def start
+      super
+
+      @metrics = {
+        position: get_gauge(
+          :fluentd_tail_file_position,
+          'Current position of file.'),
+        inode: get_gauge(
+          :fluentd_tail_file_inode,
+          'Current inode of file.'),
+      }
+      timer_execute(:in_prometheus_tail_monitor, @interval, &method(:update_monitor_info))
+    end
+
+    def update_monitor_info
+      opts = {
+        ivars: MONITOR_IVARS,
+      }
+
+      agent_info = @monitor_agent.plugins_info_all(opts).select {|info|
+        info['type'] == 'tail'.freeze
+      }
+
+      agent_info.each do |info|
+        tails = info['instance_variables'][:tails]
+        next if tails.nil?
+
+        tails.clone.each do |_, watcher|
+          # Access to internal variable of internal class...
+          # Very fragile implementation
+          pe = watcher.instance_variable_get(:@pe)
+          label = labels(info, watcher.path)
+          @metrics[:inode].set(pe.read_inode, labels: label)
+          @metrics[:position].set(pe.read_pos, labels: label)
+        end
+      end
+    end
+
+    def labels(plugin_info, path)
+      @base_labels.merge(
+        plugin_id: plugin_info["plugin_id"],
+        type: plugin_info["type"],
+        path: path,
+      )
+    end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type, :path])
+      end
+    end
+  end
+end