fluent-plugin-prometheus 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bc459ea9922aeb0b292c93035e176c4a3c2dfa03
-  data.tar.gz: c2598bbe1b5e3cd7de282d7db394c0f5b0a89723
+  metadata.gz: 85a4bc3e525bcf86677e899b7e6fc565cff5cfff
+  data.tar.gz: 76db50326565b85f18ef1ddf5bbf08dd7d4f3de8
 SHA512:
-  metadata.gz: 7da56621296f4f7f279334f998e5f238fadf881ce08d4aef8dfaab2d96a81e490db42689a276b544e3830b87ba83a8507168d33c3006df17164ed3551a14c2b0
-  data.tar.gz: ea09dd18c3d52d3b852c917c661318d5143ba4a9946c34eaa03adfc160d6fc1179676fdb635d44ca15fa99dacb3b3676f1ab6e688883835c17de3c8ad7b79ea7
+  metadata.gz: ca87c98ff6ffb9b0c2b7ad82b8f5d5a875fcc70f5b6e539e59fa15f1d3624d08d0cfc13ae47ccaa1eeed6c5d2087655ec9d21b8c8ff342f69fe96c42789fb095
+  data.tar.gz: 80d39b58eb5289f0d8f2b5d5b4f8e7eec58ce3b7bcae4dcfa00abdf6699e6dc73c61498806803f7d2eac80ce9f95dfa42a8ffc1d220ef243396701ac64d3ae80

data/.travis.yml

@@ -1,13 +1,18 @@
 language: ruby
 
 rvm:
-  - 2.2
-  - 2.3.1
+  - "2.3.3"
+  - "2.4.0"
 
 gemfile:
   - Gemfile
-  - Gemfile.fluentd.0.10
   - Gemfile.fluentd.0.12
+  - Gemfile.fluentd.0.10
+
+matrix:
+  allow_failures:
+    - gemfile: Gemfile.fluentd.0.10
+      rvm: 2.4.0
 
 script:
   - bundle exec rake spec

data/README.md

@@ -73,6 +73,70 @@ More configuration parameters:
 - `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
 - `interval`: interval to update monitor_agent information in seconds (default: 5)
 
+### prometheus_output_monitor input plugin
+
+This plugin collects internal metrics for output plugin in Fluentd. This is similar to `prometheus_monitor` plugin, but specialized for input plugin. There are Many metrics `prometheus_monitor` does not include, such as `num_errors`, `retry_wait` and so on.
+
+Current exposed metrics:
+
+- `fluentd_output_status_buffer_queue_length`
+- `fluentd_output_status_buffer_total_bytes`
+- `fluentd_output_status_retry_count`
+- `fluentd_output_status_num_errors`
+- `fluentd_output_status_emit_count`
+- `fluentd_output_status_retry_wait`
+    - current retry_wait computed from last retry time and next retry time
+- `fluentd_output_status_emit_records`
+    - only for v0.14
+- `fluentd_output_status_write_count`
+    - only for v0.14
+- `fluentd_output_status_rollback_count`
+    - only for v0.14
+
+
+With following configuration, those metrics are collected.
+
+<source>
+  type prometheus_output_monitor
+</source>
+
+More configuration parameters:
+
+- `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
+- `interval`: interval to update monitor_agent information in seconds (default: 5)
+
+### prometheus_tail_monitor input plugin
+
+**experimental**
+
+This plugin collects internal metrics for in_tail plugin in Fluentd. in_tail plugin holds internal state for files that the plugin is watching. The state is sometimes important to monitor plugins work correctly.
+
+This plugin uses internal class of Fluentd, so it's easy to break.
+
+Current exposed metrics:
+
+- `fluentd_tail_file_position`
+    - Current bytes which plugin reads from the file
+- `fluentd_tail_file_inode`
+    - inode of the file
+
+Default labels:
+
+- `plugin_id`: a value set for a plugin in configuration.
+- `type`: plugin name. `in_tail` only for now.
+- `path`: file path
+
+With following configuration, those metrics are collected.
+
+<source>
+  type prometheus_output_monitor
+</source>
+
+More configuration parameters:
+
+- `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
+- `interval`: interval to update monitor_agent information in seconds (default: 5)
+
 ### prometheus output/filter plugin
 
 Both output/filter plugins instrument metrics from records. Both plugins have no impact against values of each records, just read.
@@ -315,8 +379,8 @@ $ bundle install --path vendor/bundle
 Download pre-compiled prometheus binary and start it. It listens on 9090.
 
 ```
-$ wget https://github.com/prometheus/prometheus/releases/download/v1.0.1/prometheus-1.0.1.linux-amd64.tar.gz -O - | tar zxf -
-$ ./prometheus-1.0.1.linux-amd64/prometheus -config.file=./misc/prometheus.yaml -storage.local.path=./prometheus/metrics
+$ wget https://github.com/prometheus/prometheus/releases/download/v1.5.2/prometheus-1.5.2.linux-amd64.tar.gz -O - | tar zxf -
+$ ./prometheus-1.5.2.linux-amd64/prometheus -config.file=./misc/prometheus.yaml -storage.local.path=./prometheus/metrics
 ```
 
 Install Nginx for sample metrics. It listens on 80 and 9999.

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "0.2.1"
+  spec.version       = "0.3.0"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}

data/lib/fluent/plugin/in_prometheus.rb

@@ -22,6 +22,7 @@ module Fluent
     end
 
     def start
+      super
       @server = WEBrick::HTTPServer.new(
         BindAddress: @bind,
         Port: @port,
@@ -33,6 +34,7 @@ module Fluent
     end
 
     def shutdown
+      super
       if @server
         @server.shutdown
         @server = nil

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -1,7 +1,6 @@
 require 'fluent/input'
 require 'fluent/plugin/in_monitor_agent'
 require 'fluent/plugin/prometheus'
-require 'webrick'
 
 module Fluent
   class PrometheusMonitorInput < Input
@@ -65,6 +64,7 @@ module Fluent
     end
 
     def start
+      super
       @loop = Coolio::Loop.new
       @timer = TimerWatcher.new(@interval, true, log, &method(:update_monitor_info))
       @loop.attach(@timer)
@@ -72,6 +72,7 @@ module Fluent
     end
 
     def shutdown
+      super
       @loop.watchers.each {|w| w.detach }
       @loop.stop
       @thread.join

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -0,0 +1,178 @@
+require 'fluent/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/plugin/prometheus'
+
+module Fluent
+  class PrometheusOutputMonitorInput < Input
+    Plugin.register_input('prometheus_output_monitor', self)
+
+    config_param :interval, :time, :default => 5
+    attr_reader :registry
+
+    MONITOR_IVARS = [
+      :num_errors,
+      :emit_count,
+
+      # for v0.12
+      :last_retry_time,
+
+      # from v0.14
+      :emit_records,
+      :write_count,
+      :rollback_count,
+    ]
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def configure(conf)
+      super
+      hostname = Socket.gethostname
+      expander = Fluent::Prometheus.placeholder_expander(log)
+      placeholders = expander.prepare_placeholders({'hostname' => hostname})
+      @base_labels = Fluent::Prometheus.parse_labels_elements(conf)
+      @base_labels.each do |key, value|
+        @base_labels[key] = expander.expand(value, placeholders)
+      end
+
+      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
+        # from v0.14.6
+        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+      else
+        @monitor_agent = Fluent::MonitorAgentInput.new
+      end
+
+      @metrics = {
+        buffer_queue_length: @registry.gauge(
+          :fluentd_output_status_buffer_queue_length,
+          'Current buffer queue length.'),
+        buffer_total_queued_size: @registry.gauge(
+          :fluentd_output_status_buffer_total_bytes,
+          'Current total size of queued buffers.'),
+        retry_counts: @registry.gauge(
+          :fluentd_output_status_retry_count,
+          'Current retry counts.'),
+        num_errors: @registry.gauge(
+          :fluentd_output_status_num_errors,
+          'Current number of errors.'),
+        emit_count: @registry.gauge(
+          :fluentd_output_status_emit_count,
+          'Current emit counts.'),
+        emit_records: @registry.gauge(
+          :fluentd_output_status_emit_records,
+          'Current emit records.'),
+        write_count: @registry.gauge(
+          :fluentd_output_status_write_count,
+          'Current write counts.'),
+        rollback_count: @registry.gauge(
+          :fluentd_output_status_rollback_count,
+          'Current rollback counts.'),
+        retry_wait: @registry.gauge(
+          :fluentd_output_status_retry_wait,
+          'Current retry wait'),
+      }
+    end
+
+    class TimerWatcher < Coolio::TimerWatcher
+      def initialize(interval, repeat, log, &callback)
+        @callback = callback
+        @log = log
+        super(interval, repeat)
+      end
+
+      def on_timer
+        @callback.call
+      rescue
+        @log.error $!.to_s
+        @log.error_backtrace
+      end
+    end
+
+    def start
+      super
+      @loop = Coolio::Loop.new
+      @timer = TimerWatcher.new(@interval, true, log, &method(:update_monitor_info))
+      @loop.attach(@timer)
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      super
+      @loop.watchers.each {|w| w.detach }
+      @loop.stop
+      @thread.join
+    end
+
+    def run
+      @loop.run
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
+    end
+
+    def update_monitor_info
+      opts = {
+        ivars: MONITOR_IVARS,
+        with_retry: true,
+      }
+
+      agent_info = @monitor_agent.plugins_info_all(opts).select {|info|
+        info['plugin_category'] == 'output'.freeze
+      }
+
+      monitor_info = {
+        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        'buffer_total_queued_size' => @metrics[:buffer_total_queued_size],
+        'retry_count' => @metrics[:retry_counts],
+      }
+      instance_vars_info = {
+        num_errors: @metrics[:num_errors],
+        write_count: @metrics[:write_count],
+        emit_count: @metrics[:emit_count],
+        emit_records: @metrics[:emit_records],
+        rollback_count: @metrics[:rollback_count],
+      }
+
+      agent_info.each do |info|
+        label = labels(info)
+
+        monitor_info.each do |name, metric|
+          if info[name]
+            metric.set(label, info[name])
+          end
+        end
+
+        if info['instance_variables']
+          instance_vars_info.each do |name, metric|
+            if info['instance_variables'][name]
+              metric.set(label, info['instance_variables'][name])
+            end
+          end
+        end
+
+        # compute current retry_wait
+        if info['retry']
+          next_time = info['retry']['next_time']
+          start_time = info['retry']['start']
+          if start_time.nil? && info['instance_variables']
+            # v0.12 does not include start, use last_retry_time instead
+            start_time = info['instance_variables'][:last_retry_time]
+          end
+          if next_time && start_time
+            wait = next_time - start_time
+            @metrics[:retry_wait].set(label, wait.to_f)
+          end
+        end
+      end
+    end
+
+    def labels(plugin_info)
+      @base_labels.merge(
+        plugin_id: plugin_info["plugin_id"],
+        type: plugin_info["type"],
+      )
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -0,0 +1,117 @@
+require 'fluent/input'
+require 'fluent/plugin/in_monitor_agent'
+require 'fluent/plugin/prometheus'
+
+module Fluent
+  class PrometheusTailMonitorInput < Input
+    Plugin.register_input('prometheus_tail_monitor', self)
+
+    config_param :interval, :time, :default => 5
+    attr_reader :registry
+
+    MONITOR_IVARS = [
+      :tails,
+    ]
+
+    def initialize
+      super
+      @registry = ::Prometheus::Client.registry
+    end
+
+    def configure(conf)
+      super
+      hostname = Socket.gethostname
+      expander = Fluent::Prometheus.placeholder_expander(log)
+      placeholders = expander.prepare_placeholders({'hostname' => hostname})
+      @base_labels = Fluent::Prometheus.parse_labels_elements(conf)
+      @base_labels.each do |key, value|
+        @base_labels[key] = expander.expand(value, placeholders)
+      end
+
+      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
+        # from v0.14.6
+        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+      else
+        @monitor_agent = Fluent::MonitorAgentInput.new
+      end
+
+      @metrics = {
+        position: @registry.gauge(
+          :fluentd_tail_file_position,
+          'Current position of file.'),
+        inode: @registry.gauge(
+          :fluentd_tail_file_inode,
+          'Current inode of file.'),
+      }
+    end
+
+    class TimerWatcher < Coolio::TimerWatcher
+      def initialize(interval, repeat, log, &callback)
+        @callback = callback
+        @log = log
+        super(interval, repeat)
+      end
+
+      def on_timer
+        @callback.call
+      rescue
+        @log.error $!.to_s
+        @log.error_backtrace
+      end
+    end
+
+    def start
+      super
+      @loop = Coolio::Loop.new
+      @timer = TimerWatcher.new(@interval, true, log, &method(:update_monitor_info))
+      @loop.attach(@timer)
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      super
+      @loop.watchers.each {|w| w.detach }
+      @loop.stop
+      @thread.join
+    end
+
+    def run
+      @loop.run
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
+    end
+
+    def update_monitor_info
+      opts = {
+        ivars: MONITOR_IVARS,
+      }
+
+      agent_info = @monitor_agent.plugins_info_all(opts).select {|info|
+        info['type'] == 'tail'.freeze
+      }
+
+      agent_info.each do |info|
+        tails = info['instance_variables'][:tails]
+        next if tails.nil?
+
+        tails.each do |_, watcher|
+          # Access to internal variable of internal class...
+          # Very fragile implementation
+          pe = watcher.instance_variable_get(:@pe)
+          label = labels(info, watcher.path)
+          @metrics[:inode].set(label, pe.read_inode)
+          @metrics[:position].set(label, pe.read_pos)
+        end
+      end
+    end
+
+    def labels(plugin_info, path)
+      @base_labels.merge(
+        plugin_id: plugin_info["plugin_id"],
+        type: plugin_info["type"],
+        path: path,
+      )
+    end
+  end
+end

data/misc/fluentd_sample.conf

@@ -13,6 +13,22 @@
   </labels>
 </source>
 
+# input plugin that collects metrics for output plugin
+<source>
+  @type prometheus_output_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+# input plugin that collects metrics for in_tail plugin
+<source>
+  @type prometheus_tail_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
 ## Nginx Access Log Configuration
 
 <source>

data/misc/prometheus.yaml

@@ -1,8 +1,13 @@
 # A job to scrape an endpoint of Fluentd running on localhost.
 scrape_configs:
+- job_name: 'prometheus'
+  scrape_interval: 5s
+  static_configs:
+    - targets:
+      - 'localhost:9090'
 - job_name: fluentd
   scrape_interval: 5s
   static_configs:
     - targets:
-        - 'localhost:24231'
+      - 'localhost:24231'
   metrics_path: /metrics

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-11 00:00:00.000000000 Z
+date: 2017-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -114,6 +114,8 @@ files:
 - lib/fluent/plugin/filter_prometheus.rb
 - lib/fluent/plugin/in_prometheus.rb
 - lib/fluent/plugin/in_prometheus_monitor.rb
+- lib/fluent/plugin/in_prometheus_output_monitor.rb
+- lib/fluent/plugin/in_prometheus_tail_monitor.rb
 - lib/fluent/plugin/out_prometheus.rb
 - lib/fluent/plugin/prometheus.rb
 - misc/fluentd_sample.conf
@@ -144,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.3
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.