fluent-plugin-prometheus 1.4.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80f51eb873f6e3a90a0e68b0afbeb92c1efefbf5a5336dda94aea65e78227adb
-  data.tar.gz: 5182614c1617d843d584e749c8402e6c70f17cf09376ea1f9adcc8dd6901074a
+  metadata.gz: 32e8a803f1394e445d273b41ec9bde5f238f4193956c431a0866c93b2a2a0426
+  data.tar.gz: 281024e977c3aee02030691ffb7cd86361b759dae259bb8103934db7e195f751
 SHA512:
-  metadata.gz: 62fe2b113f75baef5bb2e5e452d64b8f1d5734edacbf7989c164b959666fb2515a26d7800299c46a0121acd71a2a091d436d45baa3d53b105dc436839a8fb635
-  data.tar.gz: 5296bfc10750933363b2d08b79d55617d4edc65a4a7a55be4a4cd9fefb3515b1f44175b61f5f4e1ae49a8048cb2856bbbab54377be8272eb2029a2d891ebf91e
+  metadata.gz: 6e784fec1bca9a2ea8f725b5970ed5f08069b6d457ae11b23e1c22017d188046cab4718573c67a94b230acff1583ac12d6378a372a3713f311470c2b9a5818e3
+  data.tar.gz: 805673a26e562f4607cefbbf256c12e3557317e115a88096357ddde30f6d5f8dea67835e580100c6bfb5aa9abfb542d0f5b0bff9374b6d4b48046034fea5527a

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 
 rvm:
-  - "2.3.6"
-  - "2.4.3"
-  - "2.5.0"
+  - "2.4.7"
+  - "2.5"
+  - "2.6"
 
 gemfile:
   - Gemfile

data/README.md

@@ -58,18 +58,25 @@ More configuration parameters:
 - `bind`: binding interface (default: '0.0.0.0')
 - `port`: listen port (default: 24231)
 - `metrics_path`: metrics HTTP endpoint (default: /metrics)
+- `aggregated_metrics_path`: metrics HTTP endpoint (default: /aggregated_metrics)
 
 When using multiple workers, each worker binds to port + `fluent_worker_id`.
+To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
 
 ### prometheus_monitor input plugin
 
-This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](http://docs.fluentd.org/articles/monitoring#monitoring-agent).
+This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](https://docs.fluentd.org/input/monitor_agent).
 
-Current exposed metrics:
 
-- `buffere_queue_length` of each BufferedOutput plugins
-- `buffer_total_queued_size` of each BufferedOutput plugins
-- `retry_count` of each BufferedOutput plugins
+#### Exposed metrics
+
+- `fluentd_status_buffer_queue_length`
+- `fluentd_status_buffer_total_queued_size`
+- `fluentd_status_retry_count`
+- `fluentd_status_buffer_newest_timekey` from fluentd v1.4.2
+- `fluentd_status_buffer_oldest_timekey` from fluentd v1.4.2
+
+#### Configuration
 
 With following configuration, those metrics are collected.
 
@@ -86,26 +93,35 @@ More configuration parameters:
 
 ### prometheus_output_monitor input plugin
 
-**experimental**
-
 This plugin collects internal metrics for output plugin in Fluentd. This is similar to `prometheus_monitor` plugin, but specialized for output plugin. There are Many metrics `prometheus_monitor` does not include, such as `num_errors`, `retry_wait` and so on.
 
-Current exposed metrics:
+#### Exposed metrics
+
+Metrics for output
 
-- `fluentd_output_status_buffer_queue_length`
-- `fluentd_output_status_buffer_total_bytes`
 - `fluentd_output_status_retry_count`
 - `fluentd_output_status_num_errors`
 - `fluentd_output_status_emit_count`
 - `fluentd_output_status_retry_wait`
     - current retry_wait computed from last retry time and next retry time
 - `fluentd_output_status_emit_records`
-    - only for v0.14
 - `fluentd_output_status_write_count`
-    - only for v0.14
 - `fluentd_output_status_rollback_count`
-    - only for v0.14
+- `fluentd_output_status_flush_time_count` from fluentd v1.6.0
+- `fluentd_output_status_slow_flush_count` from fluentd v1.6.0
+
+Metrics for buffer
+
+- `fluentd_output_status_buffer_total_bytes`
+- `fluentd_output_status_buffer_stage_length` from fluentd v1.6.0
+- `fluentd_output_status_buffer_stage_byte_size` from fluentd v1.6.0
+- `fluentd_output_status_buffer_queue_length`
+- `fluentd_output_status_buffer_queue_byte_size` from fluentd v1.6.0
+- `fluentd_output_status_buffer_newest_timekey` from fluentd v1.6.0
+- `fluentd_output_status_buffer_oldest_timekey` from fluentd v1.6.0
+- `fluentd_output_status_buffer_available_space_ratio` from fluentd v1.6.0
 
+#### Configuration
 
 With following configuration, those metrics are collected.
 
@@ -122,13 +138,11 @@ More configuration parameters:
 
 ### prometheus_tail_monitor input plugin
 
-**experimental**
-
 This plugin collects internal metrics for in_tail plugin in Fluentd. in_tail plugin holds internal state for files that the plugin is watching. The state is sometimes important to monitor plugins work correctly.
 
 This plugin uses internal class of Fluentd, so it's easy to break.
 
-Current exposed metrics:
+#### Exposed metrics
 
 - `fluentd_tail_file_position`
     - Current bytes which plugin reads from the file
@@ -141,6 +155,8 @@ Default labels:
 - `type`: plugin name. `in_tail` only for now.
 - `path`: file path
 
+#### Configuration
+
 With following configuration, those metrics are collected.
 
 ```
@@ -224,7 +240,7 @@ In output plugin style:
 
 With above configuration, the plugin collects a metric named `message_foo_counter` from key `foo` of each records.
 
-You can access nested keys in records via dot or bracket notation (https://docs.fluentd.org/v1.0/articles/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`.
+You can access nested keys in records via dot or bracket notation (https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`.
 
 See Supported Metric Type and Labels for more configuration parameters.
 
@@ -341,7 +357,7 @@ You can add labels with static value or dynamic value from records. In `promethe
 
 All labels sections has same format. Each lines have key/value for label.
 
-You can access nested fields in records via dot or bracket notation (https://docs.fluentd.org/v1.0/articles/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`. Other values are handled as raw string as is and may be expanded by placeholder described later.
+You can access nested fields in records via dot or bracket notation (https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`. Other values are handled as raw string as is and may be expanded by placeholder described later.
 
 You can use placeholder for label values. The placeholders will be expanded from reserved values and records.
 If you specify `${hostname}`, it will be expanded by value of a hostname where fluentd runs.
@@ -353,7 +369,13 @@ Reserved placeholders are:
 - `${worker_id}`: fluent worker id
 - `${tag}`: tag name
   - only available in Prometheus output/filter plugin
-
+- `${tag_parts[N]}` refers to the Nth part of the tag.
+  - only available in Prometheus output/filter plugin
+- `${tag_prefix[N]}` refers to the [0..N] part of the tag.
+  - only available in Prometheus output/filter plugin
+- `${tag_suffix[N]}` refers to the [`tagsize`-1-N..] part of the tag.
+  - where `tagsize` is the size of tag which is splitted with `.` (when tag is `1.2.3`, then `tagsize` is 3)
+  - only available in Prometheus output/filter plugin
 
 ### top-level labels and labels inside metric
 

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.4.0"
+  spec.version       = "1.7.1"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "fluentd", ">= 0.14.20", "< 2"
-  spec.add_dependency "prometheus-client"
+  spec.add_dependency "prometheus-client", "< 0.10"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/lib/fluent/plugin/filter_prometheus.rb

@@ -4,6 +4,7 @@ require 'fluent/plugin/filter'
 module Fluent::Plugin
   class PrometheusFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
     include Fluent::Plugin::Prometheus
 
     def initialize
@@ -17,13 +18,13 @@ module Fluent::Plugin
 
     def configure(conf)
       super
-      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      labels = parse_labels_elements(conf)
       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 
-    def filter_stream(tag, es)
-      instrument(tag, es, @metrics)
-      es
+    def filter(tag, time, record)
+      instrument_single(tag, time, record, @metrics)
+      record
     end
   end
 end

data/lib/fluent/plugin/in_prometheus.rb

@@ -1,5 +1,7 @@
 require 'fluent/plugin/input'
 require 'fluent/plugin/prometheus'
+require 'fluent/plugin/prometheus_metrics'
+require 'net/http'
 require 'webrick'
 
 module Fluent::Plugin
@@ -11,6 +13,7 @@ module Fluent::Plugin
     config_param :bind, :string, default: '0.0.0.0'
     config_param :port, :integer, default: 24231
     config_param :metrics_path, :string, default: '/metrics'
+    config_param :aggregated_metrics_path, :string, default: '/aggregated_metrics'
 
     desc 'Enable ssl configuration for the server'
     config_section :ssl, required: false, multi: false do
@@ -31,6 +34,10 @@ module Fluent::Plugin
 
     attr_reader :registry
 
+    attr_reader :num_workers
+    attr_reader :base_port
+    attr_reader :metrics_path
+
     def initialize
       super
       @registry = ::Prometheus::Client.registry
@@ -38,6 +45,18 @@ module Fluent::Plugin
 
     def configure(conf)
       super
+
+      # Get how many workers we have
+      sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                  owner.system_config
+                elsif self.respond_to?(:system_config)
+                  self.system_config
+                else
+                  nil
+                end
+      @num_workers = sysconf && sysconf.workers ? sysconf.workers : 1
+
+      @base_port = @port
       @port += fluentd_worker_id
     end
 
@@ -82,6 +101,7 @@ module Fluent::Plugin
 
       @server = WEBrick::HTTPServer.new(config)
       @server.mount(@metrics_path, MonitorServlet, self)
+      @server.mount(@aggregated_metrics_path, MonitorServletAll, self)
       thread_create(:in_prometheus) do
         @server.start
       end
@@ -110,5 +130,35 @@ module Fluent::Plugin
         res.body = $!.to_s
       end
     end
+
+    class MonitorServletAll < WEBrick::HTTPServlet::AbstractServlet
+      def initialize(server, prometheus)
+        @prometheus = prometheus
+      end
+
+      def do_GET(req, res)
+        res.status = 200
+        res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
+
+        full_result = PromMetricsAggregator.new
+        fluent_server_ip = @prometheus.bind == '0.0.0.0' ? '127.0.0.1' : @prometheus.bind
+        current_worker = 0
+        while current_worker < @prometheus.num_workers
+          Net::HTTP.start(fluent_server_ip, @prometheus.base_port + current_worker) do |http|
+            req = Net::HTTP::Get.new(@prometheus.metrics_path)
+            result = http.request(req)
+            if result.is_a?(Net::HTTPSuccess)
+              full_result.add_metrics(result.body)
+            end
+          end
+          current_worker += 1
+        end
+        res.body = full_result.get_metrics
+      rescue
+        res.status = 500
+        res['Content-Type'] = 'text/plain'
+        res.body = $!.to_s
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -23,14 +24,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
@@ -45,6 +46,12 @@ module Fluent::Plugin
     def start
       super
 
+      @buffer_newest_timekey = @registry.gauge(
+        :fluentd_status_buffer_newest_timekey,
+        'Newest timekey in buffer.')
+      @buffer_oldest_timekey = @registry.gauge(
+        :fluentd_status_buffer_oldest_timekey,
+        'Oldest timekey in buffer.')
       buffer_queue_length = @registry.gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
@@ -65,11 +72,19 @@ module Fluent::Plugin
 
     def update_monitor_info
       @monitor_agent.plugins_info_all.each do |info|
+        label = labels(info)
+
         @monitor_info.each do |name, metric|
           if info[name]
-            metric.set(labels(info), info[name])
+            metric.set(label, info[name])
           end
         end
+
+        timekeys = info["buffer_timekeys"]
+        if timekeys && !timekeys.empty?
+          @buffer_newest_timekey.set(label, timekeys.max)
+          @buffer_oldest_timekey.set(label, timekeys.min)
+        end
       end
     end
 

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusOutputMonitorInput < Fluent::Input
     Fluent::Plugin.register_input('prometheus_output_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -24,6 +25,10 @@ module Fluent::Plugin
       :emit_records,
       :write_count,
       :rollback_count,
+
+      # from v1.6.0
+      :flush_time_count,
+      :slow_flush_count,
     ]
 
     def initialize
@@ -38,14 +43,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.buidl({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
@@ -60,12 +65,33 @@ module Fluent::Plugin
       super
 
       @metrics = {
-        buffer_queue_length: @registry.gauge(
-          :fluentd_output_status_buffer_queue_length,
-          'Current buffer queue length.'),
+        # Buffer metrics
         buffer_total_queued_size: @registry.gauge(
           :fluentd_output_status_buffer_total_bytes,
-          'Current total size of queued buffers.'),
+          'Current total size of stage and queue buffers.'),
+        buffer_stage_length: @registry.gauge(
+          :fluentd_output_status_buffer_stage_length,
+          'Current length of stage buffers.'),
+        buffer_stage_byte_size: @registry.gauge(
+          :fluentd_output_status_buffer_stage_byte_size,
+          'Current total size of stage buffers.'),
+        buffer_queue_length: @registry.gauge(
+          :fluentd_output_status_buffer_queue_length,
+          'Current length of queue buffers.'),
+        buffer_queue_byte_size: @registry.gauge(
+          :fluentd_output_status_queue_byte_size,
+          'Current total size of queue buffers.'),
+        buffer_available_buffer_space_ratios: @registry.gauge(
+          :fluentd_output_status_buffer_available_space_ratio,
+          'Ratio of available space in buffer.'),
+        buffer_newest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_newest_timekey,
+          'Newest timekey in buffer.'),
+        buffer_oldest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_oldest_timekey,
+          'Oldest timekey in buffer.'),
+
+        # Output metrics
         retry_counts: @registry.gauge(
           :fluentd_output_status_retry_count,
           'Current retry counts.'),
@@ -84,6 +110,12 @@ module Fluent::Plugin
         rollback_count: @registry.gauge(
           :fluentd_output_status_rollback_count,
           'Current rollback counts.'),
+        flush_time_count: @registry.gauge(
+          :fluentd_output_status_flush_time_count,
+          'Total flush time.'),
+        slow_flush_count: @registry.gauge(
+          :fluentd_output_status_slow_flush_count,
+          'Current slow flush counts.'),
         retry_wait: @registry.gauge(
           :fluentd_output_status_retry_wait,
           'Current retry wait'),
@@ -102,8 +134,17 @@ module Fluent::Plugin
       }
 
       monitor_info = {
-        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        # buffer metrics
         'buffer_total_queued_size' => @metrics[:buffer_total_queued_size],
+        'buffer_stage_length' => @metrics[:buffer_stage_length],
+        'buffer_stage_byte_size' => @metrics[:buffer_stage_byte_size],
+        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        'buffer_queue_byte_size' => @metrics[:buffer_queue_byte_size],
+        'buffer_available_buffer_space_ratios' => @metrics[:buffer_available_buffer_space_ratios],
+        'buffer_newest_timekey' => @metrics[:buffer_newest_timekey],
+        'buffer_oldest_timekey' => @metrics[:buffer_oldest_timekey],
+
+        # output metrics
         'retry_count' => @metrics[:retry_counts],
       }
       instance_vars_info = {
@@ -112,6 +153,8 @@ module Fluent::Plugin
         emit_count: @metrics[:emit_count],
         emit_records: @metrics[:emit_records],
         rollback_count: @metrics[:rollback_count],
+        flush_time_count: @metrics[:flush_time_count],
+        slow_flush_count: @metrics[:slow_flush_count],
       }
 
       agent_info.each do |info|