fluent-plugin-prometheus 1.2.1 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64ed448a556ffbc5fda04297363a53663dc2acff960d358bb24b07e227b99b3a
-  data.tar.gz: 2ad3f13b35c60b0fad6664831e9d4edcacfcf963a3a819c658aad761bc64af2f
+  metadata.gz: 4d8cc397b9977b5dc5215828a0ef11bfe6700c2c364214a5115636012ea06480
+  data.tar.gz: 24157ee77c8cfbda5e1b5612e97be016e46373c68e78068f0cdd504d87991ae1
 SHA512:
-  metadata.gz: cb6b43bf8e18dea2ff324a1a75738b51513c652ca3afdea8bf941f5d56e0b44028d19726daef5ee704070597e243e0c5262db1edaf6027adb3f1a2f6baf3c76a
-  data.tar.gz: 0dd07480d6a1a4db6345e14727f932b1c2ce09ce9683f01f4fe4c3e008c608742ef47a68aca27371f182dc1b8cd995420bef3a4d666abe6ff0ec6ed218513916
+  metadata.gz: 4583efbf9e42ca7154f61330b0c702f0cc7190d726983d4dd6ba2f438abed1864391f45e81523a5abde8733b513de5b0a5359e7e0c97762aee83f782dafefec5
+  data.tar.gz: 6aec93c8c72d70cfbc5f328350f527d7388467d6f9be77e8cb78fce8660bd6b25dc5075943289dfad7e4ba2ebaac86f78f85b395f59ff7c1a09c8af446bf3453

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 
 rvm:
-  - "2.3.6"
-  - "2.4.3"
-  - "2.5.0"
+  - "2.4.7"
+  - "2.5"
+  - "2.6"
 
 gemfile:
   - Gemfile

data/README.md

@@ -42,8 +42,8 @@ See [sample configuration](./misc/fluentd_sample.conf), or try [tutorial](#try-p
 
 ### prometheus input plugin
 
-You have to configure this plugin to expose metrics collected by other promtheus plugins.
-This plugin provides a metrics HTTP endpoint to be scraped by a prometheus server on 24231/tcp(default).
+You have to configure this plugin to expose metrics collected by other Prometheus plugins.
+This plugin provides a metrics HTTP endpoint to be scraped by a Prometheus server on 24231/tcp(default).
 
 With following configuration, you can access http://localhost:24231/metrics on a server where fluentd running.
 
@@ -56,20 +56,27 @@ With following configuration, you can access http://localhost:24231/metrics on a
 More configuration parameters:
 
 - `bind`: binding interface (default: '0.0.0.0')
-- `port`: listen port (defaut: 24231)
+- `port`: listen port (default: 24231)
 - `metrics_path`: metrics HTTP endpoint (default: /metrics)
+- `aggregated_metrics_path`: metrics HTTP endpoint (default: /aggregated_metrics)
 
 When using multiple workers, each worker binds to port + `fluent_worker_id`.
+To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
 
 ### prometheus_monitor input plugin
 
-This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](http://docs.fluentd.org/articles/monitoring#monitoring-agent).
+This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](https://docs.fluentd.org/input/monitor_agent).
 
-Current exposed metrics:
 
-- `buffere_queue_length` of each BufferedOutput plugins
-- `buffer_total_queued_size` of each BufferedOutput plugins
-- `retry_count` of each BufferedOutput plugins
+#### Exposed metrics
+
+- `fluentd_status_buffer_queue_length`
+- `fluentd_status_buffer_total_queued_size`
+- `fluentd_status_retry_count`
+- `fluentd_status_buffer_newest_timekey` from fluentd v1.4.2
+- `fluentd_status_buffer_oldest_timekey` from fluentd v1.4.2
+
+#### Configuration
 
 With following configuration, those metrics are collected.
 
@@ -86,26 +93,35 @@ More configuration parameters:
 
 ### prometheus_output_monitor input plugin
 
-**experimental**
-
 This plugin collects internal metrics for output plugin in Fluentd. This is similar to `prometheus_monitor` plugin, but specialized for output plugin. There are Many metrics `prometheus_monitor` does not include, such as `num_errors`, `retry_wait` and so on.
 
-Current exposed metrics:
+#### Exposed metrics
+
+Metrics for output
 
-- `fluentd_output_status_buffer_queue_length`
-- `fluentd_output_status_buffer_total_bytes`
 - `fluentd_output_status_retry_count`
 - `fluentd_output_status_num_errors`
 - `fluentd_output_status_emit_count`
 - `fluentd_output_status_retry_wait`
     - current retry_wait computed from last retry time and next retry time
 - `fluentd_output_status_emit_records`
-    - only for v0.14
 - `fluentd_output_status_write_count`
-    - only for v0.14
 - `fluentd_output_status_rollback_count`
-    - only for v0.14
+- `fluentd_output_status_flush_time_count` from fluentd v1.6.0
+- `fluentd_output_status_slow_flush_count` from fluentd v1.6.0
 
+Metrics for buffer
+
+- `fluentd_output_status_buffer_total_bytes`
+- `fluentd_output_status_buffer_stage_length` from fluentd v1.6.0
+- `fluentd_output_status_buffer_stage_byte_size` from fluentd v1.6.0
+- `fluentd_output_status_buffer_queue_length`
+- `fluentd_output_status_buffer_queue_byte_size` from fluentd v1.6.0
+- `fluentd_output_status_buffer_newest_timekey` from fluentd v1.6.0
+- `fluentd_output_status_buffer_oldest_timekey` from fluentd v1.6.0
+- `fluentd_output_status_buffer_available_space_ratio` from fluentd v1.6.0
+
+#### Configuration
 
 With following configuration, those metrics are collected.
 
@@ -122,13 +138,11 @@ More configuration parameters:
 
 ### prometheus_tail_monitor input plugin
 
-**experimental**
-
 This plugin collects internal metrics for in_tail plugin in Fluentd. in_tail plugin holds internal state for files that the plugin is watching. The state is sometimes important to monitor plugins work correctly.
 
 This plugin uses internal class of Fluentd, so it's easy to break.
 
-Current exposed metrics:
+#### Exposed metrics
 
 - `fluentd_tail_file_position`
     - Current bytes which plugin reads from the file
@@ -141,6 +155,8 @@ Default labels:
 - `type`: plugin name. `in_tail` only for now.
 - `path`: file path
 
+#### Configuration
+
 With following configuration, those metrics are collected.
 
 ```
@@ -224,6 +240,8 @@ In output plugin style:
 
 With above configuration, the plugin collects a metric named `message_foo_counter` from key `foo` of each records.
 
+You can access nested keys in records via dot or bracket notation (https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`.
+
 See Supported Metric Type and Labels for more configuration parameters.
 
 ## Supported Metric Types
@@ -339,7 +357,7 @@ You can add labels with static value or dynamic value from records. In `promethe
 
 All labels sections has same format. Each lines have key/value for label.
 
-You can access nested fields in records via dot or bracket notation (https://docs.fluentd.org/v1.0/articles/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`. Other values are handled as raw string as is and may be expanded by placeholder described later.
+You can access nested fields in records via dot or bracket notation (https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`. Other values are handled as raw string as is and may be expanded by placeholder described later.
 
 You can use placeholder for label values. The placeholders will be expanded from reserved values and records.
 If you specify `${hostname}`, it will be expanded by value of a hostname where fluentd runs.
@@ -350,12 +368,12 @@ Reserved placeholders are:
 - `${hostname}`: hostname
 - `${worker_id}`: fluent worker id
 - `${tag}`: tag name
-  - only availabe in prometheus output/filter plugin
+  - only available in Prometheus output/filter plugin
 
 
 ### top-level labels and labels inside metric
 
-Prometheus output/filter plugin can have multiple metric section. Top-level labels section spcifies labels for all metrics. Labels section insede metric section specifis labels for the metric. Both are specified, labels are merged.
+Prometheus output/filter plugin can have multiple metric section. Top-level labels section specifies labels for all metrics. Labels section inside metric section specifies labels for the metric. Both are specified, labels are merged.
 
 ```
 <filter message>
@@ -399,7 +417,7 @@ $ cd fluent-plugin-prometheus
 $ bundle install --path vendor/bundle
 ```
 
-Download pre-compiled prometheus binary and start it. It listens on 9090.
+Download pre-compiled Prometheus binary and start it. It listens on 9090.
 
 ```
 $ wget https://github.com/prometheus/prometheus/releases/download/v1.5.2/prometheus-1.5.2.linux-amd64.tar.gz -O - | tar zxf -

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.2.1"
+  spec.version       = "1.6.1"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "fluentd", ">= 0.14.20", "< 2"
-  spec.add_dependency "prometheus-client"
+  spec.add_dependency "prometheus-client", "< 0.10"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/lib/fluent/plugin/filter_prometheus.rb

@@ -4,6 +4,7 @@ require 'fluent/plugin/filter'
 module Fluent::Plugin
   class PrometheusFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
     include Fluent::Plugin::Prometheus
 
     def initialize
@@ -11,9 +12,13 @@ module Fluent::Plugin
       @registry = ::Prometheus::Client.registry
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def configure(conf)
       super
-      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      labels = parse_labels_elements(conf)
       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 

data/lib/fluent/plugin/in_prometheus.rb

@@ -1,5 +1,7 @@
 require 'fluent/plugin/input'
 require 'fluent/plugin/prometheus'
+require 'fluent/plugin/prometheus_metrics'
+require 'net/http'
 require 'webrick'
 
 module Fluent::Plugin
@@ -11,6 +13,7 @@ module Fluent::Plugin
     config_param :bind, :string, default: '0.0.0.0'
     config_param :port, :integer, default: 24231
     config_param :metrics_path, :string, default: '/metrics'
+    config_param :aggregated_metrics_path, :string, default: '/aggregated_metrics'
 
     desc 'Enable ssl configuration for the server'
     config_section :ssl, required: false, multi: false do
@@ -31,6 +34,10 @@ module Fluent::Plugin
 
     attr_reader :registry
 
+    attr_reader :num_workers
+    attr_reader :base_port
+    attr_reader :metrics_path
+
     def initialize
       super
       @registry = ::Prometheus::Client.registry
@@ -38,6 +45,18 @@ module Fluent::Plugin
 
     def configure(conf)
       super
+
+      # Get how many workers we have
+      sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                  owner.system_config
+                elsif self.respond_to?(:system_config)
+                  self.system_config
+                else
+                  nil
+                end
+      @num_workers = sysconf && sysconf.workers ? sysconf.workers : 1
+
+      @base_port = @port
       @port += fluentd_worker_id
     end
 
@@ -82,6 +101,7 @@ module Fluent::Plugin
 
       @server = WEBrick::HTTPServer.new(config)
       @server.mount(@metrics_path, MonitorServlet, self)
+      @server.mount(@aggregated_metrics_path, MonitorServletAll, self)
       thread_create(:in_prometheus) do
         @server.start
       end
@@ -110,5 +130,35 @@ module Fluent::Plugin
         res.body = $!.to_s
       end
     end
+
+    class MonitorServletAll < WEBrick::HTTPServlet::AbstractServlet
+      def initialize(server, prometheus)
+        @prometheus = prometheus
+      end
+
+      def do_GET(req, res)
+        res.status = 200
+        res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
+
+        full_result = PromMetricsAggregator.new
+        fluent_server_ip = @prometheus.bind == '0.0.0.0' ? '127.0.0.1' : @prometheus.bind
+        current_worker = 0
+        while current_worker < @prometheus.num_workers
+          Net::HTTP.start(fluent_server_ip, @prometheus.base_port + current_worker) do |http|
+            req = Net::HTTP::Get.new(@prometheus.metrics_path)
+            result = http.request(req)
+            if result.is_a?(Net::HTTPSuccess)
+              full_result.add_metrics(result.body)
+            end
+          end
+          current_worker += 1
+        end
+        res.body = full_result.get_metrics
+      rescue
+        res.status = 500
+        res['Content-Type'] = 'text/plain'
+        res.body = $!.to_s
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -25,11 +26,12 @@ module Fluent::Plugin
       hostname = Socket.gethostname
       expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
-        if value.is_a?(String)
-          @base_labels[key] = expander.expand(value, placeholders)
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
         end
+        @base_labels[key] = expander.expand(value, placeholders)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
@@ -39,6 +41,17 @@ module Fluent::Plugin
         @monitor_agent = Fluent::MonitorAgentInput.new
       end
 
+    end
+
+    def start
+      super
+
+      @buffer_newest_timekey = @registry.gauge(
+        :fluentd_status_buffer_newest_timekey,
+        'Newest timekey in buffer.')
+      @buffer_oldest_timekey = @registry.gauge(
+        :fluentd_status_buffer_oldest_timekey,
+        'Oldest timekey in buffer.')
       buffer_queue_length = @registry.gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
@@ -54,20 +67,24 @@ module Fluent::Plugin
         'buffer_total_queued_size' => buffer_total_queued_size,
         'retry_count' => retry_counts,
       }
-    end
-
-    def start
-      super
       timer_execute(:in_prometheus_monitor, @interval, &method(:update_monitor_info))
     end
 
     def update_monitor_info
       @monitor_agent.plugins_info_all.each do |info|
+        label = labels(info)
+
         @monitor_info.each do |name, metric|
           if info[name]
-            metric.set(labels(info), info[name])
+            metric.set(label, info[name])
           end
         end
+
+        timekeys = info["buffer_timekeys"]
+        if timekeys && !timekeys.empty?
+          @buffer_newest_timekey.set(label, timekeys.max)
+          @buffer_oldest_timekey.set(label, timekeys.min)
+        end
       end
     end
 

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusOutputMonitorInput < Fluent::Input
     Fluent::Plugin.register_input('prometheus_output_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -24,6 +25,10 @@ module Fluent::Plugin
       :emit_records,
       :write_count,
       :rollback_count,
+
+      # from v1.6.0
+      :flush_time_count,
+      :slow_flush_count,
     ]
 
     def initialize
@@ -40,8 +45,11 @@ module Fluent::Plugin
       hostname = Socket.gethostname
       expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
+        end
         @base_labels[key] = expander.expand(value, placeholders)
       end
 
@@ -51,14 +59,39 @@ module Fluent::Plugin
       else
         @monitor_agent = Fluent::MonitorAgentInput.new
       end
+    end
+
+    def start
+      super
 
       @metrics = {
-        buffer_queue_length: @registry.gauge(
-          :fluentd_output_status_buffer_queue_length,
-          'Current buffer queue length.'),
+        # Buffer metrics
         buffer_total_queued_size: @registry.gauge(
           :fluentd_output_status_buffer_total_bytes,
-          'Current total size of queued buffers.'),
+          'Current total size of stage and queue buffers.'),
+        buffer_stage_length: @registry.gauge(
+          :fluentd_output_status_buffer_stage_length,
+          'Current length of stage buffers.'),
+        buffer_stage_byte_size: @registry.gauge(
+          :fluentd_output_status_buffer_stage_byte_size,
+          'Current total size of stage buffers.'),
+        buffer_queue_length: @registry.gauge(
+          :fluentd_output_status_buffer_queue_length,
+          'Current length of queue buffers.'),
+        buffer_queue_byte_size: @registry.gauge(
+          :fluentd_output_status_queue_byte_size,
+          'Current total size of queue buffers.'),
+        buffer_available_buffer_space_ratios: @registry.gauge(
+          :fluentd_output_status_buffer_available_space_ratio,
+          'Ratio of available space in buffer.'),
+        buffer_newest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_newest_timekey,
+          'Newest timekey in buffer.'),
+        buffer_oldest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_oldest_timekey,
+          'Oldest timekey in buffer.'),
+
+        # Output metrics
         retry_counts: @registry.gauge(
           :fluentd_output_status_retry_count,
           'Current retry counts.'),
@@ -77,14 +110,16 @@ module Fluent::Plugin
         rollback_count: @registry.gauge(
           :fluentd_output_status_rollback_count,
           'Current rollback counts.'),
+        flush_time_count: @registry.gauge(
+          :fluentd_output_status_flush_time_count,
+          'Total flush time.'),
+        slow_flush_count: @registry.gauge(
+          :fluentd_output_status_slow_flush_count,
+          'Current slow flush counts.'),
         retry_wait: @registry.gauge(
           :fluentd_output_status_retry_wait,
           'Current retry wait'),
       }
-    end
-
-    def start
-      super
       timer_execute(:in_prometheus_output_monitor, @interval, &method(:update_monitor_info))
     end
 
@@ -99,8 +134,17 @@ module Fluent::Plugin
       }
 
       monitor_info = {
-        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        # buffer metrics
         'buffer_total_queued_size' => @metrics[:buffer_total_queued_size],
+        'buffer_stage_length' => @metrics[:buffer_stage_length],
+        'buffer_stage_byte_size' => @metrics[:buffer_stage_byte_size],
+        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        'buffer_queue_byte_size' => @metrics[:buffer_queue_byte_size],
+        'buffer_available_buffer_space_ratios' => @metrics[:buffer_available_buffer_space_ratios],
+        'buffer_newest_timekey' => @metrics[:buffer_newest_timekey],
+        'buffer_oldest_timekey' => @metrics[:buffer_oldest_timekey],
+
+        # output metrics
         'retry_count' => @metrics[:retry_counts],
       }
       instance_vars_info = {
@@ -109,6 +153,8 @@ module Fluent::Plugin
         emit_count: @metrics[:emit_count],
         emit_records: @metrics[:emit_records],
         rollback_count: @metrics[:rollback_count],
+        flush_time_count: @metrics[:flush_time_count],
+        slow_flush_count: @metrics[:slow_flush_count],
       }
 
       agent_info.each do |info|

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusTailMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_tail_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -29,8 +30,11 @@ module Fluent::Plugin
       hostname = Socket.gethostname
       expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"
+        end
         @base_labels[key] = expander.expand(value, placeholders)
       end
 
@@ -40,6 +44,10 @@ module Fluent::Plugin
       else
         @monitor_agent = Fluent::MonitorAgentInput.new
       end
+    end
+
+    def start
+      super
 
       @metrics = {
         position: @registry.gauge(
@@ -49,10 +57,6 @@ module Fluent::Plugin
           :fluentd_tail_file_inode,
           'Current inode of file.'),
       }
-    end
-
-    def start
-      super
       timer_execute(:in_prometheus_tail_monitor, @interval, &method(:update_monitor_info))
     end
 

data/lib/fluent/plugin/out_prometheus.rb

@@ -4,6 +4,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusOutput < Fluent::Plugin::Output
     Fluent::Plugin.register_output('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
     include Fluent::Plugin::Prometheus
 
     def initialize
@@ -11,9 +12,13 @@ module Fluent::Plugin
       @registry = ::Prometheus::Client.registry
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def configure(conf)
       super
-      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      labels = parse_labels_elements(conf)
       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 

data/lib/fluent/plugin/prometheus.rb

@@ -4,6 +4,31 @@ require 'fluent/plugin/filter_record_transformer'
 
 module Fluent
   module Plugin
+    module PrometheusLabelParser
+      def configure(conf)
+        super
+        # Check if running with multiple workers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+          owner.system_config
+        elsif self.respond_to?(:system_config)
+          self.system_config
+        else
+          nil
+        end
+        @multi_worker = sysconf && sysconf.workers ? (sysconf.workers > 1) : false
+      end
+
+      def parse_labels_elements(conf)
+        base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+
+        if @multi_worker
+          base_labels[:worker_id] = fluentd_worker_id.to_s
+        end
+
+        base_labels
+      end
+    end
+
     module Prometheus
       class AlreadyRegisteredError < StandardError; end
 
@@ -36,6 +61,10 @@ module Fluent
         conf.elements.select { |element|
           element.name == 'metric'
         }.each { |element|
+          if element.has_key?('key') && (element['key'].start_with?('$.') || element['key'].start_with?('$['))
+            value = element['key']
+            element['key'] = PluginHelper::RecordAccessor::Accessor.new(value)
+          end
           case element['type']
           when 'summary'
             metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels)
@@ -147,8 +176,13 @@ module Fluent
         end
 
         def instrument(record, expander, placeholders)
-          if record[@key]
-            @gauge.set(labels(record, expander, placeholders), record[@key])
+          if @key.is_a?(String)
+            value = record[@key]
+          else
+            value = @key.call(record)
+          end
+          if value
+            @gauge.set(labels(record, expander, placeholders), value)
           end
         end
       end
@@ -165,7 +199,13 @@ module Fluent
 
         def instrument(record, expander, placeholders)
           # use record value of the key if key is specified, otherwise just increment
-          value = @key ? record[@key] : 1
+          if @key.nil?
+            value = 1
+          elsif @key.is_a?(String)
+            value = record[@key]
+          else
+            value = @key.call(record)
+          end
 
           # ignore if record value is nil
           return if value.nil?
@@ -189,8 +229,13 @@ module Fluent
         end
 
         def instrument(record, expander, placeholders)
-          if record[@key]
-            @summary.observe(labels(record, expander, placeholders), record[@key])
+          if @key.is_a?(String)
+            value = record[@key]
+          else
+            value = @key.call(record)
+          end
+          if value
+            @summary.observe(labels(record, expander, placeholders), value)
           end
         end
       end
@@ -217,8 +262,13 @@ module Fluent
         end
 
         def instrument(record, expander, placeholders)
-          if record[@key]
-            @histogram.observe(labels(record, expander, placeholders), record[@key])
+          if @key.is_a?(String)
+            value = record[@key]
+          else
+            value = @key.call(record)
+          end
+          if value
+            @histogram.observe(labels(record, expander, placeholders), value)
           end
         end
       end

data/lib/fluent/plugin/prometheus_metrics.rb

@@ -0,0 +1,77 @@
+module Fluent::Plugin
+
+  ##
+  # PromMetricsAggregator aggregates multiples metrics exposed using Prometheus text-based format
+  # see https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
+
+
+  class PrometheusMetrics
+    def initialize
+      @comments = []
+      @metrics = []
+    end
+
+    def to_string
+      (@comments + @metrics).join("\n")
+    end
+
+    def add_comment(comment)
+      @comments << comment
+    end
+
+    def add_metric_value(value)
+      @metrics << value
+    end
+
+    attr_writer :comments, :metrics
+  end
+
+  class PromMetricsAggregator
+    def initialize
+      @metrics = {}
+    end
+
+    def get_metric_name_from_comment(line)
+      tokens = line.split(' ')
+      if ['HELP', 'TYPE'].include?(tokens[1])
+        tokens[2]
+      else
+        ''
+      end
+    end
+
+    def add_metrics(metrics)
+      current_metric = ''
+      new_metric = false
+      lines = metrics.split("\n")
+      for line in lines
+        if line[0] == '#'
+          # Metric comment (# TYPE, # HELP)
+          parsed_metric = get_metric_name_from_comment(line)
+          if parsed_metric != ''
+            if parsed_metric != current_metric
+              # Starting a new metric comment block
+              new_metric = !@metrics.key?(parsed_metric)
+              if new_metric
+                @metrics[parsed_metric] = PrometheusMetrics.new()
+              end
+              current_metric = parsed_metric
+            end
+
+            if new_metric && parsed_metric == current_metric
+              # New metric, inject comments (# TYPE, # HELP)
+              @metrics[parsed_metric].add_comment(line)
+            end
+          end
+        else
+          # Metric value, simply append line
+          @metrics[current_metric].add_metric_value(line)
+        end
+      end
+    end
+
+    def get_metrics
+      @metrics.map{|k,v| v.to_string()}.join("\n") + (@metrics.length ? "\n" : "")
+    end
+  end
+end

data/misc/prometheus_alerts.yaml

@@ -47,7 +47,7 @@ ALERT FluentdQueueLength
   }
 
 ALERT FluentdRecordsCountsHigh
-  IF sum(rate(fluentd_record_counts{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_record_counts{job="fluentd"}[15m])) BY (instance))
+  IF sum(rate(fluentd_output_status_emit_records{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_output_status_emit_records{job="fluentd"}[15m])) BY (instance))
   FOR 1m
   LABELS {
     service = "fluentd",

data/spec/fluent/plugin/in_prometheus_monitor_spec.rb

@@ -8,8 +8,17 @@ describe Fluent::Plugin::PrometheusMonitorInput do
   <labels>
     host ${hostname}
     foo bar
-    no_effect1 $.foo.bar
-    no_effect2 $[0][1]
+  </labels>
+]
+
+  INVALID_MONITOR_CONFIG = %[
+  @type prometheus_monitor
+
+  <labels>
+    host ${hostname}
+    foo bar
+    invalid_use1 $.foo.bar
+    invalid_use2 $[0][1]
   </labels>
 ]
 
@@ -18,8 +27,17 @@ describe Fluent::Plugin::PrometheusMonitorInput do
   let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusMonitorInput).configure(config) }
 
   describe '#configure' do
-    it 'does not raise error' do
-      expect{driver}.not_to raise_error
+    describe 'valid' do
+      it 'does not raise error' do
+        expect{driver}.not_to raise_error
+      end
+    end
+
+    describe 'invalid' do
+      let(:config) { INVALID_MONITOR_CONFIG }
+      it 'expect raise error' do
+        expect{driver}.to raise_error
+      end
     end
   end
 end

data/spec/fluent/plugin/prometheus_metrics_spec.rb

@@ -0,0 +1,138 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus'
+require 'fluent/test/driver/input'
+
+require 'net/http'
+
+describe Fluent::Plugin::PromMetricsAggregator do
+
+  metrics_worker_1 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+]
+
+  metrics_worker_2 = %[# TYPE fluentd_output_status_buffer_queue_length gauge
+# HELP fluentd_output_status_buffer_queue_length Current buffer queue length.
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+# TYPE fluentd_output_status_buffer_total_bytes gauge
+# HELP fluentd_output_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+]
+
+  metrics_worker_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  metrics_merged_1_and_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  describe 'add_metrics' do
+    context '1st_metrics' do
+      it 'adds all fields' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1)
+      end
+    end
+    context '2nd_metrics' do
+      it 'append new metrics' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1 + metrics_worker_2)
+      end
+    end
+
+    context '3rd_metrics' do
+      it 'append existing metrics in the right place' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        all_metrics.add_metrics(metrics_worker_3)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_merged_1_and_3 + metrics_worker_2)
+      end
+    end
+  end
+end

data/spec/fluent/plugin/shared.rb

@@ -52,6 +52,24 @@ FULL_CONFIG = BASE_CONFIG + %[
       key foo4
     </labels>
   </metric>
+  <metric>
+    name full_accessor1
+    type summary
+    desc Something with accessor.
+    key $.foo
+    <labels>
+      key foo5
+    </labels>
+  </metric>
+  <metric>
+    name full_accessor2
+    type counter
+    desc Something with accessor.
+    key $.foo
+    <labels>
+      key foo6
+    </labels>
+  </metric>
   <labels>
     test_key test_value
   </labels>
@@ -110,6 +128,8 @@ shared_context 'full_config' do
   let(:gauge) { registry.get(:full_bar) }
   let(:summary) { registry.get(:full_baz) }
   let(:histogram) { registry.get(:full_qux) }
+  let(:summary_with_accessor) { registry.get(:full_accessor1) }
+  let(:counter_with_accessor) { registry.get(:full_accessor2) }
 end
 
 shared_context 'placeholder_config' do
@@ -192,15 +212,20 @@ shared_examples_for 'instruments record' do
       expect(registry.metrics.map(&:name)).to include(:full_foo)
       expect(registry.metrics.map(&:name)).to include(:full_bar)
       expect(registry.metrics.map(&:name)).to include(:full_baz)
+      expect(registry.metrics.map(&:name)).to include(:full_accessor1)
+      expect(registry.metrics.map(&:name)).to include(:full_accessor2)
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
       expect(summary).to be_kind_of(::Prometheus::Client::Metric)
+      expect(summary_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
+      expect(counter_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
       expect(histogram).to be_kind_of(::Prometheus::Client::Metric)
     end
 
     it 'instruments counter metric' do
       expect(counter.type).to eq(:counter)
       expect(counter.get({test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
+      expect(counter_with_accessor.get({test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
     end
 
     it 'instruments gauge metric' do
@@ -212,6 +237,7 @@ shared_examples_for 'instruments record' do
       expect(summary.type).to eq(:summary)
       expect(summary.get({test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
       expect(summary.get({test_key: 'test_value', key: 'foo3'})[0.99]).to eq(100)
+      expect(summary_with_accessor.get({test_key: 'test_value', key: 'foo5'})[0.99]).to eq(100)
     end
 
     it 'instruments histogram metric' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.6.1
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-10 00:00:00.000000000 Z
+date: 2019-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,7 @@ files:
 - lib/fluent/plugin/in_prometheus_tail_monitor.rb
 - lib/fluent/plugin/out_prometheus.rb
 - lib/fluent/plugin/prometheus.rb
+- lib/fluent/plugin/prometheus_metrics.rb
 - misc/fluentd_sample.conf
 - misc/nginx_proxy.conf
 - misc/prometheus.yaml
@@ -129,6 +130,7 @@ files:
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus_metrics_spec.rb
 - spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb
@@ -151,8 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.
@@ -160,6 +161,7 @@ test_files:
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus_metrics_spec.rb
 - spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb