fluent-plugin-prometheus 1.3.0 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 274c9ef3f67ca084c8b46d6a24ca97ce8313d2ade4409426dc2bb664ec227f31
-  data.tar.gz: 1c3cc72e17e834cf5144315086856bc8a8394a1a893771f7521e0a0fd4adf73f
+  metadata.gz: 95008f0031db355e5a5bbf4ed21f8f9f0a07839480e40bf44e974046b04f562f
+  data.tar.gz: 994a30da7b1ff0ad77f12e5b47d9218a2deb97effc31a36fa83175077cc6ed69
 SHA512:
-  metadata.gz: 791e3bf0e513d98d32f0bb8a0f936e305f373ae2706fd9ad6bd3cdd12eb9adcd288d811ea3bcc9eb06c50b7eb583ec5edda3fb52751f7def52252c1174780407
-  data.tar.gz: 2aa16a95bedb91f4f04bac51a8fe96ee540aefe13532429d12803f97f0537e9fd3c10bb54a37c8145f8a58fe0e06e7975d7e99e1a8addb686eb12311ab9951c5
+  metadata.gz: e0dc26e78e242e51e0fa87257b85c68af2ae32c11f7dabd1b4245e3ad7e1b1a7182b1b8432578f98af8a5a22d14c14e664285a238f74ebc40df0c0461026f323
+  data.tar.gz: fb31a79f327ce31e8203fa1fa9a5b9d9166410d71f9b62d4fd5411d246c768dfde5c9c076ff6e72f0d9f5bd4c7657c9180eb91ba4ee383729856da0704930404

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 
 rvm:
-  - "2.3.6"
-  - "2.4.3"
-  - "2.5.0"
+  - "2.4.7"
+  - "2.5"
+  - "2.6"
 
 gemfile:
   - Gemfile

data/README.md

@@ -58,18 +58,25 @@ More configuration parameters:
 - `bind`: binding interface (default: '0.0.0.0')
 - `port`: listen port (default: 24231)
 - `metrics_path`: metrics HTTP endpoint (default: /metrics)
+- `aggregated_metrics_path`: metrics HTTP endpoint (default: /aggregated_metrics)
 
 When using multiple workers, each worker binds to port + `fluent_worker_id`.
+To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
 
 ### prometheus_monitor input plugin
 
-This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](http://docs.fluentd.org/articles/monitoring#monitoring-agent).
+This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](https://docs.fluentd.org/input/monitor_agent).
 
-Current exposed metrics:
 
-- `buffere_queue_length` of each BufferedOutput plugins
-- `buffer_total_queued_size` of each BufferedOutput plugins
-- `retry_count` of each BufferedOutput plugins
+#### Exposed metrics
+
+- `fluentd_status_buffer_queue_length`
+- `fluentd_status_buffer_total_queued_size`
+- `fluentd_status_retry_count`
+- `fluentd_status_buffer_newest_timekey` from fluentd v1.4.2
+- `fluentd_status_buffer_oldest_timekey` from fluentd v1.4.2
+
+#### Configuration
 
 With following configuration, those metrics are collected.
 
@@ -86,26 +93,35 @@ More configuration parameters:
 
 ### prometheus_output_monitor input plugin
 
-**experimental**
-
 This plugin collects internal metrics for output plugin in Fluentd. This is similar to `prometheus_monitor` plugin, but specialized for output plugin. There are Many metrics `prometheus_monitor` does not include, such as `num_errors`, `retry_wait` and so on.
 
-Current exposed metrics:
+#### Exposed metrics
+
+Metrics for output
 
-- `fluentd_output_status_buffer_queue_length`
-- `fluentd_output_status_buffer_total_bytes`
 - `fluentd_output_status_retry_count`
 - `fluentd_output_status_num_errors`
 - `fluentd_output_status_emit_count`
 - `fluentd_output_status_retry_wait`
     - current retry_wait computed from last retry time and next retry time
 - `fluentd_output_status_emit_records`
-    - only for v0.14
 - `fluentd_output_status_write_count`
-    - only for v0.14
 - `fluentd_output_status_rollback_count`
-    - only for v0.14
+- `fluentd_output_status_flush_time_count` from fluentd v1.6.0
+- `fluentd_output_status_slow_flush_count` from fluentd v1.6.0
 
+Metrics for buffer
+
+- `fluentd_output_status_buffer_total_bytes`
+- `fluentd_output_status_buffer_stage_length` from fluentd v1.6.0
+- `fluentd_output_status_buffer_stage_byte_size` from fluentd v1.6.0
+- `fluentd_output_status_buffer_queue_length`
+- `fluentd_output_status_buffer_queue_byte_size` from fluentd v1.6.0
+- `fluentd_output_status_buffer_newest_timekey` from fluentd v1.6.0
+- `fluentd_output_status_buffer_oldest_timekey` from fluentd v1.6.0
+- `fluentd_output_status_buffer_available_space_ratio` from fluentd v1.6.0
+
+#### Configuration
 
 With following configuration, those metrics are collected.
 
@@ -122,13 +138,11 @@ More configuration parameters:
 
 ### prometheus_tail_monitor input plugin
 
-**experimental**
-
 This plugin collects internal metrics for in_tail plugin in Fluentd. in_tail plugin holds internal state for files that the plugin is watching. The state is sometimes important to monitor plugins work correctly.
 
 This plugin uses internal class of Fluentd, so it's easy to break.
 
-Current exposed metrics:
+#### Exposed metrics
 
 - `fluentd_tail_file_position`
     - Current bytes which plugin reads from the file
@@ -141,6 +155,8 @@ Default labels:
 - `type`: plugin name. `in_tail` only for now.
 - `path`: file path
 
+#### Configuration
+
 With following configuration, those metrics are collected.
 
 ```
@@ -224,7 +240,7 @@ In output plugin style:
 
 With above configuration, the plugin collects a metric named `message_foo_counter` from key `foo` of each records.
 
-You can access nested keys in records via dot or bracket notation (https://docs.fluentd.org/v1.0/articles/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`.
+You can access nested keys in records via dot or bracket notation (https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`.
 
 See Supported Metric Type and Labels for more configuration parameters.
 
@@ -341,7 +357,7 @@ You can add labels with static value or dynamic value from records. In `promethe
 
 All labels sections has same format. Each lines have key/value for label.
 
-You can access nested fields in records via dot or bracket notation (https://docs.fluentd.org/v1.0/articles/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`. Other values are handled as raw string as is and may be expanded by placeholder described later.
+You can access nested fields in records via dot or bracket notation (https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`. Other values are handled as raw string as is and may be expanded by placeholder described later.
 
 You can use placeholder for label values. The placeholders will be expanded from reserved values and records.
 If you specify `${hostname}`, it will be expanded by value of a hostname where fluentd runs.

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.3.0"
+  spec.version       = "1.7.0"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "fluentd", ">= 0.14.20", "< 2"
-  spec.add_dependency "prometheus-client"
+  spec.add_dependency "prometheus-client", "< 0.10"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/lib/fluent/plugin/filter_prometheus.rb

@@ -4,6 +4,7 @@ require 'fluent/plugin/filter'
 module Fluent::Plugin
   class PrometheusFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
     include Fluent::Plugin::Prometheus
 
     def initialize
@@ -11,15 +12,19 @@ module Fluent::Plugin
       @registry = ::Prometheus::Client.registry
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def configure(conf)
       super
-      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      labels = parse_labels_elements(conf)
       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 
-    def filter_stream(tag, es)
-      instrument(tag, es, @metrics)
-      es
+    def filter(tag, time, record)
+      instrument_single(tag, time, record, @metrics)
+      record
     end
   end
 end

data/lib/fluent/plugin/in_prometheus.rb

@@ -1,5 +1,7 @@
 require 'fluent/plugin/input'
 require 'fluent/plugin/prometheus'
+require 'fluent/plugin/prometheus_metrics'
+require 'net/http'
 require 'webrick'
 
 module Fluent::Plugin
@@ -11,6 +13,7 @@ module Fluent::Plugin
     config_param :bind, :string, default: '0.0.0.0'
     config_param :port, :integer, default: 24231
     config_param :metrics_path, :string, default: '/metrics'
+    config_param :aggregated_metrics_path, :string, default: '/aggregated_metrics'
 
     desc 'Enable ssl configuration for the server'
     config_section :ssl, required: false, multi: false do
@@ -31,6 +34,10 @@ module Fluent::Plugin
 
     attr_reader :registry
 
+    attr_reader :num_workers
+    attr_reader :base_port
+    attr_reader :metrics_path
+
     def initialize
       super
       @registry = ::Prometheus::Client.registry
@@ -38,6 +45,18 @@ module Fluent::Plugin
 
     def configure(conf)
       super
+
+      # Get how many workers we have
+      sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                  owner.system_config
+                elsif self.respond_to?(:system_config)
+                  self.system_config
+                else
+                  nil
+                end
+      @num_workers = sysconf && sysconf.workers ? sysconf.workers : 1
+
+      @base_port = @port
       @port += fluentd_worker_id
     end
 
@@ -82,6 +101,7 @@ module Fluent::Plugin
 
       @server = WEBrick::HTTPServer.new(config)
       @server.mount(@metrics_path, MonitorServlet, self)
+      @server.mount(@aggregated_metrics_path, MonitorServletAll, self)
       thread_create(:in_prometheus) do
         @server.start
       end
@@ -110,5 +130,35 @@ module Fluent::Plugin
         res.body = $!.to_s
       end
     end
+
+    class MonitorServletAll < WEBrick::HTTPServlet::AbstractServlet
+      def initialize(server, prometheus)
+        @prometheus = prometheus
+      end
+
+      def do_GET(req, res)
+        res.status = 200
+        res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
+
+        full_result = PromMetricsAggregator.new
+        fluent_server_ip = @prometheus.bind == '0.0.0.0' ? '127.0.0.1' : @prometheus.bind
+        current_worker = 0
+        while current_worker < @prometheus.num_workers
+          Net::HTTP.start(fluent_server_ip, @prometheus.base_port + current_worker) do |http|
+            req = Net::HTTP::Get.new(@prometheus.metrics_path)
+            result = http.request(req)
+            if result.is_a?(Net::HTTPSuccess)
+              full_result.add_metrics(result.body)
+            end
+          end
+          current_worker += 1
+        end
+        res.body = full_result.get_metrics
+      rescue
+        res.status = 500
+        res['Content-Type'] = 'text/plain'
+        res.body = $!.to_s
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -25,11 +26,12 @@ module Fluent::Plugin
       hostname = Socket.gethostname
       expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
-        if value.is_a?(String)
-          @base_labels[key] = expander.expand(value, placeholders)
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
         end
+        @base_labels[key] = expander.expand(value, placeholders)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
@@ -39,6 +41,17 @@ module Fluent::Plugin
         @monitor_agent = Fluent::MonitorAgentInput.new
       end
 
+    end
+
+    def start
+      super
+
+      @buffer_newest_timekey = @registry.gauge(
+        :fluentd_status_buffer_newest_timekey,
+        'Newest timekey in buffer.')
+      @buffer_oldest_timekey = @registry.gauge(
+        :fluentd_status_buffer_oldest_timekey,
+        'Oldest timekey in buffer.')
       buffer_queue_length = @registry.gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
@@ -54,20 +67,24 @@ module Fluent::Plugin
         'buffer_total_queued_size' => buffer_total_queued_size,
         'retry_count' => retry_counts,
       }
-    end
-
-    def start
-      super
       timer_execute(:in_prometheus_monitor, @interval, &method(:update_monitor_info))
     end
 
     def update_monitor_info
       @monitor_agent.plugins_info_all.each do |info|
+        label = labels(info)
+
         @monitor_info.each do |name, metric|
           if info[name]
-            metric.set(labels(info), info[name])
+            metric.set(label, info[name])
           end
         end
+
+        timekeys = info["buffer_timekeys"]
+        if timekeys && !timekeys.empty?
+          @buffer_newest_timekey.set(label, timekeys.max)
+          @buffer_oldest_timekey.set(label, timekeys.min)
+        end
       end
     end
 

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusOutputMonitorInput < Fluent::Input
     Fluent::Plugin.register_input('prometheus_output_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -24,6 +25,10 @@ module Fluent::Plugin
       :emit_records,
       :write_count,
       :rollback_count,
+
+      # from v1.6.0
+      :flush_time_count,
+      :slow_flush_count,
     ]
 
     def initialize
@@ -40,8 +45,11 @@ module Fluent::Plugin
       hostname = Socket.gethostname
       expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
+        end
         @base_labels[key] = expander.expand(value, placeholders)
       end
 
@@ -51,14 +59,39 @@ module Fluent::Plugin
       else
         @monitor_agent = Fluent::MonitorAgentInput.new
       end
+    end
+
+    def start
+      super
 
       @metrics = {
-        buffer_queue_length: @registry.gauge(
-          :fluentd_output_status_buffer_queue_length,
-          'Current buffer queue length.'),
+        # Buffer metrics
         buffer_total_queued_size: @registry.gauge(
           :fluentd_output_status_buffer_total_bytes,
-          'Current total size of queued buffers.'),
+          'Current total size of stage and queue buffers.'),
+        buffer_stage_length: @registry.gauge(
+          :fluentd_output_status_buffer_stage_length,
+          'Current length of stage buffers.'),
+        buffer_stage_byte_size: @registry.gauge(
+          :fluentd_output_status_buffer_stage_byte_size,
+          'Current total size of stage buffers.'),
+        buffer_queue_length: @registry.gauge(
+          :fluentd_output_status_buffer_queue_length,
+          'Current length of queue buffers.'),
+        buffer_queue_byte_size: @registry.gauge(
+          :fluentd_output_status_queue_byte_size,
+          'Current total size of queue buffers.'),
+        buffer_available_buffer_space_ratios: @registry.gauge(
+          :fluentd_output_status_buffer_available_space_ratio,
+          'Ratio of available space in buffer.'),
+        buffer_newest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_newest_timekey,
+          'Newest timekey in buffer.'),
+        buffer_oldest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_oldest_timekey,
+          'Oldest timekey in buffer.'),
+
+        # Output metrics
         retry_counts: @registry.gauge(
           :fluentd_output_status_retry_count,
           'Current retry counts.'),
@@ -77,14 +110,16 @@ module Fluent::Plugin
         rollback_count: @registry.gauge(
           :fluentd_output_status_rollback_count,
           'Current rollback counts.'),
+        flush_time_count: @registry.gauge(
+          :fluentd_output_status_flush_time_count,
+          'Total flush time.'),
+        slow_flush_count: @registry.gauge(
+          :fluentd_output_status_slow_flush_count,
+          'Current slow flush counts.'),
         retry_wait: @registry.gauge(
           :fluentd_output_status_retry_wait,
           'Current retry wait'),
       }
-    end
-
-    def start
-      super
       timer_execute(:in_prometheus_output_monitor, @interval, &method(:update_monitor_info))
     end
 
@@ -99,8 +134,17 @@ module Fluent::Plugin
       }
 
       monitor_info = {
-        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        # buffer metrics
         'buffer_total_queued_size' => @metrics[:buffer_total_queued_size],
+        'buffer_stage_length' => @metrics[:buffer_stage_length],
+        'buffer_stage_byte_size' => @metrics[:buffer_stage_byte_size],
+        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        'buffer_queue_byte_size' => @metrics[:buffer_queue_byte_size],
+        'buffer_available_buffer_space_ratios' => @metrics[:buffer_available_buffer_space_ratios],
+        'buffer_newest_timekey' => @metrics[:buffer_newest_timekey],
+        'buffer_oldest_timekey' => @metrics[:buffer_oldest_timekey],
+
+        # output metrics
         'retry_count' => @metrics[:retry_counts],
       }
       instance_vars_info = {
@@ -109,6 +153,8 @@ module Fluent::Plugin
         emit_count: @metrics[:emit_count],
         emit_records: @metrics[:emit_records],
         rollback_count: @metrics[:rollback_count],
+        flush_time_count: @metrics[:flush_time_count],
+        slow_flush_count: @metrics[:slow_flush_count],
       }
 
       agent_info.each do |info|

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusTailMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_tail_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -29,8 +30,11 @@ module Fluent::Plugin
       hostname = Socket.gethostname
       expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
+        unless value.is_a?(String)
+          raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"
+        end
         @base_labels[key] = expander.expand(value, placeholders)
       end
 
@@ -40,6 +44,10 @@ module Fluent::Plugin
       else
         @monitor_agent = Fluent::MonitorAgentInput.new
       end
+    end
+
+    def start
+      super
 
       @metrics = {
         position: @registry.gauge(
@@ -49,10 +57,6 @@ module Fluent::Plugin
           :fluentd_tail_file_inode,
           'Current inode of file.'),
       }
-    end
-
-    def start
-      super
       timer_execute(:in_prometheus_tail_monitor, @interval, &method(:update_monitor_info))
     end
 

data/lib/fluent/plugin/out_prometheus.rb

@@ -4,6 +4,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusOutput < Fluent::Plugin::Output
     Fluent::Plugin.register_output('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
     include Fluent::Plugin::Prometheus
 
     def initialize
@@ -11,9 +12,13 @@ module Fluent::Plugin
       @registry = ::Prometheus::Client.registry
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def configure(conf)
       super
-      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      labels = parse_labels_elements(conf)
       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 

data/lib/fluent/plugin/prometheus.rb

@@ -4,6 +4,31 @@ require 'fluent/plugin/filter_record_transformer'
 
 module Fluent
   module Plugin
+    module PrometheusLabelParser
+      def configure(conf)
+        super
+        # Check if running with multiple workers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+          owner.system_config
+        elsif self.respond_to?(:system_config)
+          self.system_config
+        else
+          nil
+        end
+        @multi_worker = sysconf && sysconf.workers ? (sysconf.workers > 1) : false
+      end
+
+      def parse_labels_elements(conf)
+        base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+
+        if @multi_worker
+          base_labels[:worker_id] = fluentd_worker_id.to_s
+        end
+
+        base_labels
+      end
+    end
+
     module Prometheus
       class AlreadyRegisteredError < StandardError; end
 
@@ -63,10 +88,30 @@ module Fluent
 
       def configure(conf)
         super
+        @placeholder_values = {}
         @placeholder_expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
         @hostname = Socket.gethostname
       end
 
+      def instrument_single(tag, time, record, metrics)
+        @placeholder_values[tag] ||= {
+          'tag' => tag,
+          'hostname' => @hostname,
+          'worker_id' => fluentd_worker_id,
+        }
+
+        placeholders = record.merge(@placeholder_values[tag])
+        placeholders = @placeholder_expander.prepare_placeholders(placeholders)
+        metrics.each do |metric|
+          begin
+            metric.instrument(record, @placeholder_expander, placeholders)
+          rescue => e
+            log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
+            router.emit_error_event(tag, time, record, e)
+          end
+        end
+      end
+
       def instrument(tag, es, metrics)
         placeholder_values = {
           'tag' => tag,

data/lib/fluent/plugin/prometheus_metrics.rb

@@ -0,0 +1,77 @@
+module Fluent::Plugin
+
+  ##
+  # PromMetricsAggregator aggregates multiples metrics exposed using Prometheus text-based format
+  # see https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
+
+
+  class PrometheusMetrics
+    def initialize
+      @comments = []
+      @metrics = []
+    end
+
+    def to_string
+      (@comments + @metrics).join("\n")
+    end
+
+    def add_comment(comment)
+      @comments << comment
+    end
+
+    def add_metric_value(value)
+      @metrics << value
+    end
+
+    attr_writer :comments, :metrics
+  end
+
+  class PromMetricsAggregator
+    def initialize
+      @metrics = {}
+    end
+
+    def get_metric_name_from_comment(line)
+      tokens = line.split(' ')
+      if ['HELP', 'TYPE'].include?(tokens[1])
+        tokens[2]
+      else
+        ''
+      end
+    end
+
+    def add_metrics(metrics)
+      current_metric = ''
+      new_metric = false
+      lines = metrics.split("\n")
+      for line in lines
+        if line[0] == '#'
+          # Metric comment (# TYPE, # HELP)
+          parsed_metric = get_metric_name_from_comment(line)
+          if parsed_metric != ''
+            if parsed_metric != current_metric
+              # Starting a new metric comment block
+              new_metric = !@metrics.key?(parsed_metric)
+              if new_metric
+                @metrics[parsed_metric] = PrometheusMetrics.new()
+              end
+              current_metric = parsed_metric
+            end
+
+            if new_metric && parsed_metric == current_metric
+              # New metric, inject comments (# TYPE, # HELP)
+              @metrics[parsed_metric].add_comment(line)
+            end
+          end
+        else
+          # Metric value, simply append line
+          @metrics[current_metric].add_metric_value(line)
+        end
+      end
+    end
+
+    def get_metrics
+      @metrics.map{|k,v| v.to_string()}.join("\n") + (@metrics.length ? "\n" : "")
+    end
+  end
+end

data/misc/prometheus_alerts.yaml

@@ -47,7 +47,7 @@ ALERT FluentdQueueLength
   }
 
 ALERT FluentdRecordsCountsHigh
-  IF sum(rate(fluentd_record_counts{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_record_counts{job="fluentd"}[15m])) BY (instance))
+  IF sum(rate(fluentd_output_status_emit_records{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_output_status_emit_records{job="fluentd"}[15m])) BY (instance))
   FOR 1m
   LABELS {
     service = "fluentd",

data/spec/fluent/plugin/in_prometheus_monitor_spec.rb

@@ -8,8 +8,17 @@ describe Fluent::Plugin::PrometheusMonitorInput do
   <labels>
     host ${hostname}
     foo bar
-    no_effect1 $.foo.bar
-    no_effect2 $[0][1]
+  </labels>
+]
+
+  INVALID_MONITOR_CONFIG = %[
+  @type prometheus_monitor
+
+  <labels>
+    host ${hostname}
+    foo bar
+    invalid_use1 $.foo.bar
+    invalid_use2 $[0][1]
   </labels>
 ]
 
@@ -18,8 +27,17 @@ describe Fluent::Plugin::PrometheusMonitorInput do
   let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusMonitorInput).configure(config) }
 
   describe '#configure' do
-    it 'does not raise error' do
-      expect{driver}.not_to raise_error
+    describe 'valid' do
+      it 'does not raise error' do
+        expect{driver}.not_to raise_error
+      end
+    end
+
+    describe 'invalid' do
+      let(:config) { INVALID_MONITOR_CONFIG }
+      it 'expect raise error' do
+        expect{driver}.to raise_error
+      end
     end
   end
 end

data/spec/fluent/plugin/prometheus_metrics_spec.rb

@@ -0,0 +1,138 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus'
+require 'fluent/test/driver/input'
+
+require 'net/http'
+
+describe Fluent::Plugin::PromMetricsAggregator do
+
+  metrics_worker_1 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+]
+
+  metrics_worker_2 = %[# TYPE fluentd_output_status_buffer_queue_length gauge
+# HELP fluentd_output_status_buffer_queue_length Current buffer queue length.
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+# TYPE fluentd_output_status_buffer_total_bytes gauge
+# HELP fluentd_output_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+]
+
+  metrics_worker_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  metrics_merged_1_and_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  describe 'add_metrics' do
+    context '1st_metrics' do
+      it 'adds all fields' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1)
+      end
+    end
+    context '2nd_metrics' do
+      it 'append new metrics' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1 + metrics_worker_2)
+      end
+    end
+
+    context '3rd_metrics' do
+      it 'append existing metrics in the right place' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        all_metrics.add_metrics(metrics_worker_3)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_merged_1_and_3 + metrics_worker_2)
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.7.0
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-05 00:00:00.000000000 Z
+date: 2019-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,7 @@ files:
 - lib/fluent/plugin/in_prometheus_tail_monitor.rb
 - lib/fluent/plugin/out_prometheus.rb
 - lib/fluent/plugin/prometheus.rb
+- lib/fluent/plugin/prometheus_metrics.rb
 - misc/fluentd_sample.conf
 - misc/nginx_proxy.conf
 - misc/prometheus.yaml
@@ -129,6 +130,7 @@ files:
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus_metrics_spec.rb
 - spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb
@@ -151,8 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.
@@ -160,6 +161,7 @@ test_files:
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus_metrics_spec.rb
 - spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb