fluent-plugin-prometheus 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de38cb79041db3b0a78ead61eee23d58b568b71292886b19db9ca30ad38ad00b
-  data.tar.gz: a1867c481cd9527cc4ada9125a266a2455aacca7868c4f00d8489926c845a665
+  metadata.gz: 3f079822fa7231b77912e5ac1324b691f38dbf1f1f32fd8700dee19bc636fd8c
+  data.tar.gz: 58ad623a333cc1600e971fdd35b9b2c7d4a381c03a954d99206233758e4562f6
 SHA512:
-  metadata.gz: eb397416c207fcc7d6111aaa9346bca990e5c44764b25072a7a0c68ef32748e767ba04637ec9576be25273caf2345303e6ab7bacbbb261e572cd3288b01475ff
-  data.tar.gz: 3e4a0484386035bf791b49f39d567b52675b97c5ccf7738edaeeb90e23c4adfce852ee1f5a5c69ee0905bbc33f57358e0322761dbbdf86b559d10524f7c21abf
+  metadata.gz: 4394983e398e9e4b116f54927b7ad82778ab672b27e41bfc1ef3eb5421f3f3cde7899ce906d86de8862179c0293c1831c0a555553f21c10787dddba22c68fe1e
+  data.tar.gz: f8d52d61a493f4c8abfc88a1cfdc415779a4946eeed639b03545574b2a86304d7a560c026391897ab93596280be306a02c24d3dcdbc161c30f625c211d237711

data/.travis.yml

@@ -1,9 +1,9 @@
 language: ruby
 
 rvm:
-  - "2.3.6"
-  - "2.4.3"
-  - "2.5.0"
+  - "2.4.7"
+  - "2.5"
+  - "2.6"
 
 gemfile:
   - Gemfile

data/README.md

@@ -58,18 +58,25 @@ More configuration parameters:
 - `bind`: binding interface (default: '0.0.0.0')
 - `port`: listen port (default: 24231)
 - `metrics_path`: metrics HTTP endpoint (default: /metrics)
+- `aggregated_metrics_path`: metrics HTTP endpoint (default: /aggregated_metrics)
 
 When using multiple workers, each worker binds to port + `fluent_worker_id`.
+To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
 
 ### prometheus_monitor input plugin
 
 This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](https://docs.fluentd.org/input/monitor_agent).
 
-Current exposed metrics:
 
-- `buffer_queue_length` of each BufferedOutput plugins
-- `buffer_total_queued_size` of each BufferedOutput plugins
-- `retry_count` of each BufferedOutput plugins
+#### Exposed metrics
+
+- `fluentd_status_buffer_queue_length`
+- `fluentd_status_buffer_total_queued_size`
+- `fluentd_status_retry_count`
+- `fluentd_status_buffer_newest_timekey` from fluentd v1.4.2
+- `fluentd_status_buffer_oldest_timekey` from fluentd v1.4.2
+
+#### Configuration
 
 With following configuration, those metrics are collected.
 
@@ -86,28 +93,35 @@ More configuration parameters:
 
 ### prometheus_output_monitor input plugin
 
-**experimental**
-
 This plugin collects internal metrics for output plugin in Fluentd. This is similar to `prometheus_monitor` plugin, but specialized for output plugin. There are Many metrics `prometheus_monitor` does not include, such as `num_errors`, `retry_wait` and so on.
 
-Current exposed metrics:
+#### Exposed metrics
+
+Metrics for output
 
-- `fluentd_output_status_buffer_queue_length`
-- `fluentd_output_status_buffer_total_bytes`
 - `fluentd_output_status_retry_count`
 - `fluentd_output_status_num_errors`
 - `fluentd_output_status_emit_count`
-- `fluentd_output_status_flush_time_count`
-- `fluentd_output_status_slow_flush_count`
 - `fluentd_output_status_retry_wait`
     - current retry_wait computed from last retry time and next retry time
 - `fluentd_output_status_emit_records`
-    - only for v0.14
 - `fluentd_output_status_write_count`
-    - only for v0.14
 - `fluentd_output_status_rollback_count`
-    - only for v0.14
+- `fluentd_output_status_flush_time_count` from fluentd v1.6.0
+- `fluentd_output_status_slow_flush_count` from fluentd v1.6.0
 
+Metrics for buffer
+
+- `fluentd_output_status_buffer_total_bytes`
+- `fluentd_output_status_buffer_stage_length` from fluentd v1.6.0
+- `fluentd_output_status_buffer_stage_byte_size` from fluentd v1.6.0
+- `fluentd_output_status_buffer_queue_length`
+- `fluentd_output_status_buffer_queue_byte_size` from fluentd v1.6.0
+- `fluentd_output_status_buffer_newest_timekey` from fluentd v1.6.0
+- `fluentd_output_status_buffer_oldest_timekey` from fluentd v1.6.0
+- `fluentd_output_status_buffer_available_space_ratio` from fluentd v1.6.0
+
+#### Configuration
 
 With following configuration, those metrics are collected.
 
@@ -124,13 +138,11 @@ More configuration parameters:
 
 ### prometheus_tail_monitor input plugin
 
-**experimental**
-
 This plugin collects internal metrics for in_tail plugin in Fluentd. in_tail plugin holds internal state for files that the plugin is watching. The state is sometimes important to monitor plugins work correctly.
 
 This plugin uses internal class of Fluentd, so it's easy to break.
 
-Current exposed metrics:
+#### Exposed metrics
 
 - `fluentd_tail_file_position`
     - Current bytes which plugin reads from the file
@@ -143,6 +155,8 @@ Default labels:
 - `type`: plugin name. `in_tail` only for now.
 - `path`: file path
 
+#### Configuration
+
 With following configuration, those metrics are collected.
 
 ```

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.5.0"
+  spec.version       = "1.6.0"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}

data/lib/fluent/plugin/filter_prometheus.rb

@@ -4,6 +4,7 @@ require 'fluent/plugin/filter'
 module Fluent::Plugin
   class PrometheusFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
     include Fluent::Plugin::Prometheus
 
     def initialize
@@ -17,7 +18,7 @@ module Fluent::Plugin
 
     def configure(conf)
       super
-      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      labels = parse_labels_elements(conf)
       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 

data/lib/fluent/plugin/in_prometheus.rb

@@ -1,5 +1,7 @@
 require 'fluent/plugin/input'
 require 'fluent/plugin/prometheus'
+require 'fluent/plugin/prometheus_metrics'
+require 'net/http'
 require 'webrick'
 
 module Fluent::Plugin
@@ -11,6 +13,7 @@ module Fluent::Plugin
     config_param :bind, :string, default: '0.0.0.0'
     config_param :port, :integer, default: 24231
     config_param :metrics_path, :string, default: '/metrics'
+    config_param :aggregated_metrics_path, :string, default: '/aggregated_metrics'
 
     desc 'Enable ssl configuration for the server'
     config_section :ssl, required: false, multi: false do
@@ -31,6 +34,10 @@ module Fluent::Plugin
 
     attr_reader :registry
 
+    attr_reader :num_workers
+    attr_reader :base_port
+    attr_reader :metrics_path
+
     def initialize
       super
       @registry = ::Prometheus::Client.registry
@@ -38,6 +45,18 @@ module Fluent::Plugin
 
     def configure(conf)
       super
+
+      # Get how many workers we have
+      sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                  owner.system_config
+                elsif self.respond_to?(:system_config)
+                  self.system_config
+                else
+                  nil
+                end
+      @num_workers = sysconf && sysconf.workers ? sysconf.workers : 1
+
+      @base_port = @port
       @port += fluentd_worker_id
     end
 
@@ -82,6 +101,7 @@ module Fluent::Plugin
 
       @server = WEBrick::HTTPServer.new(config)
       @server.mount(@metrics_path, MonitorServlet, self)
+      @server.mount(@aggregated_metrics_path, MonitorServletAll, self)
       thread_create(:in_prometheus) do
         @server.start
       end
@@ -110,5 +130,35 @@ module Fluent::Plugin
         res.body = $!.to_s
       end
     end
+
+    class MonitorServletAll < WEBrick::HTTPServlet::AbstractServlet
+      def initialize(server, prometheus)
+        @prometheus = prometheus
+      end
+
+      def do_GET(req, res)
+        res.status = 200
+        res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
+
+        full_result = PromMetricsAggregator.new
+        fluent_server_ip = @prometheus.bind == '0.0.0.0' ? '127.0.0.1' : @prometheus.bind
+        current_worker = 0
+        while current_worker < @prometheus.num_workers
+          Net::HTTP.start(fluent_server_ip, @prometheus.base_port + current_worker) do |http|
+            req = Net::HTTP::Get.new(@prometheus.metrics_path)
+            result = http.request(req)
+            if result.is_a?(Net::HTTPSuccess)
+              full_result.add_metrics(result.body)
+            end
+          end
+          current_worker += 1
+        end
+        res.body = full_result.get_metrics
+      rescue
+        res.status = 500
+        res['Content-Type'] = 'text/plain'
+        res.body = $!.to_s
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -25,7 +26,7 @@ module Fluent::Plugin
       hostname = Socket.gethostname
       expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
@@ -45,6 +46,12 @@ module Fluent::Plugin
     def start
       super
 
+      @buffer_newest_timekey = @registry.gauge(
+        :fluentd_status_buffer_newest_timekey,
+        'Newest timekey in buffer.')
+      @buffer_oldest_timekey = @registry.gauge(
+        :fluentd_status_buffer_oldest_timekey,
+        'Oldest timekey in buffer.')
       buffer_queue_length = @registry.gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
@@ -65,11 +72,19 @@ module Fluent::Plugin
 
     def update_monitor_info
       @monitor_agent.plugins_info_all.each do |info|
+        label = labels(info)
+
         @monitor_info.each do |name, metric|
           if info[name]
-            metric.set(labels(info), info[name])
+            metric.set(label, info[name])
           end
         end
+
+        timekeys = info["buffer_timekeys"]
+        if timekeys && !timekeys.empty?
+          @buffer_newest_timekey.set(label, timekeys.max)
+          @buffer_oldest_timekey.set(label, timekeys.min)
+        end
       end
     end
 

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusOutputMonitorInput < Fluent::Input
     Fluent::Plugin.register_input('prometheus_output_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -44,7 +45,7 @@ module Fluent::Plugin
       hostname = Socket.gethostname
       expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
@@ -64,12 +65,33 @@ module Fluent::Plugin
       super
 
       @metrics = {
-        buffer_queue_length: @registry.gauge(
-          :fluentd_output_status_buffer_queue_length,
-          'Current buffer queue length.'),
+        # Buffer metrics
         buffer_total_queued_size: @registry.gauge(
           :fluentd_output_status_buffer_total_bytes,
-          'Current total size of queued buffers.'),
+          'Current total size of stage and queue buffers.'),
+        buffer_stage_length: @registry.gauge(
+          :fluentd_output_status_buffer_stage_length,
+          'Current length of stage buffers.'),
+        buffer_stage_byte_size: @registry.gauge(
+          :fluentd_output_status_buffer_stage_byte_size,
+          'Current total size of stage buffers.'),
+        buffer_queue_length: @registry.gauge(
+          :fluentd_output_status_buffer_queue_length,
+          'Current length of queue buffers.'),
+        buffer_queue_byte_size: @registry.gauge(
+          :fluentd_output_status_queue_byte_size,
+          'Current total size of queue buffers.'),
+        buffer_available_buffer_space_ratios: @registry.gauge(
+          :fluentd_output_status_buffer_available_space_ratio,
+          'Ratio of available space in buffer.'),
+        buffer_newest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_newest_timekey,
+          'Newest timekey in buffer.'),
+        buffer_oldest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_oldest_timekey,
+          'Oldest timekey in buffer.'),
+
+        # Output metrics
         retry_counts: @registry.gauge(
           :fluentd_output_status_retry_count,
           'Current retry counts.'),
@@ -112,8 +134,17 @@ module Fluent::Plugin
       }
 
       monitor_info = {
-        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        # buffer metrics
         'buffer_total_queued_size' => @metrics[:buffer_total_queued_size],
+        'buffer_stage_length' => @metrics[:buffer_stage_length],
+        'buffer_stage_byte_size' => @metrics[:buffer_stage_byte_size],
+        'buffer_queue_length' => @metrics[:buffer_queue_length],
+        'buffer_queue_byte_size' => @metrics[:buffer_queue_byte_size],
+        'buffer_available_buffer_space_ratios' => @metrics[:buffer_available_buffer_space_ratios],
+        'buffer_newest_timekey' => @metrics[:buffer_newest_timekey],
+        'buffer_oldest_timekey' => @metrics[:buffer_oldest_timekey],
+
+        # output metrics
         'retry_count' => @metrics[:retry_counts],
       }
       instance_vars_info = {

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusTailMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_tail_monitor', self)
+    include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
@@ -29,7 +30,7 @@ module Fluent::Plugin
       hostname = Socket.gethostname
       expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
-      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"

data/lib/fluent/plugin/out_prometheus.rb

@@ -4,6 +4,7 @@ require 'fluent/plugin/prometheus'
 module Fluent::Plugin
   class PrometheusOutput < Fluent::Plugin::Output
     Fluent::Plugin.register_output('prometheus', self)
+    include Fluent::Plugin::PrometheusLabelParser
     include Fluent::Plugin::Prometheus
 
     def initialize
@@ -17,7 +18,7 @@ module Fluent::Plugin
 
     def configure(conf)
       super
-      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      labels = parse_labels_elements(conf)
       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 

data/lib/fluent/plugin/prometheus.rb

@@ -4,6 +4,31 @@ require 'fluent/plugin/filter_record_transformer'
 
 module Fluent
   module Plugin
+    module PrometheusLabelParser
+      def configure(conf)
+        super
+        # Check if running with multiple workers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+          owner.system_config
+        elsif self.respond_to?(:system_config)
+          self.system_config
+        else
+          nil
+        end
+        @multi_worker = sysconf && sysconf.workers ? (sysconf.workers > 1) : false
+      end
+
+      def parse_labels_elements(conf)
+        base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+
+        if @multi_worker
+          base_labels[:worker_id] = fluentd_worker_id.to_s
+        end
+
+        base_labels
+      end
+    end
+
     module Prometheus
       class AlreadyRegisteredError < StandardError; end
 

data/lib/fluent/plugin/prometheus_metrics.rb

@@ -0,0 +1,77 @@
+module Fluent::Plugin
+
+  ##
+  # PromMetricsAggregator aggregates multiples metrics exposed using Prometheus text-based format
+  # see https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
+
+
+  class PrometheusMetrics
+    def initialize
+      @comments = []
+      @metrics = []
+    end
+
+    def to_string
+      (@comments + @metrics).join("\n")
+    end
+
+    def add_comment(comment)
+      @comments << comment
+    end
+
+    def add_metric_value(value)
+      @metrics << value
+    end
+
+    attr_writer :comments, :metrics
+  end
+
+  class PromMetricsAggregator
+    def initialize
+      @metrics = {}
+    end
+
+    def get_metric_name_from_comment(line)
+      tokens = line.split(' ')
+      if ['HELP', 'TYPE'].include?(tokens[1])
+        tokens[2]
+      else
+        ''
+      end
+    end
+
+    def add_metrics(metrics)
+      current_metric = ''
+      new_metric = false
+      lines = metrics.split("\n")
+      for line in lines
+        if line[0] == '#'
+          # Metric comment (# TYPE, # HELP)
+          parsed_metric = get_metric_name_from_comment(line)
+          if parsed_metric != ''
+            if parsed_metric != current_metric
+              # Starting a new metric comment block
+              new_metric = !@metrics.key?(parsed_metric)
+              if new_metric
+                @metrics[parsed_metric] = PrometheusMetrics.new()
+              end
+              current_metric = parsed_metric
+            end
+
+            if new_metric && parsed_metric == current_metric
+              # New metric, inject comments (# TYPE, # HELP)
+              @metrics[parsed_metric].add_comment(line)
+            end
+          end
+        else
+          # Metric value, simply append line
+          @metrics[current_metric].add_metric_value(line)
+        end
+      end
+    end
+
+    def get_metrics
+      @metrics.map{|k,v| v.to_string()}.join("\n") + (@metrics.length ? "\n" : "")
+    end
+  end
+end

data/misc/prometheus_alerts.yaml

@@ -47,7 +47,7 @@ ALERT FluentdQueueLength
   }
 
 ALERT FluentdRecordsCountsHigh
-  IF sum(rate(fluentd_record_counts{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_record_counts{job="fluentd"}[15m])) BY (instance))
+  IF sum(rate(fluentd_output_status_emit_records{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_output_status_emit_records{job="fluentd"}[15m])) BY (instance))
   FOR 1m
   LABELS {
     service = "fluentd",

data/spec/fluent/plugin/prometheus_metrics_spec.rb

@@ -0,0 +1,138 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus'
+require 'fluent/test/driver/input'
+
+require 'net/http'
+
+describe Fluent::Plugin::PromMetricsAggregator do
+
+  metrics_worker_1 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+]
+
+  metrics_worker_2 = %[# TYPE fluentd_output_status_buffer_queue_length gauge
+# HELP fluentd_output_status_buffer_queue_length Current buffer queue length.
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+# TYPE fluentd_output_status_buffer_total_bytes gauge
+# HELP fluentd_output_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+]
+
+  metrics_worker_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  metrics_merged_1_and_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  describe 'add_metrics' do
+    context '1st_metrics' do
+      it 'adds all fields' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1)
+      end
+    end
+    context '2nd_metrics' do
+      it 'append new metrics' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1 + metrics_worker_2)
+      end
+    end
+
+    context '3rd_metrics' do
+      it 'append existing metrics in the right place' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        all_metrics.add_metrics(metrics_worker_3)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_merged_1_and_3 + metrics_worker_2)
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-01 00:00:00.000000000 Z
+date: 2019-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -122,6 +122,7 @@ files:
 - lib/fluent/plugin/in_prometheus_tail_monitor.rb
 - lib/fluent/plugin/out_prometheus.rb
 - lib/fluent/plugin/prometheus.rb
+- lib/fluent/plugin/prometheus_metrics.rb
 - misc/fluentd_sample.conf
 - misc/nginx_proxy.conf
 - misc/prometheus.yaml
@@ -129,6 +130,7 @@ files:
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus_metrics_spec.rb
 - spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb
@@ -151,7 +153,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.
@@ -159,6 +162,7 @@ test_files:
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus_metrics_spec.rb
 - spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb