fluent-plugin-prometheus 0.5.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: c8de232abd7ede7d5988f3280f6c8735de2c0371c831830ee56574ca5d38a056
-  data.tar.gz: 5042393a2e68e75dbcc35bf4ae7a857498f9d2e6ebbbd164f62a270e3274be65
+SHA1:
+  metadata.gz: b6e00809503106fad10f9d831e67091e69ed56da
+  data.tar.gz: 5f84aeff3ed1a79a6da4c7ea18a5332c02304baa
 SHA512:
-  metadata.gz: 8acba008b1f5beafae1e7cac6214083ad033def64ea5af572724cbec3337b03c122cef09b1935a9617e375abc0f83b7b33269904f567f7e17ec90d7f79e8da40
-  data.tar.gz: 7187a5995e479c445bb0457081606b869456ad46bacd539962fb74147f5b9d72d0b78cb45b1d43a64c247328f673c91aa12ce86c53e915e171d6f5fb70e23afa
+  metadata.gz: 7658cc206cc8063ea61d83730b37db12a311f152a560c3c5d669c9f73dd9994c417b1a30810531d101c56821f4395f5d875fe5ef5cabb67e72948974186cdad9
+  data.tar.gz: ac72182e9fea379125a487f521d42a29bbf354d857abf3dc778daffdec8aa8bd346d444b3ab001cbcac951fde2b72c3ad208e0e2748aa9d5a14d35b34e73e1c3

data/.travis.yml

@@ -1,8 +1,8 @@
 language: ruby
 
 rvm:
-  - "2.3.3"
-  - "2.4.0"
+  - "2.3.4"
+  - "2.4.1"
 
 gemfile:
   - Gemfile

data/Gemfile

@@ -1,3 +1,4 @@
-source 'https://rubygems.org/'
-gem 'fluentd', '~> 0.12.0'
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-prometheus.gemspec
 gemspec

data/README.md

@@ -1,11 +1,18 @@
 # fluent-plugin-prometheus, a plugin for [Fluentd](https://www.fluentd.org)
 
-This is the branch to support Fluentd v0.12. If you want to use Fluentd v0.14 or later, see [master branch](https://github.com/fluent/fluent-plugin-prometheus).
-
-[![Build Status](https://travis-ci.org/fluent/fluent-plugin-prometheus.svg?branch=v0.12)](https://travis-ci.org/fluent/fluent-plugin-prometheus)
+[![Build Status](https://travis-ci.org/fluent/fluent-plugin-prometheus.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-prometheus)
 
 A fluent plugin that instruments metrics from records and exposes them via web interface. Intended to be used together with a [Prometheus server](https://github.com/prometheus/prometheus).
 
+If you are using Fluentd v0.10, you have to explicitly install [fluent-plugin-record-reformer](https://github.com/sonots/fluent-plugin-record-reformer) together. With Fluentd v0.12 or v0.14, there is no additional dependency.
+
+## Requirements
+
+| fluent-plugin-prometheus | fluentd    | ruby   |
+|--------------------------|------------|--------|
+| 1.x.y                    | >= v0.14.8 | >= 2.1 |
+| 0.x.y                    | >= v0.12.0 | >= 1.9 |
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -72,7 +79,7 @@ With following configuration, those metrics are collected.
 
 More configuration parameters:
 
-- `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
+- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 - `interval`: interval to update monitor_agent information in seconds (default: 5)
 
 ### prometheus_output_monitor input plugin
@@ -106,7 +113,7 @@ With following configuration, those metrics are collected.
 
 More configuration parameters:
 
-- `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
+- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 - `interval`: interval to update monitor_agent information in seconds (default: 5)
 
 ### prometheus_tail_monitor input plugin
@@ -133,12 +140,12 @@ Default labels:
 With following configuration, those metrics are collected.
 
 <source>
-  @type prometheus_output_monitor
+  @type prometheus_tail_monitor
 </source>
 
 More configuration parameters:
 
-- `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
+- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 - `interval`: interval to update monitor_agent information in seconds (default: 5)
 
 ### prometheus output/filter plugin
@@ -237,7 +244,7 @@ For details of each metric type, see [Prometheus documentation](http://prometheu
 - `type`: metric type (required)
 - `desc`: description of this metric (required)
 - `key`: key name of record for instrumentation (**optional**)
-- `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
+- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 
 If key is empty, the metric values is treated as 1, so the counter increments by 1 on each record regardless of contents of the record.
 
@@ -261,7 +268,7 @@ If key is empty, the metric values is treated as 1, so the counter increments by
 - `type`: metric type (required)
 - `desc`: description of metric (required)
 - `key`: key name of record for instrumentation (required)
-- `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
+- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 
 ### summary type
 
@@ -283,7 +290,7 @@ If key is empty, the metric values is treated as 1, so the counter increments by
 - `type`: metric type (required)
 - `desc`: description of metric (required)
 - `key`: key name of record for instrumentation (required)
-- `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
+- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 
 ### histogram type
 
@@ -307,7 +314,7 @@ If key is empty, the metric values is treated as 1, so the counter increments by
 - `desc`: description of metric (required)
 - `key`: key name of record for instrumentation (required)
 - `buckets`: buckets of record for instrumentation (optional)
-- `<labels>`: additional labels for this metric (optional). See [Labels](#Labels)
+- `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 
 ## Labels
 

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "0.5.0"
+  spec.version       = "1.0.0"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "fluentd", ">= 0.12.0", "< 0.14.0"
+  spec.add_dependency "fluentd", ">= 0.14.8", "< 2"
   spec.add_dependency "prometheus-client"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"

data/lib/fluent/plugin/filter_prometheus.rb

@@ -1,9 +1,10 @@
 require 'fluent/plugin/prometheus'
+require 'fluent/plugin/filter'
 
-module Fluent
-  class PrometheusFilter < Filter
-    Plugin.register_filter('prometheus', self)
-    include Fluent::Prometheus
+module Fluent::Plugin
+  class PrometheusFilter < Fluent::Plugin::Filter
+    Fluent::Plugin.register_filter('prometheus', self)
+    include Fluent::Plugin::Prometheus
 
     def initialize
       super
@@ -12,8 +13,8 @@ module Fluent
 
     def configure(conf)
       super
-      labels = Fluent::Prometheus.parse_labels_elements(conf)
-      @metrics = Fluent::Prometheus.parse_metrics_elements(conf, @registry, labels)
+      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 
     def filter_stream(tag, es)

data/lib/fluent/plugin/in_prometheus.rb

@@ -1,31 +1,16 @@
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/plugin/prometheus'
 require 'webrick'
 
-module Fluent
-  class PrometheusInput < Input
-    Plugin.register_input('prometheus', self)
+module Fluent::Plugin
+  class PrometheusInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus', self)
 
-    config_param :bind, :string, :default => '0.0.0.0'
-    config_param :port, :integer, :default => 24231
-    config_param :metrics_path, :string, :default => '/metrics'
+    helpers :thread
 
-    desc 'Enable ssl configuration for the server'
-    config_section :ssl, multi: false, required: false do
-      config_param :enable, :bool, required: false, default: false
-
-      desc 'Path to the ssl certificate in PEM format.  Read from file and added to conf as "SSLCertificate"'
-      config_param :certificate_path, :string, required: false, default: nil
-
-      desc 'Path to the ssl private key in PEM format.  Read from file and added to conf as "SSLPrivateKey"'
-      config_param :private_key_path, :string, required: false, default: nil
-
-      desc 'Path to CA in PEM format.  Read from file and added to conf as "SSLCACertificateFile"'
-      config_param :ca_path, :string, required: false, default: nil
-
-      desc 'Additional ssl conf for the server.  Ref: https://github.com/ruby/webrick/blob/master/lib/webrick/ssl.rb'
-      config_param :extra_conf, :hash, multi: false, required: false, default: {:SSLCertName => [['CN','nobody'],['DC','example']]}, symbolize_keys: true
-    end
+    config_param :bind, :string, default: '0.0.0.0'
+    config_param :port, :integer, default: 24231
+    config_param :metrics_path, :string, default: '/metrics'
 
     attr_reader :registry
 
@@ -34,59 +19,27 @@ module Fluent
       @registry = ::Prometheus::Client.registry
     end
 
-    def configure(conf)
-      super
-    end
-
     def start
       super
-      config = {
+      @server = WEBrick::HTTPServer.new(
         BindAddress: @bind,
         Port: @port,
         MaxClients: 5,
         Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
         AccessLog: [],
-      }
-      unless @ssl.nil? || !@ssl['enable']
-        require 'webrick/https'
-        require 'openssl'
-        if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
-            raise RuntimeError.new("certificate_path and private_key_path most both be defined")
-        end
-        ssl_config = {
-            SSLEnable: true
-        }
-        if @ssl['certificate_path']
-          cert = OpenSSL::X509::Certificate.new(File.read(@ssl['certificate_path']))
-          ssl_config[:SSLCertificate] = cert
-        end
-        if @ssl['private_key_path']
-          key = OpenSSL::PKey::RSA.new(File.read(@ssl['private_key_path']))
-          ssl_config[:SSLPrivateKey] = key
-        end
-        ssl_config[:SSLCACertificateFile] = @ssl['ca_path'] if @ssl['ca_path']
-        ssl_config = ssl_config.merge(@ssl['extra_conf'])
-        config = ssl_config.merge(config)
-      end
-      @log.on_debug do
-        @log.debug("WEBrick conf: #{config}")
-      end
-
-      @server = WEBrick::HTTPServer.new(config)
+      )
       @server.mount(@metrics_path, MonitorServlet, self)
-      @thread = Thread.new { @server.start }
+      thread_create(:in_prometheus) do
+        @server.start
+      end
     end
 
     def shutdown
-      super
       if @server
         @server.shutdown
         @server = nil
       end
-      if @thread
-        @thread.join
-        @thread = nil
-      end
+      super
     end
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -1,12 +1,14 @@
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/plugin/in_monitor_agent'
 require 'fluent/plugin/prometheus'
 
-module Fluent
-  class PrometheusMonitorInput < Input
-    Plugin.register_input('prometheus_monitor', self)
+module Fluent::Plugin
+  class PrometheusMonitorInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus_monitor', self)
 
-    config_param :interval, :time, :default => 5
+    helpers :timer
+
+    config_param :interval, :time, default: 5
     attr_reader :registry
 
     def initialize
@@ -17,9 +19,9 @@ module Fluent
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Prometheus.placeholder_expander(log)
+      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname})
-      @base_labels = Fluent::Prometheus.parse_labels_elements(conf)
+      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
       @base_labels.each do |key, value|
         @base_labels[key] = expander.expand(value, placeholders)
       end
@@ -48,41 +50,9 @@ module Fluent
       }
     end
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, log, &callback)
-        @callback = callback
-        @log = log
-        super(interval, repeat)
-      end
-
-      def on_timer
-        @callback.call
-      rescue
-        @log.error $!.to_s
-        @log.error_backtrace
-      end
-    end
-
     def start
       super
-      @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@interval, true, log, &method(:update_monitor_info))
-      @loop.attach(@timer)
-      @thread = Thread.new(&method(:run))
-    end
-
-    def shutdown
-      super
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @thread.join
-    end
-
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", :error=>$!.to_s
-      log.error_backtrace
+      timer_execute(:in_prometheus_monitor, @interval, &method(:update_monitor_info))
     end
 
     def update_monitor_info

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -2,11 +2,13 @@ require 'fluent/input'
 require 'fluent/plugin/in_monitor_agent'
 require 'fluent/plugin/prometheus'
 
-module Fluent
-  class PrometheusOutputMonitorInput < Input
-    Plugin.register_input('prometheus_output_monitor', self)
+module Fluent::Plugin
+  class PrometheusOutputMonitorInput < Fluent::Input
+    Fluent::Plugin.register_input('prometheus_output_monitor', self)
 
-    config_param :interval, :time, :default => 5
+    helpers :timer
+
+    config_param :interval, :time, default: 5
     attr_reader :registry
 
     MONITOR_IVARS = [
@@ -32,9 +34,9 @@ module Fluent
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Prometheus.placeholder_expander(log)
+      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname})
-      @base_labels = Fluent::Prometheus.parse_labels_elements(conf)
+      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
       @base_labels.each do |key, value|
         @base_labels[key] = expander.expand(value, placeholders)
       end
@@ -77,41 +79,9 @@ module Fluent
       }
     end
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, log, &callback)
-        @callback = callback
-        @log = log
-        super(interval, repeat)
-      end
-
-      def on_timer
-        @callback.call
-      rescue
-        @log.error $!.to_s
-        @log.error_backtrace
-      end
-    end
-
     def start
       super
-      @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@interval, true, log, &method(:update_monitor_info))
-      @loop.attach(@timer)
-      @thread = Thread.new(&method(:run))
-    end
-
-    def shutdown
-      super
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @thread.join
-    end
-
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", :error=>$!.to_s
-      log.error_backtrace
+      timer_execute(:in_prometheus_output_monitor, @interval, &method(:update_monitor_info))
     end
 
     def update_monitor_info

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -1,12 +1,14 @@
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/plugin/in_monitor_agent'
 require 'fluent/plugin/prometheus'
 
-module Fluent
-  class PrometheusTailMonitorInput < Input
-    Plugin.register_input('prometheus_tail_monitor', self)
+module Fluent::Plugin
+  class PrometheusTailMonitorInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('prometheus_tail_monitor', self)
 
-    config_param :interval, :time, :default => 5
+    helpers :timer
+
+    config_param :interval, :time, default: 5
     attr_reader :registry
 
     MONITOR_IVARS = [
@@ -21,9 +23,9 @@ module Fluent
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Prometheus.placeholder_expander(log)
+      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
       placeholders = expander.prepare_placeholders({'hostname' => hostname})
-      @base_labels = Fluent::Prometheus.parse_labels_elements(conf)
+      @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
       @base_labels.each do |key, value|
         @base_labels[key] = expander.expand(value, placeholders)
       end
@@ -45,41 +47,9 @@ module Fluent
       }
     end
 
-    class TimerWatcher < Coolio::TimerWatcher
-      def initialize(interval, repeat, log, &callback)
-        @callback = callback
-        @log = log
-        super(interval, repeat)
-      end
-
-      def on_timer
-        @callback.call
-      rescue
-        @log.error $!.to_s
-        @log.error_backtrace
-      end
-    end
-
     def start
       super
-      @loop = Coolio::Loop.new
-      @timer = TimerWatcher.new(@interval, true, log, &method(:update_monitor_info))
-      @loop.attach(@timer)
-      @thread = Thread.new(&method(:run))
-    end
-
-    def shutdown
-      super
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @thread.join
-    end
-
-    def run
-      @loop.run
-    rescue
-      log.error "unexpected error", :error=>$!.to_s
-      log.error_backtrace
+      timer_execute(:in_prometheus_tail_monitor, @interval, &method(:update_monitor_info))
     end
 
     def update_monitor_info

data/lib/fluent/plugin/out_prometheus.rb

@@ -1,10 +1,10 @@
-require 'fluent/output'
+require 'fluent/plugin/output'
 require 'fluent/plugin/prometheus'
 
-module Fluent
-  class PrometheusOutput < Output
-    Plugin.register_output('prometheus', self)
-    include Fluent::Prometheus
+module Fluent::Plugin
+  class PrometheusOutput < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('prometheus', self)
+    include Fluent::Plugin::Prometheus
 
     def initialize
       super
@@ -13,13 +13,12 @@ module Fluent
 
     def configure(conf)
       super
-      labels = Fluent::Prometheus.parse_labels_elements(conf)
-      @metrics = Fluent::Prometheus.parse_metrics_elements(conf, @registry, labels)
+      labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
+      @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 
-    def emit(tag, es, chain)
+    def process(tag, es)
       instrument(tag, es, @metrics)
-      chain.next
     end
   end
 end

data/lib/fluent/plugin/prometheus.rb

@@ -1,219 +1,213 @@
 require 'prometheus/client'
 require 'prometheus/client/formats/text'
+require 'fluent/plugin/filter_record_transformer'
 
 module Fluent
-  module Prometheus
-    class AlreadyRegisteredError < StandardError; end
+  module Plugin
+    module Prometheus
+      class AlreadyRegisteredError < StandardError; end
 
-    def self.parse_labels_elements(conf)
-      labels = conf.elements.select { |e| e.name == 'labels' }
-      if labels.size > 1
-        raise ConfigError, "labels section must have at most 1"
-      end
-
-      base_labels = {}
-      unless labels.empty?
-        labels.first.each do |key, value|
-          labels.first.has_key?(key)
-          base_labels[key.to_sym] = value
+      def self.parse_labels_elements(conf)
+        labels = conf.elements.select { |e| e.name == 'labels' }
+        if labels.size > 1
+          raise ConfigError, "labels section must have at most 1"
         end
-      end
-
-      base_labels
-    end
 
-    def self.parse_metrics_elements(conf, registry, labels = {})
-      metrics = []
-      conf.elements.select { |element|
-        element.name == 'metric'
-      }.each { |element|
-        case element['type']
-        when 'summary'
-          metrics << Fluent::Prometheus::Summary.new(element, registry, labels)
-        when 'gauge'
-          metrics << Fluent::Prometheus::Gauge.new(element, registry, labels)
-        when 'counter'
-          metrics << Fluent::Prometheus::Counter.new(element, registry, labels)
-        when 'histogram'
-          metrics << Fluent::Prometheus::Histogram.new(element, registry, labels)
-        else
-          raise ConfigError, "type option must be 'counter', 'gauge', 'summary' or 'histogram'"
+        base_labels = {}
+        unless labels.empty?
+          labels.first.each do |key, value|
+            labels.first.has_key?(key)
+            base_labels[key.to_sym] = value
+          end
         end
-      }
-      metrics
-    end
 
-    def self.placeholder_expander(log)
-      # Use internal class in order to expand placeholder
-      require 'fluent/plugin/filter_record_transformer'
-      if defined?(Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander)
-        # for v0.14
-        return Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander.new(log: log)
-      else
-        # for v0.12
-        return Fluent::RecordTransformerFilter::PlaceholderExpander.new(log: log)
+        base_labels
       end
-    rescue LoadError => e
-      raise ConfigError, "cannot find filter_record_transformer plugin: #{e.message}"
-    end
 
-    def configure(conf)
-      super
-      @placeholder_expander = Fluent::Prometheus.placeholder_expander(log)
-      @hostname = Socket.gethostname
-    end
+      def self.parse_metrics_elements(conf, registry, labels = {})
+        metrics = []
+        conf.elements.select { |element|
+          element.name == 'metric'
+        }.each { |element|
+          case element['type']
+          when 'summary'
+            metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels)
+          when 'gauge'
+            metrics << Fluent::Plugin::Prometheus::Gauge.new(element, registry, labels)
+          when 'counter'
+            metrics << Fluent::Plugin::Prometheus::Counter.new(element, registry, labels)
+          when 'histogram'
+            metrics << Fluent::Plugin::Prometheus::Histogram.new(element, registry, labels)
+          else
+            raise ConfigError, "type option must be 'counter', 'gauge', 'summary' or 'histogram'"
+          end
+        }
+        metrics
+      end
 
-    def instrument(tag, es, metrics)
-      placeholder_values = {
-        'tag' => tag,
-        'hostname' => @hostname,
-      }
+      def self.placeholder_expander(log)
+        # Use internal class in order to expand placeholder
+        Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander.new(log: log)
+      end
 
-      es.each do |time, record|
-        placeholders = record.merge(placeholder_values)
-        placeholders = @placeholder_expander.prepare_placeholders(placeholders)
-        metrics.each do |metric|
-          begin
-            metric.instrument(record, @placeholder_expander, placeholders)
-          rescue => e
-            log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
-            router.emit_error_event(tag, time, record, e)
+      def configure(conf)
+        super
+        @placeholder_expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
+        @hostname = Socket.gethostname
+      end
+
+      def instrument(tag, es, metrics)
+        placeholder_values = {
+          'tag' => tag,
+          'hostname' => @hostname,
+        }
+
+        es.each do |time, record|
+          placeholders = record.merge(placeholder_values)
+          placeholders = @placeholder_expander.prepare_placeholders(placeholders)
+          metrics.each do |metric|
+            begin
+              metric.instrument(record, @placeholder_expander, placeholders)
+            rescue => e
+              log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
+              router.emit_error_event(tag, time, record, e)
+            end
           end
         end
       end
-    end
 
-    class Metric
-      attr_reader :type
-      attr_reader :name
-      attr_reader :key
-      attr_reader :desc
+      class Metric
+        attr_reader :type
+        attr_reader :name
+        attr_reader :key
+        attr_reader :desc
 
-      def initialize(element, registry, labels)
-        ['name', 'desc'].each do |key|
-          if element[key].nil?
-            raise ConfigError, "metric requires '#{key}' option"
+        def initialize(element, registry, labels)
+          ['name', 'desc'].each do |key|
+            if element[key].nil?
+              raise ConfigError, "metric requires '#{key}' option"
+            end
           end
-        end
-        @type = element['type']
-        @name = element['name']
-        @key = element['key']
-        @desc = element['desc']
+          @type = element['type']
+          @name = element['name']
+          @key = element['key']
+          @desc = element['desc']
 
-        @base_labels = Fluent::Prometheus.parse_labels_elements(element)
-        @base_labels = labels.merge(@base_labels)
-      end
+          @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(element)
+          @base_labels = labels.merge(@base_labels)
+        end
 
-      def labels(record, expander, placeholders)
-        label = {}
-        @base_labels.each do |k, v|
-          label[k] = expander.expand(v, placeholders)
+        def labels(record, expander, placeholders)
+          label = {}
+          @base_labels.each do |k, v|
+            label[k] = expander.expand(v, placeholders)
+          end
+          label
         end
-        label
-      end
 
-      def self.get(registry, name, type, docstring)
-        metric = registry.get(name)
+        def self.get(registry, name, type, docstring)
+          metric = registry.get(name)
 
-        # should have same type, docstring
-        if metric.type != type
-          raise AlreadyRegisteredError, "#{name} has already been registered as #{type} type"
-        end
-        if metric.docstring != docstring
-          raise AlreadyRegisteredError, "#{name} has already been registered with different docstring"
-        end
+          # should have same type, docstring
+          if metric.type != type
+            raise AlreadyRegisteredError, "#{name} has already been registered as #{type} type"
+          end
+          if metric.docstring != docstring
+            raise AlreadyRegisteredError, "#{name} has already been registered with different docstring"
+          end
 
-        metric
+          metric
+        end
       end
-    end
 
-    class Gauge < Metric
-      def initialize(element, registry, labels)
-        super
-        if @key.nil?
-          raise ConfigError, "gauge metric requires 'key' option"
-        end
+      class Gauge < Metric
+        def initialize(element, registry, labels)
+          super
+          if @key.nil?
+            raise ConfigError, "gauge metric requires 'key' option"
+          end
 
-        begin
-          @gauge = registry.gauge(element['name'].to_sym, element['desc'])
-        rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
-          @gauge = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
+          begin
+            @gauge = registry.gauge(element['name'].to_sym, element['desc'])
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
+          end
         end
-      end
 
-      def instrument(record, expander, placeholders)
-        if record[@key]
-          @gauge.set(labels(record, expander, placeholders), record[@key])
+        def instrument(record, expander, placeholders)
+          if record[@key]
+            @gauge.set(labels(record, expander, placeholders), record[@key])
+          end
         end
       end
-    end
 
-    class Counter < Metric
-      def initialize(element, registry, labels)
-        super
-        begin
-          @counter = registry.counter(element['name'].to_sym, element['desc'])
-        rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
-          @counter = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
+      class Counter < Metric
+        def initialize(element, registry, labels)
+          super
+          begin
+            @counter = registry.counter(element['name'].to_sym, element['desc'])
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
+          end
         end
-      end
 
-      def instrument(record, expander, placeholders)
-        # use record value of the key if key is specified, otherwise just increment
-        value = @key ? record[@key] : 1
+        def instrument(record, expander, placeholders)
+          # use record value of the key if key is specified, otherwise just increment
+          value = @key ? record[@key] : 1
 
-        # ignore if record value is nil
-        return if value.nil?
+          # ignore if record value is nil
+          return if value.nil?
 
-        @counter.increment(labels(record, expander, placeholders), value)
+          @counter.increment(labels(record, expander, placeholders), value)
+        end
       end
-    end
 
-    class Summary < Metric
-      def initialize(element, registry, labels)
-        super
-        if @key.nil?
-          raise ConfigError, "summary metric requires 'key' option"
-        end
+      class Summary < Metric
+        def initialize(element, registry, labels)
+          super
+          if @key.nil?
+            raise ConfigError, "summary metric requires 'key' option"
+          end
 
-        begin
-          @summary = registry.summary(element['name'].to_sym, element['desc'])
-        rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
-          @summary = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
+          begin
+            @summary = registry.summary(element['name'].to_sym, element['desc'])
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
+          end
         end
-      end
 
-      def instrument(record, expander, placeholders)
-        if record[@key]
-          @summary.observe(labels(record, expander, placeholders), record[@key])
+        def instrument(record, expander, placeholders)
+          if record[@key]
+            @summary.observe(labels(record, expander, placeholders), record[@key])
+          end
         end
       end
-    end
 
-    class Histogram < Metric
-      def initialize(element, registry, labels)
-        super
-        if @key.nil?
-          raise ConfigError, "histogram metric requires 'key' option"
-        end
+      class Histogram < Metric
+        def initialize(element, registry, labels)
+          super
+          if @key.nil?
+            raise ConfigError, "histogram metric requires 'key' option"
+          end
 
-        begin
-          if element['buckets']
-            buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
-              e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
+          begin
+            if element['buckets']
+              buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
+                e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
+              end
+              @histogram = registry.histogram(element['name'].to_sym, element['desc'], {}, buckets)
+            else
+              @histogram = registry.histogram(element['name'].to_sym, element['desc'])
             end
-            @histogram = registry.histogram(element['name'].to_sym, element['desc'], {}, buckets)
-          else
-            @histogram = registry.histogram(element['name'].to_sym, element['desc'])
+          rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
+            @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
           end
-        rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
-          @histogram = Fluent::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
         end
-      end
 
-      def instrument(record, expander, placeholders)
-        if record[@key]
-          @histogram.observe(labels(record, expander, placeholders), record[@key])
+        def instrument(record, expander, placeholders)
+          if record[@key]
+            @histogram.observe(labels(record, expander, placeholders), record[@key])
+          end
         end
       end
     end

data/misc/fluentd_sample.conf

@@ -85,6 +85,13 @@
     buffer_type memory
     flush_interval 1s
     max_retry_wait 2s
+    <buffer>
+    # max_retry_wait 10s
+    flush_interval 1s
+    # retry_type periodic
+    disable_retry_limit
+</buffer>
+    # retry_limit 3
     disable_retry_limit
    <server>
      host 127.0.0.1

data/spec/fluent/plugin/filter_prometheus_spec.rb

@@ -1,10 +1,11 @@
 require 'spec_helper'
+require 'fluent/test/driver/filter'
 require 'fluent/plugin/filter_prometheus'
 require_relative 'shared'
 
-describe Fluent::PrometheusFilter do
+describe Fluent::Plugin::PrometheusFilter do
   let(:tag) { 'prometheus.test' }
-  let(:driver) { Fluent::Test::FilterTestDriver.new(Fluent::PrometheusFilter, tag).configure(config, true) }
+  let(:driver) { Fluent::Test::Driver::Filter.new(Fluent::Plugin::PrometheusFilter).configure(config) }
   let(:registry) { ::Prometheus::Client.registry }
 
   describe '#configure' do
@@ -13,7 +14,10 @@ describe Fluent::PrometheusFilter do
 
   describe '#run' do
     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
-    let(:es) { driver.run { driver.emit(message, Time.now) }.filtered }
+    let(:es) {
+      driver.run(default_tag: tag) { driver.feed(event_time, message) }
+      driver.filtered_records
+    }
 
     context 'simple config' do
       include_context 'simple_config'
@@ -25,7 +29,7 @@ describe Fluent::PrometheusFilter do
       end
 
       it 'should keep original message' do
-        expect(es.first[1]).to eq(message)
+        expect(es.first).to eq(message)
       end
     end
 

data/spec/fluent/plugin/out_prometheus_spec.rb

@@ -1,10 +1,11 @@
 require 'spec_helper'
+require 'fluent/test/driver/output'
 require 'fluent/plugin/out_prometheus'
 require_relative 'shared'
 
-describe Fluent::PrometheusOutput do
+describe Fluent::Plugin::PrometheusOutput do
   let(:tag) { 'prometheus.test' }
-  let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::PrometheusOutput, tag).configure(config) }
+  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::PrometheusOutput).configure(config) }
   let(:registry) { ::Prometheus::Client.registry }
 
   describe '#configure' do
@@ -13,7 +14,10 @@ describe Fluent::PrometheusOutput do
 
   describe '#run' do
     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
-    let(:es) { driver.run { driver.emit(message, Time.now) } }
+    let(:es) {
+      driver.run(default_tag: tag) { driver.feed(event_time, message) }
+      driver.events
+    }
 
     context 'simple config' do
       include_context 'simple_config'

data/spec/fluent/plugin/prometheus_spec.rb

@@ -1,9 +1,10 @@
 require 'spec_helper'
 require 'fluent/plugin/in_prometheus'
+require 'fluent/test/driver/input'
 
 require 'net/http'
 
-describe Fluent::PrometheusInput do
+describe Fluent::Plugin::PrometheusInput do
   CONFIG = %[
   type prometheus
 ]
@@ -15,7 +16,7 @@ describe Fluent::PrometheusInput do
 
   let(:config) { CONFIG }
   let(:port) { 24231 }
-  let(:driver) { Fluent::Test::InputTestDriver.new(Fluent::PrometheusInput).configure(config) }
+  let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusInput).configure(config) }
 
   describe '#configure' do
     describe 'bind' do
@@ -50,7 +51,7 @@ describe Fluent::PrometheusInput do
     context '/metrics' do
       let(:config) { LOCAL_CONFIG }
       it 'returns 200' do
-        driver.run do
+        driver.run(timeout: 1) do
           Net::HTTP.start("127.0.0.1", port) do |http|
             req = Net::HTTP::Get.new("/metrics")
             res = http.request(req)
@@ -63,7 +64,7 @@ describe Fluent::PrometheusInput do
     context '/foo' do
       let(:config) { LOCAL_CONFIG }
       it 'does not return 200' do
-        driver.run do
+        driver.run(timeout: 1) do
           Net::HTTP.start("127.0.0.1", port) do |http|
             req = Net::HTTP::Get.new("/foo")
             res = http.request(req)

data/spec/spec_helper.rb

@@ -1,8 +1,10 @@
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'fluent/test'
+require 'fluent/test/helpers'
 require 'fluent/plugin/prometheus'
 
 # Disable Test::Unit
 Test::Unit::AutoRunner.need_auto_run = false
 
 Fluent::Test.setup
+include Fluent::Test::Helpers

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-09 00:00:00.000000000 Z
+date: 2017-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,20 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.8
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.8
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
@@ -151,7 +151,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.