fluent-plugin-prometheus 1.7.0 → 1.8.1

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -24,14 +24,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
       @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -43,14 +43,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
       @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -28,14 +28,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
       @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)

data/lib/fluent/plugin/prometheus.rb

@@ -1,6 +1,6 @@
 require 'prometheus/client'
 require 'prometheus/client/formats/text'
-require 'fluent/plugin/filter_record_transformer'
+require 'fluent/plugin/prometheus/placeholder_expander'
 
 module Fluent
   module Plugin
@@ -82,14 +82,13 @@ module Fluent
       end
 
       def self.placeholder_expander(log)
-        # Use internal class in order to expand placeholder
-        Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander.new(log: log)
+        Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
       end
 
       def configure(conf)
         super
         @placeholder_values = {}
-        @placeholder_expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
+        @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
         @hostname = Socket.gethostname
       end
 
@@ -101,10 +100,10 @@ module Fluent
         }
 
         placeholders = record.merge(@placeholder_values[tag])
-        placeholders = @placeholder_expander.prepare_placeholders(placeholders)
+        expander = @placeholder_expander_builder.build(placeholders)
         metrics.each do |metric|
           begin
-            metric.instrument(record, @placeholder_expander, placeholders)
+            metric.instrument(record, expander)
           rescue => e
             log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
             router.emit_error_event(tag, time, record, e)
@@ -121,10 +120,10 @@ module Fluent
 
         es.each do |time, record|
           placeholders = record.merge(placeholder_values)
-          placeholders = @placeholder_expander.prepare_placeholders(placeholders)
+          expander = @placeholder_expander_builder.build(placeholders)
           metrics.each do |metric|
             begin
-              metric.instrument(record, @placeholder_expander, placeholders)
+              metric.instrument(record, expander)
             rescue => e
               log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
               router.emit_error_event(tag, time, record, e)
@@ -154,11 +153,11 @@ module Fluent
           @base_labels = labels.merge(@base_labels)
         end
 
-        def labels(record, expander, placeholders)
+        def labels(record, expander)
           label = {}
           @base_labels.each do |k, v|
             if v.is_a?(String)
-              label[k] = expander.expand(v, placeholders)
+              label[k] = expander.expand(v)
             else
               label[k] = v.call(record)
             end
@@ -195,14 +194,14 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @gauge.set(labels(record, expander, placeholders), value)
+            @gauge.set(labels(record, expander), value)
           end
         end
       end
@@ -217,7 +216,7 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           # use record value of the key if key is specified, otherwise just increment
           if @key.nil?
             value = 1
@@ -230,7 +229,7 @@ module Fluent
           # ignore if record value is nil
           return if value.nil?
 
-          @counter.increment(labels(record, expander, placeholders), value)
+          @counter.increment(labels(record, expander), value)
         end
       end
 
@@ -248,14 +247,14 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @summary.observe(labels(record, expander, placeholders), value)
+            @summary.observe(labels(record, expander), value)
           end
         end
       end
@@ -281,14 +280,14 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @histogram.observe(labels(record, expander, placeholders), value)
+            @histogram.observe(labels(record, expander), value)
           end
         end
       end

data/lib/fluent/plugin/prometheus/placeholder_expander.rb

@@ -0,0 +1,132 @@
+module Fluent
+  module Plugin
+    module Prometheus
+      class ExpandBuilder
+        def self.build(placeholder, log:)
+          new(log: log).build(placeholder)
+        end
+
+        def initialize(log:)
+          @log = log
+        end
+
+        def build(placeholder_values)
+          placeholders = {}
+          placeholder_values.each do |key, value|
+            case value
+            when Array
+              size = value.size
+              value.each_with_index do |v, i|
+                placeholders["${#{key}[#{i}]}"] = v
+                placeholders["${#{key}[#{i - size}]}"] = v
+              end
+            when Hash
+              value.each do |k, v|
+                placeholders[%(${#{key}["#{k}"]})] = v
+              end
+            else
+              if key == 'tag'
+                placeholders.merge!(build_tag(value))
+              else
+                placeholders["${#{key}}"] = value
+              end
+            end
+          end
+
+          Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander.new(@log, placeholders)
+        end
+
+        private
+
+        def build_tag(tag)
+          tags = tag.split('.')
+
+          placeholders = { '${tag}' => tag }
+
+          size = tags.size
+
+          tags.each_with_index do |v, i|
+            placeholders["${tag_parts[#{i}]}"] = v
+            placeholders["${tag_parts[#{i - size}]}"] = v
+          end
+
+          tag_prefix(tags).each_with_index do |v, i|
+            placeholders["${tag_prefix[#{i}]}"] = v
+          end
+
+          tag_suffix(tags).each_with_index do |v, i|
+            placeholders["${tag_suffix[#{i}]}"] = v
+          end
+
+          placeholders
+        end
+
+        def tag_prefix(tags)
+          tags = tags.dup
+          return [] if tags.empty?
+
+          ret = [tags.shift]
+          tags.each.with_index(1) do |tag, i|
+            ret[i] = "#{ret[i-1]}.#{tag}"
+          end
+          ret
+        end
+
+        def tag_suffix(tags)
+          return [] if tags.empty?
+
+          tags = tags.dup.reverse
+          ret = [tags.shift]
+          tags.each.with_index(1) do |tag, i|
+            ret[i] = "#{tag}.#{ret[i-1]}"
+          end
+          ret
+        end
+
+        class PlaceholderExpander
+          PLACEHOLDER_REGEX = /(\${[^\[}]+(\[[^\]]+\])?})/.freeze
+
+          attr_reader :placeholder
+
+          def initialize(log, placeholder)
+            @placeholder = placeholder
+            @log = log
+            @expander_cache = {}
+          end
+
+          def merge_placeholder(placeholder)
+            @placeholder.merge!(placeholder)
+          end
+
+          def expand(str, dynamic_placeholders: nil)
+            expander = if dynamic_placeholders
+                         if @expander_cache[dynamic_placeholders]
+                           @expander_cache[dynamic_placeholders]
+                         else
+                           e = ExpandBuilder.build(dynamic_placeholders, log: @log)
+                           e.merge_placeholder(@placeholder)
+                           @expander_cache[dynamic_placeholders] = e
+                           e
+                         end
+                       else
+                         self
+                       end
+
+            expander.expand!(str)
+          end
+
+          protected
+
+          def expand!(str)
+            str.gsub(PLACEHOLDER_REGEX) { |value|
+              @placeholder.fetch(value) do
+                @log.warn("unknown placeholder `#{value}` found")
+                value # return as it is
+              end
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/spec/fluent/plugin/filter_prometheus_spec.rb

@@ -6,7 +6,11 @@ require_relative 'shared'
 describe Fluent::Plugin::PrometheusFilter do
   let(:tag) { 'prometheus.test' }
   let(:driver) { Fluent::Test::Driver::Filter.new(Fluent::Plugin::PrometheusFilter).configure(config) }
-  let(:registry) { ::Prometheus::Client.registry }
+  let(:registry) { ::Prometheus::Client::Registry.new }
+
+  before do
+    allow(Prometheus::Client).to receive(:registry).and_return(registry)
+  end
 
   describe '#configure' do
     it_behaves_like 'output configuration'
@@ -14,22 +18,28 @@ describe Fluent::Plugin::PrometheusFilter do
 
   describe '#run' do
     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
-    let(:es) {
-      driver.run(default_tag: tag) { driver.feed(event_time, message) }
-      driver.filtered_records
-    }
 
     context 'simple config' do
-      include_context 'simple_config'
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
 
       it 'adds a new counter metric' do
-        expect(registry.metrics.map(&:name)).not_to include(name)
-        es
-        expect(registry.metrics.map(&:name)).to include(name)
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
       end
 
       it 'should keep original message' do
-        expect(es.first).to eq(message)
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(driver.filtered_records.first).to eq(message)
       end
     end
 

data/spec/fluent/plugin/in_prometheus_monitor_spec.rb

@@ -23,7 +23,6 @@ describe Fluent::Plugin::PrometheusMonitorInput do
 ]
 
   let(:config) { MONITOR_CONFIG }
-  let(:port) { 24231 }
   let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusMonitorInput).configure(config) }
 
   describe '#configure' do

data/spec/fluent/plugin/in_prometheus_spec.rb

@@ -0,0 +1,225 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus'
+require 'fluent/test/driver/input'
+
+require 'net/http'
+
+describe Fluent::Plugin::PrometheusInput do
+  CONFIG = %[
+  @type prometheus
+]
+
+  LOCAL_CONFIG = %[
+  @type prometheus
+  bind 127.0.0.1
+]
+
+  let(:config) { CONFIG }
+  let(:port) { 24231 }
+  let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusInput).configure(config) }
+
+  describe '#configure' do
+    describe 'bind' do
+      let(:config) { CONFIG + %[
+  bind 127.0.0.1
+] }
+      it 'should be configurable' do
+        expect(driver.instance.bind).to eq('127.0.0.1')
+      end
+    end
+
+    describe 'port' do
+      let(:config) { CONFIG + %[
+  port 8888
+] }
+      it 'should be configurable' do
+        expect(driver.instance.port).to eq(8888)
+      end
+    end
+
+    describe 'metrics_path' do
+      let(:config) { CONFIG + %[
+  metrics_path /_test
+] }
+      it 'should be configurable' do
+        expect(driver.instance.metrics_path).to eq('/_test')
+      end
+    end
+  end
+
+  describe '#start' do
+    context 'with transport section' do
+      let(:config) do
+        %[
+           @type prometheus
+           bind 127.0.0.1
+           <transport tls>
+             insecure true
+           </transport>
+         ]
+      end
+
+      it 'returns 200' do
+        driver.run(timeout: 1) do
+          Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
+            req = Net::HTTP::Get.new('/metrics')
+            res = http.request(req)
+            expect(res.code).to eq('200')
+          end
+        end
+      end
+    end
+
+    context 'old parameters are given' do
+      context 'when extra_conf is used' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              extra_conf { "SSLCertName": [["CN", "nobody"], ["DC", "example"]] }
+            </ssl>
+         ]
+        end
+
+        it 'uses webrick' do
+          expect(driver.instance).to receive(:start_webrick).once
+          driver.run(timeout: 1)
+        end
+
+        it 'returns 200' do
+          driver.run(timeout: 1) do
+            Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
+              req = Net::HTTP::Get.new('/metrics')
+              res = http.request(req)
+              expect(res.code).to eq('200')
+            end
+          end
+        end
+      end
+
+      context 'cert_path and private_key_path combination' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              certificate_path path
+              private_key_path path1
+            </ssl>
+          ]
+        end
+
+        it 'converts them into new transport section' do
+          expect(driver.instance).to receive(:http_server_create_http_server).with(
+                                       :in_prometheus_server,
+                                       addr: anything,
+                                       logger: anything,
+                                       port: anything,
+                                       proto: :tls,
+                                       tls_opts: { 'cert_path' => 'path', 'private_key_path' => 'path1' }
+                                     ).once
+
+          driver.run(timeout: 1)
+        end
+      end
+
+      context 'insecure and ca_path' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              ca_path path
+            </ssl>
+           ]
+        end
+
+        it 'converts them into new transport section' do
+          expect(driver.instance).to receive(:http_server_create_http_server).with(
+                                       :in_prometheus_server,
+                                       addr: anything,
+                                       logger: anything,
+                                       port: anything,
+                                       proto: :tls,
+                                       tls_opts: { 'ca_path' => 'path', 'insecure' => true }
+                                     ).once
+
+          driver.run(timeout: 1)
+        end
+      end
+
+      context 'when only private_key_path is geven' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              private_key_path path
+            </ssl>
+           ]
+        end
+
+        it 'raises ConfigError' do
+          expect { driver.run(timeout: 1) }.to raise_error(Fluent::ConfigError, 'both certificate_path and private_key_path must be defined')
+        end
+      end
+    end
+  end
+
+  describe '#run' do
+    context '/metrics' do
+      let(:config) { LOCAL_CONFIG }
+      it 'returns 200' do
+        driver.run(timeout: 1) do
+          Net::HTTP.start("127.0.0.1", port) do |http|
+            req = Net::HTTP::Get.new("/metrics")
+            res = http.request(req)
+            expect(res.code).to eq('200')
+          end
+        end
+      end
+    end
+
+    context '/foo' do
+      let(:config) { LOCAL_CONFIG }
+      it 'does not return 200' do
+        driver.run(timeout: 1) do
+          Net::HTTP.start("127.0.0.1", port) do |http|
+            req = Net::HTTP::Get.new("/foo")
+            res = http.request(req)
+            expect(res.code).not_to eq('200')
+          end
+        end
+      end
+    end
+  end
+
+  describe '#run_multi_workers' do
+    context '/metrics' do
+      Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
+        let(:config) { CONFIG + %[
+          port #{port - 2}
+        ] }
+
+        it 'should configure port using sequential number' do
+          driver = Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusInput)
+          driver.instance.instance_eval{ @_fluentd_worker_id = 2 }
+          driver.configure(config)
+          expect(driver.instance.port).to eq(port)
+          driver.run(timeout: 1) do
+            Net::HTTP.start("127.0.0.1", port) do |http|
+              req = Net::HTTP::Get.new("/metrics")
+              res = http.request(req)
+              expect(res.code).to eq('200')
+            end
+          end
+        end
+      end
+    end
+  end
+end