fluent-plugin-prometheus 1.6.0 → 1.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f079822fa7231b77912e5ac1324b691f38dbf1f1f32fd8700dee19bc636fd8c
-  data.tar.gz: 58ad623a333cc1600e971fdd35b9b2c7d4a381c03a954d99206233758e4562f6
+  metadata.gz: 1f14f2b2162bace2a18a80f885df28261ed260a7b8e865a12e7828d46af9b02f
+  data.tar.gz: 20749cd6ee743ac3726719e654e1a4f046aa2f953245604e6aa75cd87ed00e62
 SHA512:
-  metadata.gz: 4394983e398e9e4b116f54927b7ad82778ab672b27e41bfc1ef3eb5421f3f3cde7899ce906d86de8862179c0293c1831c0a555553f21c10787dddba22c68fe1e
-  data.tar.gz: f8d52d61a493f4c8abfc88a1cfdc415779a4946eeed639b03545574b2a86304d7a560c026391897ab93596280be306a02c24d3dcdbc161c30f625c211d237711
+  metadata.gz: 28ca37140a4b634665a9ba184c1615380bc7c458ed745d5548cb75b92d75d3b1069cd342d3119785a11d1beb369e7b41b5fa0319021bb0f69400ffe327dea83e
+  data.tar.gz: b04770fec96856fc37b6943c7129c5e8ef16ce8b1415b425932e4419594e58a3255406de54e3aad1c28af1e3d62074391bd2b40e4aded8b5f598fbece53cab82

data/README.md

@@ -369,7 +369,13 @@ Reserved placeholders are:
 - `${worker_id}`: fluent worker id
 - `${tag}`: tag name
   - only available in Prometheus output/filter plugin
-
+- `${tag_parts[N]}` refers to the Nth part of the tag.
+  - only available in Prometheus output/filter plugin
+- `${tag_prefix[N]}` refers to the [0..N] part of the tag.
+  - only available in Prometheus output/filter plugin
+- `${tag_suffix[N]}` refers to the [`tagsize`-1-N..] part of the tag.
+  - where `tagsize` is the size of tag which is splitted with `.` (when tag is `1.2.3`, then `tagsize` is 3)
+  - only available in Prometheus output/filter plugin
 
 ### top-level labels and labels inside metric
 

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.6.0"
+  spec.version       = "1.7.3"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "fluentd", ">= 0.14.20", "< 2"
-  spec.add_dependency "prometheus-client"
+  spec.add_dependency "prometheus-client", "< 0.10"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/lib/fluent/plugin/filter_prometheus.rb

@@ -22,9 +22,9 @@ module Fluent::Plugin
       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
     end
 
-    def filter_stream(tag, es)
-      instrument(tag, es, @metrics)
-      es
+    def filter(tag, time, record)
+      instrument_single(tag, time, record, @metrics)
+      record
     end
   end
 end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -24,14 +24,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
       @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -43,14 +43,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
       @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -28,14 +28,14 @@ module Fluent::Plugin
     def configure(conf)
       super
       hostname = Socket.gethostname
-      expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
-      placeholders = expander.prepare_placeholders({'hostname' => hostname, 'worker_id' => fluentd_worker_id})
+      expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
+      expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
       @base_labels = parse_labels_elements(conf)
       @base_labels.each do |key, value|
         unless value.is_a?(String)
           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"
         end
-        @base_labels[key] = expander.expand(value, placeholders)
+        @base_labels[key] = expander.expand(value)
       end
 
       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)

data/lib/fluent/plugin/prometheus.rb

@@ -1,6 +1,6 @@
 require 'prometheus/client'
 require 'prometheus/client/formats/text'
-require 'fluent/plugin/filter_record_transformer'
+require 'fluent/plugin/prometheus/placeholder_expander'
 
 module Fluent
   module Plugin
@@ -82,16 +82,35 @@ module Fluent
       end
 
       def self.placeholder_expander(log)
-        # Use internal class in order to expand placeholder
-        Fluent::Plugin::RecordTransformerFilter::PlaceholderExpander.new(log: log)
+        Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
       end
 
       def configure(conf)
         super
-        @placeholder_expander = Fluent::Plugin::Prometheus.placeholder_expander(log)
+        @placeholder_values = {}
+        @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
         @hostname = Socket.gethostname
       end
 
+      def instrument_single(tag, time, record, metrics)
+        @placeholder_values[tag] ||= {
+          'tag' => tag,
+          'hostname' => @hostname,
+          'worker_id' => fluentd_worker_id,
+        }
+
+        placeholders = record.merge(@placeholder_values[tag])
+        expander = @placeholder_expander_builder.build(placeholders)
+        metrics.each do |metric|
+          begin
+            metric.instrument(record, expander)
+          rescue => e
+            log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
+            router.emit_error_event(tag, time, record, e)
+          end
+        end
+      end
+
       def instrument(tag, es, metrics)
         placeholder_values = {
           'tag' => tag,
@@ -101,10 +120,10 @@ module Fluent
 
         es.each do |time, record|
           placeholders = record.merge(placeholder_values)
-          placeholders = @placeholder_expander.prepare_placeholders(placeholders)
+          expander = @placeholder_expander_builder.build(placeholders)
           metrics.each do |metric|
             begin
-              metric.instrument(record, @placeholder_expander, placeholders)
+              metric.instrument(record, expander)
             rescue => e
               log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
               router.emit_error_event(tag, time, record, e)
@@ -134,11 +153,11 @@ module Fluent
           @base_labels = labels.merge(@base_labels)
         end
 
-        def labels(record, expander, placeholders)
+        def labels(record, expander)
           label = {}
           @base_labels.each do |k, v|
             if v.is_a?(String)
-              label[k] = expander.expand(v, placeholders)
+              label[k] = expander.expand(v)
             else
               label[k] = v.call(record)
             end
@@ -175,14 +194,14 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @gauge.set(labels(record, expander, placeholders), value)
+            @gauge.set(labels(record, expander), value)
           end
         end
       end
@@ -197,7 +216,7 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           # use record value of the key if key is specified, otherwise just increment
           if @key.nil?
             value = 1
@@ -210,7 +229,7 @@ module Fluent
           # ignore if record value is nil
           return if value.nil?
 
-          @counter.increment(labels(record, expander, placeholders), value)
+          @counter.increment(labels(record, expander), value)
         end
       end
 
@@ -228,14 +247,14 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @summary.observe(labels(record, expander, placeholders), value)
+            @summary.observe(labels(record, expander), value)
           end
         end
       end
@@ -261,14 +280,14 @@ module Fluent
           end
         end
 
-        def instrument(record, expander, placeholders)
+        def instrument(record, expander)
           if @key.is_a?(String)
             value = record[@key]
           else
             value = @key.call(record)
           end
           if value
-            @histogram.observe(labels(record, expander, placeholders), value)
+            @histogram.observe(labels(record, expander), value)
           end
         end
       end

data/lib/fluent/plugin/prometheus/placeholder_expander.rb

@@ -0,0 +1,132 @@
+module Fluent
+  module Plugin
+    module Prometheus
+      class ExpandBuilder
+        def self.build(placeholder, log:)
+          new(log: log).build(placeholder)
+        end
+
+        def initialize(log:)
+          @log = log
+        end
+
+        def build(placeholder_values)
+          placeholders = {}
+          placeholder_values.each do |key, value|
+            case value
+            when Array
+              size = value.size
+              value.each_with_index do |v, i|
+                placeholders["${#{key}[#{i}]}"] = v
+                placeholders["${#{key}[#{i - size}]}"] = v
+              end
+            when Hash
+              value.each do |k, v|
+                placeholders[%(${#{key}["#{k}"]})] = v
+              end
+            else
+              if key == 'tag'
+                placeholders.merge!(build_tag(value))
+              else
+                placeholders["${#{key}}"] = value
+              end
+            end
+          end
+
+          Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander.new(@log, placeholders)
+        end
+
+        private
+
+        def build_tag(tag)
+          tags = tag.split('.')
+
+          placeholders = { '${tag}' => tag }
+
+          size = tags.size
+
+          tags.each_with_index do |v, i|
+            placeholders["${tag_parts[#{i}]}"] = v
+            placeholders["${tag_parts[#{i - size}]}"] = v
+          end
+
+          tag_prefix(tags).each_with_index do |v, i|
+            placeholders["${tag_prefix[#{i}]}"] = v
+          end
+
+          tag_suffix(tags).each_with_index do |v, i|
+            placeholders["${tag_suffix[#{i}]}"] = v
+          end
+
+          placeholders
+        end
+
+        def tag_prefix(tags)
+          tags = tags.dup
+          return [] if tags.empty?
+
+          ret = [tags.shift]
+          tags.each.with_index(1) do |tag, i|
+            ret[i] = "#{ret[i-1]}.#{tag}"
+          end
+          ret
+        end
+
+        def tag_suffix(tags)
+          return [] if tags.empty?
+
+          tags = tags.dup.reverse
+          ret = [tags.shift]
+          tags.each.with_index(1) do |tag, i|
+            ret[i] = "#{tag}.#{ret[i-1]}"
+          end
+          ret
+        end
+
+        class PlaceholderExpander
+          PLACEHOLDER_REGEX = /(\${[^\[}]+(\[[^\]]+\])?})/.freeze
+
+          attr_reader :placeholder
+
+          def initialize(log, placeholder)
+            @placeholder = placeholder
+            @log = log
+            @expander_cache = {}
+          end
+
+          def merge_placeholder(placeholder)
+            @placeholder.merge!(placeholder)
+          end
+
+          def expand(str, dynamic_placeholders: nil)
+            expander = if dynamic_placeholders
+                         if @expander_cache[dynamic_placeholders]
+                           @expander_cache[dynamic_placeholders]
+                         else
+                           e = ExpandBuilder.build(dynamic_placeholders, log: @log)
+                           e.merge_placeholder(@placeholder)
+                           @expander_cache[dynamic_placeholders] = e
+                           e
+                         end
+                       else
+                         self
+                       end
+
+            expander.expand!(str)
+          end
+
+          protected
+
+          def expand!(str)
+            str.gsub(PLACEHOLDER_REGEX) { |value|
+              @placeholder.fetch(value) do
+                @log.warn("unknown placeholder `#{value}` found")
+                value # return as it is
+              end
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/spec/fluent/plugin/filter_prometheus_spec.rb

@@ -6,7 +6,11 @@ require_relative 'shared'
 describe Fluent::Plugin::PrometheusFilter do
   let(:tag) { 'prometheus.test' }
   let(:driver) { Fluent::Test::Driver::Filter.new(Fluent::Plugin::PrometheusFilter).configure(config) }
-  let(:registry) { ::Prometheus::Client.registry }
+  let(:registry) { ::Prometheus::Client::Registry.new }
+
+  before do
+    allow(Prometheus::Client).to receive(:registry).and_return(registry)
+  end
 
   describe '#configure' do
     it_behaves_like 'output configuration'
@@ -14,22 +18,28 @@ describe Fluent::Plugin::PrometheusFilter do
 
   describe '#run' do
     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
-    let(:es) {
-      driver.run(default_tag: tag) { driver.feed(event_time, message) }
-      driver.filtered_records
-    }
 
     context 'simple config' do
-      include_context 'simple_config'
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
 
       it 'adds a new counter metric' do
-        expect(registry.metrics.map(&:name)).not_to include(name)
-        es
-        expect(registry.metrics.map(&:name)).to include(name)
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
       end
 
       it 'should keep original message' do
-        expect(es.first).to eq(message)
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(driver.filtered_records.first).to eq(message)
       end
     end
 

data/spec/fluent/plugin/in_prometheus_monitor_spec.rb

@@ -23,7 +23,6 @@ describe Fluent::Plugin::PrometheusMonitorInput do
 ]
 
   let(:config) { MONITOR_CONFIG }
-  let(:port) { 24231 }
   let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusMonitorInput).configure(config) }
 
   describe '#configure' do

data/spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus_tail_monitor'
+require 'fluent/test/driver/input'
+
+describe Fluent::Plugin::PrometheusTailMonitorInput do
+  MONITOR_CONFIG = %[
+  @type prometheus_tail_monitor
+  <labels>
+    host ${hostname}
+    foo bar
+  </labels>
+]
+
+  INVALID_MONITOR_CONFIG = %[
+  @type prometheus_tail_monitor
+
+  <labels>
+    host ${hostname}
+    foo bar
+    invalid_use1 $.foo.bar
+    invalid_use2 $[0][1]
+  </labels>
+]
+
+  let(:config) { MONITOR_CONFIG }
+  let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusTailMonitorInput).configure(config) }
+
+  describe '#configure' do
+    describe 'valid' do
+      it 'does not raise error' do
+        expect { driver }.not_to raise_error
+      end
+    end
+
+    describe 'invalid' do
+      let(:config) { INVALID_MONITOR_CONFIG }
+      it 'expect raise error' do
+        expect { driver }.to raise_error
+      end
+    end
+  end
+end

data/spec/fluent/plugin/out_prometheus_spec.rb

@@ -6,7 +6,11 @@ require_relative 'shared'
 describe Fluent::Plugin::PrometheusOutput do
   let(:tag) { 'prometheus.test' }
   let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::PrometheusOutput).configure(config) }
-  let(:registry) { ::Prometheus::Client.registry }
+  let(:registry) { ::Prometheus::Client::Registry.new }
+
+  before do
+    allow(Prometheus::Client).to receive(:registry).and_return(registry)
+  end
 
   describe '#configure' do
     it_behaves_like 'output configuration'
@@ -14,18 +18,23 @@ describe Fluent::Plugin::PrometheusOutput do
 
   describe '#run' do
     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
-    let(:es) {
-      driver.run(default_tag: tag) { driver.feed(event_time, message) }
-      driver.events
-    }
 
     context 'simple config' do
-      include_context 'simple_config'
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
 
       it 'adds a new counter metric' do
-        expect(registry.metrics.map(&:name)).not_to include(name)
-        es
-        expect(registry.metrics.map(&:name)).to include(name)
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
       end
     end
 

data/spec/fluent/plugin/prometheus/placeholder_expander_spec.rb

@@ -0,0 +1,110 @@
+require 'logger'
+
+require 'spec_helper'
+require 'fluent/plugin/prometheus/placeholder_expander'
+require_relative '../shared'
+
+describe Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander do
+  let(:log) do
+    Logger.new('/dev/null')
+  end
+
+  let(:builder) do
+    Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
+  end
+
+  describe '#expand' do
+    context 'with static placeholder' do
+      let(:static_placeholder) do
+        {
+          'hostname' => 'host_value',
+          'tag' => '1.2.3',
+          'ary_value' => ['1', '2', '3'],
+          'hash_value' => { 'key1' => 'val1' },
+        }
+      end
+
+      let(:dynamic_placeholder) do
+      end
+
+      it 'expands values' do
+        expander = builder.build(static_placeholder)
+        expect(expander.expand('${hostname}')).to eq('host_value')
+        expect(expander.expand('${ary_value[0]}.${ary_value[1]}.${ary_value[2]}')).to eq('1.2.3')
+        expect(expander.expand('${ary_value[-3]}.${ary_value[-2]}.${ary_value[-1]}')).to eq('1.2.3')
+        expect(expander.expand('${hash_value["key1"]}')).to eq('val1')
+
+        expect(expander.expand('${tag}')).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}')).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[-3]}.${tag_parts[-2]}.${tag_parts[-1]}')).to eq('1.2.3')
+        expect(expander.expand('${tag_prefix[0]}.${tag_prefix[1]}.${tag_prefix[2]}')).to eq('1.1.2.1.2.3')
+        expect(expander.expand('${tag_suffix[0]}.${tag_suffix[1]}.${tag_suffix[2]}')).to eq('3.2.3.1.2.3')
+      end
+
+      it 'does not create new expander' do
+        builder # cached before mock
+
+        expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).never
+        expander = builder.build(static_placeholder)
+        expander.expand('${hostname}')
+        expander.expand('${hostname}')
+      end
+
+      context 'when not found placeholder' do
+        it 'prints wanring log and as it is' do
+          expect(log).to receive(:warn).with('unknown placeholder `${tag_prefix[100]}` found').once
+
+          expander = builder.build(static_placeholder)
+          expect(expander.expand('${tag_prefix[100]}')).to eq('${tag_prefix[100]}')
+        end
+      end
+    end
+
+    context 'with dynamic placeholder' do
+      let(:static_placeholder) do
+        {
+          'hostname' => 'host_value',
+          'ary_value' => ['1', '2', '3'],
+          'hash_value' => { 'key1' => 'val1' },
+        }
+      end
+
+      let(:dynamic_placeholder) do
+        { 'tag' => '1.2.3'}
+      end
+
+      it 'expands values' do
+        expander = builder.build(static_placeholder)
+        expect(expander.expand('${hostname}', dynamic_placeholders: dynamic_placeholder)).to eq('host_value')
+        expect(expander.expand('${ary_value[0]}.${ary_value[1]}.${ary_value[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${ary_value[-3]}.${ary_value[-2]}.${ary_value[-1]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${hash_value["key1"]}', dynamic_placeholders: dynamic_placeholder)).to eq('val1')
+
+        expect(expander.expand('${tag}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[-3]}.${tag_parts[-2]}.${tag_parts[-1]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${tag_prefix[0]}.${tag_prefix[1]}.${tag_prefix[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.1.2.1.2.3')
+        expect(expander.expand('${tag_suffix[0]}.${tag_suffix[1]}.${tag_suffix[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('3.2.3.1.2.3')
+      end
+
+      it 'does not create expander twice if given the same placeholder' do
+        builder # cached before mock
+
+        expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).once.and_call_original
+        expander = builder.build(static_placeholder)
+        placeholder = { 'tag' => 'val.test' }
+        expander.expand('${hostname}', dynamic_placeholders: placeholder)
+        expander.expand('${hostname}', dynamic_placeholders: placeholder)
+      end
+
+      it 'creates new expander for each placeholder' do
+        builder # cached before mock
+
+        expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).twice.and_call_original
+        expander = builder.build(static_placeholder)
+        expander.expand('${hostname}', dynamic_placeholders: { 'tag' => 'val.test' })
+        expander.expand('${hostname}', dynamic_placeholders: { 'tag' => 'val.test2' })
+      end
+    end
+  end
+end

data/spec/fluent/plugin/shared.rb

@@ -3,9 +3,7 @@ BASE_CONFIG = %[
   type prometheus
 ]
 
-
 SIMPLE_CONFIG = BASE_CONFIG + %[
-  type prometheus
   <metric>
     name simple_foo
     type counter
@@ -111,109 +109,65 @@ COUNTER_WITHOUT_KEY_CONFIG = BASE_CONFIG + %[
   </metric>
 ]
 
-def gen_time_suffix
-  return Time.now.to_f.to_s.gsub('.', '')
-end
-
-shared_context 'simple_config' do
-  let(:orig_name) { 'simple_foo' }
-  let(:config) { SIMPLE_CONFIG.gsub(orig_name, name.to_s) }
-  let(:name) { "#{orig_name}_#{gen_time_suffix}".to_sym }
-  let(:counter) { registry.get(name) }
-end
-
-shared_context 'full_config' do
-  let(:config) { FULL_CONFIG }
-  let(:counter) { registry.get(:full_foo) }
-  let(:gauge) { registry.get(:full_bar) }
-  let(:summary) { registry.get(:full_baz) }
-  let(:histogram) { registry.get(:full_qux) }
-  let(:summary_with_accessor) { registry.get(:full_accessor1) }
-  let(:counter_with_accessor) { registry.get(:full_accessor2) }
-end
-
-shared_context 'placeholder_config' do
-  let(:orig_name) { 'placeholder_foo' }
-  let(:config) { PLACEHOLDER_CONFIG.gsub(orig_name, name.to_s) }
-  let(:name) { "#{orig_name}_#{gen_time_suffix}".to_sym }
-  let(:counter) { registry.get(name) }
-end
-
-shared_context 'accessor_config' do
-  let(:orig_name) { 'accessor_foo' }
-  let(:config) { ACCESSOR_CONFIG.gsub(orig_name, name.to_s) }
-  let(:name) { "#{orig_name}_#{gen_time_suffix}".to_sym }
-  let(:counter) { registry.get(name) }
-end
-
-shared_context 'counter_without_key_config' do
-  let(:orig_name) { 'without_key_foo' }
-  let(:config) { COUNTER_WITHOUT_KEY_CONFIG.gsub(orig_name, name.to_s) }
-  let(:name) { "#{orig_name}_#{gen_time_suffix}".to_sym }
-  let(:counter) { registry.get(name) }
-end
-
 shared_examples_for 'output configuration' do
   context 'base config' do
     let(:config) { BASE_CONFIG }
-    it 'does not raise error' do
-      expect{driver}.not_to raise_error
-    end
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure simple configuration' do
-    include_context 'simple_config'
-    it { expect{driver}.not_to raise_error }
+  context 'with simple configuration' do
+    let(:config) { SIMPLE_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure full configuration' do
-    include_context 'full_config'
-    it { expect{driver}.not_to raise_error }
+  context 'with full configuration' do
+    let(:config) { FULL_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure placeholder configuration' do
-    include_context 'placeholder_config'
-    it { expect{driver}.not_to raise_error }
+  context 'with placeholder configuration' do
+    let(:config) { PLACEHOLDER_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure accessor configuration' do
-    include_context 'accessor_config'
-    it { expect{driver}.not_to raise_error }
+  context 'with accessor configuration' do
+    let(:config) {  ACCESSOR_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure counter without key configuration' do
-    include_context 'counter_without_key_config'
-    it { expect{driver}.not_to raise_error }
+  describe 'with counter without key configuration' do
+    let(:config) { COUNTER_WITHOUT_KEY_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  context 'unknown type' do
-    let(:config) { BASE_CONFIG + %[
-<metric>
-  type foo
-</metric>
-] }
-    it 'raises ConfigError' do
-      expect{driver}.to raise_error Fluent::ConfigError
+  context 'with unknown type' do
+    let(:config) do
+      BASE_CONFIG + %[
+      <metric>
+        type foo
+      </metric>
+      ]
     end
+    it { expect { driver }.to raise_error(Fluent::ConfigError) }
   end
 end
 
-emit_count = 0
 shared_examples_for 'instruments record' do
-  context 'full config' do
-    include_context 'full_config'
+  before do
+    driver.run(default_tag: tag) { driver.feed(event_time, message) }
+  end
 
-    before :each do
-      es
-      emit_count += 1
-    end
+  context 'full config' do
+    let(:config) { FULL_CONFIG }
+    let(:counter) { registry.get(:full_foo) }
+    let(:gauge) { registry.get(:full_bar) }
+    let(:summary) { registry.get(:full_baz) }
+    let(:histogram) { registry.get(:full_qux) }
+    let(:summary_with_accessor) { registry.get(:full_accessor1) }
+    let(:counter_with_accessor) { registry.get(:full_accessor2) }
 
     it 'adds all metrics' do
-      expect(registry.metrics.map(&:name)).to include(:full_foo)
-      expect(registry.metrics.map(&:name)).to include(:full_bar)
-      expect(registry.metrics.map(&:name)).to include(:full_baz)
-      expect(registry.metrics.map(&:name)).to include(:full_accessor1)
-      expect(registry.metrics.map(&:name)).to include(:full_accessor2)
+      expect(registry.metrics.map(&:name)).to eq(%i[full_foo full_bar full_baz full_qux full_accessor1 full_accessor2])
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
       expect(summary).to be_kind_of(::Prometheus::Client::Metric)
@@ -241,21 +195,22 @@ shared_examples_for 'instruments record' do
     end
 
     it 'instruments histogram metric' do
+      driver.run(default_tag: tag) do
+        4.times { driver.feed(event_time, message) }
+      end
+
       expect(histogram.type).to eq(:histogram)
       expect(histogram.get({test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
-      expect(histogram.get({test_key: 'test_value', key: 'foo4'})[10]).to eq(emit_count)
+      expect(histogram.get({test_key: 'test_value', key: 'foo4'})[10]).to eq(5) # 4 + `es` in before
     end
   end
 
   context 'placeholder config' do
-    include_context 'placeholder_config'
-
-    before :each do
-      es
-    end
+    let(:config) { PLACEHOLDER_CONFIG }
+    let(:counter) { registry.get(:placeholder_foo) }
 
     it 'expands placeholders with record values' do
-      expect(registry.metrics.map(&:name)).to include(name)
+      expect(registry.metrics.map(&:name)).to eq([:placeholder_foo])
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       key, _ = counter.values.find {|k,v| v ==  100 }
       expect(key).to be_kind_of(Hash)
@@ -268,14 +223,11 @@ shared_examples_for 'instruments record' do
   end
 
   context 'accessor config' do
-    include_context 'accessor_config'
-
-    before :each do
-      es
-    end
+    let(:config) { ACCESSOR_CONFIG }
+    let(:counter) { registry.get(:accessor_foo) }
 
     it 'expands accessor with record values' do
-      expect(registry.metrics.map(&:name)).to include(name)
+      expect(registry.metrics.map(&:name)).to eq([:accessor_foo])
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       key, _ = counter.values.find {|k,v| v ==  100 }
       expect(key).to be_kind_of(Hash)
@@ -284,14 +236,11 @@ shared_examples_for 'instruments record' do
   end
 
   context 'counter_without config' do
-    include_context 'counter_without_key_config'
-
-    before :each do
-      es
-    end
+    let(:config) { COUNTER_WITHOUT_KEY_CONFIG }
+    let(:counter) { registry.get(:without_key_foo) }
 
     it 'just increments by 1' do
-      expect(registry.metrics.map(&:name)).to include(name)
+      expect(registry.metrics.map(&:name)).to eq([:without_key_foo])
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       _, value = counter.values.find {|k,v| k == {} }
       expect(value).to eq(1)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.7.3
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-15 00:00:00.000000000 Z
+date: 2020-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,7 @@ files:
 - lib/fluent/plugin/in_prometheus_tail_monitor.rb
 - lib/fluent/plugin/out_prometheus.rb
 - lib/fluent/plugin/prometheus.rb
+- lib/fluent/plugin/prometheus/placeholder_expander.rb
 - lib/fluent/plugin/prometheus_metrics.rb
 - misc/fluentd_sample.conf
 - misc/nginx_proxy.conf
@@ -129,9 +130,11 @@ files:
 - misc/prometheus_alerts.yaml
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
+- spec/fluent/plugin/in_prometheus_spec.rb
+- spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus/placeholder_expander_spec.rb
 - spec/fluent/plugin/prometheus_metrics_spec.rb
-- spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb
 homepage: https://github.com/fluent/fluent-plugin-prometheus
@@ -153,16 +156,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.
 test_files:
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
+- spec/fluent/plugin/in_prometheus_spec.rb
+- spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus/placeholder_expander_spec.rb
 - spec/fluent/plugin/prometheus_metrics_spec.rb
-- spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb