fluent-plugin-prometheus 1.7.0 → 2.0.2

data/spec/fluent/plugin/out_prometheus_spec.rb

@@ -6,7 +6,11 @@ require_relative 'shared'
 describe Fluent::Plugin::PrometheusOutput do
   let(:tag) { 'prometheus.test' }
   let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::PrometheusOutput).configure(config) }
-  let(:registry) { ::Prometheus::Client.registry }
+  let(:registry) { ::Prometheus::Client::Registry.new }
+
+  before do
+    allow(Prometheus::Client).to receive(:registry).and_return(registry)
+  end
 
   describe '#configure' do
     it_behaves_like 'output configuration'
@@ -14,18 +18,48 @@ describe Fluent::Plugin::PrometheusOutput do
 
   describe '#run' do
     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
-    let(:es) {
-      driver.run(default_tag: tag) { driver.feed(event_time, message) }
-      driver.events
-    }
 
     context 'simple config' do
-      include_context 'simple_config'
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
+
+      it 'adds a new counter metric' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
+      end
+    end
+
+    it_behaves_like 'instruments record'
+  end
+
+  describe '#run with symbolized keys' do
+    let(:message) { {:foo => 100, :bar => 100, :baz => 100, :qux => 10} }
+
+    context 'simple config' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
 
       it 'adds a new counter metric' do
-        expect(registry.metrics.map(&:name)).not_to include(name)
-        es
-        expect(registry.metrics.map(&:name)).to include(name)
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
       end
     end
 

data/spec/fluent/plugin/prometheus/placeholder_expander_spec.rb

@@ -0,0 +1,110 @@
+require 'logger'
+
+require 'spec_helper'
+require 'fluent/plugin/prometheus/placeholder_expander'
+require_relative '../shared'
+
+describe Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander do
+  let(:log) do
+    Logger.new('/dev/null')
+  end
+
+  let(:builder) do
+    Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
+  end
+
+  describe '#expand' do
+    context 'with static placeholder' do
+      let(:static_placeholder) do
+        {
+          'hostname' => 'host_value',
+          'tag' => '1.2.3',
+          'ary_value' => ['1', '2', '3'],
+          'hash_value' => { 'key1' => 'val1' },
+        }
+      end
+
+      let(:dynamic_placeholder) do
+      end
+
+      it 'expands values' do
+        expander = builder.build(static_placeholder)
+        expect(expander.expand('${hostname}')).to eq('host_value')
+        expect(expander.expand('${ary_value[0]}.${ary_value[1]}.${ary_value[2]}')).to eq('1.2.3')
+        expect(expander.expand('${ary_value[-3]}.${ary_value[-2]}.${ary_value[-1]}')).to eq('1.2.3')
+        expect(expander.expand('${hash_value["key1"]}')).to eq('val1')
+
+        expect(expander.expand('${tag}')).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}')).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[-3]}.${tag_parts[-2]}.${tag_parts[-1]}')).to eq('1.2.3')
+        expect(expander.expand('${tag_prefix[0]}.${tag_prefix[1]}.${tag_prefix[2]}')).to eq('1.1.2.1.2.3')
+        expect(expander.expand('${tag_suffix[0]}.${tag_suffix[1]}.${tag_suffix[2]}')).to eq('3.2.3.1.2.3')
+      end
+
+      it 'does not create new expander' do
+        builder # cached before mock
+
+        expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).never
+        expander = builder.build(static_placeholder)
+        expander.expand('${hostname}')
+        expander.expand('${hostname}')
+      end
+
+      context 'when not found placeholder' do
+        it 'prints wanring log and as it is' do
+          expect(log).to receive(:warn).with('unknown placeholder `${tag_prefix[100]}` found').once
+
+          expander = builder.build(static_placeholder)
+          expect(expander.expand('${tag_prefix[100]}')).to eq('${tag_prefix[100]}')
+        end
+      end
+    end
+
+    context 'with dynamic placeholder' do
+      let(:static_placeholder) do
+        {
+          'hostname' => 'host_value',
+          'ary_value' => ['1', '2', '3'],
+          'hash_value' => { 'key1' => 'val1' },
+        }
+      end
+
+      let(:dynamic_placeholder) do
+        { 'tag' => '1.2.3'}
+      end
+
+      it 'expands values' do
+        expander = builder.build(static_placeholder)
+        expect(expander.expand('${hostname}', dynamic_placeholders: dynamic_placeholder)).to eq('host_value')
+        expect(expander.expand('${ary_value[0]}.${ary_value[1]}.${ary_value[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${ary_value[-3]}.${ary_value[-2]}.${ary_value[-1]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${hash_value["key1"]}', dynamic_placeholders: dynamic_placeholder)).to eq('val1')
+
+        expect(expander.expand('${tag}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[-3]}.${tag_parts[-2]}.${tag_parts[-1]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${tag_prefix[0]}.${tag_prefix[1]}.${tag_prefix[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.1.2.1.2.3')
+        expect(expander.expand('${tag_suffix[0]}.${tag_suffix[1]}.${tag_suffix[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('3.2.3.1.2.3')
+      end
+
+      it 'does not create expander twice if given the same placeholder' do
+        builder # cached before mock
+
+        expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).once.and_call_original
+        expander = builder.build(static_placeholder)
+        placeholder = { 'tag' => 'val.test' }
+        expander.expand('${hostname}', dynamic_placeholders: placeholder)
+        expander.expand('${hostname}', dynamic_placeholders: placeholder)
+      end
+
+      it 'creates new expander for each placeholder' do
+        builder # cached before mock
+
+        expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).twice.and_call_original
+        expander = builder.build(static_placeholder)
+        expander.expand('${hostname}', dynamic_placeholders: { 'tag' => 'val.test' })
+        expander.expand('${hostname}', dynamic_placeholders: { 'tag' => 'val.test2' })
+      end
+    end
+  end
+end

data/spec/fluent/plugin/shared.rb

@@ -1,11 +1,9 @@
 
 BASE_CONFIG = %[
-  type prometheus
+  @type prometheus
 ]
 
-
 SIMPLE_CONFIG = BASE_CONFIG + %[
-  type prometheus
   <metric>
     name simple_foo
     type counter
@@ -111,109 +109,65 @@ COUNTER_WITHOUT_KEY_CONFIG = BASE_CONFIG + %[
   </metric>
 ]
 
-def gen_time_suffix
-  return Time.now.to_f.to_s.gsub('.', '')
-end
-
-shared_context 'simple_config' do
-  let(:orig_name) { 'simple_foo' }
-  let(:config) { SIMPLE_CONFIG.gsub(orig_name, name.to_s) }
-  let(:name) { "#{orig_name}_#{gen_time_suffix}".to_sym }
-  let(:counter) { registry.get(name) }
-end
-
-shared_context 'full_config' do
-  let(:config) { FULL_CONFIG }
-  let(:counter) { registry.get(:full_foo) }
-  let(:gauge) { registry.get(:full_bar) }
-  let(:summary) { registry.get(:full_baz) }
-  let(:histogram) { registry.get(:full_qux) }
-  let(:summary_with_accessor) { registry.get(:full_accessor1) }
-  let(:counter_with_accessor) { registry.get(:full_accessor2) }
-end
-
-shared_context 'placeholder_config' do
-  let(:orig_name) { 'placeholder_foo' }
-  let(:config) { PLACEHOLDER_CONFIG.gsub(orig_name, name.to_s) }
-  let(:name) { "#{orig_name}_#{gen_time_suffix}".to_sym }
-  let(:counter) { registry.get(name) }
-end
-
-shared_context 'accessor_config' do
-  let(:orig_name) { 'accessor_foo' }
-  let(:config) { ACCESSOR_CONFIG.gsub(orig_name, name.to_s) }
-  let(:name) { "#{orig_name}_#{gen_time_suffix}".to_sym }
-  let(:counter) { registry.get(name) }
-end
-
-shared_context 'counter_without_key_config' do
-  let(:orig_name) { 'without_key_foo' }
-  let(:config) { COUNTER_WITHOUT_KEY_CONFIG.gsub(orig_name, name.to_s) }
-  let(:name) { "#{orig_name}_#{gen_time_suffix}".to_sym }
-  let(:counter) { registry.get(name) }
-end
-
 shared_examples_for 'output configuration' do
   context 'base config' do
     let(:config) { BASE_CONFIG }
-    it 'does not raise error' do
-      expect{driver}.not_to raise_error
-    end
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure simple configuration' do
-    include_context 'simple_config'
-    it { expect{driver}.not_to raise_error }
+  context 'with simple configuration' do
+    let(:config) { SIMPLE_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure full configuration' do
-    include_context 'full_config'
-    it { expect{driver}.not_to raise_error }
+  context 'with full configuration' do
+    let(:config) { FULL_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure placeholder configuration' do
-    include_context 'placeholder_config'
-    it { expect{driver}.not_to raise_error }
+  context 'with placeholder configuration' do
+    let(:config) { PLACEHOLDER_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure accessor configuration' do
-    include_context 'accessor_config'
-    it { expect{driver}.not_to raise_error }
+  context 'with accessor configuration' do
+    let(:config) {  ACCESSOR_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  describe 'configure counter without key configuration' do
-    include_context 'counter_without_key_config'
-    it { expect{driver}.not_to raise_error }
+  describe 'with counter without key configuration' do
+    let(:config) { COUNTER_WITHOUT_KEY_CONFIG }
+    it { expect { driver }.not_to raise_error }
   end
 
-  context 'unknown type' do
-    let(:config) { BASE_CONFIG + %[
-<metric>
-  type foo
-</metric>
-] }
-    it 'raises ConfigError' do
-      expect{driver}.to raise_error Fluent::ConfigError
+  context 'with unknown type' do
+    let(:config) do
+      BASE_CONFIG + %[
+      <metric>
+        type foo
+      </metric>
+      ]
     end
+    it { expect { driver }.to raise_error(Fluent::ConfigError) }
   end
 end
 
-emit_count = 0
 shared_examples_for 'instruments record' do
-  context 'full config' do
-    include_context 'full_config'
+  before do
+    driver.run(default_tag: tag) { driver.feed(event_time, message) }
+  end
 
-    before :each do
-      es
-      emit_count += 1
-    end
+  context 'full config' do
+    let(:config) { FULL_CONFIG }
+    let(:counter) { registry.get(:full_foo) }
+    let(:gauge) { registry.get(:full_bar) }
+    let(:summary) { registry.get(:full_baz) }
+    let(:histogram) { registry.get(:full_qux) }
+    let(:summary_with_accessor) { registry.get(:full_accessor1) }
+    let(:counter_with_accessor) { registry.get(:full_accessor2) }
 
     it 'adds all metrics' do
-      expect(registry.metrics.map(&:name)).to include(:full_foo)
-      expect(registry.metrics.map(&:name)).to include(:full_bar)
-      expect(registry.metrics.map(&:name)).to include(:full_baz)
-      expect(registry.metrics.map(&:name)).to include(:full_accessor1)
-      expect(registry.metrics.map(&:name)).to include(:full_accessor2)
+      expect(registry.metrics.map(&:name)).to eq(%i[full_foo full_bar full_baz full_qux full_accessor1 full_accessor2])
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
       expect(summary).to be_kind_of(::Prometheus::Client::Metric)
@@ -224,38 +178,38 @@ shared_examples_for 'instruments record' do
 
     it 'instruments counter metric' do
       expect(counter.type).to eq(:counter)
-      expect(counter.get({test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
-      expect(counter_with_accessor.get({test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
+      expect(counter.get(labels: {test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
+      expect(counter_with_accessor.get(labels: {test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
     end
 
     it 'instruments gauge metric' do
       expect(gauge.type).to eq(:gauge)
-      expect(gauge.get({test_key: 'test_value', key: 'foo2'})).to eq(100)
+      expect(gauge.get(labels: {test_key: 'test_value', key: 'foo2'})).to eq(100)
     end
 
     it 'instruments summary metric' do
       expect(summary.type).to eq(:summary)
-      expect(summary.get({test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
-      expect(summary.get({test_key: 'test_value', key: 'foo3'})[0.99]).to eq(100)
-      expect(summary_with_accessor.get({test_key: 'test_value', key: 'foo5'})[0.99]).to eq(100)
+      expect(summary.get(labels: {test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
+      expect(summary_with_accessor.get(labels: {test_key: 'test_value', key: 'foo5'})["sum"]).to eq(100)
     end
 
     it 'instruments histogram metric' do
+      driver.run(default_tag: tag) do
+        4.times { driver.feed(event_time, message) }
+      end
+
       expect(histogram.type).to eq(:histogram)
-      expect(histogram.get({test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
-      expect(histogram.get({test_key: 'test_value', key: 'foo4'})[10]).to eq(emit_count)
+      expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
+      expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})["10"]).to eq(5) # 4 + `es` in before
     end
   end
 
   context 'placeholder config' do
-    include_context 'placeholder_config'
-
-    before :each do
-      es
-    end
+    let(:config) { PLACEHOLDER_CONFIG }
+    let(:counter) { registry.get(:placeholder_foo) }
 
     it 'expands placeholders with record values' do
-      expect(registry.metrics.map(&:name)).to include(name)
+      expect(registry.metrics.map(&:name)).to eq([:placeholder_foo])
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       key, _ = counter.values.find {|k,v| v ==  100 }
       expect(key).to be_kind_of(Hash)
@@ -268,30 +222,24 @@ shared_examples_for 'instruments record' do
   end
 
   context 'accessor config' do
-    include_context 'accessor_config'
-
-    before :each do
-      es
-    end
+    let(:config) { ACCESSOR_CONFIG }
+    let(:counter) { registry.get(:accessor_foo) }
 
     it 'expands accessor with record values' do
-      expect(registry.metrics.map(&:name)).to include(name)
+      expect(registry.metrics.map(&:name)).to eq([:accessor_foo])
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       key, _ = counter.values.find {|k,v| v ==  100 }
       expect(key).to be_kind_of(Hash)
-      expect(key[:foo]).to eq(100)
+      expect(key[:foo]).to eq("100")
     end
   end
 
   context 'counter_without config' do
-    include_context 'counter_without_key_config'
-
-    before :each do
-      es
-    end
+    let(:config) { COUNTER_WITHOUT_KEY_CONFIG }
+    let(:counter) { registry.get(:without_key_foo) }
 
     it 'just increments by 1' do
-      expect(registry.metrics.map(&:name)).to include(name)
+      expect(registry.metrics.map(&:name)).to eq([:without_key_foo])
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       _, value = counter.values.find {|k,v| k == {} }
       expect(value).to eq(1)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 2.0.2
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-31 00:00:00.000000000 Z
+date: 2021-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.20
+        version: 1.9.1
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.20
+        version: 1.9.1
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -34,16 +34,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -107,9 +107,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- ChangeLog
 - Gemfile
 - LICENSE
 - README.md
@@ -117,11 +119,13 @@ files:
 - fluent-plugin-prometheus.gemspec
 - lib/fluent/plugin/filter_prometheus.rb
 - lib/fluent/plugin/in_prometheus.rb
+- lib/fluent/plugin/in_prometheus/async_wrapper.rb
 - lib/fluent/plugin/in_prometheus_monitor.rb
 - lib/fluent/plugin/in_prometheus_output_monitor.rb
 - lib/fluent/plugin/in_prometheus_tail_monitor.rb
 - lib/fluent/plugin/out_prometheus.rb
 - lib/fluent/plugin/prometheus.rb
+- lib/fluent/plugin/prometheus/placeholder_expander.rb
 - lib/fluent/plugin/prometheus_metrics.rb
 - misc/fluentd_sample.conf
 - misc/nginx_proxy.conf
@@ -129,9 +133,11 @@ files:
 - misc/prometheus_alerts.yaml
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
+- spec/fluent/plugin/in_prometheus_spec.rb
+- spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus/placeholder_expander_spec.rb
 - spec/fluent/plugin/prometheus_metrics_spec.rb
-- spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb
 homepage: https://github.com/fluent/fluent-plugin-prometheus
@@ -153,15 +159,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.
 test_files:
 - spec/fluent/plugin/filter_prometheus_spec.rb
 - spec/fluent/plugin/in_prometheus_monitor_spec.rb
+- spec/fluent/plugin/in_prometheus_spec.rb
+- spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb
 - spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus/placeholder_expander_spec.rb
 - spec/fluent/plugin/prometheus_metrics_spec.rb
-- spec/fluent/plugin/prometheus_spec.rb
 - spec/fluent/plugin/shared.rb
 - spec/spec_helper.rb