fluent-plugin-prometheus 2.2.2 → 2.3.0

data/spec/fluent/plugin/in_prometheus_spec.rb

@@ -22,9 +22,16 @@ describe Fluent::Plugin::PrometheusInput do
   describe '#configure' do
     describe 'bind' do
       let(:config) { CONFIG + %[
-  bind 127.0.0.1
+  bind ::1
 ] }
       it 'should be configurable' do
+        expect(driver.instance.bind).to eq('::1')
+      end
+    end
+
+    describe 'default bind' do
+      let(:config) { CONFIG }
+      it 'should be accessible from 127.0.0.1 by default' do
         expect(driver.instance.bind).to eq('127.0.0.1')
       end
     end
@@ -64,6 +71,22 @@ describe Fluent::Plugin::PrometheusInput do
         expect(driver.instance.content_encoding).to eq(:gzip)
       end
     end
+
+    describe 'default ignore_error_log_interval' do
+      let(:config) { CONFIG }
+      it 'should be 3600 seconds by default' do
+        expect(driver.instance.ignore_error_log_interval).to eq(3600)
+      end
+    end
+
+    describe 'error_log_interval' do
+      let(:config) { CONFIG + %[
+    ignore_error_log_interval 60
+] }
+      it 'should be configurable' do
+        expect(driver.instance.ignore_error_log_interval).to eq(60)
+      end
+    end
   end
 
   describe '#start' do
@@ -331,4 +354,184 @@ describe Fluent::Plugin::PrometheusInput do
       include_examples 'IPv6 server binding', '[::1]', '::1', 'handles pre-bracketed address correctly'
     end
   end
+
+  describe 'error handling (information disclosure)' do
+    let(:config) { LOCAL_CONFIG }
+    let(:secret_message) { 'dummy secret detail: password=deadbeef' }
+
+    shared_examples 'suppressed exception response' do
+      it 'returns 500 with text/plain' do
+        status, header, _body = subject
+        expect(status).to eq(500)
+        expect(header['Content-Type']).to eq('text/plain')
+      end
+
+      it 'exposes the exception class only' do
+        _status, _header, body = subject
+        expect(body).to eq('in_prometheus server error: <RuntimeError>')
+        expect(body).not_to include(secret_message)
+      end
+
+      it 'logs the detail on the server side' do
+        subject
+        expect(driver.logs.any? { |log| log.include?(log_message) }).to be true
+        expect(driver.logs.any? { |log| log.include?(secret_message) }).to be true
+      end
+    end
+
+    context '#all_metrics' do
+      subject { driver.instance.send(:all_metrics) }
+
+      let(:log_message) { 'in_prometheus: failed to render metrics' }
+
+      before do
+        allow(::Prometheus::Client::Formats::Text).to receive(:marshal).and_raise(RuntimeError, secret_message)
+      end
+
+      include_examples 'suppressed exception response'
+    end
+
+    context '#all_workers_metrics' do
+      subject { driver.instance.send(:all_workers_metrics) }
+
+      let(:log_message) { 'in_prometheus: failed to render workers metrics' }
+
+      before do
+        allow(driver.instance).to receive(:send_request_to_each_worker).and_raise(RuntimeError, secret_message)
+      end
+
+      include_examples 'suppressed exception response'
+    end
+
+    context 'over HTTP' do
+      before do
+        allow(::Prometheus::Client::Formats::Text).to receive(:marshal).and_raise(RuntimeError, secret_message)
+      end
+
+      it 'does not leak the exception message to the client' do
+        driver.run(timeout: 1) do
+          Net::HTTP.start('127.0.0.1', port) do |http|
+            req = Net::HTTP::Get.new('/metrics')
+            res = http.request(req)
+            expect(res.code).to eq('500')
+            expect(res.body).to eq('in_prometheus server error: <RuntimeError>')
+            expect(res.body).not_to include(secret_message)
+          end
+        end
+      end
+    end
+  end
+
+  # The LogThrottle spec covers the throttling itself. These examples cover how
+  # in_prometheus uses it: the scope it throttles on, the fingerprint it makes
+  # from an error, the suppressed_log_count in the log, and what the client
+  # gets meanwhile.
+  describe 'error log throttling' do
+    let(:config) { LOCAL_CONFIG }
+    let(:secret_message) { 'dummy secret detail: password=deadbeef' }
+    let(:log_message) { 'in_prometheus: failed to render metrics' }
+    let(:workers_log_message) { 'in_prometheus: failed to render workers metrics' }
+
+    # Fluent::Clock.now is monotonic, so a plain Hash is enough to drive it
+    let(:clock) { { now: 1000.0 } }
+
+    def error_logs(message)
+      driver.logs.select { |log| log.include?(message) }
+    end
+
+    context 'when rendering metrics keeps failing' do
+      before do
+        allow(Fluent::Clock).to receive(:now) { clock[:now] }
+        allow(::Prometheus::Client::Formats::Text).to receive(:marshal).and_raise(RuntimeError, secret_message)
+      end
+
+      # every iteration raises from the same line, so the exceptions are equal
+      # to each other and only ignore_error_log_interval can let a log through
+      it 'logs the repeated same failure only once within ignore_error_log_interval' do
+        5.times { driver.instance.send(:all_metrics) }
+        expect(error_logs(log_message).size).to eq(1)
+      end
+
+      it 'keeps returning 500 to the client even while the log is suppressed' do
+        responses = 5.times.map { driver.instance.send(:all_metrics) }
+        expect(error_logs(log_message).size).to eq(1)
+        responses.each do |status, _header, body|
+          expect(status).to eq(500)
+          expect(body).to eq('in_prometheus server error: <RuntimeError>')
+        end
+      end
+
+      it 'reports how many logs were suppressed in the meantime' do
+        3.times { driver.instance.send(:all_metrics) }
+        clock[:now] += driver.instance.ignore_error_log_interval
+        driver.instance.send(:all_metrics)
+        logs = error_logs(log_message)
+        expect(logs.size).to eq(2)
+        expect(logs.first).not_to include('suppressed_log_count')
+        expect(logs.last).to include('suppressed_log_count=2')
+      end
+    end
+
+    describe 'telling the errors apart' do
+      before do
+        allow(Fluent::Clock).to receive(:now) { clock[:now] }
+      end
+
+      def log_error(scope, message, error)
+        driver.instance.send(:log_error_throttled, scope, message, error: error)
+      end
+
+      # the class and the message both take part in the fingerprint
+      it 'logs immediately when the error class differs' do
+        log_error(:metrics, log_message, RuntimeError.new(secret_message))
+        log_error(:metrics, log_message, ArgumentError.new(secret_message))
+        expect(error_logs(log_message).size).to eq(2)
+      end
+
+      it 'logs immediately when the error differs' do
+        log_error(:metrics, log_message, RuntimeError.new(secret_message))
+        log_error(:metrics, log_message, RuntimeError.new('another failure'))
+        expect(error_logs(log_message).size).to eq(2)
+      end
+
+      # the scope, not the log message, picks the slot to throttle on
+      it 'suppresses an equal error within a scope even when the log message differs' do
+        error = RuntimeError.new(secret_message)
+        log_error(:metrics, log_message, error)
+        log_error(:metrics, workers_log_message, error)
+        expect(error_logs(workers_log_message)).to be_empty
+      end
+
+      # the only example which takes the interval from the configuration
+      context 'with ignore_error_log_interval 0' do
+        let(:config) { LOCAL_CONFIG + %[
+  ignore_error_log_interval 0
+] }
+
+        it 'logs every occurrence of the same error' do
+          3.times { log_error(:metrics, log_message, RuntimeError.new(secret_message)) }
+          expect(error_logs(log_message).size).to eq(3)
+        end
+      end
+    end
+
+    # /metrics and /aggregated_metrics are usually scraped in turn, so both of
+    # them must be throttled on their own slot
+    context 'when both endpoints keep failing alternately' do
+      before do
+        allow(Fluent::Clock).to receive(:now) { clock[:now] }
+        allow(::Prometheus::Client::Formats::Text).to receive(:marshal).and_raise(RuntimeError, secret_message)
+        allow(driver.instance).to receive(:send_request_to_each_worker).and_raise(ArgumentError, 'another failure')
+      end
+
+      it 'logs each failure only once within ignore_error_log_interval' do
+        5.times do
+          driver.instance.send(:all_metrics)
+          driver.instance.send(:all_workers_metrics)
+        end
+        expect(error_logs(log_message).size).to eq(1)
+        expect(error_logs(workers_log_message).size).to eq(1)
+      end
+    end
+  end
 end

data/spec/fluent/plugin/out_prometheus_spec.rb

@@ -19,7 +19,35 @@ describe Fluent::Plugin::PrometheusOutput do
   describe '#testinitlabels' do
     it_behaves_like 'initalized metrics'
   end
-  
+
+  describe 'limiting label expansion' do
+    it_behaves_like 'limits label expansion'
+  end
+
+  # filter_prometheus routes such a record to @ERROR already. The output has to
+  # do the same, instead of failing on the router itself.
+  describe 'a record which cannot be instrumented' do
+    let(:config) {
+      BASE_CONFIG + %(
+        <metric>
+          name failing
+          type counter
+          desc Something foo.
+          key foo
+        </metric>
+      )
+    }
+
+    it 'emits an error event' do
+      driver.run(default_tag: tag) do
+        # a non numeric value is refused when the metric is instrumented
+        driver.feed(event_time, {'foo' => 'not a number'})
+      end
+
+      expect(driver.error_events.size).to eq(1)
+    end
+  end
+
   describe '#run' do
     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
 

data/spec/fluent/plugin/prometheus/log_throttle_spec.rb

@@ -0,0 +1,127 @@
+require 'spec_helper'
+require 'fluent/plugin/prometheus/log_throttle'
+
+describe Fluent::Plugin::Prometheus::LogThrottle do
+  # Fluent::Clock.now is monotonic, so a Hash is enough to fake it
+  let(:clock) { { now: 1000.0 } }
+  let(:interval) { 3600 }
+  # in_prometheus builds it out of an error
+  let(:fingerprint) { [RuntimeError, 'a'] }
+  subject(:throttle) { described_class.new(interval) }
+
+  before do
+    allow(Fluent::Clock).to receive(:now) { clock[:now] }
+  end
+
+  describe '#check' do
+    it 'emits on the first occurrence of a key' do
+      emit, suppressed = throttle.check(:foo, fingerprint)
+      expect(emit).to be true
+      expect(suppressed).to eq(0)
+    end
+
+    it 'suppresses the same key within the interval' do
+      throttle.check(:foo, fingerprint)
+      clock[:now] += interval - 1
+      emit, _ = throttle.check(:foo, fingerprint)
+      expect(emit).to be false
+    end
+
+    it 'emits again once the interval has elapsed' do
+      throttle.check(:foo, fingerprint)
+      clock[:now] += interval
+      emit, _ = throttle.check(:foo, fingerprint)
+      expect(emit).to be true
+    end
+
+    it 'reports how many occurrences were suppressed in the meantime' do
+      throttle.check(:foo, fingerprint)             # emits, suppressed=0
+      2.times { throttle.check(:foo, fingerprint) } # suppressed 1, then 2
+      clock[:now] += interval
+      emit, suppressed = throttle.check(:foo, fingerprint)
+      expect(emit).to be true
+      expect(suppressed).to eq(2)
+    end
+
+    it 'resets the suppressed count after emitting' do
+      throttle.check(:foo, fingerprint)
+      2.times { throttle.check(:foo, fingerprint) }
+      clock[:now] += interval
+      throttle.check(:foo, fingerprint)             # emits with suppressed=2
+      clock[:now] += interval
+      _, suppressed = throttle.check(:foo, fingerprint)
+      expect(suppressed).to eq(0)
+    end
+
+    it 'keeps a separate slot per key' do
+      expect(throttle.check(:foo, fingerprint).first).to be true
+      expect(throttle.check(:bar, fingerprint).first).to be true
+    end
+
+    # filter/out_prometheus throttles on the metric alone, since every drop of
+    # a metric reads the same
+    context 'without a fingerprint' do
+      it 'throttles on the key alone' do
+        expect(throttle.check(:foo).first).to be true
+        expect(throttle.check(:foo).first).to be false
+        expect(throttle.check(:bar).first).to be true
+      end
+
+      it 'reports how many occurrences were suppressed in the meantime' do
+        throttle.check(:foo)
+        2.times { throttle.check(:foo) }
+        clock[:now] += interval
+        emit, suppressed = throttle.check(:foo)
+        expect(emit).to be true
+        expect(suppressed).to eq(2)
+      end
+    end
+
+    it 'emits immediately when the fingerprint changes within the interval' do
+      expect(throttle.check(:foo, [RuntimeError, 'a']).first).to be true
+      expect(throttle.check(:foo, [RuntimeError, 'b']).first).to be true
+    end
+
+    # the caller makes a new fingerprint for each event, so it has to be
+    # compared by value, not by object identity
+    it 'suppresses an equal fingerprint given as a different object' do
+      expect(throttle.check(:foo, [RuntimeError, 'a']).first).to be true
+      expect(throttle.check(:foo, [RuntimeError, 'a']).first).to be false
+    end
+
+    it 'does not carry the suppressed count across a fingerprint change' do
+      throttle.check(:foo, [RuntimeError, 'a'])
+      2.times { throttle.check(:foo, [RuntimeError, 'a']) }
+      emit, suppressed = throttle.check(:foo, [RuntimeError, 'b'])
+      expect(emit).to be true
+      expect(suppressed).to eq(0)
+    end
+
+    context 'when interval is zero' do
+      let(:interval) { 0 }
+
+      it 'always emits without consulting the clock' do
+        expect(Fluent::Clock).not_to receive(:now)
+        3.times do
+          emit, suppressed = throttle.check(:foo, fingerprint)
+          expect(emit).to be true
+          expect(suppressed).to eq(0)
+        end
+      end
+    end
+
+    context 'when interval is negative' do
+      let(:interval) { -1 }
+
+      it 'always emits' do
+        expect(throttle.check(:foo, fingerprint).first).to be true
+        expect(throttle.check(:foo, fingerprint).first).to be true
+      end
+    end
+
+    it 'serializes concurrent checks for the same key into a single emission' do
+      results = 10.times.map { Thread.new { throttle.check(:foo, fingerprint).first } }.map(&:value)
+      expect(results.count(true)).to eq(1)
+    end
+  end
+end

data/spec/fluent/plugin/prometheus/placeholder_expander_spec.rb

@@ -50,6 +50,94 @@ describe Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander do
         expander.expand('${hostname}')
       end
 
+      # tag_parts, tag_prefix and tag_suffix must be expanded with the tag,
+      # not with "forged"
+      context 'with a value named after a tag placeholder' do
+        let(:forged_placeholder) do
+          {
+            'tag' => '1.2.3',
+            'tag_parts' => %w[forged forged forged],
+            'tag_prefix' => %w[forged forged forged],
+            'tag_suffix' => %w[forged forged forged],
+          }
+        end
+
+        it 'expands the placeholders with the tag' do
+          expander = builder.build(forged_placeholder)
+
+          expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}')).to eq('1.2.3')
+          expect(expander.expand('${tag_parts[-3]}.${tag_parts[-2]}.${tag_parts[-1]}')).to eq('1.2.3')
+          expect(expander.expand('${tag_prefix[0]},${tag_prefix[1]},${tag_prefix[2]}')).to eq('1,1.2,1.2.3')
+          expect(expander.expand('${tag_suffix[0]},${tag_suffix[1]},${tag_suffix[2]}')).to eq('3,2.3,1.2.3')
+        end
+
+        it 'does not expand an index which the tag does not have' do
+          # tag_prefix and tag_suffix have no negative index, so they are kept
+          # as they are. This checks that the record does not fill them.
+          expander = builder.build(forged_placeholder)
+
+          expect(expander.expand('${tag_prefix[-1]}')).to eq('${tag_prefix[-1]}')
+          expect(expander.expand('${tag_suffix[-1]}')).to eq('${tag_suffix[-1]}')
+
+          # the tag has 3 parts, so the 4th value of the record is out of it.
+          longer = forged_placeholder.merge('tag_parts' => %w[forged forged forged forged])
+          expander = builder.build(longer)
+
+          expect(expander.expand('${tag_parts[3]}')).to eq('${tag_parts[3]}')
+          expect(expander.expand('${tag_parts[-4]}')).to eq('${tag_parts[-4]}')
+        end
+      end
+
+      # a record may also have a key named "tag_parts[0]", which makes the
+      # same placeholder as the tag
+      context 'with a value whose key is a tag placeholder' do
+        it 'expands the placeholders with the tag' do
+          spelled = {
+            'tag' => '1.2.3',
+            'tag_parts[0]' => 'forged',
+            'tag_prefix[0]' => 'forged',
+            'tag_suffix[0]' => 'forged',
+          }
+          expander = builder.build(spelled)
+
+          expect(expander.expand('${tag_parts[0]}')).to eq('1')
+          expect(expander.expand('${tag_prefix[0]}')).to eq('1')
+          expect(expander.expand('${tag_suffix[0]}')).to eq('3')
+        end
+
+        it 'does not expand an index which the tag does not have' do
+          spelled = {
+            'tag' => '1',
+            'tag_parts[1]' => 'forged',
+            'tag_parts[-2]' => 'forged',
+            'tag_prefix[-1]' => 'forged',
+            'tag_suffix[-1]' => 'forged',
+          }
+          expander = builder.build(spelled)
+
+          expect(expander.expand('${tag_parts[1]}')).to eq('${tag_parts[1]}')
+          expect(expander.expand('${tag_parts[-2]}')).to eq('${tag_parts[-2]}')
+          expect(expander.expand('${tag_prefix[-1]}')).to eq('${tag_prefix[-1]}')
+          expect(expander.expand('${tag_suffix[-1]}')).to eq('${tag_suffix[-1]}')
+        end
+
+        it 'keeps the tag itself' do
+          spelled = {
+            'tag' => '1.2.3',
+            'tag_parts' => 'forged',
+            'tag_prefix' => 'forged',
+            'tag_suffix' => 'forged',
+          }
+          expander = builder.build(spelled)
+
+          expect(expander.expand('${tag}')).to eq('1.2.3')
+          # these are not built from the tag, so they are kept as they are
+          expect(expander.expand('${tag_parts}')).to eq('${tag_parts}')
+          expect(expander.expand('${tag_prefix}')).to eq('${tag_prefix}')
+          expect(expander.expand('${tag_suffix}')).to eq('${tag_suffix}')
+        end
+      end
+
       context 'when not found placeholder' do
         it 'prints wanring log and as it is' do
           expect(log).to receive(:warn).with('unknown placeholder `${tag_prefix[100]}` found').once
@@ -87,6 +175,14 @@ describe Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander do
         expect(expander.expand('${tag_suffix[0]}.${tag_suffix[1]}.${tag_suffix[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('3.2.3.1.2.3')
       end
 
+      it 'expands the placeholders with the dynamic tag' do
+        forged = static_placeholder.merge('tag_parts' => %w[forged forged forged])
+        expander = builder.build(forged)
+
+        expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}',
+                               dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+      end
+
       it 'does not create expander twice if given the same placeholder' do
         builder # cached before mock