fluent-plugin-prometheus 2.2.1 → 2.3.0

data/spec/fluent/plugin/filter_prometheus_spec.rb

@@ -45,4 +45,145 @@ describe Fluent::Plugin::PrometheusFilter do
 
     it_behaves_like 'instruments record'
   end
+
+  # a label which uses the tag must be expanded with the tag, not with the
+  # record.
+  describe 'a record which has a tag placeholder name' do
+    let(:config) {
+      BASE_CONFIG + %(
+        <metric>
+          name tagged
+          type counter
+          desc Something foo.
+          key foo
+          <labels>
+            part ${tag_parts[0]}
+          </labels>
+        </metric>
+      )
+    }
+
+    it 'labels the metric with the tag' do
+      driver.run(default_tag: tag) do
+        driver.feed(event_time, {'tag' => 'forged.tag', 'foo' => 1, 'tag_parts' => ['forged']})
+      end
+      # even though tag_parts exists in the record, it should not be taken as "part".
+      expect(registry.get(:tagged).values.keys).to eq([{part: 'prometheus'}])
+    end
+
+    it 'labels the metric with the tag, not with the record key' do
+      driver.run(default_tag: tag) do
+        driver.feed(event_time, {'tag' => 'forged.tag', 'foo' => 1, 'tag_parts[0]' => 'forged'})
+      end
+      # the record has a key named "tag_parts[0]", but the tag must win.
+      expect(registry.get(:tagged).values.keys).to eq([{part: 'prometheus'}])
+    end
+  end
+
+  # a record must not fill an index which the tag does not have.
+  describe 'a record which uses an index out of the tag' do
+    let(:config) {
+      BASE_CONFIG + %(
+        <metric>
+          name out_of_range
+          type counter
+          desc Something foo.
+          key foo
+          <labels>
+            part ${tag_parts[2]}
+            prefix ${tag_prefix[-1]}
+            suffix ${tag_suffix[-1]}
+          </labels>
+        </metric>
+      )
+    }
+
+    it 'leaves the label unexpanded' do
+      driver.run(default_tag: tag) do
+        driver.feed(event_time, {
+                      'foo' => 1,
+                      'tag_parts[2]' => 'forged',
+                      'tag_prefix[-1]' => 'forged',
+                      'tag_suffix[-1]' => 'forged',
+                    })
+      end
+      # the tag "prometheus.test" has 2 parts, and tag_prefix and tag_suffix
+      # have no negative index. So these labels must not be expanded.
+      expect(registry.get(:out_of_range).values.keys).to eq(
+        [{part: '${tag_parts[2]}', prefix: '${tag_prefix[-1]}', suffix: '${tag_suffix[-1]}'}]
+      )
+    end
+  end
+
+  describe 'limiting label expansion' do
+    it_behaves_like 'limits label expansion'
+  end
+
+  # the throttling itself is covered by the LogThrottle spec; what is left here
+  # is the warning warn_label_set_limit builds out of it, and the interval it
+  # takes from the configuration
+  describe 'drop log throttling' do
+    let(:config) {
+      BASE_CONFIG + %[
+        ignore_error_log_interval 3600
+        <metric>
+          name throttled
+          type counter
+          desc Something foo.
+          key foo
+        </metric>
+      ]
+    }
+    # Fluent::Clock.now is monotonic, so a plain Hash is enough to drive it
+    let(:clock) { { now: 1000.0 } }
+    let(:metric) { double('metric', name: :throttled, max_series_per_metric: 5) }
+    let(:text) { 'dropped a label set' }
+
+    before do
+      allow(Fluent::Clock).to receive(:now) { clock[:now] }
+    end
+
+    def logs_about(text)
+      driver.logs.select { |log| log.include?(text) }
+    end
+
+    def drop
+      driver.instance.send(:warn_label_set_limit, metric)
+    end
+
+    it 'warns only once within ignore_error_log_interval' do
+      5.times { drop }
+      expect(logs_about(text).size).to eq(1)
+    end
+
+    it 'reports how many warnings were suppressed in the meantime' do
+      3.times { drop }
+      clock[:now] += driver.instance.ignore_error_log_interval
+      drop
+      logs = logs_about(text)
+      expect(logs.size).to eq(2)
+      expect(logs.first).not_to include('suppressed_log_count')
+      expect(logs.last).to include('suppressed_log_count=2')
+    end
+
+    # the only example which takes the interval from the configuration
+    context 'with ignore_error_log_interval 0' do
+      let(:config) {
+        BASE_CONFIG + %[
+          ignore_error_log_interval 0
+          <metric>
+            name throttled
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        ]
+      }
+
+      it 'warns about every drop' do
+        3.times { drop }
+        expect(logs_about(text).size).to eq(3)
+      end
+    end
+  end
 end

data/spec/fluent/plugin/in_prometheus_spec.rb

@@ -22,9 +22,16 @@ describe Fluent::Plugin::PrometheusInput do
   describe '#configure' do
     describe 'bind' do
       let(:config) { CONFIG + %[
-  bind 127.0.0.1
+  bind ::1
 ] }
       it 'should be configurable' do
+        expect(driver.instance.bind).to eq('::1')
+      end
+    end
+
+    describe 'default bind' do
+      let(:config) { CONFIG }
+      it 'should be accessible from 127.0.0.1 by default' do
         expect(driver.instance.bind).to eq('127.0.0.1')
       end
     end
@@ -64,6 +71,22 @@ describe Fluent::Plugin::PrometheusInput do
         expect(driver.instance.content_encoding).to eq(:gzip)
       end
     end
+
+    describe 'default ignore_error_log_interval' do
+      let(:config) { CONFIG }
+      it 'should be 3600 seconds by default' do
+        expect(driver.instance.ignore_error_log_interval).to eq(3600)
+      end
+    end
+
+    describe 'error_log_interval' do
+      let(:config) { CONFIG + %[
+    ignore_error_log_interval 60
+] }
+      it 'should be configurable' do
+        expect(driver.instance.ignore_error_log_interval).to eq(60)
+      end
+    end
   end
 
   describe '#start' do
@@ -89,6 +112,22 @@ describe Fluent::Plugin::PrometheusInput do
       end
     end
 
+    context 'IPv6 with TLS' do
+      let(:config) do
+        %[
+           @type prometheus
+           bind ::1
+           <transport tls>
+             insecure true
+           </transport>
+         ]
+      end
+
+      it 'raises ConfigError for unsupported combination' do
+        expect { driver.run(timeout: 1) }.to raise_error(Fluent::ConfigError, /IPv6 with <transport tls> is not currently supported/)
+      end
+    end
+
     context 'old parameters are given' do
       context 'when extra_conf is used' do
         let(:config) do
@@ -278,4 +317,221 @@ describe Fluent::Plugin::PrometheusInput do
       end
     end
   end
+
+  describe '#run with IPv6' do
+    shared_examples 'IPv6 server binding' do |bind_addr, connect_addr, description|
+      let(:config) do
+        # Quote the bind address if it contains brackets
+        bind_value = bind_addr.include?('[') ? "\"#{bind_addr}\"" : bind_addr
+        %[
+          @type prometheus
+          bind #{bind_value}
+        ]
+      end
+
+      it description do
+        skip 'IPv6 not available on this system' unless ipv6_enabled?
+
+        driver.run(timeout: 3) do
+          Net::HTTP.start(connect_addr, port) do |http|
+            req = Net::HTTP::Get.new('/metrics')
+            res = http.request(req)
+            expect(res.code).to eq('200')
+          end
+        end
+      end
+    end
+
+    context 'IPv6 loopback address ::1' do
+      include_examples 'IPv6 server binding', '::1', '::1', 'binds and serves on IPv6 loopback address'
+    end
+
+    context 'IPv6 any address ::' do
+      include_examples 'IPv6 server binding', '::', '::1', 'binds on :: and connects via ::1'
+    end
+
+    context 'pre-bracketed IPv6 address [::1]' do
+      include_examples 'IPv6 server binding', '[::1]', '::1', 'handles pre-bracketed address correctly'
+    end
+  end
+
+  describe 'error handling (information disclosure)' do
+    let(:config) { LOCAL_CONFIG }
+    let(:secret_message) { 'dummy secret detail: password=deadbeef' }
+
+    shared_examples 'suppressed exception response' do
+      it 'returns 500 with text/plain' do
+        status, header, _body = subject
+        expect(status).to eq(500)
+        expect(header['Content-Type']).to eq('text/plain')
+      end
+
+      it 'exposes the exception class only' do
+        _status, _header, body = subject
+        expect(body).to eq('in_prometheus server error: <RuntimeError>')
+        expect(body).not_to include(secret_message)
+      end
+
+      it 'logs the detail on the server side' do
+        subject
+        expect(driver.logs.any? { |log| log.include?(log_message) }).to be true
+        expect(driver.logs.any? { |log| log.include?(secret_message) }).to be true
+      end
+    end
+
+    context '#all_metrics' do
+      subject { driver.instance.send(:all_metrics) }
+
+      let(:log_message) { 'in_prometheus: failed to render metrics' }
+
+      before do
+        allow(::Prometheus::Client::Formats::Text).to receive(:marshal).and_raise(RuntimeError, secret_message)
+      end
+
+      include_examples 'suppressed exception response'
+    end
+
+    context '#all_workers_metrics' do
+      subject { driver.instance.send(:all_workers_metrics) }
+
+      let(:log_message) { 'in_prometheus: failed to render workers metrics' }
+
+      before do
+        allow(driver.instance).to receive(:send_request_to_each_worker).and_raise(RuntimeError, secret_message)
+      end
+
+      include_examples 'suppressed exception response'
+    end
+
+    context 'over HTTP' do
+      before do
+        allow(::Prometheus::Client::Formats::Text).to receive(:marshal).and_raise(RuntimeError, secret_message)
+      end
+
+      it 'does not leak the exception message to the client' do
+        driver.run(timeout: 1) do
+          Net::HTTP.start('127.0.0.1', port) do |http|
+            req = Net::HTTP::Get.new('/metrics')
+            res = http.request(req)
+            expect(res.code).to eq('500')
+            expect(res.body).to eq('in_prometheus server error: <RuntimeError>')
+            expect(res.body).not_to include(secret_message)
+          end
+        end
+      end
+    end
+  end
+
+  # The LogThrottle spec covers the throttling itself. These examples cover how
+  # in_prometheus uses it: the scope it throttles on, the fingerprint it makes
+  # from an error, the suppressed_log_count in the log, and what the client
+  # gets meanwhile.
+  describe 'error log throttling' do
+    let(:config) { LOCAL_CONFIG }
+    let(:secret_message) { 'dummy secret detail: password=deadbeef' }
+    let(:log_message) { 'in_prometheus: failed to render metrics' }
+    let(:workers_log_message) { 'in_prometheus: failed to render workers metrics' }
+
+    # Fluent::Clock.now is monotonic, so a plain Hash is enough to drive it
+    let(:clock) { { now: 1000.0 } }
+
+    def error_logs(message)
+      driver.logs.select { |log| log.include?(message) }
+    end
+
+    context 'when rendering metrics keeps failing' do
+      before do
+        allow(Fluent::Clock).to receive(:now) { clock[:now] }
+        allow(::Prometheus::Client::Formats::Text).to receive(:marshal).and_raise(RuntimeError, secret_message)
+      end
+
+      # every iteration raises from the same line, so the exceptions are equal
+      # to each other and only ignore_error_log_interval can let a log through
+      it 'logs the repeated same failure only once within ignore_error_log_interval' do
+        5.times { driver.instance.send(:all_metrics) }
+        expect(error_logs(log_message).size).to eq(1)
+      end
+
+      it 'keeps returning 500 to the client even while the log is suppressed' do
+        responses = 5.times.map { driver.instance.send(:all_metrics) }
+        expect(error_logs(log_message).size).to eq(1)
+        responses.each do |status, _header, body|
+          expect(status).to eq(500)
+          expect(body).to eq('in_prometheus server error: <RuntimeError>')
+        end
+      end
+
+      it 'reports how many logs were suppressed in the meantime' do
+        3.times { driver.instance.send(:all_metrics) }
+        clock[:now] += driver.instance.ignore_error_log_interval
+        driver.instance.send(:all_metrics)
+        logs = error_logs(log_message)
+        expect(logs.size).to eq(2)
+        expect(logs.first).not_to include('suppressed_log_count')
+        expect(logs.last).to include('suppressed_log_count=2')
+      end
+    end
+
+    describe 'telling the errors apart' do
+      before do
+        allow(Fluent::Clock).to receive(:now) { clock[:now] }
+      end
+
+      def log_error(scope, message, error)
+        driver.instance.send(:log_error_throttled, scope, message, error: error)
+      end
+
+      # the class and the message both take part in the fingerprint
+      it 'logs immediately when the error class differs' do
+        log_error(:metrics, log_message, RuntimeError.new(secret_message))
+        log_error(:metrics, log_message, ArgumentError.new(secret_message))
+        expect(error_logs(log_message).size).to eq(2)
+      end
+
+      it 'logs immediately when the error differs' do
+        log_error(:metrics, log_message, RuntimeError.new(secret_message))
+        log_error(:metrics, log_message, RuntimeError.new('another failure'))
+        expect(error_logs(log_message).size).to eq(2)
+      end
+
+      # the scope, not the log message, picks the slot to throttle on
+      it 'suppresses an equal error within a scope even when the log message differs' do
+        error = RuntimeError.new(secret_message)
+        log_error(:metrics, log_message, error)
+        log_error(:metrics, workers_log_message, error)
+        expect(error_logs(workers_log_message)).to be_empty
+      end
+
+      # the only example which takes the interval from the configuration
+      context 'with ignore_error_log_interval 0' do
+        let(:config) { LOCAL_CONFIG + %[
+  ignore_error_log_interval 0
+] }
+
+        it 'logs every occurrence of the same error' do
+          3.times { log_error(:metrics, log_message, RuntimeError.new(secret_message)) }
+          expect(error_logs(log_message).size).to eq(3)
+        end
+      end
+    end
+
+    # /metrics and /aggregated_metrics are usually scraped in turn, so both of
+    # them must be throttled on their own slot
+    context 'when both endpoints keep failing alternately' do
+      before do
+        allow(Fluent::Clock).to receive(:now) { clock[:now] }
+        allow(::Prometheus::Client::Formats::Text).to receive(:marshal).and_raise(RuntimeError, secret_message)
+        allow(driver.instance).to receive(:send_request_to_each_worker).and_raise(ArgumentError, 'another failure')
+      end
+
+      it 'logs each failure only once within ignore_error_log_interval' do
+        5.times do
+          driver.instance.send(:all_metrics)
+          driver.instance.send(:all_workers_metrics)
+        end
+        expect(error_logs(log_message).size).to eq(1)
+        expect(error_logs(workers_log_message).size).to eq(1)
+      end
+    end
+  end
 end

data/spec/fluent/plugin/out_prometheus_spec.rb

@@ -19,7 +19,35 @@ describe Fluent::Plugin::PrometheusOutput do
   describe '#testinitlabels' do
     it_behaves_like 'initalized metrics'
   end
-  
+
+  describe 'limiting label expansion' do
+    it_behaves_like 'limits label expansion'
+  end
+
+  # filter_prometheus routes such a record to @ERROR already. The output has to
+  # do the same, instead of failing on the router itself.
+  describe 'a record which cannot be instrumented' do
+    let(:config) {
+      BASE_CONFIG + %(
+        <metric>
+          name failing
+          type counter
+          desc Something foo.
+          key foo
+        </metric>
+      )
+    }
+
+    it 'emits an error event' do
+      driver.run(default_tag: tag) do
+        # a non numeric value is refused when the metric is instrumented
+        driver.feed(event_time, {'foo' => 'not a number'})
+      end
+
+      expect(driver.error_events.size).to eq(1)
+    end
+  end
+
   describe '#run' do
     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
 

data/spec/fluent/plugin/prometheus/log_throttle_spec.rb

@@ -0,0 +1,127 @@
+require 'spec_helper'
+require 'fluent/plugin/prometheus/log_throttle'
+
+describe Fluent::Plugin::Prometheus::LogThrottle do
+  # Fluent::Clock.now is monotonic, so a Hash is enough to fake it
+  let(:clock) { { now: 1000.0 } }
+  let(:interval) { 3600 }
+  # in_prometheus builds it out of an error
+  let(:fingerprint) { [RuntimeError, 'a'] }
+  subject(:throttle) { described_class.new(interval) }
+
+  before do
+    allow(Fluent::Clock).to receive(:now) { clock[:now] }
+  end
+
+  describe '#check' do
+    it 'emits on the first occurrence of a key' do
+      emit, suppressed = throttle.check(:foo, fingerprint)
+      expect(emit).to be true
+      expect(suppressed).to eq(0)
+    end
+
+    it 'suppresses the same key within the interval' do
+      throttle.check(:foo, fingerprint)
+      clock[:now] += interval - 1
+      emit, _ = throttle.check(:foo, fingerprint)
+      expect(emit).to be false
+    end
+
+    it 'emits again once the interval has elapsed' do
+      throttle.check(:foo, fingerprint)
+      clock[:now] += interval
+      emit, _ = throttle.check(:foo, fingerprint)
+      expect(emit).to be true
+    end
+
+    it 'reports how many occurrences were suppressed in the meantime' do
+      throttle.check(:foo, fingerprint)             # emits, suppressed=0
+      2.times { throttle.check(:foo, fingerprint) } # suppressed 1, then 2
+      clock[:now] += interval
+      emit, suppressed = throttle.check(:foo, fingerprint)
+      expect(emit).to be true
+      expect(suppressed).to eq(2)
+    end
+
+    it 'resets the suppressed count after emitting' do
+      throttle.check(:foo, fingerprint)
+      2.times { throttle.check(:foo, fingerprint) }
+      clock[:now] += interval
+      throttle.check(:foo, fingerprint)             # emits with suppressed=2
+      clock[:now] += interval
+      _, suppressed = throttle.check(:foo, fingerprint)
+      expect(suppressed).to eq(0)
+    end
+
+    it 'keeps a separate slot per key' do
+      expect(throttle.check(:foo, fingerprint).first).to be true
+      expect(throttle.check(:bar, fingerprint).first).to be true
+    end
+
+    # filter/out_prometheus throttles on the metric alone, since every drop of
+    # a metric reads the same
+    context 'without a fingerprint' do
+      it 'throttles on the key alone' do
+        expect(throttle.check(:foo).first).to be true
+        expect(throttle.check(:foo).first).to be false
+        expect(throttle.check(:bar).first).to be true
+      end
+
+      it 'reports how many occurrences were suppressed in the meantime' do
+        throttle.check(:foo)
+        2.times { throttle.check(:foo) }
+        clock[:now] += interval
+        emit, suppressed = throttle.check(:foo)
+        expect(emit).to be true
+        expect(suppressed).to eq(2)
+      end
+    end
+
+    it 'emits immediately when the fingerprint changes within the interval' do
+      expect(throttle.check(:foo, [RuntimeError, 'a']).first).to be true
+      expect(throttle.check(:foo, [RuntimeError, 'b']).first).to be true
+    end
+
+    # the caller makes a new fingerprint for each event, so it has to be
+    # compared by value, not by object identity
+    it 'suppresses an equal fingerprint given as a different object' do
+      expect(throttle.check(:foo, [RuntimeError, 'a']).first).to be true
+      expect(throttle.check(:foo, [RuntimeError, 'a']).first).to be false
+    end
+
+    it 'does not carry the suppressed count across a fingerprint change' do
+      throttle.check(:foo, [RuntimeError, 'a'])
+      2.times { throttle.check(:foo, [RuntimeError, 'a']) }
+      emit, suppressed = throttle.check(:foo, [RuntimeError, 'b'])
+      expect(emit).to be true
+      expect(suppressed).to eq(0)
+    end
+
+    context 'when interval is zero' do
+      let(:interval) { 0 }
+
+      it 'always emits without consulting the clock' do
+        expect(Fluent::Clock).not_to receive(:now)
+        3.times do
+          emit, suppressed = throttle.check(:foo, fingerprint)
+          expect(emit).to be true
+          expect(suppressed).to eq(0)
+        end
+      end
+    end
+
+    context 'when interval is negative' do
+      let(:interval) { -1 }
+
+      it 'always emits' do
+        expect(throttle.check(:foo, fingerprint).first).to be true
+        expect(throttle.check(:foo, fingerprint).first).to be true
+      end
+    end
+
+    it 'serializes concurrent checks for the same key into a single emission' do
+      results = 10.times.map { Thread.new { throttle.check(:foo, fingerprint).first } }.map(&:value)
+      expect(results.count(true)).to eq(1)
+    end
+  end
+end