fluent-plugin-prometheus 1.8.2 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6be199d6eb3693d4fe35be96961c05475a667ce8a7fe4b344fb6c5f7c269e820
-  data.tar.gz: 3cb81f50d84773da036d7bafadfa41c05928a7fb062310eeea0eea6546382900
+  metadata.gz: 53ce03bc02b89a80dbb46f166e91ad0e4a71011c54dcf1b2999fe5ef24aab7b8
+  data.tar.gz: d8a764f2a0ba7ffeb6ae383002aef2705e0a8c7a76550d8647452bfa032edf37
 SHA512:
-  metadata.gz: fc05d9f8283546700cc401c771d5444470e806fd4193cc47f661e01fd91bcd402ac2b0aa7fd52f2fa92ff5811f76d19aca378e3ae9919b93de27bca8e0300581
-  data.tar.gz: e614db8e23a6dad5a0d416b224e49ca1895b641b4b0d41933954d709080344f90cb79bc51869533d0efc43e93422dd401e4b547fa85ea316fa1f13457236933f
+  metadata.gz: d4187d6c44ae90bcbcb357efe3b6502c4a62250aa61d77779ded61c80e6be76d04e33a87ade92e9b5e4684d3b7b8d8d89a3eef9d622e9824d4801fc93ee0d6ca
+  data.tar.gz: c9a8db411e69763e3759ac7b3b0b9b14bd4eaf64d0c33ac0b97ac33fdcfdae2bf014c1e76de13e4c400251000faf9494e0258eddd8802f052885637003392960

data/.github/workflows/linux.yml

@@ -0,0 +1,26 @@
+name: linux
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake spec

data/ChangeLog

@@ -1,3 +1,23 @@
+Release 2.0.1 - 2021/04/08
+
+	* out_prometheus: Allow for lookup by symbol as well as string
+
+Release 2.0.0 - 2021/02/18
+
+	* Update prometheus-client dependency to 2.1.0 or later
+
+Release 1.8.5 - 2020/11/24
+
+	* in_prometheus_monitor: Support USR2 reload
+
+Release 1.8.4 - 2020/09/24
+
+	* in_prometheus_output_monitor: Add gauge_all parameter
+
+Release 1.8.3 - 2020/08/24
+
+	* Fix resourcr leak in async-http based server
+
 Release 1.8.2 - 2020/07/17
 
 	* in_prometheus_output_monitor/in_prometheus_tail_monitor: Support USR2 reload

data/README.md

@@ -152,6 +152,7 @@ More configuration parameters:
 
 - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 - `interval`: interval to update monitor_agent information in seconds (default: 5)
+- `gauge_all`: Specify metric type. If `true`, use `gauge` type. If `false`, use `counter` type. Since v2, this parameter will be removed and use `counter` type.
 
 ### prometheus_tail_monitor input plugin
 

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.8.2"
+  spec.version       = "2.0.1"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "fluentd", ">= 1.9.1", "< 2"
-  spec.add_dependency "prometheus-client", "< 0.10"
+  spec.add_dependency "prometheus-client", ">= 2.1.0"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/lib/fluent/plugin/in_prometheus/async_wrapper.rb

@@ -13,8 +13,9 @@ module Fluent::Plugin
             Async::HTTP::Endpoint.parse("http://#{host}:#{port}")
           end
 
-        client = Async::HTTP::Client.new(endpoint)
-        yield(AsyncHttpWrapper.new(client))
+        Async::HTTP::Client.open(endpoint) do |client|
+          yield(AsyncHttpWrapper.new(client))
+        end
       end
 
       Response = Struct.new(:code, :body, :headers)
@@ -38,7 +39,7 @@ module Fluent::Plugin
             raise error
           end
 
-          Response.new(response.status.to_s, response.body.read, response.headers)
+          Response.new(response.status.to_s, response.read || '', response.headers)
         end
       end
     end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -46,19 +46,19 @@ module Fluent::Plugin
     def start
       super
 
-      @buffer_newest_timekey = @registry.gauge(
+      @buffer_newest_timekey = get_gauge(
         :fluentd_status_buffer_newest_timekey,
         'Newest timekey in buffer.')
-      @buffer_oldest_timekey = @registry.gauge(
+      @buffer_oldest_timekey = get_gauge(
         :fluentd_status_buffer_oldest_timekey,
         'Oldest timekey in buffer.')
-      buffer_queue_length = @registry.gauge(
+      buffer_queue_length = get_gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
-      buffer_total_queued_size = @registry.gauge(
+      buffer_total_queued_size = get_gauge(
         :fluentd_status_buffer_total_bytes,
         'Current total size of queued buffers.')
-      retry_counts = @registry.gauge(
+      retry_counts = get_gauge(
         :fluentd_status_retry_count,
         'Current retry counts.')
 
@@ -76,14 +76,14 @@ module Fluent::Plugin
 
         @monitor_info.each do |name, metric|
           if info[name]
-            metric.set(label, info[name])
+            metric.set(info[name], labels: label)
           end
         end
 
         timekeys = info["buffer_timekeys"]
         if timekeys && !timekeys.empty?
-          @buffer_newest_timekey.set(label, timekeys.max)
-          @buffer_oldest_timekey.set(label, timekeys.min)
+          @buffer_newest_timekey.set(timekeys.max, labels: label)
+          @buffer_oldest_timekey.set(timekeys.min, labels: label)
         end
       end
     end
@@ -95,5 +95,13 @@ module Fluent::Plugin
         type: plugin_info["type"],
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :plugin_category, :type])
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -10,6 +10,7 @@ module Fluent::Plugin
     helpers :timer
 
     config_param :interval, :time, default: 5
+    config_param :gauge_all, :bool, default: true
     attr_reader :registry
 
     MONITOR_IVARS = [
@@ -54,6 +55,8 @@ module Fluent::Plugin
       end
 
       @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+
+      @gauge_or_counter = @gauge_all ? :gauge : :counter
     end
 
     def start
@@ -87,28 +90,28 @@ module Fluent::Plugin
           'Oldest timekey in buffer.'),
 
         # Output metrics
-        retry_counts: get_gauge(
+        retry_counts: get_gauge_or_counter(
           :fluentd_output_status_retry_count,
           'Current retry counts.'),
-        num_errors: get_gauge(
+        num_errors: get_gauge_or_counter(
           :fluentd_output_status_num_errors,
           'Current number of errors.'),
-        emit_count: get_gauge(
+        emit_count: get_gauge_or_counter(
           :fluentd_output_status_emit_count,
           'Current emit counts.'),
-        emit_records: get_gauge(
+        emit_records: get_gauge_or_counter(
           :fluentd_output_status_emit_records,
           'Current emit records.'),
-        write_count: get_gauge(
+        write_count: get_gauge_or_counter(
           :fluentd_output_status_write_count,
           'Current write counts.'),
         rollback_count: get_gauge(
           :fluentd_output_status_rollback_count,
           'Current rollback counts.'),
-        flush_time_count: get_gauge(
+        flush_time_count: get_gauge_or_counter(
           :fluentd_output_status_flush_time_count,
           'Total flush time.'),
-        slow_flush_count: get_gauge(
+        slow_flush_count: get_gauge_or_counter(
           :fluentd_output_status_slow_flush_count,
           'Current slow flush counts.'),
         retry_wait: get_gauge(
@@ -157,14 +160,22 @@ module Fluent::Plugin
 
         monitor_info.each do |name, metric|
           if info[name]
-            metric.set(label, info[name])
+            if metric.is_a?(::Prometheus::Client::Gauge)
+              metric.set(info[name], labels: label)
+            elsif metric.is_a?(::Prometheus::Client::Counter)
+              metric.increment(by: info[name] - metric.get(labels: label), labels: label)
+            end
           end
         end
 
         if info['instance_variables']
           instance_vars_info.each do |name, metric|
             if info['instance_variables'][name]
-              metric.set(label, info['instance_variables'][name])
+              if metric.is_a?(::Prometheus::Client::Gauge)
+                metric.set(info['instance_variables'][name], labels: label)
+              elsif metric.is_a?(::Prometheus::Client::Counter)
+                metric.increment(by: info['instance_variables'][name] - metric.get(labels: label), labels: label)
+              end
             end
           end
         end
@@ -182,7 +193,7 @@ module Fluent::Plugin
           if next_time && start_time
             wait = next_time - start_time
           end
-          @metrics[:retry_wait].set(label, wait.to_f)
+          @metrics[:retry_wait].set(wait.to_f, labels: label)
         end
       end
     end
@@ -198,7 +209,15 @@ module Fluent::Plugin
       if @registry.exist?(name)
         @registry.get(name)
       else
-        @registry.gauge(name, docstring)
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
+      end
+    end
+
+    def get_gauge_or_counter(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.public_send(@gauge_or_counter, name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
       end
     end
   end

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -73,8 +73,8 @@ module Fluent::Plugin
           # Very fragile implementation
           pe = watcher.instance_variable_get(:@pe)
           label = labels(info, watcher.path)
-          @metrics[:inode].set(label, pe.read_inode)
-          @metrics[:position].set(label, pe.read_pos)
+          @metrics[:inode].set(pe.read_inode, labels: label)
+          @metrics[:position].set(pe.read_pos, labels: label)
         end
       end
     end
@@ -91,7 +91,7 @@ module Fluent::Plugin
       if @registry.exist?(name)
         @registry.get(name)
       else
-        @registry.gauge(name, docstring)
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type, :path])
       end
     end
   end

data/lib/fluent/plugin/prometheus.rb

@@ -85,6 +85,18 @@ module Fluent
         Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
       end
 
+      def stringify_keys(hash_to_stringify)
+        # Adapted from: https://www.jvt.me/posts/2019/09/07/ruby-hash-keys-string-symbol/
+        hash_to_stringify.map do |k,v|
+          value_or_hash = if v.instance_of? Hash
+                            stringify_keys(v)
+                          else
+                            v
+                          end
+          [k.to_s, value_or_hash]
+        end.to_h
+      end
+
       def configure(conf)
         super
         @placeholder_values = {}
@@ -99,6 +111,7 @@ module Fluent
           'worker_id' => fluentd_worker_id,
         }
 
+        record = stringify_keys(record)
         placeholders = record.merge(@placeholder_values[tag])
         expander = @placeholder_expander_builder.build(placeholders)
         metrics.each do |metric|
@@ -119,6 +132,7 @@ module Fluent
         }
 
         es.each do |time, record|
+          record = stringify_keys(record)
           placeholders = record.merge(placeholder_values)
           expander = @placeholder_expander_builder.build(placeholders)
           metrics.each do |metric|
@@ -188,7 +202,7 @@ module Fluent
           end
 
           begin
-            @gauge = registry.gauge(element['name'].to_sym, element['desc'])
+            @gauge = registry.gauge(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
           end
@@ -201,7 +215,7 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @gauge.set(labels(record, expander), value)
+            @gauge.set(value, labels: labels(record, expander))
           end
         end
       end
@@ -210,7 +224,7 @@ module Fluent
         def initialize(element, registry, labels)
           super
           begin
-            @counter = registry.counter(element['name'].to_sym, element['desc'])
+            @counter = registry.counter(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
           end
@@ -229,7 +243,7 @@ module Fluent
           # ignore if record value is nil
           return if value.nil?
 
-          @counter.increment(labels(record, expander), value)
+          @counter.increment(by: value, labels: labels(record, expander))
         end
       end
 
@@ -241,7 +255,7 @@ module Fluent
           end
 
           begin
-            @summary = registry.summary(element['name'].to_sym, element['desc'])
+            @summary = registry.summary(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
           end
@@ -254,7 +268,7 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @summary.observe(labels(record, expander), value)
+            @summary.observe(value, labels: labels(record, expander))
           end
         end
       end
@@ -271,9 +285,9 @@ module Fluent
               buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
                 e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
               end
-              @histogram = registry.histogram(element['name'].to_sym, element['desc'], {}, buckets)
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys, buckets: buckets)
             else
-              @histogram = registry.histogram(element['name'].to_sym, element['desc'])
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
             end
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
@@ -287,7 +301,7 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @histogram.observe(labels(record, expander), value)
+            @histogram.observe(value, labels: labels(record, expander))
           end
         end
       end

data/spec/fluent/plugin/out_prometheus_spec.rb

@@ -40,4 +40,29 @@ describe Fluent::Plugin::PrometheusOutput do
 
     it_behaves_like 'instruments record'
   end
+
+  describe '#run with symbolized keys' do
+    let(:message) { {:foo => 100, :bar => 100, :baz => 100, :qux => 10} }
+
+    context 'simple config' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
+
+      it 'adds a new counter metric' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
+      end
+    end
+
+    it_behaves_like 'instruments record'
+  end
 end

data/spec/fluent/plugin/shared.rb

@@ -178,20 +178,19 @@ shared_examples_for 'instruments record' do
 
     it 'instruments counter metric' do
       expect(counter.type).to eq(:counter)
-      expect(counter.get({test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
-      expect(counter_with_accessor.get({test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
+      expect(counter.get(labels: {test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
+      expect(counter_with_accessor.get(labels: {test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
     end
 
     it 'instruments gauge metric' do
       expect(gauge.type).to eq(:gauge)
-      expect(gauge.get({test_key: 'test_value', key: 'foo2'})).to eq(100)
+      expect(gauge.get(labels: {test_key: 'test_value', key: 'foo2'})).to eq(100)
     end
 
     it 'instruments summary metric' do
       expect(summary.type).to eq(:summary)
-      expect(summary.get({test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
-      expect(summary.get({test_key: 'test_value', key: 'foo3'})[0.99]).to eq(100)
-      expect(summary_with_accessor.get({test_key: 'test_value', key: 'foo5'})[0.99]).to eq(100)
+      expect(summary.get(labels: {test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
+      expect(summary_with_accessor.get(labels: {test_key: 'test_value', key: 'foo5'})["sum"]).to eq(100)
     end
 
     it 'instruments histogram metric' do
@@ -200,8 +199,8 @@ shared_examples_for 'instruments record' do
       end
 
       expect(histogram.type).to eq(:histogram)
-      expect(histogram.get({test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
-      expect(histogram.get({test_key: 'test_value', key: 'foo4'})[10]).to eq(5) # 4 + `es` in before
+      expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
+      expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})["10"]).to eq(5) # 4 + `es` in before
     end
   end
 
@@ -231,7 +230,7 @@ shared_examples_for 'instruments record' do
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       key, _ = counter.values.find {|k,v| v ==  100 }
       expect(key).to be_kind_of(Hash)
-      expect(key[:foo]).to eq(100)
+      expect(key[:foo]).to eq("100")
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.8.2
+  version: 2.0.1
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-17 00:00:00.000000000 Z
+date: 2021-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +107,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -158,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.