fluent-plugin-prometheus 1.8.4 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c268ed901a0c48ea1885ce64b621c3abc79983a95696180236156b7844029a4d
-  data.tar.gz: ac9aea40b3eab787f14ba1e402732898ab2cf355a3836d252b85867100914a93
+  metadata.gz: 949d2ec196fae78bdfcbbfeae20d718e7a3a34a6ceba9ba665d1a7d40326d529
+  data.tar.gz: c1adf9a63565a38141247034157f84aadb85336e5add24ca6d99c126cd8e01d7
 SHA512:
-  metadata.gz: 8bead7bf6d8534398ef24706348ae55d6437e3b898d89e6edb9981c6311d92be2e6d25f841a6bbf6be973a870966ff1296f3f4fb5314be62d06b900668f51b85
-  data.tar.gz: 8e9d8a48bdb3a3ba73626c01acda706a8af9f6dc309a1c005a240eb3cf051ef50ffe3b0012e966765ba8ffec415b2d2fd4a24852938c7fad84882ee842a96899
+  metadata.gz: 7199fc669d4c389fa5638615d16a4fdb29a27cc959b60c3ec6e3031328a9cb052bb6169733c73e66f659ec929aa3faffd8869187dc70f6eff361263b3304ecb0
+  data.tar.gz: acfbe23681c5bf801ae10c41a265dfb358b05516b79d0e2944317469b0b4be8aad5b1fbda246d9e4caa0e75f473428046015232c0900689ccb629a3b7c5b72a8

data/.github/workflows/linux.yml

@@ -0,0 +1,32 @@
+name: linux
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
+        os:
+          - ubuntu-latest
+        experimental: [false]
+        include:
+          - ruby: head
+            os: ubuntu-latest
+            experimental: true
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake spec

data/ChangeLog

@@ -1,3 +1,19 @@
+Release 2.0.2 - 2021/08/02
+
+	* in_prometheus_output_monitor: Follow Fluentd's core metrics mechanism
+
+Release 2.0.1 - 2021/04/08
+
+	* out_prometheus: Allow for lookup by symbol as well as string
+
+Release 2.0.0 - 2021/02/18
+
+	* Update prometheus-client dependency to 2.1.0 or later
+
+Release 1.8.5 - 2020/11/24
+
+	* in_prometheus_monitor: Support USR2 reload
+
 Release 1.8.4 - 2020/09/24
 
 	* in_prometheus_output_monitor: Add gauge_all parameter

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.8.4"
+  spec.version       = "2.0.2"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "fluentd", ">= 1.9.1", "< 2"
-  spec.add_dependency "prometheus-client", "< 0.10"
+  spec.add_dependency "prometheus-client", ">= 2.1.0"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -46,19 +46,19 @@ module Fluent::Plugin
     def start
       super
 
-      @buffer_newest_timekey = @registry.gauge(
+      @buffer_newest_timekey = get_gauge(
         :fluentd_status_buffer_newest_timekey,
         'Newest timekey in buffer.')
-      @buffer_oldest_timekey = @registry.gauge(
+      @buffer_oldest_timekey = get_gauge(
         :fluentd_status_buffer_oldest_timekey,
         'Oldest timekey in buffer.')
-      buffer_queue_length = @registry.gauge(
+      buffer_queue_length = get_gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
-      buffer_total_queued_size = @registry.gauge(
+      buffer_total_queued_size = get_gauge(
         :fluentd_status_buffer_total_bytes,
         'Current total size of queued buffers.')
-      retry_counts = @registry.gauge(
+      retry_counts = get_gauge(
         :fluentd_status_retry_count,
         'Current retry counts.')
 
@@ -76,14 +76,14 @@ module Fluent::Plugin
 
         @monitor_info.each do |name, metric|
           if info[name]
-            metric.set(label, info[name])
+            metric.set(info[name], labels: label)
           end
         end
 
         timekeys = info["buffer_timekeys"]
         if timekeys && !timekeys.empty?
-          @buffer_newest_timekey.set(label, timekeys.max)
-          @buffer_oldest_timekey.set(label, timekeys.min)
+          @buffer_newest_timekey.set(timekeys.max, labels: label)
+          @buffer_oldest_timekey.set(timekeys.min, labels: label)
         end
       end
     end
@@ -95,5 +95,13 @@ module Fluent::Plugin
         type: plugin_info["type"],
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :plugin_category, :type])
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -144,7 +144,16 @@ module Fluent::Plugin
 
         # output metrics
         'retry_count' => @metrics[:retry_counts],
+        # Needed since Fluentd v1.14 due to metrics extensions.
+        'num_errors' => @metrics[:num_errors],
+        'write_count' => @metrics[:write_count],
+        'emit_count' => @metrics[:emit_count],
+        'emit_records' => @metrics[:emit_records],
+        'rollback_count' => @metrics[:rollback_count],
+        'flush_time_count' => @metrics[:flush_time_count],
+        'slow_flush_count' => @metrics[:slow_flush_count],
       }
+      # No needed for Fluentd v1.14 but leave as-is for backward compatibility.
       instance_vars_info = {
         num_errors: @metrics[:num_errors],
         write_count: @metrics[:write_count],
@@ -161,9 +170,9 @@ module Fluent::Plugin
         monitor_info.each do |name, metric|
           if info[name]
             if metric.is_a?(::Prometheus::Client::Gauge)
-              metric.set(label, info[name])
+              metric.set(info[name], labels: label)
             elsif metric.is_a?(::Prometheus::Client::Counter)
-              metric.increment(label, info[name] - metric.get(label))
+              metric.increment(by: info[name] - metric.get(labels: label), labels: label)
             end
           end
         end
@@ -172,9 +181,9 @@ module Fluent::Plugin
           instance_vars_info.each do |name, metric|
             if info['instance_variables'][name]
               if metric.is_a?(::Prometheus::Client::Gauge)
-                metric.set(label, info['instance_variables'][name])
+                metric.set(info['instance_variables'][name], labels: label)
               elsif metric.is_a?(::Prometheus::Client::Counter)
-                metric.increment(label, info['instance_variables'][name] - metric.get(label))
+                metric.increment(by: info['instance_variables'][name] - metric.get(labels: label), labels: label)
               end
             end
           end
@@ -193,7 +202,7 @@ module Fluent::Plugin
           if next_time && start_time
             wait = next_time - start_time
           end
-          @metrics[:retry_wait].set(label, wait.to_f)
+          @metrics[:retry_wait].set(wait.to_f, labels: label)
         end
       end
     end
@@ -209,7 +218,7 @@ module Fluent::Plugin
       if @registry.exist?(name)
         @registry.get(name)
       else
-        @registry.gauge(name, docstring)
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
       end
     end
 
@@ -217,7 +226,7 @@ module Fluent::Plugin
       if @registry.exist?(name)
         @registry.get(name)
       else
-        @registry.public_send(@gauge_or_counter, name, docstring)
+        @registry.public_send(@gauge_or_counter, name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
       end
     end
   end

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -73,8 +73,8 @@ module Fluent::Plugin
           # Very fragile implementation
           pe = watcher.instance_variable_get(:@pe)
           label = labels(info, watcher.path)
-          @metrics[:inode].set(label, pe.read_inode)
-          @metrics[:position].set(label, pe.read_pos)
+          @metrics[:inode].set(pe.read_inode, labels: label)
+          @metrics[:position].set(pe.read_pos, labels: label)
         end
       end
     end
@@ -91,7 +91,7 @@ module Fluent::Plugin
       if @registry.exist?(name)
         @registry.get(name)
       else
-        @registry.gauge(name, docstring)
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type, :path])
       end
     end
   end

data/lib/fluent/plugin/prometheus.rb

@@ -85,6 +85,18 @@ module Fluent
         Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
       end
 
+      def stringify_keys(hash_to_stringify)
+        # Adapted from: https://www.jvt.me/posts/2019/09/07/ruby-hash-keys-string-symbol/
+        hash_to_stringify.map do |k,v|
+          value_or_hash = if v.instance_of? Hash
+                            stringify_keys(v)
+                          else
+                            v
+                          end
+          [k.to_s, value_or_hash]
+        end.to_h
+      end
+
       def configure(conf)
         super
         @placeholder_values = {}
@@ -99,6 +111,7 @@ module Fluent
           'worker_id' => fluentd_worker_id,
         }
 
+        record = stringify_keys(record)
         placeholders = record.merge(@placeholder_values[tag])
         expander = @placeholder_expander_builder.build(placeholders)
         metrics.each do |metric|
@@ -119,6 +132,7 @@ module Fluent
         }
 
         es.each do |time, record|
+          record = stringify_keys(record)
           placeholders = record.merge(placeholder_values)
           expander = @placeholder_expander_builder.build(placeholders)
           metrics.each do |metric|
@@ -188,7 +202,7 @@ module Fluent
           end
 
           begin
-            @gauge = registry.gauge(element['name'].to_sym, element['desc'])
+            @gauge = registry.gauge(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
           end
@@ -201,7 +215,7 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @gauge.set(labels(record, expander), value)
+            @gauge.set(value, labels: labels(record, expander))
           end
         end
       end
@@ -210,7 +224,7 @@ module Fluent
         def initialize(element, registry, labels)
           super
           begin
-            @counter = registry.counter(element['name'].to_sym, element['desc'])
+            @counter = registry.counter(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
           end
@@ -229,7 +243,7 @@ module Fluent
           # ignore if record value is nil
           return if value.nil?
 
-          @counter.increment(labels(record, expander), value)
+          @counter.increment(by: value, labels: labels(record, expander))
         end
       end
 
@@ -241,7 +255,7 @@ module Fluent
           end
 
           begin
-            @summary = registry.summary(element['name'].to_sym, element['desc'])
+            @summary = registry.summary(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
           end
@@ -254,7 +268,7 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @summary.observe(labels(record, expander), value)
+            @summary.observe(value, labels: labels(record, expander))
           end
         end
       end
@@ -271,9 +285,9 @@ module Fluent
               buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
                 e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
               end
-              @histogram = registry.histogram(element['name'].to_sym, element['desc'], {}, buckets)
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys, buckets: buckets)
             else
-              @histogram = registry.histogram(element['name'].to_sym, element['desc'])
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
             end
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
@@ -287,7 +301,7 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @histogram.observe(labels(record, expander), value)
+            @histogram.observe(value, labels: labels(record, expander))
           end
         end
       end

data/spec/fluent/plugin/out_prometheus_spec.rb

@@ -40,4 +40,29 @@ describe Fluent::Plugin::PrometheusOutput do
 
     it_behaves_like 'instruments record'
   end
+
+  describe '#run with symbolized keys' do
+    let(:message) { {:foo => 100, :bar => 100, :baz => 100, :qux => 10} }
+
+    context 'simple config' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
+
+      it 'adds a new counter metric' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
+      end
+    end
+
+    it_behaves_like 'instruments record'
+  end
 end

data/spec/fluent/plugin/shared.rb

@@ -178,20 +178,19 @@ shared_examples_for 'instruments record' do
 
     it 'instruments counter metric' do
       expect(counter.type).to eq(:counter)
-      expect(counter.get({test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
-      expect(counter_with_accessor.get({test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
+      expect(counter.get(labels: {test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
+      expect(counter_with_accessor.get(labels: {test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
     end
 
     it 'instruments gauge metric' do
       expect(gauge.type).to eq(:gauge)
-      expect(gauge.get({test_key: 'test_value', key: 'foo2'})).to eq(100)
+      expect(gauge.get(labels: {test_key: 'test_value', key: 'foo2'})).to eq(100)
     end
 
     it 'instruments summary metric' do
       expect(summary.type).to eq(:summary)
-      expect(summary.get({test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
-      expect(summary.get({test_key: 'test_value', key: 'foo3'})[0.99]).to eq(100)
-      expect(summary_with_accessor.get({test_key: 'test_value', key: 'foo5'})[0.99]).to eq(100)
+      expect(summary.get(labels: {test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
+      expect(summary_with_accessor.get(labels: {test_key: 'test_value', key: 'foo5'})["sum"]).to eq(100)
     end
 
     it 'instruments histogram metric' do
@@ -200,8 +199,8 @@ shared_examples_for 'instruments record' do
       end
 
       expect(histogram.type).to eq(:histogram)
-      expect(histogram.get({test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
-      expect(histogram.get({test_key: 'test_value', key: 'foo4'})[10]).to eq(5) # 4 + `es` in before
+      expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
+      expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})["10"]).to eq(5) # 4 + `es` in before
     end
   end
 
@@ -231,7 +230,7 @@ shared_examples_for 'instruments record' do
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       key, _ = counter.values.find {|k,v| v ==  100 }
       expect(key).to be_kind_of(Hash)
-      expect(key[:foo]).to eq(100)
+      expect(key[:foo]).to eq("100")
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.8.4
+  version: 2.0.2
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-24 00:00:00.000000000 Z
+date: 2021-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +107,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -158,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.22
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.