fluent-plugin-prometheus 1.8.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f48e673585638a68fa128172db768d7f35035a04f4d55bb98737e7cfd5d7cdf2
-  data.tar.gz: 1030ddd92d55c5e6603e49933760eadac266f5081cbb54169b0ebc237bcfe933
+  metadata.gz: e3250eed2a2f1156410b9e67005f7e23fdb822111dd780690ce61ee4967db7cf
+  data.tar.gz: dd1d3aa0defd1ac16f7d4e6e75dd7b6b7c9caff11f9e16288dc9b787d8f3ef9e
 SHA512:
-  metadata.gz: adb129f3a83c6be98247e2180d6cb26037a1e8e32e10c555232cf9d267867b4b5b6d5168adebbb9b8ed16b1193c9ff7e8879b379ccb92406b66cb2abc67075e1
-  data.tar.gz: 2e7a8eecf131d31d08266507620939f7f9b73613d39009d6c2349bbb166b778dae82baa6ea84021793b5374c1372b5cd0673b126ec5b6981c9dfd368c8da0922
+  metadata.gz: ed531d2e8e383f54f3ff68c695cb63c0a2922a17ced661a109e50c50546a571aeb45747cba079274b451f41bdd506737d43c035140e28db7e5a427a83a1126b9
+  data.tar.gz: 7fab4fb99f73a770f0a87d88341078f089e78fd865ba28aba4634d04dfa5b5baa7ab8e7b1942fdc5ef96f6306ef379d7da2335bc7856ea150cb69c8749635f94

data/.github/workflows/linux.yml

@@ -0,0 +1,26 @@
+name: linux
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake spec

data/ChangeLog

@@ -1,3 +1,23 @@
+Release 2.0.0 - 2021/02/18
+
+	* Update prometheus-client dependency to 2.1.0 or later
+
+Release 1.8.5 - 2020/11/24
+
+	* in_prometheus_monitor: Support USR2 reload
+
+Release 1.8.4 - 2020/09/24
+
+	* in_prometheus_output_monitor: Add gauge_all parameter
+
+Release 1.8.3 - 2020/08/24
+
+	* Fix resourcr leak in async-http based server
+
+Release 1.8.2 - 2020/07/17
+
+	* in_prometheus_output_monitor/in_prometheus_tail_monitor: Support USR2 reload
+
 Release 1.8.1 - 2020/07/06
 
 	* Fix aggregate bug with async-http

data/README.md

@@ -152,6 +152,7 @@ More configuration parameters:
 
 - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 - `interval`: interval to update monitor_agent information in seconds (default: 5)
+- `gauge_all`: Specify metric type. If `true`, use `gauge` type. If `false`, use `counter` type. Since v2, this parameter will be removed and use `counter` type.
 
 ### prometheus_tail_monitor input plugin
 

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.8.1"
+  spec.version       = "2.0.0"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "fluentd", ">= 1.9.1", "< 2"
-  spec.add_dependency "prometheus-client", "< 0.10"
+  spec.add_dependency "prometheus-client", ">= 2.1.0"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

data/lib/fluent/plugin/in_prometheus/async_wrapper.rb

@@ -13,8 +13,9 @@ module Fluent::Plugin
             Async::HTTP::Endpoint.parse("http://#{host}:#{port}")
           end
 
-        client = Async::HTTP::Client.new(endpoint)
-        yield(AsyncHttpWrapper.new(client))
+        Async::HTTP::Client.open(endpoint) do |client|
+          yield(AsyncHttpWrapper.new(client))
+        end
       end
 
       Response = Struct.new(:code, :body, :headers)
@@ -38,7 +39,7 @@ module Fluent::Plugin
             raise error
           end
 
-          Response.new(response.status.to_s, response.body.read, response.headers)
+          Response.new(response.status.to_s, response.read || '', response.headers)
         end
       end
     end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -46,19 +46,19 @@ module Fluent::Plugin
     def start
       super
 
-      @buffer_newest_timekey = @registry.gauge(
+      @buffer_newest_timekey = get_gauge(
         :fluentd_status_buffer_newest_timekey,
         'Newest timekey in buffer.')
-      @buffer_oldest_timekey = @registry.gauge(
+      @buffer_oldest_timekey = get_gauge(
         :fluentd_status_buffer_oldest_timekey,
         'Oldest timekey in buffer.')
-      buffer_queue_length = @registry.gauge(
+      buffer_queue_length = get_gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
-      buffer_total_queued_size = @registry.gauge(
+      buffer_total_queued_size = get_gauge(
         :fluentd_status_buffer_total_bytes,
         'Current total size of queued buffers.')
-      retry_counts = @registry.gauge(
+      retry_counts = get_gauge(
         :fluentd_status_retry_count,
         'Current retry counts.')
 
@@ -76,14 +76,14 @@ module Fluent::Plugin
 
         @monitor_info.each do |name, metric|
           if info[name]
-            metric.set(label, info[name])
+            metric.set(info[name], labels: label)
           end
         end
 
         timekeys = info["buffer_timekeys"]
         if timekeys && !timekeys.empty?
-          @buffer_newest_timekey.set(label, timekeys.max)
-          @buffer_oldest_timekey.set(label, timekeys.min)
+          @buffer_newest_timekey.set(timekeys.max, labels: label)
+          @buffer_oldest_timekey.set(timekeys.min, labels: label)
         end
       end
     end
@@ -95,5 +95,13 @@ module Fluent::Plugin
         type: plugin_info["type"],
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :plugin_category, :type])
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -1,15 +1,16 @@
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/plugin/in_monitor_agent'
 require 'fluent/plugin/prometheus'
 
 module Fluent::Plugin
-  class PrometheusOutputMonitorInput < Fluent::Input
+  class PrometheusOutputMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_output_monitor', self)
     include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
     config_param :interval, :time, default: 5
+    config_param :gauge_all, :bool, default: true
     attr_reader :registry
 
     MONITOR_IVARS = [
@@ -53,12 +54,9 @@ module Fluent::Plugin
         @base_labels[key] = expander.expand(value)
       end
 
-      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
-        # from v0.14.6
-        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
-      else
-        @monitor_agent = Fluent::MonitorAgentInput.new
-      end
+      @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+
+      @gauge_or_counter = @gauge_all ? :gauge : :counter
     end
 
     def start
@@ -66,57 +64,57 @@ module Fluent::Plugin
 
       @metrics = {
         # Buffer metrics
-        buffer_total_queued_size: @registry.gauge(
+        buffer_total_queued_size: get_gauge(
           :fluentd_output_status_buffer_total_bytes,
           'Current total size of stage and queue buffers.'),
-        buffer_stage_length: @registry.gauge(
+        buffer_stage_length: get_gauge(
           :fluentd_output_status_buffer_stage_length,
           'Current length of stage buffers.'),
-        buffer_stage_byte_size: @registry.gauge(
+        buffer_stage_byte_size: get_gauge(
           :fluentd_output_status_buffer_stage_byte_size,
           'Current total size of stage buffers.'),
-        buffer_queue_length: @registry.gauge(
+        buffer_queue_length: get_gauge(
           :fluentd_output_status_buffer_queue_length,
           'Current length of queue buffers.'),
-        buffer_queue_byte_size: @registry.gauge(
+        buffer_queue_byte_size: get_gauge(
           :fluentd_output_status_queue_byte_size,
           'Current total size of queue buffers.'),
-        buffer_available_buffer_space_ratios: @registry.gauge(
+        buffer_available_buffer_space_ratios: get_gauge(
           :fluentd_output_status_buffer_available_space_ratio,
           'Ratio of available space in buffer.'),
-        buffer_newest_timekey: @registry.gauge(
+        buffer_newest_timekey: get_gauge(
           :fluentd_output_status_buffer_newest_timekey,
           'Newest timekey in buffer.'),
-        buffer_oldest_timekey: @registry.gauge(
+        buffer_oldest_timekey: get_gauge(
           :fluentd_output_status_buffer_oldest_timekey,
           'Oldest timekey in buffer.'),
 
         # Output metrics
-        retry_counts: @registry.gauge(
+        retry_counts: get_gauge_or_counter(
           :fluentd_output_status_retry_count,
           'Current retry counts.'),
-        num_errors: @registry.gauge(
+        num_errors: get_gauge_or_counter(
           :fluentd_output_status_num_errors,
           'Current number of errors.'),
-        emit_count: @registry.gauge(
+        emit_count: get_gauge_or_counter(
           :fluentd_output_status_emit_count,
           'Current emit counts.'),
-        emit_records: @registry.gauge(
+        emit_records: get_gauge_or_counter(
           :fluentd_output_status_emit_records,
           'Current emit records.'),
-        write_count: @registry.gauge(
+        write_count: get_gauge_or_counter(
           :fluentd_output_status_write_count,
           'Current write counts.'),
-        rollback_count: @registry.gauge(
+        rollback_count: get_gauge(
           :fluentd_output_status_rollback_count,
           'Current rollback counts.'),
-        flush_time_count: @registry.gauge(
+        flush_time_count: get_gauge_or_counter(
           :fluentd_output_status_flush_time_count,
           'Total flush time.'),
-        slow_flush_count: @registry.gauge(
+        slow_flush_count: get_gauge_or_counter(
           :fluentd_output_status_slow_flush_count,
           'Current slow flush counts.'),
-        retry_wait: @registry.gauge(
+        retry_wait: get_gauge(
           :fluentd_output_status_retry_wait,
           'Current retry wait'),
       }
@@ -162,14 +160,22 @@ module Fluent::Plugin
 
         monitor_info.each do |name, metric|
           if info[name]
-            metric.set(label, info[name])
+            if metric.is_a?(::Prometheus::Client::Gauge)
+              metric.set(info[name], labels: label)
+            elsif metric.is_a?(::Prometheus::Client::Counter)
+              metric.increment(by: info[name] - metric.get(labels: label), labels: label)
+            end
           end
         end
 
         if info['instance_variables']
           instance_vars_info.each do |name, metric|
             if info['instance_variables'][name]
-              metric.set(label, info['instance_variables'][name])
+              if metric.is_a?(::Prometheus::Client::Gauge)
+                metric.set(info['instance_variables'][name], labels: label)
+              elsif metric.is_a?(::Prometheus::Client::Counter)
+                metric.increment(by: info['instance_variables'][name] - metric.get(labels: label), labels: label)
+              end
             end
           end
         end
@@ -187,7 +193,7 @@ module Fluent::Plugin
           if next_time && start_time
             wait = next_time - start_time
           end
-          @metrics[:retry_wait].set(label, wait.to_f)
+          @metrics[:retry_wait].set(wait.to_f, labels: label)
         end
       end
     end
@@ -198,5 +204,21 @@ module Fluent::Plugin
         type: plugin_info["type"],
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
+      end
+    end
+
+    def get_gauge_or_counter(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.public_send(@gauge_or_counter, name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -38,22 +38,17 @@ module Fluent::Plugin
         @base_labels[key] = expander.expand(value)
       end
 
-      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
-        # from v0.14.6
-        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
-      else
-        @monitor_agent = Fluent::MonitorAgentInput.new
-      end
+      @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
     end
 
     def start
       super
 
       @metrics = {
-        position: @registry.gauge(
+        position: get_gauge(
           :fluentd_tail_file_position,
           'Current position of file.'),
-        inode: @registry.gauge(
+        inode: get_gauge(
           :fluentd_tail_file_inode,
           'Current inode of file.'),
       }
@@ -78,8 +73,8 @@ module Fluent::Plugin
           # Very fragile implementation
           pe = watcher.instance_variable_get(:@pe)
           label = labels(info, watcher.path)
-          @metrics[:inode].set(label, pe.read_inode)
-          @metrics[:position].set(label, pe.read_pos)
+          @metrics[:inode].set(pe.read_inode, labels: label)
+          @metrics[:position].set(pe.read_pos, labels: label)
         end
       end
     end
@@ -91,5 +86,13 @@ module Fluent::Plugin
         path: path,
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type, :path])
+      end
+    end
   end
 end

data/lib/fluent/plugin/prometheus.rb

@@ -188,7 +188,7 @@ module Fluent
           end
 
           begin
-            @gauge = registry.gauge(element['name'].to_sym, element['desc'])
+            @gauge = registry.gauge(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
           end
@@ -201,7 +201,7 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @gauge.set(labels(record, expander), value)
+            @gauge.set(value, labels: labels(record, expander))
           end
         end
       end
@@ -210,7 +210,7 @@ module Fluent
         def initialize(element, registry, labels)
           super
           begin
-            @counter = registry.counter(element['name'].to_sym, element['desc'])
+            @counter = registry.counter(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
           end
@@ -229,7 +229,7 @@ module Fluent
           # ignore if record value is nil
           return if value.nil?
 
-          @counter.increment(labels(record, expander), value)
+          @counter.increment(by: value, labels: labels(record, expander))
         end
       end
 
@@ -241,7 +241,7 @@ module Fluent
           end
 
           begin
-            @summary = registry.summary(element['name'].to_sym, element['desc'])
+            @summary = registry.summary(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
           end
@@ -254,7 +254,7 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @summary.observe(labels(record, expander), value)
+            @summary.observe(value, labels: labels(record, expander))
           end
         end
       end
@@ -271,9 +271,9 @@ module Fluent
               buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
                 e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
               end
-              @histogram = registry.histogram(element['name'].to_sym, element['desc'], {}, buckets)
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys, buckets: buckets)
             else
-              @histogram = registry.histogram(element['name'].to_sym, element['desc'])
+              @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
             end
           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
             @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
@@ -287,7 +287,7 @@ module Fluent
             value = @key.call(record)
           end
           if value
-            @histogram.observe(labels(record, expander), value)
+            @histogram.observe(value, labels: labels(record, expander))
           end
         end
       end

data/spec/fluent/plugin/shared.rb

@@ -178,20 +178,19 @@ shared_examples_for 'instruments record' do
 
     it 'instruments counter metric' do
       expect(counter.type).to eq(:counter)
-      expect(counter.get({test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
-      expect(counter_with_accessor.get({test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
+      expect(counter.get(labels: {test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
+      expect(counter_with_accessor.get(labels: {test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
     end
 
     it 'instruments gauge metric' do
       expect(gauge.type).to eq(:gauge)
-      expect(gauge.get({test_key: 'test_value', key: 'foo2'})).to eq(100)
+      expect(gauge.get(labels: {test_key: 'test_value', key: 'foo2'})).to eq(100)
     end
 
     it 'instruments summary metric' do
       expect(summary.type).to eq(:summary)
-      expect(summary.get({test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
-      expect(summary.get({test_key: 'test_value', key: 'foo3'})[0.99]).to eq(100)
-      expect(summary_with_accessor.get({test_key: 'test_value', key: 'foo5'})[0.99]).to eq(100)
+      expect(summary.get(labels: {test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
+      expect(summary_with_accessor.get(labels: {test_key: 'test_value', key: 'foo5'})["sum"]).to eq(100)
     end
 
     it 'instruments histogram metric' do
@@ -200,8 +199,8 @@ shared_examples_for 'instruments record' do
       end
 
       expect(histogram.type).to eq(:histogram)
-      expect(histogram.get({test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
-      expect(histogram.get({test_key: 'test_value', key: 'foo4'})[10]).to eq(5) # 4 + `es` in before
+      expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
+      expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})["10"]).to eq(5) # 4 + `es` in before
     end
   end
 
@@ -231,7 +230,7 @@ shared_examples_for 'instruments record' do
       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
       key, _ = counter.values.find {|k,v| v ==  100 }
       expect(key).to be_kind_of(Hash)
-      expect(key[:foo]).to eq(100)
+      expect(key[:foo]).to eq("100")
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.8.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-05 00:00:00.000000000 Z
+date: 2021-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,16 @@ dependencies:
   name: prometheus-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "<"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +107,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -158,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A fluent plugin that collects metrics and exposes for Prometheus.