fluent-plugin-prometheus 1.8.0 → 1.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f98778420f98da85a1325661e897dce930a9eaec03675f3112ca9223a8435a4
-  data.tar.gz: 864fd85893d2cbe49c2aa5d8e59026405098d9075e54499cbe3efc8e30025a3d
+  metadata.gz: 5679e5c8d8872be0448075a8c1e05c395b1f9c06a0267f568a788c77dc12dc62
+  data.tar.gz: 786191b3092cf3745ab97425e4aebb4b1cb524c907433d0f4e091dbb6d14b7f9
 SHA512:
-  metadata.gz: ae6ec1075cf1fa4ab375a888c0a37c2653cf067e25a87b588ef5b5a332aa090cb1794b9add155314e4b867a188cca55450b064ea04ec99cb9299ad2fcfbde37b
-  data.tar.gz: c0d269902c7efff6fe7cb153c853fc17eaf9b0a740994ece3701854308fff45af2fa906c17efa2d0af32f882dfdcdd40340c5f3b8ad21a58469655104064a0c1
+  metadata.gz: 1b7c49f483ad596641edf01226e4282db10b72dd0117867884d0df4cf6ca0cf23c48e54eab6a2a22ca3ecae845ad8edda6e75d3532853a200b7a57ac8caca423
+  data.tar.gz: 344b9a363c49067056c4366f46b6b9f3bddf955e22a0155215158b13420027479c0a5e41da1a86b2baaf71e0b01256d90d0a0c8269c322ebe3616c861ce35a98

data/ChangeLog

@@ -0,0 +1,27 @@
+Release 1.8.5 - 2020/11/24
+
+	* in_prometheus_monitor: Support USR2 reload
+
+Release 1.8.4 - 2020/09/24
+
+	* in_prometheus_output_monitor: Add gauge_all parameter
+
+Release 1.8.3 - 2020/08/24
+
+	* Fix resourcr leak in async-http based server
+
+Release 1.8.2 - 2020/07/17
+
+	* in_prometheus_output_monitor/in_prometheus_tail_monitor: Support USR2 reload
+
+Release 1.8.1 - 2020/07/06
+
+	* Fix aggregate bug with async-http
+
+Release 1.8.0 - 2020/04/17
+
+	* Use http_server helper
+	* Require fluentd v1.9.1 or later
+
+
+For older releases, see commits on github repository.

data/README.md

@@ -124,7 +124,7 @@ Metrics for output
 - `fluentd_output_status_emit_records`
 - `fluentd_output_status_write_count`
 - `fluentd_output_status_rollback_count`
-- `fluentd_output_status_flush_time_count` from fluentd v1.6.0
+- `fluentd_output_status_flush_time_count` in milliseconds from fluentd v1.6.0
 - `fluentd_output_status_slow_flush_count` from fluentd v1.6.0
 
 Metrics for buffer
@@ -152,6 +152,7 @@ More configuration parameters:
 
 - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
 - `interval`: interval to update monitor_agent information in seconds (default: 5)
+- `gauge_all`: Specify metric type. If `true`, use `gauge` type. If `false`, use `counter` type. Since v2, this parameter will be removed and use `counter` type.
 
 ### prometheus_tail_monitor input plugin
 

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.8.0"
+  spec.version       = "1.8.5"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}

data/lib/fluent/plugin/in_prometheus.rb

@@ -73,6 +73,14 @@ module Fluent::Plugin
         return
       end
 
+      begin
+        require 'async'
+        require 'fluent/plugin/in_prometheus/async_wrapper'
+        extend AsyncWrapper
+      rescue LoadError => _
+        # ignore
+      end
+
       tls_opt = if @ssl && @ssl['enable']
                   ssl_config = {}
 
@@ -185,7 +193,7 @@ module Fluent::Plugin
       full_result = PromMetricsAggregator.new
 
       send_request_to_each_worker do |resp|
-        if resp.is_a?(Net::HTTPSuccess)
+        if resp.code.to_s == '200'
           full_result.add_metrics(resp.body)
         end
       end
@@ -197,14 +205,14 @@ module Fluent::Plugin
 
     def send_request_to_each_worker
       bind = (@bind == '0.0.0.0') ? '127.0.0.1' : @bind
-      req = Net::HTTP::Get.new(@metrics_path)
       [*(@base_port...(@base_port + @num_workers))].each do |worker_port|
         do_request(host: bind, port: worker_port, secure: @secure) do |http|
-          yield(http.request(req))
+          yield(http.get(@metrics_path))
         end
       end
     end
 
+    # might be replaced by AsyncWrapper if async gem is installed
     def do_request(host:, port:, secure:)
       http = Net::HTTP.new(host, port)
 
@@ -214,7 +222,7 @@ module Fluent::Plugin
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
 
-       http.start do
+      http.start do
         yield(http)
       end
     end

data/lib/fluent/plugin/in_prometheus/async_wrapper.rb

@@ -0,0 +1,47 @@
+require 'async'
+
+module Fluent::Plugin
+  class PrometheusInput
+    module AsyncWrapper
+      def do_request(host:, port:, secure:)
+        endpoint =
+          if secure
+            context = OpenSSL::SSL::SSLContext.new
+            context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            Async::HTTP::Endpoint.parse("https://#{host}:#{port}", ssl_context: context)
+          else
+            Async::HTTP::Endpoint.parse("http://#{host}:#{port}")
+          end
+
+        Async::HTTP::Client.open(endpoint) do |client|
+          yield(AsyncHttpWrapper.new(client))
+        end
+      end
+
+      Response = Struct.new(:code, :body, :headers)
+
+      class AsyncHttpWrapper
+        def initialize(http)
+          @http = http
+        end
+
+        def get(path)
+          error = nil
+          response = Async::Task.current.async {
+            begin
+              @http.get(path)
+            rescue => e               # Async::Reactor rescue all error. handle it by itself
+              error = e
+            end
+          }.wait
+
+          if error
+            raise error
+          end
+
+          Response.new(response.status.to_s, response.read || '', response.headers)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_prometheus_monitor.rb

@@ -46,19 +46,19 @@ module Fluent::Plugin
     def start
       super
 
-      @buffer_newest_timekey = @registry.gauge(
+      @buffer_newest_timekey = get_gauge(
         :fluentd_status_buffer_newest_timekey,
         'Newest timekey in buffer.')
-      @buffer_oldest_timekey = @registry.gauge(
+      @buffer_oldest_timekey = get_gauge(
         :fluentd_status_buffer_oldest_timekey,
         'Oldest timekey in buffer.')
-      buffer_queue_length = @registry.gauge(
+      buffer_queue_length = get_gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
-      buffer_total_queued_size = @registry.gauge(
+      buffer_total_queued_size = get_gauge(
         :fluentd_status_buffer_total_bytes,
         'Current total size of queued buffers.')
-      retry_counts = @registry.gauge(
+      retry_counts = get_gauge(
         :fluentd_status_retry_count,
         'Current retry counts.')
 
@@ -95,5 +95,13 @@ module Fluent::Plugin
         type: plugin_info["type"],
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring)
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_output_monitor.rb

@@ -1,15 +1,16 @@
-require 'fluent/input'
+require 'fluent/plugin/input'
 require 'fluent/plugin/in_monitor_agent'
 require 'fluent/plugin/prometheus'
 
 module Fluent::Plugin
-  class PrometheusOutputMonitorInput < Fluent::Input
+  class PrometheusOutputMonitorInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus_output_monitor', self)
     include Fluent::Plugin::PrometheusLabelParser
 
     helpers :timer
 
     config_param :interval, :time, default: 5
+    config_param :gauge_all, :bool, default: true
     attr_reader :registry
 
     MONITOR_IVARS = [
@@ -53,12 +54,9 @@ module Fluent::Plugin
         @base_labels[key] = expander.expand(value)
       end
 
-      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
-        # from v0.14.6
-        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
-      else
-        @monitor_agent = Fluent::MonitorAgentInput.new
-      end
+      @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
+
+      @gauge_or_counter = @gauge_all ? :gauge : :counter
     end
 
     def start
@@ -66,57 +64,57 @@ module Fluent::Plugin
 
       @metrics = {
         # Buffer metrics
-        buffer_total_queued_size: @registry.gauge(
+        buffer_total_queued_size: get_gauge(
           :fluentd_output_status_buffer_total_bytes,
           'Current total size of stage and queue buffers.'),
-        buffer_stage_length: @registry.gauge(
+        buffer_stage_length: get_gauge(
           :fluentd_output_status_buffer_stage_length,
           'Current length of stage buffers.'),
-        buffer_stage_byte_size: @registry.gauge(
+        buffer_stage_byte_size: get_gauge(
           :fluentd_output_status_buffer_stage_byte_size,
           'Current total size of stage buffers.'),
-        buffer_queue_length: @registry.gauge(
+        buffer_queue_length: get_gauge(
           :fluentd_output_status_buffer_queue_length,
           'Current length of queue buffers.'),
-        buffer_queue_byte_size: @registry.gauge(
+        buffer_queue_byte_size: get_gauge(
           :fluentd_output_status_queue_byte_size,
           'Current total size of queue buffers.'),
-        buffer_available_buffer_space_ratios: @registry.gauge(
+        buffer_available_buffer_space_ratios: get_gauge(
           :fluentd_output_status_buffer_available_space_ratio,
           'Ratio of available space in buffer.'),
-        buffer_newest_timekey: @registry.gauge(
+        buffer_newest_timekey: get_gauge(
           :fluentd_output_status_buffer_newest_timekey,
           'Newest timekey in buffer.'),
-        buffer_oldest_timekey: @registry.gauge(
+        buffer_oldest_timekey: get_gauge(
           :fluentd_output_status_buffer_oldest_timekey,
           'Oldest timekey in buffer.'),
 
         # Output metrics
-        retry_counts: @registry.gauge(
+        retry_counts: get_gauge_or_counter(
           :fluentd_output_status_retry_count,
           'Current retry counts.'),
-        num_errors: @registry.gauge(
+        num_errors: get_gauge_or_counter(
           :fluentd_output_status_num_errors,
           'Current number of errors.'),
-        emit_count: @registry.gauge(
+        emit_count: get_gauge_or_counter(
           :fluentd_output_status_emit_count,
           'Current emit counts.'),
-        emit_records: @registry.gauge(
+        emit_records: get_gauge_or_counter(
           :fluentd_output_status_emit_records,
           'Current emit records.'),
-        write_count: @registry.gauge(
+        write_count: get_gauge_or_counter(
           :fluentd_output_status_write_count,
           'Current write counts.'),
-        rollback_count: @registry.gauge(
+        rollback_count: get_gauge(
           :fluentd_output_status_rollback_count,
           'Current rollback counts.'),
-        flush_time_count: @registry.gauge(
+        flush_time_count: get_gauge_or_counter(
           :fluentd_output_status_flush_time_count,
           'Total flush time.'),
-        slow_flush_count: @registry.gauge(
+        slow_flush_count: get_gauge_or_counter(
           :fluentd_output_status_slow_flush_count,
           'Current slow flush counts.'),
-        retry_wait: @registry.gauge(
+        retry_wait: get_gauge(
           :fluentd_output_status_retry_wait,
           'Current retry wait'),
       }
@@ -162,14 +160,22 @@ module Fluent::Plugin
 
         monitor_info.each do |name, metric|
           if info[name]
-            metric.set(label, info[name])
+            if metric.is_a?(::Prometheus::Client::Gauge)
+              metric.set(label, info[name])
+            elsif metric.is_a?(::Prometheus::Client::Counter)
+              metric.increment(label, info[name] - metric.get(label))
+            end
           end
         end
 
         if info['instance_variables']
           instance_vars_info.each do |name, metric|
             if info['instance_variables'][name]
-              metric.set(label, info['instance_variables'][name])
+              if metric.is_a?(::Prometheus::Client::Gauge)
+                metric.set(label, info['instance_variables'][name])
+              elsif metric.is_a?(::Prometheus::Client::Counter)
+                metric.increment(label, info['instance_variables'][name] - metric.get(label))
+              end
             end
           end
         end
@@ -198,5 +204,21 @@ module Fluent::Plugin
         type: plugin_info["type"],
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring)
+      end
+    end
+
+    def get_gauge_or_counter(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.public_send(@gauge_or_counter, name, docstring)
+      end
+    end
   end
 end

data/lib/fluent/plugin/in_prometheus_tail_monitor.rb

@@ -38,22 +38,17 @@ module Fluent::Plugin
         @base_labels[key] = expander.expand(value)
       end
 
-      if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
-        # from v0.14.6
-        @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
-      else
-        @monitor_agent = Fluent::MonitorAgentInput.new
-      end
+      @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
     end
 
     def start
       super
 
       @metrics = {
-        position: @registry.gauge(
+        position: get_gauge(
           :fluentd_tail_file_position,
           'Current position of file.'),
-        inode: @registry.gauge(
+        inode: get_gauge(
           :fluentd_tail_file_inode,
           'Current inode of file.'),
       }
@@ -91,5 +86,13 @@ module Fluent::Plugin
         path: path,
       )
     end
+
+    def get_gauge(name, docstring)
+      if @registry.exist?(name)
+        @registry.get(name)
+      else
+        @registry.gauge(name, docstring)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.8.5
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-21 00:00:00.000000000 Z
+date: 2020-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -110,6 +110,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- ChangeLog
 - Gemfile
 - LICENSE
 - README.md
@@ -117,6 +118,7 @@ files:
 - fluent-plugin-prometheus.gemspec
 - lib/fluent/plugin/filter_prometheus.rb
 - lib/fluent/plugin/in_prometheus.rb
+- lib/fluent/plugin/in_prometheus/async_wrapper.rb
 - lib/fluent/plugin/in_prometheus_monitor.rb
 - lib/fluent/plugin/in_prometheus_output_monitor.rb
 - lib/fluent/plugin/in_prometheus_tail_monitor.rb