fluent-plugin-prometheus 1.7.3 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f14f2b2162bace2a18a80f885df28261ed260a7b8e865a12e7828d46af9b02f
-  data.tar.gz: 20749cd6ee743ac3726719e654e1a4f046aa2f953245604e6aa75cd87ed00e62
+  metadata.gz: 3f98778420f98da85a1325661e897dce930a9eaec03675f3112ca9223a8435a4
+  data.tar.gz: 864fd85893d2cbe49c2aa5d8e59026405098d9075e54499cbe3efc8e30025a3d
 SHA512:
-  metadata.gz: 28ca37140a4b634665a9ba184c1615380bc7c458ed745d5548cb75b92d75d3b1069cd342d3119785a11d1beb369e7b41b5fa0319021bb0f69400ffe327dea83e
-  data.tar.gz: b04770fec96856fc37b6943c7129c5e8ef16ce8b1415b425932e4419594e58a3255406de54e3aad1c28af1e3d62074391bd2b40e4aded8b5f598fbece53cab82
+  metadata.gz: ae6ec1075cf1fa4ab375a888c0a37c2653cf067e25a87b588ef5b5a332aa090cb1794b9add155314e4b867a188cca55450b064ea04ec99cb9299ad2fcfbde37b
+  data.tar.gz: c0d269902c7efff6fe7cb153c853fc17eaf9b0a740994ece3701854308fff45af2fa906c17efa2d0af32f882dfdcdd40340c5f3b8ad21a58469655104064a0c1

data/README.md

@@ -8,9 +8,13 @@ A fluent plugin that instruments metrics from records and exposes them via web i
 
 | fluent-plugin-prometheus | fluentd    | ruby   |
 |--------------------------|------------|--------|
-| 1.x.y                    | >= v0.14.8 | >= 2.1 |
+| 1.x.y                    | >= v1.9.1  | >= 2.4 |
+| 1.[0-7].y                | >= v0.14.8 | >= 2.1 |
 | 0.x.y                    | >= v0.12.0 | >= 1.9 |
 
+Since v1.8.0, fluent-plugin-prometheus uses [http_server helper](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-http_server) to launch HTTP server.
+If you want to handle lots of connections, install `async-http` gem.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -63,6 +67,19 @@ More configuration parameters:
 When using multiple workers, each worker binds to port + `fluent_worker_id`.
 To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
 
+#### TLS setting
+
+Use `<trasnport tls>`. See [transport config article](https://docs.fluentd.org/configuration/transport-section) for more details.
+
+```
+<source>
+  @type prometheus
+  <transport tls>
+    # TLS parameters...
+  </transport
+</source>
+```
+
 ### prometheus_monitor input plugin
 
 This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](https://docs.fluentd.org/input/monitor_agent).
@@ -71,7 +88,7 @@ This plugin collects internal metrics in Fluentd. The metrics are similar to/par
 #### Exposed metrics
 
 - `fluentd_status_buffer_queue_length`
-- `fluentd_status_buffer_total_queued_size`
+- `fluentd_status_buffer_total_bytes`
 - `fluentd_status_retry_count`
 - `fluentd_status_buffer_newest_timekey` from fluentd v1.4.2
 - `fluentd_status_buffer_oldest_timekey` from fluentd v1.4.2

data/fluent-plugin-prometheus.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-prometheus"
-  spec.version       = "1.7.3"
+  spec.version       = "1.8.0"
   spec.authors       = ["Masahiro Sano"]
   spec.email         = ["sabottenda@gmail.com"]
   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "fluentd", ">= 0.14.20", "< 2"
+  spec.add_dependency "fluentd", ">= 1.9.1", "< 2"
   spec.add_dependency "prometheus-client", "< 0.10"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"

data/lib/fluent/plugin/in_prometheus.rb

@@ -2,13 +2,13 @@ require 'fluent/plugin/input'
 require 'fluent/plugin/prometheus'
 require 'fluent/plugin/prometheus_metrics'
 require 'net/http'
-require 'webrick'
+require 'openssl'
 
 module Fluent::Plugin
   class PrometheusInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('prometheus', self)
 
-    helpers :thread
+    helpers :thread, :http_server
 
     config_param :bind, :string, default: '0.0.0.0'
     config_param :port, :integer, default: 24231
@@ -17,30 +17,25 @@ module Fluent::Plugin
 
     desc 'Enable ssl configuration for the server'
     config_section :ssl, required: false, multi: false do
-      config_param :enable, :bool, default: false
+      config_param :enable, :bool, default: false, deprecated: 'Use <transport tls> section'
 
       desc 'Path to the ssl certificate in PEM format.  Read from file and added to conf as "SSLCertificate"'
-      config_param :certificate_path, :string, default: nil
+      config_param :certificate_path, :string, default: nil, deprecated: 'Use cert_path in <transport tls> section'
 
       desc 'Path to the ssl private key in PEM format.  Read from file and added to conf as "SSLPrivateKey"'
-      config_param :private_key_path, :string, default: nil
+      config_param :private_key_path, :string, default: nil, deprecated: 'Use private_key_path in <transport tls> section'
 
       desc 'Path to CA in PEM format.  Read from file and added to conf as "SSLCACertificateFile"'
-      config_param :ca_path, :string, default: nil
+      config_param :ca_path, :string, default: nil, deprecated: 'Use ca_path in <transport tls> section'
 
       desc 'Additional ssl conf for the server.  Ref: https://github.com/ruby/webrick/blob/master/lib/webrick/ssl.rb'
-      config_param :extra_conf, :hash, default: {:SSLCertName => [['CN','nobody'],['DC','example']]}, symbolize_keys: true
+      config_param :extra_conf, :hash, default: nil, symbolize_keys: true, deprecated: 'See http helper config'
     end
 
-    attr_reader :registry
-
-    attr_reader :num_workers
-    attr_reader :base_port
-    attr_reader :metrics_path
-
     def initialize
       super
       @registry = ::Prometheus::Client.registry
+      @secure = nil
     end
 
     def configure(conf)
@@ -55,6 +50,7 @@ module Fluent::Plugin
                   nil
                 end
       @num_workers = sysconf && sysconf.workers ? sysconf.workers : 1
+      @secure = @transport_config.protocol == :tls || (@ssl && @ssl['enable'])
 
       @base_port = @port
       @port += fluentd_worker_id
@@ -66,7 +62,63 @@ module Fluent::Plugin
 
     def start
       super
-      log.debug "listening prometheus http server on http://#{@bind}:#{@port}/#{@metrics_path} for worker#{fluentd_worker_id}"
+
+      scheme = @secure ? 'https' : 'http'
+      log.debug "listening prometheus http server on #{scheme}:://#{@bind}:#{@port}/#{@metrics_path} for worker#{fluentd_worker_id}"
+
+      proto = @secure ? :tls : :tcp
+
+      if @ssl && @ssl['enable'] && @ssl['extra_conf']
+        start_webrick
+        return
+      end
+
+      tls_opt = if @ssl && @ssl['enable']
+                  ssl_config = {}
+
+                  if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
+                    raise Fluent::ConfigError.new('both certificate_path and private_key_path must be defined')
+                  end
+
+                  if @ssl['certificate_path']
+                    ssl_config['cert_path'] = @ssl['certificate_path']
+                  end
+
+                  if @ssl['private_key_path']
+                    ssl_config['private_key_path'] = @ssl['private_key_path']
+                  end
+
+                  if @ssl['ca_path']
+                    ssl_config['ca_path'] = @ssl['ca_path']
+                    # Only ca_path is insecure in fluentd
+                    # https://github.com/fluent/fluentd/blob/2236ad45197ba336fd9faf56f442252c8b226f25/lib/fluent/plugin_helper/cert_option.rb#L68
+                    ssl_config['insecure'] = true
+                  end
+
+                  ssl_config
+                end
+
+      http_server_create_http_server(:in_prometheus_server, addr: @bind, port: @port, logger: log, proto: proto, tls_opts: tls_opt) do |server|
+        server.get(@metrics_path) { |_req| all_metrics }
+        server.get(@aggregated_metrics_path) { |_req| all_workers_metrics }
+      end
+    end
+
+    def shutdown
+      if @webrick_server
+        @webrick_server.shutdown
+        @webrick_server = nil
+      end
+      super
+    end
+
+    private
+
+    # For compatiblity because http helper can't support extra_conf option
+    def start_webrick
+      require 'webrick/https'
+      require 'webrick'
+
       config = {
         BindAddress: @bind,
         Port: @port,
@@ -74,90 +126,96 @@ module Fluent::Plugin
         Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
         AccessLog: [],
       }
-      unless @ssl.nil? || !@ssl['enable']
-        require 'webrick/https'
-        require 'openssl'
-        if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
-            raise RuntimeError.new("certificate_path and private_key_path most both be defined")
-        end
-        ssl_config = {
-            SSLEnable: true
-        }
-        if @ssl['certificate_path']
-          cert = OpenSSL::X509::Certificate.new(File.read(@ssl['certificate_path']))
-          ssl_config[:SSLCertificate] = cert
-        end
-        if @ssl['private_key_path']
-          key = OpenSSL::PKey::RSA.new(File.read(@ssl['private_key_path']))
-          ssl_config[:SSLPrivateKey] = key
-        end
-        ssl_config[:SSLCACertificateFile] = @ssl['ca_path'] if @ssl['ca_path']
-        ssl_config = ssl_config.merge(@ssl['extra_conf'])
-        config = ssl_config.merge(config)
+      if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
+        raise RuntimeError.new("certificate_path and private_key_path most both be defined")
+      end
+
+      ssl_config = {
+        SSLEnable: true,
+        SSLCertName: [['CN', 'nobody'], ['DC', 'example']]
+      }
+
+      if @ssl['certificate_path']
+        cert = OpenSSL::X509::Certificate.new(File.read(@ssl['certificate_path']))
+        ssl_config[:SSLCertificate] = cert
       end
+
+      if @ssl['private_key_path']
+        key = OpenSSL::PKey.read(@ssl['private_key_path'])
+        ssl_config[:SSLPrivateKey] = key
+      end
+
+      ssl_config[:SSLCACertificateFile] = @ssl['ca_path'] if @ssl['ca_path']
+      ssl_config = ssl_config.merge(@ssl['extra_conf']) if @ssl['extra_conf']
+      config = ssl_config.merge(config)
+
       @log.on_debug do
         @log.debug("WEBrick conf: #{config}")
       end
 
-      @server = WEBrick::HTTPServer.new(config)
-      @server.mount(@metrics_path, MonitorServlet, self)
-      @server.mount(@aggregated_metrics_path, MonitorServletAll, self)
-      thread_create(:in_prometheus) do
-        @server.start
+      @webrick_server = WEBrick::HTTPServer.new(config)
+      @webrick_server.mount_proc(@metrics_path) do |_req, res|
+        status, header, body = all_metrics
+        res.status = status
+        res['Content-Type'] = header['Content-Type']
+        res.body = body
+        res
       end
-    end
 
-    def shutdown
-      if @server
-        @server.shutdown
-        @server = nil
+      @webrick_server.mount_proc(@aggregated_metrics_path) do |_req, res|
+        status, header, body = all_workers_metrics
+        res.status = status
+        res['Content-Type'] = header['Content-Type']
+        res.body = body
+        res
+      end
+
+      thread_create(:in_prometheus_webrick) do
+        @webrick_server.start
       end
-      super
     end
 
-    class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
-      def initialize(server, prometheus)
-        @prometheus = prometheus
+    def all_metrics
+      [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, ::Prometheus::Client::Formats::Text.marshal(@registry)]
+    rescue => e
+      [500, { 'Content-Type' => 'text/plain' }, e.to_s]
+    end
+
+    def all_workers_metrics
+      full_result = PromMetricsAggregator.new
+
+      send_request_to_each_worker do |resp|
+        if resp.is_a?(Net::HTTPSuccess)
+          full_result.add_metrics(resp.body)
+        end
       end
 
-      def do_GET(req, res)
-        res.status = 200
-        res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
-        res.body = ::Prometheus::Client::Formats::Text.marshal(@prometheus.registry)
-      rescue
-        res.status = 500
-        res['Content-Type'] = 'text/plain'
-        res.body = $!.to_s
+      [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE }, full_result.get_metrics]
+    rescue => e
+      [500, { 'Content-Type' => 'text/plain' }, e.to_s]
+    end
+
+    def send_request_to_each_worker
+      bind = (@bind == '0.0.0.0') ? '127.0.0.1' : @bind
+      req = Net::HTTP::Get.new(@metrics_path)
+      [*(@base_port...(@base_port + @num_workers))].each do |worker_port|
+        do_request(host: bind, port: worker_port, secure: @secure) do |http|
+          yield(http.request(req))
+        end
       end
     end
 
-    class MonitorServletAll < WEBrick::HTTPServlet::AbstractServlet
-      def initialize(server, prometheus)
-        @prometheus = prometheus
+    def do_request(host:, port:, secure:)
+      http = Net::HTTP.new(host, port)
+
+      if secure
+        http.use_ssl = true
+        # target is our child process. so it's secure.
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
 
-      def do_GET(req, res)
-        res.status = 200
-        res['Content-Type'] = ::Prometheus::Client::Formats::Text::CONTENT_TYPE
-
-        full_result = PromMetricsAggregator.new
-        fluent_server_ip = @prometheus.bind == '0.0.0.0' ? '127.0.0.1' : @prometheus.bind
-        current_worker = 0
-        while current_worker < @prometheus.num_workers
-          Net::HTTP.start(fluent_server_ip, @prometheus.base_port + current_worker) do |http|
-            req = Net::HTTP::Get.new(@prometheus.metrics_path)
-            result = http.request(req)
-            if result.is_a?(Net::HTTPSuccess)
-              full_result.add_metrics(result.body)
-            end
-          end
-          current_worker += 1
-        end
-        res.body = full_result.get_metrics
-      rescue
-        res.status = 500
-        res['Content-Type'] = 'text/plain'
-        res.body = $!.to_s
+       http.start do
+        yield(http)
       end
     end
   end

data/spec/fluent/plugin/in_prometheus_spec.rb

@@ -6,11 +6,11 @@ require 'net/http'
 
 describe Fluent::Plugin::PrometheusInput do
   CONFIG = %[
-  type prometheus
+  @type prometheus
 ]
 
   LOCAL_CONFIG = %[
-  type prometheus
+  @type prometheus
   bind 127.0.0.1
 ]
 
@@ -47,6 +47,130 @@ describe Fluent::Plugin::PrometheusInput do
     end
   end
 
+  describe '#start' do
+    context 'with transport section' do
+      let(:config) do
+        %[
+           @type prometheus
+           bind 127.0.0.1
+           <transport tls>
+             insecure true
+           </transport>
+         ]
+      end
+
+      it 'returns 200' do
+        driver.run(timeout: 1) do
+          Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
+            req = Net::HTTP::Get.new('/metrics')
+            res = http.request(req)
+            expect(res.code).to eq('200')
+          end
+        end
+      end
+    end
+
+    context 'old parameters are given' do
+      context 'when extra_conf is used' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              extra_conf { "SSLCertName": [["CN", "nobody"], ["DC", "example"]] }
+            </ssl>
+         ]
+        end
+
+        it 'uses webrick' do
+          expect(driver.instance).to receive(:start_webrick).once
+          driver.run(timeout: 1)
+        end
+
+        it 'returns 200' do
+          driver.run(timeout: 1) do
+            Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
+              req = Net::HTTP::Get.new('/metrics')
+              res = http.request(req)
+              expect(res.code).to eq('200')
+            end
+          end
+        end
+      end
+
+      context 'cert_path and private_key_path combination' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              certificate_path path
+              private_key_path path1
+            </ssl>
+          ]
+        end
+
+        it 'converts them into new transport section' do
+          expect(driver.instance).to receive(:http_server_create_http_server).with(
+                                       :in_prometheus_server,
+                                       addr: anything,
+                                       logger: anything,
+                                       port: anything,
+                                       proto: :tls,
+                                       tls_opts: { 'cert_path' => 'path', 'private_key_path' => 'path1' }
+                                     ).once
+
+          driver.run(timeout: 1)
+        end
+      end
+
+      context 'insecure and ca_path' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              ca_path path
+            </ssl>
+           ]
+        end
+
+        it 'converts them into new transport section' do
+          expect(driver.instance).to receive(:http_server_create_http_server).with(
+                                       :in_prometheus_server,
+                                       addr: anything,
+                                       logger: anything,
+                                       port: anything,
+                                       proto: :tls,
+                                       tls_opts: { 'ca_path' => 'path', 'insecure' => true }
+                                     ).once
+
+          driver.run(timeout: 1)
+        end
+      end
+
+      context 'when only private_key_path is geven' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              private_key_path path
+            </ssl>
+           ]
+        end
+
+        it 'raises ConfigError' do
+          expect { driver.run(timeout: 1) }.to raise_error(Fluent::ConfigError, 'both certificate_path and private_key_path must be defined')
+        end
+      end
+    end
+  end
+
   describe '#run' do
     context '/metrics' do
       let(:config) { LOCAL_CONFIG }

data/spec/fluent/plugin/shared.rb

@@ -1,6 +1,6 @@
 
 BASE_CONFIG = %[
-  type prometheus
+  @type prometheus
 ]
 
 SIMPLE_CONFIG = BASE_CONFIG + %[

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-prometheus
 version: !ruby/object:Gem::Version
-  version: 1.7.3
+  version: 1.8.0
 platform: ruby
 authors:
 - Masahiro Sano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-14 00:00:00.000000000 Z
+date: 2020-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.20
+        version: 1.9.1
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.20
+        version: 1.9.1
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'