fluentd 1.9.0-x86-mingw32 → 1.9.1-x86-mingw32

data/lib/fluent/plugin/out_forward.rb

@@ -17,6 +17,7 @@
 require 'fluent/output'
 require 'fluent/config/error'
 require 'fluent/clock'
+require 'fluent/tls'
 require 'base64'
 require 'forwardable'
 
@@ -89,9 +90,9 @@ module Fluent::Plugin
     config_param :compress, :enum, list: [:text, :gzip], default: :text
 
     desc 'The default version of TLS transport.'
-    config_param :tls_version, :enum, list: Fluent::PluginHelper::Socket::TLS_SUPPORTED_VERSIONS, default: Fluent::PluginHelper::Socket::TLS_DEFAULT_VERSION
+    config_param :tls_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
     desc 'The cipher configuration of TLS transport.'
-    config_param :tls_ciphers, :string, default: Fluent::PluginHelper::Socket::CIPHERS_DEFAULT
+    config_param :tls_ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT
     desc 'Skip all verification of certificates or not.'
     config_param :tls_insecure_mode, :bool, default: false
     desc 'Allow self signed certificates or not.'

data/lib/fluent/plugin/out_http.rb

@@ -17,6 +17,7 @@
 require 'net/http'
 require 'uri'
 require 'openssl'
+require 'fluent/tls'
 require 'fluent/plugin/output'
 require 'fluent/plugin_helper/socket'
 
@@ -57,14 +58,14 @@ module Fluent::Plugin
     desc 'The verify mode of TLS'
     config_param :tls_verify_mode, :enum, list: [:none, :peer], default: :peer
     desc 'The default version of TLS'
-    config_param :tls_version, :enum, list: Fluent::PluginHelper::Socket::TLS_SUPPORTED_VERSIONS, default: Fluent::PluginHelper::Socket::TLS_DEFAULT_VERSION
+    config_param :tls_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
     desc 'The cipher configuration of TLS'
-    config_param :tls_ciphers, :string, default: Fluent::PluginHelper::Socket::CIPHERS_DEFAULT
+    config_param :tls_ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT
 
     desc 'Raise UnrecoverableError when the response is non success, 4xx/5xx'
     config_param :error_response_as_unrecoverable, :bool, default: true
     desc 'The list of retryable response code'
-    config_param :retryable_response_codes, :array, value_type: :integer, default: [503]
+    config_param :retryable_response_codes, :array, value_type: :integer, default: nil
 
     config_section :format do
       config_set_default :@type, 'json'
@@ -90,6 +91,11 @@ module Fluent::Plugin
     def configure(conf)
       super
 
+      if @retryable_response_codes.nil?
+        log.warn('Status code 503 is going to be removed from default `retryable_response_codes` from fluentd v2. Please add it by yourself if you wish')
+        @retryable_response_codes = [503]
+      end
+
       @http_opt = setup_http_option
       @proxy_uri = URI.parse(@proxy) if @proxy
       @formatter = formatter_create
@@ -172,7 +178,7 @@ module Fluent::Plugin
     end
 
     def parse_endpoint(chunk)
-      endpoint = extract_placeholders(@endpoint, chunk)    
+      endpoint = extract_placeholders(@endpoint, chunk)
       URI.parse(endpoint)
     end
 

data/lib/fluent/plugin/output.rb

@@ -1091,7 +1091,7 @@ module Fluent
         end
       end
 
-      UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError, MessagePack::UnpackError]
+      UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError, MessagePack::UnpackError, EncodingError]
 
       def try_flush
         chunk = @buffer.dequeue_chunk

data/lib/fluent/plugin/parser_syslog.rb

@@ -27,8 +27,8 @@ module Fluent
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
-      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
+      REGEXP_RFC5424 = /\A(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?\z/m
+      REGEXP_RFC5424_WITH_PRI = /\A\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?\z/m
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
@@ -141,6 +141,9 @@ module Fluent
                 end
               end
               record[name] = value if @keep_time_key
+            when "message"
+              value.chomp!
+              record[name] = value
             else
               record[name] = value
             end

data/lib/fluent/plugin_helper/cert_option.rb

@@ -17,14 +17,16 @@
 require 'openssl'
 require 'socket'
 
-# this module is only for Socket/Server plugin helpers
+require 'fluent/tls'
+
+# this module is only for Socket/Server/HttpServer plugin helpers
 module Fluent
   module PluginHelper
     module CertOption
       def cert_option_create_context(version, insecure, ciphers, conf)
         cert, key, extra = cert_option_server_validate!(conf)
 
-        ctx = OpenSSL::SSL::SSLContext.new(version)
+        ctx = OpenSSL::SSL::SSLContext.new
         unless insecure
           # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
           # https://bugs.ruby-lang.org/issues/9424
@@ -43,6 +45,7 @@ module Fluent
         if extra && !extra.empty?
           ctx.extra_chain_cert = extra
         end
+        Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)
 
         ctx
       end

data/lib/fluent/plugin_helper/http_server.rb

@@ -23,27 +23,77 @@ rescue LoadError => _
 end
 
 require 'fluent/plugin_helper/thread'
+require 'fluent/plugin_helper/server' # For Server::ServerTransportParams
+require 'fluent/plugin_helper/http_server/ssl_context_builder'
 
 module Fluent
   module PluginHelper
     module HttpServer
       include Fluent::PluginHelper::Thread
+      include Fluent::Configurable
+
       # stop     : stop http server and mark callback thread as stopped
       # shutdown : [-]
       # close    : correct stopped threads
       # terminate: kill thread
 
+      def self.included(mod)
+        mod.include Fluent::PluginHelper::Server::ServerTransportParams
+      end
+
+      def initialize(*)
+        super
+        @_http_server = nil
+      end
+
+      def create_http_server(title, addr:, port:, logger:, default_app: nil, proto: nil, tls_opts: nil, &block)
+        logger.warn('this method is deprecated. Use #http_server_create_http_server instead')
+        http_server_create_http_server(title, addr: addr, port: port, logger: logger, default_app: default_app, proto: proto, tls_opts: tls_opts, &block)
+      end
+
       # @param title [Symbol] the thread name. this value should be unique.
       # @param addr [String] Listen address
       # @param port [String] Listen port
       # @param logger [Logger] logger used in this server
       # @param default_app [Object] This method must have #call.
-      def create_http_server(title, addr:, port:, logger:, default_app: nil)
+      # @param proto [Symbol] :tls or :tcp
+      # @param tls_opts [Hash] options for TLS.
+      def http_server_create_http_server(title, addr:, port:, logger:, default_app: nil, proto: nil, tls_opts: nil, &block)
         unless block_given?
           raise ArgumentError, 'BUG: callback not specified'
         end
 
-        @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app) do |serv|
+        if proto == :tls || (@transport_config && @transport_config.protocol == :tls)
+          http_server_create_https_server(title, addr: addr, port: port, logger: logger, default_app: default_app, tls_opts: tls_opts, &block)
+        else
+          @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app) do |serv|
+            yield(serv)
+          end
+
+          _block_until_http_server_start do |notify|
+            thread_create(title) do
+              @_http_server.start(notify)
+            end
+          end
+        end
+      end
+
+      # @param title [Symbol] the thread name. this value should be unique.
+      # @param addr [String] Listen address
+      # @param port [String] Listen port
+      # @param logger [Logger] logger used in this server
+      # @param default_app [Object] This method must have #call.
+      # @param tls_opts [Hash] options for TLS.
+      def http_server_create_https_server(title, addr:, port:, logger:, default_app: nil, tls_opts: nil)
+        topt =
+          if tls_opts
+            _http_server_overwrite_config(@transport_config, tls_opts)
+          else
+            @transport_config
+          end
+        ctx = Fluent::PluginHelper::HttpServer::SSLContextBuilder.new($log).build(topt)
+
+        @_http_server = HttpServer::Server.new(addr: addr, port: port, logger: logger, default_app: default_app, tls_context: ctx) do |serv|
           yield(serv)
         end
 
@@ -64,6 +114,16 @@ module Fluent
 
       private
 
+      def _http_server_overwrite_config(config, opts)
+        conf = config.dup
+        Fluent::PluginHelper::Server::SERVER_TRANSPORT_PARAMS.map(&:to_s).each do |param|
+          if opts.key?(param)
+            conf[param] = opts[param]
+          end
+        end
+        conf
+      end
+
       # To block until server is ready to listen
       def _block_until_http_server_start
         que = Queue.new

data/lib/fluent/plugin_helper/http_server/compat/server.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/plugin_helper/http_server/methods'
 require 'fluent/plugin_helper/http_server/compat/webrick_handler'
+require 'fluent/plugin_helper/http_server/compat/ssl_context_extractor'
 
 module Fluent
   module PluginHelper
@@ -24,16 +25,26 @@ module Fluent
         class Server
           # @param logger [Logger]
           # @param default_app [Object] ignored option. only for compat
-          def initialize(addr:, port:, logger:, default_app: nil)
+          # @param tls_context [OpenSSL::SSL::SSLContext]
+          def initialize(addr:, port:, logger:, default_app: nil, tls_context: nil)
             @addr = addr
             @port = port
             @logger = logger
-            @server = WEBrick::HTTPServer.new(
+
+            config = {
               BindAddress: @addr,
               Port: @port,
               Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
               AccessLog: [],
-            )
+            }
+            if tls_context
+              require 'webrick/https'
+              @logger.warn('Webrick ignores given TLS version')
+              tls_opt = Fluent::PluginHelper::HttpServer::Compat::SSLContextExtractor.extract(tls_context)
+              config = tls_opt.merge(**config)
+            end
+
+            @server = WEBrick::HTTPServer.new(config)
 
             # @example ["/example.json", :get, handler object]
             @methods = []

data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb

@@ -0,0 +1,52 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      module Compat
+        # This class converts OpenSSL::SSL::SSLContext to Webrick SSL Config because webrick does not have interface to pass OpenSSL::SSL::SSLContext directory
+        # https://github.com/ruby/webrick/blob/v1.6.0/lib/webrick/ssl.rb#L67-L88
+        class SSLContextExtractor
+
+          #
+          # memo: https://github.com/ruby/webrick/blob/v1.6.0/lib/webrick/ssl.rb#L180-L205
+          # @param ctx [OpenSSL::SSL::SSLContext]
+          def self.extract(ctx)
+            {
+              SSLEnable: true,
+              SSLPrivateKey: ctx.key,
+              SSLCertificate: ctx.cert,
+              SSLClientCA: ctx.client_ca,
+              SSLExtraChainCert: ctx.extra_chain_cert,
+              SSLCACertificateFile: ctx.ca_file,
+              SSLCACertificatePath: ctx.ca_path,
+              SSLCertificateStore: ctx.cert_store,
+              SSLTmpDhCallback: ctx.tmp_dh_callback,
+              SSLVerifyClient: ctx.verify_mode,
+              SSLVerifyDepth: ctx.verify_depth,
+              SSLVerifyCallback: ctx.verify_callback,
+              SSLServerNameCallback: ctx.servername_cb,
+              SSLTimeout: ctx.timeout,
+              SSLOptions: ctx.options,
+              SSLCiphers: ctx.ciphers,
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/http_server/server.rb

@@ -26,20 +26,26 @@ module Fluent
   module PluginHelper
     module HttpServer
       class Server
+        # @param logger [Logger]
         # @param default_app [Object] This method must have #call.
-        def initialize(addr:, port:, logger:, default_app: nil)
+        # @param tls_context [OpenSSL::SSL::SSLContext]
+        def initialize(addr:, port:, logger:, default_app: nil, tls_context: nil)
           @addr = addr
           @port = port
           @logger = logger
 
-          # TODO: support https and http2
-          @uri = URI("http://#{@addr}:#{@port}").to_s
+          # TODO: support http2
+          scheme = tls_context ? 'https' : 'http'
+          @uri = URI("#{scheme}://#{@addr}:#{@port}").to_s
           @router = Router.new(default_app)
-          @reactor = Async::Reactor.new
-          @server = Async::HTTP::Server.new(
-            App.new(@router, @logger),
-            Async::HTTP::Endpoint.parse(@uri)
-          )
+          @reactor = Async::Reactor.new(nil, logger: @logger)
+
+          opts = if tls_context
+                   { ssl_context: tls_context }
+                 else
+                   {}
+                 end
+          @server = Async::HTTP::Server.new(App.new(@router, @logger), Async::HTTP::Endpoint.parse(@uri, **opts))
 
           if block_given?
             yield(self)

data/lib/fluent/plugin_helper/http_server/ssl_context_builder.rb

@@ -0,0 +1,41 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/cert_option'
+
+module Fluent
+  module PluginHelper
+    module HttpServer
+      # In order not to expose CertOption's methods unnecessary
+      class SSLContextBuilder
+        include Fluent::PluginHelper::CertOption
+
+        def initialize(log)
+          @log = log
+        end
+
+        # @param config [Fluent::Config::Section] @transport_config
+        def build(config)
+          cert_option_create_context(config.version, config.insecure, config.ciphers, config)
+        end
+
+        private
+
+        attr_reader :log
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/server.rb

@@ -240,7 +240,7 @@ module Fluent
       end
 
       SERVER_TRANSPORT_PARAMS = [
-        :protocol, :version, :ciphers, :insecure,
+        :protocol, :version, :min_version, :max_version, :ciphers, :insecure,
         :ca_path, :cert_path, :private_key_path, :private_key_passphrase, :client_cert_auth,
         :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
         :generate_private_key_length,
@@ -260,18 +260,13 @@ module Fluent
       end
 
       module ServerTransportParams
-        TLS_DEFAULT_VERSION = :'TLSv1_2'
-        TLS_SUPPORTED_VERSIONS = [:'TLSv1_1', :'TLSv1_2']
-        ### follow httpclient configuration by nahi
-        # OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
-        CIPHERS_DEFAULT = "ALL:!aNULL:!eNULL:!SSLv2" # OpenSSL >1.0.0 default
-
         include Fluent::Configurable
         config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
           config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
-          config_param :version, :enum, list: TLS_SUPPORTED_VERSIONS, default: TLS_DEFAULT_VERSION
-
-          config_param :ciphers, :string, default: CIPHERS_DEFAULT
+          config_param :version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
+          config_param :min_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
+          config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
+          config_param :ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT
           config_param :insecure, :bool, default: false
 
           # Cert signed by public CA

data/lib/fluent/plugin_helper/socket.rb

@@ -21,6 +21,7 @@ if Fluent.windows?
   require 'certstore'
 end
 
+require 'fluent/tls'
 require_relative 'socket_option'
 
 module Fluent
@@ -33,12 +34,6 @@ module Fluent
 
       include Fluent::PluginHelper::SocketOption
 
-      TLS_DEFAULT_VERSION = :'TLSv1_2'
-      TLS_SUPPORTED_VERSIONS = [:'TLSv1_1', :'TLSv1_2']
-      ### follow httpclient configuration by nahi
-      # OpenSSL 0.9.8 default: "ALL:!ADH:!LOW:!EXP:!MD5:+SSLv2:@STRENGTH"
-      CIPHERS_DEFAULT = "ALL:!aNULL:!eNULL:!SSLv2" # OpenSSL >1.0.0 default
-
       attr_reader :_sockets # for tests
 
       # TODO: implement connection pool for specified host
@@ -97,7 +92,7 @@ module Fluent
 
       def socket_create_tls(
           host, port,
-          version: TLS_DEFAULT_VERSION, ciphers: CIPHERS_DEFAULT, insecure: false, verify_fqdn: true, fqdn: nil,
+          version: Fluent::TLS::DEFAULT_VERSION, min_version: nil, max_version: nil, ciphers: Fluent::TLS::CIPHERS_DEFAULT, insecure: false, verify_fqdn: true, fqdn: nil,
           enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,
           cert_path: nil, private_key_path: nil, private_key_passphrase: nil,
           cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,
@@ -106,7 +101,7 @@ module Fluent
         host_is_ipaddress = IPAddr.new(host) rescue false
         fqdn ||= host unless host_is_ipaddress
 
-        context = OpenSSL::SSL::SSLContext.new(version)
+        context = OpenSSL::SSL::SSLContext.new
 
         if insecure
           log.trace "setting TLS verify_mode NONE"
@@ -154,6 +149,7 @@ module Fluent
           context.cert = OpenSSL::X509::Certificate.new(File.read(cert_path)) if cert_path
           context.key = OpenSSL::PKey::read(File.read(private_key_path), private_key_passphrase) if private_key_path
         end
+        Fluent::TLS.set_version_to_context(context, version, min_version, max_version)
 
         tcpsock = socket_create_tcp(host, port, **kwargs)
         sock = WrappedSocket::TLS.new(tcpsock, context)