fluentd 1.7.4-x64-mingw32 → 1.8.0-x64-mingw32

data/lib/fluent/plugin/sd_static.rb

@@ -0,0 +1,58 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/service_discovery'
+
+module Fluent
+  module Plugin
+    class StaticServiceDiscovery < ServiceDiscovery
+      Plugin.register_sd('static', self)
+
+      LISTEN_PORT = 24224
+
+      config_section :service, param_name: :service_configs do
+        desc 'The IP address or host name of the server.'
+        config_param :host, :string
+        desc 'The name of the server. Used for logging and certificate verification in TLS transport (when host is address).'
+        config_param :name, :string, default: nil
+        desc 'The port number of the host.'
+        config_param :port, :integer, default: LISTEN_PORT
+        desc 'The shared key per server.'
+        config_param :shared_key, :string, default: nil, secret: true
+        desc 'The username for authentication.'
+        config_param :username, :string, default: ''
+        desc 'The password for authentication.'
+        config_param :password, :string, default: '', secret: true
+        desc 'Marks a node as the standby node for an Active-Standby model between Fluentd nodes.'
+        config_param :standby, :bool, default: false
+        desc 'The load balancing weight.'
+        config_param :weight, :integer, default: 60
+      end
+
+      def configure(conf)
+        super
+
+        @services = @service_configs.map do |s|
+          ServiceDiscovery::Service.new(:static, s.host, s.port, s.name, s.weight, s.standby, s.username, s.password, s.shared_key)
+        end
+      end
+
+      def start(queue = nil)
+        super()
+      end
+    end
+  end
+end

data/lib/fluent/plugin/service_discovery.rb

@@ -0,0 +1,80 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/unique_id'
+require 'fluent/plugin_id'
+
+module Fluent
+  module Plugin
+    class ServiceDiscovery < Base
+      include PluginId
+      include PluginLoggerMixin
+      include UniqueId::Mixin
+
+      configured_in :service_discovery
+
+      attr_reader :services
+
+      Service = Struct.new(:plugin_name, :host, :port, :name, :weight, :standby, :username, :password, :shared_key) do
+        def discovery_id
+          @discovery_id ||= Base64.encode64(to_h.to_s)
+        end
+      end
+
+      SERVICE_IN = :service_in
+      SERVICE_OUT = :service_out
+      DiscoveryMessage = Struct.new(:type, :service)
+
+      class << self
+        def service_in_msg(service)
+          DiscoveryMessage.new(SERVICE_IN, service)
+        end
+
+        def service_out_msg(service)
+          DiscoveryMessage.new(SERVICE_OUT, service)
+        end
+      end
+
+      def initialize
+        @services = []
+
+        super
+      end
+
+      def start(queue = nil)
+        super()
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper.rb

@@ -31,6 +31,7 @@ require 'fluent/plugin_helper/counter'
 require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/record_accessor'
 require 'fluent/plugin_helper/compat_parameters'
+require 'fluent/plugin_helper/service_discovery'
 
 module Fluent
   module PluginHelper

data/lib/fluent/plugin_helper/child_process.rb

@@ -267,19 +267,19 @@ module Fluent
         end
 
         if mode.include?(:write)
-          writeio.set_encoding(external_encoding, internal_encoding, encoding_options)
+          writeio.set_encoding(external_encoding, internal_encoding, **encoding_options)
           writeio_in_use = true
         else
           writeio.reopen(IO::NULL) if writeio
         end
         if mode.include?(:read) || mode.include?(:read_with_stderr)
-          readio.set_encoding(external_encoding, internal_encoding, encoding_options)
+          readio.set_encoding(external_encoding, internal_encoding, **encoding_options)
           readio_in_use = true
         else
           readio.reopen(IO::NULL) if readio
         end
         if mode.include?(:stderr)
-          stderrio.set_encoding(external_encoding, internal_encoding, encoding_options)
+          stderrio.set_encoding(external_encoding, internal_encoding, **encoding_options)
           stderrio_in_use = true
         else
           stderrio.reopen(IO::NULL) if stderrio && stderr == :discard

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -194,7 +194,9 @@ module Fluent
           hash['time_type'] = 'unixtime'
         end
         if conf.has_key?('localtime') || conf.has_key?('utc')
-          if conf.has_key?('localtime') && conf.has_key?('utc')
+          utc = to_bool(conf['utc'])
+          localtime = to_bool(conf['localtime'])
+          if conf.has_key?('localtime') && conf.has_key?('utc') && !(localtime ^ utc)
             raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
           elsif conf.has_key?('localtime')
             hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
@@ -217,6 +219,14 @@ module Fluent
         conf
       end
 
+      def to_bool(v)
+        if  v.is_a?(FalseClass) || v == 'false' || v.nil?
+          false
+        else
+          true
+        end
+      end
+
       def compat_parameters_extract(conf)
         return unless conf.elements('extract').empty?
         return if EXTRACT_PARAMS.keys.all?{|k| !conf.has_key?(k) } && !conf.has_key?('format')

data/lib/fluent/plugin_helper/extract.rb

@@ -55,7 +55,7 @@ module Fluent
           config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
 
           Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
-            config_param name, type, opts
+            config_param(name, type, **opts)
           end
         end
       end

data/lib/fluent/plugin_helper/inject.rb

@@ -79,7 +79,7 @@ module Fluent
           config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
 
           Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
-            config_param name, type, opts
+            config_param(name, type, **opts)
           end
         end
       end

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -31,6 +31,10 @@ module Fluent
         Accessor.new(param)
       end
 
+      def record_accessor_nested?(param)
+        Accessor.parse_parameter(param).is_a?(Array)
+      end
+
       class Accessor
         attr_reader :keys
 
@@ -42,25 +46,19 @@ module Fluent
             @dig_keys = @keys[0..-2]
             if @dig_keys.empty?
               @keys = @keys.first
-              mcall = method(:call_index)
-              mdelete = method(:delete_top)
             else
               mcall = method(:call_dig)
               mdelete = method(:delete_nest)
+              singleton_class.module_eval do
+                define_method(:call, mcall)
+                define_method(:delete, mdelete)
+              end
             end
-          else
-            # Call [] for single key to reduce dig overhead
-            mcall = method(:call_index)
-            mdelete = method(:delete_top)
-          end
-
-          singleton_class.module_eval do
-            define_method(:call, mcall)
-            define_method(:delete, mdelete)
           end
         end
 
         def call(r)
+          r[@keys]
         end
 
         # To optimize the performance, use class_eval with pre-expanding @keys
@@ -69,11 +67,8 @@ module Fluent
           r.dig(*@keys)
         end
 
-        def call_index(r)
-          r[@keys]
-        end
-
         def delete(r)
+          r.delete(@keys)
         end
 
         def delete_nest(r)
@@ -88,10 +83,6 @@ module Fluent
           nil
         end
 
-        def delete_top(r)
-          r.delete(@keys)
-        end
-
         def self.parse_parameter(param)
           if param.start_with?('$.')
             parse_dot_notation(param)

data/lib/fluent/plugin_helper/server.rb

@@ -355,7 +355,11 @@ module Fluent
         sock = if shared
                  server_socket_manager_client.listen_tcp(bind, port)
                else
-                 TCPServer.new(bind, port) # this method call can create sockets for AF_INET6
+                 # TCPServer.new doesn't set IPV6_V6ONLY flag, so use Addrinfo class instead.
+                 # backlog will be set by the caller, we don't need to set backlog here
+                 tsock = Addrinfo.tcp(bind, port).listen
+                 tsock.autoclose = false
+                 TCPServer.for_fd(tsock.fileno)
                end
         # close-on-exec is set by default in Ruby 2.0 or later (, and it's unavailable on Windows)
         sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK) # nonblock
@@ -690,9 +694,9 @@ module Fluent
           end
 
           if RUBY_VERSION.to_f >= 2.3
-            NONBLOCK_ARG = {exception: false}
+            NONBLOCK_ARG = { exception: false }
             def try_handshake
-              @_handler_socket.accept_nonblock(NONBLOCK_ARG)
+              @_handler_socket.accept_nonblock(**NONBLOCK_ARG)
             end
           else
             def try_handshake
@@ -722,7 +726,7 @@ module Fluent
 
                 return true
               end
-            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, OpenSSL::SSL::SSLError => e
+            rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, OpenSSL::SSL::SSLError => e
               @log.trace "unexpected error before accepting TLS connection", error: e
               close rescue nil
             end

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -0,0 +1,80 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin_helper/timer'
+require 'fluent/plugin_helper/service_discovery/manager'
+
+module Fluent
+  module PluginHelper
+    module ServiceDiscovery
+      include Fluent::PluginHelper::Timer
+
+      def self.included(mod)
+        mod.include ServiceDiscoveryParams
+      end
+
+      def start
+        unless @discovery_manager
+          log.warn('There is no discovery_manager. skip start them')
+          super
+          return
+        end
+
+        @discovery_manager.start
+        unless @discovery_manager.static_config?
+          timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_iterval) do
+            @discovery_manager.run_once
+          end
+        end
+
+        super
+      end
+
+      private
+
+      # @param title [Symbol] the thread name. this value should be unique.
+      # @param configurations [Hash] hash which must has discivery_service type and its configuration like `{ type: :static, conf: <Fluent::Config::Element> }`
+      # @param load_balancer [Object] object which has two methods #rebalance and #select_service
+      # @param custom_build_method [Proc]
+      def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
+        @_plugin_helper_service_discovery_title = title
+        @_plugin_helper_service_discovery_iterval = interval
+
+        @discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(
+          log: log,
+          load_balancer: load_balancer,
+          custom_build_method: custom_build_method,
+        )
+
+        @discovery_manager.configure(configurations, parent: self)
+
+        @discovery_manager
+      end
+
+      def discovery_manager
+        @discovery_manager
+      end
+
+      module ServiceDiscoveryParams
+        include Fluent::Configurable
+
+        config_section :service_discovery do
+          config_param :@type, :string
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/service_discovery/manager.rb

@@ -0,0 +1,132 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/service_discovery'
+require 'fluent/plugin_helper/service_discovery/round_robin_balancer'
+
+module Fluent
+  module PluginHelper
+    module ServiceDiscovery
+      class Manager
+        def initialize(log:, load_balancer: nil, custom_build_method: nil)
+          @log = log
+          @load_balancer = load_balancer || RoundRobinBalancer.new
+          @custom_build_method = custom_build_method
+
+          @discoveries = []
+          @services = {}
+          @queue = Queue.new
+          @static_config = true
+        end
+
+        def configure(opts, parent: nil)
+          opts.each do |opt|
+            sd = Fluent::Plugin.new_sd(opt[:type], parent: parent)
+            sd.configure(opt[:conf])
+
+            sd.services.each do |s|
+              @services[s.discovery_id] = build_service(s)
+            end
+            @discoveries << sd
+
+            if @static_config && opt[:type] != :static
+              @static_config = false
+            end
+          end
+
+          rebalance
+        end
+
+        def static_config?
+          @static_config
+        end
+
+        def start
+          @discoveries.each do |d|
+            d.start(@queue)
+          end
+        end
+
+        def run_once
+          # Don't care race in this loop intentionally
+          s = @queue.size
+
+          if s == 0
+            return
+          end
+
+          s.times do
+            msg = @queue.pop
+
+            unless msg.is_a?(Fluent::Plugin::ServiceDiscovery::DiscoveryMessage)
+              @log.warn("BUG: #{msg}")
+              next
+            end
+
+            begin
+              handle_message(msg)
+            rescue => e
+              @log.error(e)
+            end
+          end
+
+          rebalance
+        end
+
+        def rebalance
+          @load_balancer.rebalance(services)
+        end
+
+        def select_service(&block)
+          @load_balancer.select_service(&block)
+        end
+
+        def services
+          @services.values
+        end
+
+        private
+
+        def handle_message(msg)
+          service = msg.service
+
+          case msg.type
+          when Fluent::Plugin::ServiceDiscovery::SERVICE_IN
+            if (n = build_service(service))
+              @log.info("Service in: name=#{service.name} #{service.host}:#{service.port}")
+              @services[service.discovery_id] = n
+            else
+              raise "failed to build service in name=#{service.name} #{service.host}:#{service.port}"
+            end
+          when Fluent::Plugin::ServiceDiscovery::SERVICE_OUT
+            s = @services.delete(service.discovery_id)
+            if s
+              @log.info("Service out: name=#{service.name} #{service.host}:#{service.port}")
+            else
+              @log.warn("Not found service: name=#{service.name} #{service.host}:#{service.port}")
+            end
+          else
+            @log.error("BUG: unknow message type: #{msg.type}")
+          end
+        end
+
+        def build_service(n)
+          @custom_build_method ? @custom_build_method.call(n) : n
+        end
+      end
+    end
+  end
+end