fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

data/lib/fluent/plugin_helper/socket.rb

@@ -0,0 +1,101 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'socket'
+require 'ipaddr'
+
+require_relative 'socket_option'
+
+module Fluent
+  module PluginHelper
+    module Socket
+      # stop     : [-]
+      # shutdown : [-]
+      # close    : [-]
+      # terminate: [-]
+
+      include Fluent::PluginHelper::SocketOption
+
+      attr_reader :_sockets # for tests
+
+      # TODO: implement connection pool for specified host
+
+      def socket_create(proto, host, port, **kwargs, &block)
+        case proto
+        when :tcp
+          socket_create_tcp(host, port, **kwargs, &block)
+        when :udp
+          socket_create_udp(host, port, **kwargs, &block)
+        when :tls
+          socket_create_tls(host, port, **kwargs, &block)
+        when :unix
+          raise "not implemented yet"
+        else
+          raise ArgumentError, "invalid protocol: #{proto}"
+        end
+      end
+
+      def socket_create_tcp(host, port, resolve_name: false, **kwargs, &block)
+        sock = TCPSocket.new(host, port)
+        socket_option_set(sock, resolve_name: resolve_name, **kwargs)
+        if block
+          begin
+            block.call(sock)
+          ensure
+            sock.close_write rescue nil
+            sock.close rescue nil
+          end
+        else
+          sock
+        end
+      end
+
+      def socket_create_udp(host, port, resolve_name: false, connect: false, **kwargs, &block)
+        family = IPAddr.new(IPSocket.getaddress(host)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
+        sock = UDPSocket.new(family)
+        socket_option_set(sock, resolve_name: resolve_name, **kwargs)
+        sock.connect(host, port) if connect
+        if block
+          begin
+            block.call(sock)
+          ensure
+            sock.close rescue nil
+          end
+        else
+          sock
+        end
+      end
+
+      def socket_create_tls(host, port, resolve_name: false, certopts: {}, &block)
+        raise "not implemented yet"
+      end
+
+      # socket_create_socks ?
+
+      def initialize
+        super
+        # @_sockets = [] # for keepalived sockets / connection pool
+      end
+
+      # def close
+      #   @_sockets.each do |sock|
+      #     sock.close
+      #   end
+      #   super
+      # end
+    end
+  end
+end

data/lib/fluent/plugin_helper/socket_option.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'socket'
+require 'fcntl'
+
+# this module is only for Socket/Server plugin helpers
+module Fluent
+  module PluginHelper
+    module SocketOption
+      FORMAT_STRUCT_LINGER  = 'I!I!' # { int l_onoff; int l_linger; }
+      FORMAT_STRUCT_TIMEVAL = 'L!L!' # { time_t tv_sec; suseconds_t tv_usec; }
+
+      def socket_option_validate!(protocol, resolve_name: nil, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, certopts: nil)
+        unless resolve_name.nil?
+          if protocol != :tcp && protocol != :udp && protocol != :tls
+            raise ArgumentError, "BUG: resolve_name in available for tcp/udp/tls"
+          end
+        end
+        if linger_timeout
+          if protocol != :tcp && protocol != :tls
+            raise ArgumentError, "BUG: linger_timeout is available for tcp/tls"
+          end
+        end
+        if certopts
+          if protocol != :tls
+            raise ArgumentError, "BUG: certopts is available only for tls"
+          end
+        else
+          if protocol == :tls
+            raise ArgumentError, "BUG: certopts (certificate options) not specified for TLS"
+            socket_option_certopts_validate!(certopts)
+          end
+        end
+      end
+
+      def socket_option_certopts_validate!(certopts)
+        raise "not implemented yet"
+      end
+
+      def socket_option_set(sock, resolve_name: nil, nonblock: false, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, certopts: nil)
+        unless resolve_name.nil?
+          sock.do_not_reverse_lookup = !resolve_name
+        end
+        if nonblock
+          sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
+        end
+        if linger_timeout
+          optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
+          socket_option_set_one(sock, :SO_LINGER, optval)
+        end
+        if recv_timeout
+          optval = [recv_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)
+          socket_option_set_one(sock, :SO_RCVTIMEO, optval)
+        end
+        if send_timeout
+          optval = [send_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)
+          socket_option_set_one(sock, :SO_SNDTIMEO, optval)
+        end
+        # TODO: certopts for TLS
+        sock
+      end
+
+      def socket_option_set_one(sock, option, value)
+        sock.setsockopt(::Socket::SOL_SOCKET, option, value)
+      rescue => e
+        log.warn "failed to set socket option", sock: sock.class, option: option, value: value, error: e
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/timer.rb

@@ -36,6 +36,7 @@ module Fluent
         timer = TimerWatcher.new(title, interval, repeat, log, checker, &block)
         @_timers << title
         event_loop_attach(timer)
+        timer
       end
 
       def timer_running?

data/lib/fluent/root_agent.rb

@@ -233,7 +233,7 @@ module Fluent
       # <source> emits events to the top-level event router (RootAgent#event_router).
       # Input#configure overwrites event_router to a label's event_router if it has `@label` parameter.
       # See also 'fluentd/plugin/input.rb'
-      input.router = @event_router
+      input.context_router = @event_router
       input.configure(conf)
       @inputs << input
 

data/lib/fluent/test/driver/base.rb

@@ -18,32 +18,38 @@ require 'fluent/config'
 require 'fluent/config/element'
 require 'fluent/log'
 
+require 'serverengine/socket_manager'
+require 'fileutils'
 require 'timeout'
 
 module Fluent
   module Test
     module Driver
+      class TestTimedOut < RuntimeError; end
+
       class Base
         attr_reader :instance, :logs
 
+        DEFAULT_TIMEOUT = 300
+
         def initialize(klass, opts: {}, &block)
           if klass.is_a?(Class)
+            @instance = klass.new
             if block
-              # Create new class for test w/ overwritten methods
-              #   klass.dup is worse because its ancestors does NOT include original class name
-              klass_name = klass.name
-              klass = Class.new(klass)
-              klass.define_singleton_method("name") { klass_name }
-              klass.module_eval(&block)
+              @instance.singleton_class.module_eval(&block)
+              @instance.send(:initialize)
             end
-            @instance = klass.new
           else
             @instance = klass
           end
           @instance.under_plugin_development = true
 
+          @socket_manager_server = nil
+
           @logs = []
 
+          @test_clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
+
           @run_post_conditions = []
           @run_breaking_conditions = []
           @broken = false
@@ -70,97 +76,137 @@ module Fluent
         def run(timeout: nil, start: true, shutdown: true, &block)
           instance_start if start
 
-          if @instance.respond_to?(:thread_wait_until_start)
-            @instance.thread_wait_until_start
-          end
-          if @instance.respond_to?(:event_loop_wait_until_start)
-            @instance.event_loop_wait_until_start
+          timeout ||= DEFAULT_TIMEOUT
+          stop_at = Process.clock_gettime(@test_clock_id) + timeout
+          @run_breaking_conditions << ->(){ Process.clock_gettime(@test_clock_id) >= stop_at }
+
+          if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?
+            raise ArgumentError, "no stop conditions nor block specified"
           end
 
+          sleep_with_watching_threads = ->(){
+            if @instance.respond_to?(:_threads)
+              @instance._threads.values.each{|t| t.join(0) }
+            end
+            sleep 0.1
+          }
+
           begin
-            run_actual(timeout: timeout, &block)
+            retval = run_actual(timeout: timeout, &block)
+            sleep_with_watching_threads.call until stop?
+            retval
           ensure
             instance_shutdown if shutdown
           end
         end
 
         def instance_start
+          if @instance.respond_to?(:server_wait_until_start)
+            @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
+            end
+            @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
+            ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+          end
+
           unless @instance.started?
             @instance.start
-            instance_hook_after_started
           end
           unless @instance.after_started?
             @instance.after_start
           end
+
+          if @instance.respond_to?(:thread_wait_until_start)
+            @instance.thread_wait_until_start
+          end
+
+          if @instance.respond_to?(:event_loop_wait_until_start)
+            @instance.event_loop_wait_until_start
+          end
+
+          instance_hook_after_started
         end
 
         def instance_hook_after_started
           # insert hooks for tests available after instance.start
         end
 
+        def instance_hook_before_stopped
+          # same with above
+        end
+
         def instance_shutdown
-          @instance.stop            unless @instance.stopped?
-          @instance.before_shutdown unless @instance.before_shutdown?
-          @instance.shutdown        unless @instance.shutdown?
+          instance_hook_before_stopped
+
+          unless @instance.stopped?
+            @instance.stop rescue nil
+          end
+          unless @instance.before_shutdown?
+            @instance.before_shutdown rescue nil
+          end
+          unless @instance.shutdown?
+            @instance.shutdown rescue nil
+          end
 
           if @instance.respond_to?(:event_loop_wait_until_stop)
             @instance.event_loop_wait_until_stop
           end
 
-          @instance.after_shutdown  unless @instance.after_shutdown?
-          @instance.close     unless @instance.closed?
+          unless @instance.after_shutdown?
+            @instance.after_shutdown rescue nil
+          end
+          unless @instance.closed?
+            @instance.close rescue nil
+          end
 
           if @instance.respond_to?(:thread_wait_until_stop)
             @instance.thread_wait_until_stop
           end
 
-          @instance.terminate unless @instance.terminated?
-        end
+          if @instance.respond_to?(:server_wait_until_stop)
+            @instance.server_wait_until_stop
+          end
 
-        def run_actual(timeout: nil, &block)
-          if @instance.respond_to?(:_threads)
-            until @instance._threads.values.all?(&:alive?)
-              sleep 0.01
-            end
+          unless @instance.terminated?
+            @instance.terminate rescue nil
           end
 
-          if @instance.respond_to?(:event_loop_running?)
-            until @instance.event_loop_running?
-              sleep 0.01
+          if @socket_manager_server
+            @socket_manager_server.close
+            if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+              FileUtils.rm_f @socket_manager_path
             end
           end
+        end
 
-          if timeout
-            stop_at = Time.now + timeout
-            @run_breaking_conditions << ->(){ Time.now >= stop_at }
+        def run_actual(timeout: DEFAULT_TIMEOUT, &block)
+          if @instance.respond_to?(:_threads)
+            sleep 0.1 until @instance._threads.values.all?(&:alive?)
           end
 
-          if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?
-            raise ArgumentError, "no stop conditions nor block specified"
+          if @instance.respond_to?(:event_loop_running?)
+            sleep 0.1 until @instance.event_loop_running?
           end
 
-          proc = if block_given?
-                   ->(){ block.call; sleep(0.1) until stop? }
-                 else
-                   ->(){ sleep(0.1) until stop? }
-                 end
+          if @instance.respond_to?(:_child_process_processes)
+            sleep 0.1 until @instance._child_process_processes.values.all?{|pinfo| pinfo.alive }
+          end
 
-          if timeout
-            begin
-              Timeout.timeout(timeout * 1.1) do |sec|
-                proc.call
-              end
-            rescue Timeout::Error
-              @broken = true
+          return_value = nil
+          begin
+            Timeout.timeout(timeout * 1.1) do |sec|
+              return_value = block.call if block_given?
             end
-          else
-            proc.call
+          rescue Timeout::Error
+            raise TestTimedOut, "Test case timed out with hard limit."
           end
+          return_value
         end
 
         def stop?
           # Should stop running if post conditions are not registered.
-          return true unless @run_post_conditions
+          return true unless @run_post_conditions || @run_post_conditions.empty?
 
           # Should stop running if all of the post conditions are true.
           return true if @run_post_conditions.all? {|proc| proc.call }

data/lib/fluent/test/driver/base_owner.rb

@@ -71,25 +71,35 @@ module Fluent
         end
 
         def events(tag: nil)
-          return [] if @event_streams.nil?
-          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
           if block_given?
-            selected.each do |e|
-              e.es.each do |time, record|
-                yield e.tag, time, record
+            event_streams(tag: tag) do |t, es|
+              es.each do |time, record|
+                yield t, time, record
               end
             end
           else
             list = []
-            selected.each do |e|
-              e.es.each do |time, record|
-                list << [e.tag, time, record]
+            event_streams(tag: tag) do |t, es|
+              es.each do |time, record|
+                list << [t, time, record]
               end
             end
             list
           end
         end
 
+        def event_streams(tag: nil)
+          return [] if @event_streams.nil?
+          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              yield e.tag, e.es
+            end
+          else
+            selected.map{|e| [e.tag, e.es] }
+          end
+        end
+
         def emit_count
           @event_streams.size
         end