fluentd 0.14.8 → 0.14.9

data/lib/fluent/test/driver/base_owner.rb

@@ -71,25 +71,35 @@ module Fluent
         end
 
         def events(tag: nil)
-          return [] if @event_streams.nil?
-          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
           if block_given?
-            selected.each do |e|
-              e.es.each do |time, record|
-                yield e.tag, time, record
+            event_streams(tag: tag) do |t, es|
+              es.each do |time, record|
+                yield t, time, record
               end
             end
           else
             list = []
-            selected.each do |e|
-              e.es.each do |time, record|
-                list << [e.tag, time, record]
+            event_streams(tag: tag) do |t, es|
+              es.each do |time, record|
+                list << [t, time, record]
               end
             end
             list
           end
         end
 
+        def event_streams(tag: nil)
+          return [] if @event_streams.nil?
+          selected = @event_streams.select{|e| tag.nil? ? true : e.tag == tag }
+          if block_given?
+            selected.each do |e|
+              yield e.tag, e.es
+            end
+          else
+            selected.map{|e| [e.tag, e.es] }
+          end
+        end
+
         def emit_count
           @event_streams.size
         end

data/lib/fluent/test/driver/multi_output.rb

@@ -37,10 +37,11 @@ module Fluent
         end
 
         def run_actual(**kwargs, &block)
-          super(**kwargs, &block)
+          val = super(**kwargs, &block)
           if @flush_buffer_at_cleanup
             @instance.outputs.each{|o| o.force_flush }
           end
+          val
         end
 
         def flush

data/lib/fluent/test/driver/output.rb

@@ -29,23 +29,31 @@ module Fluent
         def initialize(klass, opts: {}, &block)
           super
           raise ArgumentError, "plugin is not an instance of Fluent::Plugin::Output" unless @instance.is_a? Fluent::Plugin::Output
-          @instance.in_tests = true
           @flush_buffer_at_cleanup = nil
+          @wait_flush_completion = nil
+          @force_flush_retry = nil
           @format_hook = nil
           @format_results = []
         end
 
-        def run(flush: true, **kwargs, &block)
+        def run(flush: true, wait_flush_completion: true, force_flush_retry: false, **kwargs, &block)
           @flush_buffer_at_cleanup = flush
+          @wait_flush_completion = wait_flush_completion
+          @force_flush_retry = force_flush_retry
           super(**kwargs, &block)
         end
 
         def run_actual(**kwargs, &block)
-          super(**kwargs, &block)
+          if @force_flush_retry
+            @instance.retry_for_error_chunk = true
+          end
+          val = super(**kwargs, &block)
           if @flush_buffer_at_cleanup
-            @instance.force_flush
-            Timeout.timeout(10){ sleep 0.1 until !@instance.buffer || @instance.buffer.queue.size == 0 }
+            self.flush
           end
+          val
+        ensure
+          @instance.retry_for_error_chunk = false
         end
 
         def formatted
@@ -54,7 +62,22 @@ module Fluent
 
         def flush
           @instance.force_flush
-          Timeout.timeout(10){ sleep 0.1 until !@instance.buffer || @instance.buffer.queue.size == 0 }
+          wait_flush_completion if @wait_flush_completion
+        end
+
+        def wait_flush_completion
+          buffer_queue = ->(){ @instance.buffer && @instance.buffer.queue.size > 0 }
+          dequeued_chunks = ->(){
+            @instance.dequeued_chunks_mutex &&
+            @instance.dequeued_chunks &&
+            @instance.dequeued_chunks_mutex.synchronize{ @instance.dequeued_chunks.size > 0 }
+          }
+
+          Timeout.timeout(10) do
+            while buffer_queue.call || dequeued_chunks.call
+              sleep 0.1
+            end
+          end
         end
 
         def instance_hook_after_started

data/lib/fluent/test/helpers.rb

@@ -23,7 +23,9 @@ module Fluent
     module Helpers
       # See "Example Custom Assertion: http://test-unit.github.io/test-unit/en/Test/Unit/Assertions.html
       def assert_equal_event_time(expected, actual, message = nil)
-        message = build_message(message, <<EOT, expected, actual)
+        expected_s = "#{Time.at(expected.sec)} (nsec #{expected.nsec})"
+        actual_s   = "#{Time.at(actual.sec)  } (nsec #{actual.nsec})"
+        message = build_message(message, <<EOT, expected_s, actual_s)
 <?> expected but was
 <?>.
 EOT

data/lib/fluent/test/log.rb

@@ -26,6 +26,7 @@ module Fluent
       def initialize
         @logs = []
         @flush_logs = true
+        @use_stderr = false
       end
 
       def reset
@@ -41,6 +42,9 @@ module Fluent
       end
 
       def write(message)
+        if @use_stderr
+          STDERR.write message
+        end
         @logs.push message
       end
 

data/lib/fluent/test/startup_shutdown.rb

@@ -15,6 +15,7 @@
 #
 
 require 'serverengine'
+require 'fileutils'
 
 module Fluent
   module Test
@@ -28,6 +29,18 @@ module Fluent
       def shutdown
         @server.close
       end
+
+      def self.setup
+        @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        @server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+      end
+
+      def self.teardown
+        @server.close
+        # on Windows, socket_manager_path is a TCP port number
+        FileUtils.rm_f @socket_manager_path unless Fluent.windows?
+      end
     end
   end
 end

data/lib/fluent/time.rb

@@ -50,6 +50,10 @@ module Fluent
       @sec
     end
 
+    def to_f
+      @sec + @nsec / 1_000_000_000.0
+    end
+
     # for Time.at
     def to_r
       Rational(@sec * 1_000_000_000 + @nsec, 1_000_000_000)
@@ -107,6 +111,8 @@ module Fluent
   end
 
   module TimeMixin
+    TIME_TYPES = ['string', 'unixtime', 'float']
+
     TIME_PARAMETERS = [
       [:time_format, :string, {default: nil}],
       [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil
@@ -115,7 +121,7 @@ module Fluent
     ]
     TIME_FULL_PARAMETERS = [
       # To avoid to define :time_type twice (in plugin_helper/inject)
-      [:time_type, :enum, {default: :string, list: [:string, :unixtime, :float]}],
+      [:time_type, :enum, {default: :string, list: TIME_TYPES.map(&:to_sym)}],
     ] + TIME_PARAMETERS
 
     module TimeParameters
@@ -197,7 +203,7 @@ module Fluent
                     else Time.now.localtime.utc_offset # utc
                     end
 
-      strptime = format && (Strptime.new(time_format) rescue nil)
+      strptime = format && (Strptime.new(format) rescue nil)
 
       @parse = case
                when format_with_timezone && strptime then ->(v){ Fluent::EventTime.from_time(strptime.exec(v)) }
@@ -248,8 +254,8 @@ module Fluent
     end
 
     def parse_unixtime(value)
-      unless value.is_a?(String)
-        raise TimeParseError, "value must be a string: #{value}"
+      unless value.is_a?(String) || value.is_a?(Numeric)
+        raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
       end
 
       if @cache1_key == value
@@ -279,8 +285,8 @@ module Fluent
     ## parse_by_to_r  (full): 28.722362 sec
     ## parse_by_to_r  (msec): 28.232856 sec
     def parse_float(value)
-      unless value.is_a?(String)
-        raise TimeParseError, "value must be a string: #{value}"
+      unless value.is_a?(String) || value.is_a?(Numeric)
+        raise TimeParseError, "value must be a string or a number: #{value}(value.class)"
       end
 
       if @cache1_key == value
@@ -290,8 +296,8 @@ module Fluent
       end
 
       begin
-        sec_s, nsec_s, _ = value.split('.', 3) # throw away second-dot and later
-        nsec_s = nsec_s[0..9]
+        sec_s, nsec_s, _ = value.to_s.split('.', 3) # throw away second-dot and later
+        nsec_s = nsec_s && nsec_s[0..9] || '0'
         nsec_s += '0' * (9 - nsec_s.size) if nsec_s.size < 9
         time = Fluent::EventTime.new(sec_s.to_i, nsec_s.to_i)
       rescue => e

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.8'
+  VERSION = '0.14.9'
 
 end

data/test/command/test_binlog_reader.rb

@@ -58,9 +58,13 @@ class TestBaseCommand < ::Test::Unit::TestCase
   def create_message_packed_file(path, times = [event_time], records = [{ 'message' => 'dummy' }])
     es = Fluent::MultiEventStream.new(times, records)
     v = es.to_msgpack_stream
-    File.open("#{TMP_DIR}/#{path}", 'w') do |f|
+    out_path = "#{TMP_DIR}/#{path}"
+    File.open(out_path, 'wb') do |f|
       f.print(v)
     end
+    waiting(5) do
+      sleep 0.5 until File.size(out_path) == v.bytesize
+    end
   end
 
   def setup

data/test/config/test_configurable.rb

@@ -334,6 +334,12 @@ module ConfigurableSpec
       end
     end
 
+    class NilOwner < Owner
+      config_section :buffer do
+        config_set_default :size_of_something, nil
+      end
+    end
+
     class FlatChild
       include Fluent::Configurable
       attr_accessor :owner
@@ -346,6 +352,19 @@ module ConfigurableSpec
       configured_in :buffer
       config_param :size_of_something, :size, default: 128
     end
+
+    class BufferBase
+      include Fluent::Configurable
+    end
+
+    class BufferSubclass < BufferBase
+      attr_accessor :owner
+      configured_in :buffer
+      config_param :size_of_something, :size, default: 512
+    end
+
+    class BufferSubSubclass < BufferSubclass
+    end
   end
   class UnRecommended
     include Fluent::Configurable
@@ -1038,6 +1057,8 @@ module Fluent::Config
       test 'for feature plugin which has flat parameters with parent' do
         owner = ConfigurableSpec::OverwriteDefaults::Owner.new
         child = ConfigurableSpec::OverwriteDefaults::FlatChild.new
+        assert_nil child.class.merged_configure_proxy.configured_in_section
+
         child.owner = owner
         child.configure(config_element('ROOT', '', {}, []))
         assert_equal "V1", child.key1
@@ -1046,6 +1067,8 @@ module Fluent::Config
       test 'for feature plugin which has parameters in subsection of parent' do
         owner = ConfigurableSpec::OverwriteDefaults::Owner.new
         child = ConfigurableSpec::OverwriteDefaults::BufferChild.new
+        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section
+
         child.owner = owner
         child.configure(config_element('ROOT', '', {}, []))
         assert_equal 1024, child.size_of_something
@@ -1054,10 +1077,48 @@ module Fluent::Config
       test 'even in subclass of owner' do
         owner = ConfigurableSpec::OverwriteDefaults::SubOwner.new
         child = ConfigurableSpec::OverwriteDefaults::BufferChild.new
+        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section
+
         child.owner = owner
         child.configure(config_element('ROOT', '', {}, []))
         assert_equal 2048, child.size_of_something
       end
+
+      test 'default values can be overwritten with nil' do
+        owner = ConfigurableSpec::OverwriteDefaults::NilOwner.new
+        child = ConfigurableSpec::OverwriteDefaults::BufferChild.new
+        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section
+
+        child.owner = owner
+        child.configure(config_element('ROOT', '', {}, []))
+        assert_nil child.size_of_something
+      end
+
+      test 'the first configured_in (in the order from base class) will be applied' do
+        child = ConfigurableSpec::OverwriteDefaults::BufferSubclass.new
+        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section
+
+        child.configure(config_element('ROOT', '', {}, []))
+        assert_equal 512, child.size_of_something
+      end
+
+      test 'the first configured_in is valid with owner classes' do
+        owner = ConfigurableSpec::OverwriteDefaults::Owner.new
+        child = ConfigurableSpec::OverwriteDefaults::BufferSubclass.new
+        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section
+
+        child.owner = owner
+        child.configure(config_element('ROOT', '', {}, []))
+        assert_equal 1024, child.size_of_something
+      end
+
+      test 'the only first configured_in is valid even in subclasses of a class with configured_in' do
+        child = ConfigurableSpec::OverwriteDefaults::BufferSubSubclass.new
+        assert_equal :buffer, child.class.merged_configure_proxy.configured_in_section
+
+        child.configure(config_element('ROOT', '', {}, []))
+        assert_equal 512, child.size_of_something
+      end
     end
 
     sub_test_case ':secret option' do
@@ -1236,5 +1297,117 @@ module Fluent::Config
         end
       end
     end
+
+    sub_test_case '#config_param without default values cause error if section is configured as init:true' do
+      setup do
+        @type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }
+        @proxy = Fluent::Config::ConfigureProxy.new(:section, type_lookup: @type_lookup)
+      end
+
+      test 'with simple config_param with default value' do
+        class InitTestClass01
+          include Fluent::Configurable
+          config_section :subsection, init: true do
+            config_param :param1, :integer, default: 1
+          end
+        end
+        c = InitTestClass01.new
+        c.configure(config_element('root', ''))
+
+        assert_equal 1, c.subsection.size
+        assert_equal 1, c.subsection.first.param1
+      end
+
+      test 'with simple config_param without default value' do
+        class InitTestClass02
+          include Fluent::Configurable
+          config_section :subsection, init: true do
+            config_param :param1, :integer
+          end
+        end
+        c = InitTestClass02.new
+        assert_raises ArgumentError.new("subsection: init is specified, but there're parameters without default values:param1") do
+          c.configure(config_element('root', ''))
+        end
+
+        c.configure(config_element('root', '', {}, [config_element('subsection', '', {'param1' => '1'})]))
+
+        assert_equal 1, c.subsection.size
+        assert_equal 1, c.subsection.first.param1
+      end
+
+      test 'with config_param with config_set_default' do
+        module InitTestModule03
+          include Fluent::Configurable
+          config_section :subsection, init: true do
+            config_param :param1, :integer
+          end
+        end
+        class InitTestClass03
+          include Fluent::Configurable
+          include InitTestModule03
+          config_section :subsection do
+            config_set_default :param1, 1
+          end
+        end
+
+        c = InitTestClass03.new
+        c.configure(config_element('root', ''))
+
+        assert_equal 1, c.subsection.size
+        assert_equal 1, c.subsection.first.param1
+      end
+
+      test 'with config_argument with default value' do
+        class InitTestClass04
+          include Fluent::Configurable
+          config_section :subsection, init: true do
+            config_argument :param0, :string, default: 'yay'
+          end
+        end
+
+        c = InitTestClass04.new
+        c.configure(config_element('root', ''))
+
+        assert_equal 1, c.subsection.size
+        assert_equal 'yay', c.subsection.first.param0
+      end
+
+      test 'with config_argument without default value' do
+        class InitTestClass04
+          include Fluent::Configurable
+          config_section :subsection, init: true do
+            config_argument :param0, :string
+          end
+        end
+
+        c = InitTestClass04.new
+        assert_raise ArgumentError.new("subsection: init is specified, but default value of argument is missing") do
+          c.configure(config_element('root', ''))
+        end
+      end
+
+      test 'with config_argument with config_set_default' do
+        module InitTestModule05
+          include Fluent::Configurable
+          config_section :subsection, init: true do
+            config_argument :param0, :string
+          end
+        end
+        class InitTestClass05
+          include Fluent::Configurable
+          include InitTestModule05
+          config_section :subsection do
+            config_set_default :param0, 'foo'
+          end
+        end
+
+        c = InitTestClass05.new
+        c.configure(config_element('root', ''))
+
+        assert_equal 1, c.subsection.size
+        assert_equal 'foo', c.subsection.first.param0
+      end
+    end
   end
 end