fluentd 0.14.0 → 0.14.1

data/test/scripts/fluent/plugin/out_test2.rb

@@ -0,0 +1,80 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent::Plugin
+  class Test2Output < Output
+    Fluent::Plugin.register_output('test2', self)
+
+    helpers :event_emitter
+
+    config_param :name, :string
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+    end
+
+    def initialize
+      super
+      @emit_streams = []
+    end
+
+    attr_reader :emit_streams
+
+    def emits
+      all = []
+      @emit_streams.each {|tag,events|
+        events.each {|time,record|
+          all << [tag, time, record]
+        }
+      }
+      all
+    end
+
+    def events
+      all = []
+      @emit_streams.each {|tag,events|
+        all.concat events
+      }
+      all
+    end
+
+    def records
+      all = []
+      @emit_streams.each {|tag,events|
+        events.each {|time,record|
+          all << record
+        }
+      }
+      all
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def process(tag, es)
+      @emit_streams << [tag, es.to_a]
+    end
+
+    def write(chunk)
+      es = Fluent::ArrayEventStream.new
+      chunk.each do |time, record|
+        es.add(time, record)
+      end
+      @emit_streams << [tag, es]
+    end
+  end
+end

data/test/test_event.rb

@@ -3,8 +3,26 @@ require 'fluent/test'
 require 'fluent/event'
 
 module EventTest
+  module DeepCopyAssertion
+    def assert_duplicated_records(es1, es2)
+      ary1 = []
+      es1.each do |_, record|
+        ary1 << record
+      end
+      ary2 = []
+      es2.each do |_, record|
+        ary2 << record
+      end
+      assert_equal ary1.size, ary2.size
+      ary1.each_with_index do |r, i|
+        assert_not_equal r.object_id, ary2[i].object_id
+      end
+    end
+  end
+
   class OneEventStreamTest < ::Test::Unit::TestCase
     include Fluent
+    include DeepCopyAssertion
 
     def setup
       @time = event_time()
@@ -12,6 +30,14 @@ module EventTest
       @es = OneEventStream.new(@time, @record)
     end
 
+    test 'empty?' do
+      assert_false @es.empty?
+    end
+
+    test 'size' do
+      assert_equal 1, @es.size
+    end
+
     test 'repeatable?' do
       assert_true @es.repeatable?
     end
@@ -20,6 +46,21 @@ module EventTest
       dupped = @es.dup
       assert_kind_of OneEventStream, dupped
       assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+    end
+
+    test 'slice' do
+      assert_equal 0, @es.slice(1, 1).size
+      assert_equal 0, @es.slice(0, 0).size
+
+      sliced = @es.slice(0, 1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @time, time
+        assert_equal @record, record
+      end
     end
 
     test 'each' do
@@ -48,6 +89,7 @@ module EventTest
 
   class ArrayEventStreamTest < ::Test::Unit::TestCase
     include Fluent
+    include DeepCopyAssertion
 
     def setup
       time = Engine.now
@@ -64,6 +106,7 @@ module EventTest
       dupped = @es.dup
       assert_kind_of ArrayEventStream, dupped
       assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
     end
 
     test 'empty?' do
@@ -93,6 +136,7 @@ module EventTest
 
   class MultiEventStreamTest < ::Test::Unit::TestCase
     include Fluent
+    include DeepCopyAssertion
 
     def setup
       time = Engine.now
@@ -112,6 +156,7 @@ module EventTest
       dupped = @es.dup
       assert_kind_of MultiEventStream, dupped
       assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
     end
 
     test 'empty?' do
@@ -141,6 +186,7 @@ module EventTest
 
   class MessagePackEventStreamTest < ::Test::Unit::TestCase
     include Fluent
+    include DeepCopyAssertion
 
     def setup
       pk = Fluent::Engine.msgpack_factory.packer
@@ -153,6 +199,17 @@ module EventTest
       @es = MessagePackEventStream.new(pk.to_s)
     end
 
+    test 'dup' do
+      dupped = @es.dup
+      assert_kind_of MessagePackEventStream, dupped
+      assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+    end
+
+    test 'empty?' do
+      assert_false @es.empty?
+    end
+
     test 'repeatable?' do
       assert_true @es.repeatable?
     end

data/test/test_formatter.rb

@@ -430,184 +430,6 @@ module FormatterTest
     end
   end
 
-  class TimeFormatterTest < ::Test::Unit::TestCase
-    include FormatterTest
-
-    def setup
-      @time = Time.new(2014, 9, 27, 0, 0, 0, 0).to_i
-      @fmt  = "%Y%m%d %H%M%z"  # YYYYMMDD HHMM[+-]HHMM
-    end
-
-    def format(format, localtime, timezone)
-      formatter = Fluent::TimeFormatter.new(format, localtime, timezone)
-      formatter.format(@time)
-    end
-
-    def test_default_utc_nil
-      assert_equal("2014-09-27T00:00:00Z", format(nil, false, nil))
-    end
-
-    def test_default_utc_pHH_MM
-      assert_equal("2014-09-27T01:30:00+01:30", format(nil, false, "+01:30"))
-    end
-
-    def test_default_utc_nHH_MM
-      assert_equal("2014-09-26T22:30:00-01:30", format(nil, false, "-01:30"))
-    end
-
-    def test_default_utc_pHHMM
-      assert_equal("2014-09-27T02:30:00+02:30", format(nil, false, "+0230"))
-    end
-
-    def test_default_utc_nHHMM
-      assert_equal("2014-09-26T21:30:00-02:30", format(nil, false, "-0230"))
-    end
-
-    def test_default_utc_pHH
-      assert_equal("2014-09-27T03:00:00+03:00", format(nil, false, "+03"))
-    end
-
-    def test_default_utc_nHH
-      assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "-03"))
-    end
-
-    def test_default_utc_timezone_1
-      # Asia/Tokyo (+09:00) does not have daylight saving time.
-      assert_equal("2014-09-27T09:00:00+09:00", format(nil, false, "Asia/Tokyo"))
-    end
-
-    def test_default_utc_timezone_2
-      # Pacific/Honolulu (-10:00) does not have daylight saving time.
-      assert_equal("2014-09-26T14:00:00-10:00", format(nil, false, "Pacific/Honolulu"))
-    end
-
-    def test_default_utc_timezone_3
-      # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.
-      assert_equal("2014-09-26T21:00:00-03:00", format(nil, false, "America/Argentina/Buenos_Aires"))
-    end
-
-    def test_default_utc_timezone_4
-      # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its
-      # UTC offset in DST is +02:00. In September, Europe/Paris is in DST.
-      assert_equal("2014-09-27T02:00:00+02:00", format(nil, false, "Europe/Paris"))
-    end
-
-    def test_default_utc_timezone_5
-      # Europe/Paris has daylight saving time. Its UTC offset is +01:00 and its
-      # UTC offset in DST is +02:00. In January, Europe/Paris is not in DST.
-      @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i
-      assert_equal("2014-01-24T01:00:00+01:00", format(nil, false, "Europe/Paris"))
-    end
-
-    def test_default_utc_invalid
-      assert_equal("2014-09-27T00:00:00Z", format(nil, false, "Invalid"))
-    end
-
-    def test_default_localtime_nil_1
-      with_timezone("UTC-04") do
-        assert_equal("2014-09-27T04:00:00+04:00", format(nil, true, nil))
-      end
-    end
-
-    def test_default_localtime_nil_2
-      with_timezone("UTC+05") do
-        assert_equal("2014-09-26T19:00:00-05:00", format(nil, true, nil))
-      end
-    end
-
-    def test_default_localtime_timezone
-      # 'timezone' takes precedence over 'localtime'.
-      with_timezone("UTC-06") do
-        assert_equal("2014-09-27T07:00:00+07:00", format(nil, true, "+07"))
-      end
-    end
-
-    def test_specific_utc_nil
-      assert_equal("20140927 0000+0000", format(@fmt, false, nil))
-    end
-
-    def test_specific_utc_pHH_MM
-      assert_equal("20140927 0830+0830", format(@fmt, false, "+08:30"))
-    end
-
-    def test_specific_utc_nHH_MM
-      assert_equal("20140926 1430-0930", format(@fmt, false, "-09:30"))
-    end
-
-    def test_specific_utc_pHHMM
-      assert_equal("20140927 1030+1030", format(@fmt, false, "+1030"))
-    end
-
-    def test_specific_utc_nHHMM
-      assert_equal("20140926 1230-1130", format(@fmt, false, "-1130"))
-    end
-
-    def test_specific_utc_pHH
-      assert_equal("20140927 1200+1200", format(@fmt, false, "+12"))
-    end
-
-    def test_specific_utc_nHH
-      assert_equal("20140926 1100-1300", format(@fmt, false, "-13"))
-    end
-
-    def test_specific_utc_timezone_1
-      # Europe/Moscow (+04:00) does not have daylight saving time.
-      assert_equal("20140927 0400+0400", format(@fmt, false, "Europe/Moscow"))
-    end
-
-    def test_specific_utc_timezone_2
-      # Pacific/Galapagos (-06:00) does not have daylight saving time.
-      assert_equal("20140926 1800-0600", format(@fmt, false, "Pacific/Galapagos"))
-    end
-
-    def test_specific_utc_timezone_3
-      # America/Argentina/Buenos_Aires (-03:00) does not have daylight saving time.
-      assert_equal("20140926 2100-0300", format(@fmt, false, "America/Argentina/Buenos_Aires"))
-    end
-
-    def test_specific_utc_timezone_4
-      # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its
-      # UTC offset in DST is -07:00. In September, America/Los_Angeles is in DST.
-      assert_equal("20140926 1700-0700", format(@fmt, false, "America/Los_Angeles"))
-    end
-
-    def test_specific_utc_timezone_5
-      # America/Los_Angeles has daylight saving time. Its UTC offset is -08:00 and its
-      # UTC offset in DST is -07:00. In January, America/Los_Angeles is not in DST.
-      @time = Time.new(2014, 1, 24, 0, 0, 0, 0).to_i
-      assert_equal("20140123 1600-0800", format(@fmt, false, "America/Los_Angeles"))
-    end
-
-    def test_specific_utc_invalid
-      assert_equal("20140927 0000+0000", format(@fmt, false, "Invalid"))
-    end
-
-    def test_specific_localtime_nil_1
-      with_timezone("UTC-07") do
-        assert_equal("20140927 0700+0700", format(@fmt, true, nil))
-      end
-    end
-
-    def test_specific_localtime_nil_2
-      with_timezone("UTC+08") do
-        assert_equal("20140926 1600-0800", format(@fmt, true, nil))
-      end
-    end
-
-    def test_specific_localtime_timezone
-      # 'timezone' takes precedence over 'localtime'.
-      with_timezone("UTC-09") do
-        assert_equal("20140926 1400-1000", format(@fmt, true, "-10"))
-      end
-    end
-
-    def test_format_with_subsec
-      time = Fluent::EventTime.new(@time)
-      formatter = Fluent::TimeFormatter.new("%Y%m%d %H%M.%N", false, nil)
-      assert_equal("20140927 0000.000000000", formatter.format(time))
-    end
-  end
-
   class TimeConfigTest < ::Test::Unit::TestCase
     include FormatterTest
 

data/test/test_output.rb

@@ -16,6 +16,7 @@ module FluentOutputTest
       def startup
         $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), 'scripts'))
         require 'fluent/plugin/out_test'
+        require 'fluent/plugin/out_test2'
       end
 
       def shutdown
@@ -64,54 +65,6 @@ module FluentOutputTest
       # assert_equal Float, d.instance.retry_wait.class
     end
 
-    def test_calc_retry_wait
-      omit "too internal test"
-
-      # default
-      d = create_driver
-      d.instance.retry_limit.times {
-        # "d.instance.instance_variable_get(:@num_errors) += 1" causes SyntaxError
-        d.instance.instance_eval { @num_errors += 1 }
-      }
-      wait = d.instance.retry_wait * (2 ** (d.instance.retry_limit - 1))
-      assert( d.instance.calc_retry_wait > wait - wait / 8.0 )
-
-      # max_retry_wait
-      d = create_driver(CONFIG + %[max_retry_wait 4])
-      d.instance.retry_limit.times {
-        d.instance.instance_eval { @num_errors += 1 }
-      }
-      assert_equal 4, d.instance.calc_retry_wait
-    end
-
-    def test_calc_retry_wait_with_integer_retry_wait
-      omit "too internal test"
-
-      d = create_driver(CONFIG + %[retry_wait 2s])
-      d.instance.retry_limit.times {
-        d.instance.instance_eval { @num_errors += 1 }
-      }
-      assert_equal true, d.instance.calc_retry_wait.finite?
-    end
-
-    def test_large_num_retries
-      omit "too internal test"
-
-      # Test that everything works properly after a very large number of
-      # retries and we hit the expected max_retry_wait.
-      exp_max_retry_wait = 300
-      d = create_driver(CONFIG + %[
-        disable_retry_limit true
-        max_retry_wait #{exp_max_retry_wait}
-      ])
-      d.instance.instance_eval { @num_errors += 1000 }
-      assert_equal exp_max_retry_wait, d.instance.calc_retry_wait
-      d.instance.instance_eval { @num_errors += 1000 }
-      assert_equal exp_max_retry_wait, d.instance.calc_retry_wait
-      d.instance.instance_eval { @num_errors += 1000 }
-      assert_equal exp_max_retry_wait, d.instance.calc_retry_wait
-    end
-
     def create_mock_driver(conf=CONFIG)
       Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
         attr_accessor :submit_flush_threads
@@ -141,78 +94,15 @@ module FluentOutputTest
       end.configure(conf)
     end
 
-    def test_submit_flush_target
-      omit "too internal test"
-
-      # default
-      d = create_mock_driver
-      d.instance.start_mock
-      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.submit_flush
-      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.submit_flush
-      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.submit_flush
-      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.submit_flush
-      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.shutdown
-      assert_equal 1, d.instance.submit_flush_threads.size
-
-      # num_threads 4
-      d = create_mock_driver(CONFIG + %[num_threads 4])
-      d.instance.start
-      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.submit_flush
-      assert_equal 1, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.submit_flush
-      assert_equal 2, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.submit_flush
-      assert_equal 3, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.submit_flush
-      assert_equal 0, d.instance.instance_variable_get('@writer_current_position')
-      d.instance.shutdown
-      assert (d.instance.submit_flush_threads.size > 1), "fails if only one thread works to submit flush"
-    end
-
     def test_secondary
       d = create_driver(CONFIG + %[
         <secondary>
-          type test
+          type test2
           name c0
         </secondary>
       ])
       assert_not_nil d.instance.instance_variable_get(:@secondary).router
     end
-
-    sub_test_case "test_force_flush" do
-      setup do
-        time = Time.parse("2011-01-02 13:14:15 UTC")
-        Timecop.freeze(time)
-        @time = time.to_i
-      end
-
-      teardown do
-        Timecop.return
-      end
-
-      test "force_flush works on retrying" do
-        omit "too internal test"
-
-        d = create_driver(CONFIG)
-        d.instance.start
-        buffer = d.instance.instance_variable_get(:@buffer)
-        # imitate 10 failures
-        d.instance.instance_variable_set(:@num_errors, 10)
-        d.instance.instance_variable_set(:@next_retry_time, @time + d.instance.calc_retry_wait)
-        # buffer should be popped (flushed) immediately
-        flexmock(buffer).should_receive(:pop).once
-        # force_flush
-        buffer.emit("test", 'test', NullOutputChain.instance)
-        d.instance.force_flush
-        10.times { sleep 0.05 }
-      end
-    end
   end
 
   class ObjectBufferedOutputTest < ::Test::Unit::TestCase
@@ -262,33 +152,6 @@ module FluentOutputTest
       Fluent::Test::TimeSlicedOutputTestDriver.new(TimeSlicedOutputTestPlugin).configure(conf, true)
     end
 
-    sub_test_case "force_flush test" do
-      setup do
-        time = Time.parse("2011-01-02 13:14:15 UTC")
-        Timecop.freeze(time)
-        @es = OneEventStream.new(time.to_i, {"message" => "foo"})
-      end
-
-      teardown do
-        Timecop.return
-      end
-
-      test "force_flush immediately flushes" do
-        omit "too internal test"
-
-        d = create_driver(CONFIG + %[
-          time_format %Y%m%d%H%M%S
-        ])
-        d.instance.start
-        # buffer should be popped (flushed) immediately
-        flexmock(d.instance.instance_variable_get(:@buffer)).should_receive(:pop).once
-        # force_flush
-        d.instance.emit('test', @es, NullOutputChain.instance)
-        d.instance.force_flush
-        10.times { sleep 0.05 }
-      end
-    end
-
     sub_test_case "test emit" do
       setup do
         @time = Time.parse("2011-01-02 13:14:15 UTC")
@@ -299,19 +162,6 @@ module FluentOutputTest
         Timecop.return
       end
 
-      test "emit with valid event" do
-        omit "there's no #emit method anymore in output plugins"
-
-        d = create_driver
-        d.instance.start
-        if d.instance.method(:emit).arity == 3
-          d.instance.emit('test', OneEventStream.new(@time.to_i, {"message" => "foo"}), NullOutputChain.instance)
-        else
-          d.instance.emit('test', OneEventStream.new(@time.to_i, {"message" => "foo"}))
-        end
-        assert_equal 0, d.instance.log.logs.size
-      end
-
       test "emit with invalid event" do
         d = create_driver
         d.instance.start