fluentd 0.12.43 → 0.14.0

data/test/plugin_helper/test_timer.rb

@@ -0,0 +1,100 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/timer'
+require 'fluent/plugin/base'
+
+class TimerTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :timer
+  end
+
+  test 'can be instantiated under state that timer is not running' do
+    d1 = Dummy.new
+    assert d1.respond_to?(:timer_running?)
+    assert !d1.timer_running?
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'can start timers by start' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    assert !d1.timer_running?
+    d1.start
+    assert d1.timer_running?
+
+    counter = 0
+    d1.timer_execute(:test, 1) do
+      counter += 1
+    end
+
+    sleep 5
+
+    d1.stop
+    assert !d1.timer_running?
+
+    assert{ counter >= 3 && counter <= 6 }
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can run many timers' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    d1.start
+
+    counter1 = 0
+    counter2 = 0
+
+    d1.timer_execute(:t1, 1) do
+      counter1 += 1
+    end
+    d1.timer_execute(:t2, 2) do
+      counter2 += 1
+    end
+
+    sleep 6
+
+    d1.stop
+
+    assert{ counter1 >= 4 && counter1 <= 7 }
+    assert{ counter2 >= 2 && counter2 <= 4 }
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'aborts timer which raises exceptions' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    d1.start
+
+    counter1 = 0
+    counter2 = 0
+
+    d1.timer_execute(:t1, 1) do
+      counter1 += 1
+    end
+    d1.timer_execute(:t2, 1) do
+      raise "abort!!!!!!" if counter2 > 1
+      counter2 += 1
+    end
+
+    sleep 5
+
+    d1.stop
+
+    assert{ counter1 >= 3 && counter1 <= 6 }
+    assert{ counter2 == 2 }
+    msg = "Unexpected error raised. Stopping the timer. title=:t2"
+    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") } }
+    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") } }
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+end

data/test/scripts/exec_script.rb

@@ -17,10 +17,6 @@ def gen_raw_string(time)
   "#{time} hello"
 end
 
-def gen_invalid_json(time)
-  %({"tag": "tag1", "time": #{time}, "k1": "ok", })
-end
-
 time = ARGV.first
 time = Integer(time) rescue time
 
@@ -33,6 +29,4 @@ when 2
   print gen_msgpack(time)
 when 3
   print gen_raw_string(time)
-when 4
-  print gen_invalid_json(time)
 end

data/test/scripts/fluent/plugin/out_test.rb

@@ -19,6 +19,7 @@ module Fluent
     Plugin.register_output('test', self)
 
     def initialize
+      super
       @emit_streams = []
       @name = nil
     end
@@ -60,9 +61,11 @@ module Fluent
     end
 
     def start
+      super
     end
 
     def shutdown
+      super
     end
 
     def emit(tag, es, chain)

data/test/test_config.rb

@@ -157,14 +157,23 @@ class ConfigTest < Test::Unit::TestCase
   end
 
   def test_inline
-    prepare_config 
+    prepare_config
     opts = {
-      config_path: "#{TMP_DIR}/config_test_1.conf",
-      inline_config: "<source>\n  type http\n  port 2222\n </source>"
+        :config_path => "#{TMP_DIR}/config_test_1.conf",
+        :inline_config => "<source>\n  type http\n  port 2222\n </source>"
     }
     assert_nothing_raised do
       Fluent::Supervisor.new(opts)
-    end  
+    end
+    create_warn_dummy_logger
+  end
+
+  def create_warn_dummy_logger
+    dl_opts = {}
+    dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+    logdev = Fluent::Test::DummyLogDevice.new
+    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+    $log = Fluent::Log.new(logger)
   end
 end
 

data/test/test_event.rb

@@ -7,7 +7,7 @@ module EventTest
     include Fluent
 
     def setup
-      @time = Engine.now
+      @time = event_time()
       @record = {'k' => 'v', 'n' => 1}
       @es = OneEventStream.new(@time, @record)
     end
@@ -31,18 +31,27 @@ module EventTest
 
     test 'to_msgpack_stream' do
       stream = @es.to_msgpack_stream
-      MessagePack::Unpacker.new.feed_each(stream) { |time, record|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
         assert_equal @time, time
         assert_equal @record, record
       }
     end
+
+    test 'to_msgpack_stream with time_int argument' do
+      stream = @es.to_msgpack_stream(time_int: true)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @time.to_i, time
+        assert_equal @record, record
+      }
+    end
   end
 
   class ArrayEventStreamTest < ::Test::Unit::TestCase
     include Fluent
 
     def setup
-      @times = [Engine.now, Engine.now + 1]
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
       @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
       @es = ArrayEventStream.new(@times.zip(@records))
     end
@@ -74,7 +83,7 @@ module EventTest
     test 'to_msgpack_stream' do
       i = 0
       stream = @es.to_msgpack_stream
-      MessagePack::Unpacker.new.feed_each(stream) { |time, record|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
@@ -86,11 +95,12 @@ module EventTest
     include Fluent
 
     def setup
-      @times = [Engine.now, Engine.now + 1]
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
       @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
       @es = MultiEventStream.new
-      @times.zip(@records).each { |time, record|
-        @es.add(time, record)
+      @times.zip(@records).each { |_time, record|
+        @es.add(_time, record)
       }
     end
 
@@ -121,7 +131,7 @@ module EventTest
     test 'to_msgpack_stream' do
       i = 0
       stream = @es.to_msgpack_stream
-      MessagePack::Unpacker.new.feed_each(stream) { |time, record|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1
@@ -133,11 +143,12 @@ module EventTest
     include Fluent
 
     def setup
-      pk = MessagePack::Packer.new
-      @times = [Engine.now, Engine.now + 1]
+      pk = Fluent::Engine.msgpack_factory.packer
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
       @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
-      @times.zip(@records).each { |time, record|
-        pk.write([time, record])
+      @times.zip(@records).each { |_time, record|
+        pk.write([_time, record])
       }
       @es = MessagePackEventStream.new(pk.to_s)
     end
@@ -158,7 +169,7 @@ module EventTest
     test 'to_msgpack_stream' do
       i = 0
       stream = @es.to_msgpack_stream
-      MessagePack::Unpacker.new.feed_each(stream) { |time, record|
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
         assert_equal @times[i], time
         assert_equal @records[i], record
         i += 1

data/test/test_event_router.rb

@@ -97,7 +97,7 @@ class EventRouterTest < ::Test::Unit::TestCase
 
     test 'set one output' do
       @pipeline.set_output(output)
-      @pipeline.emit('test', @es, nil)
+      @pipeline.emit_events('test', @es)
       assert_equal 1, output.events.size
       assert_equal 'value', output.events['test'].first['key']
     end
@@ -109,7 +109,7 @@ class EventRouterTest < ::Test::Unit::TestCase
 
       test 'set one filer' do
         @pipeline.add_filter(filter)
-        @pipeline.emit('test', @es, nil)
+        @pipeline.emit_events('test', @es)
         assert_equal 1, output.events.size
         assert_equal 'value', output.events['test'].first['key']
         assert_equal 0, output.events['test'].first['__test__']
@@ -117,7 +117,7 @@ class EventRouterTest < ::Test::Unit::TestCase
 
       test 'set one filer with multi events' do
         @pipeline.add_filter(filter)
-        @pipeline.emit('test', events, nil)
+        @pipeline.emit_events('test', events)
         assert_equal 1, output.events.size
         assert_equal 5, output.events['test'].size
         DEFAULT_EVENT_NUM.times { |i|
@@ -140,7 +140,7 @@ class EventRouterTest < ::Test::Unit::TestCase
     sub_test_case 'default collector' do
       test 'call default collector when no output' do
         assert_rr do
-          mock(default_collector).emit('test', is_a(OneEventStream), NullOutputChain.instance)
+          mock(default_collector).emit_events('test', is_a(OneEventStream))
           event_router.emit('test', Engine.now, 'k' => 'v')
         end
       end
@@ -149,7 +149,7 @@ class EventRouterTest < ::Test::Unit::TestCase
         event_router.add_rule('test', filter)
         assert_rr do
           # After apply Filter, EventStream becomes MultiEventStream by default
-          mock(default_collector).emit('test', is_a(MultiEventStream), NullOutputChain.instance)
+          mock(default_collector).emit_events('test', is_a(MultiEventStream))
           event_router.emit('test', Engine.now, 'k' => 'v')
         end
         assert_equal 1, filter.num
@@ -158,7 +158,7 @@ class EventRouterTest < ::Test::Unit::TestCase
       test "call default collector when no matched with output" do
         event_router.add_rule('test', output)
         assert_rr do
-          mock(default_collector).emit('dummy', is_a(OneEventStream), NullOutputChain.instance)
+          mock(default_collector).emit_events('dummy', is_a(OneEventStream))
           event_router.emit('dummy', Engine.now, 'k' => 'v')
         end
       end
@@ -166,7 +166,7 @@ class EventRouterTest < ::Test::Unit::TestCase
       test "don't call default collector when tag matched" do
         event_router.add_rule('test', output)
         assert_rr do
-          dont_allow(default_collector).emit('test', is_a(OneEventStream), NullOutputChain.instance)
+          dont_allow(default_collector).emit_events('test', is_a(OneEventStream))
           event_router.emit('test', Engine.now, 'k' => 'v')
         end
         # check emit handler doesn't catch rr error
@@ -224,6 +224,7 @@ class EventRouterTest < ::Test::Unit::TestCase
       test 'call handle_emits_error when emit failed' do
         event_router.add_rule('test', error_output)
 
+        event_router.emit('test', Engine.now, 'k' => 'v')
         assert_rr do
           mock(emit_handler).handle_emits_error('test', is_a(OneEventStream), is_a(RuntimeError))
           event_router.emit('test', Engine.now, 'k' => 'v')

data/test/test_event_time.rb

@@ -0,0 +1,187 @@
+require_relative 'helper'
+require 'timecop'
+require 'oj'
+require 'yajl'
+
+class EventTimeTest < Test::Unit::TestCase
+  setup do
+    @now = Time.now
+    Timecop.freeze(@now)
+  end
+
+  teardown do
+    Timecop.return
+  end
+
+  test '#sec' do
+    assert_equal(1, Fluent::EventTime.new(1, 2).sec)
+  end
+
+  test '#nsec' do
+    assert_equal(2, Fluent::EventTime.new(1, 2).nsec)
+    assert_equal(0, Fluent::EventTime.new(1).nsec)
+  end
+
+  test '#to_int' do
+    assert_equal(1, Fluent::EventTime.new(1, 2).to_int)
+  end
+
+  test '#to_r' do
+    assert_equal(Rational(1_000_000_002, 1_000_000_000), Fluent::EventTime.new(1, 2).to_r)
+  end
+
+  test '#to_s' do
+    time = Fluent::EventTime.new(100)
+    assert_equal('100', time.to_s)
+    assert_equal('100', "#{time}")
+  end
+
+  test '#to_json' do
+    time = Fluent::EventTime.new(100)
+    assert_equal('100', time.to_json)
+    assert_equal('{"time":100}', {'time' => time}.to_json)
+    assert_equal('["tag",100,{"key":"value"}]', ["tag", time, {"key" => "value"}].to_json)
+  end
+
+  test 'JSON.dump' do
+    time = Fluent::EventTime.new(100)
+    assert_equal('100', JSON.dump(time))
+    assert_equal('{"time":100}', JSON.dump({'time' => time}))
+    assert_equal('["tag",100,{"key":"value"}]', JSON.dump(["tag", time, {"key" => "value"}]))
+  end
+
+  test 'Oj.dump' do
+    time = Fluent::EventTime.new(100)
+    assert_equal('100', Oj.dump(time, mode: :compat))
+    assert_equal('{"time":100}', Oj.dump({'time' => time}, mode: :compat))
+    assert_equal('["tag",100,{"key":"value"}]', Oj.dump(["tag", time, {"key" => "value"}], mode: :compat))
+  end
+
+  test 'Yajl.dump' do
+    time = Fluent::EventTime.new(100)
+    assert_equal('100', Yajl.dump(time))
+    assert_equal('{"time":100}', Yajl.dump({'time' => time}))
+    assert_equal('["tag",100,{"key":"value"}]', Yajl.dump(["tag", time, {"key" => "value"}]))
+  end
+
+  test '.from_time' do
+    sec = 1000
+    usec = 2
+    time = Fluent::EventTime.from_time(Time.at(sec, usec))
+    assert_equal(time.sec, sec)
+    assert_equal(time.nsec, usec * 1000)
+  end
+
+  test 'now' do
+    assert_equal(@now.to_i, Fluent::EventTime.now.sec)
+    assert_equal(@now.nsec, Fluent::EventTime.now.nsec)
+  end
+
+  test 'parse' do
+    assert_equal(Time.parse("2011-01-02 13:14:15").to_i, Fluent::EventTime.parse("2011-01-02 13:14:15").sec)
+    assert_equal(Time.parse("2011-01-02 13:14:15").nsec, Fluent::EventTime.parse("2011-01-02 13:14:15").nsec)
+  end
+
+  test 'eq?' do
+    assert(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), Fluent::EventTime.new(1, 2)))
+    refute(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), Fluent::EventTime.new(1, 3)))
+    refute(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), Fluent::EventTime.new(3, 2)))
+    refute(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), Fluent::EventTime.new(3, 4)))
+
+    assert(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), 1))
+    refute(Fluent::EventTime.eq?(Fluent::EventTime.new(1, 2), 2))
+
+    assert(Fluent::EventTime.eq?(1, Fluent::EventTime.new(1, 2)))
+    refute(Fluent::EventTime.eq?(2, Fluent::EventTime.new(1, 2)))
+  end
+
+  test '==' do
+    assert(Fluent::EventTime.new(1, 2) == Fluent::EventTime.new(1, 2))
+    assert(Fluent::EventTime.new(1, 2) == Fluent::EventTime.new(1, 3))
+    refute(Fluent::EventTime.new(1, 2) == Fluent::EventTime.new(3, 2))
+    refute(Fluent::EventTime.new(1, 2) == Fluent::EventTime.new(3, 4))
+
+    assert(Fluent::EventTime.new(1, 2) == 1)
+    refute(Fluent::EventTime.new(1, 2) == 2)
+
+    assert(1 == Fluent::EventTime.new(1, 2))
+    refute(2 == Fluent::EventTime.new(1, 2))
+  end
+
+  test '+' do
+    assert_equal(4, Fluent::EventTime.new(1, 2) + Fluent::EventTime.new(3, 4))
+    assert_equal(6, Fluent::EventTime.new(1, 2) + 5)
+    assert_equal(6, 5 + Fluent::EventTime.new(1, 2))
+  end
+
+  test '-' do
+    assert_equal(-2, Fluent::EventTime.new(1, 2) - Fluent::EventTime.new(3, 4))
+    assert_equal(-4, Fluent::EventTime.new(1, 2) - 5)
+    assert_equal(4, 5 - Fluent::EventTime.new(1, 2))
+  end
+
+  test '>' do
+    assert(Fluent::EventTime.new(2) > Fluent::EventTime.new(1))
+    refute(Fluent::EventTime.new(1) > Fluent::EventTime.new(1))
+    refute(Fluent::EventTime.new(1) > Fluent::EventTime.new(2))
+
+    assert(Fluent::EventTime.new(2) > 1)
+    refute(Fluent::EventTime.new(1) > 1)
+    refute(Fluent::EventTime.new(1) > 2)
+
+    assert(2 > Fluent::EventTime.new(1))
+    refute(1 > Fluent::EventTime.new(1))
+    refute(1 > Fluent::EventTime.new(2))
+  end
+
+  test '>=' do
+    assert(Fluent::EventTime.new(2) >= Fluent::EventTime.new(1))
+    assert(Fluent::EventTime.new(1) >= Fluent::EventTime.new(1))
+    refute(Fluent::EventTime.new(1) >= Fluent::EventTime.new(2))
+
+    assert(Fluent::EventTime.new(2) >= 1)
+    assert(Fluent::EventTime.new(1) >= 1)
+    refute(Fluent::EventTime.new(1) >= 2)
+
+    assert(2 >= Fluent::EventTime.new(1))
+    assert(1 >= Fluent::EventTime.new(1))
+    refute(1 >= Fluent::EventTime.new(2))
+  end
+
+  test '<' do
+    assert(Fluent::EventTime.new(1) < Fluent::EventTime.new(2))
+    refute(Fluent::EventTime.new(1) < Fluent::EventTime.new(1))
+    refute(Fluent::EventTime.new(2) < Fluent::EventTime.new(1))
+
+    assert(Fluent::EventTime.new(1) < 2)
+    refute(Fluent::EventTime.new(1) < 1)
+    refute(Fluent::EventTime.new(2) < 1)
+
+    assert(1 < Fluent::EventTime.new(2))
+    refute(1 < Fluent::EventTime.new(1))
+    refute(2 < Fluent::EventTime.new(1))
+  end
+
+  test '=<' do
+    assert(Fluent::EventTime.new(1) <= Fluent::EventTime.new(2))
+    assert(Fluent::EventTime.new(1) <= Fluent::EventTime.new(1))
+    refute(Fluent::EventTime.new(2) <= Fluent::EventTime.new(1))
+
+    assert(Fluent::EventTime.new(1) <= 2)
+    assert(Fluent::EventTime.new(1) <= 1)
+    refute(Fluent::EventTime.new(2) <= 1)
+
+    assert(1 <= Fluent::EventTime.new(2))
+    assert(1 <= Fluent::EventTime.new(1))
+    refute(2 <= Fluent::EventTime.new(1))
+  end
+
+  test 'Time.at' do
+    sec = 1000
+    nsec = 2000
+    ntime = Fluent::EventTime.new(sec, nsec)
+    time = Time.at(ntime)
+    assert_equal(sec, time.to_i)
+    assert_equal(nsec, time.nsec)
+  end
+end