fluentd 0.12.40 → 1.6.2

data/test/plugin_helper/test_timer.rb

@@ -0,0 +1,132 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/timer'
+require 'fluent/plugin/base'
+
+class TimerTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :timer
+  end
+
+  test 'can be instantiated under state that timer is not running' do
+    d1 = Dummy.new
+    assert d1.respond_to?(:timer_running?)
+    assert !d1.timer_running?
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'can start timers by start' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    assert !d1.timer_running?
+    d1.start
+    assert d1.timer_running?
+
+    counter = 0
+    d1.timer_execute(:test, 1) do
+      counter += 1
+    end
+
+    sleep 5
+
+    d1.stop
+    assert !d1.timer_running?
+
+    assert{ counter >= 3 && counter <= 6 }
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can run many timers' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    d1.start
+
+    counter1 = 0
+    counter2 = 0
+
+    d1.timer_execute(:t1, 1) do
+      counter1 += 1
+    end
+    d1.timer_execute(:t2, 2) do
+      counter2 += 1
+    end
+
+    sleep 6
+
+    d1.stop
+
+    assert{ counter1 >= 4 && counter1 <= 7 }
+    assert{ counter2 >= 2 && counter2 <= 4 }
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'aborts timer which raises exceptions' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    d1.start
+
+    counter1 = 0
+    counter2 = 0
+
+    d1.timer_execute(:t1, 1) do
+      counter1 += 1
+    end
+    d1.timer_execute(:t2, 1) do
+      raise "abort!!!!!!" if counter2 > 1
+      counter2 += 1
+    end
+
+    sleep 5
+
+    d1.stop
+
+    assert{ counter1 >= 3 && counter1 <= 6 }
+    assert{ counter2 == 2 }
+    msg = "Unexpected error raised. Stopping the timer. title=:t2"
+    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") } }
+    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") } }
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can run at once' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    assert !d1.timer_running?
+    d1.start
+    assert d1.timer_running?
+
+    waiting_assertion = true
+    waiting_timer = true
+    counter = 0
+    d1.timer_execute(:test, 1, repeat: false) do
+      sleep(0.1) while waiting_assertion
+      counter += 1
+      waiting_timer = false
+    end
+
+    watchers = d1._event_loop.watchers.reject {|w| w.is_a?(Fluent::PluginHelper::EventLoop::DefaultWatcher) }
+    assert_equal(1, watchers.size)
+    assert(watchers.first.attached?)
+
+    waiting_assertion = false
+    sleep(0.1) while waiting_timer
+
+    assert_equal(1, counter)
+    waiting(4){ sleep 0.1 while watchers.first.attached? }
+    assert_false(watchers.first.attached?)
+    watchers = d1._event_loop.watchers.reject {|w| w.is_a?(Fluent::PluginHelper::EventLoop::DefaultWatcher) }
+    assert_equal(0, watchers.size)
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+end

data/test/scripts/exec_script.rb

@@ -17,10 +17,6 @@ def gen_raw_string(time)
   "#{time} hello"
 end
 
-def gen_invalid_json(time)
-  %({"tag": "tag1", "time": #{time}, "k1": "ok", })
-end
-
 time = ARGV.first
 time = Integer(time) rescue time
 
@@ -33,6 +29,4 @@ when 2
   print gen_msgpack(time)
 when 3
   print gen_raw_string(time)
-when 4
-  print gen_invalid_json(time)
 end

data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb

@@ -0,0 +1,7 @@
+module Fluent
+  Plugin.register_formatter(
+    'test1',
+    Proc.new { |tag, time, record|
+      "#{tag}:#{time}:#{record.size}"
+    })
+end

data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb

@@ -0,0 +1,7 @@
+module Fluent
+  Plugin.register_formatter(
+    'test2',
+    Proc.new { |tag, time, record|
+      "#{tag}:#{time}:#{record.size}"
+    })
+end

data/test/scripts/fluent/plugin/out_test.rb

@@ -14,16 +14,25 @@
 #    limitations under the License.
 #
 
-module Fluent
+require 'fluent/plugin/output'
+require 'fluent/event'
+
+module Fluent::Plugin
   class TestOutput < Output
-    Plugin.register_output('test', self)
+    Fluent::Plugin.register_output('test', self)
+
+    config_param :name, :string
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+    end
 
     def initialize
+      super
       @emit_streams = []
-      @name = nil
     end
 
-    attr_reader :emit_streams, :name
+    attr_reader :emit_streams
 
     def emits
       all = []
@@ -53,21 +62,20 @@ module Fluent
       all
     end
 
-    def configure(conf)
-      if name = conf['name']
-        @name = name
-      end
-    end
-
-    def start
+    def prefer_buffered_processing
+      false
     end
 
-    def shutdown
+    def process(tag, es)
+      @emit_streams << [tag, es.to_a]
     end
 
-    def emit(tag, es, chain)
-      chain.next
-      @emit_streams << [tag, es.to_a]
+    def write(chunk)
+      es = Fluent::ArrayEventStream.new
+      chunk.each do |time, record|
+        es.add(time, record)
+      end
+      @emit_streams << [tag, es]
     end
   end
 end

data/test/scripts/fluent/plugin/out_test2.rb

@@ -0,0 +1,80 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent::Plugin
+  class Test2Output < Output
+    Fluent::Plugin.register_output('test2', self)
+
+    helpers :event_emitter
+
+    config_param :name, :string
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+    end
+
+    def initialize
+      super
+      @emit_streams = []
+    end
+
+    attr_reader :emit_streams
+
+    def emits
+      all = []
+      @emit_streams.each {|tag,events|
+        events.each {|time,record|
+          all << [tag, time, record]
+        }
+      }
+      all
+    end
+
+    def events
+      all = []
+      @emit_streams.each {|tag,events|
+        all.concat events
+      }
+      all
+    end
+
+    def records
+      all = []
+      @emit_streams.each {|tag,events|
+        events.each {|time,record|
+          all << record
+        }
+      }
+      all
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def process(tag, es)
+      @emit_streams << [tag, es.to_a]
+    end
+
+    def write(chunk)
+      es = Fluent::ArrayEventStream.new
+      chunk.each do |time, record|
+        es.add(time, record)
+      end
+      @emit_streams << [tag, es]
+    end
+  end
+end

data/test/test_clock.rb

@@ -0,0 +1,164 @@
+require_relative 'helper'
+require 'fluent/clock'
+
+require 'timecop'
+
+class ClockTest < ::Test::Unit::TestCase
+  teardown do
+    Fluent::Clock.return # call it always not to affect other tests
+  end
+
+  sub_test_case 'without any pre-operation' do
+    test 'clock can provides incremental floating point number based on second' do
+      c1 = Fluent::Clock.now
+      assert_kind_of Float, c1
+      sleep 1.1
+      c2 = Fluent::Clock.now
+      assert{ c2 >= c1 + 1.0 && c2 < c1 + 9.0 } # if clock returns deci-second (fantastic!), c2 should be larger than c1 + 10
+    end
+
+    test 'clock value will proceed even if timecop freezes Time' do
+      Timecop.freeze(Time.now) do
+        c1 = Fluent::Clock.now
+        assert_kind_of Float, c1
+        sleep 1.1
+        c2 = Fluent::Clock.now
+        assert{ c2 >= c1 + 1.0 && c2 < c1 + 9.0 }
+      end
+    end
+  end
+
+  sub_test_case 'using #freeze without any arguments' do
+    test 'Clock.freeze without arguments freezes clock with current clock value' do
+      c0 = Fluent::Clock.now
+      Fluent::Clock.freeze
+      c1 = Fluent::Clock.now
+      Fluent::Clock.return
+      c2 = Fluent::Clock.now
+      assert{ c0 <= c1 && c1 <= c2 }
+    end
+
+    test 'Clock.return raises an error if it is called in block' do
+      assert_raise RuntimeError.new("invalid return while running code in blocks") do
+        Fluent::Clock.freeze do
+          Fluent::Clock.return
+        end
+      end
+    end
+  end
+
+  sub_test_case 'using #freeze with clock value' do
+    test 'Clock.now always returns frozen time until #return called' do
+      c0 = Fluent::Clock.now
+      Fluent::Clock.freeze(c0)
+      assert_equal c0, Fluent::Clock.now
+      sleep 0.5
+      assert_equal c0, Fluent::Clock.now
+      sleep 0.6
+      assert_equal c0, Fluent::Clock.now
+
+      Fluent::Clock.return
+      c1 = Fluent::Clock.now
+      assert{ c1 >= c0 + 1.0 }
+    end
+
+    test 'Clock.now returns frozen time in the block argument of #freeze' do
+      c0 = Fluent::Clock.now
+      Fluent::Clock.freeze(c0) do
+        assert_equal c0, Fluent::Clock.now
+        sleep 0.5
+        assert_equal c0, Fluent::Clock.now
+        sleep 0.6
+        assert_equal c0, Fluent::Clock.now
+      end
+      c1 = Fluent::Clock.now
+      assert{ c1 >= c0 + 1.0 }
+    end
+
+    test 'Clock.now returns unfrozen value after jumping out from block by raising errors' do
+      c0 = Fluent::Clock.now
+      rescued_error = nil
+      begin
+        Fluent::Clock.freeze(c0) do
+          assert_equal c0, Fluent::Clock.now
+          sleep 0.5
+          assert_equal c0, Fluent::Clock.now
+          sleep 0.6
+          assert_equal c0, Fluent::Clock.now
+          raise "bye!"
+        end
+      rescue => e
+        rescued_error = e
+      end
+      assert rescued_error # ensure to rescue an error
+      c1 = Fluent::Clock.now
+      assert{ c1 >= c0 + 1.0 }
+    end
+
+    test 'Clock.return cancels all Clock.freeze effects by just once' do
+      c0 = Fluent::Clock.now
+      sleep 0.1
+      c1 = Fluent::Clock.now
+      sleep 0.1
+      c2 = Fluent::Clock.now
+      Fluent::Clock.freeze(c0)
+      sleep 0.1
+      assert_equal c0, Fluent::Clock.now
+      Fluent::Clock.freeze(c1)
+      sleep 0.1
+      assert_equal c1, Fluent::Clock.now
+      Fluent::Clock.freeze(c2)
+      sleep 0.1
+      assert_equal c2, Fluent::Clock.now
+
+      Fluent::Clock.return
+      assert{ Fluent::Clock.now > c2 }
+    end
+
+    test 'Clock.freeze allows nested blocks by itself' do
+      c0 = Fluent::Clock.now
+      sleep 0.1
+      c1 = Fluent::Clock.now
+      sleep 0.1
+      c2 = Fluent::Clock.now
+      Fluent::Clock.freeze(c0) do
+        sleep 0.1
+        assert_equal c0, Fluent::Clock.now
+        Fluent::Clock.freeze(c1) do
+          sleep 0.1
+          assert_equal c1, Fluent::Clock.now
+          Fluent::Clock.freeze(c2) do
+            sleep 0.1
+            assert_equal c2, Fluent::Clock.now
+          end
+          assert_equal c1, Fluent::Clock.now
+        end
+        assert_equal c0, Fluent::Clock.now
+      end
+      assert{ Fluent::Clock.now > c0 }
+    end
+  end
+
+  sub_test_case 'using #freeze with Time argument' do
+    test 'Clock.freeze returns the clock value which should be produced when the time is at the specified time' do
+      c0 = Fluent::Clock.now
+      t0 = Time.now
+      t1 = t0 - 30
+      assert_kind_of Time, t1
+      t2 = t0 + 30
+      assert_kind_of Time, t2
+
+      # 31 is for error of floating point value
+      Fluent::Clock.freeze(t1) do
+        c1 = Fluent::Clock.now
+        assert{ c1 >= c0 - 31 && c1 <= c0 - 31 + 10 } # +10 is for threading schedule error
+      end
+
+      # 29 is for error of floating point value
+      Fluent::Clock.freeze(t2) do
+        c2 = Fluent::Clock.now
+        assert{ c2 >= c0 + 29 && c2 <= c0 + 29 + 10 } # +10 is for threading schedule error
+      end
+    end
+  end
+end

data/test/test_config.rb

@@ -53,7 +53,7 @@ class ConfigTest < Test::Unit::TestCase
       </elem2>
     ]
     write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
-      k9 embeded
+      k9 embedded
       <elem3 name>
         nested nested_value
         include hoge
@@ -98,7 +98,7 @@ class ConfigTest < Test::Unit::TestCase
     elem2 = c.elements.find { |e| e.name == 'elem2' }
     assert_not_nil elem2
     assert_equal 'name', elem2.arg
-    assert_equal 'embeded', elem2['k9']
+    assert_equal 'embedded', elem2['k9']
     assert !elem2.has_key?('include')
 
     elem3 = elem2.elements.find { |e| e.name == 'elem3' }
@@ -145,7 +145,7 @@ class ConfigTest < Test::Unit::TestCase
     not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
     assert_equal %w[pattern replace], not_fetched
 
-    # repeateadly accessing should not grow memory usage
+    # repeatedly accessing should not grow memory usage
     before_size = match_conf.unused.size
     10.times { match_conf['type'] }
     assert_equal before_size, match_conf.unused.size
@@ -157,14 +157,23 @@ class ConfigTest < Test::Unit::TestCase
   end
 
   def test_inline
-    prepare_config 
+    prepare_config
     opts = {
-      config_path: "#{TMP_DIR}/config_test_1.conf",
-      inline_config: "<source>\n  type http\n  port 2222\n </source>"
+        :config_path => "#{TMP_DIR}/config_test_1.conf",
+        :inline_config => "<source>\n  type http\n  port 2222\n </source>"
     }
     assert_nothing_raised do
       Fluent::Supervisor.new(opts)
-    end  
+    end
+    create_warn_dummy_logger
+  end
+
+  def create_warn_dummy_logger
+    dl_opts = {}
+    dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+    logdev = Fluent::Test::DummyLogDevice.new
+    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+    $log = Fluent::Log.new(logger)
   end
 end
 

data/test/test_configdsl.rb

@@ -65,13 +65,13 @@ match('aa')
     e0 = root.elements[0]
     assert_equal 'source', e0.name
     assert_equal '', e0.arg
-    assert_equal 'forward', e0['type']
+    assert_equal 'forward', e0['@type']
     assert_equal '24224', e0['port']
 
     e1 = root.elements[1]
     assert_equal 'match', e1.name
     assert_equal 'test.**', e1.arg
-    assert_equal 'forward', e1['type']
+    assert_equal 'forward', e1['@type']
     assert_equal '1s', e1['flush_interval']
     assert_equal 2, e1.elements.size
     e1s0 = e1.elements[0]