fluentd 0.14.4-x64-mingw32

data/test/plugin_helper/test_thread.rb

@@ -0,0 +1,164 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/thread'
+require 'fluent/plugin/base'
+require 'timeout'
+
+class ThreadTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :thread
+    def configure(conf)
+      super
+      @_thread_wait_seconds = 0.1
+      self
+    end
+  end
+
+  test 'can be instantiated to be able to create threads' do
+    d1 = Dummy.new
+    assert d1.respond_to?(:thread_current_running?)
+    assert d1.respond_to?(:thread_create)
+    assert d1.respond_to?(:_threads)
+    assert !d1.thread_current_running?
+    assert d1._threads.empty?
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'can create thread after prepared' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    d1.start
+
+    m1 = Mutex.new
+    m2 = Mutex.new
+
+    m1.lock
+    thread_run = false
+
+    Timeout.timeout(10) do
+      t = d1.thread_create(:test1) do
+        m2.lock
+
+        assert !d1._threads.empty? # this must be true always
+        assert d1.thread_current_running?
+
+        thread_run = true
+        m2.unlock
+        m1.lock
+      end
+      Thread.pass until m2.locked? || thread_run
+
+      m2.lock; m2.unlock
+      assert_equal 1, d1._threads.size
+
+      assert_equal :test1, t[:_fluentd_plugin_helper_thread_title]
+      assert t[:_fluentd_plugin_helper_thread_running]
+      assert !d1._threads.empty?
+
+      m1.unlock
+
+      while t[:_fluentd_plugin_helper_thread_running]
+        Thread.pass
+      end
+    end
+
+    assert d1._threads.empty?
+
+    d1.stop; d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can wait until all threads start' do
+    d1 = Dummy.new.configure(config_element()).start
+    ary = []
+    d1.thread_create(:t1) do
+      ary << 1
+    end
+    d1.thread_create(:t2) do
+      ary << 2
+    end
+    d1.thread_create(:t3) do
+      ary << 3
+    end
+    Timeout.timeout(10) do
+      d1.thread_wait_until_start
+    end
+    assert_equal [1,2,3], ary.sort
+
+    d1.stop; d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can stop threads which is watching thread_current_running?, and then close it' do
+    d1 = Dummy.new.configure(config_element()).start
+
+    m1 = Mutex.new
+    thread_in_run = false
+    Timeout.timeout(10) do
+      t = d1.thread_create(:test2) do
+        thread_in_run = true
+        m1.lock
+        while d1.thread_current_running?
+          Thread.pass
+        end
+        thread_in_run = false
+        m1.unlock
+      end
+      Thread.pass until m1.locked?
+
+      assert thread_in_run
+      assert !d1._threads.empty?
+
+      d1.stop
+      Thread.pass while m1.locked?
+      assert !t[:_fluentd_plugin_helper_thread_running]
+      assert t.stop?
+    end
+
+    assert d1._threads.empty?
+
+    d1.stop; d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can terminate threads forcedly which is running forever' do
+    d1 = Dummy.new.configure(config_element()).start
+
+    m1 = Mutex.new
+    thread_in_run = false
+    Timeout.timeout(10) do
+      t = d1.thread_create(:test2) do
+        thread_in_run = true
+        m1.lock
+        while true
+          Thread.pass
+        end
+        thread_in_run = false
+      end
+      Thread.pass until m1.locked?
+
+      assert thread_in_run
+      assert !d1._threads.empty?
+
+      d1.stop
+      assert !t[:_fluentd_plugin_helper_thread_running]
+      assert t.alive?
+
+      d1.shutdown
+      assert t.alive?
+      assert !d1._threads.empty?
+
+      d1.close
+      assert t.alive?
+      assert !d1._threads.empty?
+
+      d1.terminate
+      assert t.stop?
+      assert d1._threads.empty?
+    end
+  end
+end

data/test/plugin_helper/test_timer.rb

@@ -0,0 +1,131 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/timer'
+require 'fluent/plugin/base'
+
+class TimerTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :timer
+  end
+
+  test 'can be instantiated under state that timer is not running' do
+    d1 = Dummy.new
+    assert d1.respond_to?(:timer_running?)
+    assert !d1.timer_running?
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'can start timers by start' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    assert !d1.timer_running?
+    d1.start
+    assert d1.timer_running?
+
+    counter = 0
+    d1.timer_execute(:test, 1) do
+      counter += 1
+    end
+
+    sleep 5
+
+    d1.stop
+    assert !d1.timer_running?
+
+    assert{ counter >= 3 && counter <= 6 }
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can run many timers' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    d1.start
+
+    counter1 = 0
+    counter2 = 0
+
+    d1.timer_execute(:t1, 1) do
+      counter1 += 1
+    end
+    d1.timer_execute(:t2, 2) do
+      counter2 += 1
+    end
+
+    sleep 6
+
+    d1.stop
+
+    assert{ counter1 >= 4 && counter1 <= 7 }
+    assert{ counter2 >= 2 && counter2 <= 4 }
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'aborts timer which raises exceptions' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    d1.start
+
+    counter1 = 0
+    counter2 = 0
+
+    d1.timer_execute(:t1, 1) do
+      counter1 += 1
+    end
+    d1.timer_execute(:t2, 1) do
+      raise "abort!!!!!!" if counter2 > 1
+      counter2 += 1
+    end
+
+    sleep 5
+
+    d1.stop
+
+    assert{ counter1 >= 3 && counter1 <= 6 }
+    assert{ counter2 == 2 }
+    msg = "Unexpected error raised. Stopping the timer. title=:t2"
+    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") } }
+    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") } }
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can run at once' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    assert !d1.timer_running?
+    d1.start
+    assert d1.timer_running?
+
+    waiting_assertion = true
+    waiting_timer = true
+    counter = 0
+    d1.timer_execute(:test, 1, repeat: false) do
+      sleep(0.1) while waiting_assertion
+      counter += 1
+      waiting_timer = false
+    end
+
+    watchers = d1._event_loop.watchers.reject {|w| w.is_a?(Fluent::PluginHelper::EventLoop::DefaultWatcher) }
+    assert_equal(1, watchers.size)
+    assert(watchers.first.attached?)
+
+    waiting_assertion = false
+    sleep(0.1) while waiting_timer
+
+    assert_equal(1, counter)
+    assert_false(watchers.first.attached?)
+    watchers = d1._event_loop.watchers.reject {|w| w.is_a?(Fluent::PluginHelper::EventLoop::DefaultWatcher) }
+    assert_equal(0, watchers.size)
+
+    d1.shutdown; d1.close; d1.terminate
+  end
+end

data/test/scripts/exec_script.rb

@@ -0,0 +1,32 @@
+require 'json'
+require 'msgpack'
+
+def gen_tsv(time)
+  "#{time}\ttag1\tok"
+end
+
+def gen_json(time)
+  {'tag' => 'tag1', 'time' => time, 'k1' => 'ok'}.to_json
+end
+
+def gen_msgpack(time)
+  {'tagger' => 'tag1', 'datetime' => time, 'k1' => 'ok'}.to_msgpack
+end
+
+def gen_raw_string(time)
+  "#{time} hello"
+end
+
+time = ARGV.first
+time = Integer(time) rescue time
+
+case ARGV.last.to_i
+when 0
+  puts gen_tsv(time)
+when 1
+  puts gen_json(time)
+when 2
+  print gen_msgpack(time)
+when 3
+  print gen_raw_string(time)
+end

data/test/scripts/fluent/plugin/formatter_known.rb

@@ -0,0 +1,8 @@
+module Fluent
+  TextFormatter.register_template('known_old', Proc.new { |tag, time, record|
+      "#{tag}:#{time}:#{record.size}"
+    })
+  Plugin.register_formatter('known', Proc.new { |tag, time, record|
+      "#{tag}:#{time}:#{record.size}"
+    })
+end

data/test/scripts/fluent/plugin/out_test.rb

@@ -0,0 +1,81 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+require 'fluent/event'
+
+module Fluent::Plugin
+  class TestOutput < Output
+    Fluent::Plugin.register_output('test', self)
+
+    config_param :name, :string
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+    end
+
+    def initialize
+      super
+      @emit_streams = []
+    end
+
+    attr_reader :emit_streams
+
+    def emits
+      all = []
+      @emit_streams.each {|tag,events|
+        events.each {|time,record|
+          all << [tag, time, record]
+        }
+      }
+      all
+    end
+
+    def events
+      all = []
+      @emit_streams.each {|tag,events|
+        all.concat events
+      }
+      all
+    end
+
+    def records
+      all = []
+      @emit_streams.each {|tag,events|
+        events.each {|time,record|
+          all << record
+        }
+      }
+      all
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def process(tag, es)
+      @emit_streams << [tag, es.to_a]
+    end
+
+    def write(chunk)
+      es = Fluent::ArrayEventStream.new
+      chunk.each do |time, record|
+        es.add(time, record)
+      end
+      @emit_streams << [tag, es]
+    end
+  end
+end

data/test/scripts/fluent/plugin/out_test2.rb

@@ -0,0 +1,80 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent::Plugin
+  class Test2Output < Output
+    Fluent::Plugin.register_output('test2', self)
+
+    helpers :event_emitter
+
+    config_param :name, :string
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+    end
+
+    def initialize
+      super
+      @emit_streams = []
+    end
+
+    attr_reader :emit_streams
+
+    def emits
+      all = []
+      @emit_streams.each {|tag,events|
+        events.each {|time,record|
+          all << [tag, time, record]
+        }
+      }
+      all
+    end
+
+    def events
+      all = []
+      @emit_streams.each {|tag,events|
+        all.concat events
+      }
+      all
+    end
+
+    def records
+      all = []
+      @emit_streams.each {|tag,events|
+        events.each {|time,record|
+          all << record
+        }
+      }
+      all
+    end
+
+    def prefer_buffered_processing
+      false
+    end
+
+    def process(tag, es)
+      @emit_streams << [tag, es.to_a]
+    end
+
+    def write(chunk)
+      es = Fluent::ArrayEventStream.new
+      chunk.each do |time, record|
+        es.add(time, record)
+      end
+      @emit_streams << [tag, es]
+    end
+  end
+end