fluentd 0.12.40 → 1.6.2

data/test/test_plugin_classes.rb

@@ -1,24 +1,49 @@
 require_relative 'helper'
-require 'fluent/input'
-require 'fluent/output'
-require 'fluent/filter'
+require 'fluent/plugin/input'
+require 'fluent/plugin/output'
+require 'fluent/plugin/bare_output'
+require 'fluent/plugin/filter'
 
 module FluentTest
-  class FluentTestInput < ::Fluent::Input
+  class FluentTestInput < ::Fluent::Plugin::Input
     ::Fluent::Plugin.register_input('test_in', self)
 
     attr_reader :started
 
     def start
+      super
       @started = true
     end
 
     def shutdown
       @started = false
+      super
     end
   end
 
-  class FluentTestOutput < ::Fluent::Output
+  class FluentTestGenInput < ::Fluent::Plugin::Input
+    ::Fluent::Plugin.register_input('test_in_gen', self)
+
+    attr_reader :started
+
+    config_param :num, :integer, default: 10000
+
+    def start
+      super
+      @started = true
+
+      @num.times { |i|
+        router.emit("test.evet", Fluent::EventTime.now, {'message' => 'Hello!', 'key' => "value#{i}", 'num' => i})
+      }
+    end
+
+    def shutdown
+      @started = false
+      super
+    end
+  end
+
+  class FluentTestOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out', self)
 
     def initialize
@@ -30,21 +55,117 @@ module FluentTest
     attr_reader :started
 
     def start
+      super
       @started = true
     end
 
     def shutdown
       @started = false
+      super
     end
 
-    def emit(tag, es, chain)
-      es.each { |time, record|
+    def process(tag, es)
+      es.each do |time, record|
         @events[tag] << record
-      }
+      end
+    end
+  end
+
+  class FluentTestDynamicOutput < ::Fluent::Plugin::BareOutput
+    ::Fluent::Plugin.register_output('test_dynamic_out', self)
+
+    attr_reader :child
+    attr_reader :started
+
+    def start
+      super
+      @started = true
+      @child = Fluent::Plugin.new_output('copy')
+      conf = config_element('DYNAMIC', '', {}, [
+          config_element('store', '', {'@type' => 'test_out', '@id' => 'dyn_out1'}),
+          config_element('store', '', {'@type' => 'test_out', '@id' => 'dyn_out2'}),
+      ])
+      @child.configure(conf)
+      @child.start
+    end
+
+    def after_start
+      super
+      @child.after_start
+    end
+
+    def stop
+      super
+      @child.stop
+    end
+
+    def before_shutdown
+      super
+      @child.before_shutdown
+    end
+
+    def shutdown
+      @started = false
+      super
+      @child.shutdown
+    end
+
+    def after_shutdown
+      super
+      @child.after_shutdown
+    end
+
+    def close
+      super
+      @child.close
+    end
+
+    def terminate
+      super
+      @child.terminate
+    end
+
+    def process(tag, es)
+      es.each do |time, record|
+        @events[tag] << record
+      end
     end
   end
 
-  class FluentTestErrorOutput < ::Fluent::BufferedOutput
+  class FluentTestBufferedOutput < ::Fluent::Plugin::Output
+    ::Fluent::Plugin.register_output('test_out_buffered', self)
+
+    attr_reader :started
+
+    def start
+      super
+      @started = true
+    end
+
+    def shutdown
+      @started = false
+      super
+    end
+
+    def write(chunk)
+      # drop everything
+    end
+  end
+
+  class FluentTestEmitOutput < ::Fluent::Plugin::Output
+    ::Fluent::Plugin.register_output('test_out_emit', self)
+    helpers :event_emitter
+    def write(chunk)
+      tag = chunk.metadata.tag || 'test'
+      array = []
+      chunk.each do |time, record|
+        array << [time, record]
+      end
+      router.emit_array(tag, array)
+    end
+  end
+
+  class FluentTestErrorOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out_error', self)
 
     def format(tag, time, record)
@@ -56,7 +177,36 @@ module FluentTest
     end
   end
 
-  class FluentTestFilter < ::Fluent::Filter
+  class FluentCompatTestFilter < ::Fluent::Filter
+    ::Fluent::Plugin.register_filter('test_compat_filter', self)
+
+    def initialize(field = '__test__')
+      super()
+      @num = 0
+      @field = field
+    end
+
+    attr_reader :num
+    attr_reader :started
+
+    def start
+      super
+      @started = true
+    end
+
+    def shutdown
+      @started = false
+      super
+    end
+
+    def filter(tag, time, record)
+      record[@field] = @num
+      @num += 1
+      record
+    end
+  end
+
+  class FluentTestFilter < ::Fluent::Plugin::Filter
     ::Fluent::Plugin.register_filter('test_filter', self)
 
     def initialize(field = '__test__')
@@ -69,11 +219,13 @@ module FluentTest
     attr_reader :started
 
     def start
+      super
       @started = true
     end
 
     def shutdown
       @started = false
+      super
     end
 
     def filter(tag, time, record)
@@ -83,6 +235,21 @@ module FluentTest
     end
   end
 
+  class FluentTestBuffer < Fluent::Plugin::Buffer
+    ::Fluent::Plugin.register_buffer('test_buffer', self)
+
+    def resume
+      return {}, []
+    end
+
+    def generate_chunk(metadata)
+    end
+
+    def multi_workers_ready?
+      false
+    end
+  end
+
   class TestEmitErrorHandler
     def initialize
       @events = Hash.new { |h, k| h[k] = [] }

data/test/test_plugin_helper.rb

@@ -0,0 +1,81 @@
+require_relative 'helper'
+require 'fluent/plugin_helper'
+require 'fluent/plugin/base'
+
+class ConfigTest < Test::Unit::TestCase
+  module FluentTest; end
+
+  sub_test_case 'Fluent::Plugin::Base.helpers method works as shortcut to include helper modules' do
+    class FluentTest::PluginTest1 < Fluent::Plugin::TestBase
+      helpers :event_emitter
+    end
+    class FluentTest::PluginTest2 < Fluent::Plugin::TestBase
+      helpers :thread
+    end
+    class FluentTest::PluginTest3 < Fluent::Plugin::TestBase
+      helpers :event_loop
+    end
+    class FluentTest::PluginTest4 < Fluent::Plugin::TestBase
+      helpers :timer
+    end
+    class FluentTest::PluginTest5 < Fluent::Plugin::TestBase
+      helpers :child_process
+    end
+    class FluentTest::PluginTest6 < Fluent::Plugin::TestBase
+      helpers :retry_state
+    end
+    class FluentTest::PluginTest0 < Fluent::Plugin::TestBase
+      helpers :event_emitter, :thread, :event_loop, :timer, :child_process, :retry_state
+    end
+
+    test 'plugin can include helper event_emitter' do
+      assert FluentTest::PluginTest1.include?(Fluent::PluginHelper::EventEmitter)
+      p1 = FluentTest::PluginTest1.new
+      assert p1.respond_to?(:has_router?)
+      assert p1.has_router?
+    end
+
+    test 'plugin can include helper thread' do
+      assert FluentTest::PluginTest2.include?(Fluent::PluginHelper::Thread)
+      p2 = FluentTest::PluginTest2.new
+      assert p2.respond_to?(:thread_current_running?)
+      assert p2.respond_to?(:thread_create)
+    end
+
+    test 'plugin can include helper event_loop' do
+      assert FluentTest::PluginTest3.include?(Fluent::PluginHelper::EventLoop)
+      p3 = FluentTest::PluginTest3.new
+      assert p3.respond_to?(:event_loop_attach)
+      assert p3.respond_to?(:event_loop_running?)
+    end
+
+    test 'plugin can include helper timer' do
+      assert FluentTest::PluginTest4.include?(Fluent::PluginHelper::Timer)
+      p4 = FluentTest::PluginTest4.new
+      assert p4.respond_to?(:timer_execute)
+    end
+
+    test 'plugin can include helper child_process' do
+      assert FluentTest::PluginTest5.include?(Fluent::PluginHelper::ChildProcess)
+      p5 = FluentTest::PluginTest5.new
+      assert p5.respond_to?(:child_process_execute)
+    end
+
+    test 'plugin can 2 or more helpers at once' do
+      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::EventEmitter)
+      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::Thread)
+      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::EventLoop)
+      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::Timer)
+      assert FluentTest::PluginTest0.include?(Fluent::PluginHelper::ChildProcess)
+
+      p0 = FluentTest::PluginTest0.new
+      assert p0.respond_to?(:child_process_execute)
+      assert p0.respond_to?(:timer_execute)
+      assert p0.respond_to?(:event_loop_attach)
+      assert p0.respond_to?(:event_loop_running?)
+      assert p0.respond_to?(:thread_current_running?)
+      assert p0.respond_to?(:thread_create)
+      assert p0.respond_to?(:has_router?)
+    end
+  end
+end

data/test/test_plugin_id.rb

@@ -0,0 +1,101 @@
+require_relative 'helper'
+require 'fluent/plugin/base'
+require 'fluent/system_config'
+require 'fileutils'
+
+class PluginIdTest < Test::Unit::TestCase
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/plugin_id/#{ENV['TEST_ENV_NUMBER']}")
+
+  class MyPlugin < Fluent::Plugin::Base
+    include Fluent::PluginId
+  end
+
+  setup do
+    @p = MyPlugin.new
+  end
+
+  sub_test_case '#plugin_id_for_test?' do
+    test 'returns true always in test files' do
+      assert @p.plugin_id_for_test?
+    end
+
+    test 'returns false always out of test files' do
+      # TODO: no good way to write this test....
+    end
+  end
+
+  sub_test_case 'configured without @id' do
+    setup do
+      @p.configure(config_element())
+    end
+
+    test '#plugin_id_configured? returns false' do
+      assert_false @p.plugin_id_configured?
+    end
+
+    test '#plugin_id returns object_id based string' do
+      assert_kind_of String, @p.plugin_id
+      assert @p.plugin_id =~ /^object:[0-9a-f]+/
+    end
+
+    test '#plugin_root_dir returns nil' do
+      assert_nil @p.plugin_root_dir
+    end
+  end
+
+  sub_test_case 'configured with @id' do
+    setup do
+      FileUtils.rm_rf(TMP_DIR)
+      FileUtils.mkdir_p(TMP_DIR)
+      @p.configure(config_element('ROOT', '', {'@id' => 'testing_plugin_id'}))
+    end
+
+    test '#plugin_id_configured? returns true' do
+      assert @p.plugin_id_configured?
+    end
+
+    test '#plugin_id returns the configured value' do
+      assert_equal 'testing_plugin_id', @p.plugin_id
+    end
+
+    test '#plugin_root_dir returns nil without system root directory configuration' do
+      assert_nil @p.plugin_root_dir
+    end
+
+    test '#plugin_root_dir returns an existing directory path frozen String' do
+      root_dir = Fluent::SystemConfig.overwrite_system_config('root_dir' => File.join(TMP_DIR, "myroot")) do
+        @p.plugin_root_dir
+      end
+      assert_kind_of String, root_dir
+      assert Dir.exist?(root_dir)
+      assert root_dir =~ %r!/worker0/!
+      assert root_dir.frozen?
+    end
+
+    test '#plugin_root_dir returns the same value for 2nd or more call' do
+      root_dir = Fluent::SystemConfig.overwrite_system_config('root_dir' => File.join(TMP_DIR, "myroot")) do
+        @p.plugin_root_dir
+      end
+      twice = Fluent::SystemConfig.overwrite_system_config('root_dir' => File.join(TMP_DIR, "myroot")) do
+        @p.plugin_root_dir
+      end
+      assert_equal root_dir.object_id, twice.object_id
+    end
+
+    test '#plugin_root_dir referres SERVERENGINE_WORKER_ID environment path to create it' do
+      prev_env_val = ENV['SERVERENGINE_WORKER_ID']
+      begin
+        ENV['SERVERENGINE_WORKER_ID'] = '7'
+        root_dir = Fluent::SystemConfig.overwrite_system_config('root_dir' => File.join(TMP_DIR, "myroot")) do
+          @p.plugin_root_dir
+        end
+        assert_kind_of String, root_dir
+        assert Dir.exist?(root_dir)
+        assert root_dir =~ %r!/worker7/!
+        assert root_dir.frozen?
+      ensure
+        ENV['SERVERENGINE_WORKER_ID'] = prev_env_val
+      end
+    end
+  end
+end

data/test/test_process.rb

@@ -1,48 +1,14 @@
 require_relative 'helper'
-require 'fluent/test'
-require 'fluent/event'
 require 'fluent/process'
-require 'stringio'
-require 'msgpack'
 
-module FluentProcessTest
-  class DelayedForwarderTest < Test::Unit::TestCase
-    include Fluent
-
-    test 'run and emit' do
-      io = StringIO.new
-      fwd = Fluent::DetachProcessManager::DelayedForwarder.new(io, 0.001)
-
-      num_events_per_tag = 5000
-      num_tags = 20
-
-      now = Time.now.to_i
-      record = {'key' => 'value'}
-      (num_tags * num_events_per_tag).times do |i|
-        tag = "foo.bar#{i % num_tags}"
-        fwd.emit(tag, OneEventStream.new(now, record))
-      end
-      sleep 0.5
-
-      io.pos = 0
-
-      tags = {}
-      MessagePack::Unpacker.new(io).each do |tag_and_msgpacks|
-        tag, ms = *tag_and_msgpacks
-        tags[tag] ||= ''
-        tags[tag] << ms
-      end
+class ProcessCompatibilityTest < ::Test::Unit::TestCase
+  test 'DetachProcessMixin is defined' do
+    assert defined?(::Fluent::DetachProcessMixin)
+    assert_equal ::Fluent::DetachProcessMixin, ::Fluent::Compat::DetachProcessMixin
+  end
 
-      assert_equal(num_tags, tags.size)
-      num_tags.times do |i|
-        tag = "foo.bar#{i % num_tags}"
-        ms = tags[tag]
-        count = 0
-        MessagePack::Unpacker.new(StringIO.new(ms)).each do |x|
-          count += 1
-        end
-        assert_equal(num_events_per_tag, count)
-      end
-    end
+  test 'DetachMultiProcessMixin is defined' do
+    assert defined?(::Fluent::DetachMultiProcessMixin)
+    assert_equal ::Fluent::DetachMultiProcessMixin, ::Fluent::Compat::DetachMultiProcessMixin
   end
 end