fluentd 0.12.40 → 1.6.2

data/test/plugin_helper/test_storage.rb

@@ -0,0 +1,542 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/storage'
+require 'fluent/plugin/base'
+
+class ExampleStorage < Fluent::Plugin::Storage
+  Fluent::Plugin.register_storage('example', self)
+
+  attr_reader :data, :saved, :load_times, :save_times
+
+  def initialize
+    super
+    @data = {}
+    @saved = {}
+    @load_times = 0
+    @save_times = 0
+  end
+  def load
+    @data ||= {}
+    @load_times += 1
+  end
+  def save
+    @saved = @data.dup
+    @save_times += 1
+  end
+  def get(key)
+    @data[key]
+  end
+  def fetch(key, defval)
+    @data.fetch(key, defval)
+  end
+  def put(key, value)
+    @data[key] = value
+  end
+  def delete(key)
+    @data.delete(key)
+  end
+  def update(key, &block)
+    @data[key] = block.call(@data[key])
+  end
+  def close
+    @data = {}
+    super
+  end
+  def terminate
+    @saved = {}
+    @load_times = @save_times = 0
+    super
+  end
+end
+
+class Example2Storage < ExampleStorage
+  Fluent::Plugin.register_storage('ex2', self)
+  config_param :dummy_path, :string, default: 'dummy'
+end
+class Example3Storage < ExampleStorage
+  Fluent::Plugin.register_storage('ex3', self)
+  def synchronized?
+    true
+  end
+end
+class Example4Storage < ExampleStorage
+  Fluent::Plugin.register_storage('ex4', self)
+  def persistent_always?
+    true
+  end
+  def synchronized?
+    true
+  end
+end
+
+class StorageHelperTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :storage
+  end
+
+  class Dummy2 < Fluent::Plugin::TestBase
+    helpers :storage
+    config_section :storage do
+      config_set_default :@type, 'ex2'
+      config_set_default :dummy_path, '/tmp/yay'
+    end
+  end
+
+  setup do
+    @d = nil
+  end
+
+  teardown do
+    if @d
+      @d.stop unless @d.stopped?
+      @d.shutdown unless @d.shutdown?
+      @d.close unless @d.closed?
+      @d.terminate unless @d.terminated?
+    end
+  end
+
+  test 'can be initialized without any storages at first' do
+    d = Dummy.new
+    assert_equal 0, d._storages.size
+  end
+
+  test 'can be configured with hash' do
+    d = Dummy.new
+    d.configure(config_element())
+    conf = { '@type' => 'example' }
+    assert_nothing_raised do
+      d.storage_create(conf: conf)
+    end
+  end
+
+  test 'can override default configuration parameters, but not overwrite whole definition' do
+    d = Dummy.new
+    d.configure(config_element())
+    assert_equal 1, d.storage_configs.size
+    assert_equal 'local', d.storage_configs.first[:@type]
+
+    d = Dummy2.new
+    d.configure(config_element('ROOT', '', {}, [config_element('storage', '', {}, [])]))
+    assert_raise NoMethodError do
+      d.storage
+    end
+    assert_equal 1, d.storage_configs.size
+    assert_equal 'ex2', d.storage_configs.first[:@type]
+    assert_equal '/tmp/yay', d.storage_configs.first.dummy_path
+  end
+
+  test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.storage_create(conf: config_element('format', '', {'@type' => 'example'}), default_type: 'ex2')
+    assert{ i.is_a?(Fluent::PluginHelper::Storage::SynchronizeWrapper) && i.instance_eval{ @storage }.is_a?(ExampleStorage) }
+
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.storage_create(conf: nil, default_type: 'ex2')
+    assert{ i.is_a?(Fluent::PluginHelper::Storage::SynchronizeWrapper) && i.instance_eval{ @storage }.is_a?(Example2Storage) }
+  end
+
+  test 'raises config error if config section is specified, but @type is not specified' do
+    d = Dummy2.new
+    d.configure(config_element())
+    assert_raise Fluent::ConfigError.new("@type is required in <storage>") do
+      d.storage_create(conf: config_element('storage', 'foo', {}), default_type: 'ex2')
+    end
+  end
+
+  test 'raises config error if config argument has invalid characters' do
+    d = Dummy.new
+    assert_raise Fluent::ConfigError.new("Argument in <storage ARG> uses invalid characters: 'yaa y'") do
+      d.configure(config_element('root', '', {}, [config_element('storage', 'yaa y', {'@type' => 'local'})]))
+    end
+    d.configure(config_element())
+    assert_raise Fluent::ConfigError.new("Argument in <storage ARG> uses invalid characters: 'a,b'") do
+      d.storage_create(usage: 'a,b', type: 'local')
+    end
+  end
+
+  test 'can be configured without storage sections' do
+    d = Dummy.new
+    assert_nothing_raised do
+      d.configure(config_element())
+    end
+    assert_equal 1, d._storages.size
+  end
+
+  test 'can be configured with a storage section' do
+    d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', '', {'@type' => 'example'})
+      ])
+    assert_nothing_raised do
+      d.configure(conf)
+    end
+    assert_equal 1, d._storages.size
+    assert{ d._storages.values.all?{ |s| !s.running } }
+  end
+
+  test 'can be configured with 2 or more storage sections with different usages with each other' do
+    d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'default', {'@type' => 'example'}),
+        config_element('storage', 'extra', {'@type' => 'ex2', 'dummy_path' => 'v'}),
+      ])
+    assert_nothing_raised do
+      d.configure(conf)
+    end
+    assert_equal 2, d._storages.size
+    assert{ d._storages.values.all?{ |s| !s.running } }
+  end
+
+  test 'cannot be configured with 2 storage sections with same usage' do
+    d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'default', {'@type' => 'example'}),
+        config_element('storage', 'extra', {'@type' => 'ex2', 'dummy_path' => 'v'}),
+        config_element('storage', 'extra', {'@type' => 'ex2', 'dummy_path' => 'v2'}),
+      ])
+    assert_raises Fluent::ConfigError do
+      d.configure(conf)
+    end
+  end
+
+  test 'creates a storage plugin instance which is already configured without usage' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', '', {'@type' => 'example'})
+      ])
+    d.configure(conf)
+    d.start
+
+    s = d.storage_create
+    assert{ s.implementation.is_a? ExampleStorage }
+  end
+
+  test 'creates a storage plugin instance which is already configured with usage' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example'})
+      ])
+    d.configure(conf)
+    d.start
+
+    s = d.storage_create(usage: 'mydata')
+    assert{ s.implementation.is_a? ExampleStorage }
+  end
+
+  test 'creates a storage plugin without configurations' do
+    @d = d = Dummy.new
+    d.configure(config_element())
+    d.start
+
+    s = d.storage_create(usage: 'mydata', type: 'example', conf: config_element('storage', 'mydata'))
+    assert{ s.implementation.is_a? ExampleStorage }
+  end
+
+  test 'creates 2 or more storage plugin instances' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example'}),
+        config_element('storage', 'secret', {'@type' => 'ex2', 'dummy_path' => 'yay!'}),
+      ])
+    d.configure(conf)
+    d.start
+
+    s1 = d.storage_create(usage: 'mydata')
+    s2 = d.storage_create(usage: 'secret')
+    assert{ s1.implementation.is_a? ExampleStorage }
+    assert{ s2.implementation.is_a? Example2Storage }
+    assert_equal 'yay!', s2.implementation.dummy_path
+  end
+
+  test 'creates wrapped instances for non-synchronized plugin in default' do # and check operations
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example'})
+      ])
+    d.configure(conf)
+    d.start
+
+    s = d.storage_create(usage: 'mydata')
+    assert !s.implementation.synchronized?
+    assert{ s.is_a? Fluent::PluginHelper::Storage::SynchronizeWrapper }
+    assert s.synchronized?
+    assert s.autosave
+    assert s.save_at_shutdown
+
+    assert_nil s.get('key')
+    assert_equal 'value', s.put('key', 'value')
+    assert_equal 'value', s.fetch('key', 'v1')
+    assert_equal 'v2', s.update('key'){|v| v[0] + '2' }
+    assert_equal 'v2', s.get('key')
+    assert_equal 'v2', s.delete('key')
+  end
+
+  test 'creates wrapped instances for non-persistent plugins when configured as persistent' do # and check operations
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example', 'persistent' => 'true'})
+      ])
+    d.configure(conf)
+    d.start
+
+    s = d.storage_create(usage: 'mydata')
+    assert !s.implementation.persistent_always?
+    assert{ s.is_a? Fluent::PluginHelper::Storage::PersistentWrapper }
+    assert s.persistent
+    assert s.persistent_always?
+    assert s.synchronized?
+    assert !s.autosave
+    assert s.save_at_shutdown
+
+    assert_nil s.get('key')
+    assert_equal 'value', s.put('key', 'value')
+    assert_equal 'value', s.fetch('key', 'v1')
+    assert_equal 'v2', s.update('key'){|v| v[0] + '2' }
+    assert_equal 'v2', s.get('key')
+    assert_equal 'v2', s.delete('key')
+  end
+
+  test 'creates bare instances for synchronized plugin in default' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'ex3'})
+      ])
+    d.configure(conf)
+    d.start
+
+    s = d.storage_create(usage: 'mydata')
+    assert s.implementation.synchronized?
+    assert{ s.is_a? Example3Storage }
+    assert s.synchronized?
+  end
+
+  test 'creates bare instances for persistent-always plugin when configured as persistent' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'ex4', 'persistent' => 'true'})
+      ])
+    d.configure(conf)
+    d.start
+
+    s = d.storage_create(usage: 'mydata')
+    assert s.implementation.persistent_always?
+    assert{ s.is_a? Example4Storage }
+    assert s.persistent
+    assert s.persistent_always?
+    assert s.synchronized?
+  end
+
+  test 'does not execute timer if autosave is not specified' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example', 'autosave' => 'false'})
+      ])
+    d.configure(conf)
+    d.start
+
+    d.storage_create(usage: 'mydata')
+    assert_equal 0, d._timers.size
+  end
+
+  test 'executes timer if autosave is specified and plugin is not persistent' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example', 'autosave_interval' => '1s', 'persistent' => 'false'})
+      ])
+    d.configure(conf)
+    d.start
+
+    d.storage_create(usage: 'mydata')
+    assert_equal 1, d._timers.size
+  end
+
+  test 'executes timer for autosave, which calls #save periodically' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example', 'autosave_interval' => '1s', 'persistent' => 'false'})
+      ])
+    d.configure(conf)
+    d.start
+
+    s = d.storage_create(usage: 'mydata')
+    assert_equal 1, d._timers.size
+    timeout = Time.now + 3
+    while Time.now < timeout
+      s.put('k', 'v')
+      sleep 0.2
+    end
+
+    d.stop
+    assert{ s.implementation.save_times > 0 }
+  end
+
+  test 'saves data for each operations if plugin storage is configured as persistent, and wrapped' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example', 'persistent' => 'true'})
+      ])
+    d.configure(conf)
+    d.start
+    s = d.storage_create(usage: 'mydata')
+    assert_equal 1, s.implementation.load_times
+    assert_equal 0, s.implementation.save_times
+
+    s.get('k1')
+    assert_equal 2, s.implementation.load_times
+    assert_equal 0, s.implementation.save_times
+
+    s.put('k1', 'v1')
+    assert_equal 3, s.implementation.load_times
+    assert_equal 1, s.implementation.save_times
+
+    s.fetch('k2', 'v2')
+    assert_equal 4, s.implementation.load_times
+    assert_equal 1, s.implementation.save_times
+
+    s.delete('k1')
+    assert_equal 5, s.implementation.load_times
+    assert_equal 2, s.implementation.save_times
+  end
+
+  test 'stops timer for autosave by #stop, calls #save by #shutdown if save_at_shutdown is specified' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example', 'save_at_shutdown' => 'true'})
+      ])
+    d.configure(conf)
+
+    assert !d.timer_running?
+
+    d.start
+
+    assert d.timer_running?
+
+    s = d.storage_create(usage: 'mydata')
+    assert s.autosave
+    assert_equal 1, d._timers.size
+
+    d.stop
+
+    assert !d.timer_running?
+
+    assert_equal 1, s.implementation.load_times
+    assert_equal 0, s.implementation.save_times
+
+    d.before_shutdown
+
+    d.shutdown
+
+    assert_equal 1, s.implementation.load_times
+    assert_equal 1, s.implementation.save_times
+  end
+
+  test 'calls #close and #terminate for all plugin instances by #close/#shutdown' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('storage', 'mydata', {'@type' => 'example', 'autosave' => 'false', 'save_at_shutdown' => 'true'})
+      ])
+    d.configure(conf)
+    d.start
+    s = d.storage_create(usage: 'mydata')
+
+    s.put('k1', 'v1')
+    s.put('k2', 2)
+    s.put('k3', true)
+
+    d.stop
+
+    assert_equal 3, s.implementation.data.size
+    assert_equal 0, s.implementation.saved.size
+
+    d.shutdown
+
+    assert_equal 3, s.implementation.data.size
+    assert_equal 3, s.implementation.saved.size
+
+    d.close
+
+    assert_equal 0, s.implementation.data.size
+    assert_equal 3, s.implementation.saved.size
+
+    d.shutdown
+
+    assert_equal 0, s.implementation.data.size
+    assert_equal 0, s.implementation.saved.size
+  end
+
+  test 'calls lifecycle methods for all plugin instances via owner plugin' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [ config_element('storage', '', {'@type' => 'example'}), config_element('storage', 'e2', {'@type' => 'example'}) ])
+    d.configure(conf)
+    d.start
+
+    i1 = d.storage_create(usage: '')
+    i2 = d.storage_create(usage: 'e2')
+    i3 = d.storage_create(usage: 'e3', type: 'ex2')
+
+    assert i1.started?
+    assert i2.started?
+    assert i3.started?
+
+    assert !i1.stopped?
+    assert !i2.stopped?
+    assert !i3.stopped?
+
+    d.stop
+
+    assert i1.stopped?
+    assert i2.stopped?
+    assert i3.stopped?
+
+    assert !i1.before_shutdown?
+    assert !i2.before_shutdown?
+    assert !i3.before_shutdown?
+
+    d.before_shutdown
+
+    assert i1.before_shutdown?
+    assert i2.before_shutdown?
+    assert i3.before_shutdown?
+
+    assert !i1.shutdown?
+    assert !i2.shutdown?
+    assert !i3.shutdown?
+
+    d.shutdown
+
+    assert i1.shutdown?
+    assert i2.shutdown?
+    assert i3.shutdown?
+
+    assert !i1.after_shutdown?
+    assert !i2.after_shutdown?
+    assert !i3.after_shutdown?
+
+    d.after_shutdown
+
+    assert i1.after_shutdown?
+    assert i2.after_shutdown?
+    assert i3.after_shutdown?
+
+    assert !i1.closed?
+    assert !i2.closed?
+    assert !i3.closed?
+
+    d.close
+
+    assert i1.closed?
+    assert i2.closed?
+    assert i3.closed?
+
+    assert !i1.terminated?
+    assert !i2.terminated?
+    assert !i3.terminated?
+
+    d.terminate
+
+    assert i1.terminated?
+    assert i2.terminated?
+    assert i3.terminated?
+  end
+end

data/test/plugin_helper/test_thread.rb

@@ -0,0 +1,164 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/thread'
+require 'fluent/plugin/base'
+require 'timeout'
+
+class ThreadTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :thread
+    def configure(conf)
+      super
+      @_thread_wait_seconds = 0.1
+      self
+    end
+  end
+
+  test 'can be instantiated to be able to create threads' do
+    d1 = Dummy.new
+    assert d1.respond_to?(:thread_current_running?)
+    assert d1.respond_to?(:thread_create)
+    assert d1.respond_to?(:_threads)
+    assert !d1.thread_current_running?
+    assert d1._threads.empty?
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'can create thread after prepared' do
+    d1 = Dummy.new
+    d1.configure(config_element())
+    d1.start
+
+    m1 = Mutex.new
+    m2 = Mutex.new
+
+    m1.lock
+    thread_run = false
+
+    Timeout.timeout(10) do
+      t = d1.thread_create(:test1) do
+        m2.lock
+
+        assert !d1._threads.empty? # this must be true always
+        assert d1.thread_current_running?
+
+        thread_run = true
+        m2.unlock
+        m1.lock
+      end
+      Thread.pass until m2.locked? || thread_run
+
+      m2.lock; m2.unlock
+      assert_equal 1, d1._threads.size
+
+      assert_equal :test1, t[:_fluentd_plugin_helper_thread_title]
+      assert t[:_fluentd_plugin_helper_thread_running]
+      assert !d1._threads.empty?
+
+      m1.unlock
+
+      while t[:_fluentd_plugin_helper_thread_running]
+        Thread.pass
+      end
+    end
+
+    assert d1._threads.empty?
+
+    d1.stop; d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can wait until all threads start' do
+    d1 = Dummy.new.configure(config_element()).start
+    ary = []
+    d1.thread_create(:t1) do
+      ary << 1
+    end
+    d1.thread_create(:t2) do
+      ary << 2
+    end
+    d1.thread_create(:t3) do
+      ary << 3
+    end
+    Timeout.timeout(10) do
+      d1.thread_wait_until_start
+    end
+    assert_equal [1,2,3], ary.sort
+
+    d1.stop; d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can stop threads which is watching thread_current_running?, and then close it' do
+    d1 = Dummy.new.configure(config_element()).start
+
+    m1 = Mutex.new
+    thread_in_run = false
+    Timeout.timeout(10) do
+      t = d1.thread_create(:test2) do
+        thread_in_run = true
+        m1.lock
+        while d1.thread_current_running?
+          Thread.pass
+        end
+        thread_in_run = false
+        m1.unlock
+      end
+      Thread.pass until m1.locked?
+
+      assert thread_in_run
+      assert !d1._threads.empty?
+
+      d1.stop
+      Thread.pass while m1.locked?
+      assert !t[:_fluentd_plugin_helper_thread_running]
+      assert t.stop?
+    end
+
+    assert d1._threads.empty?
+
+    d1.stop; d1.shutdown; d1.close; d1.terminate
+  end
+
+  test 'can terminate threads forcedly which is running forever' do
+    d1 = Dummy.new.configure(config_element()).start
+
+    m1 = Mutex.new
+    thread_in_run = false
+    Timeout.timeout(10) do
+      t = d1.thread_create(:test2) do
+        thread_in_run = true
+        m1.lock
+        while true
+          Thread.pass
+        end
+        thread_in_run = false
+      end
+      Thread.pass until m1.locked?
+
+      assert thread_in_run
+      assert !d1._threads.empty?
+
+      d1.stop
+      assert !t[:_fluentd_plugin_helper_thread_running]
+      assert t.alive?
+
+      d1.shutdown
+      assert t.alive?
+      assert !d1._threads.empty?
+
+      d1.close
+      assert t.alive?
+      assert !d1._threads.empty?
+
+      d1.terminate
+      assert t.stop?
+      assert d1._threads.empty?
+    end
+  end
+end