fluentd 1.2.0.pre1 → 1.2.0

data/test/counter/test_store.rb

@@ -0,0 +1,252 @@
+require_relative '../helper'
+require 'fluent/counter/store'
+require 'fluent/time'
+require 'timecop'
+
+class CounterStoreTest < ::Test::Unit::TestCase
+  setup do
+    @name = 'key_name'
+    @scope = "server\tworker\tplugin"
+
+    # timecop isn't compatible with EventTime
+    t = Time.parse('2016-09-22 16:59:59 +0900')
+    Timecop.freeze(t)
+    @now = Fluent::EventTime.now
+  end
+
+  shutdown do
+    Timecop.return
+  end
+
+  def extract_value_from_counter(counter, key)
+    store = counter.instance_variable_get(:@storage).instance_variable_get(:@store)
+    store[key]
+  end
+
+  sub_test_case 'init' do
+    setup do
+      @reset_interval = 10
+      @store = Fluent::Counter::Store.new
+      @data = { 'name' => @name, 'reset_interval' => @reset_interval }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+    end
+
+    test 'create new value in the counter' do
+      v = @store.init(@key, @data)
+
+      assert_equal @name, v['name']
+      assert_equal @reset_interval, v['reset_interval']
+
+      v2 = extract_value_from_counter(@store, @key)
+      v2 = @store.send(:build_response, v2)
+      assert_equal v, v2
+    end
+
+    test 'raise an error when a passed key already exists' do
+      @store.init(@key, @data)
+
+      assert_raise Fluent::Counter::InvalidParams do
+        @store.init(@key, @data)
+      end
+    end
+
+    test 'return a value when passed key already exists and a ignore option is true' do
+      v = @store.init(@key, @data)
+      v1 = extract_value_from_counter(@store, @key)
+      v1 = @store.send(:build_response, v1)
+      v2 = @store.init(@key, @data, ignore: true)
+      assert_equal v, v2
+      assert_equal v1, v2
+    end
+  end
+
+  sub_test_case 'get' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      data = { 'name' => @name, 'reset_interval' => 10 }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(@key, data)
+    end
+
+    test 'return a value from the counter' do
+      v = extract_value_from_counter(@store, @key)
+      expected = @store.send(:build_response, v)
+      assert_equal expected, @store.get(@key)
+    end
+
+    test 'return a raw value from the counter when raw option is true' do
+      v = extract_value_from_counter(@store, @key)
+      assert_equal v, @store.get(@key, raw: true)
+    end
+
+    test "return nil when a passed key doesn't exist" do
+      assert_equal nil, @store.get('unknown_key')
+    end
+
+    test "raise a error when when a passed key doesn't exist and raise_error option is true" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.get('unknown_key', raise_error: true)
+      end
+    end
+  end
+
+  sub_test_case 'key?' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      data = { 'name' => @name, 'reset_interval' => 10 }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(@key, data)
+    end
+
+    test 'return true when passed key exists' do
+      assert_true @store.key?(@key)
+    end
+
+    test "return false when passed key doesn't exist" do
+      assert_true !@store.key?('unknown_key')
+    end
+  end
+
+  sub_test_case 'delete' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      data = { 'name' => @name, 'reset_interval' => 10 }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @init_value = @store.init(@key, data)
+    end
+
+    test 'delete a value from the counter' do
+      v = @store.delete(@key)
+      assert_equal @init_value, v
+      assert_nil extract_value_from_counter(@store, @key)
+    end
+
+    test "raise an error when passed key doesn't exist" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.delete('unknown_key')
+      end
+    end
+  end
+
+  sub_test_case 'inc' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      @init_data = { 'name' => @name, 'reset_interval' => 10 }
+      @travel_sec = 10
+    end
+
+    data(
+      positive: 10,
+      negative: -10
+    )
+    test 'increment or decrement a value in the counter' do |value|
+      key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(key, @init_data)
+      Timecop.travel(@travel_sec)
+      v = @store.inc(key, { 'value' => value })
+
+      assert_equal value, v['total']
+      assert_equal value, v['current']
+      assert_equal @now, v['last_reset_at'] # last_reset_at doesn't change
+
+      v1 = extract_value_from_counter(@store, key)
+      v1 = @store.send(:build_response, v1)
+      assert_equal v, v1
+    end
+
+    test "raise an error when passed key doesn't exist" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.inc('unknown_key', { 'value' => 1 })
+      end
+    end
+
+    test 'raise an error when a type of passed value is incompatible with a stored value' do
+      key1 = Fluent::Counter::Store.gen_key(@scope, @name)
+      key2 = Fluent::Counter::Store.gen_key(@scope, 'name2')
+      key3 = Fluent::Counter::Store.gen_key(@scope, 'name3')
+      v1 = @store.init(key1, @init_data.merge('type' => 'integer'))
+      v2 = @store.init(key2, @init_data.merge('type' => 'float'))
+      v3 = @store.init(key3, @init_data.merge('type' => 'numeric'))
+      assert_equal 'integer', v1['type']
+      assert_equal 'float', v2['type']
+      assert_equal 'numeric', v3['type']
+
+      assert_raise Fluent::Counter::InvalidParams do
+        @store.inc(key1, { 'value' => 1.1 })
+      end
+
+      assert_raise Fluent::Counter::InvalidParams do
+        @store.inc(key2, { 'value' => 1 })
+      end
+
+      assert_nothing_raised do
+        @store.inc(key3, { 'value' => 1 })
+        @store.inc(key3, { 'value' => 1.0 })
+      end
+    end
+  end
+
+  sub_test_case 'reset' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      @travel_sec = 10
+
+      @inc_value = 10
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(@key, { 'name' => @name, 'reset_interval' => 10 })
+      @store.inc(@key, { 'value' => 10 })
+    end
+
+    test 'reset a value in the counter' do
+      Timecop.travel(@travel_sec)
+
+      v = @store.reset(@key)
+      assert_equal @travel_sec, v['elapsed_time']
+      assert_true v['success']
+      counter = v['counter_data']
+
+      assert_equal @name, counter['name']
+      assert_equal @inc_value, counter['total']
+      assert_equal @inc_value, counter['current']
+      assert_equal 'numeric', counter['type']
+      assert_equal @now, counter['last_reset_at']
+      assert_equal 10, counter['reset_interval']
+
+      v1 = extract_value_from_counter(@store, @key)
+      assert_equal 0, v1['current']
+      assert_true v1['current'].is_a?(Integer)
+      assert_equal @inc_value, v1['total']
+      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])
+      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])
+    end
+
+    test 'reset a value after `reset_interval` passed' do
+      first_travel_sec = 5
+      Timecop.travel(first_travel_sec) # jump time less than reset_interval
+      v = @store.reset(@key)
+
+      assert_equal false, v['success']
+      assert_equal first_travel_sec, v['elapsed_time']
+      store = extract_value_from_counter(@store, @key)
+      assert_equal 10, store['current']
+      assert_equal @now, Fluent::EventTime.new(*store['last_reset_at'])
+
+      # time is passed greater than reset_interval
+      Timecop.travel(@travel_sec)
+      v = @store.reset(@key)
+      assert_true v['success']
+      assert_equal @travel_sec + first_travel_sec, v['elapsed_time']
+
+      v1 = extract_value_from_counter(@store, @key)
+      assert_equal 0, v1['current']
+      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])
+      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])
+    end
+
+    test "raise an error when passed key doesn't exist" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.reset('unknown_key')
+      end
+    end
+  end
+end

data/test/counter/test_validator.rb

@@ -0,0 +1,137 @@
+require_relative '../helper'
+require 'fluent/counter/validator'
+
+class CounterValidatorTest < ::Test::Unit::TestCase
+  data(
+    invalid_name1: '',
+    invalid_name3: '_',
+    invalid_name4: 'A',
+    invalid_name5: 'a*',
+    invalid_name6: "a\t",
+    invalid_name7: "\n",
+  )
+  test 'invalid name' do |invalid_name|
+    assert_nil(Fluent::Counter::Validator::VALID_NAME =~ invalid_name)
+  end
+
+  sub_test_case 'request' do
+    test 'return an empty array' do
+      data = { 'id' => 0, 'method' => 'init' }
+      errors = Fluent::Counter::Validator.request(data)
+      assert_empty errors
+    end
+
+    data(
+      missing_id: [
+        { 'method' => 'init' },
+        { 'code' => 'invalid_request', 'message' => 'Request should include `id`' }
+      ],
+      missing_method: [
+        { 'id' => 0 },
+        { 'code' => 'invalid_request', 'message' => 'Request should include `method`' }
+      ],
+      invalid_method: [
+        { 'id' => 0, 'method' => "A\t" },
+        { 'code' => 'invalid_request', 'message' => '`method` is the invalid format' }
+      ],
+      unknown_method: [
+        { 'id' => 0, 'method' => 'unknown_method' },
+        { 'code' => 'method_not_found', 'message' => 'Unknown method name passed: unknown_method' }
+      ]
+    )
+    test 'return an error array' do |(data, expected_error)|
+      errors = Fluent::Counter::Validator.request(data)
+      assert_equal [expected_error], errors
+    end
+  end
+
+  sub_test_case 'call' do
+    test "return an error hash when passed method doesn't exist" do
+      v = Fluent::Counter::Validator.new(:unknown)
+      success, errors = v.call(['key1'])
+      assert_empty success
+      assert_equal 'internal_server_error', errors.first.to_hash['code']
+    end
+  end
+
+  test 'validate_empty!' do
+    v = Fluent::Counter::Validator.new(:empty)
+    success, errors = v.call([])
+    assert_empty success
+    assert_equal [Fluent::Counter::InvalidParams.new('One or more `params` are required')], errors
+  end
+end
+
+class CounterArrayValidatorTest < ::Test::Unit::TestCase
+  test 'validate_key!' do
+    ary = ['key', 100, '_']
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => 'The type of `key` should be String' },
+      { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' }
+    ]
+    v = Fluent::Counter::ArrayValidator.new(:key)
+    valid_params, errors = v.call(ary)
+
+    assert_equal ['key'], valid_params
+    assert_equal error_expected, errors.map(&:to_hash)
+  end
+end
+
+class CounterHashValidatorTest < ::Test::Unit::TestCase
+  test 'validate_name!' do
+    hash = [
+      { 'name' => 'key' },
+      {},
+      { 'name' => 10 },
+      { 'name' => '_' }
+    ]
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => '`name` is required' },
+      { 'code' => 'invalid_params', 'message' => 'The type of `name` should be String' },
+      { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' },
+    ]
+    v = Fluent::Counter::HashValidator.new(:name)
+    success, errors = v.call(hash)
+
+    assert_equal [{ 'name' => 'key' }], success
+    assert_equal error_expected, errors.map(&:to_hash)
+  end
+
+  test 'validate_value!' do
+    hash = [
+      { 'value' => 1 },
+      { 'value' => -1 },
+      {},
+      { 'value' => 'str' }
+    ]
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => '`value` is required' },
+      { 'code' => 'invalid_params', 'message' => 'The type of `value` type should be Numeric' },
+    ]
+    v = Fluent::Counter::HashValidator.new(:value)
+    valid_params, errors = v.call(hash)
+
+    assert_equal [{ 'value' => 1 }, { 'value' => -1 }], valid_params
+    assert_equal error_expected, errors.map(&:to_hash)
+  end
+
+  test 'validate_reset_interval!' do
+    hash = [
+      { 'reset_interval' => 1 },
+      { 'reset_interval' => 1.0 },
+      {},
+      { 'reset_interval' => -1 },
+      { 'reset_interval' => 'str' }
+    ]
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => '`reset_interval` is required' },
+      { 'code' => 'invalid_params', 'message' => '`reset_interval` should be a positive number' },
+      { 'code' => 'invalid_params', 'message' => 'The type of `reset_interval` should be Numeric' },
+    ]
+    v = Fluent::Counter::HashValidator.new(:reset_interval)
+    valid_params, errors = v.call(hash)
+
+    assert_equal [{ 'reset_interval' => 1 }, { 'reset_interval' => 1.0 }], valid_params
+    assert_equal error_expected.map(&:to_hash), errors.map(&:to_hash)
+  end
+end

data/test/plugin/test_output_as_buffered_backup.rb

@@ -0,0 +1,271 @@
+require_relative '../helper'
+require 'fluent/plugin/output'
+require 'fluent/plugin/buffer'
+require 'fluent/event'
+require 'fluent/error'
+
+require 'json'
+require 'time'
+require 'timeout'
+require 'timecop'
+
+
+class BufferedOutputBackupTest < Test::Unit::TestCase
+  class BareOutput < Fluent::Plugin::Output
+    def register(name, &block)
+      instance_variable_set("@#{name}", block)
+    end
+  end
+  class DummyOutput < BareOutput
+    def initialize
+      super
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      true
+    end
+    def prefer_delayed_commit
+      false
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      [tag, time.to_i, record].to_json + "\n"
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  class DummyOutputForSecondary < BareOutput
+    def initialize
+      super
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      true
+    end
+    def prefer_delayed_commit
+      false
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      [tag, time.to_i, record].to_json + "\n"
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  class DummyAsyncOutputForSecondary < BareOutput
+    def initialize
+      super
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      true
+    end
+    def prefer_delayed_commit
+      true
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      [tag, time.to_i, record].to_json + "\n"
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/bu#{ENV['TEST_ENV_NUMBER']}")
+
+  def create_output
+    DummyOutput.new
+  end
+  def create_metadata(timekey: nil, tag: nil, variables: nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+  def waiting(seconds)
+    begin
+      Timeout.timeout(seconds) do
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.print(*@i.log.out.logs)
+      raise
+    end
+  end
+
+  def dummy_event_stream
+    Fluent::ArrayEventStream.new([
+      [ event_time('2016-04-13 18:33:00'), {"name" => "moris", "age" => 36, "message" => "data1"} ],
+      [ event_time('2016-04-13 18:33:13'), {"name" => "moris", "age" => 36, "message" => "data2"} ],
+      [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ],
+    ])
+  end
+
+  setup do
+    @i = create_output
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+
+    Fluent::Plugin.register_output('backup_output', DummyOutput)
+    Fluent::Plugin.register_output('backup_output2', DummyOutputForSecondary)
+    Fluent::Plugin.register_output('backup_async_output', DummyAsyncOutputForSecondary)
+  end
+
+  teardown do
+    if @i
+      @i.stop unless @i.stopped?
+      @i.before_shutdown unless @i.before_shutdown?
+      @i.shutdown unless @i.shutdown?
+      @i.after_shutdown unless @i.after_shutdown?
+      @i.close unless @i.closed?
+      @i.terminate unless @i.terminated?
+    end
+    Timecop.return
+  end
+
+  sub_test_case 'buffered output for broken chunks' do
+    def flush_chunks
+      @i.start
+      @i.after_start
+
+      @i.interrupt_flushes
+
+      now = Time.parse('2016-04-13 18:33:30 -0700')
+      Timecop.freeze(now)
+      @i.emit_events("test.tag.1", dummy_event_stream())
+      now = Time.parse('2016-04-13 18:33:32 -0700')
+      Timecop.freeze(now)
+
+      @i.enqueue_thread_wait
+      @i.flush_thread_wakeup
+      waiting(4) { Thread.pass until @i.write_count > 0 }
+
+      assert { @i.write_count > 0 }
+      Timecop.freeze(now)
+      @i.flush_thread_wakeup
+    end
+
+    test 'backup chunk without secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and no secondary") } }
+      end
+    end
+
+    test 'backup chunk with same type secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_same_secondary'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        secconf = config_element('secondary','',{'@type' => 'backup_output'})
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary type is same as primary") } }
+      end
+    end
+
+    test 'backup chunk with different type secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_diff_secondary'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        secconf = config_element('secondary','',{'@type' => 'backup_output2'})
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+        @i.secondary.register(:write) { |chunk|
+          raise Fluent::UnrecoverableError, "yay, your secondary #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary. Skip retry and flush chunk to secondary") } }
+        assert { logs.any? { |l| l.include?("got an error in secondary for unrecoverable error") } }
+      end
+    end
+
+    test 'backup chunk with async secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_diff_secondary'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        secconf = config_element('secondary','',{'@type' => 'backup_async_output'})
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary is async output") } }
+      end
+    end
+  end
+end