fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/test/counter/test_store.rb

@@ -0,0 +1,258 @@
+require_relative '../helper'
+require 'fluent/counter/store'
+require 'fluent/time'
+require 'timecop'
+
+class CounterStoreTest < ::Test::Unit::TestCase
+  setup do
+    @name = 'key_name'
+    @scope = "server\tworker\tplugin"
+
+    # timecop isn't compatible with EventTime
+    t = Time.parse('2016-09-22 16:59:59 +0900')
+    Timecop.freeze(t)
+    @now = Fluent::EventTime.now
+  end
+
+  shutdown do
+    Timecop.return
+  end
+
+  def extract_value_from_counter(counter, key)
+    store = counter.instance_variable_get(:@storage).instance_variable_get(:@store)
+    store[key]
+  end
+
+  def travel(sec)
+    # Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing
+    # rounding errors to Time.now, we need to use Timecop.freeze() instead.
+    Timecop.freeze(Time.now + sec)
+  end
+
+  sub_test_case 'init' do
+    setup do
+      @reset_interval = 10
+      @store = Fluent::Counter::Store.new
+      @data = { 'name' => @name, 'reset_interval' => @reset_interval }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+    end
+
+    test 'create new value in the counter' do
+      v = @store.init(@key, @data)
+
+      assert_equal @name, v['name']
+      assert_equal @reset_interval, v['reset_interval']
+
+      v2 = extract_value_from_counter(@store, @key)
+      v2 = @store.send(:build_response, v2)
+      assert_equal v, v2
+    end
+
+    test 'raise an error when a passed key already exists' do
+      @store.init(@key, @data)
+
+      assert_raise Fluent::Counter::InvalidParams do
+        @store.init(@key, @data)
+      end
+    end
+
+    test 'return a value when passed key already exists and a ignore option is true' do
+      v = @store.init(@key, @data)
+      v1 = extract_value_from_counter(@store, @key)
+      v1 = @store.send(:build_response, v1)
+      v2 = @store.init(@key, @data, ignore: true)
+      assert_equal v, v2
+      assert_equal v1, v2
+    end
+  end
+
+  sub_test_case 'get' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      data = { 'name' => @name, 'reset_interval' => 10 }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(@key, data)
+    end
+
+    test 'return a value from the counter' do
+      v = extract_value_from_counter(@store, @key)
+      expected = @store.send(:build_response, v)
+      assert_equal expected, @store.get(@key)
+    end
+
+    test 'return a raw value from the counter when raw option is true' do
+      v = extract_value_from_counter(@store, @key)
+      assert_equal v, @store.get(@key, raw: true)
+    end
+
+    test "return nil when a passed key doesn't exist" do
+      assert_equal nil, @store.get('unknown_key')
+    end
+
+    test "raise a error when when a passed key doesn't exist and raise_error option is true" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.get('unknown_key', raise_error: true)
+      end
+    end
+  end
+
+  sub_test_case 'key?' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      data = { 'name' => @name, 'reset_interval' => 10 }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(@key, data)
+    end
+
+    test 'return true when passed key exists' do
+      assert_true @store.key?(@key)
+    end
+
+    test "return false when passed key doesn't exist" do
+      assert_true !@store.key?('unknown_key')
+    end
+  end
+
+  sub_test_case 'delete' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      data = { 'name' => @name, 'reset_interval' => 10 }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @init_value = @store.init(@key, data)
+    end
+
+    test 'delete a value from the counter' do
+      v = @store.delete(@key)
+      assert_equal @init_value, v
+      assert_nil extract_value_from_counter(@store, @key)
+    end
+
+    test "raise an error when passed key doesn't exist" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.delete('unknown_key')
+      end
+    end
+  end
+
+  sub_test_case 'inc' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      @init_data = { 'name' => @name, 'reset_interval' => 10 }
+      @travel_sec = 10
+    end
+
+    data(
+      positive: 10,
+      negative: -10
+    )
+    test 'increment or decrement a value in the counter' do |value|
+      key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(key, @init_data)
+      travel(@travel_sec)
+      v = @store.inc(key, { 'value' => value })
+
+      assert_equal value, v['total']
+      assert_equal value, v['current']
+      assert_equal @now, v['last_reset_at'] # last_reset_at doesn't change
+
+      v1 = extract_value_from_counter(@store, key)
+      v1 = @store.send(:build_response, v1)
+      assert_equal v, v1
+    end
+
+    test "raise an error when passed key doesn't exist" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.inc('unknown_key', { 'value' => 1 })
+      end
+    end
+
+    test 'raise an error when a type of passed value is incompatible with a stored value' do
+      key1 = Fluent::Counter::Store.gen_key(@scope, @name)
+      key2 = Fluent::Counter::Store.gen_key(@scope, 'name2')
+      key3 = Fluent::Counter::Store.gen_key(@scope, 'name3')
+      v1 = @store.init(key1, @init_data.merge('type' => 'integer'))
+      v2 = @store.init(key2, @init_data.merge('type' => 'float'))
+      v3 = @store.init(key3, @init_data.merge('type' => 'numeric'))
+      assert_equal 'integer', v1['type']
+      assert_equal 'float', v2['type']
+      assert_equal 'numeric', v3['type']
+
+      assert_raise Fluent::Counter::InvalidParams do
+        @store.inc(key1, { 'value' => 1.1 })
+      end
+
+      assert_raise Fluent::Counter::InvalidParams do
+        @store.inc(key2, { 'value' => 1 })
+      end
+
+      assert_nothing_raised do
+        @store.inc(key3, { 'value' => 1 })
+        @store.inc(key3, { 'value' => 1.0 })
+      end
+    end
+  end
+
+  sub_test_case 'reset' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      @travel_sec = 10
+
+      @inc_value = 10
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(@key, { 'name' => @name, 'reset_interval' => 10 })
+      @store.inc(@key, { 'value' => 10 })
+    end
+
+    test 'reset a value in the counter' do
+      travel(@travel_sec)
+
+      v = @store.reset(@key)
+      assert_equal @travel_sec, v['elapsed_time']
+      assert_true v['success']
+      counter = v['counter_data']
+
+      assert_equal @name, counter['name']
+      assert_equal @inc_value, counter['total']
+      assert_equal @inc_value, counter['current']
+      assert_equal 'numeric', counter['type']
+      assert_equal @now, counter['last_reset_at']
+      assert_equal 10, counter['reset_interval']
+
+      v1 = extract_value_from_counter(@store, @key)
+      assert_equal 0, v1['current']
+      assert_true v1['current'].is_a?(Integer)
+      assert_equal @inc_value, v1['total']
+      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])
+      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])
+    end
+
+    test 'reset a value after `reset_interval` passed' do
+      first_travel_sec = 5
+      travel(first_travel_sec) # jump time less than reset_interval
+      v = @store.reset(@key)
+
+      assert_equal false, v['success']
+      assert_equal first_travel_sec, v['elapsed_time']
+      store = extract_value_from_counter(@store, @key)
+      assert_equal 10, store['current']
+      assert_equal @now, Fluent::EventTime.new(*store['last_reset_at'])
+
+      # time is passed greater than reset_interval
+      travel(@travel_sec)
+      v = @store.reset(@key)
+      assert_true v['success']
+      assert_equal @travel_sec + first_travel_sec, v['elapsed_time']
+
+      v1 = extract_value_from_counter(@store, @key)
+      assert_equal 0, v1['current']
+      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])
+      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])
+    end
+
+    test "raise an error when passed key doesn't exist" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.reset('unknown_key')
+      end
+    end
+  end
+end

data/test/counter/test_validator.rb

@@ -0,0 +1,137 @@
+require_relative '../helper'
+require 'fluent/counter/validator'
+
+class CounterValidatorTest < ::Test::Unit::TestCase
+  data(
+    invalid_name1: '',
+    invalid_name3: '_',
+    invalid_name4: 'A',
+    invalid_name5: 'a*',
+    invalid_name6: "a\t",
+    invalid_name7: "\n",
+  )
+  test 'invalid name' do |invalid_name|
+    assert_nil(Fluent::Counter::Validator::VALID_NAME =~ invalid_name)
+  end
+
+  sub_test_case 'request' do
+    test 'return an empty array' do
+      data = { 'id' => 0, 'method' => 'init' }
+      errors = Fluent::Counter::Validator.request(data)
+      assert_empty errors
+    end
+
+    data(
+      missing_id: [
+        { 'method' => 'init' },
+        { 'code' => 'invalid_request', 'message' => 'Request should include `id`' }
+      ],
+      missing_method: [
+        { 'id' => 0 },
+        { 'code' => 'invalid_request', 'message' => 'Request should include `method`' }
+      ],
+      invalid_method: [
+        { 'id' => 0, 'method' => "A\t" },
+        { 'code' => 'invalid_request', 'message' => '`method` is the invalid format' }
+      ],
+      unknown_method: [
+        { 'id' => 0, 'method' => 'unknown_method' },
+        { 'code' => 'method_not_found', 'message' => 'Unknown method name passed: unknown_method' }
+      ]
+    )
+    test 'return an error array' do |(data, expected_error)|
+      errors = Fluent::Counter::Validator.request(data)
+      assert_equal [expected_error], errors
+    end
+  end
+
+  sub_test_case 'call' do
+    test "return an error hash when passed method doesn't exist" do
+      v = Fluent::Counter::Validator.new(:unknown)
+      success, errors = v.call(['key1'])
+      assert_empty success
+      assert_equal 'internal_server_error', errors.first.to_hash['code']
+    end
+  end
+
+  test 'validate_empty!' do
+    v = Fluent::Counter::Validator.new(:empty)
+    success, errors = v.call([])
+    assert_empty success
+    assert_equal [Fluent::Counter::InvalidParams.new('One or more `params` are required')], errors
+  end
+end
+
+class CounterArrayValidatorTest < ::Test::Unit::TestCase
+  test 'validate_key!' do
+    ary = ['key', 100, '_']
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => 'The type of `key` should be String' },
+      { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' }
+    ]
+    v = Fluent::Counter::ArrayValidator.new(:key)
+    valid_params, errors = v.call(ary)
+
+    assert_equal ['key'], valid_params
+    assert_equal error_expected, errors.map(&:to_hash)
+  end
+end
+
+class CounterHashValidatorTest < ::Test::Unit::TestCase
+  test 'validate_name!' do
+    hash = [
+      { 'name' => 'key' },
+      {},
+      { 'name' => 10 },
+      { 'name' => '_' }
+    ]
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => '`name` is required' },
+      { 'code' => 'invalid_params', 'message' => 'The type of `name` should be String' },
+      { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' },
+    ]
+    v = Fluent::Counter::HashValidator.new(:name)
+    success, errors = v.call(hash)
+
+    assert_equal [{ 'name' => 'key' }], success
+    assert_equal error_expected, errors.map(&:to_hash)
+  end
+
+  test 'validate_value!' do
+    hash = [
+      { 'value' => 1 },
+      { 'value' => -1 },
+      {},
+      { 'value' => 'str' }
+    ]
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => '`value` is required' },
+      { 'code' => 'invalid_params', 'message' => 'The type of `value` type should be Numeric' },
+    ]
+    v = Fluent::Counter::HashValidator.new(:value)
+    valid_params, errors = v.call(hash)
+
+    assert_equal [{ 'value' => 1 }, { 'value' => -1 }], valid_params
+    assert_equal error_expected, errors.map(&:to_hash)
+  end
+
+  test 'validate_reset_interval!' do
+    hash = [
+      { 'reset_interval' => 1 },
+      { 'reset_interval' => 1.0 },
+      {},
+      { 'reset_interval' => -1 },
+      { 'reset_interval' => 'str' }
+    ]
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => '`reset_interval` is required' },
+      { 'code' => 'invalid_params', 'message' => '`reset_interval` should be a positive number' },
+      { 'code' => 'invalid_params', 'message' => 'The type of `reset_interval` should be Numeric' },
+    ]
+    v = Fluent::Counter::HashValidator.new(:reset_interval)
+    valid_params, errors = v.call(hash)
+
+    assert_equal [{ 'reset_interval' => 1 }, { 'reset_interval' => 1.0 }], valid_params
+    assert_equal error_expected.map(&:to_hash), errors.map(&:to_hash)
+  end
+end

data/test/plugin/test_buf_file.rb

@@ -840,4 +840,128 @@ class FileBufferTest < Test::Unit::TestCase
       assert File.exist?(@not_chunk)
     end
   end
+
+  sub_test_case 'there are existing broken file chunks' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__)
+      FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)
+      @bufpath = File.join(@bufdir, 'broken_test.*.log')
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+      if @bufdir
+        Dir.glob(File.join(@bufdir, '*')).each do |path|
+          next if ['.', '..'].include?(File.basename(path))
+          File.delete(path)
+        end
+      end
+    end
+
+    def create_first_chunk(mode)
+      cid = Fluent::UniqueId.generate
+      path = File.join(@bufdir, "broken_test.#{mode}#{Fluent::UniqueId.hex(cid)}.log")
+      File.open(path, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        path + '.meta', cid, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),
+        4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i
+      )
+
+      return cid, path
+    end
+
+    def create_second_chunk(mode)
+      cid = Fluent::UniqueId.generate
+      path = File.join(@bufdir, "broken_test.#{mode}#{Fluent::UniqueId.hex(cid)}.log")
+      File.open(path, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        path + '.meta', cid, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),
+        3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i
+      )
+
+      return cid, path
+    end
+
+    def compare_staged_chunk(staged, id, time, num, mode)
+      assert_equal 1, staged.size
+      m = metadata(timekey: event_time(time).to_i)
+      assert_equal id, staged[m].unique_id
+      assert_equal num, staged[m].size
+      assert_equal mode, staged[m].state
+    end
+
+     def compare_queued_chunk(queued, id, num, mode)
+      assert_equal 1, queued.size
+      assert_equal id, queued[0].unique_id
+      assert_equal num, queued[0].size
+      assert_equal mode, queued[0].state
+    end
+
+    def compare_log(plugin, msg)
+      logs = plugin.log.out.logs
+      assert { logs.any? { |log| log.include?(msg) } }
+    end
+
+    test '#resume ignores staged empty chunk' do
+      _, p1 = create_first_chunk('b')
+      File.open(p1, 'wb') { |f| } # create staged empty chunk file
+      c2id, _ = create_second_chunk('b')
+
+      @p.start
+      compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged)
+      compare_log(@p, 'staged file chunk is empty')
+    end
+
+    test '#resume ignores staged broken metadata' do
+      c1id, _ = create_first_chunk('b')
+      _, p2 = create_second_chunk('b')
+      File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
+
+      @p.start
+      compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
+      compare_log(@p, 'staged meta file is broken')
+    end
+
+    test '#resume ignores enqueued empty chunk' do
+      _, p1 = create_first_chunk('q')
+      File.open(p1, 'wb') { |f| } # create enqueued empty chunk file
+      c2id, _ = create_second_chunk('q')
+
+      @p.start
+      compare_queued_chunk(@p.queue, c2id, 3, :queued)
+      compare_log(@p, 'enqueued file chunk is empty')
+    end
+
+    test '#resume ignores enqueued broken metadata' do
+      c1id, _ = create_first_chunk('q')
+      _, p2 = create_second_chunk('q')
+      File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create enqueued broken meta file
+
+      @p.start
+      compare_queued_chunk(@p.queue, c1id, 4, :queued)
+      compare_log(@p, 'enqueued meta file is broken')
+    end
+  end
 end