fluentd 0.12.40 → 1.6.2

data/test/counter/test_store.rb

@@ -0,0 +1,258 @@
+require_relative '../helper'
+require 'fluent/counter/store'
+require 'fluent/time'
+require 'timecop'
+
+class CounterStoreTest < ::Test::Unit::TestCase
+  setup do
+    @name = 'key_name'
+    @scope = "server\tworker\tplugin"
+
+    # timecop isn't compatible with EventTime
+    t = Time.parse('2016-09-22 16:59:59 +0900')
+    Timecop.freeze(t)
+    @now = Fluent::EventTime.now
+  end
+
+  shutdown do
+    Timecop.return
+  end
+
+  def extract_value_from_counter(counter, key)
+    store = counter.instance_variable_get(:@storage).instance_variable_get(:@store)
+    store[key]
+  end
+
+  def travel(sec)
+    # Since Timecop.travel() causes test failures on Windows/AppVeyor by inducing
+    # rounding errors to Time.now, we need to use Timecop.freeze() instead.
+    Timecop.freeze(Time.now + sec)
+  end
+
+  sub_test_case 'init' do
+    setup do
+      @reset_interval = 10
+      @store = Fluent::Counter::Store.new
+      @data = { 'name' => @name, 'reset_interval' => @reset_interval }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+    end
+
+    test 'create new value in the counter' do
+      v = @store.init(@key, @data)
+
+      assert_equal @name, v['name']
+      assert_equal @reset_interval, v['reset_interval']
+
+      v2 = extract_value_from_counter(@store, @key)
+      v2 = @store.send(:build_response, v2)
+      assert_equal v, v2
+    end
+
+    test 'raise an error when a passed key already exists' do
+      @store.init(@key, @data)
+
+      assert_raise Fluent::Counter::InvalidParams do
+        @store.init(@key, @data)
+      end
+    end
+
+    test 'return a value when passed key already exists and a ignore option is true' do
+      v = @store.init(@key, @data)
+      v1 = extract_value_from_counter(@store, @key)
+      v1 = @store.send(:build_response, v1)
+      v2 = @store.init(@key, @data, ignore: true)
+      assert_equal v, v2
+      assert_equal v1, v2
+    end
+  end
+
+  sub_test_case 'get' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      data = { 'name' => @name, 'reset_interval' => 10 }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(@key, data)
+    end
+
+    test 'return a value from the counter' do
+      v = extract_value_from_counter(@store, @key)
+      expected = @store.send(:build_response, v)
+      assert_equal expected, @store.get(@key)
+    end
+
+    test 'return a raw value from the counter when raw option is true' do
+      v = extract_value_from_counter(@store, @key)
+      assert_equal v, @store.get(@key, raw: true)
+    end
+
+    test "return nil when a passed key doesn't exist" do
+      assert_equal nil, @store.get('unknown_key')
+    end
+
+    test "raise a error when a passed key doesn't exist and raise_error option is true" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.get('unknown_key', raise_error: true)
+      end
+    end
+  end
+
+  sub_test_case 'key?' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      data = { 'name' => @name, 'reset_interval' => 10 }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(@key, data)
+    end
+
+    test 'return true when passed key exists' do
+      assert_true @store.key?(@key)
+    end
+
+    test "return false when passed key doesn't exist" do
+      assert_true !@store.key?('unknown_key')
+    end
+  end
+
+  sub_test_case 'delete' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      data = { 'name' => @name, 'reset_interval' => 10 }
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @init_value = @store.init(@key, data)
+    end
+
+    test 'delete a value from the counter' do
+      v = @store.delete(@key)
+      assert_equal @init_value, v
+      assert_nil extract_value_from_counter(@store, @key)
+    end
+
+    test "raise an error when passed key doesn't exist" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.delete('unknown_key')
+      end
+    end
+  end
+
+  sub_test_case 'inc' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      @init_data = { 'name' => @name, 'reset_interval' => 10 }
+      @travel_sec = 10
+    end
+
+    data(
+      positive: 10,
+      negative: -10
+    )
+    test 'increment or decrement a value in the counter' do |value|
+      key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(key, @init_data)
+      travel(@travel_sec)
+      v = @store.inc(key, { 'value' => value })
+
+      assert_equal value, v['total']
+      assert_equal value, v['current']
+      assert_equal @now, v['last_reset_at'] # last_reset_at doesn't change
+
+      v1 = extract_value_from_counter(@store, key)
+      v1 = @store.send(:build_response, v1)
+      assert_equal v, v1
+    end
+
+    test "raise an error when passed key doesn't exist" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.inc('unknown_key', { 'value' => 1 })
+      end
+    end
+
+    test 'raise an error when a type of passed value is incompatible with a stored value' do
+      key1 = Fluent::Counter::Store.gen_key(@scope, @name)
+      key2 = Fluent::Counter::Store.gen_key(@scope, 'name2')
+      key3 = Fluent::Counter::Store.gen_key(@scope, 'name3')
+      v1 = @store.init(key1, @init_data.merge('type' => 'integer'))
+      v2 = @store.init(key2, @init_data.merge('type' => 'float'))
+      v3 = @store.init(key3, @init_data.merge('type' => 'numeric'))
+      assert_equal 'integer', v1['type']
+      assert_equal 'float', v2['type']
+      assert_equal 'numeric', v3['type']
+
+      assert_raise Fluent::Counter::InvalidParams do
+        @store.inc(key1, { 'value' => 1.1 })
+      end
+
+      assert_raise Fluent::Counter::InvalidParams do
+        @store.inc(key2, { 'value' => 1 })
+      end
+
+      assert_nothing_raised do
+        @store.inc(key3, { 'value' => 1 })
+        @store.inc(key3, { 'value' => 1.0 })
+      end
+    end
+  end
+
+  sub_test_case 'reset' do
+    setup do
+      @store = Fluent::Counter::Store.new
+      @travel_sec = 10
+
+      @inc_value = 10
+      @key = Fluent::Counter::Store.gen_key(@scope, @name)
+      @store.init(@key, { 'name' => @name, 'reset_interval' => 10 })
+      @store.inc(@key, { 'value' => 10 })
+    end
+
+    test 'reset a value in the counter' do
+      travel(@travel_sec)
+
+      v = @store.reset(@key)
+      assert_equal @travel_sec, v['elapsed_time']
+      assert_true v['success']
+      counter = v['counter_data']
+
+      assert_equal @name, counter['name']
+      assert_equal @inc_value, counter['total']
+      assert_equal @inc_value, counter['current']
+      assert_equal 'numeric', counter['type']
+      assert_equal @now, counter['last_reset_at']
+      assert_equal 10, counter['reset_interval']
+
+      v1 = extract_value_from_counter(@store, @key)
+      assert_equal 0, v1['current']
+      assert_true v1['current'].is_a?(Integer)
+      assert_equal @inc_value, v1['total']
+      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])
+      assert_equal (@now + @travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])
+    end
+
+    test 'reset a value after `reset_interval` passed' do
+      first_travel_sec = 5
+      travel(first_travel_sec) # jump time less than reset_interval
+      v = @store.reset(@key)
+
+      assert_equal false, v['success']
+      assert_equal first_travel_sec, v['elapsed_time']
+      store = extract_value_from_counter(@store, @key)
+      assert_equal 10, store['current']
+      assert_equal @now, Fluent::EventTime.new(*store['last_reset_at'])
+
+      # time is passed greater than reset_interval
+      travel(@travel_sec)
+      v = @store.reset(@key)
+      assert_true v['success']
+      assert_equal @travel_sec + first_travel_sec, v['elapsed_time']
+
+      v1 = extract_value_from_counter(@store, @key)
+      assert_equal 0, v1['current']
+      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_reset_at'])
+      assert_equal (@now + @travel_sec + first_travel_sec), Fluent::EventTime.new(*v1['last_modified_at'])
+    end
+
+    test "raise an error when passed key doesn't exist" do
+      assert_raise Fluent::Counter::UnknownKey do
+        @store.reset('unknown_key')
+      end
+    end
+  end
+end

data/test/counter/test_validator.rb

@@ -0,0 +1,137 @@
+require_relative '../helper'
+require 'fluent/counter/validator'
+
+class CounterValidatorTest < ::Test::Unit::TestCase
+  data(
+    invalid_name1: '',
+    invalid_name3: '_',
+    invalid_name4: 'A',
+    invalid_name5: 'a*',
+    invalid_name6: "a\t",
+    invalid_name7: "\n",
+  )
+  test 'invalid name' do |invalid_name|
+    assert_nil(Fluent::Counter::Validator::VALID_NAME =~ invalid_name)
+  end
+
+  sub_test_case 'request' do
+    test 'return an empty array' do
+      data = { 'id' => 0, 'method' => 'init' }
+      errors = Fluent::Counter::Validator.request(data)
+      assert_empty errors
+    end
+
+    data(
+      missing_id: [
+        { 'method' => 'init' },
+        { 'code' => 'invalid_request', 'message' => 'Request should include `id`' }
+      ],
+      missing_method: [
+        { 'id' => 0 },
+        { 'code' => 'invalid_request', 'message' => 'Request should include `method`' }
+      ],
+      invalid_method: [
+        { 'id' => 0, 'method' => "A\t" },
+        { 'code' => 'invalid_request', 'message' => '`method` is the invalid format' }
+      ],
+      unknown_method: [
+        { 'id' => 0, 'method' => 'unknown_method' },
+        { 'code' => 'method_not_found', 'message' => 'Unknown method name passed: unknown_method' }
+      ]
+    )
+    test 'return an error array' do |(data, expected_error)|
+      errors = Fluent::Counter::Validator.request(data)
+      assert_equal [expected_error], errors
+    end
+  end
+
+  sub_test_case 'call' do
+    test "return an error hash when passed method doesn't exist" do
+      v = Fluent::Counter::Validator.new(:unknown)
+      success, errors = v.call(['key1'])
+      assert_empty success
+      assert_equal 'internal_server_error', errors.first.to_hash['code']
+    end
+  end
+
+  test 'validate_empty!' do
+    v = Fluent::Counter::Validator.new(:empty)
+    success, errors = v.call([])
+    assert_empty success
+    assert_equal [Fluent::Counter::InvalidParams.new('One or more `params` are required')], errors
+  end
+end
+
+class CounterArrayValidatorTest < ::Test::Unit::TestCase
+  test 'validate_key!' do
+    ary = ['key', 100, '_']
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => 'The type of `key` should be String' },
+      { 'code' => 'invalid_params', 'message' => '`key` is the invalid format' }
+    ]
+    v = Fluent::Counter::ArrayValidator.new(:key)
+    valid_params, errors = v.call(ary)
+
+    assert_equal ['key'], valid_params
+    assert_equal error_expected, errors.map(&:to_hash)
+  end
+end
+
+class CounterHashValidatorTest < ::Test::Unit::TestCase
+  test 'validate_name!' do
+    hash = [
+      { 'name' => 'key' },
+      {},
+      { 'name' => 10 },
+      { 'name' => '_' }
+    ]
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => '`name` is required' },
+      { 'code' => 'invalid_params', 'message' => 'The type of `name` should be String' },
+      { 'code' => 'invalid_params', 'message' => '`name` is the invalid format' },
+    ]
+    v = Fluent::Counter::HashValidator.new(:name)
+    success, errors = v.call(hash)
+
+    assert_equal [{ 'name' => 'key' }], success
+    assert_equal error_expected, errors.map(&:to_hash)
+  end
+
+  test 'validate_value!' do
+    hash = [
+      { 'value' => 1 },
+      { 'value' => -1 },
+      {},
+      { 'value' => 'str' }
+    ]
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => '`value` is required' },
+      { 'code' => 'invalid_params', 'message' => 'The type of `value` type should be Numeric' },
+    ]
+    v = Fluent::Counter::HashValidator.new(:value)
+    valid_params, errors = v.call(hash)
+
+    assert_equal [{ 'value' => 1 }, { 'value' => -1 }], valid_params
+    assert_equal error_expected, errors.map(&:to_hash)
+  end
+
+  test 'validate_reset_interval!' do
+    hash = [
+      { 'reset_interval' => 1 },
+      { 'reset_interval' => 1.0 },
+      {},
+      { 'reset_interval' => -1 },
+      { 'reset_interval' => 'str' }
+    ]
+    error_expected = [
+      { 'code' => 'invalid_params', 'message' => '`reset_interval` is required' },
+      { 'code' => 'invalid_params', 'message' => '`reset_interval` should be a positive number' },
+      { 'code' => 'invalid_params', 'message' => 'The type of `reset_interval` should be Numeric' },
+    ]
+    v = Fluent::Counter::HashValidator.new(:reset_interval)
+    valid_params, errors = v.call(hash)
+
+    assert_equal [{ 'reset_interval' => 1 }, { 'reset_interval' => 1.0 }], valid_params
+    assert_equal error_expected.map(&:to_hash), errors.map(&:to_hash)
+  end
+end

data/test/helper.rb

@@ -39,9 +39,29 @@ require 'rr'
 require 'test/unit'
 require 'test/unit/rr'
 require 'fileutils'
+require 'fluent/config/element'
 require 'fluent/log'
 require 'fluent/test'
 require 'fluent/test/helpers'
+require 'fluent/plugin/base'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+require 'fluent/msgpack_factory'
+require 'fluent/time'
+require 'serverengine'
+require 'helpers/fuzzy_assert'
+
+module Fluent
+  module Plugin
+    class TestBase < Base
+      # a base plugin class, but not input nor output
+      # mainly for helpers and owned plugins
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+    end
+  end
+end
 
 unless defined?(Test::Unit::AssertionFailedError)
   class Test::Unit::AssertionFailedError < StandardError
@@ -50,11 +70,70 @@ end
 
 include Fluent::Test::Helpers
 
-def unused_port
-  s = TCPServer.open(0)
-  port = s.addr[1]
-  s.close
-  port
+def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
+  case protocol
+  when :tcp
+    unused_port_tcp(num)
+  when :udp
+    unused_port_udp(num, bind: bind)
+  else
+    raise ArgumentError, "unknown protocol: #{protocol}"
+  end
+end
+
+def unused_port_tcp(num = 1)
+  ports = []
+  sockets = []
+  num.times do
+    s = TCPServer.open(0)
+    sockets << s
+    ports << s.addr[1]
+  end
+  sockets.each{|s| s.close }
+  if num == 1
+    return ports.first
+  else
+    return *ports
+  end
+end
+
+PORT_RANGE_AVAILABLE = (1024...65535)
+
+def unused_port_udp(num = 1, bind: "0.0.0.0")
+  family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
+  ports = []
+  sockets = []
+  while ports.size < num
+    port = rand(PORT_RANGE_AVAILABLE)
+    u = UDPSocket.new(family)
+    if (u.bind(bind, port) rescue nil)
+      ports << port
+      sockets << u
+    else
+      u.close
+    end
+  end
+  sockets.each{|s| s.close }
+  if num == 1
+    return ports.first
+  else
+    return *ports
+  end
+end
+
+def waiting(seconds, logs: nil, plugin: nil)
+  begin
+    Timeout.timeout(seconds) do
+      yield
+    end
+  rescue Timeout::Error
+    if logs
+      STDERR.print(*logs)
+    elsif plugin
+      STDERR.print(*plugin.log.out.logs)
+    end
+    raise
+  end
 end
 
 def ipv6_enabled?
@@ -68,4 +147,8 @@ def ipv6_enabled?
   end
 end
 
-$log = Fluent::Log.new(Fluent::Test::DummyLogDevice.new, Fluent::Log::LEVEL_WARN)
+dl_opts = {}
+dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+logdev = Fluent::Test::DummyLogDevice.new
+logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+$log ||= Fluent::Log.new(logger)

data/test/helpers/fuzzy_assert.rb

@@ -0,0 +1,89 @@
+require 'test/unit'
+
+class FuzzyIncludeAssertion
+  include Test::Unit::Assertions
+
+  def self.assert(expected, actual, message = nil)
+    new(expected, actual, message).assert
+  end
+
+  def initialize(expected, actual, message)
+    @expected = expected
+    @actual = actual
+    @message = message
+  end
+
+  def assert
+    if collection?
+      assert_same_collection
+    else
+      assert_same_value
+    end
+  end
+
+  private
+
+  def assert_same_value
+    m = "expected(#{@expected}) !== actual(#{@actual.inspect})"
+    if @message
+      m = "#{@message}: #{m}"
+    end
+    assert_true(@expected === @actual, m)
+  end
+
+  def assert_same_class
+    if @expected.class != @actual.class
+     if (@expected.class.ancestors | @actual.class.ancestors).empty?
+       assert_equal(@expected.class, @actual.class, @message)
+     end
+    end
+  end
+
+  def assert_same_collection
+    assert_same_class
+    assert_same_values
+  end
+
+  def assert_same_values
+    if @expected.is_a?(Array)
+      @expected.each_with_index do |val, i|
+        self.class.assert(val, @actual[i], @message)
+      end
+    else
+      @expected.each do |key, val|
+        self.class.assert(val, @actual[key], "#{key}: ")
+      end
+    end
+  end
+
+  def collection?
+    @actual.is_a?(Array) || @actual.is_a?(Hash)
+  end
+end
+
+class FuzzyAssertion < FuzzyIncludeAssertion
+  private
+
+  def assert_same_collection
+    super
+    assert_same_keys
+  end
+
+  def assert_same_keys
+    if @expected.is_a?(Array)
+      assert_equal(@expected.size, @actual.size, "expected.size(#{@expected}) != actual.size(#{@expected})")
+    else
+      assert_equal(@expected.keys.sort, @actual.keys.sort)
+    end
+  end
+end
+
+module FuzzyAssert
+  def assert_fuzzy_include(left, right, message = nil)
+    FuzzyIncludeAssertion.new(left, right, message).assert
+  end
+
+  def assert_fuzzy_equal(left, right, message = nil)
+    FuzzyAssertion.new(left, right, message).assert
+  end
+end