fluentd-hubspot 0.14.14.1

data/test/config/test_system_config.rb

@@ -0,0 +1,168 @@
+require_relative '../helper'
+require 'fluent/configurable'
+require 'fluent/config/element'
+require 'fluent/config/section'
+require 'fluent/system_config'
+
+module Fluent::Config
+  class FakeLoggerInitializer
+    attr_accessor :level
+    def initalize
+      @level = nil
+    end
+  end
+
+  class FakeSupervisor
+    attr_writer :log_level
+
+    def initialize
+      @workers = nil
+      @root_dir = nil
+      @log = FakeLoggerInitializer.new
+      @log_level = Fluent::Log::LEVEL_INFO
+      @suppress_interval = nil
+      @suppress_config_dump = nil
+      @suppress_repeated_stacktrace = nil
+      @log_event_label = nil
+      @log_event_verbose = nil
+      @without_source = nil
+      @emit_error_log_interval = nil
+      @file_permission = nil
+      @dir_permission = nil
+    end
+  end
+
+  class TestSystemConfig < ::Test::Unit::TestCase
+    TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/system_config/#{ENV['TEST_ENV_NUMBER']}")
+
+    def parse_text(text)
+      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
+    end
+
+    test 'should not override default configurations when no parameters' do
+      conf = parse_text(<<-EOS)
+        <system>
+        </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.apply(s)
+      assert_equal(1, sc.workers)
+      assert_nil(sc.root_dir)
+      assert_nil(sc.log_level)
+      assert_nil(sc.suppress_repeated_stacktrace)
+      assert_nil(sc.emit_error_log_interval)
+      assert_nil(sc.suppress_config_dump)
+      assert_nil(sc.without_source)
+      assert_equal(1, s.instance_variable_get(:@workers))
+      assert_nil(s.instance_variable_get(:@root_dir))
+      assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
+      assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
+      assert_nil(s.instance_variable_get(:@emit_error_log_interval))
+      assert_nil(s.instance_variable_get(:@suppress_config_dump))
+      assert_nil(s.instance_variable_get(:@log_event_verbose))
+      assert_nil(s.instance_variable_get(:@without_source))
+      assert_nil(s.instance_variable_get(:@file_permission))
+      assert_nil(s.instance_variable_get(:@dir_permission))
+    end
+
+    data(
+      'workers' => ['workers', 3],
+      'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
+      'log_level' => ['log_level', 'error'],
+      'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
+      'emit_error_log_interval' => ['emit_error_log_interval', 60],
+      'log_event_verbose' => ['log_event_verbose', true],
+      'suppress_config_dump' => ['suppress_config_dump', true],
+      'without_source' => ['without_source', true],
+    )
+    test "accepts parameters" do |(k, v)|
+      conf = parse_text(<<-EOS)
+          <system>
+            #{k} #{v}
+          </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.apply(s)
+      assert_not_nil(sc.instance_variable_get("@#{k}"))
+      key = (k == 'emit_error_log_interval' ? 'suppress_interval' : k)
+      assert_not_nil(s.instance_variable_get("@#{key}"))
+    end
+
+    data(
+      'foo' => ['foo', 'bar'],
+      'hoge' => ['hoge', 'fuga'],
+    )
+    test "should not affect settable parameters with unknown parameters" do |(k, v)|
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new({k => v})
+      sc.apply(s)
+      assert_equal(1, s.instance_variable_get(:@workers))
+      assert_nil(s.instance_variable_get(:@root_dir))
+      assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
+      assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
+      assert_nil(s.instance_variable_get(:@emit_error_log_interval))
+      assert_nil(s.instance_variable_get(:@suppress_config_dump))
+      assert_nil(s.instance_variable_get(:@log_event_verbose))
+      assert_nil(s.instance_variable_get(:@without_source))
+      assert_nil(s.instance_variable_get(:@file_permission))
+      assert_nil(s.instance_variable_get(:@dir_permission))
+    end
+
+    data('trace' => Fluent::Log::LEVEL_TRACE,
+         'debug' => Fluent::Log::LEVEL_DEBUG,
+         'info' => Fluent::Log::LEVEL_INFO,
+         'warn' => Fluent::Log::LEVEL_WARN,
+         'error' => Fluent::Log::LEVEL_ERROR,
+         'fatal' => Fluent::Log::LEVEL_FATAL)
+    test 'log_level is applied when log_level related command line option is not passed' do |level|
+      conf = parse_text(<<-EOS)
+        <system>
+          log_level #{Fluent::Log::LEVEL_TEXT[level]}
+        </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.attach(s)
+      sc.apply(s)
+      assert_equal(level, s.instance_variable_get("@log").level)
+    end
+
+    # info is removed because info level can't be specified via command line
+    data('trace' => Fluent::Log::LEVEL_TRACE,
+         'debug' => Fluent::Log::LEVEL_DEBUG,
+         'warn' => Fluent::Log::LEVEL_WARN,
+         'error' => Fluent::Log::LEVEL_ERROR,
+         'fatal' => Fluent::Log::LEVEL_FATAL)
+    test 'log_level is ignored when log_level related command line option is passed' do |level|
+      conf = parse_text(<<-EOS)
+        <system>
+          log_level info
+        </system>
+      EOS
+      s = FakeSupervisor.new
+      s.log_level = level
+      sc = Fluent::SystemConfig.new(conf)
+      sc.attach(s)
+      sc.apply(s)
+      assert_equal(level, s.instance_variable_get("@log").level)
+    end
+
+    test 'process global overridable variables' do
+      conf = parse_text(<<-EOS)
+        <system>
+          file_permission 0655
+          dir_permission 0765
+        </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.attach(s)
+      sc.apply(s)
+      assert_equal(0655, s.instance_variable_get(:@file_permission))
+      assert_equal(0765, s.instance_variable_get(:@dir_permission))
+    end
+  end
+end

data/test/config/test_types.rb

@@ -0,0 +1,191 @@
+require 'helper'
+require 'fluent/config/types'
+
+class TestConfigTypes < ::Test::Unit::TestCase
+  include Fluent
+
+  sub_test_case 'Config.size_value' do
+    test 'normal case' do
+      assert_equal(2048, Config.size_value("2k"))
+      assert_equal(2048, Config.size_value("2K"))
+      assert_equal(3145728, Config.size_value("3m"))
+      assert_equal(3145728, Config.size_value("3M"))
+      assert_equal(4294967296, Config.size_value("4g"))
+      assert_equal(4294967296, Config.size_value("4G"))
+      assert_equal(5497558138880, Config.size_value("5t"))
+      assert_equal(5497558138880, Config.size_value("5T"))
+      assert_equal(6, Config.size_value("6"))
+    end
+
+    test 'not assumed case' do
+      assert_equal(6, Config.size_value(6))
+      assert_equal(0, Config.size_value("hoge"))
+      assert_equal(0, Config.size_value(""))
+      assert_equal(0, Config.size_value(nil))
+    end
+  end
+
+  sub_test_case 'Config.time_value' do
+    test 'normal case' do
+      assert_equal(10, Config.time_value("10s"))
+      assert_equal(10, Config.time_value("10sec"))
+      assert_equal(120, Config.time_value("2m"))
+      assert_equal(10800, Config.time_value("3h"))
+      assert_equal(345600, Config.time_value("4d"))
+    end
+
+    test 'not assumed case' do
+      assert_equal(4.0, Config.time_value(4))
+      assert_equal(0.4, Config.time_value(0.4))
+      assert_equal(0.0, Config.time_value("hoge"))
+      assert_equal(0.0, Config.time_value(""))
+      assert_equal(0.0, Config.time_value(nil))
+    end
+  end
+
+  sub_test_case 'Config.bool_value' do
+    test 'normal case' do
+      assert_true Config.bool_value("true")
+      assert_true Config.bool_value("yes")
+      assert_true Config.bool_value("")
+      assert_false Config.bool_value("false")
+      assert_false Config.bool_value("no")
+    end
+
+    test 'not assumed case' do
+      assert_true Config.bool_value(true)
+      assert_false Config.bool_value(false)
+      assert_nil Config.bool_value("hoge")
+      assert_nil Config.bool_value(nil)
+      assert_nil Config.bool_value(10)
+    end
+  end
+
+  sub_test_case 'type converters for config_param definitions' do
+    test 'string' do
+      assert_equal 'test', Config::STRING_TYPE.call('test', {})
+      assert_equal '1', Config::STRING_TYPE.call('1', {})
+      assert_equal '   ', Config::STRING_TYPE.call('   ', {})
+      assert_equal Encoding::UTF_8, Config::STRING_TYPE.call('test', {}).encoding
+    end
+
+    data('latin' => 'Märch',
+         'ascii' => 'ascii',
+         'space' => '     ',
+         'number' => '1',
+         'Hiragana' => 'あいうえお')
+    test 'string w/ binary' do |str|
+      actual = Config::STRING_TYPE.call(str.b, {})
+      assert_equal str, actual
+      assert_equal Encoding::UTF_8, actual.encoding
+    end
+
+    test 'enum' do
+      assert_equal :val, Config::ENUM_TYPE.call('val', {list: [:val, :value, :v]})
+      assert_equal :v, Config::ENUM_TYPE.call('v', {list: [:val, :value, :v]})
+      assert_equal :value, Config::ENUM_TYPE.call('value', {list: [:val, :value, :v]})
+      assert_raises(Fluent::ConfigError.new("valid options are val,value,v but got x")){ Config::ENUM_TYPE.call('x', {list: [:val, :value, :v]}) }
+      assert_raises(RuntimeError.new("Plugin BUG: config type 'enum' requires :list of symbols")){ Config::ENUM_TYPE.call('val', {}) }
+      assert_raises(RuntimeError.new("Plugin BUG: config type 'enum' requires :list of symbols")){ Config::ENUM_TYPE.call('val', {list: ["val", "value", "v"]}) }
+    end
+
+    test 'integer' do
+      assert_equal 1, Config::INTEGER_TYPE.call('1', {})
+      assert_equal 1, Config::INTEGER_TYPE.call('1.0', {})
+      assert_equal 1000, Config::INTEGER_TYPE.call('1_000', {})
+      assert_equal 1, Config::INTEGER_TYPE.call('1x', {})
+    end
+
+    test 'float' do
+      assert_equal 1.0, Config::FLOAT_TYPE.call('1', {})
+      assert_equal 1.0, Config::FLOAT_TYPE.call('1.0', {})
+      assert_equal 1.0, Config::FLOAT_TYPE.call('1.00', {})
+      assert_equal 1.0, Config::FLOAT_TYPE.call('1e0', {})
+    end
+
+    test 'size' do
+      assert_equal 1000, Config::SIZE_TYPE.call('1000', {})
+      assert_equal 1024, Config::SIZE_TYPE.call('1k', {})
+      assert_equal 1024*1024, Config::SIZE_TYPE.call('1m', {})
+    end
+
+    test 'bool' do
+      assert_equal true, Config::BOOL_TYPE.call('true', {})
+      assert_equal true, Config::BOOL_TYPE.call('yes', {})
+      assert_equal false, Config::BOOL_TYPE.call('no', {})
+      assert_equal false, Config::BOOL_TYPE.call('false', {})
+
+      assert_equal nil, Config::BOOL_TYPE.call('TRUE', {})
+      assert_equal nil, Config::BOOL_TYPE.call('True', {})
+      assert_equal nil, Config::BOOL_TYPE.call('Yes', {})
+      assert_equal nil, Config::BOOL_TYPE.call('No', {})
+
+      assert_equal true, Config::BOOL_TYPE.call('', {})
+      assert_equal nil, Config::BOOL_TYPE.call('unexpected_string', {})
+    end
+
+    test 'time' do
+      assert_equal 0, Config::TIME_TYPE.call('0', {})
+      assert_equal 1.0, Config::TIME_TYPE.call('1', {})
+      assert_equal 1.01, Config::TIME_TYPE.call('1.01', {})
+      assert_equal 1, Config::TIME_TYPE.call('1s', {})
+      assert_equal 60, Config::TIME_TYPE.call('1m', {})
+      assert_equal 3600, Config::TIME_TYPE.call('1h', {})
+      assert_equal 86400, Config::TIME_TYPE.call('1d', {})
+    end
+
+    test 'hash' do
+      assert_equal({"x"=>"v","k"=>1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v,k:1', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v, k:1', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call(' x:v, k:1 ', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v , k:1 ', {}))
+
+      assert_equal({"x"=>"v:v","k"=>"1"}, Config::HASH_TYPE.call('x:v:v, k:1', {}))
+
+      assert_equal({x: "v", k: 1},   Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true}))
+      assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('x:v,k:1',         {symbolize_keys: true, value_type: :string}))
+      assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true, value_type: :string}))
+      assert_equal({x: 0, k: 1},     Config::HASH_TYPE.call('x:0,k:1',         {symbolize_keys: true, value_type: :integer}))
+
+      assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('{"x":"1s","y":"1m","z":"1h"}', {value_type: :time}))
+      assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('x:1s,y:1m,z:1h',               {value_type: :time}))
+
+      assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo}) }
+    end
+
+    data('latin' => ['3:Märch', {"3"=>"Märch"}],
+         'ascii' => ['ascii:ascii', {"ascii"=>"ascii"}],
+         'number' => ['number:1', {"number"=>"1"}],
+         'Hiragana' => ['hiragana:あいうえお', {"hiragana"=>"あいうえお"}])
+    test 'hash w/ binary' do |(target, expected)|
+      assert_equal(expected, Config::HASH_TYPE.call(target.b, { value_type: :string }))
+    end
+
+    test 'array' do
+      assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {}))
+      assert_equal(["1","2","1"], Config::ARRAY_TYPE.call('1,2,1', {}))
+
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('["a","b","c"]', {}))
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a,b,c', {}))
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a, b, c', {}))
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a , b , c', {}))
+
+      assert_equal(["a a","b,b"," c "], Config::ARRAY_TYPE.call('["a a","b,b"," c "]', {}))
+
+      assert_equal(["a a","b","c"], Config::ARRAY_TYPE.call('a a,b,c', {}))
+
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('[1,2,1]', {}))
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('["1","2","1"]', {value_type: :integer}))
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('1,2,1', {value_type: :integer}))
+
+      array_options = {
+        default: [],
+      }
+      assert_equal(["1","2"], Config::ARRAY_TYPE.call('["1","2"]', array_options))
+      assert_equal(["3"], Config::ARRAY_TYPE.call('["3"]', array_options))
+
+      assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::ARRAY_TYPE.call("1,2", {value_type: :foo}) }
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,153 @@
+# simplecov must be loaded before any of target code
+if ENV['SIMPLE_COV']
+  require 'simplecov'
+  if defined?(SimpleCov::SourceFile)
+    mod = SimpleCov::SourceFile
+    def mod.new(*args, &block)
+      m = allocate
+      m.instance_eval do
+        begin
+          initialize(*args, &block)
+        rescue Encoding::UndefinedConversionError
+          @src = "".force_encoding('UTF-8')
+        end
+      end
+      m
+    end
+  end
+  unless SimpleCov.running
+    SimpleCov.start do
+      add_filter '/test/'
+      add_filter '/gems/'
+    end
+  end
+end
+
+# Some tests use Hash instead of Element for configure.
+# We should rewrite these tests in the future and remove this ad-hoc code
+class Hash
+  def corresponding_proxies
+    @corresponding_proxies ||= []
+  end
+
+  def to_masked_element
+    self
+  end
+end
+
+require 'rr'
+require 'test/unit'
+require 'test/unit/rr'
+require 'fileutils'
+require 'fluent/config/element'
+require 'fluent/log'
+require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/plugin/base'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+require 'fluent/msgpack_factory'
+require 'fluent/time'
+require 'serverengine'
+
+module Fluent
+  module Plugin
+    class TestBase < Base
+      # a base plugin class, but not input nor output
+      # mainly for helpers and owned plugins
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+    end
+  end
+end
+
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
+include Fluent::Test::Helpers
+
+def unused_port(num = 1, protocol: :tcp, bind: "0.0.0.0")
+  case protocol
+  when :tcp
+    unused_port_tcp(num)
+  when :udp
+    unused_port_udp(num, bind: bind)
+  else
+    raise ArgumentError, "unknown protocol: #{protocol}"
+  end
+end
+
+def unused_port_tcp(num = 1)
+  ports = []
+  sockets = []
+  num.times do
+    s = TCPServer.open(0)
+    sockets << s
+    ports << s.addr[1]
+  end
+  sockets.each{|s| s.close }
+  if num == 1
+    return ports.first
+  else
+    return *ports
+  end
+end
+
+PORT_RANGE_AVAILABLE = (1024...65535)
+
+def unused_port_udp(num = 1, bind: "0.0.0.0")
+  family = IPAddr.new(IPSocket.getaddress(bind)).ipv4? ? ::Socket::AF_INET : ::Socket::AF_INET6
+  ports = []
+  sockets = []
+  while ports.size < num
+    port = rand(PORT_RANGE_AVAILABLE)
+    u = UDPSocket.new(family)
+    if (u.bind(bind, port) rescue nil)
+      ports << port
+      sockets << u
+    else
+      u.close
+    end
+  end
+  sockets.each{|s| s.close }
+  if num == 1
+    return ports.first
+  else
+    return *ports
+  end
+end
+
+def waiting(seconds, logs: nil, plugin: nil)
+  begin
+    Timeout.timeout(seconds) do
+      yield
+    end
+  rescue Timeout::Error
+    if logs
+      STDERR.print(*logs)
+    elsif plugin
+      STDERR.print(*plugin.log.out.logs)
+    end
+    raise
+  end
+end
+
+def ipv6_enabled?
+  require 'socket'
+
+  begin
+    TCPServer.open("::1", 0)
+    true
+  rescue
+    false
+  end
+end
+
+dl_opts = {}
+dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+logdev = Fluent::Test::DummyLogDevice.new
+logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+$log ||= Fluent::Log.new(logger)