fluentd-hubspot 0.14.14.1

data/test/scripts/fluent/plugin/parser_known.rb

@@ -0,0 +1,4 @@
+module Fluent
+  TextParser.register_template('known_old', /^(?<message>.*)$/)
+  Plugin.register_parser('known', /^(?<message>.*)$/)
+end

data/test/test_clock.rb

@@ -0,0 +1,164 @@
+require_relative 'helper'
+require 'fluent/clock'
+
+require 'timecop'
+
+class ClockTest < ::Test::Unit::TestCase
+  teardown do
+    Fluent::Clock.return # call it always not to affect other tests
+  end
+
+  sub_test_case 'without any pre-operation' do
+    test 'clock can provides incremental floating point number based on second' do
+      c1 = Fluent::Clock.now
+      assert_kind_of Float, c1
+      sleep 1.1
+      c2 = Fluent::Clock.now
+      assert{ c2 >= c1 + 1.0 && c2 < c1 + 9.0 } # if clock returns deci-second (fantastic!), c2 should be larger than c1 + 10
+    end
+
+    test 'clock value will proceed even if timecop freezes Time' do
+      Timecop.freeze(Time.now) do
+        c1 = Fluent::Clock.now
+        assert_kind_of Float, c1
+        sleep 1.1
+        c2 = Fluent::Clock.now
+        assert{ c2 >= c1 + 1.0 && c2 < c1 + 9.0 }
+      end
+    end
+  end
+
+  sub_test_case 'using #freeze without any arguments' do
+    test 'Clock.freeze without arguments freezes clock with current clock value' do
+      c0 = Fluent::Clock.now
+      Fluent::Clock.freeze
+      c1 = Fluent::Clock.now
+      Fluent::Clock.return
+      c2 = Fluent::Clock.now
+      assert{ c0 <= c1 && c1 <= c2 }
+    end
+
+    test 'Clock.return raises an error if it is called in block' do
+      assert_raise RuntimeError.new("invalid return while running code in blocks") do
+        Fluent::Clock.freeze do
+          Fluent::Clock.return
+        end
+      end
+    end
+  end
+
+  sub_test_case 'using #freeze with clock value' do
+    test 'Clock.now always returns frozen time until #return called' do
+      c0 = Fluent::Clock.now
+      Fluent::Clock.freeze(c0)
+      assert_equal c0, Fluent::Clock.now
+      sleep 0.5
+      assert_equal c0, Fluent::Clock.now
+      sleep 0.6
+      assert_equal c0, Fluent::Clock.now
+
+      Fluent::Clock.return
+      c1 = Fluent::Clock.now
+      assert{ c1 >= c0 + 1.0 }
+    end
+
+    test 'Clock.now returns frozen time in the block argument of #freeze' do
+      c0 = Fluent::Clock.now
+      Fluent::Clock.freeze(c0) do
+        assert_equal c0, Fluent::Clock.now
+        sleep 0.5
+        assert_equal c0, Fluent::Clock.now
+        sleep 0.6
+        assert_equal c0, Fluent::Clock.now
+      end
+      c1 = Fluent::Clock.now
+      assert{ c1 >= c0 + 1.0 }
+    end
+
+    test 'Clock.now returns unfrozen value after jumping out from block by raising errors' do
+      c0 = Fluent::Clock.now
+      rescued_error = nil
+      begin
+        Fluent::Clock.freeze(c0) do
+          assert_equal c0, Fluent::Clock.now
+          sleep 0.5
+          assert_equal c0, Fluent::Clock.now
+          sleep 0.6
+          assert_equal c0, Fluent::Clock.now
+          raise "bye!"
+        end
+      rescue => e
+        rescued_error = e
+      end
+      assert rescued_error # ensure to rescue an error
+      c1 = Fluent::Clock.now
+      assert{ c1 >= c0 + 1.0 }
+    end
+
+    test 'Clock.return cancels all Clock.freeze effects by just once' do
+      c0 = Fluent::Clock.now
+      sleep 0.1
+      c1 = Fluent::Clock.now
+      sleep 0.1
+      c2 = Fluent::Clock.now
+      Fluent::Clock.freeze(c0)
+      sleep 0.1
+      assert_equal c0, Fluent::Clock.now
+      Fluent::Clock.freeze(c1)
+      sleep 0.1
+      assert_equal c1, Fluent::Clock.now
+      Fluent::Clock.freeze(c2)
+      sleep 0.1
+      assert_equal c2, Fluent::Clock.now
+
+      Fluent::Clock.return
+      assert{ Fluent::Clock.now > c2 }
+    end
+
+    test 'Clock.freeze allows nested blocks by itself' do
+      c0 = Fluent::Clock.now
+      sleep 0.1
+      c1 = Fluent::Clock.now
+      sleep 0.1
+      c2 = Fluent::Clock.now
+      Fluent::Clock.freeze(c0) do
+        sleep 0.1
+        assert_equal c0, Fluent::Clock.now
+        Fluent::Clock.freeze(c1) do
+          sleep 0.1
+          assert_equal c1, Fluent::Clock.now
+          Fluent::Clock.freeze(c2) do
+            sleep 0.1
+            assert_equal c2, Fluent::Clock.now
+          end
+          assert_equal c1, Fluent::Clock.now
+        end
+        assert_equal c0, Fluent::Clock.now
+      end
+      assert{ Fluent::Clock.now > c0 }
+    end
+  end
+
+  sub_test_case 'using #freeze with Time argument' do
+    test 'Clock.freeze returns the clock value which should be produced when the time is at the specified time' do
+      c0 = Fluent::Clock.now
+      t0 = Time.now
+      t1 = t0 - 30
+      assert_kind_of Time, t1
+      t2 = t0 + 30
+      assert_kind_of Time, t2
+
+      # 31 is for error of floating point value
+      Fluent::Clock.freeze(t1) do
+        c1 = Fluent::Clock.now
+        assert{ c1 >= c0 - 31 && c1 <= c0 - 31 + 10 } # +10 is for threading schedule error
+      end
+
+      # 29 is for error of floating point value
+      Fluent::Clock.freeze(t2) do
+        c2 = Fluent::Clock.now
+        assert{ c2 >= c0 + 29 && c2 <= c0 + 29 + 10 } # +10 is for threading schedule error
+      end
+    end
+  end
+end

data/test/test_config.rb

@@ -0,0 +1,179 @@
+require_relative 'helper'
+require 'fluent/config'
+require 'fluent/config/parser'
+require 'fluent/supervisor'
+require 'fluent/load'
+require 'fileutils'
+
+class ConfigTest < Test::Unit::TestCase
+  include Fluent
+
+  TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
+
+  def read_config(path)
+    path = File.expand_path(path)
+    File.open(path) { |io|
+      Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
+    }
+  end
+
+  def prepare_config
+    write_config "#{TMP_DIR}/config_test_1.conf", %[
+      k1 root_config
+      include dir/config_test_2.conf  #
+      include #{TMP_DIR}/config_test_4.conf
+      include file://#{TMP_DIR}/config_test_5.conf
+      <include config.d/*.conf />
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+      k2 relative_path_include
+      include ../config_test_3.conf
+    ]
+    write_config "#{TMP_DIR}/config_test_3.conf", %[
+      k3 relative_include_in_included_file
+    ]
+    write_config "#{TMP_DIR}/config_test_4.conf", %[
+      k4 absolute_path_include
+    ]
+    write_config "#{TMP_DIR}/config_test_5.conf", %[
+      k5 uri_include
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+      k6 wildcard_include_1
+      <elem1 name>
+        include normal_parameter
+      </elem1>
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+      k7 wildcard_include_2
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+      <elem2 name>
+        <include ../dir/config_test_9.conf />
+      </elem2>
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+      k9 embeded
+      <elem3 name>
+        nested nested_value
+        include hoge
+      </elem3>
+    ]
+    write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+      k8 wildcard_include_3
+      <elem4 name>
+        include normal_parameter
+      </elem4>
+    ]
+
+  end
+
+  def test_include
+    prepare_config
+    c = read_config("#{TMP_DIR}/config_test_1.conf")
+    assert_equal 'root_config', c['k1']
+    assert_equal 'relative_path_include', c['k2']
+    assert_equal 'relative_include_in_included_file', c['k3']
+    assert_equal 'absolute_path_include', c['k4']
+    assert_equal 'uri_include', c['k5']
+    assert_equal 'wildcard_include_1', c['k6']
+    assert_equal 'wildcard_include_2', c['k7']
+    assert_equal 'wildcard_include_3', c['k8']
+    assert_equal [
+      'k1',
+      'k2',
+      'k3',
+      'k4',
+      'k5',
+      'k8', # Because of the file name this comes first.
+      'k6',
+      'k7',
+    ], c.keys
+
+    elem1 = c.elements.find { |e| e.name == 'elem1' }
+    assert_not_nil elem1
+    assert_equal 'name', elem1.arg
+    assert_equal 'normal_parameter', elem1['include']
+
+    elem2 = c.elements.find { |e| e.name == 'elem2' }
+    assert_not_nil elem2
+    assert_equal 'name', elem2.arg
+    assert_equal 'embeded', elem2['k9']
+    assert !elem2.has_key?('include')
+
+    elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+    assert_not_nil elem3
+    assert_equal 'nested_value', elem3['nested']
+    assert_equal 'hoge', elem3['include']
+  end
+
+  def test_check_not_fetchd
+    write_config "#{TMP_DIR}/config_test_not_fetched.conf", %[
+      <match dummy>
+       type          rewrite
+       add_prefix    filtered
+       <rule>
+         key     path
+         pattern ^[A-Z]+
+         replace
+       </rule>
+     </match>
+    ]
+    root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.conf")
+    match_conf = root_conf.elements.first
+    rule_conf  = match_conf.elements.first
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[key pattern replace], not_fetched
+
+    # accessing should delete
+    match_conf['type']
+    rule_conf['key']
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[pattern replace], not_fetched
+
+    # repeateadly accessing should not grow memory usage
+    before_size = match_conf.unused.size
+    10.times { match_conf['type'] }
+    assert_equal before_size, match_conf.unused.size
+  end
+
+  def write_config(path, data)
+    FileUtils.mkdir_p(File.dirname(path))
+    File.open(path, "w") {|f| f.write data }
+  end
+
+  def test_inline
+    prepare_config
+    opts = {
+        :config_path => "#{TMP_DIR}/config_test_1.conf",
+        :inline_config => "<source>\n  type http\n  port 2222\n </source>"
+    }
+    assert_nothing_raised do
+      Fluent::Supervisor.new(opts)
+    end
+    create_warn_dummy_logger
+  end
+
+  def create_warn_dummy_logger
+    dl_opts = {}
+    dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+    logdev = Fluent::Test::DummyLogDevice.new
+    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+    $log = Fluent::Log.new(logger)
+  end
+end
+

data/test/test_configdsl.rb

@@ -0,0 +1,148 @@
+require_relative 'helper'
+require 'fluent/config/dsl'
+require 'fluent/test'
+
+class ConfigDSLTest < Test::Unit::TestCase
+  # TEST_CONFIG1 = %[
+  #   <source>
+  #     type forward
+  #     port 24224
+  #   </source>
+  #   <match test.**>
+  #     type forward
+  #     flush_interval 1s
+  #     <server>
+  #       host host0.example.com
+  #       port 24224
+  #     </server>
+  #     <server>
+  #       host host1.example.com
+  #       port 24224
+  #     </server>
+  #   </match>
+  # ]
+  TEST_DSL_CONFIG1 = %q[
+source {
+  type "forward"
+  port 24224
+}
+match('test.**') {
+  type "forward"
+  flush_interval "1s"
+  (0..1).each do |i|
+    server {
+      host "host#{i}.example.com"
+      port 24224
+    }
+  end
+}
+]
+
+  TEST_DSL_CONFIG2 = %q[
+v = [0, 1, 2]
+]
+
+  TEST_DSL_CONFIG3 = %q[
+match
+]
+
+  TEST_DSL_CONFIG4 = %q[
+match('aa', 'bb'){
+  type :null
+}
+]
+
+  TEST_DSL_CONFIG5 = %q[
+match('aa')
+]
+
+  def test_parse
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG1)
+
+    assert_equal 0, root.keys.size
+    assert_equal 2, root.elements.size
+
+    e0 = root.elements[0]
+    assert_equal 'source', e0.name
+    assert_equal '', e0.arg
+    assert_equal 'forward', e0['type']
+    assert_equal '24224', e0['port']
+
+    e1 = root.elements[1]
+    assert_equal 'match', e1.name
+    assert_equal 'test.**', e1.arg
+    assert_equal 'forward', e1['type']
+    assert_equal '1s', e1['flush_interval']
+    assert_equal 2, e1.elements.size
+    e1s0 = e1.elements[0]
+    assert_equal 'server', e1s0.name
+    assert_equal 'host0.example.com', e1s0['host']
+    assert_equal '24224', e1s0['port']
+    e1s1 = e1.elements[1]
+    assert_equal 'server', e1s1.name
+    assert_equal 'host1.example.com', e1s1['host']
+    assert_equal '24224', e1s1['port']
+  end
+
+  def test_parse2
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG2)
+
+    assert_equal 0, root.keys.size
+    assert_equal 0, root.elements.size
+  end
+
+  def test_config_error
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG3)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG4)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG5)
+    }
+  end
+
+  def test_with_ruby_keyword
+    uname_string = `uname -a`
+    root1 = Fluent::Config::DSL::Parser.parse(<<DSL)
+uname_str = ruby.open('|uname -a'){|out| out.read}
+source {
+  uname uname_str
+}
+DSL
+    source1 = root1.elements.first
+    assert_equal 'source', source1.name
+    assert_equal 1, source1.keys.size
+    assert_equal uname_string, source1['uname']
+
+    root2 = Fluent::Config::DSL::Parser.parse(<<DSL)
+ruby_version = ruby {
+  require 'erb'
+  ERB.new('<%= RUBY_VERSION %> from erb').result
+}
+source {
+  version ruby_version
+}
+DSL
+    source2 = root2.elements.first
+    assert_equal 'source', source2.name
+    assert_equal 1, source2.keys.size
+    assert_equal "#{RUBY_VERSION} from erb", source2['version']
+
+    # Parser#parse raises NoMethodError when configuration dsl elements are written in ruby block
+    conf3 = <<DSL
+ruby {
+  source {
+    type "tail"
+  }
+}
+source {
+  uname uname_str
+}
+DSL
+    assert_raise (NoMethodError) { Fluent::Config::DSL::Parser.parse(conf3) }
+  end
+end