fluentd 0.14.4-x64-mingw32

data/test/scripts/fluent/plugin/parser_known.rb

@@ -0,0 +1,4 @@
+module Fluent
+  TextParser.register_template('known_old', /^(?<message>.*)$/)
+  Plugin.register_parser('known', /^(?<message>.*)$/)
+end

data/test/test_config.rb

@@ -0,0 +1,179 @@
+require_relative 'helper'
+require 'fluent/config'
+require 'fluent/config/parser'
+require 'fluent/supervisor'
+require 'fluent/load'
+require 'fileutils'
+
+class ConfigTest < Test::Unit::TestCase
+  include Fluent
+
+  TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
+
+  def read_config(path)
+    path = File.expand_path(path)
+    File.open(path) { |io|
+      Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
+    }
+  end
+
+  def prepare_config
+    write_config "#{TMP_DIR}/config_test_1.conf", %[
+      k1 root_config
+      include dir/config_test_2.conf  #
+      include #{TMP_DIR}/config_test_4.conf
+      include file://#{TMP_DIR}/config_test_5.conf
+      <include config.d/*.conf />
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+      k2 relative_path_include
+      include ../config_test_3.conf
+    ]
+    write_config "#{TMP_DIR}/config_test_3.conf", %[
+      k3 relative_include_in_included_file
+    ]
+    write_config "#{TMP_DIR}/config_test_4.conf", %[
+      k4 absolute_path_include
+    ]
+    write_config "#{TMP_DIR}/config_test_5.conf", %[
+      k5 uri_include
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+      k6 wildcard_include_1
+      <elem1 name>
+        include normal_parameter
+      </elem1>
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+      k7 wildcard_include_2
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+      <elem2 name>
+        <include ../dir/config_test_9.conf />
+      </elem2>
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+      k9 embeded
+      <elem3 name>
+        nested nested_value
+        include hoge
+      </elem3>
+    ]
+    write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+      k8 wildcard_include_3
+      <elem4 name>
+        include normal_parameter
+      </elem4>
+    ]
+
+  end
+
+  def test_include
+    prepare_config
+    c = read_config("#{TMP_DIR}/config_test_1.conf")
+    assert_equal 'root_config', c['k1']
+    assert_equal 'relative_path_include', c['k2']
+    assert_equal 'relative_include_in_included_file', c['k3']
+    assert_equal 'absolute_path_include', c['k4']
+    assert_equal 'uri_include', c['k5']
+    assert_equal 'wildcard_include_1', c['k6']
+    assert_equal 'wildcard_include_2', c['k7']
+    assert_equal 'wildcard_include_3', c['k8']
+    assert_equal [
+      'k1',
+      'k2',
+      'k3',
+      'k4',
+      'k5',
+      'k8', # Because of the file name this comes first.
+      'k6',
+      'k7',
+    ], c.keys
+
+    elem1 = c.elements.find { |e| e.name == 'elem1' }
+    assert_not_nil elem1
+    assert_equal 'name', elem1.arg
+    assert_equal 'normal_parameter', elem1['include']
+
+    elem2 = c.elements.find { |e| e.name == 'elem2' }
+    assert_not_nil elem2
+    assert_equal 'name', elem2.arg
+    assert_equal 'embeded', elem2['k9']
+    assert !elem2.has_key?('include')
+
+    elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+    assert_not_nil elem3
+    assert_equal 'nested_value', elem3['nested']
+    assert_equal 'hoge', elem3['include']
+  end
+
+  def test_check_not_fetchd
+    write_config "#{TMP_DIR}/config_test_not_fetched.conf", %[
+      <match dummy>
+       type          rewrite
+       add_prefix    filtered
+       <rule>
+         key     path
+         pattern ^[A-Z]+
+         replace
+       </rule>
+     </match>
+    ]
+    root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.conf")
+    match_conf = root_conf.elements.first
+    rule_conf  = match_conf.elements.first
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[type add_prefix key pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[key pattern replace], not_fetched
+
+    # accessing should delete
+    match_conf['type']
+    rule_conf['key']
+
+    not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[add_prefix pattern replace], not_fetched
+
+    not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+    assert_equal %w[pattern replace], not_fetched
+
+    # repeateadly accessing should not grow memory usage
+    before_size = match_conf.unused.size
+    10.times { match_conf['type'] }
+    assert_equal before_size, match_conf.unused.size
+  end
+
+  def write_config(path, data)
+    FileUtils.mkdir_p(File.dirname(path))
+    File.open(path, "w") {|f| f.write data }
+  end
+
+  def test_inline
+    prepare_config
+    opts = {
+        :config_path => "#{TMP_DIR}/config_test_1.conf",
+        :inline_config => "<source>\n  type http\n  port 2222\n </source>"
+    }
+    assert_nothing_raised do
+      Fluent::Supervisor.new(opts)
+    end
+    create_warn_dummy_logger
+  end
+
+  def create_warn_dummy_logger
+    dl_opts = {}
+    dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
+    logdev = Fluent::Test::DummyLogDevice.new
+    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
+    $log = Fluent::Log.new(logger)
+  end
+end
+

data/test/test_configdsl.rb

@@ -0,0 +1,148 @@
+require_relative 'helper'
+require 'fluent/config/dsl'
+require 'fluent/test'
+
+class ConfigDSLTest < Test::Unit::TestCase
+  # TEST_CONFIG1 = %[
+  #   <source>
+  #     type forward
+  #     port 24224
+  #   </source>
+  #   <match test.**>
+  #     type forward
+  #     flush_interval 1s
+  #     <server>
+  #       host host0.example.com
+  #       port 24224
+  #     </server>
+  #     <server>
+  #       host host1.example.com
+  #       port 24224
+  #     </server>
+  #   </match>
+  # ]
+  TEST_DSL_CONFIG1 = %q[
+source {
+  type "forward"
+  port 24224
+}
+match('test.**') {
+  type "forward"
+  flush_interval "1s"
+  (0..1).each do |i|
+    server {
+      host "host#{i}.example.com"
+      port 24224
+    }
+  end
+}
+]
+
+  TEST_DSL_CONFIG2 = %q[
+v = [0, 1, 2]
+]
+
+  TEST_DSL_CONFIG3 = %q[
+match
+]
+
+  TEST_DSL_CONFIG4 = %q[
+match('aa', 'bb'){
+  type :null
+}
+]
+
+  TEST_DSL_CONFIG5 = %q[
+match('aa')
+]
+
+  def test_parse
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG1)
+
+    assert_equal 0, root.keys.size
+    assert_equal 2, root.elements.size
+
+    e0 = root.elements[0]
+    assert_equal 'source', e0.name
+    assert_equal '', e0.arg
+    assert_equal 'forward', e0['type']
+    assert_equal '24224', e0['port']
+
+    e1 = root.elements[1]
+    assert_equal 'match', e1.name
+    assert_equal 'test.**', e1.arg
+    assert_equal 'forward', e1['type']
+    assert_equal '1s', e1['flush_interval']
+    assert_equal 2, e1.elements.size
+    e1s0 = e1.elements[0]
+    assert_equal 'server', e1s0.name
+    assert_equal 'host0.example.com', e1s0['host']
+    assert_equal '24224', e1s0['port']
+    e1s1 = e1.elements[1]
+    assert_equal 'server', e1s1.name
+    assert_equal 'host1.example.com', e1s1['host']
+    assert_equal '24224', e1s1['port']
+  end
+
+  def test_parse2
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG2)
+
+    assert_equal 0, root.keys.size
+    assert_equal 0, root.elements.size
+  end
+
+  def test_config_error
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG3)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG4)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG5)
+    }
+  end
+
+  def test_with_ruby_keyword
+    uname_string = `uname -a`
+    root1 = Fluent::Config::DSL::Parser.parse(<<DSL)
+uname_str = ruby.open('|uname -a'){|out| out.read}
+source {
+  uname uname_str
+}
+DSL
+    source1 = root1.elements.first
+    assert_equal 'source', source1.name
+    assert_equal 1, source1.keys.size
+    assert_equal uname_string, source1['uname']
+
+    root2 = Fluent::Config::DSL::Parser.parse(<<DSL)
+ruby_version = ruby {
+  require 'erb'
+  ERB.new('<%= RUBY_VERSION %> from erb').result
+}
+source {
+  version ruby_version
+}
+DSL
+    source2 = root2.elements.first
+    assert_equal 'source', source2.name
+    assert_equal 1, source2.keys.size
+    assert_equal "#{RUBY_VERSION} from erb", source2['version']
+
+    # Parser#parse raises NoMethodError when configuration dsl elements are written in ruby block
+    conf3 = <<DSL
+ruby {
+  source {
+    type "tail"
+  }
+}
+source {
+  uname uname_str
+}
+DSL
+    assert_raise (NoMethodError) { Fluent::Config::DSL::Parser.parse(conf3) }
+  end
+end

data/test/test_event.rb

@@ -0,0 +1,329 @@
+require_relative 'helper'
+require 'fluent/test'
+require 'fluent/event'
+
+module EventTest
+  module DeepCopyAssertion
+    def assert_duplicated_records(es1, es2)
+      ary1 = []
+      es1.each do |_, record|
+        ary1 << record
+      end
+      ary2 = []
+      es2.each do |_, record|
+        ary2 << record
+      end
+      assert_equal ary1.size, ary2.size
+      ary1.each_with_index do |r, i|
+        assert_not_equal r.object_id, ary2[i].object_id
+      end
+    end
+  end
+
+  class OneEventStreamTest < ::Test::Unit::TestCase
+    include Fluent
+    include DeepCopyAssertion
+
+    def setup
+      @time = event_time()
+      @record = {'k' => 'v', 'n' => 1}
+      @es = OneEventStream.new(@time, @record)
+    end
+
+    test 'empty?' do
+      assert_false @es.empty?
+    end
+
+    test 'size' do
+      assert_equal 1, @es.size
+    end
+
+    test 'repeatable?' do
+      assert_true @es.repeatable?
+    end
+
+    test 'dup' do
+      dupped = @es.dup
+      assert_kind_of OneEventStream, dupped
+      assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+    end
+
+    test 'slice' do
+      assert_equal 0, @es.slice(1, 1).size
+      assert_equal 0, @es.slice(0, 0).size
+
+      sliced = @es.slice(0, 1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @time, time
+        assert_equal @record, record
+      end
+    end
+
+    test 'each' do
+      @es.each { |time, record|
+        assert_equal @time, time
+        assert_equal @record, record
+      }
+    end
+
+    test 'to_msgpack_stream' do
+      stream = @es.to_msgpack_stream
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @time, time
+        assert_equal @record, record
+      }
+    end
+
+    test 'to_msgpack_stream with time_int argument' do
+      stream = @es.to_msgpack_stream(time_int: true)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @time.to_i, time
+        assert_equal @record, record
+      }
+    end
+  end
+
+  class ArrayEventStreamTest < ::Test::Unit::TestCase
+    include Fluent
+    include DeepCopyAssertion
+
+    def setup
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
+      @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
+      @es = ArrayEventStream.new(@times.zip(@records))
+    end
+
+    test 'repeatable?' do
+      assert_true @es.repeatable?
+    end
+
+    test 'dup' do
+      dupped = @es.dup
+      assert_kind_of ArrayEventStream, dupped
+      assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+    end
+
+    test 'empty?' do
+      assert_not_empty @es
+      assert_true ArrayEventStream.new([]).empty?
+    end
+
+    test 'size' do
+      assert_equal 2, @es.size
+      assert_equal 0, ArrayEventStream.new([]).size
+    end
+
+    test 'slice' do
+      sliced = @es.slice(1,1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
+    test 'each' do
+      i = 0
+      @es.each { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_msgpack_stream' do
+      i = 0
+      stream = @es.to_msgpack_stream
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+  end
+
+  class MultiEventStreamTest < ::Test::Unit::TestCase
+    include Fluent
+    include DeepCopyAssertion
+
+    def setup
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
+      @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
+      @es = MultiEventStream.new
+      @times.zip(@records).each { |_time, record|
+        @es.add(_time, record)
+      }
+    end
+
+    test 'repeatable?' do
+      assert_true @es.repeatable?
+    end
+
+    test 'dup' do
+      dupped = @es.dup
+      assert_kind_of MultiEventStream, dupped
+      assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+    end
+
+    test 'empty?' do
+      assert_not_empty @es
+      assert_true MultiEventStream.new.empty?
+    end
+
+    test 'size' do
+      assert_equal 2, @es.size
+      assert_equal 0, MultiEventStream.new.size
+    end
+
+    test 'slice' do
+      sliced = @es.slice(1,1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
+    test 'each' do
+      i = 0
+      @es.each { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_msgpack_stream' do
+      i = 0
+      stream = @es.to_msgpack_stream
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+  end
+
+  class MessagePackEventStreamTest < ::Test::Unit::TestCase
+    include Fluent
+    include DeepCopyAssertion
+
+    def setup
+      pk = Fluent::Engine.msgpack_factory.packer
+      time = Engine.now
+      @times = [Fluent::EventTime.new(time.sec), Fluent::EventTime.new(time.sec + 1)]
+      @records = [{'k' => 'v1', 'n' => 1}, {'k' => 'v2', 'n' => 2}]
+      @times.zip(@records).each { |_time, record|
+        pk.write([_time, record])
+      }
+      @es = MessagePackEventStream.new(pk.to_s)
+    end
+
+    test 'dup' do
+      dupped = @es.dup
+      assert_kind_of MessagePackEventStream, dupped
+      assert_not_equal @es.object_id, dupped.object_id
+      assert_duplicated_records @es, dupped
+
+      # After iteration of events (done in assert_duplicated_records),
+      # duplicated event stream still has unpacked objects and correct size
+      dupped = @es.dup
+      assert_equal 2, dupped.instance_eval{ @size }
+    end
+
+    test 'empty?' do
+      assert_false @es.empty?
+      assert_true MessagePackEventStream.new('', 0).empty?
+    end
+
+    test 'size' do
+      assert_equal 2, @es.size
+      assert_equal 0, MessagePackEventStream.new('').size
+    end
+
+    test 'repeatable?' do
+      assert_true @es.repeatable?
+    end
+
+    test 'slice' do
+      sliced = @es.slice(1,1)
+      assert_kind_of EventStream, sliced
+      assert_equal 1, sliced.size
+
+      sliced.each do |time, record|
+        assert_equal @times[1], time
+        assert_equal 'v2', record['k']
+        assert_equal 2, record['n']
+      end
+
+      sliced = @es.slice(0,2)
+      assert_kind_of EventStream, sliced
+      assert_equal 2, sliced.size
+
+      counter = 0
+      sliced.each do |time, record|
+        assert_equal @times[counter], time
+        assert_equal @records[counter]['k'], record['k']
+        assert_equal @records[counter]['n'], record['n']
+        counter += 1
+      end
+    end
+
+    test 'each' do
+      i = 0
+      @es.each { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+
+    test 'to_msgpack_stream' do
+      i = 0
+      stream = @es.to_msgpack_stream
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(stream) { |time, record|
+        assert_equal @times[i], time
+        assert_equal @records[i], record
+        i += 1
+      }
+    end
+  end
+end