fluentd 0.12.0.pre.2 → 0.12.0.pre.3

data/test/plugin/test_out_exec.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/test'
 require 'fileutils'
 

data/test/plugin/test_out_exec_filter.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/test'
 require 'fileutils'
 

data/test/plugin/test_out_file.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/test'
 require 'fileutils'
 require 'time'
@@ -126,7 +126,7 @@ class FileOutputTest < Test::Unit::TestCase
 
     # FileOutput#write returns path
     path = d.run
-    expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
+    expect_path = "#{TMP_DIR}/out_file_test.20110102_0.log.gz"
     assert_equal expect_path, path
 
     check_gzipped_result(path, %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
@@ -178,13 +178,13 @@ class FileOutputTest < Test::Unit::TestCase
 
     # FileOutput#write returns path
     path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test._0.log.gz", path
+    assert_equal "#{TMP_DIR}/out_file_test.20110102_0.log.gz", path
     check_gzipped_result(path, formatted_lines)
     path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test._1.log.gz", path
+    assert_equal "#{TMP_DIR}/out_file_test.20110102_1.log.gz", path
     check_gzipped_result(path, formatted_lines)
     path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test._2.log.gz", path
+    assert_equal "#{TMP_DIR}/out_file_test.20110102_2.log.gz", path
     check_gzipped_result(path, formatted_lines)
   end
 
@@ -203,13 +203,13 @@ class FileOutputTest < Test::Unit::TestCase
 
     # FileOutput#write returns path
     path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test..log.gz", path
+    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
     check_gzipped_result(path, formatted_lines)
     path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test..log.gz", path
+    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
     check_gzipped_result(path, formatted_lines * 2)
     path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test..log.gz", path
+    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
     check_gzipped_result(path, formatted_lines * 3)
   end
 
@@ -241,5 +241,58 @@ class FileOutputTest < Test::Unit::TestCase
       FileUtils.rm_rf(symlink_path)
     end
   end
+
+  sub_test_case 'path' do
+    test 'normal' do
+      d = create_driver(%[
+        path #{TMP_DIR}/out_file_test
+        time_slice_format %Y-%m-%d-%H
+        utc true
+      ])
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      d.emit({"a"=>1}, time)
+      # FileOutput#write returns path
+      path = d.run
+      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.log", path
+    end
+
+    test 'normal with append' do
+      d = create_driver(%[
+        path #{TMP_DIR}/out_file_test
+        time_slice_format %Y-%m-%d-%H
+        utc true
+        append true
+      ])
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      d.emit({"a"=>1}, time)
+      path = d.run
+      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.log", path
+    end
+
+    test '*' do
+      d = create_driver(%[
+        path #{TMP_DIR}/out_file_test.*.txt
+        time_slice_format %Y-%m-%d-%H
+        utc true
+      ])
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      d.emit({"a"=>1}, time)
+      path = d.run
+      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.txt", path
+    end
+
+    test '* with append' do
+      d = create_driver(%[
+        path #{TMP_DIR}/out_file_test.*.txt
+        time_slice_format %Y-%m-%d-%H
+        utc true
+        append true
+      ])
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      d.emit({"a"=>1}, time)
+      path = d.run
+      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.txt", path
+    end
+  end
 end
 

data/test/plugin/test_out_forward.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/test'
 
 class ForwardOutputTest < Test::Unit::TestCase

data/test/plugin/test_out_roundrobin.rb

@@ -1,7 +1,18 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/test'
 
 class RoundRobinOutputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
+      require 'fluent/plugin/out_test'
+    end
+
+    def shutdown
+      $LOAD_PATH.shift
+    end
+  end
+
   def setup
     Fluent::Test.setup
   end

data/test/plugin/test_out_stdout.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/test'
 
 class StdoutOutputTest < Test::Unit::TestCase

data/test/plugin/test_out_stream.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/test'
 
 module StreamOutputTest

data/test/scripts/fluent/plugin/formatter_known.rb

@@ -1,5 +1,8 @@
 module Fluent
-  TextFormatter.register_template('known', Proc.new { |tag, time, record|
+  TextFormatter.register_template('known_old', Proc.new { |tag, time, record|
+      "#{tag}:#{time}:#{record.size}"
+    })
+  Plugin.register_formatter('known', Proc.new { |tag, time, record|
       "#{tag}:#{time}:#{record.size}"
     })
 end

data/test/scripts/fluent/plugin/parser_known.rb

@@ -1,3 +1,4 @@
 module Fluent
-  TextParser.register_template('known', /^(?<message>.*)$/)
+  TextParser.register_template('known_old', /^(?<message>.*)$/)
+  Plugin.register_parser('known', /^(?<message>.*)$/)
 end

data/test/test_buffer.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'fluent/test'
 require 'fluent/buffer'
 

data/test/test_config.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'fluent/config'
 require 'fluent/config/parser'
 require 'fluent/supervisor'

data/test/test_configdsl.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'fluent/config/dsl'
 require 'fluent/test'
 

data/test/test_event_router.rb

@@ -0,0 +1,233 @@
+require 'fluent/event_router'
+require_relative 'test_plugin_classes'
+
+class EventRouterTest < ::Test::Unit::TestCase
+  include Fluent
+  include FluentTest
+
+  teardown do
+    @output = nil
+    @filter = nil
+    @error_output = nil
+    @emit_handler = nil
+    @default_collector = nil
+  end
+
+  def output
+    @output ||= FluentTestOutput.new
+  end
+
+  def filter
+    @filter ||= FluentTestFilter.new
+  end
+
+  def error_output
+    @error_output ||= FluentTestErrorOutput.new
+  end
+
+  def emit_handler
+    @emit_handler ||= TestEmitErrorHandler.new
+  end
+
+  def default_collector
+    @default_collector ||= FluentTestOutput.new
+  end
+
+  def event(record, time = Engine.now)
+    OneEventStream.new(time, record)
+  end
+
+  DEFAULT_EVENT_NUM = 5
+
+  def events(num = DEFAULT_EVENT_NUM)
+    es = MultiEventStream.new
+    num.times { |i|
+      es.add(Engine.now, 'key' => "value#{i}")
+    }
+    es
+  end
+
+  sub_test_case EventRouter::MatchCache do
+    setup do
+      @match_cache = EventRouter::MatchCache.new
+    end
+
+    test "call block when non-cached key" do
+      assert_raise(RuntimeError.new('Test!')) {
+        @match_cache.get('test') { raise 'Test!' }
+      }
+    end
+
+    test "don't call block when cached key" do
+      @match_cache.get('test') { "I'm cached" }
+      assert_nothing_raised {
+        @match_cache.get('test') { raise 'Test!' }
+      }
+      assert_equal "I'm cached", @match_cache.get('test') { raise 'Test!' }
+    end
+
+    test "call block when keys are expired" do
+      cache_size = EventRouter::MatchCache::MATCH_CACHE_SIZE
+      cache_size.times { |i|
+        @match_cache.get("test#{i}") { "I'm cached #{i}" }
+      }
+      assert_nothing_raised {
+        cache_size.times { |i|
+          @match_cache.get("test#{i}") { raise "Why called?" }
+        }
+      }
+      # expire old keys
+      cache_size.times { |i|
+        @match_cache.get("new_test#{i}") { "I'm young #{i}" }
+      }
+      num_called = 0
+      cache_size.times { |i|
+        @match_cache.get("test#{i}") { num_called += 1 }
+      }
+      assert_equal cache_size, num_called
+    end
+  end
+
+  sub_test_case EventRouter::Pipeline do
+    setup do
+      @pipeline = EventRouter::Pipeline.new
+      @es = event('key' => 'value')
+    end
+
+    test 'set one output' do
+      @pipeline.set_output(output)
+      @pipeline.emit('test', @es, nil)
+      assert_equal 1, output.events.size
+      assert_equal 'value', output.events['test'].first['key']
+    end
+
+    sub_test_case 'with filter' do
+      setup do
+        @pipeline.set_output(output)
+      end
+
+      test 'set one filer' do
+        @pipeline.add_filter(filter)
+        @pipeline.emit('test', @es, nil)
+        assert_equal 1, output.events.size
+        assert_equal 'value', output.events['test'].first['key']
+        assert_equal 0, output.events['test'].first['__test__']
+      end
+
+      test 'set one filer with multi events' do
+        @pipeline.add_filter(filter)
+        @pipeline.emit('test', events, nil)
+        assert_equal 1, output.events.size
+        assert_equal 5, output.events['test'].size
+        DEFAULT_EVENT_NUM.times { |i|
+          assert_equal "value#{i}", output.events['test'][i]['key']
+          assert_equal i, output.events['test'][i]['__test__']
+        }
+      end
+    end
+  end
+
+  sub_test_case EventRouter do
+    teardown do
+      @event_router = nil
+    end
+
+    def event_router
+      @event_router ||= EventRouter.new(default_collector, emit_handler)
+    end
+
+    sub_test_case 'default collector' do
+      test 'call default collector when no output' do
+        assert_rr do
+          mock(default_collector).emit('test', is_a(OneEventStream), NullOutputChain.instance)
+          event_router.emit('test', Engine.now, 'k' => 'v')
+        end
+      end
+
+      test "call default collector when only filter" do
+        event_router.add_rule('test', filter)
+        assert_rr do
+          # After apply Filter, EventStream becomes MultiEventStream by default
+          mock(default_collector).emit('test', is_a(MultiEventStream), NullOutputChain.instance)
+          event_router.emit('test', Engine.now, 'k' => 'v')
+        end
+        assert_equal 1, filter.num
+      end
+
+      test "call default collector when no matched with output" do
+        event_router.add_rule('test', output)
+        assert_rr do
+          mock(default_collector).emit('dummy', is_a(OneEventStream), NullOutputChain.instance)
+          event_router.emit('dummy', Engine.now, 'k' => 'v')
+        end
+      end
+
+      test "don't call default collector when tag matched" do
+        event_router.add_rule('test', output)
+        assert_rr do
+          dont_allow(default_collector).emit('test', is_a(OneEventStream), NullOutputChain.instance)
+          event_router.emit('test', Engine.now, 'k' => 'v')
+        end
+        # check emit handler doesn't catch rr error
+        assert_empty emit_handler.events
+      end
+    end
+
+    sub_test_case 'filter' do
+      test 'filter should be called when tag matched' do
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          mock(filter).filter_stream('test', is_a(OneEventStream)) { events }
+          event_router.emit('test', Engine.now, 'k' => 'v')
+        end
+      end
+
+      test 'filter should not be called when tag mismatched' do
+        event_router.add_rule('test', filter)
+
+        assert_rr do
+          dont_allow(filter).filter_stream('test', is_a(OneEventStream)) { events }
+          event_router.emit('foo', Engine.now, 'k' => 'v')
+        end
+      end
+
+      test 'filter changes records' do
+        event_router.add_rule('test', filter)
+        event_router.add_rule('test', output)
+        event_router.emit('test', Engine.now, 'k' => 'v')
+
+        assert_equal 1, filter.num
+        assert_equal 1, output.events['test'].size
+        assert_equal 0, output.events['test'].first['__test__']
+        assert_equal 'v', output.events['test'].first['k']
+      end
+
+      test 'filter can be chained' do
+        other_filter = FluentTestFilter.new('__hoge__')
+        event_router.add_rule('test', filter)
+        event_router.add_rule('test', other_filter)
+        event_router.add_rule('test', output)
+        event_router.emit('test', Engine.now, 'k' => 'v')
+
+        assert_equal 1, filter.num
+        assert_equal 1, other_filter.num
+        assert_equal 1, output.events['test'].size
+        assert_equal 0, output.events['test'].first['__test__']
+        assert_equal 0, output.events['test'].first['__hoge__']
+        assert_equal 'v', output.events['test'].first['k']
+      end
+    end
+
+    sub_test_case 'emit_error_handler' do
+      test 'call handle_emits_error when emit failed' do
+        event_router.add_rule('test', error_output)
+
+        assert_rr do
+          mock(emit_handler).handle_emits_error('test', is_a(OneEventStream), is_a(RuntimeError))
+          event_router.emit('test', Engine.now, 'k' => 'v')
+        end
+      end
+    end
+  end
+end

data/test/test_formatter.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative 'helper'
 require 'fluent/test'
 require 'fluent/formatter'
 
@@ -120,6 +120,49 @@ module FormatterTest
     end
   end
 
+  class MessagePackFormatterTest < ::Test::Unit::TestCase
+    include FormatterTest
+
+    def setup
+      @formatter = TextFormatter::MessagePackFormatter.new
+      @time = Engine.now
+    end
+
+    def test_format
+      @formatter.configure({})
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal(record.to_msgpack, formatted)
+    end
+
+    def test_format_with_include_tag
+      @formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo')
+      formatted = @formatter.format(tag, @time, record.dup)
+
+      r = record
+      r['foo'] = tag
+      assert_equal(r.to_msgpack, formatted)
+    end
+
+    def test_format_with_include_time
+      @formatter.configure('include_time_key' => 'true', 'localtime' => '')
+      formatted = @formatter.format(tag, @time, record.dup)
+
+      r = record
+      r['time'] = time2str(@time, true)
+      assert_equal(r.to_msgpack, formatted)
+    end
+
+    def test_format_with_include_time_as_number
+      @formatter.configure('include_time_key' => 'true', 'time_as_epoch' => 'true', 'time_key' => 'epoch')
+      formatted = @formatter.format(tag, @time, record.dup)
+
+      r = record
+      r['epoch'] = @time
+      assert_equal(r.to_msgpack, formatted)
+    end
+  end
+
   class LabeledTSVFormatterTest < ::Test::Unit::TestCase
     include FormatterTest
 
@@ -174,6 +217,118 @@ module FormatterTest
     end
   end
 
+  class CsvFormatterTest < ::Test::Unit::TestCase
+    include FormatterTest
+
+    def setup
+      @formatter = TextFormatter::CsvFormatter.new
+      @time = Engine.now
+    end
+    
+    def test_config_params
+      assert_equal ',', @formatter.delimiter
+      assert_equal true, @formatter.force_quotes
+      assert_equal [], @formatter.fields
+    end
+
+    data(
+      'tab_char' => ["\t", '\t'],
+      'tab_string' => ["\t", 'TAB'],
+      'pipe' => ['|', '|'])
+    def test_config_params_with_customized_delimiters(data)
+      expected, target = data
+      @formatter.configure('delimiter' => target)
+      assert_equal expected, @formatter.delimiter
+    end
+
+    def test_format
+      @formatter.configure('fields' => 'message,message2')
+      formatted = @formatter.format(tag, @time, {
+        'message' => 'awesome',
+        'message2' => 'awesome2'
+      })
+      assert_equal("\"awesome\",\"awesome2\"\n", formatted)
+    end
+
+    def test_format_with_tag
+      @formatter.configure(
+        'fields' => 'tag,message,message2',
+        'include_tag_key' => 'true'
+      )
+      formatted = @formatter.format(tag, @time, {
+        'message' => 'awesome',
+        'message2' => 'awesome2'
+      })
+      assert_equal("\"tag\",\"awesome\",\"awesome2\"\n", formatted)
+    end
+
+    def test_format_with_time
+      @formatter.configure(
+        'fields' => 'time,message,message2',
+        'include_time_key' => 'true',
+        'time_format' => '%Y'
+      )
+      formatted = @formatter.format(tag, @time, {
+        'message' => 'awesome',
+        'message2' => 'awesome2'
+      })
+      assert_equal("\"#{Time.now.year}\",\"awesome\",\"awesome2\"\n",
+                   formatted)
+    end
+
+    def test_format_with_customized_delimiters
+      @formatter.configure(
+        'fields' => 'message,message2',
+        'delimiter' => '\t'
+      )
+      formatted = @formatter.format(tag, @time, {
+        'message' => 'awesome',
+        'message2' => 'awesome2'
+      })
+      assert_equal("\"awesome\"\t\"awesome2\"\n", formatted)
+    end
+
+    def test_format_with_non_quote
+      @formatter.configure(
+        'fields' => 'message,message2',
+        'force_quotes' => 'false'
+      )
+      formatted = @formatter.format(tag, @time, {
+        'message' => 'awesome',
+        'message2' => 'awesome2'
+      })
+      assert_equal("awesome,awesome2\n", formatted)
+    end
+
+    data(
+      'nil' => {
+        'message' => 'awesome',
+        'message2' => nil,
+        'message3' => 'awesome3'
+      },
+      'blank' => {
+        'message' => 'awesome',
+        'message2' => '',
+        'message3' => 'awesome3'
+      })
+    def test_format_with_empty_fields(data)
+      @formatter.configure(
+        'fields' => 'message,message2,message3'
+      )
+      formatted = @formatter.format(tag, @time, data)
+      assert_equal("\"awesome\",\"\",\"awesome3\"\n", formatted)
+    end
+
+    data(
+      'normally' => 'one,two,three',
+      'white_space' => 'one , two , three',
+      'blank' => 'one,,two,three')
+    def test_config_params_with_fields(data)
+      @formatter.configure('fields' => data)
+      assert_equal %w(one two three), @formatter.fields
+    end
+  end
+
   class SingleValueFormatterTest < ::Test::Unit::TestCase
     include FormatterTest
 
@@ -216,11 +371,13 @@ module FormatterTest
       end
     end
 
-    def test_find_formatter
+    data('register_formatter' => 'known', 'register_template' => 'known_old')
+    def test_find_formatter(data)
       $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), 'scripts'))
       assert_nothing_raised ConfigError do
-        TextFormatter::TEMPLATE_REGISTRY.lookup('known')
+        TextFormatter::TEMPLATE_REGISTRY.lookup(data)
       end
+      $LOAD_PATH.shift
     end
   end