fluentd 0.10.38 → 0.10.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7dd0061dacd1f7c9ea0cbd4852cfa656a913f71b
-  data.tar.gz: f73b52509cdc043da1a07ed0f46bcd0e36ddf820
+  metadata.gz: 14f93577d21ab248d08466b142442d503ba9ad55
+  data.tar.gz: 9036d89bf55c57a454120dc766c3da6fe445fbf8
 SHA512:
-  metadata.gz: cf4231745ae0f5afe243a2e41901445575b63e0dd7fc91a532faa9416a73a3681a2e9282b06e429d2c35ad4b2a8c6782938ceb2ff18a5efd2bdc0e31d8f204d7
-  data.tar.gz: 02f39e0e614f08bbb860fad4849cc0e6c92aaea20b0e0151bb7ba2c09515374181570359cc0e931a391a5aab52496cab8834b9a3edef1283601a65159fb5f5e8
+  metadata.gz: 5c4ae2365e4e3a59cc22e7089203ea57b3cf7bc89d759915e4f02b795312903116484dd65f43a7e07ed9e98a1da13f5e1aa6112a0448d03706e4171cd3dad63a
+  data.tar.gz: 806d690bccc9674f2bd45d693eb523a07e06d0aa4e2dc63e42396421d34ce8f27977bba119b0e5470d8844ee62c140cd1cf8c4bff1b8ce2aa16bd8d40e7fb419

data/.travis.yml

@@ -11,3 +11,7 @@ branches:
     - v11
 
 script: bundle exec rake
+
+matrix:
+  allow_failures:
+    - rvm: rbx-19mode

data/ChangeLog

@@ -1,3 +1,13 @@
+Release 0.10.39 - 2013/09/18
+
+* out_file: Improve symlink handling with buf_file
+* out_copy: Add deep_copy parameter for actual record copy
+* in_tail/parser: Add none format for supporting non-parse tailing like fluent-agent-lite
+* in_tail/parser: Improve performance using parsed time caching
+* Engine: Fix signal related exception in trap context at Ruby 2.0
+* buffer: Fix race condition when remove flushed chunk
+* add --suppress-config-dump option to disable dump configuration at start
+
 Release 0.10.38 - 2013/08/29
 
 * Fix require related problem in running test

data/Rakefile

@@ -1,6 +1,7 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
 
+require 'fileutils'
 require 'rake/testtask'
 require 'rake/clean'
 
@@ -13,4 +14,10 @@ Rake::TestTask.new(:base_test) do |t|
   #t.warning = true
 end
 
+task :parallel_test do
+  FileUtils.rm_rf('./test/tmp')
+  sh("parallel_test ./test/*.rb ./test/plugin/*.rb")
+  FileUtils.rm_rf('./test/tmp')
+end
+
 task :default => [:test, :build]

data/fluentd.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency(%q<http_parser.rb>, ["~> 0.5.1"])
 
   gem.add_development_dependency(%q<rake>, [">= 0.9.2"])
+  gem.add_development_dependency(%q<parallel_tests>, [">= 0.15.3"])
   gem.add_development_dependency(%q<rr>, [">= 1.0.0"])
   gem.add_development_dependency(%q<timecop>, [">= 0.3.0"])
 end

data/lib/fluent/buffer.rb

@@ -272,13 +272,17 @@ module Fluent
           write_chunk(chunk, out)
         end
 
-        @queue.delete_if {|c|
-          c.object_id == chunk.object_id
-        }
+        empty = false
+        @queue.synchronize do
+          @queue.delete_if {|c|
+            c.object_id == chunk.object_id
+          }
+          empty = @queue.empty?
+        end
 
         chunk.purge
 
-        return !@queue.empty?
+        return !empty
       ensure
         chunk.mon_exit
       end

data/lib/fluent/command/fluentd.rb

@@ -98,6 +98,10 @@ op.on('-q', '--quiet', "decrease verbose level (-q: warn, -qq: error)", TrueClas
   end
 }
 
+op.on('--suppress-config-dump', "suppress config dumping when fluentd starts", TrueClass) {|b|
+  opts[:suppress_config_dump] = b
+}
+
 (class<<self;self;end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s

data/lib/fluent/config_dsl.rb

@@ -3,7 +3,7 @@ require 'fluent/config'
 module Fluent
   module Config
     module DSL
-      module DSLParser
+      module Parser
         def self.read(path)
           path = File.expand_path(path)
           data = File.read(path)
@@ -11,62 +11,75 @@ module Fluent
         end
 
         def self.parse(source, source_path="config.rb")
-          DSLElement.new('ROOT', nil).__eval(source, source_path).__to_config_element
+          Proxy.new('ROOT', nil).eval(source, source_path).to_config_element
         end
       end
 
-      class DSLElement < BasicObject
+      class Proxy
         def initialize(name, arg)
-          @name = name
-          @arg = arg || ''
-          @attrs = {}
-          @elements = []
+          @element = Element.new(name, arg, self)
+        end
+
+        def element
+          @element
         end
 
-        def __eval(source, source_path)
-          instance_eval(source, source_path)
+        def eval(source, source_path)
+          @element.instance_eval(source, source_path)
           self
         end
 
-        def __to_config_element
-          Config::Element.new(@name, @arg, @attrs, @elements)
+        def to_config_element
+          @element.instance_eval do
+            Config::Element.new(@name, @arg, @attrs, @elements)
+          end
+        end
+
+        def add_element(name, arg, block)
+          ::Kernel.raise ::ArgumentError, "#{name} block must be specified" if block.nil?
+
+          proxy = self.class.new(name.to_s, arg)
+          proxy.element.instance_exec(&block)
+
+          @element.instance_eval do
+            @elements.push(proxy.to_config_element)
+          end
+
+          self
+        end
+      end
+
+      class Element < BasicObject
+        def initialize(name, arg, proxy)
+          @name     = name
+          @arg      = arg || ''
+          @attrs    = {}
+          @elements = []
+          @proxy    = proxy
         end
 
         def method_missing(name, *args, &block)
-          raise ArgumentError, "Configuration DSL Syntax Error: only one argument allowed" if args.size > 1
+          ::Kernel.raise ::ArgumentError, "Configuration DSL Syntax Error: only one argument allowed" if args.size > 1
           value = args.first
+
           if block
-            element = DSLElement.new(name.to_s, value)
-            element.instance_exec(&block)
-            @elements.push(element.__to_config_element)
+            proxy = Proxy.new(name.to_s, value)
+            proxy.element.instance_exec(&block)
+            @elements.push(proxy.to_config_element)
           else
-            # @attrs[name.to_s] = value.is_a?(::Symbol) ? value.to_s : value
             @attrs[name.to_s] = value.to_s
           end
+
           self
         end
 
         def source(&block)
-          __element('source', nil, block)
+          @proxy.add_element('source', nil, block)
         end
 
         def match(*args, &block)
-          __need_arg('match',  args); __element('match',  args.first, block)
-        end
-
-        private
-
-        def __element(name, arg, block)
-          raise ArgumentError, "#{name} block must be specified" if block.nil?
-          element = DSLElement.new(name.to_s, arg)
-          element.instance_exec(&block)
-          @elements.push(element.__to_config_element)
-          self
-        end
-
-        def __need_arg(name, args)
-          raise ArgumentError, "#{name} block requires arguments for match pattern" if args.nil? || args.size != 1
-          true
+          ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for match pattern" if args.nil? || args.size != 1
+          @proxy.add_element('match', args.first, block)
         end
       end
     end

data/lib/fluent/engine.rb

@@ -31,6 +31,8 @@ module Fluent
 
       @suppress_emit_error_log_interval = 0
       @next_emit_error_log_time = nil
+
+      @suppress_config_dump = false
     end
 
     MATCH_CACHE_SIZE = 1024
@@ -54,6 +56,10 @@ module Fluent
       @next_emit_error_log_time = Time.now.to_i
     end
 
+    def suppress_config_dump=(flag)
+      @suppress_config_dump = flag
+    end
+
     def read_config(path)
       $log.info "reading config file", :path=>path
       File.open(path) {|io|
@@ -63,7 +69,7 @@ module Fluent
 
     def parse_config(io, fname, basepath=Dir.pwd)
       conf = if fname =~ /\.rb$/
-               Config::DSL::DSLParser.parse(io, fname)
+               Config::DSL::Parser.parse(io, File.join(basepath, fname))
              else
                Config.parse(io, fname, basepath)
              end
@@ -74,7 +80,9 @@ module Fluent
     end
 
     def configure(conf)
-      $log.info "using configuration file: #{conf.to_s.rstrip}"
+      unless @suppress_config_dump
+        $log.info "using configuration file: #{conf.to_s.rstrip}"
+      end
 
       conf.elements.select {|e|
         e.name == 'source'

data/lib/fluent/event.rb

@@ -42,6 +42,10 @@ module Fluent
       @record = record
     end
 
+    def dup
+      OneEventStream.new(@time, @record.dup)
+    end
+
     def repeatable?
       true
     end
@@ -58,6 +62,11 @@ module Fluent
       @entries = entries
     end
 
+    def dup
+      entries = @entries.map(:dup)
+      ArrayEventStream.new(entries)
+    end
+
     def repeatable?
       true
     end
@@ -85,6 +94,14 @@ module Fluent
       @record_array = []
     end
 
+    def dup
+      es = MultiEventStream.new
+      @time_array.zip(@record_array).each { |time, record|
+        es.add(time, record.dup)
+      }
+      es
+    end
+
     def add(time, record)
       @time_array << time
       @record_array << record

data/lib/fluent/output.rb

@@ -35,6 +35,18 @@ module Fluent
     end
   end
 
+  class CopyOutputChain < OutputChain
+    def next
+      if @array.length <= @offset
+        return @chain.next
+      end
+      @offset += 1
+      es = @array.length > @offset ? @es.dup : @es
+      result = @array[@offset-1].emit(@tag, es, self)
+      result
+    end
+  end
+
   class NullOutputChain
     require 'singleton'
     include Singleton

data/lib/fluent/parser.rb

@@ -17,6 +17,40 @@
 #
 module Fluent
   class TextParser
+    class TimeParser
+      def initialize(time_format)
+        @cache1_key = nil
+        @cache1_time = nil
+        @cache2_key = nil
+        @cache2_time = nil
+        @parser =
+          if time_format
+            Proc.new { |value| Time.strptime(value, time_format) }
+          else
+            Time.method(:parse)
+          end
+      end
+
+      def parse(value)
+        unless value.is_a?(String)
+          raise ArgumentError, "Value must be string: #{value}"
+        end
+
+        if @cache1_key == value
+          return @cache1_time
+        elsif @cache2_key == value
+          return @cache2_time
+        else
+          time = @parser.call(value).to_i
+          @cache1_key = @cache2_key
+          @cache1_time = @cache2_time
+          @cache2_key = value
+          @cache2_time = time
+          return time
+        end
+      end
+    end
+
     class RegexpParser
       include Configurable
 
@@ -28,6 +62,9 @@ module Fluent
         unless conf.empty?
           configure(conf)
         end
+
+        @time_parser = TimeParser.new(@time_format)
+        @mutex = Mutex.new
       end
 
       def call(text)
@@ -44,11 +81,7 @@ module Fluent
           if value = m[name]
             case name
             when "time"
-              if @time_format
-                time = Time.strptime(value, @time_format).to_i
-              else
-                time = Time.parse(value).to_i
-              end
+              time = @mutex.synchronize { @time_parser.parse(value) }
             else
               record[name] = value
             end
@@ -67,12 +100,21 @@ module Fluent
       config_param :time_key, :string, :default => 'time'
       config_param :time_format, :string, :default => nil
 
+      def configure(conf)
+        super
+
+        unless @time_format.nil?
+          @time_parser = TimeParser.new(@time_format)
+          @mutex = Mutex.new
+        end
+      end
+
       def call(text)
         record = Yajl.load(text)
 
         if value = record.delete(@time_key)
           if @time_format
-            time = Time.strptime(value, @time_format).to_i
+            time = @mutex.synchronize { @time_parser.parse(value) }
           else
             time = value.to_i
           end
@@ -106,6 +148,9 @@ module Fluent
         if @time_format && !@time_key
           raise ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
         end
+
+        @time_parser = TimeParser.new(@time_format)
+        @mutex = Mutex.new
       end
 
       def values_map(values)
@@ -113,11 +158,7 @@ module Fluent
 
         if @time_key
           value = record.delete(@time_key)
-          if @time_format
-            time = Time.strptime(value, @time_format).to_i
-          else
-            time = Time.parse(value).to_i
-          end
+          time = @mutex.synchronize { @time_parser.parse(value) }
         else
           time = Engine.now
         end
@@ -169,11 +210,28 @@ module Fluent
       end
     end
 
+    class NoneParser
+      include Configurable
+
+      config_param :message_key, :string, :default => 'message'
+
+      def call(text)
+        record = {}
+        record[@message_key] = text
+        return Engine.now, record
+      end
+    end
+
     class ApacheParser
       include Configurable
 
       REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
 
+      def initialize
+        @time_parser = TimeParser.new("%d/%b/%Y:%H:%M:%S %z")
+        @mutex = Mutex.new
+      end
+
       def call(text)
         m = REGEXP.match(text)
         unless m
@@ -188,7 +246,7 @@ module Fluent
         user = (user == '-') ? nil : user
 
         time = m['time']
-        time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
+        time = @mutex.synchronize { @time_parser.parse(time) }
 
         method = m['method']
         path = m['path']
@@ -229,6 +287,7 @@ module Fluent
       'ltsv' => Proc.new { LabeledTSVParser.new },
       'csv' => Proc.new { CSVParser.new },
       'nginx' => Proc.new { RegexpParser.new(/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/,  {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
+      'none' => Proc.new { NoneParser.new },
     }
 
     def self.register_template(name, regexp_or_proc, time_format=nil)

data/lib/fluent/plugin/buf_file.rb

@@ -17,13 +17,14 @@
 #
 module Fluent
   class FileBufferChunk < BufferChunk
-    def initialize(key, path, unique_id, mode="a+")
+    def initialize(key, path, unique_id, mode="a+", symlink_path = nil)
       super(key)
       @path = path
       @unique_id = unique_id
       @file = File.open(@path, mode, DEFAULT_FILE_PERMISSION)
       @file.sync = true
       @size = @file.stat.size
+      FileUtils.ln_sf(@path, symlink_path) if symlink_path
     end
 
     attr_reader :unique_id
@@ -84,6 +85,8 @@ module Fluent
 
     config_param :buffer_path, :string
 
+    attr_accessor :symlink_path
+
     def configure(conf)
       super
 
@@ -119,7 +122,7 @@ module Fluent
       encoded_key = encode_key(key)
       path, tsuffix = make_path(encoded_key, "b")
       unique_id = tsuffix_to_unique_id(tsuffix)
-      FileBufferChunk.new(key, path, unique_id)
+      FileBufferChunk.new(key, path, unique_id, "a+", @symlink_path)
     end
 
     def resume

data/lib/fluent/plugin/in_tail.rb

@@ -39,11 +39,7 @@ module Fluent
         raise ConfigError, "tail: 'path' parameter is required on tail input"
       end
 
-      if @pos_file
-        @pf_file = File.open(@pos_file, File::RDWR|File::CREAT, DEFAULT_FILE_PERMISSION)
-        @pf_file.sync = true
-        @pf = PositionFile.parse(@pf_file)
-      else
+      unless @pos_file
         $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
         $log.warn "this parameter is highly recommended to save the position to resume tailing."
       end
@@ -57,6 +53,12 @@ module Fluent
     end
 
     def start
+      if @pos_file
+        @pf_file = File.open(@pos_file, File::RDWR|File::CREAT, DEFAULT_FILE_PERMISSION)
+        @pf_file.sync = true
+        @pf = PositionFile.parse(@pf_file)
+      end
+
       @loop = Coolio::Loop.new
       @tails = @paths.map {|path|
         pe = @pf ? @pf[path] : MemoryPositionEntry.new

data/lib/fluent/plugin/out_copy.rb

@@ -19,13 +19,17 @@ module Fluent
   class CopyOutput < MultiOutput
     Plugin.register_output('copy', self)
 
+    config_param :deep_copy, :bool, :default => false
+
     def initialize
+      super
       @outputs = []
     end
 
     attr_reader :outputs
 
     def configure(conf)
+      super
       conf.elements.select {|e|
         e.name == 'store'
       }.each {|e|
@@ -61,7 +65,11 @@ module Fluent
         }
         es = m
       end
-      chain = OutputChain.new(@outputs, tag, es, chain)
+      if @deep_copy
+        chain = CopyOutputChain.new(@outputs, tag, es, chain)
+      else
+        chain = OutputChain.new(@outputs, tag, es, chain)
+      end
       chain.next
     end
   end

data/lib/fluent/plugin/out_file.rb

@@ -65,6 +65,8 @@ module Fluent
       super
 
       @timef = TimeFormatter.new(@time_format, @localtime)
+
+      @buffer.symlink_path = @symlink_path if @symlink_path
     end
 
     def format(tag, time, record)
@@ -97,7 +99,6 @@ module Fluent
           chunk.write_to(f)
         }
       end
-      create_symlink(path, suffix) if @symlink_path
 
       return path  # for test
     end
@@ -105,11 +106,5 @@ module Fluent
     def secondary_init(primary)
       # don't warn even if primary.class is not FileOutput
     end
-
-    private
-
-    def create_symlink(path, suffix)
-      FileUtils.ln_sf(path, "#{@symlink_path}#{suffix}")
-    end
   end
 end

data/lib/fluent/supervisor.rb

@@ -67,6 +67,7 @@ module Fluent
       @inline_config = opt[:inline_config]
       @suppress_interval = opt[:suppress_interval]
       @dry_run = opt[:dry_run]
+      @suppress_config_dump = opt[:suppress_config_dump]
 
       @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup)
       @finished = false
@@ -320,6 +321,8 @@ module Fluent
         Fluent::Engine.suppress_interval(@suppress_interval)
       end
 
+      Fluent::Engine.suppress_config_dump = @suppress_config_dump
+
       @libs.each {|lib|
         require lib
       }
@@ -363,7 +366,17 @@ module Fluent
         $log.debug "fluentd main process get SIGUSR1"
         $log.info "force flushing buffered events"
         @log.reopen!
-        Fluent::Engine.flush!
+
+        # Creating new thread due to mutex can't lock
+        # in main thread during trap context
+        Thread.new {
+          begin
+            Fluent::Engine.flush!
+            $log.debug "flushing thread: flushed"
+          rescue Exception => e
+            $log.warn "flushing thread error: #{e}"
+          end
+        }.run
       end
     end
 

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.38'
+  VERSION = '0.10.39'
 
 end

data/test/config.rb

@@ -7,7 +7,7 @@ require 'fileutils'
 class ConfigTest < Test::Unit::TestCase
   include Fluent
 
-  TMP_DIR = File.dirname(__FILE__) + "/tmp"
+  TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
 
   def prepare_config
     write_config "#{TMP_DIR}/config_test_1.conf", %[

data/test/configdsl.rb

@@ -1,4 +1,6 @@
 require 'fluent/config_dsl'
+require 'fluent/test'
+require 'helper'
 
 class ConfigDSLTest < Test::Unit::TestCase
   # TEST_CONFIG1 = %[
@@ -38,10 +40,24 @@ match('test.**') {
 
   TEST_DSL_CONFIG2 = %q[
 v = [0, 1, 2]
+]
+
+  TEST_DSL_CONFIG3 = %q[
+match
+]
+
+  TEST_DSL_CONFIG4 = %q[
+match('aa', 'bb'){
+  type :null
+}
+]
+
+  TEST_DSL_CONFIG5 = %q[
+match('aa')
 ]
 
   def test_parse
-    root = Fluent::Config::DSL::DSLParser.parse(TEST_DSL_CONFIG1)
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG1)
 
     assert_equal 0, root.keys.size
     assert_equal 2, root.elements.size
@@ -69,9 +85,23 @@ v = [0, 1, 2]
   end
 
   def test_parse2
-    root = Fluent::Config::DSL::DSLParser.parse(TEST_DSL_CONFIG2)
+    root = Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG2)
 
     assert_equal 0, root.keys.size
     assert_equal 0, root.elements.size
   end
+
+  def test_config_error
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG3)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG4)
+    }
+
+    assert_raise(ArgumentError) {
+      Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG5)
+    }
+  end
 end

data/test/helper.rb

@@ -8,8 +8,11 @@ unless defined?(Test::Unit::AssertionFailedError)
   end
 end
 
-class Test::Unit::TestCase
-  include RR::Adapters::TestUnit
+def unused_port
+  s = TCPServer.open(0)
+  port = s.addr[1]
+  s.close
+  port
 end
 
 def ipv6_enabled?

data/test/parser.rb

@@ -9,7 +9,35 @@ module ParserTest
     if format
       Time.strptime(str_time, format).to_i
     else
-      Time.parse(str_time)
+      Time.parse(str_time).to_i
+    end
+  end
+
+  class TimeParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def test_call_with_parse
+      parser = TextParser::TimeParser.new(nil)
+
+      time = str2time('2013-09-18 12:00:00 +0900')
+      assert_equal(time, parser.parse('2013-09-18 12:00:00 +0900'))
+    end
+
+    def test_call_with_strptime
+      parser = TextParser::TimeParser.new('%d/%b/%Y:%H:%M:%S %z')
+
+      time = str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z')
+      assert_equal(time, parser.parse('28/Feb/2013:12:00:00 +0900'))
+    end
+
+    def test_call_with_invalid_argument
+      parser = TextParser::TimeParser.new(nil)
+
+      [[], {}, nil, true, 10000].each { |v|
+        assert_raise ArgumentError do
+          parser.parse(v)
+        end
+      }
     end
   end
 
@@ -151,6 +179,7 @@ module ParserTest
 
     def test_call
       parser = TextParser::LabeledTSVParser.new
+      parser.configure({})
       time, record = parser.call("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111")
 
       assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
@@ -190,4 +219,32 @@ module ParserTest
       }, record)
     end
   end
+
+  class NoneParserTest < ::Test::Unit::TestCase
+    include ParserTest
+
+    def test_config_params
+      parser = TextParser::NoneParser.new
+      assert_equal "message", parser.message_key
+
+      parser.configure('message_key' => 'foobar')
+      assert_equal "foobar", parser.message_key
+    end
+
+    def test_call
+      parser = TextParser::TEMPLATE_FACTORIES['none'].call
+      time, record = parser.call('log message!')
+
+      assert_equal({'message' => 'log message!'}, record)
+    end
+
+    def test_call_with_message_key
+      parser = TextParser::NoneParser.new
+      parser.configure('message_key' => 'foobar')
+      time, record = parser.call('log message!')
+
+      assert_equal({'foobar' => 'log message!'}, record)
+    end
+  end
+
 end

data/test/plugin/in_forward.rb

@@ -1,12 +1,14 @@
 require 'fluent/test'
+require 'helper'
 
 class ForwardInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
+  PORT = unused_port
   CONFIG = %[
-    port 13998
+    port #{PORT}
     bind 127.0.0.1
   ]
 
@@ -16,12 +18,12 @@ class ForwardInputTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver
-    assert_equal 13998, d.instance.port
+    assert_equal PORT, d.instance.port
     assert_equal '127.0.0.1', d.instance.bind
   end
 
   def connect
-    TCPSocket.new('127.0.0.1', 13998)
+    TCPSocket.new('127.0.0.1', PORT)
   end
 
   def test_time
@@ -120,4 +122,3 @@ class ForwardInputTest < Test::Unit::TestCase
 
   # TODO heartbeat
 end
-

data/test/plugin/in_http.rb

@@ -1,4 +1,5 @@
 require 'fluent/test'
+require 'helper'
 require 'net/http'
 
 class HttpInputTest < Test::Unit::TestCase
@@ -6,8 +7,9 @@ class HttpInputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
+  PORT = unused_port
   CONFIG = %[
-    port 9911
+    port #{PORT}
     bind 127.0.0.1
     body_size_limit 10m
     keepalive_timeout 5
@@ -19,7 +21,7 @@ class HttpInputTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver
-    assert_equal 9911, d.instance.port
+    assert_equal PORT, d.instance.port
     assert_equal '127.0.0.1', d.instance.bind
     assert_equal 10*1024*1024, d.instance.body_size_limit
     assert_equal 5, d.instance.keepalive_timeout
@@ -68,7 +70,7 @@ class HttpInputTest < Test::Unit::TestCase
 
     d.run do
       d.expected_emits.each {|tag,time,record|
-        http = Net::HTTP.new("127.0.0.1", 9911)
+        http = Net::HTTP.new("127.0.0.1", PORT)
         req = Net::HTTP::Post.new("/#{tag}?time=#{time.to_s}", {"content-type"=>"application/json; charset=utf-8"})
         req.body = record.to_json
         res = http.request(req)
@@ -94,7 +96,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def post(path, params)
-    http = Net::HTTP.new("127.0.0.1", 9911)
+    http = Net::HTTP.new("127.0.0.1", PORT)
     req = Net::HTTP::Post.new(path, {})
     req.set_form_data(params)
     http.request(req)

data/test/plugin/in_stream.rb

@@ -1,4 +1,5 @@
 require 'fluent/test'
+require 'helper'
 
 module StreamInputTest
   def setup
@@ -107,8 +108,9 @@ end
 class TcpInputTest < Test::Unit::TestCase
   include StreamInputTest
 
+  PORT = unused_port
   CONFIG = %[
-    port 13998
+    port #{PORT}
     bind 127.0.0.1
   ]
 
@@ -118,20 +120,19 @@ class TcpInputTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver
-    assert_equal 13998, d.instance.port
+    assert_equal PORT, d.instance.port
     assert_equal '127.0.0.1', d.instance.bind
   end
 
   def connect
-    TCPSocket.new('127.0.0.1', 13998)
+    TCPSocket.new('127.0.0.1', PORT)
   end
 end
 
 class UnixInputTest < Test::Unit::TestCase
   include StreamInputTest
 
-  TMP_DIR = File.dirname(__FILE__) + "/../tmp"
-
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/in_unix#{ENV['TEST_ENV_NUMBER']}"
   CONFIG = %[
     path #{TMP_DIR}/unix
     backlog 1000
@@ -151,4 +152,3 @@ class UnixInputTest < Test::Unit::TestCase
     UNIXSocket.new("#{TMP_DIR}/unix")
   end
 end
-

data/test/plugin/in_syslog.rb

@@ -7,14 +7,15 @@ class SyslogInputTest < Test::Unit::TestCase
     require 'fluent/plugin/socket_util'
   end
 
+  PORT = unused_port
   CONFIG = %[
-    port 9911
+    port #{PORT}
     bind 127.0.0.1
     tag syslog
   ]
 
   IPv6_CONFIG = %[
-    port 9911
+    port #{PORT}
     bind ::1
     tag syslog
   ]
@@ -29,7 +30,7 @@ class SyslogInputTest < Test::Unit::TestCase
 
     configs.each_pair { |k, v|
       d = create_driver(v)
-      assert_equal 9911, d.instance.port
+      assert_equal PORT, d.instance.port
       assert_equal k, d.instance.bind
     }
   end
@@ -48,7 +49,7 @@ class SyslogInputTest < Test::Unit::TestCase
 
       d.run do
         u = Fluent::SocketUtil.create_udp_socket(k)
-        u.connect(k, 9911)
+        u.connect(k, PORT)
         tests.each {|test|
           u.send(test['msg'], 0)
         }
@@ -72,7 +73,7 @@ class SyslogInputTest < Test::Unit::TestCase
 
     d.run do
       u = UDPSocket.new
-      u.connect('127.0.0.1', 9911)
+      u.connect('127.0.0.1', PORT)
       tests.each {|test|
         u.send(test['msg'], 0)
       }

data/test/plugin/in_tail.rb

@@ -8,7 +8,7 @@ class TailInputTest < Test::Unit::TestCase
     FileUtils.mkdir_p(TMP_DIR)
   end
 
-  TMP_DIR = File.dirname(__FILE__) + "/../tmp"
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/tail#{ENV['TEST_ENV_NUMBER']}"
 
   CONFIG = %[
     path #{TMP_DIR}/tail.txt

data/test/plugin/out_copy.rb

@@ -88,5 +88,83 @@ class CopyOutputTest < Test::Unit::TestCase
       ], o.events
     }
   end
+
+  def create_event_test_driver(is_deep_copy = false)
+    deep_copy_config = %[
+deep_copy true
+]
+
+    output1 = Fluent::Plugin.new_output('test')
+    output1.configure('name' => 'output1')
+    output1.define_singleton_method(:emit) do |tag, es, chain|
+      es.each do |time, record|
+        record['foo'] = 'bar'
+        super(tag, [[time, record]], chain)
+      end
+    end
+
+    output2 = Fluent::Plugin.new_output('test')
+    output2.configure('name' => 'output2')
+    output2.define_singleton_method(:emit) do |tag, es, chain|
+      es.each do |time, record|
+        super(tag, [[time, record]], chain)
+      end
+    end
+
+    outputs = [output1, output2]
+
+    d = Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput)
+    d = d.configure(deep_copy_config) if is_deep_copy
+    d.instance.instance_eval { @outputs = outputs }
+    d
+  end
+
+  def test_one_event
+    time = Time.parse("2013-05-26 06:37:22 UTC").to_i
+
+    d = create_event_test_driver(false)
+    es = Fluent::OneEventStream.new(time, {"a" => 1})
+    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
+
+    assert_equal [
+      [[time, {"a"=>1, "foo"=>"bar"}]],
+      [[time, {"a"=>1, "foo"=>"bar"}]]
+    ], d.instance.outputs.map{ |o| o.events }
+
+    d = create_event_test_driver(true)
+    es = Fluent::OneEventStream.new(time, {"a" => 1})
+    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
+
+    assert_equal [
+      [[time, {"a"=>1, "foo"=>"bar"}]],
+      [[time, {"a"=>1}]]
+    ], d.instance.outputs.map{ |o| o.events }
+  end
+
+  def test_multi_event
+    time = Time.parse("2013-05-26 06:37:22 UTC").to_i
+
+    d = create_event_test_driver(false)
+    es = Fluent::MultiEventStream.new
+    es.add(time, {"a" => 1})
+    es.add(time, {"b" => 2})
+    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
+
+    assert_equal [
+      [[time, {"a"=>1, "foo"=>"bar"}], [time, {"b"=>2, "foo"=>"bar"}]],
+      [[time, {"a"=>1, "foo"=>"bar"}], [time, {"b"=>2, "foo"=>"bar"}]]
+    ], d.instance.outputs.map{ |o| o.events }
+
+    d = create_event_test_driver(true)
+    es = Fluent::MultiEventStream.new
+    es.add(time, {"a" => 1})
+    es.add(time, {"b" => 2})
+    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
+
+    assert_equal [
+      [[time, {"a"=>1, "foo"=>"bar"}], [time, {"b"=>2, "foo"=>"bar"}]],
+      [[time, {"a"=>1}], [time, {"b"=>2}]]
+    ], d.instance.outputs.map{ |o| o.events }
+  end
 end
 

data/test/plugin/out_exec.rb

@@ -8,7 +8,7 @@ class ExecOutputTest < Test::Unit::TestCase
     FileUtils.mkdir_p(TMP_DIR)
   end
 
-  TMP_DIR = File.dirname(__FILE__) + "/../tmp"
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/out_exec#{ENV['TEST_ENV_NUMBER']}"
 
   CONFIG = %[
     buffer_path #{TMP_DIR}/buffer

data/test/plugin/out_file.rb

@@ -9,7 +9,7 @@ class FileOutputTest < Test::Unit::TestCase
     FileUtils.mkdir_p(TMP_DIR)
   end
 
-  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp")
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file#{ENV['TEST_ENV_NUMBER']}")
   SYMLINK_PATH = File.expand_path("#{TMP_DIR}/current")
 
   CONFIG = %[
@@ -81,20 +81,32 @@ class FileOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_symlink
-    d = create_driver(CONFIG + %[
+    conf = CONFIG + %[
       symlink_path #{SYMLINK_PATH}
-    ])
-    symlink_path = "#{SYMLINK_PATH}.gz"
+    ]
+    symlink_path = "#{SYMLINK_PATH}"
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
+    Fluent::FileBuffer.clear_buffer_paths
+    d = Fluent::Test::TestDriver.new(Fluent::FileOutput).configure(conf)
 
-    # FileOutput#write returns path
-    path = d.run
-    assert File.exists?(symlink_path)
-    assert File.symlink?(symlink_path)
-  ensure
-    FileUtils.rm_rf(symlink_path)
+    begin
+      d.instance.start
+      10.times { sleep 0.05 }
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      es = Fluent::OneEventStream.new(time, {"a"=>1})
+      d.instance.emit('tag', es, Fluent::NullOutputChain.instance)
+
+      assert File.exists?(symlink_path)
+      assert File.symlink?(symlink_path)
+
+      d.instance.enqueue_buffer
+
+      assert !File.exists?(symlink_path)
+      assert File.symlink?(symlink_path)
+    ensure
+      d.instance.shutdown
+      FileUtils.rm_rf(symlink_path)
+    end
   end
 end
 

data/test/plugin/out_stream.rb

@@ -1,4 +1,5 @@
 require 'fluent/test'
+require 'helper'
 
 module StreamOutputTest
   def setup
@@ -33,8 +34,9 @@ end
 class TcpOutputTest < Test::Unit::TestCase
   include StreamOutputTest
 
+  PORT = unused_port
   CONFIG = %[
-    port 13999
+    port #{PORT}
     host 127.0.0.1
     send_timeout 51
   ]
@@ -45,7 +47,7 @@ class TcpOutputTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver
-    assert_equal 13999, d.instance.port
+    assert_equal PORT, d.instance.port
     assert_equal '127.0.0.1', d.instance.host
     assert_equal 51, d.instance.send_timeout
   end
@@ -54,8 +56,7 @@ end
 class UnixOutputTest < Test::Unit::TestCase
   include StreamOutputTest
 
-  TMP_DIR = File.dirname(__FILE__) + "/../tmp"
-
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/out_unix#{ENV['TEST_ENV_NUMBER']}"
   CONFIG = %[
     path #{TMP_DIR}/unix
     send_timeout 52

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.38
+  version: 0.10.39
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-29 00:00:00.000000000 Z
+date: 2013-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -124,6 +124,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: parallel_tests
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.15.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.15.3
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement