fluentd 0.10.33 → 0.10.34

data/ChangeLog

@@ -1,10 +1,27 @@
 
+Release 0.10.34 - 2013/06/03
+
+* fluentd: supports --emit-error-log-interval to suppress log messages with the
+  same message for a certain seconds
+* BufferedOutput: fixed warning messages so that errors with the same meaning
+  use the same message
+* Log: ignores ThreadError which could happen in signal handler
+* Engine: prevents signal handlers from calling stop() method of plugins twice
+* MessagePackEventStream/in_forward: doesn't cache MessagePack::Unpacker (if
+  it uses msgpack-0.5.5 which doesn't cause performance impact) to not cause
+  problems with out_copy
+* TextParser/in_tail,in_syslog: 'json' format type removes time key from records
+* TextParser/in_tail,in_syslog: raises exception if time_format is set while
+  time_key is not set
+* in_tail: fixed minor bugs
+
+
 Release 0.10.33 - 2013/03/28
 
-* add in_monitor_agent
+* added in_monitor_agent
 * switched from jeweler to bundler
 * updated dependency for msgpack from '~> 0.4.4' to '~> 0.4.4 OR ~> 0.5.4'
-* dded PluginId module so that in_monitor_agent can search plugins by id
+* added PluginId module so that in_monitor_agent can search plugins by id
 * BufferedOutput: added queued_chunk_flush_interval option to insert delay
 time before flushing queued chunks
 * BufferedOutput: randomize retry interval to prevent multiple servers from

data/lib/fluent/command/fluentd.rb

@@ -35,6 +35,7 @@ opts = {
   :setup_path => nil,
   :chuser => nil,
   :chgroup => nil,
+  :suppress_interval => 0,
 }
 
 op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
@@ -77,6 +78,9 @@ op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to
   opts[:inline_config] = s
 }
 
+op.on('--emit-error-log-interval SECONDS', "suppress interval seconds of emit error logs") {|s|
+  opts[:suppress_interval] = s.to_i
+}
 
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b

data/lib/fluent/engine.rb

@@ -26,6 +26,9 @@ class EngineClass
     @match_cache_keys = []
     @started = []
     @default_loop = nil
+
+    @suppress_emit_error_log_interval = 0
+    @next_emit_error_log_time = nil
   end
 
   MATCH_CACHE_SIZE = 1024
@@ -42,6 +45,11 @@ class EngineClass
     self
   end
 
+  def suppress_interval(interval_time)
+    @suppress_emit_error_log_interval = interval_time
+    @next_emit_error_log_time = Time.now.to_i
+  end
+
   def read_config(path)
     $log.info "reading config file", :path=>path
     File.open(path) {|io|
@@ -119,8 +127,14 @@ class EngineClass
     end
     target.emit(tag, es)
   rescue
-    $log.warn "emit transaction failed ", :error=>$!.to_s
-    $log.warn_backtrace
+    if @suppress_emit_error_log_interval == 0 || now > @next_emit_error_log_time
+      $log.warn "emit transaction failed ", :error=>$!.to_s
+      $log.warn_backtrace
+      # $log.debug "current next_emit_error_log_time: #{Time.at(@next_emit_error_log_time)}"
+      @next_emit_error_log_time = Time.now.to_i + @suppress_emit_error_log_interval
+      # $log.debug "next emit failure log suppressed"
+      # $log.debug "next logged time is #{Time.at(@next_emit_error_log_time)}"
+    end
     raise
   end
 

data/lib/fluent/event.rb

@@ -110,29 +110,54 @@ class MultiEventStream < EventStream
   end
 end
 
+if $use_msgpack_5
 
-class MessagePackEventStream < EventStream
-  def initialize(data, cached_unpacker=nil)
-    @data = data
-    @unpacker = cached_unpacker || MessagePack::Unpacker.new
-  end
+  class MessagePackEventStream < EventStream
+    def initialize(data, cached_unpacker=nil)
+      @data = data
+    end
 
-  def repeatable?
-    true
-  end
+    def repeatable?
+      true
+    end
 
-  def each(&block)
-    @unpacker.reset
-    # TODO format check
-    @unpacker.feed_each(@data, &block)
-    nil
+    def each(&block)
+      # TODO format check
+      unpacker = MessagePack::Unpacker.new
+      unpacker.feed_each(@data, &block)
+      nil
+    end
+
+    def to_msgpack_stream
+      @data
+    end
   end
 
-  def to_msgpack_stream
-    @data
+else # for 0.4.x. Will be removed after 0.5.x is stable
+
+  class MessagePackEventStream < EventStream
+    def initialize(data, cached_unpacker=nil)
+      @data = data
+      @unpacker = cached_unpacker || MessagePack::Unpacker.new
+    end
+
+    def repeatable?
+      true
+    end
+
+    def each(&block)
+      @unpacker.reset
+      # TODO format check
+      @unpacker.feed_each(@data, &block)
+      nil
+    end
+
+    def to_msgpack_stream
+      @data
+    end
   end
-end
 
+end
 
 #class IoEventStream < EventStream
 #  def initialize(io, num)

data/lib/fluent/load.rb

@@ -9,8 +9,10 @@ require 'json'
 require 'yajl'
 require 'uri'
 require 'msgpack'
+# I hate global variable but we suffer pain now for the sake of future.
+# We will switch to MessagePack 0.5 and deprecate 0.4.
+$use_msgpack_5 = defined?(MessagePack::Packer) ? true : false
 require 'cool.io'
-require 'cool.io/eventmachine'
 require 'fluent/env'
 require 'fluent/version'
 require 'fluent/log'

data/lib/fluent/log.rb

@@ -241,11 +241,11 @@ class Log
   def event(level, args)
     time = Time.now
     message = ''
-    record = {'message'=>message}
+    map = {}
     args.each {|a|
       if a.is_a?(Hash)
         a.each_pair {|k,v|
-          record[k.to_s] = v
+          map[k.to_s] = v
         }
       else
         message << a.to_s
@@ -255,14 +255,22 @@ class Log
     if @self_event
       c = caller
       if c.count(c.shift) <= 0
-        Engine.emit("#{@tag}.#{level}", time.to_i, record)
+        record = map.dup
+        record['message'] = message.dup
+        ### In signal trap context, 'Engine.emit' cannot get Monitor lock (Thread level lock),
+        ### and ThreadError will be raised
+        begin
+          Engine.emit("#{@tag}.#{level}", time.to_i, record)
+        rescue ThreadError # message: can't be called from trap context
+          # ignore
+        end
+        # TODO: We should check 'GET_THREAD()->interrupt_mask & TRAP_INTERRUPT_MASK'
+        #       for replacement of begin-rescue if any methods exists.
       end
     end
 
-    record.each_pair {|k,v|
-      if v.object_id != message.object_id
-        message << " #{k}=#{v.inspect}"
-      end
+    map.each_pair {|k,v|
+      message << " #{k}=#{v.inspect}"
     }
 
     return time, message

data/lib/fluent/mixin.rb

@@ -93,8 +93,9 @@ module RecordFilterMixin
   def format_stream(tag, es)
     out = ''
     es.each {|time,record|
-      filter_record(tag, time, record)
-      out << format(tag, time, record)
+      tag_temp = tag.dup
+      filter_record(tag_temp, time, record)
+      out << format(tag_temp, time, record)
     }
     out
   end

data/lib/fluent/output.rb

@@ -324,20 +324,20 @@ class BufferedOutput < Output
       end
 
       if error_count < @retry_limit
-        $log.warn "temporarily failed to flush the buffer, next retry will be at #{Time.at(@next_retry_time)}.", :error=>e.to_s, :instance=>object_id
+        $log.warn "temporarily failed to flush the buffer.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
         $log.warn_backtrace e.backtrace
 
       elsif @secondary
         if error_count == @retry_limit
-          $log.warn "failed to flush the buffer.", :error=>e.to_s, :instance=>object_id
+          $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
           $log.warn "retry count exceededs limit. falling back to secondary output."
           $log.warn_backtrace e.backtrace
           retry  # retry immediately
         elsif error_count <= @retry_limit + @secondary_limit
-          $log.warn "failed to flush the buffer, next retry will be with secondary output at #{Time.at(@next_retry_time)}.", :error=>e.to_s, :instance=>object_id
+          $log.warn "failed to flush the buffer, next retry will be with secondary output.", :next_retry=>Time.at(@next_retry_time), :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
           $log.warn_backtrace e.backtrace
         else
-          $log.warn "failed to flush the buffer.", :error=>e.to_s, :instance=>object_id
+          $log.warn "failed to flush the buffer.", :error_class=>e.class, :error=>e.to_s, :instance=>object_id
           $log.warn "secondary retry count exceededs limit."
           $log.warn_backtrace e.backtrace
           write_abort
@@ -345,7 +345,7 @@ class BufferedOutput < Output
         end
 
       else
-        $log.warn "failed to flush the buffer.", :error=>e.to_s, :instance=>object_id
+        $log.warn "failed to flush the buffer.", :error_class=>e.class.to_s, :error=>e.to_s, :instance=>object_id
         $log.warn "retry count exceededs limit."
         $log.warn_backtrace e.backtrace
         write_abort

data/lib/fluent/parser.rb

@@ -106,7 +106,7 @@ class TextParser
       end
 
       if @time_format && !@time_key
-        $log.warn "time_format parameter is ignored because time_key parameter is not set"
+        raise ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
       end
     end
 
@@ -114,7 +114,7 @@ class TextParser
       record = Hash[keys.zip(values)]
 
       if @time_key
-        value = record[@time_key]
+        value = record.delete(@time_key)
         if @time_format
           time = Time.strptime(value, @time_format).to_i
         else
@@ -139,9 +139,10 @@ class TextParser
   class LabeledTSVParser < ValuesParser
     config_param :delimiter,       :string, :default => "\t"
     config_param :label_delimiter, :string, :default =>  ":"
+    config_param :time_key, :string, :default =>  "time"
 
     def configure(conf)
-      conf['keys'] = conf['time_key'] || ''
+      conf['keys'] = conf['time_key'] || 'time'
       super(conf)
     end
 

data/lib/fluent/plugin/in_forward.rb

@@ -46,7 +46,7 @@ class ForwardInput < Input
     @loop.attach(@hbr)
 
     @thread = Thread.new(&method(:run))
-    @cached_unpacker = MessagePack::Unpacker.new
+    @cached_unpacker = $use_msgpack_5 ? nil : MessagePack::Unpacker.new
   end
 
   def shutdown

data/lib/fluent/plugin/in_stream.rb

@@ -31,7 +31,7 @@ class StreamInput < Input
     @lsock = listen
     @loop.attach(@lsock)
     @thread = Thread.new(&method(:run))
-    @cached_unpacker = MessagePack::Unpacker.new
+    @cached_unpacker = $use_msgpack_5 ? nil : MessagePack::Unpacker.new
   end
 
   def shutdown

data/lib/fluent/plugin/in_tail.rb

@@ -271,7 +271,7 @@ class TailInput < Input
         @wait = wait
       end
 
-      attr_reader :io
+      attr_reader :io, :wait
 
       def tick
         @wait -= 1

data/lib/fluent/plugin/out_exec_filter.rb

@@ -71,6 +71,9 @@ class ExecFilterOutput < BufferedOutput
   # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only
   config_param :child_respawn, :string, :default => nil
 
+  # 0: output logs for all of messages to emit
+  config_param :suppress_error_log_interval, :time, :default => 0
+
   config_set_default :flush_interval, 1
 
   def configure(conf)
@@ -166,6 +169,9 @@ class ExecFilterOutput < BufferedOutput
                 else
                   raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
                 end
+
+    @suppress_error_log_interval ||= 0
+    @next_log_time = Time.now.to_i
   end
 
   def start
@@ -187,6 +193,10 @@ class ExecFilterOutput < BufferedOutput
 
   def before_shutdown
     super
+    $log.debug "out_exec_filter#before_shutdown called"
+    @children.each {|c|
+      c.finished = true
+    }
     sleep 0.5  # TODO wait time before killing child process
   end
 
@@ -227,12 +237,15 @@ class ExecFilterOutput < BufferedOutput
   end
 
   class ChildProcess
+    attr_accessor :finished
+
     def initialize(parser,respawns=0)
       @pid = nil
       @thread = nil
       @parser = parser
       @respawns = respawns
       @mutex = Mutex.new
+      @finished = nil
     end
 
     def start(command)
@@ -372,8 +385,11 @@ class ExecFilterOutput < BufferedOutput
     Engine.emit(tag, time, record)
 
   rescue
-    $log.error "exec_filter failed to emit", :error=>$!.to_s, :record=>Yajl.dump(record)
-    $log.warn_backtrace $!.backtrace
+    if @suppress_error_log_interval == 0 || Time.now.to_i > @next_log_time
+      $log.error "exec_filter failed to emit", :error=>$!.to_s, :error_class=>$!.class.to_s, :record=>Yajl.dump(record)
+      $log.warn_backtrace $!.backtrace
+      @next_log_time = Time.now.to_i + @suppress_error_log_interval
+    end
   end
 
   class Parser

data/lib/fluent/plugin/out_forward.rb

@@ -128,7 +128,9 @@ class ForwardOutput < ObjectBufferedOutput
     $log.error_backtrace
   end
 
-  def write_objects(tag, es)
+  def write_objects(tag, chunk)
+    return if chunk.empty?
+
     error = nil
 
     wlen = @weight_array.length
@@ -138,7 +140,7 @@ class ForwardOutput < ObjectBufferedOutput
 
       if node.available?
         begin
-          send_data(node, tag, es)
+          send_data(node, tag, chunk)
           return
         rescue
           # for load balancing during detecting crashed servers
@@ -217,7 +219,7 @@ class ForwardOutput < ObjectBufferedOutput
     end
   end
 
-  def send_data(node, tag, es)
+  def send_data(node, tag, chunk)
     sock = connect(node)
     begin
       opt = [1, @send_timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
@@ -233,7 +235,7 @@ class ForwardOutput < ObjectBufferedOutput
       sock.write tag.to_msgpack  # tag
 
       # beginRaw(size)
-      sz = es.size
+      sz = chunk.size
       #if sz < 32
       #  # FixRaw
       #  sock.write [0xa0 | sz].pack('C')
@@ -246,7 +248,7 @@ class ForwardOutput < ObjectBufferedOutput
       #end
 
       # writeRawBody(packed_es)
-      es.write_to(sock)
+      chunk.write_to(sock)
 
       node.heartbeat(false)
     ensure

data/lib/fluent/process.rb

@@ -180,7 +180,7 @@ class DetachProcessManager
 
   def read_event_stream(r, &block)
     u = MessagePack::Unpacker.new(r)
-    cached_unpacker = MessagePack::Unpacker.new
+    cached_unpacker = $use_msgpack_5 ? nil : MessagePack::Unpacker.new
     begin
       #buf = ''
       #map = {}

data/lib/fluent/supervisor.rb

@@ -67,6 +67,7 @@ class Supervisor
     @libs = opt[:libs]
     @plugin_dirs = opt[:plugin_dirs]
     @inline_config = opt[:inline_config]
+    @suppress_interval = opt[:suppress_interval]
 
     @log = LoggerInitializer.new(@log_path, @log_level, @chuser, @chgroup)
     @finished = false
@@ -90,6 +91,7 @@ class Supervisor
         run_engine
         exit 0
       end
+      $log.error "fluentd main process died unexpectedly. restarting." unless @finished
     end
   end
 
@@ -203,30 +205,50 @@ class Supervisor
 
   def install_supervisor_signal_handlers
     trap :INT do
+      $log.debug "fluentd supervisor process get SIGINT"
       @finished = true
       if pid = @main_pid
-        Process.kill(:INT, pid)
+        # kill processes only still exists
+        unless Process.waitpid(pid, Process::WNOHANG)
+          begin
+            Process.kill(:INT, pid)
+          rescue Errno::ESRCH
+            # ignore processes already died
+          end
+        end
       end
     end
 
     trap :TERM do
+      $log.debug "fluentd supervisor process get SIGTERM"
       @finished = true
       if pid = @main_pid
-        Process.kill(:TERM, pid)
+        # kill processes only still exists
+        unless Process.waitpid(pid, Process::WNOHANG)
+          begin
+            Process.kill(:TERM, pid)
+          rescue Errno::ESRCH
+            # ignore processes already died
+          end
+        end
       end
     end
 
     trap :HUP do
+      $log.debug "fluentd supervisor process get SIGHUP"
       $log.info "restarting"
       if pid = @main_pid
         Process.kill(:TERM, pid)
+        # don't resuce Erro::ESRSH here (invalid status)
       end
     end
 
     trap :USR1 do
+      $log.debug "fluentd supervisor process get SIGUSR1"
       @log.reopen!
       if pid = @main_pid
         Process.kill(:USR1, pid)
+        # don't resuce Erro::ESRSH here (invalid status)
       end
     end
   end
@@ -281,6 +303,9 @@ class Supervisor
   def init_engine
     require 'fluent/load'
     Fluent::Engine.init
+    if @suppress_interval
+      Fluent::Engine.suppress_interval(@suppress_interval)
+    end
 
     @libs.each {|lib|
       require lib
@@ -295,19 +320,34 @@ class Supervisor
   end
 
   def install_main_process_signal_handlers
+    # Strictly speaking, these signal handling is not thread safe.
+    # But enough safe to limit twice call of Fluent::Engine.stop.
+
     trap :INT do
-      Fluent::Engine.stop
+      $log.debug "fluentd main process get SIGINT"
+      unless @finished
+        @finished = true
+        $log.debug "getting start to shutdown main process"
+        Fluent::Engine.stop
+      end
     end
 
     trap :TERM do
-      Fluent::Engine.stop
+      $log.debug "fluentd main process get SIGTERM"
+      unless @finished
+        @finished = true
+        $log.debug "getting start to shutdown main process"
+        Fluent::Engine.stop
+      end
     end
 
     trap :HUP do
       # TODO
+      $log.debug "fluentd main process get SIGHUP"
     end
 
     trap :USR1 do
+      $log.debug "fluentd main process get SIGUSR1"
       $log.info "force flushing buffered events"
       @log.reopen!
       Fluent::Engine.flush!

data/lib/fluent/test/base.rb

@@ -38,7 +38,7 @@ class TestDriver
   attr_reader :instance, :config
 
   def configure(str)
-    if str.is_a?(Config)
+    if str.is_a?(Fluent::Config::Element)
       @config = str
     else
       @config = Config.parse(str, "(test)")

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.33'
+VERSION = '0.10.34'
 
 end

data/test/helper.rb

@@ -3,6 +3,11 @@ require 'fileutils'
 require 'fluent/log'
 require 'rr'
 
+unless defined?(Test::Unit::AssertionFailedError)
+  class Test::Unit::AssertionFailedError < StandardError
+  end
+end
+
 class Test::Unit::TestCase
   include RR::Adapters::TestUnit
 end

data/test/mixin.rb

@@ -35,7 +35,7 @@ module MixinTest
       mock(Checker).format_check(tagname, @time.to_i, hash)
     end
 
-    def create_driver(include_klass, conf = '')
+    def create_driver(include_klass, conf = '', tag = "test")
       register_output_name = create_register_output_name
       include_klasses = [include_klass].flatten
 
@@ -51,7 +51,7 @@ module MixinTest
         def write(chunk); end
       }
 
-      Fluent::Test::BufferedOutputTestDriver.new(klass) {
+      Fluent::Test::BufferedOutputTestDriver.new(klass, tag) {
       }.configure("type #{register_output_name}" + conf)
     end
   end
@@ -164,6 +164,9 @@ module MixinTest
       format_check({
         'a' => 1
       }, 'tag_prefix.test')
+      format_check({
+        'a' => 2
+      }, 'tag_prefix.test')
 
       d = create_driver(Fluent::HandleTagNameMixin, %[
         add_tag_prefix tag_prefix.
@@ -171,6 +174,7 @@ module MixinTest
       ])
 
       d.emit({'a' => 1})
+      d.emit({'a' => 2})
       d.run
     end
 
@@ -178,6 +182,9 @@ module MixinTest
       format_check({
         'a' => 1
       }, 'test.test_suffix')
+      format_check({
+        'a' => 2
+      }, 'test.test_suffix')
 
       d = create_driver(Fluent::HandleTagNameMixin, %[
         add_tag_suffix .test_suffix
@@ -185,34 +192,43 @@ module MixinTest
       ])
 
       d.emit({'a' => 1})
+      d.emit({'a' => 2})
       d.run
     end
 
     def test_remove_tag_prefix
       format_check({
         'a' => 1
-      }, 'st')
+      }, 'test')
+      format_check({
+        'a' => 2
+      }, 'test')
 
       d = create_driver(Fluent::HandleTagNameMixin, %[
         remove_tag_prefix te
         include_tag_key true
-      ])
+      ], "tetest")
 
       d.emit({'a' => 1})
+      d.emit({'a' => 2})
       d.run
     end
 
     def test_remove_tag_suffix
       format_check({
         'a' => 1
-      }, 'te')
+      }, 'test')
+      format_check({
+        'a' => 2
+      }, 'test')
 
       d = create_driver(Fluent::HandleTagNameMixin, %[
         remove_tag_suffix st
         include_tag_key true
-      ])
+      ], "testst")
 
       d.emit({'a' => 1})
+      d.emit({'a' => 2})
       d.run
     end
 

data/test/parser.rb

@@ -151,12 +151,12 @@ module ParserTest
 
     def test_call
       parser = TextParser::LabeledTSVParser.new
-      time, record = parser.call("time:[28/Feb/2013:12:00:00 +0900]\thost:192.168.0.1\treq:GET /list HTTP/1.1")
+      time, record = parser.call("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111")
 
+      assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
       assert_equal({
-        'time' => '[28/Feb/2013:12:00:00 +0900]',
-        'host' => '192.168.0.1',
-        'req'  => 'GET /list HTTP/1.1',
+        'host'   => '192.168.0.1',
+        'req_id' => '111',
       }, record)
     end
 
@@ -166,27 +166,27 @@ module ParserTest
         'delimiter'       => ',',
         'label_delimiter' => '=',
       )
-      time, record = parser.call('time=[28/Feb/2013:12:00:00 +0900],host=192.168.0.1,req=GET /list HTTP/1.1')
+      time, record = parser.call('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111')
 
+      assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
       assert_equal({
-        'time' => '[28/Feb/2013:12:00:00 +0900]',
-        'host' => '192.168.0.1',
-        'req'  => 'GET /list HTTP/1.1',
+        'host'   => '192.168.0.1',
+        'req_id' => '111',
       }, record)
     end
 
     def test_call_with_customized_time_format
       parser = TextParser::LabeledTSVParser.new
       parser.configure(
-        'time_key'    => 'time',
-        'time_format' => '[%d/%b/%Y:%H:%M:%S %z]',
+        'time_key'    => 'mytime',
+        'time_format' => '%d/%b/%Y:%H:%M:%S %z',
       )
-      time, record = parser.call("time:[28/Feb/2013:12:00:00 +0900]\thost:192.168.0.1\treq:GET /list HTTP/1.1")
+      time, record = parser.call("mytime:28/Feb/2013:12:00:00 +0900\thost:192.168.0.1\treq_id:111")
 
+      assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
       assert_equal({
-        'time' => '[28/Feb/2013:12:00:00 +0900]',
-        'host' => '192.168.0.1',
-        'req'  => 'GET /list HTTP/1.1',
+        'host'   => '192.168.0.1',
+        'req_id' => '111',
       }, record)
     end
   end

data/test/plugin/out_copy.rb

@@ -51,5 +51,42 @@ class CopyOutputTest < Test::Unit::TestCase
         ], o.events
     }
   end
+
+  def test_msgpack_es_emit_bug
+    d = Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput)
+
+    outputs = %w(p1 p2).map do |pname|
+      p = Fluent::Plugin.new_output('test')
+      p.configure('name' => pname)
+      p.define_singleton_method(:emit) do |tag, es, chain|
+        es.each do |time, record|
+          super(tag, [[time, record]], chain)
+        end
+      end
+      p
+    end
+
+    d.instance.instance_eval { @outputs = outputs }
+
+    es = if defined?(MessagePack::Packer)
+           time = Time.parse("2013-05-26 06:37:22 UTC").to_i
+           packer = MessagePack::Packer.new
+           packer.pack([time, {"a" => 1}])
+           packer.pack([time, {"a" => 2}])
+           Fluent::MessagePackEventStream.new(packer.to_s)
+         else
+           events = "#{[time, {"a" => 1}].to_msgpack}#{[time, {"a" => 2}].to_msgpack}"
+           Fluent::MessagePackEventStream.new(events)
+         end
+
+    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
+
+    d.instance.outputs.each { |o|
+      assert_equal [
+        [time, {"a"=>1}],
+        [time, {"a"=>2}],
+      ], o.events
+    }
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.33
+  version: 0.10.34
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-28 00:00:00.000000000 Z
+date: 2013-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack