fluentd 0.10.13 → 0.10.15

data/ChangeLog

@@ -1,4 +1,20 @@
 
+Release 0.10.15 - 2012/03/09
+
+* Added experimental in_status plugin
+* out_forward: normalize the phi value of the failure detector into 1sec
+* out_forward: improved memory efficiency
+* out_forward: use weighted moving average for the phi accrual failure detector
+
+
+Release 0.10.14 - 2012/03/05
+
+* Show warnings if in_tcp or out_unix is used
+* BasicBuffer shows warnings when data.bytesize > @buffer_chunk_limit
+* out_forward: randomize roundrobin queue correctly
+* Added development dependencies to the gemspec
+
+
 Release 0.10.13 - 2012/02/21
 
 * Rewrote in_tail

data/README.rdoc

@@ -61,21 +61,4 @@ License:: Apache License, Version 2.0
 
 == Contributors:
 
-Patches contributed by:
-
-* {Abhishek Parolkar}[https://github.com/parolkar]
-* {Chad Skidmore}[https://github.com/chad-skidmore]
-* {Eiichiro Iwata}[http://github.com/eiichiroi]
-* {Francesc Esplugas}[https://github.com/fesplugas]
-* {hirochachacha}[https://github.com/hirochachacha]
-* {Hiro Yoshikawa}[https://github.com/hiroyoshikawa]
-* {Masahiro Nakagawa}[https://github.com/repeatedly]
-* {Nathan Parry}[https://github.com/nparry]
-* {Nobuyuki Kubota}[https://github.com/nobu-k]
-* {Sakuro Ozawa}[https://github.com/sakuro]
-* {Satoshi Tagomori}[https://github.com/tagomoris]
-* {Takashi Nagayasu}[https://github.com/sumipan]
-* {Tsuyoshi Ozawa}[https://github.com/oza]
-* {uu59}[https://github.com/uu59]
-* {Waldemar Quevedo Salinas}[https://github.com/wallyqs]
-* {Yuichi Tateno}[https://github.com/hotchpotch]
+Patches contributed by {those people}[https://github.com/fluent/fluentd/contributors].

data/Rakefile

@@ -18,9 +18,12 @@ begin
     gemspec.add_dependency "cool.io", "~> 1.1.0"
     gemspec.add_dependency "http_parser.rb", "~> 0.5.1"
     gemspec.add_development_dependency "rake", ">= 0.9.2"
+    gemspec.add_development_dependency "rr", ">= 1.0.0"
+    gemspec.add_development_dependency "timecop", ">= 0.3.0"
+    gemspec.add_development_dependency "jeweler", ">= 1.0.0"
     gemspec.test_files = Dir["test/**/*.rb"]
     gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
-      %w[fluent.conf VERSION AUTHORS Rakefile COPYING fluentd.gemspec]
+      %w[fluent.conf VERSION AUTHORS Rakefile COPYING fluentd.gemspec Gemfile]
     gemspec.executables = ['fluentd', 'fluent-cat', 'fluent-gem']
     gemspec.required_ruby_version = '~> 1.9.2'
   end
@@ -53,5 +56,15 @@ Rake::TestTask.new(:base_test) do |t|
   #t.warning = true
 end
 
-task :default => [VERSION_FILE, :build]
+# workaround for fluentd >= 0 dependency
+task :mv_gemfile do
+  File.rename "Gemfile", "Gemfile.bak" rescue nil
+end
+
+# workaround for fluentd >= 0 dependency
+task :revert_gemfile do
+  File.rename "Gemfile.bak", "Gemfile" rescue nil
+end
+
+task :default => [VERSION_FILE, :mv_gemfile, :build, :revert_gemfile]
 

data/VERSION

@@ -1 +1 @@
-0.10.13
+0.10.15

data/fluentd.gemspec

@@ -0,0 +1,160 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "fluentd"
+  s.version = "0.10.14"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Sadayuki Furuhashi"]
+  s.date = "2012-03-07"
+  s.email = "frsyuki@gmail.com"
+  s.executables = ["fluentd", "fluent-cat", "fluent-gem"]
+  s.extra_rdoc_files = [
+    "ChangeLog",
+    "README",
+    "README.rdoc"
+  ]
+  s.files = [
+    "AUTHORS",
+    "COPYING",
+    "Gemfile",
+    "Rakefile",
+    "VERSION",
+    "bin/fluent-cat",
+    "bin/fluent-gem",
+    "bin/fluentd",
+    "fluent.conf",
+    "fluentd.gemspec",
+    "lib/fluent/buffer.rb",
+    "lib/fluent/command/cat.rb",
+    "lib/fluent/command/fluentd.rb",
+    "lib/fluent/config.rb",
+    "lib/fluent/engine.rb",
+    "lib/fluent/env.rb",
+    "lib/fluent/event.rb",
+    "lib/fluent/input.rb",
+    "lib/fluent/load.rb",
+    "lib/fluent/log.rb",
+    "lib/fluent/match.rb",
+    "lib/fluent/mixin.rb",
+    "lib/fluent/output.rb",
+    "lib/fluent/parser.rb",
+    "lib/fluent/plugin.rb",
+    "lib/fluent/plugin/buf_file.rb",
+    "lib/fluent/plugin/buf_memory.rb",
+    "lib/fluent/plugin/buf_zfile.rb",
+    "lib/fluent/plugin/in_exec.rb",
+    "lib/fluent/plugin/in_forward.rb",
+    "lib/fluent/plugin/in_http.rb",
+    "lib/fluent/plugin/in_status.rb",
+    "lib/fluent/plugin/in_stream.rb",
+    "lib/fluent/plugin/in_syslog.rb",
+    "lib/fluent/plugin/in_tail.rb",
+    "lib/fluent/plugin/out_copy.rb",
+    "lib/fluent/plugin/out_exec.rb",
+    "lib/fluent/plugin/out_exec_filter.rb",
+    "lib/fluent/plugin/out_file.rb",
+    "lib/fluent/plugin/out_forward.rb",
+    "lib/fluent/plugin/out_null.rb",
+    "lib/fluent/plugin/out_roundrobin.rb",
+    "lib/fluent/plugin/out_stdout.rb",
+    "lib/fluent/plugin/out_stream.rb",
+    "lib/fluent/plugin/out_test.rb",
+    "lib/fluent/process.rb",
+    "lib/fluent/status.rb",
+    "lib/fluent/supervisor.rb",
+    "lib/fluent/test.rb",
+    "lib/fluent/test/base.rb",
+    "lib/fluent/test/input_test.rb",
+    "lib/fluent/test/output_test.rb",
+    "lib/fluent/version.rb",
+    "test/config.rb",
+    "test/helper.rb",
+    "test/match.rb",
+    "test/mixin.rb",
+    "test/plugin/in_exec.rb",
+    "test/plugin/in_forward.rb",
+    "test/plugin/in_http.rb",
+    "test/plugin/in_stream.rb",
+    "test/plugin/out_copy.rb",
+    "test/plugin/out_exec.rb",
+    "test/plugin/out_exec_filter.rb",
+    "test/plugin/out_file.rb",
+    "test/plugin/out_forward.rb",
+    "test/plugin/out_roundrobin.rb",
+    "test/plugin/out_stream.rb"
+  ]
+  s.homepage = "http://fluentd.org/"
+  s.require_paths = ["lib"]
+  s.required_ruby_version = Gem::Requirement.new("~> 1.9.2")
+  s.rubygems_version = "1.8.12"
+  s.summary = "Fluent event collector"
+  s.test_files = ["test/config.rb", "test/helper.rb", "test/match.rb", "test/mixin.rb", "test/plugin/in_exec.rb", "test/plugin/in_forward.rb", "test/plugin/in_http.rb", "test/plugin/in_stream.rb", "test/plugin/out_copy.rb", "test/plugin/out_exec.rb", "test/plugin/out_exec_filter.rb", "test/plugin/out_file.rb", "test/plugin/out_forward.rb", "test/plugin/out_roundrobin.rb", "test/plugin/out_stream.rb"]
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<fluentd>, [">= 0"])
+      s.add_development_dependency(%q<rake>, [">= 0.9.2"])
+      s.add_development_dependency(%q<rr>, [">= 1.0.0"])
+      s.add_development_dependency(%q<timecop>, [">= 0.3.0"])
+      s.add_development_dependency(%q<jeweler>, [">= 1.0.0"])
+      s.add_development_dependency(%q<rake>, [">= 0.9.2"])
+      s.add_development_dependency(%q<rr>, [">= 1.0.0"])
+      s.add_development_dependency(%q<timecop>, [">= 0.3.0"])
+      s.add_development_dependency(%q<jeweler>, [">= 1.0.0"])
+      s.add_runtime_dependency(%q<msgpack>, ["~> 0.4.4"])
+      s.add_runtime_dependency(%q<json>, [">= 1.4.3"])
+      s.add_runtime_dependency(%q<yajl-ruby>, ["~> 1.0.0"])
+      s.add_runtime_dependency(%q<cool.io>, ["~> 1.1.0"])
+      s.add_runtime_dependency(%q<http_parser.rb>, ["~> 0.5.1"])
+      s.add_development_dependency(%q<rake>, [">= 0.9.2"])
+      s.add_development_dependency(%q<rr>, [">= 1.0.0"])
+      s.add_development_dependency(%q<timecop>, [">= 0.3.0"])
+      s.add_development_dependency(%q<jeweler>, [">= 1.0.0"])
+    else
+      s.add_dependency(%q<fluentd>, [">= 0"])
+      s.add_dependency(%q<rake>, [">= 0.9.2"])
+      s.add_dependency(%q<rr>, [">= 1.0.0"])
+      s.add_dependency(%q<timecop>, [">= 0.3.0"])
+      s.add_dependency(%q<jeweler>, [">= 1.0.0"])
+      s.add_dependency(%q<rake>, [">= 0.9.2"])
+      s.add_dependency(%q<rr>, [">= 1.0.0"])
+      s.add_dependency(%q<timecop>, [">= 0.3.0"])
+      s.add_dependency(%q<jeweler>, [">= 1.0.0"])
+      s.add_dependency(%q<msgpack>, ["~> 0.4.4"])
+      s.add_dependency(%q<json>, [">= 1.4.3"])
+      s.add_dependency(%q<yajl-ruby>, ["~> 1.0.0"])
+      s.add_dependency(%q<cool.io>, ["~> 1.1.0"])
+      s.add_dependency(%q<http_parser.rb>, ["~> 0.5.1"])
+      s.add_dependency(%q<rake>, [">= 0.9.2"])
+      s.add_dependency(%q<rr>, [">= 1.0.0"])
+      s.add_dependency(%q<timecop>, [">= 0.3.0"])
+      s.add_dependency(%q<jeweler>, [">= 1.0.0"])
+    end
+  else
+    s.add_dependency(%q<fluentd>, [">= 0"])
+    s.add_dependency(%q<rake>, [">= 0.9.2"])
+    s.add_dependency(%q<rr>, [">= 1.0.0"])
+    s.add_dependency(%q<timecop>, [">= 0.3.0"])
+    s.add_dependency(%q<jeweler>, [">= 1.0.0"])
+    s.add_dependency(%q<rake>, [">= 0.9.2"])
+    s.add_dependency(%q<rr>, [">= 1.0.0"])
+    s.add_dependency(%q<timecop>, [">= 0.3.0"])
+    s.add_dependency(%q<jeweler>, [">= 1.0.0"])
+    s.add_dependency(%q<msgpack>, ["~> 0.4.4"])
+    s.add_dependency(%q<json>, [">= 1.4.3"])
+    s.add_dependency(%q<yajl-ruby>, ["~> 1.0.0"])
+    s.add_dependency(%q<cool.io>, ["~> 1.1.0"])
+    s.add_dependency(%q<http_parser.rb>, ["~> 0.5.1"])
+    s.add_dependency(%q<rake>, [">= 0.9.2"])
+    s.add_dependency(%q<rr>, [">= 1.0.0"])
+    s.add_dependency(%q<timecop>, [">= 0.3.0"])
+    s.add_dependency(%q<jeweler>, [">= 1.0.0"])
+  end
+end
+

data/lib/fluent/buffer.rb

@@ -21,6 +21,12 @@ module Fluent
 class BufferError < StandardError
 end
 
+class BufferChunkLimitError < BufferError
+end
+
+class BufferQueueLimitError < BufferError
+end
+
 
 class Buffer
   include Configurable
@@ -165,11 +171,17 @@ class BasicBuffer < Buffer
       ## FIXME
       #elsif data.bytesize > @buffer_chunk_limit
       #  # TODO
-      #  raise BufferError, "received data too large"
+      #  raise BufferChunkLimitError, "received data too large"
 
       elsif @queue.size >= @buffer_queue_limit
-        # TODO
-        raise BufferError, "queue size exceeds limit"
+        raise BufferQueueLimitError, "queue size exceeds limit"
+      end
+
+      if data.bytesize > @buffer_chunk_limit
+        $log.warn "Size of the emitted data exceeds buffer_chunk_limit."
+        $log.warn "This may occur problems in the output plugins ``at this server.``"
+        $log.warn "To avoid problems, set a smaller number to the buffer_chunk_limit"
+        $log.warn "in the forward output ``at the log forwarding server.``"
       end
 
       nc = new_chunk(key)  # TODO generate unique chunk id

data/lib/fluent/command/fluentd.rb

@@ -41,7 +41,7 @@ op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "insta
   opts[:setup_path] = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)
 }
 
-op.on('-c', '--config PATH', "config flie path (default: #{Fluent::DEFAULT_CONFIG_PATH})") {|s|
+op.on('-c', '--config PATH', "config file path (default: #{Fluent::DEFAULT_CONFIG_PATH})") {|s|
   opts[:config_path] = s
 }
 

data/lib/fluent/load.rb

@@ -12,6 +12,7 @@ require 'cool.io/eventmachine'
 require 'fluent/env'
 require 'fluent/version'
 require 'fluent/log'
+require 'fluent/status'
 require 'fluent/config'
 require 'fluent/engine'
 require 'fluent/mixin'

data/lib/fluent/output.rb

@@ -168,6 +168,7 @@ class BufferedOutput < Output
     @error_history = []
     @error_history.extend(MonitorMixin)
     @secondary_limit = 8
+    @emit_count = 0
   end
 
   config_param :buffer_type, :string, :default => 'memory'
@@ -210,6 +211,9 @@ class BufferedOutput < Output
 
       @secondary.secondary_init(self)
     end
+
+    Status.register(self, "queue_size") { @buffer.queue_size }
+    Status.register(self, "emit_count") { @emit_count }
   end
 
   def start
@@ -226,6 +230,7 @@ class BufferedOutput < Output
   end
 
   def emit(tag, es, chain, key="")
+    @emit_count += 1
     data = format_stream(tag, es)
     if @buffer.emit(key, data, chain)
       submit_flush

data/lib/fluent/plugin/in_status.rb

@@ -0,0 +1,81 @@
+
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class StatusInput < Input
+  Plugin.register_input('status', self)
+
+  def initialize
+    super
+  end
+
+  config_param :emit_interval, :time, :default => 60
+  config_param :tag, :string
+
+  class TimerWatcher < Coolio::TimerWatcher
+    def initialize(interval, repeat, &callback)
+      @callback = callback
+      super(interval, repeat)
+    end
+
+    def on_timer
+      @callback.call
+    rescue
+      # TODO log?
+      $log.error $!.to_s
+      $log.error_backtrace
+    end
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def start
+    @loop = Coolio::Loop.new
+    @timer = TimerWatcher.new(@emit_interval, true, &method(:on_timer))
+    @loop.attach(@timer)
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @loop.watchers.each {|w| w.detach }
+    @loop.stop
+    @thread.join
+  end
+
+  def run
+    @loop.run
+  rescue
+    $log.error "unexpected error", :error=>$!.to_s
+    $log.error_backtrace
+  end
+
+  def on_timer
+    now = Engine.now
+    Status.each {|record|
+      Engine.emit(@tag, now, record)
+    }
+  end
+end
+
+
+end
+

data/lib/fluent/plugin/in_stream.rb

@@ -170,8 +170,7 @@ class TcpInput < ForwardInput
 
   def initialize
     super
-    ## TODO
-    #$log.warn "'tcp' input is obsoleted and will be removed. Use 'forward' instead."
+    $log.warn "'tcp' input is obsoleted and will be removed soon. Use 'forward' instead."
   end
 end
 
@@ -184,8 +183,7 @@ class UnixInput < StreamInput
 
   def configure(conf)
     super
-    ## TODO
-    #$log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
+    $log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
   end
 
   def listen

data/lib/fluent/plugin/out_forward.rb

@@ -71,7 +71,7 @@ class ForwardOutput < ObjectBufferedOutput
         name = "#{host}:#{port}"
       end
 
-      failure = FailureDetector.new(@heartbeat_interval.to_f/2, @hard_timeout, Time.now.to_i.to_f)
+      failure = FailureDetector.new(@heartbeat_interval, @hard_timeout, Time.now.to_i.to_f)
       sockaddr = Socket.pack_sockaddr_in(port, host)
       port, host = Socket.unpack_sockaddr_in(sockaddr)
       @nodes[sockaddr] = Node.new(name, host, port, weight, standby, failure,
@@ -171,7 +171,7 @@ class ForwardOutput < ObjectBufferedOutput
     }
 
     r = Random.new(@rand_seed)
-    weight_array.sort_by { r.rand }
+    weight_array.sort_by! { r.rand }
 
     @weight_array = weight_array
   end
@@ -350,11 +350,14 @@ class ForwardOutput < ObjectBufferedOutput
     PHI_FACTOR = 1.0 / Math.log(10.0)
     SAMPLE_SIZE = 1000
 
-    def initialize(init_int, hard_timeout, init_last)
+    def initialize(heartbeat_interval, hard_timeout, init_last)
+      @heartbeat_interval = heartbeat_interval
       @last = init_last
-      @init_int = init_int
       @hard_timeout = hard_timeout
-      @window = [init_int]
+
+      # microsec
+      @init_gap = (heartbeat_interval * 1e6).to_i
+      @window = [@init_gap]
     end
 
     def hard_timeout?(now)
@@ -363,11 +366,11 @@ class ForwardOutput < ObjectBufferedOutput
 
     def add(now)
       if @window.empty?
-        @window << @init_int
+        @window << @init_gap
         @last = now
       else
-        int = now - @last
-        @window << int
+        gap = now - @last
+        @window << (gap * 1e6).to_i
         @window.shift if @window.length > SAMPLE_SIZE
         @last = now
       end
@@ -376,9 +379,24 @@ class ForwardOutput < ObjectBufferedOutput
     def phi(now)
       size = @window.size
       return 0.0 if size == 0
+
+      # Calculate weighted moving average
+      mean_usec = 0
+      fact = 0
+      @window.each_with_index {|gap,i|
+        mean_usec += gap * (1+i)
+        fact += (1+i)
+      }
+      mean_usec = mean_usec / fact
+
+      # Normalize arrive intervals into 1sec
+      mean = (mean_usec.to_f / 1e6) - @heartbeat_interval + 1
+
+      # Calculate phi of the phi accrual failure detector
       t = now - @last
-      mean = @window.inject(0) {|r,v| r + v } / size
-      return PHI_FACTOR * t / mean
+      phi = PHI_FACTOR * t / mean
+
+      return phi
     end
 
     def sample_size

data/lib/fluent/plugin/out_stream.rb

@@ -91,8 +91,8 @@ class TcpOutput < StreamOutput
 
   def initialize
     super
-    ## TODO
-    #$log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
+    $log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
+    $log.warn "see 'forward' section in http://fluentd.org/doc/plugin.html for the high-availability configuration."
   end
 
   config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
@@ -114,8 +114,7 @@ class UnixOutput < StreamOutput
 
   def initialize
     super
-    ## TODO
-    #$log.warn "'unix' output is obsoleted and will be removed."
+    $log.warn "'unix' output is obsoleted and will be removed."
   end
 
   config_param :path, :string

data/lib/fluent/status.rb

@@ -0,0 +1,49 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+
+class StatusClass
+  def initialize
+    @entries = {}
+  end
+
+  def register(instance, name, &block)
+    (@entries[instance.object_id] ||= {})[name] = block
+    nil
+  end
+
+  def each(&block)
+    @entries.each {|obj_id,hash|
+      record = {}
+      hash.each_pair {|name,block|
+        record[name] = block.call
+      }
+      block.call(record)
+    }
+  end
+end
+
+# Don't use this class from plugins.
+# The interface may be changed
+Status = StatusClass.new
+
+
+end
+

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.13'
+VERSION = '0.10.15'
 
 end

data/test/helper.rb

@@ -1,8 +1,10 @@
 require 'test/unit'
 require 'fileutils'
 require 'fluent/log'
+require 'rr'
 
 class Test::Unit::TestCase
+  include RR::Adapters::TestUnit
 end
 
 $log = Fluent::Log.new(STDOUT, Fluent::Log::LEVEL_WARN)

data/test/mixin.rb

@@ -1,232 +1,158 @@
 require 'helper'
 require 'fluent/mixin'
+require 'fluent/env'
+require 'fluent/plugin'
+require 'fluent/config'
+require 'fluent/test'
+require 'timecop'
 
-#class MixinTest < Test::Unit::TestCase
-#  class MixinOutputTester < Fluent::BufferedOutput
-#    Fluent::Plugin.register_output('mixintest', self)
-#    include Fluent::PlainTextFormatterMixin
-#    def configure(conf)
-#      super
-#    end
-#    def write(chunk)
-#      chunk.read
-#    end
-#  end
-#
-#  def create_driver(conf='')
-#    Fluent::Test::BufferedOutputTestDriver.new(MixinOutputTester).configure(conf)
-#  end
-#
-#  def test_default_config
-#    d = create_driver
-#    assert_equal true, d.instance.output_include_time
-#    assert_equal true, d.instance.output_include_tag
-#    assert_equal 'json', d.instance.output_data_type
-#    assert_equal "\t", d.instance.output_field_separator
-#    assert_equal true, d.instance.output_add_newline
-#    assert_equal nil, d.instance.instance_eval{@localtime}
-#
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\t{"foo":1,"bar":501}\n2011-11-29T12:02:50Z\ttest\t{"foo":2,"bar":502}\n], text
-#  end
-#
-#  def test_timezone
-#    d = create_driver %[
-#utc
-#]
-#    assert_equal false, d.instance.instance_eval{@localtime}
-#
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\t{"foo":1,"bar":501}\n2011-11-29T12:02:50Z\ttest\t{"foo":2,"bar":502}\n], text
-#
-#    d = create_driver %[
-#localtime
-#]
-#    assert_equal true, d.instance.instance_eval{@localtime}
-#
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    time_s = Time.parse("2011-11-29 12:02:50 UTC").getlocal.iso8601
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal time_s + %[\ttest\t{"foo":1,"bar":501}\n] + time_s + %[\ttest\t{"foo":2,"bar":502}\n], text
-#  end
-#
-#  def test_time_tag_onoff
-#    d = create_driver %[
-#output_include_time true
-#output_include_tag false
-#]
-#    assert_equal true, d.instance.output_include_time
-#    assert_equal false, d.instance.output_include_tag
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\t{"foo":1,"bar":501}\n2011-11-29T12:02:50Z\t{"foo":2,"bar":502}\n], text
-#
-#    d = create_driver %[
-#output_include_time false
-#output_include_tag true
-#]
-#    assert_equal false, d.instance.output_include_time
-#    assert_equal true, d.instance.output_include_tag
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal %[test\t{"foo":1,"bar":501}\ntest\t{"foo":2,"bar":502}\n], text
-#    
-#    d = create_driver %[
-#output_include_time false
-#output_include_tag false
-#]
-#    assert_equal false, d.instance.output_include_time
-#    assert_equal false, d.instance.output_include_tag
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal %[{"foo":1,"bar":501}\n{"foo":2,"bar":502}\n], text
-#  end
-#
-#  def test_data_type
-#    d = create_driver %[
-#output_data_type json
-#]
-#    assert_equal 'json', d.instance.output_data_type
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>"This is what you want"}, time)
-#    d.emit({"foo"=>2,"bar"=>"Is this what you want or not"}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\t{"foo":1,"bar":"This is what you want"}\n2011-11-29T12:02:50Z\ttest\t{"foo":2,"bar":"Is this what you want or not"}\n], text
-#
-#    d = create_driver %[
-#output_data_type attr:foo
-#]
-#    assert_equal 'attr:foo', d.instance.output_data_type
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>"This is what you want"}, time)
-#    d.emit({"foo"=>2,"bar"=>"Is this what you want or not"}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\t1\n2011-11-29T12:02:50Z\ttest\t2\n], text
-#
-#    d = create_driver %[
-#output_data_type attr:bar
-#]
-#    assert_equal 'attr:bar', d.instance.output_data_type
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>"This is what you want"}, time)
-#    d.emit({"foo"=>2,"bar"=>"Is this what you want or not"}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\tThis is what you want\n2011-11-29T12:02:50Z\ttest\tIs this what you want or not\n], text
-#
-#    d = create_driver %[
-#output_data_type attr:foo,bar
-#]
-#    assert_equal 'attr:foo,bar', d.instance.output_data_type
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>"This is what you want"}, time)
-#    d.emit({"foo"=>2,"bar"=>"Is this what you want or not"}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\t1\tThis is what you want\n2011-11-29T12:02:50Z\ttest\t2\tIs this what you want or not\n], text
-#  end
-#
-#  def test_add_newline
-#    d = create_driver %[
-#output_add_newline false
-#]
-#    assert_equal false, d.instance.output_add_newline
-#
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>"This is what you want"}, time)
-#    d.emit({"foo"=>2,"bar"=>"Is this what you want or not"}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\t{"foo":1,"bar":"This is what you want"}2011-11-29T12:02:50Z\ttest\t{"foo":2,"bar":"Is this what you want or not"}], text
-#    
-#    d = create_driver %[
-#output_add_newline false
-#]
-#    assert_equal false, d.instance.output_add_newline
-#
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>"This is what you want\n"}, time)
-#    d.emit({"foo"=>2,"bar"=>"Is this what you want or not\n"}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\t{"foo":1,"bar":"This is what you want\\n"}2011-11-29T12:02:50Z\ttest\t{"foo":2,"bar":"Is this what you want or not\\n"}], text
-#
-#    d = create_driver %[
-#output_data_type attr:bar
-#output_add_newline false
-#]
-#    assert_equal false, d.instance.output_add_newline
-#
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>"This is what you want\n"}, time)
-#    d.emit({"foo"=>2,"bar"=>"Is this what you want or not\n"}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\tThis is what you want\n2011-11-29T12:02:50Z\ttest\tIs this what you want or not\n], text
-#
-#    d = create_driver %[
-#output_data_type attr:bar
-#output_add_newline true
-#]
-#    assert_equal true, d.instance.output_add_newline
-#
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#    d.emit({"foo"=>1,"bar"=>"This is what you want\n"}, time)
-#    d.emit({"foo"=>2,"bar"=>"Is this what you want or not\n"}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z\ttest\tThis is what you want\n\n2011-11-29T12:02:50Z\ttest\tIs this what you want or not\n\n], text
-#  end
-#
-#  def test_field_separator
-#    time = Time.parse("2011-11-29 12:02:50 UTC").to_i
-#
-#    d = create_driver %[
-#output_field_separator SPACE
-#]
-#    assert_equal " ", d.instance.output_field_separator
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z test {"foo":1,"bar":501}\n2011-11-29T12:02:50Z test {"foo":2,"bar":502}\n], text
-#
-#    d = create_driver %[
-#output_field_separator SPACE
-#output_data_type attr:bar,foo
-#]
-#    assert_equal " ", d.instance.output_field_separator
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z test 501 1\n2011-11-29T12:02:50Z test 502 2\n], text
-#
-#    d = create_driver %[
-#output_field_separator COMMA
-#]
-#    assert_equal ",", d.instance.output_field_separator
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z,test,{"foo":1,"bar":501}\n2011-11-29T12:02:50Z,test,{"foo":2,"bar":502}\n], text
-#
-#    d = create_driver %[
-#output_field_separator COMMA
-#output_data_type attr:foo,bar
-#]
-#    assert_equal ",", d.instance.output_field_separator
-#    d.emit({"foo"=>1,"bar"=>501}, time)
-#    d.emit({"foo"=>2,"bar"=>502}, time)
-#    text = d.run
-#    assert_equal %[2011-11-29T12:02:50Z,test,1,501\n2011-11-29T12:02:50Z,test,2,502\n], text
-#  end
-#
-#end
+module MixinTest
+  module Utils
+    def setup
+      super
+      Fluent::Test.setup
+      @time = Time.utc(1,2,3,4,5,2010,nil,nil,nil,nil)
+      Timecop.freeze(@time)
+    end
 
+    def teardown
+      super
+      Timecop.return
+    end
+
+    module Checker
+      extend self
+      def format_check(tag, time, record); end
+    end
+
+    @@num = 0
+    def create_register_output_name
+      "mixin_text_#{@@num+=1}"
+    end
+
+    def format_check(hash, tagname = 'test')
+      mock(Checker).format_check(tagname, @time.to_i, hash)
+    end
+
+    def create_driver(include_klass, conf = '')
+      register_output_name = create_register_output_name
+      include_klasses = [include_klass].flatten
+
+      klass = Class.new(Fluent::BufferedOutput) {
+        include_klasses.each {|k| include k }
+
+        Fluent::Plugin.register_output(register_output_name, self)
+        def format(tag, time, record)
+          Checker.format_check(tag, time, record)
+          [tag, time, record].to_msgpack
+        end
+
+        def write(chunk); end
+      }
+
+      Fluent::Test::BufferedOutputTestDriver.new(klass) {
+      }.configure("tyep #{register_output_name}" + conf)
+    end
+  end
+
+  class SetTagKeyMixinText < Test::Unit::TestCase
+    include Utils
+
+    def test_tag_key_default
+      format_check({
+        'a' => 1
+      })
+
+      d = create_driver(Fluent::SetTagKeyMixin, %[
+      ])
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_include_tag_key_true
+      format_check({
+        'tag' => 'test',
+        'a' => 1
+      })
+
+      d = create_driver(Fluent::SetTagKeyMixin, %[
+      include_tag_key true
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_include_tag_key_false
+      format_check({
+        'a' => 1
+      })
+
+      d = create_driver(Fluent::SetTagKeyMixin, %[
+      include_tag_key false
+      ])
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_tag_key_set
+      format_check({
+        'tag_key_changed' => 'test',
+        'a' => 1
+      })
+
+      d = create_driver(Fluent::SetTagKeyMixin, %[
+      include_tag_key true
+      tag_key tag_key_changed
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+  end
+
+  class SetTimeKeyMixinText < Test::Unit::TestCase
+    include Utils
+
+    def test_time_key_default
+      format_check({
+        'a' => 1
+      })
+
+      d = create_driver(Fluent::SetTimeKeyMixin, %[
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_include_time_key_true
+      format_check({
+        'time' => "2010-05-04T03:02:01Z",
+        'a' => 1
+      })
+
+      d = create_driver(Fluent::SetTimeKeyMixin, %[
+      include_time_key true
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_time_format
+      format_check({
+        'time' => "20100504",
+        'a' => 1
+      })
+
+      d = create_driver(Fluent::SetTimeKeyMixin, %[
+      include_time_key true
+      time_format %Y%m%d
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.13
+  version: 0.10.15
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-22 00:00:00.000000000Z
+date: 2012-03-09 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70115974571520 !ruby/object:Gem::Requirement
+  requirement: &70355087964700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70115974571520
+  version_requirements: *70355087964700
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70115974563160 !ruby/object:Gem::Requirement
+  requirement: &70355087961840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70115974563160
+  version_requirements: *70355087961840
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70115974560860 !ruby/object:Gem::Requirement
+  requirement: &70355087958040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70115974560860
+  version_requirements: *70355087958040
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70115974558780 !ruby/object:Gem::Requirement
+  requirement: &70355087953920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70115974558780
+  version_requirements: *70355087953920
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70115974553640 !ruby/object:Gem::Requirement
+  requirement: &70355087948700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70115974553640
+  version_requirements: *70355087948700
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70115974549200 !ruby/object:Gem::Requirement
+  requirement: &70355087947960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,40 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70115974549200
+  version_requirements: *70355087947960
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: &70355087946840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *70355087946840
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: &70355087946240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *70355087946240
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &70355087945380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *70355087945380
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -97,6 +130,7 @@ files:
 - bin/fluent-gem
 - bin/fluentd
 - fluent.conf
+- fluentd.gemspec
 - lib/fluent/buffer.rb
 - lib/fluent/command/cat.rb
 - lib/fluent/command/fluentd.rb
@@ -118,6 +152,7 @@ files:
 - lib/fluent/plugin/in_exec.rb
 - lib/fluent/plugin/in_forward.rb
 - lib/fluent/plugin/in_http.rb
+- lib/fluent/plugin/in_status.rb
 - lib/fluent/plugin/in_stream.rb
 - lib/fluent/plugin/in_syslog.rb
 - lib/fluent/plugin/in_tail.rb
@@ -132,6 +167,7 @@ files:
 - lib/fluent/plugin/out_stream.rb
 - lib/fluent/plugin/out_test.rb
 - lib/fluent/process.rb
+- lib/fluent/status.rb
 - lib/fluent/supervisor.rb
 - lib/fluent/test.rb
 - lib/fluent/test/base.rb