fluentd 0.10.24 → 0.10.25

data/ChangeLog

@@ -1,4 +1,12 @@
 
+Release 0.10.25 - 2012/07/23
+
+* shutdown won't block
+* out_exec_filter supports 'child_respawn' option
+* in_tail doesn't remove whitespaces
+* out_roundrobin supports 'weight' option
+
+
 Release 0.10.24 - 2012/06/28
 
 * TextParser/in_tail: supports 'tsv' and 'csv' preset formats

data/VERSION

@@ -1 +1 @@
-0.10.24
+0.10.25

data/lib/fluent/engine.rb

@@ -174,13 +174,17 @@ class EngineClass
   end
 
   def shutdown
-    @started.reverse_each {|s|
-      begin
-        s.shutdown
-      rescue
-        $log.warn "unexpected error while shutting down", :error=>$!.to_s
-        $log.warn_backtrace
+    @started.map {|s|
+      Thread.new do
+        begin
+          s.shutdown
+        rescue
+          $log.warn "unexpected error while shutting down", :error=>$!.to_s
+          $log.warn_backtrace
+        end
       end
+    }.each {|t|
+      t.join
     }
   end
 

data/lib/fluent/plugin/in_tail.rb

@@ -90,7 +90,7 @@ class TailInput < Input
     es = MultiEventStream.new
     lines.each {|line|
       begin
-        line.rstrip!  # remove \n
+        line.chomp!  # remove \n
         time, record = parse_line(line)
         if time && record
           es.add(time, record)

data/lib/fluent/plugin/out_exec_filter.rb

@@ -68,6 +68,9 @@ class ExecFilterOutput < BufferedOutput
   config_param :localtime, :bool, :default => true
   config_param :num_children, :integer, :default => 1
 
+  # nil, 'none' or 0: no respawn, 'inf' or -1: infinite times, positive integer: try to respawn specified times only
+  config_param :child_respawn, :string, :default => nil
+
   config_set_default :flush_interval, 1
 
   def configure(conf)
@@ -153,6 +156,16 @@ class ExecFilterOutput < BufferedOutput
     when :msgpack
       @parser = MessagePackParser.new(method(:on_message))
     end
+
+    @respawns = if @child_respawn.nil? or @child_respawn == 'none' or @child_respawn == '0'
+                  0
+                elsif @child_respawn == 'inf' or @child_respawn == '-1'
+                  -1
+                elsif @child_respawn =~ /^\d+$/
+                  @child_respawn.to_i
+                else
+                  raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
+                end
   end
 
   def start
@@ -162,7 +175,7 @@ class ExecFilterOutput < BufferedOutput
     @rr = 0
     begin
       @num_children.times do
-        c = ChildProcess.new(@parser)
+        c = ChildProcess.new(@parser, @respawns)
         c.start(@command)
         @children << c
       end
@@ -214,49 +227,81 @@ class ExecFilterOutput < BufferedOutput
   end
 
   class ChildProcess
-    def initialize(parser)
+    def initialize(parser,respawns=0)
       @pid = nil
       @thread = nil
       @parser = parser
+      @respawns = respawns
+      @mutex = Mutex.new
     end
 
     def start(command)
       @command = command
-      @io = IO.popen(command, "r+")
-      @pid = @io.pid
-      @io.sync = true
+      @mutex.synchronize do
+        @io = IO.popen(command, "r+")
+        @pid = @io.pid
+        @io.sync = true
+        @thread = Thread.new(&method(:run))
+      end
       @finished = false
-      @thread = Thread.new(&method(:run))
     end
 
-    def shutdown
-      @finished = true
+    def kill_child(join_wait)
       begin
         Process.kill(:TERM, @pid)
       rescue Errno::ESRCH
-        if $!.message == 'No such process'
-          # child process killed by signal chained from fluentd process
-        else
-          raise
-        end
+        # Errno::ESRCH 'No such process', ignore
+        # child process killed by signal chained from fluentd process
       end
-      if @thread.join(60)  # TODO wait time
+      if @thread.join(join_wait)
+        # @thread successfully shutdown
         return
       end
       begin
         Process.kill(:KILL, @pid)
       rescue Errno::ESRCH
-        if $!.message == 'No such process'
-          # ignore if successfully killed by :TERM
-        else
-          raise
-        end
+        # ignore if successfully killed by :TERM
       end
       @thread.join
     end
 
+    def shutdown
+      @finished = true
+      @mutex.synchronize do
+        kill_child(60) # TODO wait time
+      end
+    end
+
     def write(chunk)
-      chunk.write_to(@io)
+      begin
+        chunk.write_to(@io)
+      rescue Errno::EPIPE => e
+        # Broken pipe (child process unexpectedly exited)
+        $log.warn "exec_filter Broken pipe, child process maybe exited.", :command => @command
+        if try_respawn
+          retry # retry chunk#write_to with child respawned
+        else
+          raise e # to retry #write with other ChildProcess instance (when num_children > 1)
+        end
+      end
+    end
+
+    def try_respawn
+      return false if @respawns == 0
+      @mutex.synchronize do
+        return false if @respawns == 0
+
+        kill_child(5) # TODO wait time
+
+        @io = IO.popen(@command, "r+")
+        @pid = @io.pid
+        @io.sync = true
+        @thread = Thread.new(&method(:run))
+
+        @respawns -= 1 if @respawns > 0
+      end
+      $log.warn "exec_filter child process successfully respawned.", :command => @command, :respawns => @respawns
+      true
     end
 
     def run
@@ -268,6 +313,9 @@ class ExecFilterOutput < BufferedOutput
       pid, stat = Process.waitpid2(@pid)
       unless @finished
         $log.error "exec_filter process unexpectedly exited.", :command=>@command, :ecode=>stat.to_i
+        unless @respawns == 0
+          $log.warn "exec_filter child process will respawn for next input data (respawns #{@respawns})."
+        end
       end
     end
   end

data/lib/fluent/plugin/out_roundrobin.rb

@@ -23,9 +23,11 @@ class RoundRobinOutput < MultiOutput
 
   def initialize
     @outputs = []
+    @weights = []
   end
 
-  attr_reader :outputs
+  attr_reader :outputs, :weights
+  attr_accessor :rand_seed
 
   def configure(conf)
     conf.elements.select {|e|
@@ -35,16 +37,23 @@ class RoundRobinOutput < MultiOutput
       unless type
         raise ConfigError, "Missing 'type' parameter on <store> directive"
       end
-      $log.debug "adding store type=#{type.dump}"
+
+      weight = e['weight']
+      weight = weight ? weight.to_i : 1
+      $log.debug "adding store type=#{type.dump}, weight=#{weight}"
 
       output = Plugin.new_output(type)
       output.configure(e)
       @outputs << output
+      @weights << weight
     }
     @rr = -1  # starts from @output[0]
+    @rand_seed = Random.new.seed
   end
 
   def start
+    rebuild_weight_array
+
     @outputs.each {|o|
       o.start
     }
@@ -62,8 +71,27 @@ class RoundRobinOutput < MultiOutput
 
   protected
   def next_output
-    @rr = 0 if (@rr += 1) >= @outputs.size
-    @outputs[@rr]
+    @rr = 0 if (@rr += 1) >= @weight_array.size
+    @weight_array[@rr]
+  end
+
+  def rebuild_weight_array
+    gcd = @weights.inject(0) {|r,w| r.gcd(w) }
+
+    weight_array = []
+    @outputs.zip(@weights).each {|output,weight|
+      (weight / gcd).times {
+        weight_array << output
+      }
+    }
+
+    # don't randomize order if all weight is 1 (=default)
+    if @weights.any? {|w| w > 1 }
+      r = Random.new(@rand_seed)
+      weight_array.sort_by! { r.rand }
+    end
+
+    @weight_array = weight_array
   end
 end
 

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.24'
+VERSION = '0.10.25'
 
 end

data/test/plugin/in_tail.rb

@@ -75,5 +75,33 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal(true, emits.length > 0)
     assert_equal({"message"=>"test3test4"}, emits[0][2])
   end
-end
 
+  def test_whitespace
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
+
+    d = create_driver
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts "    "		# 4 spaces
+        f.puts "    4 spaces"
+        f.puts "4 spaces    "
+        f.puts "	"	# tab
+        f.puts "	tab"
+        f.puts "tab	"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"message"=>"    "}, emits[0][2])
+    assert_equal({"message"=>"    4 spaces"}, emits[1][2])
+    assert_equal({"message"=>"4 spaces    "}, emits[2][2])
+    assert_equal({"message"=>"	"}, emits[3][2])
+    assert_equal({"message"=>"	tab"}, emits[4][2])
+    assert_equal({"message"=>"tab	"}, emits[5][2])
+  end
+end

data/test/plugin/out_exec_filter.rb

@@ -22,6 +22,17 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     Fluent::Test::OutputTestDriver.new(Fluent::ExecFilterOutput, tag).configure(conf)
   end
 
+  def sed_unbuffered_support?
+    @sed_unbuffered_support ||= lambda {
+      system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >/dev/null 2>&1")
+      $?.success?
+    }.call
+  end
+
+  def sed_unbuffered_option
+    sed_unbuffered_support? ? '--unbuffered' : ''
+  end
+
   def test_configure
     d = create_driver
 
@@ -115,7 +126,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], emits[0]
 
     d = create_driver %[
-      command sed --unbuffered -l -e s/foo/bar/
+      command sed #{sed_unbuffered_option} -l -e s/foo/bar/
       in_keys time,val1
       out_keys time,val2
       tag xxx
@@ -138,7 +149,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
 
   def test_emit_4
     d = create_driver(%[
-      command sed --unbuffered -l -e s/foo/bar/
+      command sed #{sed_unbuffered_option} -l -e s/foo/bar/
       in_keys tag,time,val1
       remove_prefix input
       out_keys tag,time,val2

data/test/plugin/out_roundrobin.rb

@@ -19,6 +19,22 @@ class RoundRobinOutputTest < Test::Unit::TestCase
       name c2
     </store>
   ]
+  CONFIG_WITH_WEIGHT = %[
+    <store>
+      type test
+      name c0
+      weight 3
+    </store>
+    <store>
+      type test
+      name c1
+      weight 3
+    </store>
+    <store>
+      type test
+      name c2
+    </store>
+  ]
 
   def create_driver(conf = CONFIG)
     Fluent::Test::OutputTestDriver.new(Fluent::RoundRobinOutput).configure(conf)
@@ -35,16 +51,41 @@ class RoundRobinOutputTest < Test::Unit::TestCase
     assert_equal "c0", outputs[0].name
     assert_equal "c1", outputs[1].name
     assert_equal "c2", outputs[2].name
+
+    weights = d.instance.weights
+    assert_equal 3, weights.size
+    assert_equal 1, weights[0]
+    assert_equal 1, weights[1]
+    assert_equal 1, weights[2]
+
+    d = create_driver(CONFIG_WITH_WEIGHT)
+
+    outputs = d.instance.outputs
+    assert_equal 3, outputs.size
+    assert_equal Fluent::TestOutput, outputs[0].class
+    assert_equal Fluent::TestOutput, outputs[1].class
+    assert_equal Fluent::TestOutput, outputs[2].class
+    assert_equal "c0", outputs[0].name
+    assert_equal "c1", outputs[1].name
+    assert_equal "c2", outputs[2].name
+
+    weights = d.instance.weights
+    assert_equal 3, weights.size
+    assert_equal 3, weights[0]
+    assert_equal 3, weights[1]
+    assert_equal 1, weights[2]
   end
 
   def test_emit
     d = create_driver
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-    d.emit({"a"=>3}, time)
-    d.emit({"a"=>4}, time)
+    d.run do
+      d.emit({"a"=>1}, time)
+      d.emit({"a"=>2}, time)
+      d.emit({"a"=>3}, time)
+      d.emit({"a"=>4}, time)
+    end
 
     os = d.instance.outputs
 
@@ -61,5 +102,22 @@ class RoundRobinOutputTest < Test::Unit::TestCase
         [time, {"a"=>3}],
       ], os[2].events
   end
+
+  def test_emit_weighted
+    d = create_driver(CONFIG_WITH_WEIGHT)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.run do
+      14.times do |i|
+        d.emit({"a"=>i}, time)
+      end
+    end
+
+    os = d.instance.outputs
+
+    assert_equal 6, os[0].events.size  # weight=3
+    assert_equal 6, os[1].events.size  # weight=3
+    assert_equal 2, os[2].events.size  # weight=1
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.24
+  version: 0.10.25
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-29 00:00:00.000000000Z
+date: 2012-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70174825287820 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,15 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70174825287820
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.4
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70174825286900 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +37,15 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70174825286900
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.3
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70174825286020 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +53,15 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70174825286020
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70174825279360 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +69,15 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70174825279360
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70174825278760 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +85,15 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70174825278760
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.1
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70174825278280 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +101,15 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70174825278280
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &70174825277720 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +117,15 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70174825277720
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: timecop
-  requirement: &70174825277040 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +133,15 @@ dependencies:
         version: 0.3.0
   type: :development
   prerelease: false
-  version_requirements: *70174825277040
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.0
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70174825276300 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +149,12 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70174825276300
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -219,7 +264,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.12
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Fluent event collector