fluentd 0.10.8 → 0.10.9

data/ChangeLog

@@ -1,4 +1,16 @@
 
+Release 0.10.9 - 2012/01/19
+
+* Fixed TimeSlicedOutputTestDriver
+* Updated cool.io 1.0.0 -> 1.1.0
+* TextParser: fixed regexp of syslog parser to work with rsyslog on CentOS
+* out_exec_filter: improve performance by using buffering
+* out_exec_filter: added num_children parameter
+* out_exec_filter: added remove_prefix/add_prefix parameters
+* out_tail: show warning if pos_file parameter is not set
+* out_copy: fixed problems when event stream is not repeatable
+
+
 Release 0.10.8 - 2011/12/03
 
 * Added Supervisor: restart process on SIGHUP or unexpected end of process

data/README.rdoc

@@ -63,6 +63,7 @@ License:: Apache License, Version 2.0
 Patches contributed by:
 
 * {Abhishek Parolkar}[https://github.com/parolkar]
+* {Chad Skidmore}[https://github.com/chad-skidmore]
 * {Eiichiro Iwata}[http://github.com/eiichiroi]
 * {Francesc Esplugas}[https://github.com/fesplugas]
 * {hirochachacha}[https://github.com/hirochachacha]

data/Rakefile

@@ -15,7 +15,7 @@ begin
     gemspec.add_dependency "msgpack", "~> 0.4.4"
     gemspec.add_dependency "json", ">= 1.4.3"
     gemspec.add_dependency "yajl-ruby", "~> 1.0.0"
-    gemspec.add_dependency "cool.io", "~> 1.0.0"
+    gemspec.add_dependency "cool.io", "~> 1.1.0"
     gemspec.add_dependency "http_parser.rb", "~> 0.5.1"
     gemspec.add_development_dependency "rake", ">= 0.9.2"
     gemspec.test_files = Dir["test/**/*.rb"]
@@ -29,12 +29,6 @@ rescue LoadError
   puts "Jeweler not available. Install it with: gem install jeweler"
 end
 
-Rake::TestTask.new(:test) do |t|
-  t.test_files = Dir['test/*_test.rb']
-  t.ruby_opts = ['-rubygems'] if defined? Gem
-  t.ruby_opts << '-I.'
-end
-
 VERSION_FILE = "lib/fluent/version.rb"
 
 file VERSION_FILE => ["VERSION"] do |t|

data/VERSION

@@ -1 +1 @@
-0.10.8
+0.10.9

data/lib/fluent/parser.rb

@@ -21,7 +21,7 @@ module Fluent
 class TextParser
   TEMPLATES = {
     'apache' => [/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, "%d/%b/%Y:%H:%M:%S %z"],
-    'syslog' => [/^(?<time>[^ ]* [^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, "%b %d %H:%M:%S"],
+    'syslog' => [/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, "%b %d %H:%M:%S"],
   }
 
   def self.register_template(name, regexp, time_format=nil)

data/lib/fluent/plugin.rb

@@ -94,7 +94,7 @@ class PluginClass
     if klass = map[type]
       return klass.new
     end
-    raise ConfigError, "Unknown #{name} plugin '#{type}'. Run 'gem search fluent-plugin' to find plugins"
+    raise ConfigError, "Unknown #{name} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"
   end
 
   def try_load_plugin(name, type)

data/lib/fluent/plugin/in_tail.rb

@@ -42,6 +42,9 @@ class TailInput < Input
       @pf_file = File.open(@pos_file, File::RDWR|File::CREAT)
       @pf_file.sync = true
       @pf = PositionFile.parse(@pf_file)
+    else
+      $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
+      $log.warn "this parameter is highly recommended to save the position to resume tailing."
     end
 
     configure_parser(conf)
@@ -61,7 +64,7 @@ class TailInput < Input
       @loop.attach h
     }
     handlers.each {|h|
-      h.on_change  # initialize call
+      h.on_change(nil, nil)  # initialize call  # TODO prev, cur
     }
     @thread = Thread.new(&method(:run))
   end
@@ -116,14 +119,14 @@ class TailInput < Input
         # seek to the end of file first.
         # logs never duplicate but may be lost if fluent is down.
         @pos = stat.size
-        @pe.update(@inode, stat.size)
+        @pe.update(@inode, @pos)
       end
       @buffer = ''
       @callback = callback
       super(path)
     end
 
-    def on_change
+    def on_change(prev, cur)
       lines = []
       inode = nil
 
@@ -170,7 +173,7 @@ class TailInput < Input
         @pe.update_pos(@pos)
       end
 
-      @callback.call(lines)
+      @callback.call(lines) unless lines.empty?
 
     rescue Errno::ENOENT
       # moved or deleted

data/lib/fluent/plugin/out_copy.rb

@@ -57,10 +57,11 @@ class CopyOutput < MultiOutput
 
   def emit(tag, es, chain)
     unless es.repeatable?
-      es = MultiEventStream.new
-      array = es.map {|time,record|
-        es.add(time, record)
+      m = MultiEventStream.new
+      es.each {|time,record|
+        m.add(time, record)
       }
+      es = m
     end
     chain = OutputChain.new(@outputs, tag, es, chain)
     chain.next

data/lib/fluent/plugin/out_exec_filter.rb

@@ -18,7 +18,7 @@
 module Fluent
 
 
-class ExecFilterOutput < Output
+class ExecFilterOutput < BufferedOutput
   Plugin.register_output('exec_filter', self)
 
   def initialize
@@ -27,12 +27,17 @@ class ExecFilterOutput < Output
 
   config_param :command, :string
   config_param :in_keys, :string
+  config_param :remove_prefix, :string, :default => nil
   config_param :out_keys, :string
+  config_param :add_prefix, :string, :default => nil
   config_param :tag, :string, :default => nil
   config_param :tag_key, :string, :default => nil
   config_param :time_key, :string, :default => nil
   config_param :time_format, :string, :default => nil
   config_param :localtime, :bool, :default => true
+  config_param :num_children, :integer, :default => 1
+
+  config_set_default :flush_interval, 1
 
   def configure(conf)
     super
@@ -61,27 +66,55 @@ class ExecFilterOutput < Output
         @time_parse_proc = Proc.new {|str| str.to_i }
       end
     end
+
+    if @remove_prefix
+      @removed_prefix_string = @remove_prefix + '.'
+      @removed_length = @removed_prefix_string.length
+    end
+    if @add_prefix
+      @added_prefix_string = @add_prefix + '.'
+    end
   end
 
   def start
-    @io = IO.popen(@command, "r+")
-    @pid = @io.pid
-    @io.sync = true
-    @thread = Thread.new(&method(:run))
+    super
+
+    @children = []
+    @rr = 0
+    begin
+      @num_children.times do
+        c = ChildProcess.new(&method(:each_line))
+        c.start(@command)
+        @children << c
+      end
+    rescue
+      shutdown
+      raise
+    end
+  end
+
+  def before_shutdown
+    super
+    sleep 0.5  # TODO wait time before killing child process
   end
 
   def shutdown
-    Process.kill(:TERM, @pid)
-    if @thread.join(60)  # TODO wait time
-      return
-    end
-    Process.kill(:KILL, @pid)
-    @thread.join
-    nil
+    super
+
+    @children.reject! {|c|
+      c.shutdown
+      true
+    }
   end
 
-  def emit(tag, es, chain)
+  def format_stream(tag, es)
     out = ''
+    if @remove_prefix
+      if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @removed_prefix
+        tag = tag[@removed_length..-1] || ''
+      end
+    end
+
     es.each {|time,record|
       last = @in_keys.length-1
       for i in 0..last
@@ -97,45 +130,99 @@ class ExecFilterOutput < Output
       end
       out << "\n"
     }
-    @io.write out
-    chain.next
+
+    out
   end
 
-  def run
-    @io.each_line {|line|
-      begin
-        line.chomp!
-        vals = line.split("\t")
-
-        tag = @tag
-        time = nil
-        record = {}
-        for i in 0..@out_keys.length-1
-          key = @out_keys[i]
-          val = vals[i]
-          if key == @time_key
-            time = @time_parse_proc.call(val)
-          elsif key == @tag_key
-            tag = val
-          else
-            record[key] = val
-          end
-        end
+  def write(chunk)
+    r = @rr = (@rr + 1) % @children.length
+    @children[r].write chunk
+  end
 
-        if tag
-          time ||= Engine.now
-          Engine.emit(tag, time, record)
-        end
-      rescue
-        $log.error "exec_filter failed to emit", :error=>$!, :line=>line
-        $log.warn_backtrace $!.backtrace
+  def each_line(line)
+    line.chomp!
+    vals = line.split("\t")
+
+    tag = @tag
+    time = nil
+    record = {}
+    for i in 0..@out_keys.length-1
+      key = @out_keys[i]
+      val = vals[i]
+      if key == @time_key
+        time = @time_parse_proc.call(val)
+      elsif key == @tag_key
+        tag = if @add_prefix
+                @added_prefix_string + val
+              else
+                val
+              end
+      else
+        record[key] = val
       end
-    }
-    Process.waitpid(@pid)
+    end
+
+    if tag
+      time ||= Engine.now
+      Engine.emit(tag, time, record)
+    end
   rescue
-    $log.error "exec_filter process exited", :error=>$!
+    $log.error "exec_filter failed to emit", :error=>$!, :line=>line
     $log.warn_backtrace $!.backtrace
   end
+
+  class ChildProcess
+    def initialize(&each_line)
+      @pid = nil
+      @thread = nil
+      @each_line = each_line
+    end
+
+    def start(command)
+      @io = IO.popen(command, "r+")
+      @pid = @io.pid
+      @io.sync = true
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      begin
+        Process.kill(:TERM, @pid)
+      rescue Errno::ESRCH
+        if $!.message == 'No such process'
+          # child process killed by signal chained from fluentd process
+        else
+          raise
+        end
+      end
+      if @thread.join(60)  # TODO wait time
+        return
+      end
+      begin
+        Process.kill(:KILL, @pid)
+      rescue Errno::ESRCH
+        if $!.message == 'No such process'
+          # ignore if successfully killed by :TERM
+        else
+          raise
+        end
+      end
+      @thread.join
+    end
+
+    def write(chunk)
+      chunk.write_to(@io)
+    end
+
+    def run
+      @io.each_line(&@each_line)
+    rescue
+      $log.error "exec_filter process exited", :error=>$!
+      $log.warn_backtrace $!.backtrace
+    ensure
+      Process.waitpid(@pid)
+    end
+  end
 end
 
 

data/lib/fluent/test/output_test.rb

@@ -112,13 +112,19 @@ class TimeSlicedOutputTestDriver < InputTestDriver
   def run(&block)
     result = []
     super {
+      buffer = ''
       @entries.keys.each {|key|
         es = ArrayEventStream.new(@entries[key])
         @instance.emit(@tag, es, NullOutputChain.instance)
+        buffer << @instance.format_stream(@tag, es)
       }
 
       block.call if block
 
+      if @expected_buffer
+        assert_equal(@expected_buffer, buffer)
+      end
+
       chunks = @instance.instance_eval {
         @buffer.instance_eval {
           chunks = []

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.8'
+VERSION = '0.10.9'
 
 end

data/test/plugin/out_exec_filter.rb

@@ -14,10 +14,11 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     time_key time
     time_format %Y-%m-%d %H:%M:%S
     localtime
+    num_children 3
   ]
 
-  def create_driver(conf = CONFIG)
-    Fluent::Test::OutputTestDriver.new(Fluent::ExecFilterOutput).configure(conf)
+  def create_driver(conf = CONFIG, tag = 'test')
+    Fluent::Test::OutputTestDriver.new(Fluent::ExecFilterOutput, tag).configure(conf)
   end
 
   def test_configure
@@ -29,6 +30,24 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal "time", d.instance.time_key
     assert_equal "%Y-%m-%d %H:%M:%S", d.instance.time_format
     assert_equal true, d.instance.localtime
+    assert_equal 3, d.instance.num_children
+
+    d = create_driver %[
+      command sed -l -e s/foo/bar/
+      in_keys time,k1
+      out_keys time,k2
+      tag xxx
+      time_key time
+      num_children 3
+    ]
+    assert_equal "sed -l -e s/foo/bar/", d.instance.command
+
+    d = create_driver(CONFIG + %[
+      remove_prefix before
+      add_prefix after
+    ])
+    assert_equal "before", d.instance.remove_prefix
+    assert_equal "after" , d.instance.add_prefix
   end
 
   def test_emit_1
@@ -39,7 +58,6 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     d.run do
       d.emit({"k1"=>1}, time)
       d.emit({"k1"=>2}, time)
-      sleep 0.5
     end
 
     emits = d.emits
@@ -55,6 +73,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
       out_keys time,k2
       tag xxx
       time_key time
+      num_children 3
     ]
 
     time = Time.parse("2011-01-02 13:14:15").to_i
@@ -62,7 +81,6 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     d.run do
       d.emit({"k1"=>1}, time)
       d.emit({"k1"=>2}, time)
-      sleep 0.5
     end
 
     emits = d.emits
@@ -70,5 +88,73 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal ["xxx", time, {"k2"=>"1"}], emits[0]
     assert_equal ["xxx", time, {"k2"=>"2"}], emits[1]
   end
+
+  def test_emit_3
+    d = create_driver %[
+      command grep --line-buffered -v poo
+      in_keys time,val1
+      out_keys time,val2
+      tag xxx
+      time_key time
+      num_children 3
+    ]
+
+    time = Time.parse("2011-01-02 13:14:15").to_i
+
+    d.run do
+      d.emit({"val1"=>"sed-ed value foo"}, time)
+      d.emit({"val1"=>"sed-ed value poo"}, time)
+    end
+
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], emits[0]
+
+    d = create_driver %[
+      command sed -l -e s/foo/bar/
+      in_keys time,val1
+      out_keys time,val2
+      tag xxx
+      time_key time
+      num_children 3
+    ]
+
+    time = Time.parse("2011-01-02 13:14:15").to_i
+
+    d.run do
+      d.emit({"val1"=>"sed-ed value foo"}, time)
+      d.emit({"val1"=>"sed-ed value poo"}, time)
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.length
+    assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], emits[0]
+    assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], emits[1]
+  end
+
+  def test_emit_4
+    d = create_driver(%[
+      command sed -l -e s/foo/bar/
+      in_keys tag,time,val1
+      remove_prefix input
+      out_keys tag,time,val2
+      add_prefix output
+      tag_key tag
+      time_key time
+      num_children 3
+    ], 'input.test')
+
+    time = Time.parse("2011-01-02 13:14:15").to_i
+
+    d.run do
+      d.emit({"val1"=>"sed-ed value foo"}, time)
+      d.emit({"val1"=>"sed-ed value poo"}, time)
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.length
+    assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], emits[0]
+    assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], emits[1]
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.8
+  version: 0.10.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-04 00:00:00.000000000Z
+date: 2012-01-20 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70282132235300 !ruby/object:Gem::Requirement
+  requirement: &70352789356340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70282132235300
+  version_requirements: *70352789356340
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70282132234740 !ruby/object:Gem::Requirement
+  requirement: &70352789355760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70282132234740
+  version_requirements: *70352789355760
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70282132234020 !ruby/object:Gem::Requirement
+  requirement: &70352789355160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,21 +43,21 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70282132234020
+  version_requirements: *70352789355160
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70282132233440 !ruby/object:Gem::Requirement
+  requirement: &70352789343800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70282132233440
+  version_requirements: *70352789343800
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70282132232700 !ruby/object:Gem::Requirement
+  requirement: &70352789343300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70282132232700
+  version_requirements: *70352789343300
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70282132231740 !ruby/object:Gem::Requirement
+  requirement: &70352789342760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70282132231740
+  version_requirements: *70352789342760
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -176,7 +176,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.12
 signing_key: 
 specification_version: 3
 summary: Fluent event collector