fluentd 0.10.22 → 0.10.23

data/ChangeLog

@@ -1,4 +1,21 @@
 
+Release 0.10.23 - 2012/06/25
+
+* out_exec_filter: show more error messages when child process unexpectedly exited
+* in_tail: Added 'apache2' format template which converts fields into integers or nil
+* TextParser creates a parser instance for each output for proper configuration
+* BufferedOutput: reset @flush_now flag
+* changed default buffer limit parameters:
+    Fluentd core:
+      buffer_chunk_limit 256MB -> 8MB
+      buffer_queue_limit 128 -> 256
+    buf_memory:
+      buffer_chunk_limit 32MB -> 8MB
+      buffer_queue_limit 32 -> 64
+    out_file:
+      buffer_chunk_limit -> 256MB
+
+
 Release 0.10.22 - 2012/05/02
 
 * Fixed in_tail and TextParser not to raise "time_format is required" error

data/VERSION

@@ -1 +1 @@
-0.10.22
+0.10.23

data/lib/fluent/buffer.rb

@@ -127,8 +127,11 @@ class BasicBuffer < Buffer
     @parallel_pop = b
   end
 
-  config_param :buffer_chunk_limit, :size, :default => 256*1024*1024
-  config_param :buffer_queue_limit, :integer, :default => 128
+  # This configuration assumes plugins to send records to a remote server.
+  # Local file based plugins which should provide more reliability and efficiency
+  # should override buffer_chunk_limit with a larger size.
+  config_param :buffer_chunk_limit, :size, :default => 8*1024*1024
+  config_param :buffer_queue_limit, :integer, :default => 256
 
   alias chunk_limit buffer_chunk_limit
   alias chunk_limit= buffer_chunk_limit=
@@ -191,14 +194,16 @@ class BasicBuffer < Buffer
         nc << data
         chain.next
 
+        flush_trigger = false
         @queue.synchronize {
           enqueue(top)
+          flush_trigger = @queue.empty?
           @queue << top
           @map[key] = nc
         }
 
         ok = true
-        return @queue.size == 1
+        return flush_trigger
       ensure
         nc.purge unless ok
       end

data/lib/fluent/output.rb

@@ -77,7 +77,6 @@ end
 class OutputThread
   def initialize(output)
     @output = output
-    @flush_now = false
     @finish = false
     @next_time = Engine.now + 1.0
   end
@@ -102,7 +101,7 @@ class OutputThread
 
   def submit_flush
     @mutex.synchronize {
-      @flush_now = true
+      @next_time = 0
       @cond.signal
     }
     Thread.pass
@@ -115,7 +114,7 @@ class OutputThread
       until @finish
         time = Engine.now
 
-        if @flush_now || @next_time <= time
+        if @next_time <= time
           @mutex.unlock
           begin
             @next_time = @output.try_flush

data/lib/fluent/parser.rb

@@ -84,24 +84,79 @@ class TextParser
       end
 
       return time, record
+    rescue Yajl::ParseError
+      # TODO?
+      return nil, nil
     end
   end
 
-  TEMPLATES = {
-    'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
-    'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
-    'json' => JSONParser.new,
+  class ApacheParser
+    include Configurable
+
+    REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+
+    def call(text)
+      m = REGEXP.match(text)
+      unless m
+        $log.debug "pattern not match: #{text}"
+        return nil, nil
+      end
+
+      host = m['host']
+      host = (host == '-') ? nil : host
+
+      user = m['user']
+      user = (user == '-') ? nil : user
+
+      time = m['time']
+      time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
+
+      method = m['method']
+      path = m['path']
+
+      code = m['code'].to_i
+      code = nil if code == 0
+
+      size = m['size']
+      size = (size == '-') ? nil : size.to_i
+
+      referer = m['referer']
+      referer = (referer == '-') ? nil : referer
+
+      agent = m['agent']
+      agent = (agent == '-') ? nil : agent
+
+      record = {
+        "host" => host,
+        "user" => user,
+        "method" => method,
+        "path" => path,
+        "code" => code,
+        "size" => size,
+        "referer" => referer,
+        "agent" => agent,
+      }
+
+      return time, record
+    end
+  end
+
+  TEMPLATE_FACTORIES = {
+    'apache' => Proc.new { RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
+    'apache2' => Proc.new { ApacheParser.new },
+    'syslog' => Proc.new { RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}) },
+    'json' => Proc.new { JSONParser.new },
   }
 
   def self.register_template(name, regexp_or_proc, time_format=nil)
     if regexp_or_proc.is_a?(Regexp)
-      pr = regexp_or_proc
-    else
       regexp = regexp_or_proc
-      pr = RegexpParser.new(regexp, {'time_format'=>time_format})
+      factory = Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
+    else
+      factory = Proc.new { regexp_or_proc }
     end
 
-    TEMPLATES[name] = pr
+    TEMPLATE_FACTORIES[name] = factory
   end
 
   def initialize
@@ -134,10 +189,11 @@ class TextParser
 
     else
       # built-in template
-      @parser = TEMPLATES[format]
-      unless @parser
+      factory = TEMPLATE_FACTORIES[format]
+      unless factory
         raise ConfigError, "Unknown format template '#{format}'"
       end
+      @parser = factory.call
     end
 
     if @parser.respond_to?(:configure)

data/lib/fluent/parser.rb.orig

@@ -0,0 +1,159 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class TextParser
+  class RegexpParser
+    include Configurable
+
+    config_param :time_format, :string, :default => nil
+
+    def initialize(regexp, conf={})
+      super()
+      @regexp = regexp
+      unless conf.empty?
+        configure(conf)
+      end
+    end
+
+    def call(text)
+      m = @regexp.match(text)
+      unless m
+        $log.debug "pattern not match: #{text}"
+        # TODO?
+        return nil, nil
+      end
+
+      time = nil
+      record = {}
+
+      m.names.each {|name|
+        if value = m[name]
+          case name
+          when "time"
+            if @time_format
+              time = Time.strptime(value, @time_format).to_i
+            else
+              time = Time.parse(value).to_i
+            end
+          else
+            record[name] = value
+          end
+        end
+      }
+
+      time ||= Engine.now
+
+      return time, record
+    end
+  end
+
+  class JSONParser
+    include Configurable
+
+    config_param :time_key, :string, :default => 'time'
+    config_param :time_format, :string, :default => nil
+
+    def call(text)
+      record = Yajl.load(text)
+
+      if value = record.delete(@time_key)
+        if @time_format
+          time = Time.strptime(value, @time_format).to_i
+        else
+          time = value.to_i
+        end
+      else
+        time = Engine.now
+      end
+
+      return time, record
+    rescue Yajl::ParseError
+      # TODO?
+      return nil, nil
+    end
+  end
+
+  TEMPLATES = {
+    'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
+    'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
+    'json' => JSONParser.new,
+  }
+
+  def self.register_template(name, regexp_or_proc, time_format=nil)
+    if regexp_or_proc.is_a?(Regexp)
+      regexp = regexp_or_proc
+      pr = RegexpParser.new(regexp, {'time_format'=>time_format})
+    else
+      pr = regexp_or_proc
+    end
+
+    TEMPLATES[name] = pr
+  end
+
+  def initialize
+    @parser = nil
+  end
+
+  def configure(conf, required=true)
+    format = conf['format']
+
+    if format == nil
+      if required
+        raise ConfigError, "'format' parameter is required"
+      else
+        return nil
+      end
+    end
+
+    if format[0] == ?/ && format[format.length-1] == ?/
+      # regexp
+      begin
+        regexp = Regexp.new(format[1..-2])
+        if regexp.named_captures.empty?
+          raise "No named captures"
+        end
+      rescue
+        raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
+      end
+
+      @parser = RegexpParser.new(regexp)
+
+    else
+      # built-in template
+      @parser = TEMPLATES[format]
+      unless @parser
+        raise ConfigError, "Unknown format template '#{format}'"
+      end
+    end
+
+    if @parser.respond_to?(:configure)
+      @parser.configure(conf)
+    end
+
+    return true
+  end
+
+  def parse(text)
+    return @parser.call(text)
+  end
+end
+
+
+end

data/lib/fluent/parser.rb.rej

@@ -0,0 +1,91 @@
+***************
+*** 90,107 ****
+      end
+    end
+  
+    TEMPLATES = {
+-     'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
+-     'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
+-     'json' => JSONParser.new,
+    }
+  
+    def self.register_template(name, regexp_or_proc, time_format=nil)
+      if regexp_or_proc.is_a?(Regexp)
+-       pr = regexp_or_proc
+      else
+        regexp = regexp_or_proc
+-       pr = RegexpParser.new(regexp, {'time_format'=>time_format})
+      end
+  
+      TEMPLATES[name] = pr
+--- 90,159 ----
+      end
+    end
+  
++   class ApacheParser
++     include Configurable
++ 
++     REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
++ 
++     def call(text)
++       m = REGEXP.match(text)
++       unless m
++         $log.debug "pattern not match: #{text}"
++         return nil, nil
++       end
++ 
++       host = m['host']
++       host = (host == '-') ? nil : host
++ 
++       user = m['user']
++       user = (user == '-') ? nil : user
++ 
++       time = m['time']
++       time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
++ 
++       method = m['method']
++       path = m['path']
++ 
++       code = m['code'].to_i
++       code = nil if code == 0
++ 
++       size = m['size']
++       size = (size == '-') ? nil : size.to_i
++ 
++       referer = m['referer']
++       referer = (referer == '-') ? nil : referer
++ 
++       agent = m['agent']
++       agent = (agent == '-') ? nil : agent
++ 
++       record = {
++         "host" => host,
++         "user" => user,
++         "method" => method,
++         "path" => path,
++         "code" => code,
++         "size" => size,
++         "referer" => referer,
++         "agent" => agent,
++       }
++ 
++       return time, record
++     end
++   end
++ 
+    TEMPLATES = {
++     'apache' => Proc.new { RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}) },
++     'apache2' => Proc.new { ApacheParser.new },
++     'syslog' => Proc.new { RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}) },
++     'json' => Proc.new { JSONParser.new },
+    }
+  
+    def self.register_template(name, regexp_or_proc, time_format=nil)
+      if regexp_or_proc.is_a?(Regexp)
++       pr = Proc.new { regexp_or_proc }
+      else
+        regexp = regexp_or_proc
++       pr = Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
+      end
+  
+      TEMPLATES[name] = pr

data/lib/fluent/plugin/buf_memory.rb

@@ -68,9 +68,9 @@ class MemoryBuffer < BasicBuffer
     super
   end
 
-  # overwrite default buffer_chunk_limit and buffer_queue_limit
-  config_set_default :buffer_chunk_limit, 32*1024*1024
-  config_set_default :buffer_queue_limit, 32
+  # Overwrite default BasicBuffer#buffer_queue_limit
+  # to limit total memory usage upto 512MB.
+  config_set_default :buffer_queue_limit, 64
 
   def configure(conf)
     super

data/lib/fluent/plugin/out_exec_filter.rb

@@ -221,13 +221,16 @@ class ExecFilterOutput < BufferedOutput
     end
 
     def start(command)
+      @command = command
       @io = IO.popen(command, "r+")
       @pid = @io.pid
       @io.sync = true
+      @finished = false
       @thread = Thread.new(&method(:run))
     end
 
     def shutdown
+      @finished = true
       begin
         Process.kill(:TERM, @pid)
       rescue Errno::ESRCH
@@ -259,10 +262,13 @@ class ExecFilterOutput < BufferedOutput
     def run
       @parser.call(@io)
     rescue
-      $log.error "exec_filter process exited", :error=>$!.to_s
+      $log.error "exec_filter thread unexpectedly failed with an error.", :command=>@command, :error=>$!.to_s
       $log.warn_backtrace $!.backtrace
     ensure
-      Process.waitpid(@pid)
+      pid, stat = Process.waitpid2(@pid)
+      unless @finished
+        $log.error "exec_filter process unexpectedly exited.", :command=>@command, :ecode=>stat.to_i
+      end
     end
   end
 

data/lib/fluent/plugin/out_file.rb

@@ -62,6 +62,9 @@ class FileOutput < TimeSlicedOutput
       conf['buffer_path'] ||= "#{@path}.*"
     end
 
+    # override default buffer_chunk_limit
+    conf['buffer_chunk_limit'] ||= '256m'
+
     super
 
     @timef = TimeFormatter.new(@time_format, @localtime)

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.22'
+VERSION = '0.10.23'
 
 end

data/test/plugin/out_exec_filter.rb

@@ -115,7 +115,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], emits[0]
 
     d = create_driver %[
-      command sed -l -e s/foo/bar/
+      command sed --unbuffered -l -e s/foo/bar/
       in_keys time,val1
       out_keys time,val2
       tag xxx
@@ -138,7 +138,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
 
   def test_emit_4
     d = create_driver(%[
-      command sed -l -e s/foo/bar/
+      command sed --unbuffered -l -e s/foo/bar/
       in_keys tag,time,val1
       remove_prefix input
       out_keys tag,time,val2

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.22
+  version: 0.10.23
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-02 00:00:00.000000000Z
+date: 2012-06-26 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70252881832620 !ruby/object:Gem::Requirement
+  requirement: &70230576091380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70252881832620
+  version_requirements: *70230576091380
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70252881832140 !ruby/object:Gem::Requirement
+  requirement: &70230576090780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70252881832140
+  version_requirements: *70230576090780
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70252881831560 !ruby/object:Gem::Requirement
+  requirement: &70230576090240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70252881831560
+  version_requirements: *70230576090240
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70252881831080 !ruby/object:Gem::Requirement
+  requirement: &70230576089620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70252881831080
+  version_requirements: *70230576089620
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70252881830600 !ruby/object:Gem::Requirement
+  requirement: &70230576088920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70252881830600
+  version_requirements: *70230576088920
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70252881830080 !ruby/object:Gem::Requirement
+  requirement: &70230576088280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70252881830080
+  version_requirements: *70230576088280
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &70252881829500 !ruby/object:Gem::Requirement
+  requirement: &70230576087680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70252881829500
+  version_requirements: *70230576087680
 - !ruby/object:Gem::Dependency
   name: timecop
-  requirement: &70252881828920 !ruby/object:Gem::Requirement
+  requirement: &70230576087040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: 0.3.0
   type: :development
   prerelease: false
-  version_requirements: *70252881828920
+  version_requirements: *70230576087040
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70252881828380 !ruby/object:Gem::Requirement
+  requirement: &70230576086400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70252881828380
+  version_requirements: *70230576086400
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -145,6 +145,8 @@ files:
 - lib/fluent/mixin.rb
 - lib/fluent/output.rb
 - lib/fluent/parser.rb
+- lib/fluent/parser.rb.orig
+- lib/fluent/parser.rb.rej
 - lib/fluent/plugin.rb
 - lib/fluent/plugin/buf_file.rb
 - lib/fluent/plugin/buf_memory.rb