fluentd 0.10.5 → 0.10.6

data/ChangeLog

@@ -1,4 +1,13 @@
 
+Release 0.10.6 - 2011/11/11
+
+* Fixed --group NAME cmdline argument
+* Run configure(conf) after changing user and group
+* Fixed SIGHUP handling
+* BasicBuffer plugin uses Configurable module
+* buf_file uses microseconds + 12 bits random number for unique file name
+
+
 Release 0.10.5 - 2011/11/08
 
 * Added out_forward, in_forward

data/README.rdoc

@@ -47,6 +47,7 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
   $ fluentd &
   $ echo '{"json":"message"}' | fluent-cat debug.test
 
+== Meta
 
 Web site::  http://fluentd.org/
 Documents:: http://fluentd.org/doc/
@@ -55,3 +56,10 @@ Author:: Sadayuki Furuhashi
 Copyright:: (c) 2011 FURUHASHI Sadayuki
 License:: Apache License, Version 2.0
 
+Contributed by:
+
+* Eiichiro Iwata
+* Masahiro Nakagawa
+* Takashi Nagayasu
+* Yuichi Tateno
+

data/VERSION

@@ -1 +1 @@
-0.10.5
+0.10.6

data/configure.in

@@ -1,6 +1,6 @@
 AC_INIT(lib/fluent/engine.rb)
 AC_CONFIG_AUX_DIR(ac)
-AM_INIT_AUTOMAKE(fluentd, 0.10.3)
+AM_INIT_AUTOMAKE(fluentd, 0.10.6)
 
 AC_CHECK_LIB(z,deflate,,
     AC_MSG_ERROR([zlib library not found]))

data/fluentd.gemspec

@@ -36,5 +36,5 @@ EOF
   gem.add_dependency "yajl-ruby", "~> 1.0.0"
   gem.add_dependency "cool.io", "~> 1.0.0"
   gem.add_dependency "http_parser.rb", "~> 0.5.1"
-  gem.add_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "rake", ">= 0.9.2"
 end

data/lib/fluent/buffer.rb

@@ -113,24 +113,14 @@ class BasicBuffer < Buffer
   include MonitorMixin
 
   def initialize
-    super()
-    @chunk_limit = 16*1024*1024  # TODO default
-    @queue_limit = 64 # TODO default
-    @parallel = false
+    super
   end
 
-  attr_accessor :queue_limit, :chunk_limit
+  config_param :buffer_chunk_limit, :size, :default => 256*1024*1024
+  config_param :buffer_queue_limit, :integer, :default => 128
 
   def configure(conf)
     super
-
-    if chunk_limit = conf['buffer_chunk_limit']
-      @chunk_limit = Config.size_value(chunk_limit)
-    end
-
-    if queue_limit = conf['buffer_queue_limit']
-      @queue_limit = queue_limit.to_i
-    end
   end
 
   def start
@@ -157,16 +147,16 @@ class BasicBuffer < Buffer
     synchronize do
       top = (@map[key] ||= new_chunk(key))  # TODO generate unique chunk id
 
-      if top.size + data.bytesize <= @chunk_limit
+      if top.size + data.bytesize <= @buffer_chunk_limit
         chain.next
         top << data
         return false
 
-      elsif data.bytesize > @chunk_limit
+      elsif data.bytesize > @buffer_chunk_limit
         # TODO
         raise BufferError, "received data too large"
 
-      elsif @queue.size >= @queue_limit
+      elsif @queue.size >= @buffer_queue_limit
         # TODO
         raise BufferError, "queue size exceeds limit"
       end

data/lib/fluent/command/fluentd.rb

@@ -134,31 +134,21 @@ require 'fluent/load'
 
 begin
   #
-  # initialize
+  # read config file
   #
-  Fluent::Engine.init
-
-  libs.each {|lib|
-    require lib
-  }
-
-  plugin_dirs.each {|dir|
-    if Dir.exist?(dir)
-      dir = File.expand_path(dir)
-      Fluent::Engine.load_plugin_dir(dir)
-    end
-  }
-
-  Fluent::Engine.read_config(config_path)
+  $log.info "reading config file", :path=>config_path
+  config_fname = File.basename(config_path)
+  config_basedir = File.dirname(config_path)
+  config_data = File.read(config_path)
 
 
   #
-  # daemonize
+  # change user
   #
   if chgroup
     chgid = chgroup.to_i
     if chgid.to_s != chgroup
-      chgid = `id -u #{chgroup}`.to_i
+      chgid = `id -g #{chgroup}`.to_i
       if $?.to_i != 0
         exit 1
       end
@@ -177,6 +167,29 @@ begin
     Process::UID.change_privilege(chuid)
   end
 
+
+  #
+  # run configure
+  #
+  Fluent::Engine.init
+
+  libs.each {|lib|
+    require lib
+  }
+
+  plugin_dirs.each {|dir|
+    if Dir.exist?(dir)
+      dir = File.expand_path(dir)
+      Fluent::Engine.load_plugin_dir(dir)
+    end
+  }
+
+  Fluent::Engine.parse_config(config_data, config_fname, config_basedir)
+
+
+  #
+  # daemonize
+  #
   trap :INT do
     Fluent::Engine.stop
   end
@@ -187,7 +200,7 @@ begin
 
   trap :HUP do
     if log_file
-      $log.reopen(log_file, "a")
+      log_out.reopen(log_file, "a")
     end
   end
 

data/lib/fluent/engine.rb

@@ -37,7 +37,13 @@ class EngineClass
 
   def read_config(path)
     $log.info "reading config file", :path=>path
-    conf = Config.read(path)
+    File.open(path) {|io|
+      parse_config(io, File.basename(path), File.dirname(path))
+    }
+  end
+
+  def parse_config(io, fname, basepath=Dir.pwd)
+    conf = Config.parse(io, fname, basepath)
     configure(conf)
     conf.check_not_fetched {|key,e|
       $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."

data/lib/fluent/output.rb

@@ -236,28 +236,34 @@ class OutputThread
         next if has_next
 
       rescue
-        $log.warn "failed to flush the buffer", :error=>$!.to_s
-        $log.warn_backtrace
-
+        e = $!
         error_count = @error_history.size
         @error_history << time
 
         if error_count < @retry_limit
-          $log.info "retrying."
+          $log.warn "failed to flush the buffer, retrying.", :error=>e.to_s
+          $log.warn_backtrace e.backtrace
 
         elsif @secondary
           if error_count == @retry_limit
+            $log.warn "failed to flush the buffer.", :error=>e.to_s
             $log.warn "retry count exceededs limit. falling back to secondary output."
+            $log.warn_backtrace e.backtrace
             next  # retry immediately
           elsif error_count <= @retry_limit + @secondary_limit
-            $log.info "retrying secondary."
+            $log.warn "failed to flush the buffer, retrying secondary.", :error=>e.to_s
+            $log.warn_backtrace e.backtrace
           else
+            $log.warn "failed to flush the buffer.", :error=>e.to_s
             $log.warn "secondary retry count exceededs limit."
+            $log.warn_backtrace e.backtrace
             write_abort
           end
 
         else
+          $log.warn "failed to flush the buffer.", :error=>e.to_s
           $log.warn "retry count exceededs limit."
+          $log.warn_backtrace e.backtrace
           write_abort
         end
 
@@ -367,12 +373,12 @@ class TimeSlicedOutput < BufferedOutput
     super
     @localtime = true
     #@ignore_old = false   # TODO
-    # TODO @flush_interval = 60  # overwrite default flush_interval
   end
 
   config_param :time_slice_format, :string, :default => '%Y%m%d'
-  config_param :time_slice_wait, :time, :default => 10*60  # TODO default value
+  config_param :time_slice_wait, :time, :default => 10*60
   config_set_default :buffer_type, 'file'  # overwrite default buffer_type
+  # config_set_default :flush_interval, 60  # TODO overwrite default flush_interval
 
   attr_accessor :localtime
 
@@ -386,14 +392,6 @@ class TimeSlicedOutput < BufferedOutput
       @localtime = true
     end
 
-    if time_slice_format = conf['time_slice_format']
-      @time_slice_format = time_slice_format
-    end
-
-    if time_slice_wait = conf['time_slice_wait']
-      @time_slice_wait = Config.time_value(time_slice_wait)
-    end
-
     if @localtime
       @time_slicer = Proc.new {|time|
         Time.at(time).strftime(@time_slice_format)

data/lib/fluent/plugin/buf_file.rb

@@ -98,12 +98,11 @@ class FileBuffer < BasicBuffer
     super
   end
 
-  PATH_MATCH = /^(.*)[\._](b|q)([0-9a-fA-F]{1,16})$/
+  PATH_MATCH = /^(.*)[\._](b|q)([0-9a-fA-F]{1,32})$/
 
   def new_chunk(key)
-    tsuffix = Engine.now.to_i.to_s(16)
     encoded_key = encode_key(key)
-    path = "#{@buffer_path_prefix}#{encoded_key}.b#{tsuffix}#{@buffer_path_suffix}"
+    path = make_path(encoded_key, "b")
     FileBufferChunk.new(key, path)
   end
 
@@ -150,10 +149,9 @@ class FileBuffer < BasicBuffer
 
     m = PATH_MATCH.match(mp)
     encoded_key = m ? m[1] : ""
-    tsuffix = Engine.now.to_i.to_s(16)
+    npath = make_path(encoded_key, "q")
 
-    path = "#{@buffer_path_prefix}#{encoded_key}.q#{tsuffix}#{@buffer_path_suffix}"
-    chunk.mv(path)
+    chunk.mv(npath)
   end
 
   protected
@@ -164,6 +162,12 @@ class FileBuffer < BasicBuffer
   def decode_key(encoded_key)
 		URI.decode(encoded_key, /[^-_.a-zA-Z0-9]/n)
   end
+
+  def make_path(encoded_key, bq)
+    now = Time.now.utc
+    tsuffix = ((now.to_i*1000*1000+now.usec) << 12 | rand(0xfff)).to_s(16)
+    "#{@buffer_path_prefix}#{encoded_key}.#{bq}#{tsuffix}#{@buffer_path_suffix}"
+  end
 end
 
 

data/lib/fluent/plugin/buf_memory.rb

@@ -66,6 +66,10 @@ class MemoryBuffer < BasicBuffer
     super
   end
 
+  # overwrite default buffer_chunk_limit and buffer_queue_limit
+  config_set_default :buffer_chunk_limit, 32*1024*1024
+  config_set_default :buffer_queue_limit, 32
+
   def configure(conf)
     super
   end

data/lib/fluent/plugin/in_forward.rb

@@ -189,7 +189,7 @@ class ForwardInput < Input
   end
 
   def on_heartbeat_request(host, port, msg)
-    $log.trace "heartbeat request from #{host}:#{port}"
+    #$log.trace "heartbeat request from #{host}:#{port}"
     @usock.send "", 0, host, port
   end
 end

data/lib/fluent/plugin/out_forward.rb

@@ -224,7 +224,7 @@ class ForwardOutput < BufferedOutput
 
   def on_heartbeat(sockaddr, msg)
     if node = @nodes[sockaddr]
-      $log.trace "heartbeat from '#{node.name}'", :host=>node.host, :port=>node.port
+      #$log.trace "heartbeat from '#{node.name}'", :host=>node.host, :port=>node.port
       node.heartbeat
     end
   end
@@ -253,7 +253,7 @@ class ForwardOutput < BufferedOutput
     def tick
       now = Time.now.to_f
       phi = @failure.phi(now)
-      $log.trace "phi '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
+      #$log.trace "phi '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
       if phi > @phi_threshold
         $log.info "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
         @available = false

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.5'
+VERSION = '0.10.6'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.5
+  version: 0.10.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-08 00:00:00.000000000Z
+date: 2011-11-10 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70344270797220 !ruby/object:Gem::Requirement
+  requirement: &70241926265840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70344270797220
+  version_requirements: *70241926265840
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70344270796760 !ruby/object:Gem::Requirement
+  requirement: &70241926265340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70344270796760
+  version_requirements: *70241926265340
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70344270796240 !ruby/object:Gem::Requirement
+  requirement: &70241926264760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70344270796240
+  version_requirements: *70241926264760
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70344270795440 !ruby/object:Gem::Requirement
+  requirement: &70241926264180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70344270795440
+  version_requirements: *70241926264180
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70344270794640 !ruby/object:Gem::Requirement
+  requirement: &70241926263580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,18 +65,18 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70344270794640
+  version_requirements: *70241926263580
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70344270793480 !ruby/object:Gem::Requirement
+  requirement: &70241926262980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
-  type: :runtime
+  type: :development
   prerelease: false
-  version_requirements: *70344270793480
+  version_requirements: *70241926262980
 description: Fluent event collector
 email: frsyuki@gmail.com
 executables:
@@ -174,6 +174,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -247010597989313430
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10