fluentd 0.10.10 → 0.10.11

data/ChangeLog

@@ -1,4 +1,20 @@
 
+Release 0.10.11 - 2012/02/10
+
+* out_forward supports 'standby' parameter
+* out_forward handles 'hard_timeout' correctly
+
+
+Release 0.10.10 - 2012/02/09
+
+* in_forward and out_forward don't raise exceptions in callback handlers of
+  cool.io not to stop Loop#run
+* TimeSlicedOutput ignores time_slice_wait if flush_interval is specified
+* in_tail follows symbolic links correctly
+* in_http supports "Content-Type: application/json"
+* TestDriver#run sleeps 0.5 seconds for the out_exec_filter plugin
+
+
 Release 0.10.9 - 2012/01/19
 
 * Fixed TimeSlicedOutputTestDriver

data/VERSION

@@ -1 +1 @@
-0.10.10
+0.10.11

data/lib/fluent/plugin/out_forward.rb

@@ -66,15 +66,17 @@ class ForwardOutput < ObjectBufferedOutput
       weight = e['weight']
       weight = weight ? weight.to_i : 60
 
+      standby = !!e['standby']
+
       name = e['name']
       unless name
         name = "#{host}:#{port}"
       end
 
-      failure = FailureDetector.new(@heartbeat_interval.to_f/2, @hard_timeout)
+      failure = FailureDetector.new(@heartbeat_interval.to_f/2, @hard_timeout, Time.now.to_i.to_f)
       sockaddr = Socket.pack_sockaddr_in(port, host)
       port, host = Socket.unpack_sockaddr_in(sockaddr)
-      @nodes[sockaddr] = Node.new(name, host, port, weight, failure,
+      @nodes[sockaddr] = Node.new(name, host, port, weight, standby, failure,
                                   @phi_threshold, recover_sample_size)
       $log.info "adding forwarding server '#{name}'", :host=>host, :port=>port, :weight=>weight
     }
@@ -83,15 +85,8 @@ class ForwardOutput < ObjectBufferedOutput
   def start
     super
 
-    @weight_array = []
-    gcd = @nodes.values.map {|n| n.weight }.inject(0) {|r,w| r.gcd(w) }
-    @nodes.each_value {|n|
-      (n.weight / gcd).times {
-        @weight_array << n
-      }
-    }
-    @weight_array.sort_by { rand }
-
+    @rand_seed = Random.new.seed
+    rebuild_weight_array
     @rr = 0
 
     @loop = Coolio::Loop.new
@@ -145,6 +140,44 @@ class ForwardOutput < ObjectBufferedOutput
   end
 
   private
+  def rebuild_weight_array
+    standby_nodes, regular_nodes = @nodes.values.partition {|n|
+      n.standby?
+    }
+
+    lost_weight = 0
+    regular_nodes.each {|n|
+      unless n.available?
+        lost_weight += n.weight
+      end
+    }
+    $log.debug "rebuilding weight array", :lost_weight=>lost_weight
+
+    if lost_weight > 0
+      standby_nodes.each {|n|
+        if n.available?
+          regular_nodes << n
+          $log.info "using standby node #{n.host}:#{n.port}", :weight=>n.weight
+          lost_weight -= n.weight
+          break if lost_weight <= 0
+        end
+      }
+    end
+
+    weight_array = []
+    gcd = regular_nodes.map {|n| n.weight }.inject(0) {|r,w| r.gcd(w) }
+    regular_nodes.each {|n|
+      (n.weight / gcd).times {
+        weight_array << n
+      }
+    }
+
+    r = Random.new(@rand_seed)
+    weight_array.sort_by { r.rand }
+
+    @weight_array = weight_array
+  end
+
   # MessagePack FixArray length = 2
   FORWARD_HEADER = [0x92].pack('C')
 
@@ -204,8 +237,11 @@ class ForwardOutput < ObjectBufferedOutput
   def on_timer
     return if @finished
     @nodes.each_pair {|sockaddr,n|
-      n.tick
+      if n.tick
+        rebuild_weight_array
+      end
       begin
+        #$log.trace "sending heartbeat #{n.host}:#{n.port}"
         @usock.send "", 0, sockaddr
       rescue
         # TODO log
@@ -235,49 +271,75 @@ class ForwardOutput < ObjectBufferedOutput
   def on_heartbeat(sockaddr, msg)
     if node = @nodes[sockaddr]
       #$log.trace "heartbeat from '#{node.name}'", :host=>node.host, :port=>node.port
-      node.heartbeat
+      if node.heartbeat
+        rebuild_weight_array
+      end
     end
   end
 
   class Node
-    def initialize(name, host, port, weight, failure,
+    def initialize(name, host, port, weight, standby, failure,
                    phi_threshold, recover_sample_size)
       @name = name
       @host = host
       @port = port
       @weight = weight
+      @standby = standby
       @failure = failure
       @phi_threshold = phi_threshold
       @recover_sample_size = recover_sample_size
       @available = true
     end
 
-    attr_reader :name, :host, :port
-    attr_reader :weight
-    attr_writer :available
+    attr_reader :name, :host, :port, :weight
+    attr_writer :weight, :standby, :available
 
     def available?
       @available
     end
 
+    def standby?
+      @standby
+    end
+
     def tick
       now = Time.now.to_f
+      if !@available
+        if @failure.hard_timeout?(now)
+          @failure.clear
+        end
+        return nil
+      end
+
+      if @failure.hard_timeout?(now)
+        $log.info "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :hard_timeout=>true
+        @available = false
+        @failure.clear
+        return true
+      end
+
       phi = @failure.phi(now)
       #$log.trace "phi '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
       if phi > @phi_threshold
         $log.info "detached forwarding server '#{@name}'", :host=>@host, :port=>@port, :phi=>phi
         @available = false
         @failure.clear
+        return true
+      else
+        return false
       end
     end
 
     def heartbeat
       now = Time.now.to_f
       @failure.add(now)
-      if !@available && @failure.sample_size > @recover_sample_size &&
-            @failure.phi(now) <= @phi_threshold
+      #$log.trace "heartbeat from '#{@name}'", :host=>@host, :port=>@port, :available=>@available, :sample_size=>@failure.sample_size
+      if !@available && @failure.sample_size > @recover_sample_size
         $log.info "recovered forwarding server '#{@name}'", :host=>@host, :port=>@port
         @available = true
+        return true
+      else
+        return nil
       end
     end
 
@@ -290,27 +352,26 @@ class ForwardOutput < ObjectBufferedOutput
     PHI_FACTOR = 1.0 / Math.log(10.0)
     SAMPLE_SIZE = 1000
 
-    def initialize(init_int, hard_timeout)
-      @last = 0
+    def initialize(init_int, hard_timeout, init_last)
+      @last = init_last
       @init_int = init_int
       @hard_timeout = hard_timeout
-      @window = []
-      @failure = false
+      @window = [init_int]
+    end
+
+    def hard_timeout?(now)
+      now - @last > @hard_timeout
     end
 
     def add(now)
-      if @last > 0
-        int = now - @last
+      if @window.empty?
+        @window << @init_int
+        @last = now
       else
-        int = @init_int
-      end
-      if int <= @hard_timeout
+        int = now - @last
         @window << int
         @window.shift if @window.length > SAMPLE_SIZE
         @last = now
-        true
-      else
-        false
       end
     end
 

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.10'
+VERSION = '0.10.11'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.10
+  version: 0.10.11
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-09 00:00:00.000000000Z
+date: 2012-02-10 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70325962479480 !ruby/object:Gem::Requirement
+  requirement: &70203141220220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70325962479480
+  version_requirements: *70203141220220
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70325962478220 !ruby/object:Gem::Requirement
+  requirement: &70203141219740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70325962478220
+  version_requirements: *70203141219740
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70325962474920 !ruby/object:Gem::Requirement
+  requirement: &70203141219240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70325962474920
+  version_requirements: *70203141219240
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70325962473640 !ruby/object:Gem::Requirement
+  requirement: &70203141218740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70325962473640
+  version_requirements: *70203141218740
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70325962469580 !ruby/object:Gem::Requirement
+  requirement: &70203141218220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70325962469580
+  version_requirements: *70203141218220
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70325962465420 !ruby/object:Gem::Requirement
+  requirement: &70203141217440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70325962465420
+  version_requirements: *70203141217440
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -86,7 +86,6 @@ executables:
 extensions: []
 extra_rdoc_files:
 - ChangeLog
-- ChangeLog.cp
 - README
 - README.rdoc
 files:
@@ -155,7 +154,6 @@ files:
 - test/plugin/out_roundrobin.rb
 - test/plugin/out_stream.rb
 - ChangeLog
-- ChangeLog.cp
 - README
 - README.rdoc
 homepage: http://fluentd.org/

data/ChangeLog.cp

@@ -1,265 +0,0 @@
-
-Release 0.10.10 - 2012/01/19
-
-* in_forward and on_forward don't raise exceptions in callback handlers of
-  cool.io not to stop Loop#run
-* TimeSlicedOutput ignores time_slice_wait if flush_interval is specified
-* in_tail follows symbolic links correctly
-* in_http supports "Content-Type: application/json"
-* TestDriver#run sleeps 0.5 seconds for the out_exec_filter plugin
-
-
-Release 0.10.9 - 2012/01/19
-
-* Fixed TimeSlicedOutputTestDriver
-* Updated cool.io 1.0.0 -> 1.1.0
-* TextParser: fixed regexp of syslog parser to work with rsyslog on CentOS
-* out_exec_filter: improve performance by using buffering
-* out_exec_filter: added num_children parameter
-* out_exec_filter: added remove_prefix/add_prefix parameters
-* out_tail: show warning if pos_file parameter is not set
-* out_copy: fixed problems when event stream is not repeatable
-
-
-Release 0.10.8 - 2011/12/03
-
-* Added Supervisor: restart process on SIGHUP or unexpected end of process
-* Added -i commandline option which allows inline config
-* Added TimeSlicedOutputTestDriver
-* BufferedOutput outputs 'retry succeeded' message
-* Use Gem::Specification instead of Gem.searcher which is obsoleted
-* Added BasicBuffer#chunk_limit -> buffer_chunk_limit alias for backward
-  compatibility
-* buf_file: fixed to work with keys which contains '/'
-
-
-Release 0.10.7 - 2011/11/16
-
-* Supports multi-threaded on buffered output plugins ('num_threads')
-* Supports multi-process on input plugins ('detach_process')
-* Added ObjectBufferedOutput
-* Ensure to call 'shutdown' method of started plugins
-* in_tail supports 'pos_file' option that stores read position to a file
-
-
-Release 0.10.6 - 2011/11/11
-
-* Fixed --group NAME cmdline argument
-* Run configure(conf) after changing user and group
-* Fixed SIGHUP handling
-* BasicBuffer plugin uses Configurable module
-* buf_file uses microseconds + 12 bits random number for unique file name
-
-
-Release 0.10.5 - 2011/11/08
-
-* Added out_forward, in_forward
-* Added out_exec, in_exec
-* Added out_exec_filter
-
-
-Release 0.10.4 - 2011/11/07
-
-* TestDriver uses klass.dup.module_eval instead of inheriting class to
-  override methods
-* fixed encoding problem of String#[] in FileBufferChunk#<<(data)
-
-
-Release 0.10.3 - 2011/10/21
-
-  * Supports 'include' in config file
-    * Supports "http://" schema
-	* Supports wildcards
-
-
-Release 0.10.2 - 2011/10/18
-
-  * Fixed EventStream#to_msgpack_stream (thanks CkReal)
-  * Added gemspec.required_ruby_version = '~> 1.9.2' (thanks sakuro)
-
-
-Release 0.10.1 - 2011/10/16
-
-  * SetTimeKeyMixin accepts include_time_key parameter
-  * SetTagKeyMixin accepts include_tag_key parameter
-  * Fixed Makefile.am
-  * Fixed MemoryBufferChunk#msgpack_each
-
-
-Release 0.10.0 - 2011/10/16
-
-  * Removed Event class
-    * def format(tag, event) -> format(time, tag, record)
-	* def emit(tag, event) -> emit(tag, time, record)
-  * Added plugin APIs
-    * Added Configurable
-    * Added TimeFormatter
-    * Added SetTimeKeyMixin
-    * Added SetTagKeyMixin
-    * Added InputTestDriver
-  * Added Gemfile, gemspec and .rvmrc files
-
-
-Release 0.9.20 - 2011/10/07
-
-  * Config#has_key? sets used flag not to show warning
-  * in_tcp supports JSON
-  * Depends on Yajl
-  * Fixed fluentd -r option
-  * Fixed autogen.sh to create plugins directory
-
-
-Release 0.9.19 - 2011/10/07
-
-  * Secondary outputs work only when error count exceededs retry_limit
-  * Secondary outputs retry upto secondary_limit
-  * Added Output#secondary_init
-  * Added unit test driver for output plugins
-  * Added Fluent::Test.test? method
-  * Added comparision methods (<=>, ==, eql? and hash) to Event class
-  * Added test cases for out_copy, out_roundrobin and out_file
-
-
-Release 0.9.18 - 2011/10/04
-
-  * Performance improvements
-    * in_tcp uses lazy deserialization
-	* Engine caches results of Match#match
-	* out_file caches results of strftime
-  * Fixed TextParser.register_template
-
-
-Release 0.9.17 - 2011/10/03
-
-  * TimeSlicedOutput caches results of strftime() for performance
-  * Fixed TextParser.register_template (thanks eiichiroi)
-  * Fixed fluent-cat --port argument to work
-
-
-Release 0.9.16 - 2011/10/01
-
-  * Fixed it can't use relative path for --plugin command line argument
-  * Fixed out_copy to work with non-repeatable event streams
-
-
-Release 0.9.15 - 2011/09/21
-
-  * Set default encoding to ASCII-8BIT
-  * Fixed thread issues in BasicBuffer
-  * Performance improvements
-
-
-Release 0.9.14 - 2011/09/21
-
-  * Fixed to write unexpected error to log files
-  * Plugin loader uses GemSpec if it is available to load newer plugin
-  * Added Config.bool_value method for plugins
-
-
-Release 0.9.13 - 2011/09/21
-
-  * Updated out_file plugin
-    * Uses localtime by default
-    * Uses \t to separate time, tag and JSON.
-    * Uses ISO-8601 for the time format
-
-
-Release 0.9.12 - 2011/09/20
-
-  * Updated out_file plugin
-    * Supports log compression
-	* Slices log files every day by default
-
-
-Release 0.9.11 - 2011/09/19
-
-  * Added out_null plugin
-  * Added out_roundrobin plugin
-  * in_http supports keep-alive
-  * in_tcp and in_unix detect network failure
-
-
-Release 0.9.10 - 2011/09/07
-
-  * Periodic timer enqueues buffer chunks only when queue is empty
-
-
-Release 0.9.9 - 2011/09/06
-
-  * in_tail strips \n at the end of log line
-  * Uses newer one if multiple plugins are found
-
-
-Release 0.9.8 - 2011/08/16
-
-  * in_http plugin supports 'Expect: 100-continue' header
-  * Added --user and --group command line arguments to change privilege
-  * Changed default retry_limit parameter from 8 to 17
-  * Changed default buffer_chunk_limit parameter from 1m to 16m
-  * Changed default buffer_queue_limit parameter from 100 to 64
-
-
-Release 0.9.7 - 2011/08/06
-
-  * Load plugins named fluent_plugin.rb installed by RubyGems
-  * Load plugins located on fluent/plugin/{buf,in,out}_{type} installed by RubyGems
-  * Removed out_tdlog plugin to fluent-plugin-td gem
-
-
-Release 0.9.6 - 2011/08/06
-
-  * Force flush buffered events on SIGUSR1
-  * out_tdlog uses event tag for the database and table name
-  * Fluent can log Fluent logs
-  * Colorize Fluent logs on console
-  * Changed default unix domain socket path to /var/run/fluent/fluent.sock
-  * Makefile.am and configure.in aware fakeroot
-
-
-Release 0.9.5 - 2011/07/23
-
-  * Added <secondary> tag that fallbacks to secondary output when it can't
-    write to primary output.
-  * <match> pattern supports **; matches any fragments recursively.
-  * <match> pattern supports {x,y,...}; matches any of the inner patterns.
-
-
-Release 0.9.4 - 2011/07/22
-
-  * Switched EventMachine to Cool.io
-  * in_tail continues to follow file if it is deleted on Linux
-  * Added fluentd --setup option that installs sample configuration file
-  * Added out_tdlog plugin
-
-
-Release 0.9.3 - 2011/07/05
-
-  * Disable EventMachine.epoll because it doesn't work correctly on
-    Ruby 1.9 with threads
-  * in_tcp and in_unix uses EventMachine instead of threads
-  * Set BasicSocket.do_not_reverse_lookup = true
-
-
-Release 0.9.2 - 2011/07/03
-
-  * Added TimeSlicedOutput that splits buffer chunk exactly based on the time
-  * Changed structure of buffers from queue to map + queue
-  * Show waring if configuration parameters are not used
-  * Added out_time_file plugin
-
-
-Release 0.9.1 - 2011/06/26
-
-  * Added in_tail plugin
-  * Added in_syslog plugin
-  * Added out_stdout plugin
-  * Added out_tcp plugin
-  * Added out_unix plugin
-  * Improved performance of in_http plugin by replacing WEBrick + thread
-     with EventMachine
-
-
-Release 0.9.0 - 2011/06/20
-
-  * First release
-