fluentd 0.10.12 → 0.10.13

data/ChangeLog

@@ -1,10 +1,15 @@
 
+Release 0.10.13 - 2012/02/21
+
+* Rewrote in_tail
+* Fixed SIGUSR1 handler to force flush logs
+
+
 Release 0.10.12 - 2012/02/13
 
 * Engine shows warnings when emitted record doesn't match any outputs
 * in_tail is rewritten to follow symbolic links correctly
 * out_forward uses independent default value as 'hard_timeout' parameter
-* 
 
 
 Release 0.10.11 - 2012/02/10

data/VERSION

@@ -1 +1 @@
-0.10.12
+0.10.13

data/lib/fluent/engine.rb

@@ -191,7 +191,7 @@ class EngineClass
           m = m.output
         end
         if m.is_a?(BufferedOutput)
-          m.try_flush
+          m.force_flush
         elsif m.is_a?(MultiOutput)
           flush_recursive(m.outputs)
         end

data/lib/fluent/output.rb

@@ -361,6 +361,11 @@ class BufferedOutput < Output
     end
   end
 
+  def force_flush
+    enqueue_buffer
+    submit_flush
+  end
+
   def before_shutdown
     begin
       @buffer.before_shutdown(self)

data/lib/fluent/plugin/in_tail.rb

@@ -58,15 +58,20 @@ class TailInput < Input
 
   def start
     @loop = Coolio::Loop.new
-    @tailers = @paths.map {|path|
+    @tails = @paths.map {|path|
       pe = @pf ? @pf[path] : NullPositionEntry.instance
-      Tailer.new(@loop, path, @rotate_wait, pe, method(:receive_lines))
+      TailWatcher.new(path, @rotate_wait, pe, &method(:receive_lines))
+    }
+    @tails.each {|tail|
+      tail.attach(@loop)
     }
     @thread = Thread.new(&method(:run))
   end
 
   def shutdown
-    @loop.watchers.each {|w| w.detach }
+    @tails.each {|tail|
+      tail.close
+    }
     @loop.stop
     @thread.join
     @pf_file.close if @pf_file
@@ -103,229 +108,245 @@ class TailInput < Input
     return @parser.parse(line)
   end
 
-  class Tailer
-    def initialize(loop, path, rotate_wait, pe, receive_lines)
-      @loop = loop
+  class TailWatcher
+    def initialize(path, rotate_wait, pe, &receive_lines)
       @path = path
       @rotate_wait = rotate_wait
-      @pe = pe
+      @pe = pe || NullPositionEntry.instance
       @receive_lines = receive_lines
 
       @rotate_queue = []
-      @rotate_timer = nil
-      @io_handler = nil
 
-      @rh = RotateHandler.new(path, method(:rotate))
-      @rh.check  # invoke rotate
-      @rh.on_rotate = method(:on_rotate)
-      @rh.attach(@loop)
-    end
+      @timer_trigger = TimerWatcher.new(1, true, &method(:on_notify))
+      @stat_trigger = StatWatcher.new(path, &method(:on_notify))
 
-    def on_rotate(io)
-      return if @rotate_queue.include?(io)
-      $log.info "detected rotation of #{@path}; waiting #{@rotate_wait} seconds"
-      @rotate_queue.push(io)
-
-      # start rotate_timer
-      unless @rotate_timer
-        @rotate_timer = RotateTimer.new(@rotate_wait, method(:on_rotate_timer))
-        @rotate_timer.attach(@loop)
-      end
+      @rotate_handler = RotateHandler.new(path, &method(:on_rotate))
+      @io_handler = nil
     end
 
-    def on_rotate_timer
-      io = @rotate_queue.first
-      rotate(io, 0)
+    def attach(loop)
+      @timer_trigger.attach(loop)
+      @stat_trigger.attach(loop)
+      on_notify
     end
 
-    def rotate(io, start_pos=nil)
-      # start_pos is nil if first
-      io_handler = IOHandler.new(io, start_pos, @pe, @receive_lines)
-
-      if @io_handler
-        @io_handler.close
-        @io_handler = nil
-      end
-      io_handler.attach(@loop)
-      @io_handler = io_handler
-      @rotate_queue.shift
-
-      if @rotate_queue.empty?
-        @rotate_timer.detach if @rotate_timer
-        @rotate_timer = nil
-      end
+    def detach
+      @timer_trigger.detach if @timer_trigger.attached?
+      @stat_trigger.detach if @stat_trigger.attached?
     end
 
-    def shutdown
-      @rotate_queue.reject! {|io|
-        io.close
+    def close
+      @rotate_queue.reject! {|req|
+        req.io.close
         true
       }
-      if @io_handler
-        @io_handler.close
-        @io_handler = nil
-      end
-      if @rotate_timer
-        @rotate_timer.detach
-        @rotate_timer = nil
-      end
+      detach
     end
-  end
 
-  class RotateHandler
-    def initialize(path, on_rotate)
-      @path = path
-      @inode = nil
-      @fsize = 0
-      @on_rotate = on_rotate
-      @path = path
-      @stat_watcher = Stat.new(self, @path)
-      @timer_watcher = Timer.new(self, 1)
-    end
+    def on_notify
+      @rotate_handler.on_notify
+      return unless @io_handler
+      @io_handler.on_notify
 
-    attr_accessor :on_rotate
+      # proceeds rotate queue
+      return if @rotate_queue.empty?
+      @rotate_queue.first.tick
 
-    def check
-      begin
-        io = File.open(@path)
-      rescue Errno::ENOENT
-        # moved or deleted
-        @inode = nil
-        @fsize = 0
-        return
+      while @rotate_queue.first.ready?
+        if io = @rotate_queue.first.io
+          io_handler = IOHandler.new(io, @pe, &@receive_lines)
+        else
+          io_handler = NullIOHandler.new
+        end
+        @io_handler.close
+        @io_handler = io_handler
+        @rotate_queue.shift
+        break if @rotate_queue.empty?
       end
+    end
 
-      begin
-        stat = io.stat
-        inode = stat.ino
-        fsize = stat.size
-
-        if @inode != inode || fsize < @fsize
-          # rotated or truncated
-          @on_rotate.call(io)
-          io = nil
+    def on_rotate(io)
+      if @io_handler == nil
+        if io
+          # first time
+          stat = io.stat
+          fsize = stat.size
+          inode = stat.ino
+          if inode == @pe.read_inode
+            # seek to the saved position
+            pos = @pe.read_pos
+          else
+            # seek to the end of the file.
+            # logs never duplicate but may be lost if fluentd is down.
+            pos = fsize
+            @pe.update(inode, pos)
+          end
+          io.seek(pos)
+
+          @io_handler = IOHandler.new(io, @pe, &@receive_lines)
+        else
+          @io_handler = NullIOHandler.new
         end
 
-        @inode = inode
-        @fsize = fsize
-      ensure
-        io.close if io
+      else
+        if io && @rotate_queue.find {|req| req.io == io }
+          return
+        end
+        last_io = @rotate_queue.empty? ? @io_handler.io : @rotate_queue.last.io
+        if last_io == nil
+          $log.info "detected rotation of #{@path}"
+          # rotate imeediately if previous file is nil
+          wait = 0
+        else
+          $log.info "detected rotation of #{@path}; waiting #{@rotate_wait} seconds"
+          wait = @rotate_wait
+          wait -= @rotate_queue.first.wait unless @rotate_queue.empty?
+        end
+        @rotate_queue << RotationRequest.new(io, wait)
       end
-
-    rescue
-      $log.error $!.to_s
-      $log.error_backtrace
-    end
-
-    def attach(loop)
-      @stat_watcher.attach(loop)
-      @timer_watcher.attach(loop)
     end
 
-    def detach
-      @stat_watcher.detach
-      @timer_watcher.detach
-    end
+    class TimerWatcher < Coolio::TimerWatcher
+      def initialize(interval, repeat, &callback)
+        @callback = callback
+        super(interval, repeat)
+      end
 
-    def attached?
-      @stat_watcher.attached?
+      def on_timer
+        @callback.call
+      rescue
+        # TODO log?
+        $log.error $!.to_s
+        $log.error_backtrace
+      end
     end
 
-    class Stat < Coolio::StatWatcher
-      def initialize(h, path)
-        @h = h
+    class StatWatcher < Coolio::StatWatcher
+      def initialize(path, &callback)
+        @callback = callback
         super(path)
       end
 
       def on_change(prev, cur)
-        @h.check
+        @callback.call
+      rescue
+        # TODO log?
+        $log.error $!.to_s
+        $log.error_backtrace
       end
     end
 
-    class Timer < Coolio::TimerWatcher
-      def initialize(h, interval)
-        @h = h
-        super(interval, true)
+    class RotationRequest
+      def initialize(io, wait)
+        @io = io
+        @wait = wait
       end
 
-      def on_timer
-        @h.check
+      attr_reader :io
+
+      def tick
+        @wait -= 1
       end
-    end
-  end
 
-  class RotateTimer < Coolio::TimerWatcher
-    def initialize(interval, callback)
-      super(interval, true)
-      @callback = callback
+      def ready?
+        @wait <= 0
+      end
     end
 
-    def on_timer
-      @callback.call
-    rescue
-      # TODO log?
-    end
-  end
+    class IOHandler
+      def initialize(io, pe, &receive_lines)
+        $log.info "following tail of #{io.path}"
+        @io = io
+        @pe = pe
+        @receive_lines = receive_lines
+        @buffer = ''
+      end
 
-  class IOHandler < Coolio::IOWatcher
-    def initialize(io, start_pos, pe, receive_lines)
-      $log.info "following tail of #{io.path}"
-      @io = io
-      @pe = pe
-      @receive_lines = receive_lines
+      attr_reader :io
 
-      if start_pos
-        # rotated
-        @pos = start_pos
+      def on_notify
+        lines = []
 
-      else
-        # first time
-        stat = io.stat
-        fsize = stat.size
-        inode = stat.ino
-        if inode == @pe.read_inode
-          # seek to the saved position
-          @pos = @pe.read_pos
-        else
-          # seek to the end of the file.
-          # logs never duplicate but may be lost if fluentd is down.
-          @pos = fsize
-          @pe.update(inode, @pos)
+        while line = @io.gets
+          @buffer << line
+          @pos = @io.pos
+          unless @buffer[@buffer.length-1] == ?\n
+            break
+          end
+          lines << line
         end
-      end
 
-      io.seek(@pos)
+        unless lines.empty?
+          @pe.update_pos(@pos)
+          @receive_lines.call(lines)
+        end
+      rescue
+        $log.error $!.to_s
+        $log.error_backtrace
+        close
+      end
 
-      @buffer = ''
-      super(io)
+      def close
+        @io.close unless @io.closed?
+      end
     end
 
-    def on_readable
-      lines = []
+    class NullIOHandler
+      def initialize
+      end
 
-      while line = @io.gets
-        @buffer << line
-        @pos = @io.pos
-        unless @buffer[@buffer.length-1] == ?\n
-          break
-        end
-        lines << line
+      def io
       end
 
-      @pe.update_pos(@pos)
-      @receive_lines.call(lines) unless lines.empty?
-    rescue
-      $log.error $!.to_s
-      $log.error_backtrace
-      close
+      def on_notify
+      end
+
+      def close
+      end
     end
 
-    def close
-      detach if attached?
-      @io.close unless @io.closed?
+    class RotateHandler
+      def initialize(path, &on_rotate)
+        @path = path
+        @inode = nil
+        @fsize = -1  # first
+        @on_rotate = on_rotate
+        @path = path
+      end
+
+      def on_notify
+        begin
+          io = File.open(@path)
+          stat = io.stat
+          inode = stat.ino
+          fsize = stat.size
+        rescue Errno::ENOENT
+          # moved or deleted
+          inode = nil
+          fsize = 0
+        end
+
+        begin
+          if @inode != inode || fsize < @fsize
+            # rotated or truncated
+            @on_rotate.call(io)
+            io = nil
+          end
+
+          @inode = inode
+          @fsize = fsize
+        ensure
+          io.close if io
+        end
+
+      rescue
+        $log.error $!.to_s
+        $log.error_backtrace
+      end
     end
   end
 
+
   class PositionFile
     def initialize(file, map, last_pos)
       @file = file

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.12'
+VERSION = '0.10.13'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.12
+  version: 0.10.13
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-14 00:00:00.000000000Z
+date: 2012-02-22 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70256836669480 !ruby/object:Gem::Requirement
+  requirement: &70115974571520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70256836669480
+  version_requirements: *70115974571520
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70256836648880 !ruby/object:Gem::Requirement
+  requirement: &70115974563160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70256836648880
+  version_requirements: *70115974563160
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70256836646880 !ruby/object:Gem::Requirement
+  requirement: &70115974560860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70256836646880
+  version_requirements: *70115974560860
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70256836645660 !ruby/object:Gem::Requirement
+  requirement: &70115974558780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70256836645660
+  version_requirements: *70115974558780
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70256836644620 !ruby/object:Gem::Requirement
+  requirement: &70115974553640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70256836644620
+  version_requirements: *70115974553640
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70256836643580 !ruby/object:Gem::Requirement
+  requirement: &70115974549200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70256836643580
+  version_requirements: *70115974549200
 description: 
 email: frsyuki@gmail.com
 executables: