fluentd 0.10.11 → 0.10.12

data/ChangeLog

@@ -1,4 +1,12 @@
 
+Release 0.10.12 - 2012/02/13
+
+* Engine shows warnings when emitted record doesn't match any outputs
+* in_tail is rewritten to follow symbolic links correctly
+* out_forward uses independent default value as 'hard_timeout' parameter
+* 
+
+
 Release 0.10.11 - 2012/02/10
 
 * out_forward supports 'standby' parameter

data/README.rdoc

@@ -71,6 +71,7 @@ Patches contributed by:
 * {Hiro Yoshikawa}[https://github.com/hiroyoshikawa]
 * {Masahiro Nakagawa}[https://github.com/repeatedly]
 * {Nathan Parry}[https://github.com/nparry]
+* {Nobuyuki Kubota}[https://github.com/nobu-k]
 * {Sakuro Ozawa}[https://github.com/sakuro]
 * {Satoshi Tagomori}[https://github.com/tagomoris]
 * {Takashi Nagayasu}[https://github.com/sumipan]

data/VERSION

@@ -1 +1 @@
-0.10.11
+0.10.12

data/lib/fluent/engine.rb

@@ -203,8 +203,25 @@ class EngineClass
   end
 
   class NoMatchMatch
+    def initialize
+      @count = 0
+    end
+
     def emit(tag, es)
-      $log.on_trace { $log.trace "no pattern matched", :tag=>tag }
+      # TODO use time instead of num of records
+      c = (@count += 1)
+      if c < 512
+        if Math.log(c) / Math.log(2) % 1.0 == 0
+          $log.warn "no patterns matched", :tag=>tag
+          return
+        end
+      else
+        if c % 512 == 0
+          $log.warn "no patterns matched", :tag=>tag
+          return
+        end
+      end
+      $log.on_trace { $log.trace "no patterns matched", :tag=>tag }
     end
 
     def start

data/lib/fluent/plugin/in_exec.rb

@@ -122,7 +122,7 @@ class ExecInput < Input
         Engine.emit(tag, time, record)
       end
     rescue
-      $log.error "exec failed to emit", :error=>$!, :line=>line
+      $log.error "exec failed to emit", :error=>$!.to_s, :line=>line
       $log.warn_backtrace $!.backtrace
     end
   end

data/lib/fluent/plugin/in_tail.rb

@@ -28,6 +28,7 @@ class TailInput < Input
 
   config_param :path, :string
   config_param :tag, :string
+  config_param :rotate_wait, :time, :default => 5
   config_param :pos_file, :string, :default => nil
 
   def configure(conf)
@@ -57,14 +58,9 @@ class TailInput < Input
 
   def start
     @loop = Coolio::Loop.new
-    handlers = @paths.map {|path|
-      $log.debug "following tail of #{path}"
+    @tailers = @paths.map {|path|
       pe = @pf ? @pf[path] : NullPositionEntry.instance
-      h = Handler.new(path, pe, method(:receive_lines))
-      @loop.attach h
-    }
-    handlers.each {|h|
-      h.on_change(nil, nil)  # initialize call  # TODO prev, cur
+      Tailer.new(@loop, path, @rotate_wait, pe, method(:receive_lines))
     }
     @thread = Thread.new(&method(:run))
   end
@@ -107,114 +103,226 @@ class TailInput < Input
     return @parser.parse(line)
   end
 
-  class Handler < Coolio::StatWatcher
-    def initialize(path, pe, callback)
-      stat = File.stat(path)
+  class Tailer
+    def initialize(loop, path, rotate_wait, pe, receive_lines)
+      @loop = loop
+      @path = path
+      @rotate_wait = rotate_wait
       @pe = pe
-      @inode = stat.ino
-      if @inode == @pe.read_inode
-        # seek to the saved position
-        @pos = @pe.read_pos
-      else
-        # seek to the end of file first.
-        # logs never duplicate but may be lost if fluent is down.
-        @pos = stat.size
-        @pe.update(@inode, @pos)
+      @receive_lines = receive_lines
+
+      @rotate_queue = []
+      @rotate_timer = nil
+      @io_handler = nil
+
+      @rh = RotateHandler.new(path, method(:rotate))
+      @rh.check  # invoke rotate
+      @rh.on_rotate = method(:on_rotate)
+      @rh.attach(@loop)
+    end
+
+    def on_rotate(io)
+      return if @rotate_queue.include?(io)
+      $log.info "detected rotation of #{@path}; waiting #{@rotate_wait} seconds"
+      @rotate_queue.push(io)
+
+      # start rotate_timer
+      unless @rotate_timer
+        @rotate_timer = RotateTimer.new(@rotate_wait, method(:on_rotate_timer))
+        @rotate_timer.attach(@loop)
       end
-      @buffer = ''
-      @callback = callback
-      super(path)
     end
 
-    def on_change(prev, cur)
-      lines = []
-      inode = nil
+    def on_rotate_timer
+      io = @rotate_queue.first
+      rotate(io, 0)
+    end
+
+    def rotate(io, start_pos=nil)
+      # start_pos is nil if first
+      io_handler = IOHandler.new(io, start_pos, @pe, @receive_lines)
+
+      if @io_handler
+        @io_handler.close
+        @io_handler = nil
+      end
+      io_handler.attach(@loop)
+      @io_handler = io_handler
+      @rotate_queue.shift
+
+      if @rotate_queue.empty?
+        @rotate_timer.detach if @rotate_timer
+        @rotate_timer = nil
+      end
+    end
 
-      File.open(path) {|f|
-        stat = f.stat
+    def shutdown
+      @rotate_queue.reject! {|io|
+        io.close
+        true
+      }
+      if @io_handler
+        @io_handler.close
+        @io_handler = nil
+      end
+      if @rotate_timer
+        @rotate_timer.detach
+        @rotate_timer = nil
+      end
+    end
+  end
+
+  class RotateHandler
+    def initialize(path, on_rotate)
+      @path = path
+      @inode = nil
+      @fsize = 0
+      @on_rotate = on_rotate
+      @path = path
+      @stat_watcher = Stat.new(self, @path)
+      @timer_watcher = Timer.new(self, 1)
+    end
+
+    attr_accessor :on_rotate
+
+    def check
+      begin
+        io = File.open(@path)
+      rescue Errno::ENOENT
+        # moved or deleted
+        @inode = nil
+        @fsize = 0
+        return
+      end
+
+      begin
+        stat = io.stat
         inode = stat.ino
+        fsize = stat.size
 
-        if @inode != inode || stat.size < @pos
-          # moved or deleted
-          @pos = 0
-        else
-          f.seek(@pos)
+        if @inode != inode || fsize < @fsize
+          # rotated or truncated
+          @on_rotate.call(io)
+          io = nil
         end
 
-        line = f.gets
-        unless line
-          break
-        end
+        @inode = inode
+        @fsize = fsize
+      ensure
+        io.close if io
+      end
 
-        @buffer << line
-        unless line[line.length-1] == ?\n
-          @pos = f.pos
-          break
-        end
+    rescue
+      $log.error $!.to_s
+      $log.error_backtrace
+    end
 
-        lines << @buffer
-        @buffer = ''
+    def attach(loop)
+      @stat_watcher.attach(loop)
+      @timer_watcher.attach(loop)
+    end
 
-        while line = f.gets
-          unless line[line.length-1] == ?\n
-            @buffer = line
-            break
-          end
-          lines << line
-        end
+    def detach
+      @stat_watcher.detach
+      @timer_watcher.detach
+    end
 
-        @pos = f.pos
-      }
+    def attached?
+      @stat_watcher.attached?
+    end
 
-      if @inode != inode
-        @pe.update(inode, @pos)
-        @inode = inode
-      else
-        @pe.update_pos(@pos)
+    class Stat < Coolio::StatWatcher
+      def initialize(h, path)
+        @h = h
+        super(path)
       end
 
-      @callback.call(lines) unless lines.empty?
+      def on_change(prev, cur)
+        @h.check
+      end
+    end
+
+    class Timer < Coolio::TimerWatcher
+      def initialize(h, interval)
+        @h = h
+        super(interval, true)
+      end
 
-    rescue Errno::ENOENT
-      # moved or deleted
-      @pos = 0
-    # TODO rescue
+      def on_timer
+        @h.check
+      end
     end
   end
 
-  # pos               inode
-  # ffffffffffffffff\tffffffff\n
-  class PositionEntry
-    POS_SIZE = 16
-    INO_OFFSET = 17
-    INO_SIZE = 8
-    LN_OFFSET = 25
-    SIZE = 26
-
-    def initialize(file, seek)
-      @file = file
-      @seek = seek
+  class RotateTimer < Coolio::TimerWatcher
+    def initialize(interval, callback)
+      super(interval, true)
+      @callback = callback
     end
 
-    def update(ino, pos)
-      @file.pos = @seek
-      @file.write "%016x\t%08x" % [pos, ino]
-      @inode = ino
+    def on_timer
+      @callback.call
+    rescue
+      # TODO log?
     end
+  end
 
-    def update_pos(pos)
-      @file.pos = @seek
-      @file.write "%016x" % pos
+  class IOHandler < Coolio::IOWatcher
+    def initialize(io, start_pos, pe, receive_lines)
+      $log.info "following tail of #{io.path}"
+      @io = io
+      @pe = pe
+      @receive_lines = receive_lines
+
+      if start_pos
+        # rotated
+        @pos = start_pos
+
+      else
+        # first time
+        stat = io.stat
+        fsize = stat.size
+        inode = stat.ino
+        if inode == @pe.read_inode
+          # seek to the saved position
+          @pos = @pe.read_pos
+        else
+          # seek to the end of the file.
+          # logs never duplicate but may be lost if fluentd is down.
+          @pos = fsize
+          @pe.update(inode, @pos)
+        end
+      end
+
+      io.seek(@pos)
+
+      @buffer = ''
+      super(io)
     end
 
-    def read_inode
-      @file.pos = @seek + INO_OFFSET
-      @file.read(8).to_i(16)
+    def on_readable
+      lines = []
+
+      while line = @io.gets
+        @buffer << line
+        @pos = @io.pos
+        unless @buffer[@buffer.length-1] == ?\n
+          break
+        end
+        lines << line
+      end
+
+      @pe.update_pos(@pos)
+      @receive_lines.call(lines) unless lines.empty?
+    rescue
+      $log.error $!.to_s
+      $log.error_backtrace
+      close
     end
 
-    def read_pos
-      @file.pos = @seek
-      @file.read(16).to_i(16)
+    def close
+      detach if attached?
+      @io.close unless @io.closed?
     end
   end
 
@@ -256,6 +364,42 @@ class TailInput < Input
     end
   end
 
+  # pos               inode
+  # ffffffffffffffff\tffffffff\n
+  class PositionEntry
+    POS_SIZE = 16
+    INO_OFFSET = 17
+    INO_SIZE = 8
+    LN_OFFSET = 25
+    SIZE = 26
+
+    def initialize(file, seek)
+      @file = file
+      @seek = seek
+    end
+
+    def update(ino, pos)
+      @file.pos = @seek
+      @file.write "%016x\t%08x" % [pos, ino]
+      @inode = ino
+    end
+
+    def update_pos(pos)
+      @file.pos = @seek
+      @file.write "%016x" % pos
+    end
+
+    def read_inode
+      @file.pos = @seek + INO_OFFSET
+      @file.read(8).to_i(16)
+    end
+
+    def read_pos
+      @file.pos = @seek
+      @file.read(16).to_i(16)
+    end
+  end
+
   class NullPositionEntry
     require 'singleton'
     include Singleton

data/lib/fluent/plugin/out_exec_filter.rb

@@ -167,7 +167,7 @@ class ExecFilterOutput < BufferedOutput
       Engine.emit(tag, time, record)
     end
   rescue
-    $log.error "exec_filter failed to emit", :error=>$!, :line=>line
+    $log.error "exec_filter failed to emit", :error=>$!.to_s, :line=>line
     $log.warn_backtrace $!.backtrace
   end
 
@@ -217,7 +217,7 @@ class ExecFilterOutput < BufferedOutput
     def run
       @io.each_line(&@each_line)
     rescue
-      $log.error "exec_filter process exited", :error=>$!
+      $log.error "exec_filter process exited", :error=>$!.to_s
       $log.warn_backtrace $!.backtrace
     ensure
       Process.waitpid(@pid)

data/lib/fluent/plugin/out_forward.rb

@@ -31,7 +31,7 @@ class ForwardOutput < ObjectBufferedOutput
   config_param :send_timeout, :time, :default => 60
   config_param :heartbeat_interval, :time, :default => 1
   config_param :recover_wait, :time, :default => 10
-  config_param :hard_timeout, :time, :default => nil
+  config_param :hard_timeout, :time, :default => 60
   config_param :phi_threshold, :integer, :default => 8
   attr_reader :nodes
 
@@ -52,8 +52,6 @@ class ForwardOutput < ObjectBufferedOutput
       e['port'] = port.to_s
     end
 
-    @hard_timeout ||= @send_timeout
-
     recover_sample_size = @recover_wait / @heartbeat_interval
 
     conf.elements.each {|e|
@@ -245,7 +243,7 @@ class ForwardOutput < ObjectBufferedOutput
         @usock.send "", 0, sockaddr
       rescue
         # TODO log
-        $log.debug "failed to send heartbeat packet to #{Socket.unpack_sockaddr_in(sockaddr).reverse.join(':')}", :error=>$!
+        $log.debug "failed to send heartbeat packet to #{Socket.unpack_sockaddr_in(sockaddr).reverse.join(':')}", :error=>$!.to_s
       end
     }
   end

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.11'
+VERSION = '0.10.12'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.11
+  version: 0.10.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-10 00:00:00.000000000Z
+date: 2012-02-14 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70203141220220 !ruby/object:Gem::Requirement
+  requirement: &70256836669480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70203141220220
+  version_requirements: *70256836669480
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70203141219740 !ruby/object:Gem::Requirement
+  requirement: &70256836648880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70203141219740
+  version_requirements: *70256836648880
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70203141219240 !ruby/object:Gem::Requirement
+  requirement: &70256836646880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *70203141219240
+  version_requirements: *70256836646880
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70203141218740 !ruby/object:Gem::Requirement
+  requirement: &70256836645660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70203141218740
+  version_requirements: *70256836645660
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70203141218220 !ruby/object:Gem::Requirement
+  requirement: &70256836644620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70203141218220
+  version_requirements: *70256836644620
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70203141217440 !ruby/object:Gem::Requirement
+  requirement: &70256836643580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +76,7 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70203141217440
+  version_requirements: *70256836643580
 description: 
 email: frsyuki@gmail.com
 executables: