fluentd 0.12.20 → 0.12.21

data/lib/fluent/plugin/in_stream.rb

@@ -14,10 +14,19 @@
 #    limitations under the License.
 #
 
+require 'fileutils'
+require 'socket'
+
+require 'cool.io'
+require 'yajl'
+
+require 'fluent/input'
+require 'fluent/event'
+
 module Fluent
   # obsolete
   class StreamInput < Input
-    config_param :blocking_timeout, :time, :default => 0.5
+    config_param :blocking_timeout, :time, default: 0.5
 
     def initialize
       require 'socket'
@@ -45,7 +54,7 @@ module Fluent
     def run
       @loop.run(@blocking_timeout)
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
@@ -142,7 +151,7 @@ module Fluent
       def on_read_json(data)
         @y << data
       rescue
-        @log.error "unexpected error", :error=>$!.to_s
+        @log.error "unexpected error", error: $!.to_s
         @log.error_backtrace
         close
       end
@@ -150,7 +159,7 @@ module Fluent
       def on_read_msgpack(data)
         @u.feed_each(data, &@on_message)
       rescue
-        @log.error "unexpected error", :error=>$!.to_s
+        @log.error "unexpected error", error: $!.to_s
         @log.error_backtrace
         close
       end
@@ -165,9 +174,9 @@ module Fluent
     Plugin.register_input('unix', self)
 
     desc 'The path to your Unix Domain Socket.'
-    config_param :path, :string, :default => DEFAULT_SOCKET_PATH
+    config_param :path, :string, default: DEFAULT_SOCKET_PATH
     desc 'The backlog of Unix Domain Socket.'
-    config_param :backlog, :integer, :default => nil
+    config_param :backlog, :integer, default: nil
 
     def configure(conf)
       super

data/lib/fluent/plugin/in_syslog.rb

@@ -14,6 +14,13 @@
 #    limitations under the License.
 #
 
+require 'cool.io'
+require 'yajl'
+
+require 'fluent/input'
+require 'fluent/config/error'
+require 'fluent/parser'
+
 module Fluent
   class SyslogInput < Input
     Plugin.register_input('syslog', self)
@@ -60,18 +67,17 @@ module Fluent
 
     def initialize
       super
-      require 'cool.io'
       require 'fluent/plugin/socket_util'
     end
 
     desc 'The port to listen to.'
-    config_param :port, :integer, :default => 5140
+    config_param :port, :integer, default: 5140
     desc 'The bind address to listen to.'
-    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :bind, :string, default: '0.0.0.0'
     desc 'The prefix of the tag. The tag itself is generated by the tag prefix, facility level, and priority.'
     config_param :tag, :string
     desc 'The transport protocol used to receive logs.(udp, tcp)'
-    config_param :protocol_type, :default => :udp do |val|
+    config_param :protocol_type, default: :udp do |val|
       case val.downcase
       when 'tcp'
         :tcp
@@ -82,10 +88,10 @@ module Fluent
       end
     end
     desc 'If true, add source host to event record.'
-    config_param :include_source_host, :bool, :default => false
+    config_param :include_source_host, :bool, default: false
     desc 'Specify key of source host when include_source_host is true.'
-    config_param :source_host_key, :string, :default => 'source_host'.freeze
-    config_param :blocking_timeout, :time, :default => 0.5
+    config_param :source_host_key, :string, default: 'source_host'.freeze
+    config_param :blocking_timeout, :time, default: 0.5
 
     def configure(conf)
       super
@@ -125,7 +131,7 @@ module Fluent
     def run
       @loop.run(@blocking_timeout)
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
@@ -150,14 +156,14 @@ module Fluent
         emit(pri, time, record)
       }
     rescue => e
-      log.error data.dump, :error => e.to_s
+      log.error data.dump, error: e.to_s
       log.error_backtrace
     end
 
     def receive_data(data, addr)
       @parser.parse(data) { |time, record|
         unless time && record
-          log.warn "invalid syslog message", :data => data
+          log.warn "invalid syslog message", data: data
           return
         end
 
@@ -166,7 +172,7 @@ module Fluent
         emit(pri, time, record)
       }
     rescue => e
-      log.error data.dump, :error => e.to_s
+      log.error data.dump, error: e.to_s
       log.error_backtrace
     end
 
@@ -192,7 +198,7 @@ module Fluent
 
       router.emit(tag, time, record)
     rescue => e
-      log.error "syslog failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
+      log.error "syslog failed to emit", error: e.to_s, error_class: e.class.to_s, tag: tag, record: Yajl.dump(record)
     end
   end
 end

data/lib/fluent/plugin/in_tail.rb

@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'cool.io'
+
+require 'fluent/input'
+require 'fluent/config/error'
+require 'fluent/event'
+
 module Fluent
   class NewTailInput < Input
     Plugin.register_input('tail', self)
@@ -29,19 +35,21 @@ module Fluent
     desc 'The tag of the event.'
     config_param :tag, :string
     desc 'The paths to exclude the files from watcher list.'
-    config_param :exclude_path, :array, :default => []
+    config_param :exclude_path, :array, default: []
     desc 'Specify interval to keep reference to old file when rotate a file.'
-    config_param :rotate_wait, :time, :default => 5
+    config_param :rotate_wait, :time, default: 5
     desc 'Fluentd will record the position it last read into this file.'
-    config_param :pos_file, :string, :default => nil
+    config_param :pos_file, :string, default: nil
     desc 'Start to read the logs from the head of file, not bottom.'
-    config_param :read_from_head, :bool, :default => false
+    config_param :read_from_head, :bool, default: false
     desc 'The interval of refreshing the list of watch file.'
-    config_param :refresh_interval, :time, :default => 60
+    config_param :refresh_interval, :time, default: 60
     desc 'The number of reading lines at each IO.'
-    config_param :read_lines_limit, :integer, :default => 1000
+    config_param :read_lines_limit, :integer, default: 1000
     desc 'The interval of flushing the buffer for multiline format'
-    config_param :multiline_flush_interval, :time, :default => nil
+    config_param :multiline_flush_interval, :time, default: nil
+    desc 'Enable the additional watch timer.'
+    config_param :enable_watch_timer, :bool, default: true
 
     attr_reader :paths
 
@@ -150,7 +158,7 @@ module Fluent
 
     def setup_watcher(path, pe)
       line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
       tw.attach(@loop)
       tw
     end
@@ -232,10 +240,11 @@ module Fluent
     def run
       @loop.run
     rescue
-      log.error "unexpected error", :error=>$!.to_s
+      log.error "unexpected error", error: $!.to_s
       log.error_backtrace
     end
 
+    # @return true if no error or unrecoverable error happens in emit action. false if got BufferQueueLimitError
     def receive_lines(lines, tail_watcher)
       es = @receive_handler.call(lines, tail_watcher)
       unless es.empty?
@@ -246,10 +255,15 @@ module Fluent
               end
         begin
           router.emit_stream(tag, es)
+        rescue BufferQueueLimitError
+          return false
         rescue
-          # ignore errors. Engine shows logs and backtraces.
+          # ignore non BufferQueueLimitError errors because in_tail can't recover. Engine shows logs and backtraces.
+          return true
         end
       end
+
+      return true
     end
 
     def convert_line_to_event(line, es)
@@ -263,7 +277,7 @@ module Fluent
           end
         }
       rescue => e
-        log.warn line.dump, :error => e.to_s
+        log.warn line.dump, error: e.to_s
         log.debug_backtrace(e.backtrace)
       end
     end
@@ -312,16 +326,18 @@ module Fluent
     end
 
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, read_from_head, read_lines_limit, update_watcher, line_buffer_timer_flusher, &receive_lines)
+      def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, read_lines_limit, update_watcher, line_buffer_timer_flusher, &receive_lines)
         @path = path
         @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
         @read_from_head = read_from_head
+        @enable_watch_timer = enable_watch_timer
         @read_lines_limit = read_lines_limit
         @receive_lines = receive_lines
         @update_watcher = update_watcher
 
-        @timer_trigger = TimerWatcher.new(1, true, log, &method(:on_notify))
+        @timer_trigger = TimerWatcher.new(1, true, log, &method(:on_notify)) if @enable_watch_timer
+
         @stat_trigger = StatWatcher.new(path, log, &method(:on_notify))
 
         @rotate_handler = RotateHandler.new(path, log, &method(:on_rotate))
@@ -344,13 +360,13 @@ module Fluent
       end
 
       def attach(loop)
-        @timer_trigger.attach(loop)
+        @timer_trigger.attach(loop) if @enable_watch_timer
         @stat_trigger.attach(loop)
         on_notify
       end
 
       def detach
-        @timer_trigger.detach if @timer_trigger.attached?
+        @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
         @stat_trigger.detach if @stat_trigger.attached?
       end
 
@@ -503,6 +519,7 @@ module Fluent
           @receive_lines = receive_lines
           @buffer = ''.force_encoding('ASCII-8BIT')
           @iobuf = ''.force_encoding('ASCII-8BIT')
+          @lines = []
         end
 
         attr_reader :io
@@ -510,31 +527,36 @@ module Fluent
 
         def on_notify
           begin
-            lines = []
             read_more = false
 
-            begin
-              while true
-                if @buffer.empty?
-                  @io.read_nonblock(2048, @buffer)
-                else
-                  @buffer << @io.read_nonblock(2048, @iobuf)
-                end
-                while line = @buffer.slice!(/.*?\n/m)
-                  lines << line
-                end
-                if lines.size >= @read_lines_limit
-                  # not to use too much memory in case the file is very large
-                  read_more = true
-                  break
+            if @lines.empty?
+              begin
+                while true
+                  if @buffer.empty?
+                    @io.readpartial(2048, @buffer)
+                  else
+                    @buffer << @io.readpartial(2048, @iobuf)
+                  end
+                  while line = @buffer.slice!(/.*?\n/m)
+                    @lines << line
+                  end
+                  if @lines.size >= @read_lines_limit
+                    # not to use too much memory in case the file is very large
+                    read_more = true
+                    break
+                  end
                 end
+              rescue EOFError
               end
-            rescue EOFError
             end
 
-            unless lines.empty?
-              @receive_lines.call(lines)
-              @pe.update_pos(@io.pos - @buffer.bytesize)
+            unless @lines.empty?
+              if @receive_lines.call(@lines)
+                @pe.update_pos(@io.pos - @buffer.bytesize)
+                @lines.clear
+              else
+                read_more = false
+              end
             end
           end while read_more
 
@@ -574,8 +596,7 @@ module Fluent
 
         def on_notify
           begin
-            io = File.open(@path)
-            stat = io.stat
+            stat = File.stat(@path)
             inode = stat.ino
             fsize = stat.size
           rescue Errno::ENOENT
@@ -587,14 +608,14 @@ module Fluent
           begin
             if @inode != inode || fsize < @fsize
               # rotated or truncated
+              begin
+                io = File.open(@path)
+              rescue Errno::ENOENT
+              end
               @on_rotate.call(io)
-              io = nil
             end
-
             @inode = inode
             @fsize = fsize
-          ensure
-            io.close if io
           end
 
         rescue

data/lib/fluent/plugin/in_tcp.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'cool.io'
+
 require 'fluent/plugin/socket_util'
 
 module Fluent
@@ -22,7 +24,7 @@ module Fluent
 
     config_set_default :port, 5170
     desc 'The payload is read up to this character.'
-    config_param :delimiter, :string, :default => "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+    config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
     def listen(callback)
       log.info "listening tcp socket on #{@bind}:#{@port}"

data/lib/fluent/plugin/in_udp.rb

@@ -21,7 +21,7 @@ module Fluent
     Plugin.register_input('udp', self)
 
     config_set_default :port, 5160
-    config_param :body_size_limit, :size, :default => 4096
+    config_param :body_size_limit, :size, default: 4096
 
     def listen(callback)
       log.info "listening udp socket on #{@bind}:#{@port}"

data/lib/fluent/plugin/out_copy.rb

@@ -14,12 +14,16 @@
 #    limitations under the License.
 #
 
+require 'fluent/output'
+require 'fluent/config/error'
+require 'fluent/event'
+
 module Fluent
   class CopyOutput < MultiOutput
     Plugin.register_output('copy', self)
 
     desc 'If true, pass different record to each `store` plugin.'
-    config_param :deep_copy, :bool, :default => false
+    config_param :deep_copy, :bool, default: false
 
     def initialize
       super

data/lib/fluent/plugin/out_exec.rb

@@ -14,6 +14,10 @@
 #    limitations under the License.
 #
 
+require 'tempfile'
+
+require 'fluent/output'
+require 'fluent/config/error'
 require 'fluent/plugin/exec_util'
 
 module Fluent
@@ -22,24 +26,23 @@ module Fluent
 
     def initialize
       super
-      require 'tempfile'
       @localtime = false
     end
 
     desc 'The command (program) to execute. The exec plugin passes the path of a TSV file as the last argumen'
     config_param :command, :string
     desc 'Specify the comma-separated keys when using the tsv format.'
-    config_param :keys, :default => [] do |val|
+    config_param :keys, default: [] do |val|
       val.split(',')
     end
     desc 'The name of the key to use as the event tag. This replaces the value in the event record.'
-    config_param :tag_key, :string, :default => nil
+    config_param :tag_key, :string, default: nil
     desc 'The name of the key to use as the event time. This replaces the the value in the event record.'
-    config_param :time_key, :string, :default => nil
+    config_param :time_key, :string, default: nil
     desc 'The format for event time used when the time_key parameter is specified. The default is UNIX time (integer).'
-    config_param :time_format, :string, :default => nil
+    config_param :time_format, :string, default: nil
     desc "The format used to map the incoming events to the program input. (#{ExecUtil::SUPPORTED_FORMAT.keys.join(',')})"
-    config_param :format, :default => :tsv do |val|
+    config_param :format, default: :tsv do |val|
       f = ExecUtil::SUPPORTED_FORMAT[val]
       raise ConfigError, "Unsupported format '#{val}'" unless f
       f