fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -30,11 +30,17 @@ module Fluent::Plugin
 
     helpers :timer, :thread
 
+    desc 'The address to bind to.'
     config_param :bind, :string, default: '0.0.0.0'
+    desc 'The port to listen to.'
     config_param :port, :integer, default: 24220
+    desc 'The tag with which internal metrics are emitted.'
     config_param :tag, :string, default: nil
+    desc 'Determine the rate to emit internal metrics as events.'
     config_param :emit_interval, :time, default: 60
+    desc 'Determine whether to include the config information.'
     config_param :include_config, :bool, default: true
+    desc 'Determine whether to include the retry information.'
     config_param :include_retry, :bool, default: true
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
@@ -279,7 +285,7 @@ module Fluent::Plugin
     MONITOR_INFO = {
       'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
       'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
-      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size },
+      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },
       'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
     }
 

data/lib/fluent/plugin/in_syslog.rb

@@ -75,6 +75,8 @@ module Fluent::Plugin
     config_param :tag, :string
     desc 'The transport protocol used to receive logs.(udp, tcp)'
     config_param :protocol_type, :enum, list: [:tcp, :udp], default: :udp
+    desc 'The message frame type.(traditional, octet_count)'
+    config_param :frame_type, :enum, list: [:traditional, :octet_count], default: :traditional
 
     desc 'If true, add source host to event record.'
     config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
@@ -83,6 +85,8 @@ module Fluent::Plugin
 
     desc 'The field name of hostname of sender.'
     config_param :source_hostname_key, :string, default: nil
+    desc 'Try to resolve hostname from IP addresses or not.'
+    config_param :resolve_hostname, :bool, default: nil
     desc 'The field name of source address of sender.'
     config_param :source_address_key, :string, default: nil
     desc 'The field name of the priority.'
@@ -116,7 +120,13 @@ module Fluent::Plugin
         end
         @source_address_key = @source_host_key
       end
-      @resolve_name = !!@source_hostname_key
+      if @source_hostname_key
+        if @resolve_hostname.nil?
+          @resolve_hostname = true
+        elsif !@resolve_hostname # user specifies "false" in config
+          raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
+        end
+      end
 
       @_event_loop_run_timeout = @blocking_timeout
     end
@@ -138,24 +148,35 @@ module Fluent::Plugin
     end
 
     def start_udp_server
-      server_create_udp(:in_syslog_udp_server, @port, bind: @bind, max_bytes: @message_length_limit, resolve_name: @resolve_name) do |data, sock|
+      server_create_udp(:in_syslog_udp_server, @port, bind: @bind, max_bytes: @message_length_limit, resolve_name: @resolve_hostname) do |data, sock|
         message_handler(data.chomp, sock)
       end
     end
 
     def start_tcp_server
-      # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
-      delimiter = "\n"
+      octet_count_frame = @frame_type == :octet_count
+
+      # syslog family adds "\n" to each message when transport is TCP and traditional frame
+      delimiter = octet_count_frame ? " " : "\n"
       delimiter_size = delimiter.size
-      server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_name) do |conn|
-        buffer = ""
+      server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
         conn.data do |data|
+          buffer = conn.buffer
           buffer << data
           pos = 0
-          while idx = buffer.index(delimiter, pos)
-            msg = buffer[pos...idx]
-            pos = idx + delimiter_size
-            message_handler(msg, conn)
+          if octet_count_frame
+            while idx = buffer.index(delimiter, pos)
+              num = Integer(buffer[pos..idx])
+              pos = idx + num
+              msg = buffer[idx + 1...pos]
+              message_handler(msg, conn)
+            end
+          else
+            while idx = buffer.index(delimiter, pos)
+              msg = buffer[pos...idx]
+              pos = idx + delimiter_size
+              message_handler(msg, conn)
+            end
           end
           buffer.slice!(0, pos) if pos > 0
         end

data/lib/fluent/plugin/in_tail.rb

@@ -34,6 +34,16 @@ module Fluent::Plugin
 
     helpers :timer, :event_loop, :parser, :compat_parameters
 
+    class WatcherSetupError < StandardError
+      def initialize(msg)
+        @message = msg
+      end
+
+      def to_s
+        @message
+      end
+    end
+
     FILE_PERMISSION = 0644
 
     def initialize
@@ -70,6 +80,8 @@ module Fluent::Plugin
     config_param :emit_unmatched_lines, :bool, default: false
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
+    desc 'Enable the stat watcher based on inotify.'
+    config_param :enable_stat_watcher, :bool, default: true
     desc 'The encoding after conversion of the input.'
     config_param :encoding, :string, default: nil
     desc 'The encoding of the input.'
@@ -87,6 +99,8 @@ module Fluent::Plugin
 
     attr_reader :paths
 
+    @@pos_file_paths = {}
+
     def configure(conf)
       compat_parameters_convert(conf, :parser)
       parser_config = conf.elements('parse').first
@@ -103,13 +117,23 @@ module Fluent::Plugin
 
       super
 
+      if !@enable_watch_timer && !@enable_stat_watcher
+        raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
+      end
+
       @paths = @path.split(',').map {|path| path.strip }
       if @paths.empty?
         raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
 
       # TODO: Use plugin_root_dir and storage plugin to store positions if available
-      unless @pos_file
+      if @pos_file
+        if @@pos_file_paths.has_key?(@pos_file) && !called_in_test?
+          plugin_id_using_this_path = @@pos_file_paths[@pos_file]
+          raise Fluent::ConfigError, "Other 'in_tail' plugin already use same pos_file path: plugin_id = #{plugin_id_using_this_path}, pos_file path = #{@pos_file}"
+        end
+        @@pos_file_paths[@pos_file] = self.plugin_id
+      else
         $log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
         $log.warn "this parameter is highly recommended to save the position to resume tailing."
       end
@@ -130,6 +154,7 @@ module Fluent::Plugin
     def configure_tag
       if @tag.index('*')
         @tag_prefix, @tag_suffix = @tag.split('*')
+        @tag_prefix ||= ''
         @tag_suffix ||= ''
       else
         @tag_prefix = nil
@@ -187,25 +212,28 @@ module Fluent::Plugin
       date = Time.now
       paths = []
 
-      excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
       @paths.each { |path|
         path = date.strftime(path)
         if path.include?('*')
           paths += Dir.glob(path).select { |p|
-            is_file = !File.directory?(p)
-            if File.readable?(p) && is_file
-              if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
-                false
+            begin
+              is_file = !File.directory?(p)
+              if File.readable?(p) && is_file
+                if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
+                  false
+                else
+                  true
+                end
               else
-                true
-              end
-            else
-              if is_file
-                unless @ignore_list.include?(path)
-                  log.warn "#{p} unreadable. It is excluded and would be examined next time."
-                  @ignore_list << path if @ignore_repeated_permission_error
+                if is_file
+                  unless @ignore_list.include?(path)
+                    log.warn "#{p} unreadable. It is excluded and would be examined next time."
+                    @ignore_list << path if @ignore_repeated_permission_error
+                  end
                 end
+                false
               end
+            rescue Errno::ENOENT
               false
             end
           }
@@ -214,6 +242,7 @@ module Fluent::Plugin
           paths << path
         end
       }
+      excluded = @exclude_path.map { |path| path = date.strftime(path); path.include?('*') ? Dir.glob(path) : path }.flatten.uniq
       paths - excluded
     end
 
@@ -235,12 +264,21 @@ module Fluent::Plugin
 
     def setup_watcher(path, pe)
       line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @enable_stat_watcher, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
       tw.attach do |watcher|
-        watcher.timer_trigger = timer_execute(:in_tail_timer_trigger, 1, &watcher.method(:on_notify)) if watcher.enable_watch_timer
-        event_loop_attach(watcher.stat_trigger)
+        event_loop_attach(watcher.timer_trigger) if watcher.timer_trigger
+        event_loop_attach(watcher.stat_trigger) if watcher.stat_trigger
       end
       tw
+    rescue => e
+      if tw
+        tw.detach { |watcher|
+          event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
+          event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
+        }
+        tw.close
+      end
+      raise e
     end
 
     def start_watchers(paths)
@@ -257,7 +295,13 @@ module Fluent::Plugin
           end
         end
 
-        @tails[path] = setup_watcher(path, pe)
+        begin
+          tw = setup_watcher(path, pe)
+        rescue WatcherSetupError => e
+          log.warn "Skip #{path} because unexpected setup error happens: #{e}"
+          next
+        end
+        @tails[path] = tw
       }
     end
 
@@ -302,7 +346,10 @@ module Fluent::Plugin
     # so adding close_io argument to avoid this problem.
     # At shutdown, IOHandler's io will be released automatically after detached the event loop
     def detach_watcher(tw, close_io = true)
-      tw.detach
+      tw.detach { |watcher|
+        event_loop_detach(watcher.timer_trigger) if watcher.timer_trigger
+        event_loop_detach(watcher.stat_trigger) if watcher.stat_trigger
+      }
       tw.close if close_io
       flush_buffer(tw)
       if tw.unwatched && @pf
@@ -311,6 +358,8 @@ module Fluent::Plugin
     end
 
     def detach_watcher_after_rotate_wait(tw)
+      # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
+      # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
       timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
         detach_watcher(tw)
       end
@@ -426,18 +475,19 @@ module Fluent::Plugin
     end
 
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
+      def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, enable_stat_watcher, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
         @path = path
         @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
         @read_from_head = read_from_head
         @enable_watch_timer = enable_watch_timer
+        @enable_stat_watcher = enable_stat_watcher
         @read_lines_limit = read_lines_limit
         @receive_lines = receive_lines
         @update_watcher = update_watcher
 
-        @stat_trigger = StatWatcher.new(self, &method(:on_notify))
-        @timer_trigger = nil
+        @stat_trigger = @enable_stat_watcher ? StatWatcher.new(self, &method(:on_notify)) : nil
+        @timer_trigger = @enable_watch_timer ? TimerTrigger.new(1, log, &method(:on_notify)) : nil
 
         @rotate_handler = RotateHandler.new(self, &method(:on_rotate))
         @io_handler = nil
@@ -452,7 +502,7 @@ module Fluent::Plugin
       attr_reader :path
       attr_reader :log, :pe, :read_lines_limit, :open_on_every_update
       attr_reader :from_encoding, :encoding
-      attr_reader :stat_trigger, :enable_watch_timer
+      attr_reader :stat_trigger, :enable_watch_timer, :enable_stat_watcher
       attr_accessor :timer_trigger
       attr_accessor :line_buffer, :line_buffer_timer_flusher
       attr_accessor :unwatched  # This is used for removing position entry from PositionFile
@@ -471,8 +521,7 @@ module Fluent::Plugin
       end
 
       def detach
-        @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
-        @stat_trigger.detach if @stat_trigger.attached?
+        yield self
         @io_handler.on_notify if @io_handler
       end
 
@@ -509,7 +558,10 @@ module Fluent::Plugin
               # assuming following situation:
               #   a) file was once renamed and backed, or
               #   b) symlink or hardlink to the same file is recreated
-              # in either case, seek to the saved position
+              # in either case of a and b, seek to the saved position
+              #   c) file was once renamed, truncated and then backed
+              # in this case, consider it truncated
+              @pe.update(inode, 0) if fsize < @pe.read_pos
             elsif last_inode != 0
               # this is FilePositionEntry and fluentd once started.
               # read data from the head of the rotated file.
@@ -539,6 +591,8 @@ module Fluent::Plugin
               @pe.update(inode, 0)
             else # file is rotated and new file found
               watcher_needs_update = true
+              # Handle the old log file before renewing TailWatcher [fluentd#1055]
+              @io_handler.on_notify
             end
           else # file is rotated and new file not found
             # Clear RotateHandler to avoid duplicated file watch in same path.
@@ -566,6 +620,21 @@ module Fluent::Plugin
         pe # This pe will be updated in on_rotate after TailWatcher is initialized
       end
 
+      class TimerTrigger < Coolio::TimerWatcher
+        def initialize(interval, log, &callback)
+          @callback = callback
+          @log = log
+          super(interval, true)
+        end
+
+        def on_timer
+          @callback.call
+        rescue => e
+          @log.error e.to_s
+          @log.error_backtrace
+        end
+      end
+
       class StatWatcher < Coolio::StatWatcher
         def initialize(watcher, &callback)
           @watcher = watcher
@@ -582,7 +651,6 @@ module Fluent::Plugin
         end
       end
 
-
       class FIFO
         def initialize(from_encoding, encoding)
           @from_encoding = from_encoding
@@ -637,10 +705,15 @@ module Fluent::Plugin
           @iobuf = ''.force_encoding('ASCII-8BIT')
           @lines = []
           @io = nil
+          @notify_mutex = Mutex.new
           @watcher.log.info "following tail of #{@watcher.path}"
         end
 
         def on_notify
+          @notify_mutex.synchronize { handle_notify }
+        end
+
+        def handle_notify
           with_io do |io|
             begin
               read_more = false
@@ -689,6 +762,9 @@ module Fluent::Plugin
           io = Fluent::FileWrapper.open(@watcher.path)
           io.seek(@watcher.pe.read_pos + @fifo.bytesize)
           io
+        rescue RangeError
+          io.close if io
+          raise WatcherSetupError, "seek error with #{@watcher.path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
         rescue Errno::ENOENT
           nil
         end
@@ -706,6 +782,9 @@ module Fluent::Plugin
               @io ||= open
               yield @io
             end
+          rescue WatcherSetupError => e
+            close
+            raise e
           rescue
             @watcher.log.error $!.to_s
             @watcher.log.error_backtrace
@@ -790,8 +869,9 @@ module Fluent::Plugin
     class PositionFile
       UNWATCHED_POSITION = 0xffffffffffffffff
 
-      def initialize(file, map, last_pos)
+      def initialize(file, file_mutex, map, last_pos)
         @file = file
+        @file_mutex = file_mutex
         @map = map
         @last_pos = last_pos
       end
@@ -801,29 +881,34 @@ module Fluent::Plugin
           return m
         end
 
-        @file.pos = @last_pos
-        @file.write path
-        @file.write "\t"
-        seek = @file.pos
-        @file.write "0000000000000000\t0000000000000000\n"
-        @last_pos = @file.pos
-
-        @map[path] = FilePositionEntry.new(@file, seek)
+        @file_mutex.synchronize {
+          @file.pos = @last_pos
+          @file.write "#{path}\t0000000000000000\t0000000000000000\n"
+          seek = @last_pos + path.bytesize + 1
+          @last_pos = @file.pos
+          @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+        }
       end
 
       def self.parse(file)
         compact(file)
 
+        file_mutex = Mutex.new
         map = {}
         file.pos = 0
         file.each_line {|line|
           m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          next unless m
+          unless m
+            $log.warn "Unparsable line in pos_file: #{line}"
+            next
+          end
           path = m[1]
+          pos = m[2].to_i(16)
+          ino = m[3].to_i(16)
           seek = file.pos - line.bytesize + path.bytesize + 1
-          map[path] = FilePositionEntry.new(file, seek)
+          map[path] = FilePositionEntry.new(file, file_mutex, seek, pos, ino)
         }
-        new(file, map, file.pos)
+        new(file, file_mutex, map, file.pos)
       end
 
       # Clean up unwatched file entries
@@ -831,7 +916,10 @@ module Fluent::Plugin
         file.pos = 0
         existent_entries = file.each_line.map { |line|
           m = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(line)
-          next unless m
+          unless m
+            $log.warn "Unparsable line in pos_file: #{line}"
+            next
+          end
           path = m[1]
           pos = m[2].to_i(16)
           ino = m[3].to_i(16)
@@ -854,36 +942,37 @@ module Fluent::Plugin
       LN_OFFSET = 33
       SIZE = 34
 
-      def initialize(file, seek)
+      def initialize(file, file_mutex, seek, pos, inode)
         @file = file
+        @file_mutex = file_mutex
         @seek = seek
-        @pos = nil
+        @pos = pos 
+        @inode = inode
       end
 
       def update(ino, pos)
-        @file.pos = @seek
-        @file.write "%016x\t%016x" % [pos, ino]
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x\t%016x" % [pos, ino]
+        }
         @pos = pos
+        @inode = ino
       end
 
       def update_pos(pos)
-        @file.pos = @seek
-        @file.write "%016x" % pos
+        @file_mutex.synchronize {
+          @file.pos = @seek
+          @file.write "%016x" % pos
+        }
         @pos = pos
       end
 
       def read_inode
-        @file.pos = @seek + INO_OFFSET
-        raw = @file.read(16)
-        raw ? raw.to_i(16) : 0
+        @inode
       end
 
       def read_pos
-        @pos ||= begin
-          @file.pos = @seek
-          raw = @file.read(16)
-          raw ? raw.to_i(16) : 0
-        end
+        @pos
       end
     end