fluentd 1.9.3 → 1.11.1

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -223,7 +223,7 @@ module Fluent::Plugin
         opts = {with_config: false, with_retry: false}
         timer_execute(:in_monitor_agent_emit, @emit_interval, repeat: true) {
           es = Fluent::MultiEventStream.new
-          now = Fluent::Engine.now
+          now = Fluent::EventTime.now
           plugins_info_all(opts).each { |record|
             es.add(now, record)
           }

data/lib/fluent/plugin/in_syslog.rb

@@ -181,12 +181,12 @@ module Fluent::Plugin
           if octet_count_frame
             while idx = buffer.index(delimiter, pos)
               num = Integer(buffer[pos..idx])
-              pos = idx + num
-              msg = buffer[idx + 1...pos]
-              if msg.size < num - 1
-                pos = pos - num - num.to_s.size
+              msg = buffer[idx + delimiter_size, num]
+              if msg.size != num
                 break
               end
+
+              pos = idx + delimiter_size + num
               message_handler(msg, conn)
             end
           else

data/lib/fluent/plugin/in_tail.rb

@@ -70,7 +70,7 @@ module Fluent::Plugin
     desc 'Fluentd will record the position it last read into this file.'
     config_param :pos_file, :string, default: nil
     desc 'The cleanup interval of pos file'
-    config_param :pos_file_compaction_interval, :integer, default: nil
+    config_param :pos_file_compaction_interval, :time, default: nil
     desc 'Start to read the logs from the head of file, not bottom.'
     config_param :read_from_head, :bool, default: false
     # When the program deletes log file and re-creates log file with same filename after passed refresh_interval,
@@ -136,7 +136,7 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
       end
 
-      @paths = @path.split(@path_delimiter).map(&:strip)
+      @paths = @path.split(@path_delimiter).map(&:strip).uniq
       if @paths.empty?
         raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
       end
@@ -271,9 +271,9 @@ module Fluent::Plugin
                 end
               else
                 if is_file
-                  unless @ignore_list.include?(path)
+                  unless @ignore_list.include?(p)
                     log.warn "#{p} unreadable. It is excluded and would be examined next time."
-                    @ignore_list << path if @ignore_repeated_permission_error
+                    @ignore_list << p if @ignore_repeated_permission_error
                   end
                 end
                 false
@@ -296,7 +296,7 @@ module Fluent::Plugin
                end
         path.include?('*') ? Dir.glob(path) : path
       }.flatten.uniq
-      paths - excluded
+      paths.uniq - excluded
     end
 
     # in_tail with '*' path doesn't check rotation file equality at refresh phase.
@@ -319,7 +319,7 @@ module Fluent::Plugin
 
     def setup_watcher(path, pe)
       line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(path, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
+      tw = TailWatcher.new(path, pe, log, @read_from_head, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
 
       if @enable_watch_timer
         tt = TimerTrigger.new(1, log) { tw.on_notify }
@@ -555,6 +555,21 @@ module Fluent::Plugin
       es
     end
 
+    private
+
+    def io_handler(watcher, path)
+      TailWatcher::IOHandler.new(
+        watcher,
+        path: path,
+        log: log,
+        read_lines_limit: @read_lines_limit,
+        open_on_every_update: @open_on_every_update,
+        from_encoding: @from_encoding,
+        encoding: @encoding,
+        &method(:receive_lines)
+      )
+    end
+
     class StatWatcher < Coolio::StatWatcher
       def initialize(path, log, &callback)
         @callback = callback
@@ -586,22 +601,16 @@ module Fluent::Plugin
     end
 
     class TailWatcher
-      def initialize(path, pe, log, read_from_head, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
+      def initialize(path, pe, log, read_from_head, update_watcher, line_buffer_timer_flusher, io_handler_build)
         @path = path
         @pe = pe || MemoryPositionEntry.new
         @read_from_head = read_from_head
-        @read_lines_limit = read_lines_limit
-        @receive_lines = receive_lines
         @update_watcher = update_watcher
-
-        @rotate_handler = RotateHandler.new(log, &method(:on_rotate))
-        @io_handler = nil
         @log = log
-
+        @rotate_handler = RotateHandler.new(log, &method(:on_rotate))
         @line_buffer_timer_flusher = line_buffer_timer_flusher
-        @from_encoding = from_encoding
-        @encoding = encoding
-        @open_on_every_update = open_on_every_update
+        @io_handler = nil
+        @io_handler_build = io_handler_build
         @watchers = []
       end
 
@@ -709,9 +718,7 @@ module Fluent::Plugin
       end
 
       def io_handler
-        IOHandler.new(self, path: @path, log: @log, read_lines_limit: @read_lines_limit, open_on_every_update: @open_on_every_update, from_encoding: @from_encoding, encoding: @encoding) do |lines|
-          @receive_lines.call(lines, self)
-        end
+        @io_handler_build.call(self, @path)
       end
 
       def swap_state(pe)
@@ -801,6 +808,19 @@ module Fluent::Plugin
           @notify_mutex.synchronize { handle_notify }
         end
 
+        def close
+          if @io && !@io.closed?
+            @io.close
+            @io = nil
+          end
+        end
+
+        def opened?
+          !!@io
+        end
+
+        private
+
         def handle_notify
           with_io do |io|
             begin
@@ -822,7 +842,7 @@ module Fluent::Plugin
               end
 
               unless @lines.empty?
-                if @receive_lines.call(@lines)
+                if @receive_lines.call(@lines, @watcher)
                   @watcher.pe.update_pos(io.pos - @fifo.bytesize)
                   @lines.clear
                 else
@@ -833,17 +853,6 @@ module Fluent::Plugin
           end
         end
 
-        def close
-          if @io && !@io.closed?
-            @io.close
-            @io = nil
-          end
-        end
-
-        def opened?
-          !!@io
-        end
-
         def open
           io = Fluent::FileWrapper.open(@path)
           io.seek(@watcher.pe.read_pos + @fifo.bytesize)

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -21,7 +21,6 @@ module Fluent::Plugin
     class PositionFile
       UNWATCHED_POSITION = 0xffffffffffffffff
       POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
-      POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
 
       def self.load(file, logger:)
         pf = new(file, logger: logger)
@@ -83,8 +82,8 @@ module Fluent::Plugin
         size = nil
 
         @file_mutex.synchronize do
-          last_modified = @file.mtime
           size = @file.size
+          last_modified = @file.mtime
         end
 
         entries = fetch_compacted_entries
@@ -93,7 +92,13 @@ module Fluent::Plugin
           if last_modified == @file.mtime && size == @file.size
             @file.pos = 0
             @file.truncate(0)
-            @file.write(entries.join)
+            @file.write(entries.values.map(&:to_entry_fmt).join)
+
+            entries.each do |path, val|
+              if (m = @map[path])
+                m.seek = val.seek
+              end
+            end
           else
             # skip
           end
@@ -104,7 +109,7 @@ module Fluent::Plugin
 
       def compact
         @file_mutex.synchronize do
-          entries = fetch_compacted_entries
+          entries = fetch_compacted_entries.values.map(&:to_entry_fmt)
 
           @file.pos = 0
           @file.truncate(0)
@@ -116,6 +121,7 @@ module Fluent::Plugin
         entries = {}
 
         @file.pos = 0
+        file_pos = 0
         @file.each_line do |line|
           m = POSITION_FILE_ENTRY_REGEX.match(line)
           if m.nil?
@@ -133,11 +139,20 @@ module Fluent::Plugin
               @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
             end
 
-            entries[path] = (POSITION_FILE_ENTRY_FORMAT % [path, pos, ino])
+            entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+            file_pos += line.size
           end
         end
 
-        entries.values
+        entries
+      end
+    end
+
+    Entry = Struct.new(:path, :pos, :ino, :seek) do
+      POSITION_FILE_ENTRY_FORMAT = "%s\t%016x\t%016x\n".freeze
+
+      def to_entry_fmt
+        POSITION_FILE_ENTRY_FORMAT % [path, pos, ino]
       end
     end
 
@@ -158,6 +173,8 @@ module Fluent::Plugin
         @inode = inode
       end
 
+      attr_writer :seek
+
       def update(ino, pos)
         @file_mutex.synchronize {
           @file.pos = @seek

data/lib/fluent/plugin/in_unix.rb

@@ -14,50 +14,68 @@
 #    limitations under the License.
 #
 
-require 'fileutils'
-require 'socket'
+require 'fluent/env'
+require 'fluent/plugin/input'
+require 'fluent/msgpack_factory'
 
 require 'cool.io'
 require 'yajl'
+require 'fileutils'
+require 'socket'
+
+module Fluent::Plugin
+  # TODO: This plugin will be 3rd party plugin
+  class UnixInput < Input
+    Fluent::Plugin.register_input('unix', self)
+
+    helpers :event_loop
+
+    def initialize
+      super
 
-require 'fluent/input'
-require 'fluent/event'
+      @lsock = nil
+    end
 
-module Fluent
-  # obsolete
-  class StreamInput < Input
-    config_param :blocking_timeout, :time, default: 0.5
+    desc 'The path to your Unix Domain Socket.'
+    config_param :path, :string, default: Fluent::DEFAULT_SOCKET_PATH
+    desc 'The backlog of Unix Domain Socket.'
+    config_param :backlog, :integer, default: nil
+    desc "New tag instead of incoming tag"
+    config_param :tag, :string, default: nil
+
+    def configure(conf)
+      super
+    end
 
     def start
       super
 
-      @loop = Coolio::Loop.new
       @lsock = listen
-      @loop.attach(@lsock)
-      @thread = Thread.new(&method(:run))
+      event_loop_attach(@lsock)
     end
 
     def shutdown
-      @loop.watchers.each {|w| w.detach }
-      @loop.stop
-      @lsock.close
-      @thread.join
+      if @lsock
+        event_loop_detach(@lsock)
+        @lsock.close
+      end
 
       super
     end
 
-    #def listen
-    #end
+    def listen
+      if File.exist?(@path)
+        log.warn "Found existing '#{@path}'. Remove this file for in_unix plugin"
+        File.unlink(@path)
+      end
+      FileUtils.mkdir_p(File.dirname(@path))
 
-    def run
-      @loop.run(@blocking_timeout)
-    rescue
-      log.error "unexpected error", error: $!.to_s
-      log.error_backtrace
+      log.info "listening fluent socket on #{@path}"
+      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
+      s.listen(@backlog) unless @backlog.nil?
+      s
     end
 
-    private
-
     # message Entry {
     #   1: long time
     #   2: object record
@@ -79,23 +97,27 @@ module Fluent
     #   3: object record
     # }
     def on_message(msg)
-      # TODO format error
-      tag = msg[0].to_s
+      unless msg.is_a?(Array)
+        log.warn "incoming data is broken:", msg: msg
+        return
+      end
+
+      tag = @tag || (msg[0].to_s)
       entries = msg[1]
 
-      if entries.class == String
+      case entries
+      when String
         # PackedForward
-        es = MessagePackEventStream.new(entries)
+        es = Fluent::MessagePackEventStream.new(entries)
         router.emit_stream(tag, es)
 
-      elsif entries.class == Array
+      when Array
         # Forward
-        es = MultiEventStream.new
+        es = Fluent::MultiEventStream.new
         entries.each {|e|
           record = e[1]
           next if record.nil?
-          time = e[0]
-          time = (now ||= Engine.now) if time.to_i == 0
+          time = convert_time(e[0])
           es.add(time, record)
         }
         router.emit_stream(tag, es)
@@ -105,25 +127,28 @@ module Fluent
         record = msg[2]
         return if record.nil?
 
-        time = msg[1]
-        time = Engine.now if time.to_i == 0
+        time = convert_time(msg[1])
         router.emit(tag, time, record)
       end
     end
 
+    def convert_time(time)
+      case time
+      when nil, 0
+        Fluent::EventTime.now
+      when Fluent::EventTime
+        time
+      else
+        Fluent::EventTime.from_time(Time.at(time))
+      end
+    end
+
     class Handler < Coolio::Socket
       def initialize(io, log, on_message)
         super(io)
-        if io.is_a?(TCPSocket)
-          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
-          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
-        end
+
         @on_message = on_message
         @log = log
-        @log.trace {
-          remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername) rescue nil
-          "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
-        }
       end
 
       def on_connect
@@ -131,13 +156,13 @@ module Fluent
 
       def on_read(data)
         first = data[0]
-        if first == '{' || first == '['
+        if first == '{'.freeze || first == '['.freeze
           m = method(:on_read_json)
-          @y = Yajl::Parser.new
-          @y.on_parse_complete = @on_message
+          @parser = Yajl::Parser.new
+          @parser.on_parse_complete = @on_message
         else
           m = method(:on_read_msgpack)
-          @u = Fluent::MessagePackFactory.msgpack_unpacker
+          @parser = Fluent::MessagePackFactory.msgpack_unpacker
         end
 
         singleton_class.module_eval do
@@ -147,17 +172,17 @@ module Fluent
       end
 
       def on_read_json(data)
-        @y << data
-      rescue
-        @log.error "unexpected error", error: $!.to_s
+        @parser << data
+      rescue => e
+        @log.error "unexpected error in json payload", error: e.to_s
         @log.error_backtrace
         close
       end
 
       def on_read_msgpack(data)
-        @u.feed_each(data, &@on_message)
-      rescue
-        @log.error "unexpected error", error: $!.to_s
+        @parser.feed_each(data, &@on_message)
+      rescue => e
+        @log.error "unexpected error in msgpack payload", error: e.to_s
         @log.error_backtrace
         close
       end
@@ -167,29 +192,4 @@ module Fluent
       end
     end
   end
-
-  class UnixInput < StreamInput
-    Plugin.register_input('unix', self)
-
-    desc 'The path to your Unix Domain Socket.'
-    config_param :path, :string, default: DEFAULT_SOCKET_PATH
-    desc 'The backlog of Unix Domain Socket.'
-    config_param :backlog, :integer, default: nil
-
-    def configure(conf)
-      super
-      #log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
-    end
-
-    def listen
-      if File.exist?(@path)
-        File.unlink(@path)
-      end
-      FileUtils.mkdir_p File.dirname(@path)
-      log.info "listening fluent socket on #{@path}"
-      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
-      s.listen(@backlog) unless @backlog.nil?
-      s
-    end
-  end
 end