fluentd 1.6.3 → 1.7.0.rc1

data/lib/fluent/plugin/buffer.rb

@@ -133,6 +133,15 @@ module Fluent
               cmp_variables(variables, variables2)
           end
         end
+
+        # This is an optimization code. Current Struct's implementation is comparing all data.
+        # https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203
+        # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
+        # This means that this class is one of the most called object in Fluentd.
+        # See https://github.com/fluent/fluentd/pull/2560
+        def hash
+          timekey.object_id
+        end
       end
 
       # for tests
@@ -155,7 +164,7 @@ module Fluent
 
         @stage_size = @queue_size = 0
         @timekeys = Hash.new(0)
-        @metadata_list = [] # keys of @stage
+        @mutex = Mutex.new
       end
 
       def persistent?
@@ -175,16 +184,18 @@ module Fluent
 
         @stage, @queue = resume
         @stage.each_pair do |metadata, chunk|
-          @metadata_list << metadata unless @metadata_list.include?(metadata)
           @stage_size += chunk.bytesize
-          add_timekey(metadata)
+          if chunk.metadata && chunk.metadata.timekey
+            add_timekey(metadata.timekey)
+          end
         end
         @queue.each do |chunk|
-          @metadata_list << chunk.metadata unless @metadata_list.include?(chunk.metadata)
           @queued_num[chunk.metadata] ||= 0
           @queued_num[chunk.metadata] += 1
           @queue_size += chunk.bytesize
-          add_timekey(chunk.metadata)
+          if chunk.metadata && chunk.metadata.timekey
+            add_timekey(chunk.metadata.timekey)
+          end
         end
         log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
       end
@@ -207,7 +218,7 @@ module Fluent
 
       def terminate
         super
-        @dequeued = @stage = @queue = @queued_num = @metadata_list = nil
+        @dequeued = @stage = @queue = @queued_num = nil
         @stage_size = @queue_size = 0
         @timekeys.clear
       end
@@ -230,61 +241,17 @@ module Fluent
         raise NotImplementedError, "Implement this method in child class"
       end
 
-      def metadata_list
-        synchronize do
-          @metadata_list.dup
-        end
-      end
-
-      # it's too dangerous, and use it so carefully to remove metadata for tests
-      def metadata_list_clear!
-        synchronize do
-          @metadata_list.clear
-        end
-      end
-
       def new_metadata(timekey: nil, tag: nil, variables: nil)
         Metadata.new(timekey, tag, variables)
       end
 
-      def add_metadata(metadata)
-        log.on_trace { log.trace "adding metadata", instance: self.object_id, metadata: metadata }
-
-        synchronize do
-          if i = @metadata_list.index(metadata)
-            @metadata_list[i]
-          else
-            @metadata_list << metadata
-            add_timekey(metadata)
-            metadata
-          end
-        end
-      end
-
       def metadata(timekey: nil, tag: nil, variables: nil)
-        meta = new_metadata(timekey: timekey, tag: tag, variables: variables)
-        add_metadata(meta)
-      end
-
-      def add_timekey(metadata)
-        if t = metadata.timekey
-          @timekeys[t] += 1
+        meta = Metadata.new(timekey, tag, variables)
+        if (t = meta.timekey)
+          add_timekey(t)
         end
-        nil
+        meta
       end
-      private :add_timekey
-      
-      def del_timekey(metadata)
-        if t = metadata.timekey
-          if @timekeys[t] <= 1
-            @timekeys.delete(t)
-          else
-            @timekeys[t] -= 1
-          end
-        end
-        nil
-      end
-      private :del_timekey
 
       def timekeys
         @timekeys.keys
@@ -408,13 +375,12 @@ module Fluent
         synchronize { @queue.reduce(0){|r, chunk| r + chunk.size } }
       end
 
-      def queued?(metadata=nil)
-        synchronize do
-          if metadata
-            n = @queued_num[metadata]
-            n && n.nonzero?
-          else
-            !@queue.empty?
+      def queued?(metadata = nil, optimistic: false)
+        if optimistic
+          optimistic_queued?(metadata)
+        else
+          synchronize do
+            optimistic_queued?(metadata)
           end
         end
       end
@@ -514,6 +480,7 @@ module Fluent
       end
 
       def purge_chunk(chunk_id)
+        metadata = nil
         synchronize do
           chunk = @dequeued.delete(chunk_id)
           return nil unless chunk # purged by other threads
@@ -532,13 +499,16 @@ module Fluent
 
           @dequeued_num[chunk.metadata] -= 1
           if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1) && @dequeued_num[metadata].zero?
-            @metadata_list.delete(metadata)
             @queued_num.delete(metadata)
             @dequeued_num.delete(metadata)
-            del_timekey(metadata)
           end
           log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end
+
+        if metadata && metadata.timekey
+          del_timekey(metadata.timekey)
+        end
+
         nil
       end
 
@@ -774,6 +744,35 @@ module Fluent
 
         { 'buffer' => stats }
       end
+
+      private
+
+      def optimistic_queued?(metadata = nil)
+        if metadata
+          n = @queued_num[metadata]
+          n && n.nonzero?
+        else
+          !@queue.empty?
+        end
+      end
+
+      def add_timekey(t)
+        @mutex.synchronize do
+          @timekeys[t] += 1
+        end
+        nil
+      end
+
+      def del_timekey(t)
+        @mutex.synchronize do
+          if @timekeys[t] <= 1
+            @timekeys.delete(t)
+          else
+            @timekeys[t] -= 1
+          end
+        end
+        nil
+      end
     end
   end
 end

data/lib/fluent/plugin/buffer/chunk.rb

@@ -57,13 +57,31 @@ module Fluent
           @state = :unstaged
 
           @size = 0
-          @created_at = Time.now
-          @modified_at = Time.now
+          @created_at = Fluent::Clock.real_now
+          @modified_at = Fluent::Clock.real_now
 
           extend Decompressable if compress == :gzip
         end
 
-        attr_reader :unique_id, :metadata, :created_at, :modified_at, :state
+        attr_reader :unique_id, :metadata, :state
+
+        def raw_create_at
+          @created_at
+        end
+
+        def raw_modified_at
+          @modified_at
+        end
+
+        # for compatibility
+        def created_at
+          @created_at_object ||= Time.at(@created_at)
+        end
+
+        # for compatibility
+        def modified_at
+          @modified_at_object ||= Time.at(@modified_at)
+        end
 
         # data is array of formatted record string
         def append(data, **kwargs)

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -24,6 +24,8 @@ module Fluent
       class FileChunk < Chunk
         class FileChunkError < StandardError; end
 
+        BUFFER_HEADER = "\xc1\x00".force_encoding(Encoding::ASCII_8BIT).freeze
+
         ### buffer path user specified : /path/to/directory/user_specified_prefix.*.log
         ### buffer chunk path          : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log
         ### buffer chunk metadata path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log.meta
@@ -74,7 +76,8 @@ module Fluent
           @size += @adding_size
           @bytesize += @adding_bytes
           @adding_bytes = @adding_size = 0
-          @modified_at = Time.now
+          @modified_at = Fluent::Clock.real_now
+          @modified_at_object = nil
 
           true
         end
@@ -214,14 +217,19 @@ module Fluent
         end
 
         def restore_metadata(bindata)
-          data = msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
+          data = restore_metadata_with_new_format(bindata)
+
+          unless data
+            # old type of restore
+            data = msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
+          end
 
-          now = Time.now
+          now = Fluent::Clock.real_now
 
           @unique_id = data[:id] || self.class.unique_id_from_path(@path) || @unique_id
           @size = data[:s] || 0
-          @created_at = Time.at(data.fetch(:c, now.to_i))
-          @modified_at = Time.at(data.fetch(:m, now.to_i))
+          @created_at = data.fetch(:c, now.to_i)
+          @modified_at = data.fetch(:m, now.to_i)
 
           @metadata.timekey = data[:timekey]
           @metadata.tag = data[:tag]
@@ -231,8 +239,8 @@ module Fluent
         def restore_metadata_partially(chunk)
           @unique_id = self.class.unique_id_from_path(chunk.path) || @unique_id
           @size = 0
-          @created_at = chunk.ctime # birthtime isn't supported on Windows (and Travis?)
-          @modified_at = chunk.mtime
+          @created_at = chunk.ctime.to_i # birthtime isn't supported on Windows (and Travis?)
+          @modified_at = chunk.mtime.to_i
 
           @metadata.timekey = nil
           @metadata.tag = nil
@@ -243,13 +251,13 @@ module Fluent
           data = @metadata.to_h.merge({
               id: @unique_id,
               s: (update ? @size + @adding_size : @size),
-              c: @created_at.to_i,
-              m: (update ? Time.now : @modified_at).to_i,
+              c: @created_at,
+              m: (update ? Fluent::Clock.real_now : @modified_at),
           })
-          bin = msgpack_packer.pack(data).to_s
+          bin = Fluent::MessagePackFactory.thread_local_msgpack_packer.pack(data).full_pack
+          size = [bin.bytesize].pack('N')
           @meta.seek(0, IO::SEEK_SET)
-          @meta.write(bin)
-          @meta.truncate(bin.bytesize)
+          @meta.write(BUFFER_HEADER + size + bin)
         end
 
         def file_rename(file, old_path, new_path, callback=nil)
@@ -377,6 +385,23 @@ module Fluent
           end
           @state = :queued
         end
+
+        private
+
+        def restore_metadata_with_new_format(chunk)
+          if chunk.size <= 6 # size of BUFFER_HEADER (2) + size of data size(4)
+            return nil
+          end
+
+          if chunk.slice(0, 2) == BUFFER_HEADER
+            size = chunk.slice(2, 4).unpack('N').first
+            if size
+              return msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
+            end
+          end
+
+          nil
+        end
       end
     end
   end

data/lib/fluent/plugin/buffer/file_single_chunk.rb

@@ -0,0 +1,314 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'uri'
+require 'fluent/plugin/buffer/chunk'
+require 'fluent/unique_id'
+require 'fluent/msgpack_factory'
+
+module Fluent
+  module Plugin
+    class Buffer
+      class FileSingleChunk < Chunk
+        class FileChunkError < StandardError; end
+
+        ## buffer path user specified : /path/to/directory
+        ## buffer chunk path          : /path/to/directory/fsb.key.b513b61c9791029c2513b61c9791029c2.buf
+        ## state: b/q - 'b'(on stage), 'q'(enqueued)
+
+        include SystemConfig::Mixin
+        include MessagePackFactory::Mixin
+
+        PATH_EXT = 'buf'
+        PATH_SUFFIX = ".#{PATH_EXT}"
+        PATH_REGEXP = /\.(b|q)([0-9a-f]+)\.#{PATH_EXT}*\Z/n  # //n switch means explicit 'ASCII-8BIT' pattern
+        FILE_PERMISSION = 0644
+
+        attr_reader :path, :permission
+
+        def initialize(metadata, path, mode, key, perm: system_config.file_permission || FILE_PERMISSION, compress: :text)
+          super(metadata, compress: compress)
+          @key = key
+          @permission = perm.is_a?(String) ? perm.to_i(8) : perm
+          @bytesize = @size = @adding_bytes = @adding_size = 0
+
+          case mode
+          when :create then create_new_chunk(path, metadata, @permission)
+          when :staged then load_existing_staged_chunk(path)
+          when :queued then load_existing_enqueued_chunk(path)
+          else
+            raise ArgumentError, "Invalid file chunk mode: #{mode}"
+          end
+        end
+
+        def concat(bulk, bulk_size)
+          raise "BUG: concatenating to unwritable chunk, now '#{self.state}'" unless self.writable?
+
+          bulk.force_encoding(Encoding::ASCII_8BIT)
+          @chunk.write(bulk)
+          @adding_bytes += bulk.bytesize
+          @adding_size += bulk_size
+          true
+        end
+
+        def commit
+          @commit_position = @chunk.pos
+          @size += @adding_size
+          @bytesize += @adding_bytes
+          @adding_bytes = @adding_size = 0
+          @modified_at = Fluent::Clock.real_now
+
+          true
+        end
+
+        def rollback
+          if @chunk.pos != @commit_position
+            @chunk.seek(@commit_position, IO::SEEK_SET)
+            @chunk.truncate(@commit_position)
+          end
+          @adding_bytes = @adding_size = 0
+          true
+        end
+
+        def bytesize
+          @bytesize + @adding_bytes
+        end
+
+        def size
+          @size + @adding_size
+        end
+
+        def empty?
+          @bytesize.zero?
+        end
+
+        def enqueued!
+          return unless self.staged?
+
+          new_chunk_path = self.class.generate_queued_chunk_path(@path, @unique_id)
+
+          begin
+            file_rename(@chunk, @path, new_chunk_path, ->(new_io) { @chunk = new_io })
+          rescue => e
+            begin
+              file_rename(@chunk, new_chunk_path, @path, ->(new_io) { @chunk = new_io }) if File.exist?(new_chunk_path)
+            rescue => re
+              # In this point, restore buffer state is hard because previous `file_rename` failed by resource problem.
+              # Retry is one possible approach but it may cause livelock under limited resources or high load environment.
+              # So we ignore such errors for now and log better message instead.
+              # "Too many open files" should be fixed by proper buffer configuration and system setting.
+              raise "can't enqueue buffer file and failed to restore. This may causes inconsistent state: path = #{@path}, error = '#{e}', retry error = '#{re}'"
+            else
+              raise "can't enqueue buffer file: path = #{@path}, error = '#{e}'"
+            end
+          end
+
+          @path = new_chunk_path
+
+          super
+        end
+
+        def close
+          super
+          size = @chunk.size
+          @chunk.close
+          if size == 0
+            File.unlink(@path)
+          end
+        end
+
+        def purge
+          super
+          @chunk.close
+          @bytesize = @size = @adding_bytes = @adding_size = 0
+          File.unlink(@path)
+        end
+
+        def read(**kwargs)
+          @chunk.seek(0, IO::SEEK_SET)
+          @chunk.read
+        end
+
+        def open(**kwargs, &block)
+          @chunk.seek(0, IO::SEEK_SET)
+          val = yield @chunk
+          @chunk.seek(0, IO::SEEK_END) if self.staged?
+          val
+        end
+
+        def self.assume_chunk_state(path)
+          return :unknown unless path.end_with?(PATH_SUFFIX)
+
+          if PATH_REGEXP =~ path
+            $1 == 'b' ? :staged : :queued
+          else
+            # files which matches to glob of buffer file pattern
+            # it includes files which are created by out_file
+            :unknown
+          end
+        end
+
+        def self.unique_id_and_key_from_path(path)
+          base = File.basename(path)
+          res = PATH_REGEXP =~ base
+          return nil unless res
+
+          key = base[4..res - 1] # remove 'fsb.' and '.'
+          hex_id = $2            # remove '.' and '.buf'
+          unique_id = hex_id.scan(/../).map {|x| x.to_i(16) }.pack('C*')
+          [unique_id, key]
+        end
+
+        def self.generate_stage_chunk_path(path, key, unique_id)
+          pos = path.index('.*.')
+          raise "BUG: buffer chunk path on stage MUST have '.*.'" unless pos
+
+          prefix = path[0...pos]
+          suffix = path[(pos + 3)..-1]
+
+          chunk_id = Fluent::UniqueId.hex(unique_id)
+          "#{prefix}.#{key}.b#{chunk_id}.#{suffix}"
+        end
+
+        def self.generate_queued_chunk_path(path, unique_id)
+          chunk_id = Fluent::UniqueId.hex(unique_id)
+          staged_path = ".b#{chunk_id}."
+          if path.index(staged_path)
+            path.sub(staged_path, ".q#{chunk_id}.")
+          else # for unexpected cases (ex: users rename files while opened by fluentd)
+            path + ".q#{chunk_id}.chunk"
+          end
+        end
+
+        def restore_metadata
+          if res = self.class.unique_id_and_key_from_path(@path)
+            @unique_id = res.first
+            key = decode_key(res.last)
+            if @key
+              @metadata.variables = {@key => key}
+            else
+              @metadata.tag = key
+            end
+          else
+            raise FileChunkError, "Invalid chunk found. unique_id and key not exist: #{@path}"
+          end
+          @size = 0
+
+          stat = File.stat(@path)
+          @created_at = stat.ctime.to_i
+          @modified_at = stat.mtime.to_i
+        end
+
+        def restore_size(chunk_format)
+          count = 0
+          File.open(@path, 'rb') { |f|
+            if chunk_format == :msgpack
+              msgpack_unpacker(f).each { |d| count += 1 }
+            else
+              f.each_line { |l| count += 1 }
+            end
+          }
+          @size = count
+        end
+
+        def file_rename(file, old_path, new_path, callback = nil)
+          pos = file.pos
+          if Fluent.windows?
+            file.close
+            File.rename(old_path, new_path)
+            file = File.open(new_path, 'rb', @permission)
+          else
+            File.rename(old_path, new_path)
+            file.reopen(new_path, 'rb')
+          end
+          file.set_encoding(Encoding::ASCII_8BIT)
+          file.sync = true
+          file.binmode
+          file.pos = pos
+          callback.call(file) if callback
+        end
+
+        URI_PARSER = URI::Parser.new
+        ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n
+
+        def encode_key(metadata)
+          k = @key ? metadata.variables[@key] : metadata.tag
+          k ||= ''
+          URI_PARSER.escape(k, ESCAPE_REGEXP)
+        end
+
+        def decode_key(key)
+          URI_PARSER.unescape(key)
+        end
+
+        def create_new_chunk(path, metadata, perm)
+          @path = self.class.generate_stage_chunk_path(path, encode_key(metadata), @unique_id)
+          begin
+            @chunk = File.open(@path, 'wb+', perm)
+            @chunk.set_encoding(Encoding::ASCII_8BIT)
+            @chunk.sync = true
+            @chunk.binmode
+          rescue => e
+            # Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
+            # If other cases are possible, we will change erorr handling with proper classes.
+            raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
+          end
+
+          @state = :unstaged
+          @bytesize = 0
+          @commit_position = @chunk.pos # must be 0
+          @adding_bytes = 0
+          @adding_size = 0
+        end
+
+        def load_existing_staged_chunk(path)
+          @path = path
+          raise FileChunkError, "staged file chunk is empty" if File.size(@path).zero?
+
+          @chunk = File.open(@path, 'rb+')
+          @chunk.set_encoding(Encoding::ASCII_8BIT)
+          @chunk.sync = true
+          @chunk.binmode
+          @chunk.seek(0, IO::SEEK_END)
+
+          restore_metadata
+
+          @state = :staged
+          @bytesize = @chunk.size
+          @commit_position = @chunk.pos
+          @adding_bytes = 0
+          @adding_size = 0
+        end
+
+        def load_existing_enqueued_chunk(path)
+          @path = path
+          raise FileChunkError, "enqueued file chunk is empty" if File.size(@path).zero?
+
+          @chunk = File.open(@path, 'rb')
+          @chunk.set_encoding(Encoding::ASCII_8BIT)
+          @chunk.binmode
+          @chunk.seek(0, IO::SEEK_SET)
+
+          restore_metadata
+
+          @state = :queued
+          @bytesize = @chunk.size
+          @commit_position = @chunk.size
+        end
+      end
+    end
+  end
+end