fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/lib/fluent/plugin/base.rb

@@ -156,6 +156,18 @@ module Fluent
         @_state.terminate
       end
 
+      def called_in_test?
+        caller_locations.each do |location|
+          # Thread::Backtrace::Location#path returns base filename or absolute path.
+          # #absolute_path returns absolute_path always.
+          # https://bugs.ruby-lang.org/issues/12159
+          if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+            return true
+          end
+        end
+        false
+      end
+
       def inspect
         # Plugin instances are sometimes too big to dump because it may have too many thins (buffer,storage, ...)
         # Original commit comment says that:

data/lib/fluent/plugin/buf_file.rb

@@ -66,7 +66,7 @@ module Fluent
         end
 
         type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
-        if @@buffer_paths.has_key?(@path) && !buffer_path_for_test?
+        if @@buffer_paths.has_key?(@path) && !called_in_test?
           type_using_this_path = @@buffer_paths[@path]
           raise ConfigError, "Other '#{type_using_this_path}' plugin already use same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
         end
@@ -114,18 +114,6 @@ module Fluent
         @multi_workers_available
       end
 
-      def buffer_path_for_test?
-        caller_locations.each do |location|
-          # Thread::Backtrace::Location#path returns base filename or absolute path.
-          # #absolute_path returns absolute_path always.
-          # https://bugs.ruby-lang.org/issues/12159
-          if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
-            return true
-          end
-        end
-        false
-      end
-
       def start
         FileUtils.mkdir_p File.dirname(@path), mode: @dir_permission
 
@@ -153,7 +141,13 @@ module Fluent
             next
           end
 
-          chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
+          begin
+            chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode) # file chunk resumes contents of metadata
+          rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
+            handle_broken_files(path, mode, e)
+            next
+          end
+
           case chunk.state
           when :staged
             stage[chunk.metadata] = chunk
@@ -170,10 +164,20 @@ module Fluent
       def generate_chunk(metadata)
         # FileChunk generates real path with unique_id
         if @file_permission
-          Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
+          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, perm: @file_permission, compress: @compress)
         else
-          Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
+          chunk = Fluent::Plugin::Buffer::FileChunk.new(metadata, @path, :create, compress: @compress)
         end
+
+        log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
+
+        return chunk
+      end
+
+      def handle_broken_files(path, mode, e)
+        log.error "found broken chunk file during resume. Deleted corresponding files:", :path => path, :mode => mode, :err_msg => e.message
+        # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
+        File.unlink(path, path + '.meta') rescue nil
       end
     end
   end

data/lib/fluent/plugin/buffer.rb

@@ -55,6 +55,9 @@ module Fluent
       # if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
       config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
 
+      desc 'The max number of queued chunks.'
+      config_param :queued_chunks_limit_size, :integer, default: nil
+
       desc 'Compress buffered data.'
       config_param :compress, :enum, list: [:text, :gzip], default: :text
 
@@ -62,6 +65,74 @@ module Fluent
         def empty?
           timekey.nil? && tag.nil? && variables.nil?
         end
+
+        def cmp_variables(v1, v2)
+          if v1.nil? && v2.nil?
+            return 0
+          elsif v1.nil? # v2 is non-nil
+            return -1
+          elsif v2.nil? # v1 is non-nil
+            return 1
+          end
+          # both of v1 and v2 are non-nil
+          v1_sorted_keys = v1.keys.sort
+          v2_sorted_keys = v2.keys.sort
+          if v1_sorted_keys != v2_sorted_keys
+            if v1_sorted_keys.size == v2_sorted_keys.size
+              v1_sorted_keys <=> v2_sorted_keys
+            else
+              v1_sorted_keys.size <=> v2_sorted_keys.size
+            end
+          else
+            v1_sorted_keys.each do |k|
+              a = v1[k]
+              b = v2[k]
+              if a && b && a != b
+                return a <=> b
+              elsif a && b || (!a && !b) # same value (including both are nil)
+                next
+              elsif a # b is nil
+                return 1
+              else # a is nil (but b is non-nil)
+                return -1
+              end
+            end
+
+            0
+          end
+        end
+
+        def <=>(o)
+          timekey2 = o.timekey
+          tag2 = o.tag
+          variables2 = o.variables
+          if (!!timekey ^ !!timekey2) || (!!tag ^ !!tag2) || (!!variables ^ !!variables2)
+            # One has value in a field, but another doesn't have value in same field
+            # This case occurs very rarely
+            if timekey == timekey2 # including the case of nil == nil
+              if tag == tag2
+                cmp_variables(variables, variables2)
+              elsif tag.nil?
+                -1
+              elsif tag2.nil?
+                1
+              else
+                tag <=> tag2
+              end
+            elsif timekey.nil?
+              -1
+            elsif timekey2.nil?
+              1
+            else
+              timekey <=> timekey2
+            end
+          else
+            # objects have values in same field pairs (comparison with non-nil and nil doesn't occur here)
+            (timekey <=> timekey2 || 0).nonzero? || # if `a <=> b` is nil, then both are nil
+              (tag <=> tag2 || 0).nonzero? ||
+              cmp_variables(variables, variables2)
+          end
+        end
       end
 
       # for tests
@@ -80,6 +151,7 @@ module Fluent
         @queue = []    #=> Array (chunks)           : already flushed (not written)
         @dequeued = {} #=> Hash (unique_id -> chunk): already written (not purged)
         @queued_num = {} # metadata => int (number of queued chunks)
+        @dequeued_num = {} # metadata => int (number of dequeued chunks)
 
         @stage_size = @queue_size = 0
         @metadata_list = [] # keys of @stage
@@ -172,7 +244,8 @@ module Fluent
       end
 
       def add_metadata(metadata)
-        log.trace "adding metadata", instance: self.object_id, metadata: metadata
+        log.on_trace { log.trace "adding metadata", instance: self.object_id, metadata: metadata }
+
         synchronize do
           if i = @metadata_list.index(metadata)
             @metadata_list[i]
@@ -195,7 +268,7 @@ module Fluent
         return if metadata_and_data.size < 1
         raise BufferOverflowError, "buffer space has too many data" unless storable?
 
-        log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size
+        log.on_trace { log.trace "writing events into buffer", instance: self.object_id, metadata_size: metadata_and_data.size }
 
         staged_bytesize = 0
         operated_chunks = []
@@ -203,7 +276,9 @@ module Fluent
         chunks_to_enqueue = []
 
         begin
-          metadata_and_data.each do |metadata, data|
+          # sort metadata to get lock of chunks in same order with other threads
+          metadata_and_data.keys.sort.each do |metadata|
+            data = metadata_and_data[metadata]
             write_once(metadata, data, format: format, size: size) do |chunk, adding_bytesize|
               chunk.mon_enter # add lock to prevent to be committed/rollbacked from other threads
               operated_chunks << chunk
@@ -296,6 +371,10 @@ module Fluent
         end
       end
 
+      def queue_full?
+        synchronize { @queue.size } >= @queued_chunks_limit_size
+      end
+
       def queued_records
         synchronize { @queue.reduce(0){|r, chunk| r + chunk.size } }
       end
@@ -312,60 +391,70 @@ module Fluent
       end
 
       def enqueue_chunk(metadata)
-        log.trace "enqueueing chunk", instance: self.object_id, metadata: metadata
-        synchronize do
-          chunk = @stage.delete(metadata)
-          return nil unless chunk
+        log.on_trace { log.trace "enqueueing chunk", instance: self.object_id, metadata: metadata }
 
-          chunk.synchronize do
+        chunk = synchronize do
+          @stage.delete(metadata)
+        end
+        return nil unless chunk
+
+        chunk.synchronize do
+          synchronize do
             if chunk.empty?
               chunk.close
             else
               @queue << chunk
               @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
-              chunk.enqueued! if chunk.respond_to?(:enqueued!)
+              chunk.enqueued!
             end
+            bytesize = chunk.bytesize
+            @stage_size -= bytesize
+            @queue_size += bytesize
           end
-          bytesize = chunk.bytesize
-          @stage_size -= bytesize
-          @queue_size += bytesize
         end
         nil
       end
 
       def enqueue_unstaged_chunk(chunk)
-        log.trace "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata
+        log.on_trace { log.trace "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata }
+
         synchronize do
           chunk.synchronize do
             metadata = chunk.metadata
             @queue << chunk
             @queued_num[metadata] = @queued_num.fetch(metadata, 0) + 1
-            chunk.enqueued! if chunk.respond_to?(:enqueued!)
+            chunk.enqueued!
           end
           @queue_size += chunk.bytesize
         end
       end
 
-      def enqueue_all
-        log.trace "enqueueing all chunks in buffer", instance: self.object_id
-        synchronize do
-          if block_given?
-            @stage.keys.each do |metadata|
-              chunk = @stage[metadata]
-              v = yield metadata, chunk
-              enqueue_chunk(metadata) if v
-            end
-          else
-            @stage.keys.each do |metadata|
-              enqueue_chunk(metadata)
-            end
+      # At flush_at_shutdown, all staged chunks should be enqueued for buffer flush. Set true to force_enqueue for it.
+      def enqueue_all(force_enqueue = false)
+        log.on_trace { log.trace "enqueueing all chunks in buffer", instance: self.object_id }
+
+        if block_given?
+          synchronize{ @stage.keys }.each do |metadata|
+            return if !force_enqueue && queue_full?
+            # NOTE: The following line might cause data race depending on Ruby implementations except CRuby
+            # cf. https://github.com/fluent/fluentd/pull/1721#discussion_r146170251
+            chunk = @stage[metadata]
+            next unless chunk
+            v = yield metadata, chunk
+            enqueue_chunk(metadata) if v
+          end
+        else
+          synchronize{ @stage.keys }.each do |metadata|
+            return if !force_enqueue && queue_full?
+            enqueue_chunk(metadata)
           end
         end
       end
 
       def dequeue_chunk
         return nil if @queue.empty?
-        log.trace "dequeueing a chunk", instance: self.object_id
+        log.on_trace { log.trace "dequeueing a chunk", instance: self.object_id }
+
         synchronize do
           chunk = @queue.shift
 
@@ -374,19 +463,23 @@ module Fluent
 
           @dequeued[chunk.unique_id] = chunk
           @queued_num[chunk.metadata] -= 1 # BUG if nil, 0 or subzero
+          @dequeued_num[chunk.metadata] ||= 0
+          @dequeued_num[chunk.metadata] += 1
           log.trace "chunk dequeued", instance: self.object_id, metadata: chunk.metadata
           chunk
         end
       end
 
       def takeback_chunk(chunk_id)
-        log.trace "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id)
+        log.on_trace { log.trace "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id) }
+
         synchronize do
           chunk = @dequeued.delete(chunk_id)
           return false unless chunk # already purged by other thread
           @queue.unshift(chunk)
           log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
           @queued_num[chunk.metadata] += 1 # BUG if nil
+          @dequeued_num[chunk.metadata] -= 1
         end
         true
       end
@@ -397,7 +490,8 @@ module Fluent
           return nil unless chunk # purged by other threads
 
           metadata = chunk.metadata
-          log.trace "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
+          log.on_trace { log.trace "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }
+
           begin
             bytesize = chunk.bytesize
             chunk.purge
@@ -407,8 +501,11 @@ module Fluent
             log.error_backtrace
           end
 
-          if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1)
+          @dequeued_num[chunk.metadata] -= 1
+          if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1) && @dequeued_num[metadata].zero?
             @metadata_list.delete(metadata)
+            @queued_num.delete(metadata)
+            @dequeued_num.delete(metadata)
           end
           log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end
@@ -416,7 +513,8 @@ module Fluent
       end
 
       def clear_queue!
-        log.trace "clearing queue", instance: self.object_id
+        log.on_trace { log.trace "clearing queue", instance: self.object_id }
+
         synchronize do
           until @queue.empty?
             begin

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -22,6 +22,8 @@ module Fluent
   module Plugin
     class Buffer
       class FileChunk < Chunk
+        class FileChunkError < StandardError; end
+
         ### buffer path user specified : /path/to/directory/user_specified_prefix.*.log
         ### buffer chunk path          : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log
         ### buffer chunk metadata path : /path/to/directory/user_specified_prefix.b513b61c9791029c2513b61c9791029c2.log.meta
@@ -244,9 +246,10 @@ module Fluent
               c: @created_at.to_i,
               m: (update ? Time.now : @modified_at).to_i,
           })
+          bin = msgpack_packer.pack(data).to_s
           @meta.seek(0, IO::SEEK_SET)
-          @meta.truncate(0)
-          @meta.write(msgpack_packer.pack(data))
+          @meta.write(bin)
+          @meta.truncate(bin.bytesize)
         end
 
         def file_rename(file, old_path, new_path, callback=nil)
@@ -284,6 +287,7 @@ module Fluent
             @meta.set_encoding(Encoding::ASCII_8BIT)
             @meta.sync = true
             @meta.binmode
+            write_metadata(update: false)
           rescue => e
             # This case is easier than enqueued!. Just removing pre-create buffer file
             @chunk.close rescue nil
@@ -307,6 +311,8 @@ module Fluent
           # staging buffer chunk without metadata is classic buffer chunk file
           # and it should be enqueued immediately
           if File.exist?(@meta_path)
+            raise FileChunkError, "staged file chunk is empty" if File.size(@path).zero?
+
             @chunk = File.open(@path, 'rb+')
             @chunk.set_encoding(Encoding::ASCII_8BIT)
             @chunk.sync = true
@@ -317,7 +323,13 @@ module Fluent
             @meta.set_encoding(Encoding::ASCII_8BIT)
             @meta.sync = true
             @meta.binmode
-            restore_metadata(@meta.read)
+            begin
+              restore_metadata(@meta.read)
+            rescue => e
+              @chunk.close
+              @meta.close
+              raise FileChunkError, "staged meta file is broken. #{e.message}"
+            end
             @meta.seek(0, IO::SEEK_SET)
 
             @state = :staged
@@ -343,6 +355,8 @@ module Fluent
 
         def load_existing_enqueued_chunk(path)
           @path = path
+          raise FileChunkError, "enqueued file chunk is empty" if File.size(@path).zero?
+
           @chunk = File.open(@path, 'rb')
           @chunk.set_encoding(Encoding::ASCII_8BIT)
           @chunk.binmode
@@ -352,7 +366,12 @@ module Fluent
 
           @meta_path = @path + '.meta'
           if File.readable?(@meta_path)
-            restore_metadata(File.open(@meta_path){|f| f.set_encoding(Encoding::ASCII_8BIT); f.binmode; f.read })
+            begin
+              restore_metadata(File.open(@meta_path){|f| f.set_encoding(Encoding::ASCII_8BIT); f.binmode; f.read })
+            rescue => e
+              @chunk.close
+              raise FileChunkError, "enqueued meta file is broken. #{e.message}"
+            end
           else
             restore_metadata_partially(@chunk)
           end

data/lib/fluent/plugin/compressable.rb

@@ -60,7 +60,7 @@ module Fluent
         out = ''
         loop do
           gz = Zlib::GzipReader.new(io)
-          out += gz.read
+          out << gz.read
           unused = gz.unused
           gz.finish