fluentd 1.6.3 → 1.7.0

data/lib/fluent/plugin/bare_output.rb

@@ -40,7 +40,7 @@ module Fluent
 
       def initialize
         super
-        @counters_monitor = Monitor.new
+        @counter_mutex = Mutex.new
         # TODO: well organized counters
         @num_errors = 0
         @emit_count = 0
@@ -48,12 +48,12 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counters_monitor.synchronize{ @emit_count += 1 }
+        @counter_mutex.synchronize{ @emit_count += 1 }
         begin
           process(tag, es)
-          @counters_monitor.synchronize{ @emit_records += es.size }
+          @counter_mutex.synchronize{ @emit_records += es.size }
         rescue
-          @counters_monitor.synchronize{ @num_errors += 1 }
+          @counter_mutex.synchronize{ @num_errors += 1 }
           raise
         end
       end

data/lib/fluent/plugin/buf_file_single.rb

@@ -0,0 +1,211 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fileutils'
+
+require 'fluent/plugin/buffer'
+require 'fluent/plugin/buffer/file_single_chunk'
+require 'fluent/system_config'
+
+module Fluent
+  module Plugin
+    class FileSingleBuffer < Fluent::Plugin::Buffer
+      Plugin.register_buffer('file_single', self)
+
+      include SystemConfig::Mixin
+
+      DEFAULT_CHUNK_LIMIT_SIZE = 256 * 1024 * 1024        # 256MB
+      DEFAULT_TOTAL_LIMIT_SIZE =  64 * 1024 * 1024 * 1024 #  64GB
+
+      PATH_SUFFIX = ".#{Fluent::Plugin::Buffer::FileSingleChunk::PATH_EXT}"
+      DIR_PERMISSION = 0755
+
+      desc 'The path where buffer chunks are stored.'
+      config_param :path, :string, default: nil
+      desc 'Calculate the number of record in chunk during resume'
+      config_param :calc_num_records, :bool, default: true
+      desc 'The format of chunk. This is used to calculate the number of record'
+      config_param :chunk_format, :enum, list: [:msgpack, :text, :auto], default: :auto
+
+      config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
+      config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
+
+      desc 'The permission of chunk file. If no specified, <system> setting or 0644 is used'
+      config_param :file_permission, :string, default: nil
+      desc 'The permission of chunk directory. If no specified, <system> setting or 0755 is used'
+      config_param :dir_permission, :string, default: nil
+
+      @@buffer_paths = {}
+
+      def initialize
+        super
+
+        @multi_workers_available = false
+        @additional_resume_path = nil
+      end
+
+      def configure(conf)
+        super
+
+        if @chunk_format == :auto
+          @chunk_format = owner.formatted_to_msgpack_binary? ? :msgpack : :text
+        end
+
+        @key_in_path = nil
+        if owner.chunk_keys.empty?
+          log.debug "use event tag for buffer key"
+        else
+          if owner.chunk_key_tag
+            raise Fluent::ConfigError, "chunk keys must be tag or one field"
+          elsif owner.chunk_keys.size > 1
+            raise Fluent::ConfigError, "2 or more chunk keys is not allowed"
+          else
+            @key_in_path = owner.chunk_keys.first.to_sym
+          end
+        end
+
+        multi_workers_configured = owner.system_config.workers > 1
+        using_plugin_root_dir = false
+        unless @path
+          if root_dir = owner.plugin_root_dir
+            @path = File.join(root_dir, 'buffer')
+            using_plugin_root_dir = true # plugin_root_dir path contains worker id
+          else
+            raise Fluent::ConfigError, "buffer path is not configured. specify 'path' in <buffer>"
+          end
+        end
+
+        type_of_owner = Plugin.lookup_type_from_class(@_owner.class)
+        if @@buffer_paths.has_key?(@path) && !called_in_test?
+          type_using_this_path = @@buffer_paths[@path]
+          raise Fluent::ConfigError, "Other '#{type_using_this_path}' plugin already uses same buffer path: type = #{type_of_owner}, buffer path = #{@path}"
+        end
+
+        @@buffer_paths[@path] = type_of_owner
+
+        specified_directory_exists = File.exist?(@path) && File.directory?(@path)
+        unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
+
+        if specified_directory_exists || unexisting_path_for_directory # directory
+          if using_plugin_root_dir || !multi_workers_configured
+            @path = File.join(@path, "fsb.*#{PATH_SUFFIX}")
+          else
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "fsb.*#{PATH_SUFFIX}")
+            if fluentd_worker_id == 0
+              # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
+              @additional_resume_path = File.join(File.expand_path("../../", @path), "fsb.*#{PATH_SUFFIX}")
+            end
+          end
+          @multi_workers_available = true
+        else # specified path is file path
+          if File.basename(@path).include?('.*.')
+            new_path = File.join(File.dirname(@path), "fsb.*#{PATH_SUFFIX}")
+            log.warn "file_single doesn't allow user specified 'prefix.*.suffix' style path. Use '#{new_path}' for file instead: #{@path}"
+            @path = new_path
+          elsif File.basename(@path).end_with?('.*')
+            @path = @path + PATH_SUFFIX
+          else
+            # existing file will be ignored
+            @path = @path + ".*#{PATH_SUFFIX}"
+          end
+          @multi_workers_available = false
+        end
+
+        @dir_permission = if @dir_permission
+                            @dir_permission.to_i(8)
+                          else
+                            system_config.dir_permission || DIR_PERMISSION
+                          end
+      end
+
+      # This method is called only when multi worker is configured
+      def multi_workers_ready?
+        unless @multi_workers_available
+          log.error "file_single buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id"
+        end
+        @multi_workers_available
+      end
+
+      def start
+        FileUtils.mkdir_p(File.dirname(@path), mode: @dir_permission)
+
+        super
+      end
+
+      def persistent?
+        true
+      end
+
+      def resume
+        stage = {}
+        queue = []
+
+        patterns = [@path]
+        patterns.unshift @additional_resume_path if @additional_resume_path
+        Dir.glob(patterns) do |path|
+          next unless File.file?(path)
+
+          log.debug { "restoring buffer file: path = #{path}" }
+
+          m = new_metadata() # this metadata will be updated in FileSingleChunk.new
+          mode = Fluent::Plugin::Buffer::FileSingleChunk.assume_chunk_state(path)
+          if mode == :unknown
+            log.debug "unknown state chunk found", path: path
+            next
+          end
+
+          begin
+            chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path)
+            chunk.restore_size(@chunk_format) if @calc_num_records
+          rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
+            handle_broken_files(path, mode, e)
+            next
+          end
+
+          case chunk.state
+          when :staged
+            stage[chunk.metadata] = chunk
+          when :queued
+            queue << chunk
+          end
+        end
+
+        queue.sort_by!(&:modified_at)
+
+        return stage, queue
+      end
+
+      def generate_chunk(metadata)
+        # FileChunk generates real path with unique_id
+        if @file_permission
+          chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, perm: @file_permission, compress: @compress)
+        else
+          chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(metadata, @path, :create, @key_in_path, compress: @compress)
+        end
+
+        log.debug "Created new chunk", chunk_id: dump_unique_id_hex(chunk.unique_id), metadata: metadata
+
+        chunk
+      end
+
+      def handle_broken_files(path, mode, e)
+        log.error "found broken chunk file during resume. Delete corresponding files:", path: path, mode: mode, err_msg: e.message
+        # After support 'backup_dir' feature, these files are moved to backup_dir instead of unlink.
+        File.unlink(path) rescue nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin/buffer.rb

@@ -133,6 +133,15 @@ module Fluent
               cmp_variables(variables, variables2)
           end
         end
+
+        # This is an optimization code. Current Struct's implementation is comparing all data.
+        # https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203
+        # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
+        # This means that this class is one of the most called object in Fluentd.
+        # See https://github.com/fluent/fluentd/pull/2560
+        def hash
+          timekey.object_id
+        end
       end
 
       # for tests
@@ -155,7 +164,7 @@ module Fluent
 
         @stage_size = @queue_size = 0
         @timekeys = Hash.new(0)
-        @metadata_list = [] # keys of @stage
+        @mutex = Mutex.new
       end
 
       def persistent?
@@ -175,16 +184,18 @@ module Fluent
 
         @stage, @queue = resume
         @stage.each_pair do |metadata, chunk|
-          @metadata_list << metadata unless @metadata_list.include?(metadata)
           @stage_size += chunk.bytesize
-          add_timekey(metadata)
+          if chunk.metadata && chunk.metadata.timekey
+            add_timekey(metadata.timekey)
+          end
         end
         @queue.each do |chunk|
-          @metadata_list << chunk.metadata unless @metadata_list.include?(chunk.metadata)
           @queued_num[chunk.metadata] ||= 0
           @queued_num[chunk.metadata] += 1
           @queue_size += chunk.bytesize
-          add_timekey(chunk.metadata)
+          if chunk.metadata && chunk.metadata.timekey
+            add_timekey(chunk.metadata.timekey)
+          end
         end
         log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
       end
@@ -207,7 +218,7 @@ module Fluent
 
       def terminate
         super
-        @dequeued = @stage = @queue = @queued_num = @metadata_list = nil
+        @dequeued = @stage = @queue = @queued_num = nil
         @stage_size = @queue_size = 0
         @timekeys.clear
       end
@@ -230,61 +241,17 @@ module Fluent
         raise NotImplementedError, "Implement this method in child class"
       end
 
-      def metadata_list
-        synchronize do
-          @metadata_list.dup
-        end
-      end
-
-      # it's too dangerous, and use it so carefully to remove metadata for tests
-      def metadata_list_clear!
-        synchronize do
-          @metadata_list.clear
-        end
-      end
-
       def new_metadata(timekey: nil, tag: nil, variables: nil)
         Metadata.new(timekey, tag, variables)
       end
 
-      def add_metadata(metadata)
-        log.on_trace { log.trace "adding metadata", instance: self.object_id, metadata: metadata }
-
-        synchronize do
-          if i = @metadata_list.index(metadata)
-            @metadata_list[i]
-          else
-            @metadata_list << metadata
-            add_timekey(metadata)
-            metadata
-          end
-        end
-      end
-
       def metadata(timekey: nil, tag: nil, variables: nil)
-        meta = new_metadata(timekey: timekey, tag: tag, variables: variables)
-        add_metadata(meta)
-      end
-
-      def add_timekey(metadata)
-        if t = metadata.timekey
-          @timekeys[t] += 1
+        meta = Metadata.new(timekey, tag, variables)
+        if (t = meta.timekey)
+          add_timekey(t)
         end
-        nil
+        meta
       end
-      private :add_timekey
-      
-      def del_timekey(metadata)
-        if t = metadata.timekey
-          if @timekeys[t] <= 1
-            @timekeys.delete(t)
-          else
-            @timekeys[t] -= 1
-          end
-        end
-        nil
-      end
-      private :del_timekey
 
       def timekeys
         @timekeys.keys
@@ -408,13 +375,12 @@ module Fluent
         synchronize { @queue.reduce(0){|r, chunk| r + chunk.size } }
       end
 
-      def queued?(metadata=nil)
-        synchronize do
-          if metadata
-            n = @queued_num[metadata]
-            n && n.nonzero?
-          else
-            !@queue.empty?
+      def queued?(metadata = nil, optimistic: false)
+        if optimistic
+          optimistic_queued?(metadata)
+        else
+          synchronize do
+            optimistic_queued?(metadata)
           end
         end
       end
@@ -514,6 +480,7 @@ module Fluent
       end
 
       def purge_chunk(chunk_id)
+        metadata = nil
         synchronize do
           chunk = @dequeued.delete(chunk_id)
           return nil unless chunk # purged by other threads
@@ -532,13 +499,16 @@ module Fluent
 
           @dequeued_num[chunk.metadata] -= 1
           if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1) && @dequeued_num[metadata].zero?
-            @metadata_list.delete(metadata)
             @queued_num.delete(metadata)
             @dequeued_num.delete(metadata)
-            del_timekey(metadata)
           end
           log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end
+
+        if metadata && metadata.timekey
+          del_timekey(metadata.timekey)
+        end
+
         nil
       end
 
@@ -774,6 +744,35 @@ module Fluent
 
         { 'buffer' => stats }
       end
+
+      private
+
+      def optimistic_queued?(metadata = nil)
+        if metadata
+          n = @queued_num[metadata]
+          n && n.nonzero?
+        else
+          !@queue.empty?
+        end
+      end
+
+      def add_timekey(t)
+        @mutex.synchronize do
+          @timekeys[t] += 1
+        end
+        nil
+      end
+
+      def del_timekey(t)
+        @mutex.synchronize do
+          if @timekeys[t] <= 1
+            @timekeys.delete(t)
+          else
+            @timekeys[t] -= 1
+          end
+        end
+        nil
+      end
     end
   end
 end

data/lib/fluent/plugin/buffer/chunk.rb

@@ -57,13 +57,31 @@ module Fluent
           @state = :unstaged
 
           @size = 0
-          @created_at = Time.now
-          @modified_at = Time.now
+          @created_at = Fluent::Clock.real_now
+          @modified_at = Fluent::Clock.real_now
 
           extend Decompressable if compress == :gzip
         end
 
-        attr_reader :unique_id, :metadata, :created_at, :modified_at, :state
+        attr_reader :unique_id, :metadata, :state
+
+        def raw_create_at
+          @created_at
+        end
+
+        def raw_modified_at
+          @modified_at
+        end
+
+        # for compatibility
+        def created_at
+          @created_at_object ||= Time.at(@created_at)
+        end
+
+        # for compatibility
+        def modified_at
+          @modified_at_object ||= Time.at(@modified_at)
+        end
 
         # data is array of formatted record string
         def append(data, **kwargs)