fluentd 0.14.11 → 0.14.12

data/lib/fluent/match.rb

@@ -138,4 +138,41 @@ module Fluent
       @patterns.any? {|pattern| pattern.match(str) }
     end
   end
+
+  class NoMatchMatch
+    def initialize(log)
+      @log = log
+      @count = 0
+      @warn_not_matched = true
+    end
+
+    def suppress_missing_match!
+      # for <label @FLUENT_LOG>
+      @warn_not_matched = false
+    end
+
+    def emit_events(tag, es)
+      return unless @warn_not_matched
+      # TODO use time instead of num of records
+      c = (@count += 1)
+      if c < 512
+        if Math.log(c) / Math.log(2) % 1.0 == 0
+          @log.warn "no patterns matched", tag: tag
+          return
+        end
+      else
+        if c % 512 == 0
+          @log.warn "no patterns matched", tag: tag
+          return
+        end
+      end
+      @log.on_trace { @log.trace "no patterns matched", tag: tag }
+    end
+
+    def start
+    end
+
+    def shutdown
+    end
+  end
 end

data/lib/fluent/plugin.rb

@@ -113,12 +113,12 @@ module Fluent
     end
 
     def self.new_parser(type, parent: nil)
-      require 'fluent/parser'
-
       if type[0] == '/' && type[-1] == '/'
         # This usage is not recommended for new API... create RegexpParser directly
         require 'fluent/parser'
-        Fluent::TextParser.lookup(type)
+        impl = Fluent::TextParser.lookup(type)
+        impl.extend FeatureAvailabilityChecker
+        impl
       else
         new_impl('parser', PARSER_REGISTRY, type, parent)
       end
@@ -155,7 +155,29 @@ module Fluent
       if parent && impl.respond_to?("owner=")
         impl.owner = parent
       end
+      impl.extend FeatureAvailabilityChecker
       impl
     end
+
+    module FeatureAvailabilityChecker
+      def configure(conf)
+        super
+
+        # extend plugin instance by this module
+        # to run this check after all #configure methods of plugins and plugin helpers
+        sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
+                    owner.system_config
+                  elsif self.respond_to?(:system_config)
+                    self.system_config
+                  else
+                    nil
+                  end
+
+        if sysconf && sysconf.workers > 1 && !self.multi_workers_ready?
+          type = Fluent::Plugin.lookup_type_from_class(self.class)
+          raise Fluent::ConfigError, "Plugin '#{type}' does not support multi workers configuration (#{self.class})"
+        end
+      end
+    end
   end
 end

data/lib/fluent/plugin/base.rb

@@ -57,6 +57,10 @@ module Fluent
         self
       end
 
+      def multi_workers_ready?
+        true
+      end
+
       def string_safe_encoding(str)
         unless str.valid_encoding?
           log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)

data/lib/fluent/plugin/buf_file.rb

@@ -46,14 +46,20 @@ module Fluent
       def initialize
         super
         @symlink_path = nil
+        @multi_workers_available = false
+        @additional_resume_path = nil
       end
 
       def configure(conf)
         super
 
+        multi_workers_configured = owner.system_config.workers > 1 ? true : false
+
+        using_plugin_root_dir = false
         unless @path
           if root_dir = owner.plugin_root_dir
             @path = File.join(root_dir, 'buffer')
+            using_plugin_root_dir = true # plugin_root_dir path contains worker id
           else
             raise Fluent::ConfigError, "buffer path is not configured. specify 'path' in <buffer>"
           end
@@ -67,33 +73,47 @@ module Fluent
 
         @@buffer_paths[@path] = type_of_owner
 
-        if File.exist?(@path)
-          if File.directory?(@path)
+        specified_directory_exists = File.exist?(@path) && File.directory?(@path)
+        unexisting_path_for_directory = !File.exist?(@path) && !@path.include?('.*')
+
+        if specified_directory_exists || unexisting_path_for_directory # directory
+          if using_plugin_root_dir || !multi_workers_configured
             @path = File.join(@path, 'buffer.*.log')
-          elsif File.basename(@path).include?('.*.')
-            # valid path (buffer.*.log will be ignored)
-          elsif File.basename(@path).end_with?('.*')
-            @path = @path + '.log'
           else
-            # existing file will be ignored
-            @path = @path + '.*.log'
+            @path = File.join(@path, "worker#{fluentd_worker_id}", 'buffer.*.log')
+            if fluentd_worker_id == 0
+              # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
+              @additional_resume_path = File.join(File.expand_path("../../", @path), 'buffer.*.log')
+            end
           end
-        else # path doesn't exist
+          @multi_workers_available = true
+        else # specified path is file path
           if File.basename(@path).include?('.*.')
-            # valid path
+            # valid file path
           elsif File.basename(@path).end_with?('.*')
             @path = @path + '.log'
           else
-            # path is handled as directory, and it will be created at #start
-            @path = File.join(@path, 'buffer.*.log')
+            # existing file will be ignored
+            @path = @path + '.*.log'
           end
+          @multi_workers_available = false
         end
 
-        unless @dir_permission
+        if @dir_permission
+          @dir_permission = @dir_permission.to_i(8) if @dir_permission.is_a?(String)
+        else
           @dir_permission = system_config.dir_permission || DIR_PERMISSION
         end
       end
 
+      # This method is called only when multi worker is configured
+      def multi_workers_ready?
+        unless @multi_workers_available
+          log.error "file buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id"
+        end
+        @multi_workers_available
+      end
+
       def buffer_path_for_test?
         caller_locations.each do |location|
           # Thread::Backtrace::Location#path returns base filename or absolute path.
@@ -120,7 +140,11 @@ module Fluent
         stage = {}
         queue = []
 
-        Dir.glob(@path) do |path|
+        patterns = [@path]
+        patterns.unshift @additional_resume_path if @additional_resume_path
+        Dir.glob(patterns) do |path|
+          next unless File.file?(path)
+
           m = new_metadata() # this metadata will be overwritten by resuming .meta file content
                              # so it should not added into @metadata_list for now
           mode = Fluent::Plugin::Buffer::FileChunk.assume_chunk_state(path)

data/lib/fluent/plugin/buffer.rb

@@ -47,10 +47,10 @@ module Fluent
 
       # If user specify this value and (chunk_size * queue_length) is smaller than total_size,
       # then total_size is automatically configured to that value
-      config_param :queue_length_limit, :integer, default: nil
+      config_param :queue_limit_length, :integer, default: nil
 
       # optional new limitations
-      config_param :chunk_records_limit, :integer, default: nil
+      config_param :chunk_limit_records, :integer, default: nil
 
       # if chunk size (or records) is 95% or more after #write, then that chunk will be enqueued
       config_param :chunk_full_threshold, :float, default: DEFAULT_CHUNK_FULL_THRESHOLD
@@ -73,8 +73,8 @@ module Fluent
 
         @chunk_limit_size = nil
         @total_limit_size = nil
-        @queue_length_limit = nil
-        @chunk_records_limit = nil
+        @queue_limit_length = nil
+        @chunk_limit_records = nil
 
         @stage = {}    #=> Hash (metadata -> chunk) : not flushed yet
         @queue = []    #=> Array (chunks)           : already flushed (not written)
@@ -92,8 +92,8 @@ module Fluent
       def configure(conf)
         super
 
-        unless @queue_length_limit.nil?
-          @total_limit_size = @chunk_limit_size * @queue_length_limit
+        unless @queue_limit_length.nil?
+          @total_limit_size = @chunk_limit_size * @queue_limit_length
         end
       end
 
@@ -142,7 +142,7 @@ module Fluent
 
       ## TODO: for back pressure feature
       # def used?(ratio)
-      #   @total_size_limit * ratio > @stage_size + @queue_size
+      #   @total_limit_size * ratio > @stage_size + @queue_size
       # end
 
       def resume
@@ -312,7 +312,7 @@ module Fluent
       end
 
       def enqueue_chunk(metadata)
-        log.debug "enqueueing chunk", instance: self.object_id, metadata: metadata
+        log.trace "enqueueing chunk", instance: self.object_id, metadata: metadata
         synchronize do
           chunk = @stage.delete(metadata)
           return nil unless chunk
@@ -334,7 +334,7 @@ module Fluent
       end
 
       def enqueue_unstaged_chunk(chunk)
-        log.debug "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata
+        log.trace "enqueueing unstaged chunk", instance: self.object_id, metadata: chunk.metadata
         synchronize do
           chunk.synchronize do
             metadata = chunk.metadata
@@ -347,7 +347,7 @@ module Fluent
       end
 
       def enqueue_all
-        log.debug "enqueueing all chunks in buffer", instance: self.object_id
+        log.trace "enqueueing all chunks in buffer", instance: self.object_id
         synchronize do
           if block_given?
             @stage.keys.each do |metadata|
@@ -365,7 +365,7 @@ module Fluent
 
       def dequeue_chunk
         return nil if @queue.empty?
-        log.debug "dequeueing a chunk", instance: self.object_id
+        log.trace "dequeueing a chunk", instance: self.object_id
         synchronize do
           chunk = @queue.shift
 
@@ -374,18 +374,18 @@ module Fluent
 
           @dequeued[chunk.unique_id] = chunk
           @queued_num[chunk.metadata] -= 1 # BUG if nil, 0 or subzero
-          log.debug "chunk dequeued", instance: self.object_id, metadata: chunk.metadata
+          log.trace "chunk dequeued", instance: self.object_id, metadata: chunk.metadata
           chunk
         end
       end
 
       def takeback_chunk(chunk_id)
-        log.debug "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id)
+        log.trace "taking back a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id)
         synchronize do
           chunk = @dequeued.delete(chunk_id)
           return false unless chunk # already purged by other thread
           @queue.unshift(chunk)
-          log.debug "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
+          log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
           @queued_num[chunk.metadata] += 1 # BUG if nil
         end
         true
@@ -397,7 +397,7 @@ module Fluent
           return nil unless chunk # purged by other threads
 
           metadata = chunk.metadata
-          log.debug "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
+          log.trace "purging a chunk", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
           begin
             bytesize = chunk.bytesize
             chunk.purge
@@ -410,18 +410,18 @@ module Fluent
           if metadata && !@stage[metadata] && (!@queued_num[metadata] || @queued_num[metadata] < 1)
             @metadata_list.delete(metadata)
           end
-          log.debug "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
+          log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end
         nil
       end
 
       def clear_queue!
-        log.debug "clearing queue", instance: self.object_id
+        log.trace "clearing queue", instance: self.object_id
         synchronize do
           until @queue.empty?
             begin
               q = @queue.shift
-              log.debug("purging a chunk in queue"){ {id: dump_unique_id_hex(chunk.unique_id), bytesize: chunk.bytesize, size: chunk.size} }
+              log.trace("purging a chunk in queue"){ {id: dump_unique_id_hex(chunk.unique_id), bytesize: chunk.bytesize, size: chunk.size} }
               q.purge
             rescue => e
               log.error "unexpected error while clearing buffer queue", error_class: e.class, error: e
@@ -433,11 +433,11 @@ module Fluent
       end
 
       def chunk_size_over?(chunk)
-        chunk.bytesize > @chunk_limit_size || (@chunk_records_limit && chunk.size > @chunk_records_limit)
+        chunk.bytesize > @chunk_limit_size || (@chunk_limit_records && chunk.size > @chunk_limit_records)
       end
 
       def chunk_size_full?(chunk)
-        chunk.bytesize >= @chunk_limit_size * @chunk_full_threshold || (@chunk_records_limit && chunk.size >= @chunk_records_limit * @chunk_full_threshold)
+        chunk.bytesize >= @chunk_limit_size * @chunk_full_threshold || (@chunk_limit_records && chunk.size >= @chunk_limit_records * @chunk_full_threshold)
       end
 
       class ShouldRetry < StandardError; end

data/lib/fluent/plugin/buffer/file_chunk.rb

@@ -42,12 +42,12 @@ module Fluent
 
         def initialize(metadata, path, mode, perm: system_config.file_permission || FILE_PERMISSION, compress: :text)
           super(metadata, compress: compress)
-          @permission = perm
+          @permission = perm.is_a?(String) ? perm.to_i(8) : perm
           @bytesize = @size = @adding_bytes = @adding_size = 0
           @meta = nil
 
           case mode
-          when :create then create_new_chunk(path, perm)
+          when :create then create_new_chunk(path, @permission)
           when :staged then load_existing_staged_chunk(path)
           when :queued then load_existing_enqueued_chunk(path)
           else

data/lib/fluent/plugin/compressable.rb

@@ -14,6 +14,7 @@
 #    limitations under the License.
 #
 
+require 'stringio'
 require 'zlib'
 
 module Fluent

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -101,6 +101,8 @@ module Fluent::Plugin
           if @renew_time_key && new_record.has_key?(@renew_time_key)
             time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
           end
+          @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
+
           new_es.add(time, new_record)
         rescue => e
           router.emit_error_event(tag, time, record, e)
@@ -129,7 +131,6 @@ module Fluent::Plugin
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
       new_record.merge!(expand_placeholders(@map, placeholders))
-      @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       new_record
     end
@@ -307,14 +308,10 @@ module Fluent::Plugin
         raise "failed to expand `#{str}` : error = #{e}"
       end
 
-      class CleanroomExpander
+      class CleanroomExpander < BasicObject
         def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
           instance_eval(__str_to_eval__)
         end
-
-        (Object.instance_methods).each do |m|
-          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
-        end
       end
     end
   end

data/lib/fluent/plugin/formatter_csv.rb

@@ -29,6 +29,7 @@ module Fluent
       # "array" looks good for type of :fields, but this implementation removes tailing comma
       # TODO: Is it needed to support tailing comma?
       config_param :fields, :array, value_type: :string
+      config_param :add_newline, :bool, default: true
 
       def configure(conf)
         super
@@ -42,7 +43,9 @@ module Fluent
         row = @fields.map do |key|
           record[key]
         end
-        CSV.generate_line(row, @generate_opts)
+        line = CSV.generate_line(row, @generate_opts)
+        line.chomp! unless @add_newline
+        line
       end
     end
   end

data/lib/fluent/plugin/formatter_hash.rb

@@ -21,8 +21,12 @@ module Fluent
     class HashFormatter < Formatter
       Plugin.register_formatter('hash', self)
 
+      config_param :add_newline, :bool, default: true
+
       def format(tag, time, record)
-        "#{record.to_s}\n"
+        line = record.to_s
+        line << "\n".freeze if @add_newline
+        line
       end
     end
   end

data/lib/fluent/plugin/formatter_json.rb

@@ -23,6 +23,7 @@ module Fluent
       Plugin.register_formatter('json', self)
 
       config_param :json_parser, :string, default: 'oj'
+      config_param :add_newline, :bool, default: true
 
       def configure(conf)
         super
@@ -35,11 +36,20 @@ module Fluent
         rescue LoadError
           @dump_proc = Yajl.method(:dump)
         end
+
+        # format json is used on various highload environment, so re-define method to skip if check
+        unless @add_newline
+          define_singleton_method(:format, method(:format_without_nl))
+        end
       end
 
       def format(tag, time, record)
         "#{@dump_proc.call(record)}\n"
       end
+
+      def format_without_nl(tag, time, record)
+        @dump_proc.call(record)
+      end
     end
   end
 end