fluentd 1.11.1-x86-mingw32 → 1.12.0-x86-mingw32

data/lib/fluent/command/plugin_config_formatter.rb

@@ -29,7 +29,8 @@ class FluentPluginConfigFormatter
   AVAILABLE_FORMATS = [:markdown, :txt, :json]
   SUPPORTED_TYPES = [
     "input", "output", "filter",
-    "buffer", "parser", "formatter", "storage"
+    "buffer", "parser", "formatter", "storage",
+    "service_discovery"
   ]
 
   DOCS_BASE_URL = "https://docs.fluentd.org/v/1.0"

data/lib/fluent/env.rb

@@ -28,4 +28,8 @@ module Fluent
   def self.windows?
     ServerEngine.windows?
   end
+
+  def self.linux?
+    /linux/ === RUBY_PLATFORM
+  end
 end

data/lib/fluent/log.rb

@@ -113,6 +113,7 @@ module Fluent
 
       @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
       @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
+      @ignore_same_log_interval = opts[:ignore_same_log_interval]
 
       @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
       @process_type ||= :standalone # to keep behavior of existing code
@@ -141,7 +142,8 @@ module Fluent
       dl_opts[:log_level] = @level - 1
       logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
       clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type,
-                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval)
+                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval,
+                             ignore_same_log_interval: @ignore_same_log_interval)
       clone.format = @format
       clone.time_format = @time_format
       clone.log_event_enabled = @log_event_enabled
@@ -151,7 +153,7 @@ module Fluent
 
     attr_reader :format
     attr_reader :time_format
-    attr_accessor :log_event_enabled, :ignore_repeated_log_interval
+    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
     attr_accessor :out
     attr_accessor :level
     attr_accessor :optional_header, :optional_attrs
@@ -428,6 +430,27 @@ module Fluent
       (cached_log.msg == message) && (time - cached_log.time <= @ignore_repeated_log_interval)
     end
 
+    def ignore_same_log?(time, message)
+      cached_log = Thread.current[:last_same_log]
+      if cached_log.nil?
+        Thread.current[:last_same_log] = {message => time}
+        return false
+      end
+
+      prev_time = cached_log[message]
+      if prev_time
+        if (time - prev_time) <= @ignore_same_log_interval
+          true
+        else
+          cached_log[message] = time
+          false
+        end
+      else
+        cached_log[message] = time
+        false
+      end
+    end
+
     def suppress_stacktrace?(backtrace)
       cached_log = Thread.current[:last_repeated_stacktrace]
       return false if cached_log.nil?
@@ -507,7 +530,11 @@ module Fluent
         end
       }
 
-      if @ignore_repeated_log_interval
+      if @ignore_same_log_interval
+        if ignore_same_log?(time, message)
+          return nil, nil
+        end
+      elsif @ignore_repeated_log_interval
         if ignore_repeated_log?(:last_repeated_log, time, message)
           return nil, nil
         else
@@ -569,6 +596,9 @@ module Fluent
       if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
         @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
       end
+      if logger.instance_variable_defined?(:@ignore_same_log_interval)
+        @ignore_same_log_interval = logger.instance_variable_get(:@ignore_same_log_interval)
+      end
 
       self.format = @logger.format
       self.time_format = @logger.time_format

data/lib/fluent/match.rb

@@ -33,6 +33,15 @@ module Fluent
 
   class GlobMatchPattern < MatchPattern
     def initialize(pat)
+      if pat.start_with?('/')
+        if pat.end_with?('/')
+          @regex = Regexp.new("\\A"+pat[1..-2]+"\\Z")
+          return
+        else
+          raise Fluent::ConfigError,  "invalid match - regex"
+        end
+      end
+
       stack = []
       regex = ['']
       escape = false

data/lib/fluent/plugin.rb

@@ -121,6 +121,11 @@ module Fluent
       new_impl('sd', SD_REGISTRY, type, parent)
     end
 
+    class << self
+      # This should be defined for fluent-plugin-config-formatter type arguments.
+      alias_method :new_service_discovery, :new_sd
+    end
+
     def self.new_parser(type, parent: nil)
       if type[0] == '/' && type[-1] == '/'
         # This usage is not recommended for new API... create RegexpParser directly

data/lib/fluent/plugin/buffer.rb

@@ -148,12 +148,9 @@ module Fluent
         # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
         # This means that this class is one of the most called object in Fluentd.
         # See https://github.com/fluent/fluentd/pull/2560
-        # But, this optimization has a side effect on Windows due to differing object_id.
-        # This difference causes flood of buffer files.
-        # So, this optimization should be enabled on non-Windows platform.
         def hash
-          timekey.object_id
-        end unless Fluent.windows?
+          timekey.hash
+        end
       end
 
       # for tests
@@ -176,6 +173,7 @@ module Fluent
 
         @stage_size = @queue_size = 0
         @timekeys = Hash.new(0)
+        @enable_update_timekeys = false
         @mutex = Mutex.new
       end
 
@@ -191,24 +189,23 @@ module Fluent
         end
       end
 
+      def enable_update_timekeys
+        @enable_update_timekeys = true
+      end
+
       def start
         super
 
         @stage, @queue = resume
         @stage.each_pair do |metadata, chunk|
           @stage_size += chunk.bytesize
-          if chunk.metadata && chunk.metadata.timekey
-            add_timekey(metadata.timekey)
-          end
         end
         @queue.each do |chunk|
           @queued_num[chunk.metadata] ||= 0
           @queued_num[chunk.metadata] += 1
           @queue_size += chunk.bytesize
-          if chunk.metadata && chunk.metadata.timekey
-            add_timekey(chunk.metadata.timekey)
-          end
         end
+        update_timekeys
         log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
       end
 
@@ -257,12 +254,9 @@ module Fluent
         Metadata.new(timekey, tag, variables)
       end
 
+      # Keep this method for existing code
       def metadata(timekey: nil, tag: nil, variables: nil)
-        meta = Metadata.new(timekey, tag, variables)
-        if (t = meta.timekey)
-          add_timekey(t)
-        end
-        meta
+        Metadata.new(timekey, tag, variables)
       end
 
       def timekeys
@@ -456,9 +450,23 @@ module Fluent
         end
       end
 
+      def update_timekeys
+        synchronize do
+          chunks = @stage.values
+          chunks.concat(@queue)
+          @timekeys = chunks.each_with_object({}) do |chunk, keys|
+            if chunk.metadata && chunk.metadata.timekey
+              t = chunk.metadata.timekey
+              keys[t] = keys.fetch(t, 0) + 1
+            end
+          end
+        end
+      end
+
       # At flush_at_shutdown, all staged chunks should be enqueued for buffer flush. Set true to force_enqueue for it.
       def enqueue_all(force_enqueue = false)
         log.on_trace { log.trace "enqueueing all chunks in buffer", instance: self.object_id }
+        update_timekeys if @enable_update_timekeys
 
         if block_given?
           synchronize{ @stage.keys }.each do |metadata|
@@ -537,10 +545,6 @@ module Fluent
           log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end
 
-        if metadata && metadata.timekey
-          del_timekey(metadata.timekey)
-        end
-
         nil
       end
 
@@ -604,7 +608,11 @@ module Fluent
 
             if chunk_size_over?(chunk)
               if format && empty_chunk
-                log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
+                if chunk.bytesize > @chunk_limit_size
+                  log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
+                else
+                  log.warn "chunk size limit exceeds for an emitted event stream: #{chunk.size}records"
+                end
               end
               chunk.rollback
 
@@ -768,11 +776,11 @@ module Fluent
           'total_queued_size' => stage_size + queue_size,
         }
 
-        if (m = timekeys.min)
+        tkeys = timekeys
+        if (m = tkeys.min)
           stats['oldest_timekey'] = m
         end
-
-        if (m = timekeys.max)
+        if (m = tkeys.max)
           stats['newest_timekey'] = m
         end
 
@@ -789,24 +797,6 @@ module Fluent
           !@queue.empty?
         end
       end
-
-      def add_timekey(t)
-        @mutex.synchronize do
-          @timekeys[t] += 1
-        end
-        nil
-      end
-
-      def del_timekey(t)
-        @mutex.synchronize do
-          if @timekeys[t] <= 1
-            @timekeys.delete(t)
-          else
-            @timekeys[t] -= 1
-          end
-        end
-        nil
-      end
     end
   end
 end

data/lib/fluent/plugin/buffer/chunk.rb

@@ -206,8 +206,9 @@ module Fluent
                 output_io = if chunk_io.is_a?(StringIO)
                               StringIO.new
                             else
-                              Tempfile.new('decompressed-data').binmode
+                              Tempfile.new('decompressed-data')
                             end
+                output_io.binmode if output_io.is_a?(Tempfile)
                 decompress(input_io: chunk_io, output_io: output_io)
                 output_io.seek(0, IO::SEEK_SET)
                 yield output_io

data/lib/fluent/plugin/formatter.rb

@@ -46,5 +46,29 @@ module Fluent
         @proc.call(tag, time, record)
       end
     end
+
+    module Newline
+      module Mixin
+        include Fluent::Configurable
+
+        DEFAULT_NEWLINE = if Fluent.windows?
+                            :crlf
+                          else
+                            :lf
+                          end
+
+        config_param :newline, :enum, list: [:lf, :crlf], default: DEFAULT_NEWLINE
+
+        def configure(conf)
+          super
+          @newline = case newline
+                     when :lf
+                       "\n".freeze
+                     when :crlf
+                       "\r\n".freeze
+                     end
+        end
+      end
+    end
   end
 end

data/lib/fluent/plugin/formatter_csv.rb

@@ -27,7 +27,7 @@ module Fluent
       helpers :record_accessor
 
       config_param :delimiter, default: ',' do |val|
-        ['\t', 'TAB'].include?(val) ? "\t" : val
+        ['\t', 'TAB'].include?(val) ? "\t".freeze : val.freeze
       end
       config_param :force_quotes, :bool, default: true
       # "array" looks good for type of :fields, but this implementation removes tailing comma

data/lib/fluent/plugin/formatter_hash.rb

@@ -19,13 +19,15 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class HashFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('hash', self)
 
       config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         line = record.to_s
-        line << "\n".freeze if @add_newline
+        line << @newline if @add_newline
         line
       end
     end

data/lib/fluent/plugin/formatter_json.rb

@@ -20,6 +20,8 @@ require 'fluent/env'
 module Fluent
   module Plugin
     class JSONFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('json', self)
 
       config_param :json_parser, :string, default: 'oj'
@@ -44,7 +46,7 @@ module Fluent
       end
 
       def format(tag, time, record)
-        "#{@dump_proc.call(record)}\n"
+        "#{@dump_proc.call(record)}#{@newline}"
       end
 
       def format_without_nl(tag, time, record)

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -19,12 +19,14 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class LabeledTSVFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('ltsv', self)
 
       # http://ltsv.org/
 
-      config_param :delimiter, :string, default: "\t"
-      config_param :label_delimiter, :string, default: ":"
+      config_param :delimiter, :string, default: "\t".freeze
+      config_param :label_delimiter, :string, default: ":".freeze
       config_param :add_newline, :bool, default: true
 
       # TODO: escaping for \t in values
@@ -34,7 +36,7 @@ module Fluent
           formatted << @delimiter if formatted.length.nonzero?
           formatted << "#{label}#{@label_delimiter}#{value}"
         end
-        formatted << "\n".freeze if @add_newline
+        formatted << @newline if @add_newline
         formatted
       end
     end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -21,15 +21,17 @@ require 'yajl'
 module Fluent
   module Plugin
     class OutFileFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('out_file', self)
 
       config_param :output_time, :bool, default: true
       config_param :output_tag, :bool, default: true
       config_param :delimiter, default: "\t" do |val|
         case val
-        when /SPACE/i then ' '
-        when /COMMA/i then ','
-        else "\t"
+        when /SPACE/i then ' '.freeze
+        when /COMMA/i then ','.freeze
+        else "\t".freeze
         end
       end
       config_set_default :time_type, :string
@@ -44,7 +46,7 @@ module Fluent
         header = ''
         header << "#{@timef.format(time)}#{@delimiter}" if @output_time
         header << "#{tag}#{@delimiter}" if @output_tag
-        "#{header}#{Yajl.dump(record)}\n"
+        "#{header}#{Yajl.dump(record)}#{@newline}"
       end
     end
   end

data/lib/fluent/plugin/formatter_single_value.rb

@@ -19,14 +19,16 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class SingleValueFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('single_value', self)
 
-      config_param :message_key, :string, default: 'message'
+      config_param :message_key, :string, default: 'message'.freeze
       config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         text = record[@message_key].to_s.dup
-        text << "\n" if @add_newline
+        text << @newline if @add_newline
         text
       end
     end

data/lib/fluent/plugin/formatter_tsv.rb

@@ -19,18 +19,20 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class TSVFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('tsv', self)
 
       desc 'Field names included in each lines'
       config_param :keys, :array, value_type: :string
       desc 'The delimiter character (or string) of TSV values'
-      config_param :delimiter, :string, default: "\t"
+      config_param :delimiter, :string, default: "\t".freeze
       desc 'The parameter to enable writing to new lines'
       config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)
-        formatted << "\n".freeze if @add_newline
+        formatted << @newline if @add_newline
         formatted
       end
     end