fluentd 1.11.0-x64-mingw32 → 1.11.5-x64-mingw32

data/lib/fluent/match.rb

@@ -33,6 +33,15 @@ module Fluent
 
   class GlobMatchPattern < MatchPattern
     def initialize(pat)
+      if pat.start_with?('/')
+        if pat.end_with?('/')
+          @regex = Regexp.new("\\A"+pat[1..-2]+"\\Z")
+          return
+        else
+          raise Fluent::ConfigError,  "invalid match - regex"
+        end
+      end
+
       stack = []
       regex = ['']
       escape = false

data/lib/fluent/plugin/buffer.rb

@@ -143,17 +143,33 @@ module Fluent
           end
         end
 
+        # timekey should be unixtime as usual.
+        # So, unixtime should be bigger than 2^30 - 1 (= 1073741823) nowadays.
+        # We should check object_id stability to use object_id as optimization for comparing operations.
+        # e.g.)
+        # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i
+        # => 1596187800
+        # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i > 2**30 -1
+        # => true
+        def self.enable_optimize?
+          a1 = 2**30 - 1
+          a2 = 2**30 - 1
+          b1 = 2**62 - 1
+          b2 = 2**62 - 1
+          (a1.object_id == a2.object_id) && (b1.object_id == b2.object_id)
+        end
+
         # This is an optimization code. Current Struct's implementation is comparing all data.
         # https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203
         # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
         # This means that this class is one of the most called object in Fluentd.
         # See https://github.com/fluent/fluentd/pull/2560
-        # But, this optimization has a side effect on Windows due to differing object_id.
+        # But, this optimization has a side effect on Windows and 32bit environment(s) due to differing object_id.
         # This difference causes flood of buffer files.
-        # So, this optimization should be enabled on non-Windows platform.
+        # So, this optimization should be enabled on `enable_optimize?` as true platforms.
         def hash
           timekey.object_id
-        end unless Fluent.windows?
+        end if enable_optimize?
       end
 
       # for tests
@@ -176,6 +192,7 @@ module Fluent
 
         @stage_size = @queue_size = 0
         @timekeys = Hash.new(0)
+        @enable_update_timekeys = false
         @mutex = Mutex.new
       end
 
@@ -191,24 +208,23 @@ module Fluent
         end
       end
 
+      def enable_update_timekeys
+        @enable_update_timekeys = true
+      end
+
       def start
         super
 
         @stage, @queue = resume
         @stage.each_pair do |metadata, chunk|
           @stage_size += chunk.bytesize
-          if chunk.metadata && chunk.metadata.timekey
-            add_timekey(metadata.timekey)
-          end
         end
         @queue.each do |chunk|
           @queued_num[chunk.metadata] ||= 0
           @queued_num[chunk.metadata] += 1
           @queue_size += chunk.bytesize
-          if chunk.metadata && chunk.metadata.timekey
-            add_timekey(chunk.metadata.timekey)
-          end
         end
+        update_timekeys
         log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
       end
 
@@ -257,12 +273,9 @@ module Fluent
         Metadata.new(timekey, tag, variables)
       end
 
+      # Keep this method for existing code
       def metadata(timekey: nil, tag: nil, variables: nil)
-        meta = Metadata.new(timekey, tag, variables)
-        if (t = meta.timekey)
-          add_timekey(t)
-        end
-        meta
+        Metadata.new(timekey, tag, variables)
       end
 
       def timekeys
@@ -456,9 +469,23 @@ module Fluent
         end
       end
 
+      def update_timekeys
+        synchronize do
+          chunks = @stage.values
+          chunks.concat(@queue)
+          @timekeys = chunks.each_with_object({}) do |chunk, keys|
+            if chunk.metadata && chunk.metadata.timekey
+              t = chunk.metadata.timekey
+              keys[t] = keys.fetch(t, 0) + 1
+            end
+          end
+        end
+      end
+
       # At flush_at_shutdown, all staged chunks should be enqueued for buffer flush. Set true to force_enqueue for it.
       def enqueue_all(force_enqueue = false)
         log.on_trace { log.trace "enqueueing all chunks in buffer", instance: self.object_id }
+        update_timekeys if @enable_update_timekeys
 
         if block_given?
           synchronize{ @stage.keys }.each do |metadata|
@@ -537,10 +564,6 @@ module Fluent
           log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end
 
-        if metadata && metadata.timekey
-          del_timekey(metadata.timekey)
-        end
-
         nil
       end
 
@@ -604,7 +627,11 @@ module Fluent
 
             if chunk_size_over?(chunk)
               if format && empty_chunk
-                log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
+                if chunk.bytesize > @chunk_limit_size
+                  log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
+                else
+                  log.warn "chunk size limit exceeds for an emitted event stream: #{chunk.size}records"
+                end
               end
               chunk.rollback
 
@@ -768,11 +795,11 @@ module Fluent
           'total_queued_size' => stage_size + queue_size,
         }
 
-        if (m = timekeys.min)
+        tkeys = timekeys
+        if (m = tkeys.min)
           stats['oldest_timekey'] = m
         end
-
-        if (m = timekeys.max)
+        if (m = tkeys.max)
           stats['newest_timekey'] = m
         end
 
@@ -789,24 +816,6 @@ module Fluent
           !@queue.empty?
         end
       end
-
-      def add_timekey(t)
-        @mutex.synchronize do
-          @timekeys[t] += 1
-        end
-        nil
-      end
-
-      def del_timekey(t)
-        @mutex.synchronize do
-          if @timekeys[t] <= 1
-            @timekeys.delete(t)
-          else
-            @timekeys[t] -= 1
-          end
-        end
-        nil
-      end
     end
   end
 end

data/lib/fluent/plugin/buffer/chunk.rb

@@ -206,8 +206,9 @@ module Fluent
                 output_io = if chunk_io.is_a?(StringIO)
                               StringIO.new
                             else
-                              Tempfile.new('decompressed-data').binmode
+                              Tempfile.new('decompressed-data')
                             end
+                output_io.binmode if output_io.is_a?(Tempfile)
                 decompress(input_io: chunk_io, output_io: output_io)
                 output_io.seek(0, IO::SEEK_SET)
                 yield output_io

data/lib/fluent/plugin/formatter.rb

@@ -46,5 +46,29 @@ module Fluent
         @proc.call(tag, time, record)
       end
     end
+
+    module Newline
+      module Mixin
+        include Fluent::Configurable
+
+        DEFAULT_NEWLINE = if Fluent.windows?
+                            :crlf
+                          else
+                            :lf
+                          end
+
+        config_param :newline, :enum, list: [:lf, :crlf], default: DEFAULT_NEWLINE
+
+        def configure(conf)
+          super
+          @newline = case newline
+                     when :lf
+                       "\n"
+                     when :crlf
+                       "\r\n"
+                     end
+        end
+      end
+    end
   end
 end

data/lib/fluent/plugin/formatter_hash.rb

@@ -19,13 +19,15 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class HashFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('hash', self)
 
       config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         line = record.to_s
-        line << "\n".freeze if @add_newline
+        line << @newline.freeze if @add_newline
         line
       end
     end

data/lib/fluent/plugin/formatter_json.rb

@@ -20,6 +20,8 @@ require 'fluent/env'
 module Fluent
   module Plugin
     class JSONFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('json', self)
 
       config_param :json_parser, :string, default: 'oj'
@@ -44,7 +46,7 @@ module Fluent
       end
 
       def format(tag, time, record)
-        "#{@dump_proc.call(record)}\n"
+        "#{@dump_proc.call(record)}#{@newline}"
       end
 
       def format_without_nl(tag, time, record)

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -19,6 +19,8 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class LabeledTSVFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('ltsv', self)
 
       # http://ltsv.org/
@@ -34,7 +36,7 @@ module Fluent
           formatted << @delimiter if formatted.length.nonzero?
           formatted << "#{label}#{@label_delimiter}#{value}"
         end
-        formatted << "\n".freeze if @add_newline
+        formatted << @newline.freeze if @add_newline
         formatted
       end
     end

data/lib/fluent/plugin/formatter_out_file.rb

@@ -21,6 +21,8 @@ require 'yajl'
 module Fluent
   module Plugin
     class OutFileFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('out_file', self)
 
       config_param :output_time, :bool, default: true
@@ -44,7 +46,7 @@ module Fluent
         header = ''
         header << "#{@timef.format(time)}#{@delimiter}" if @output_time
         header << "#{tag}#{@delimiter}" if @output_tag
-        "#{header}#{Yajl.dump(record)}\n"
+        "#{header}#{Yajl.dump(record)}#{@newline}"
       end
     end
   end

data/lib/fluent/plugin/formatter_single_value.rb

@@ -19,6 +19,8 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class SingleValueFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('single_value', self)
 
       config_param :message_key, :string, default: 'message'
@@ -26,7 +28,7 @@ module Fluent
 
       def format(tag, time, record)
         text = record[@message_key].to_s.dup
-        text << "\n" if @add_newline
+        text << @newline.freeze if @add_newline
         text
       end
     end

data/lib/fluent/plugin/formatter_tsv.rb

@@ -19,6 +19,8 @@ require 'fluent/plugin/formatter'
 module Fluent
   module Plugin
     class TSVFormatter < Formatter
+      include Fluent::Plugin::Newline::Mixin
+
       Plugin.register_formatter('tsv', self)
 
       desc 'Field names included in each lines'
@@ -30,7 +32,7 @@ module Fluent
 
       def format(tag, time, record)
         formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)
-        formatted << "\n".freeze if @add_newline
+        formatted << @newline.freeze if @add_newline
         formatted
       end
     end

data/lib/fluent/plugin/in_dummy.rb

@@ -14,126 +14,5 @@
 #    limitations under the License.
 #
 
-require 'json'
-
-require 'fluent/plugin/input'
-require 'fluent/config/error'
-
-module Fluent::Plugin
-  class DummyInput < Input
-    Fluent::Plugin.register_input('dummy', self)
-
-    helpers :thread, :storage
-
-    BIN_NUM = 10
-    DEFAULT_STORAGE_TYPE = 'local'
-
-    desc "The value is the tag assigned to the generated events."
-    config_param :tag, :string
-    desc "The number of events in event stream of each emits."
-    config_param :size, :integer, default: 1
-    desc "It configures how many events to generate per second."
-    config_param :rate, :integer, default: 1
-    desc "If specified, each generated event has an auto-incremented key field."
-    config_param :auto_increment_key, :string, default: nil
-    desc "The boolean to suspend-and-resume incremental value after restart"
-    config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
-    desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
-    config_param :dummy, default: [{"message"=>"dummy"}] do |val|
-      begin
-        parsed = JSON.parse(val)
-      rescue JSON::ParserError => ex
-        # Fluent::ConfigParseError, "got incomplete JSON" will be raised
-        # at literal_parser.rb with --use-v1-config, but I had to
-        # take care at here for the case of --use-v0-config.
-        raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
-      end
-      dummy = parsed.is_a?(Array) ? parsed : [parsed]
-      dummy.each_with_index do |e, i|
-        raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
-      end
-      dummy
-    end
-
-    def initialize
-      super
-      @storage = nil
-    end
-
-    def configure(conf)
-      super
-      @dummy_index = 0
-      config = conf.elements.select{|e| e.name == 'storage' }.first
-      @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
-    end
-
-    def multi_workers_ready?
-      true
-    end
-
-    def start
-      super
-
-      @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
-      @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
-
-      if @auto_increment_key && !@storage.get(:auto_increment_value)
-        @storage.put(:auto_increment_value, -1)
-      end
-
-      thread_create(:dummy_input, &method(:run))
-    end
-
-    def run
-      batch_num    = (@rate / BIN_NUM).to_i
-      residual_num = (@rate % BIN_NUM)
-      while thread_current_running?
-        current_time = Time.now.to_i
-        BIN_NUM.times do
-          break unless (thread_current_running? && Time.now.to_i <= current_time)
-          wait(0.1) { emit(batch_num) }
-        end
-        emit(residual_num) if thread_current_running?
-        # wait for next second
-        while thread_current_running? && Time.now.to_i <= current_time
-          sleep 0.01
-        end
-      end
-    end
-
-    def emit(num)
-      begin
-        if @size > 1
-          num.times do
-            router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
-          end
-        else
-          num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
-        end
-      rescue => _
-        # ignore all errors not to stop emits by emit errors
-      end
-    end
-
-    def generate
-      d = @dummy[@dummy_index]
-      unless d
-        @dummy_index = 0
-        d = @dummy[@dummy_index]
-      end
-      @dummy_index += 1
-      if @auto_increment_key
-        d = d.dup
-        d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
-      end
-      d
-    end
-
-    def wait(time)
-      start_time = Time.now
-      yield
-      sleep_time = time - (Time.now - start_time)
-      sleep sleep_time if sleep_time > 0
-    end
-  end
-end
+# Remove this file in fluentd v2
+require_relative 'in_sample'