fluentd 1.10.2-x86-mingw32 → 1.11.2-x86-mingw32

data/lib/fluent/plugin/buffer.rb

@@ -143,17 +143,33 @@ module Fluent
           end
         end
 
+        # timekey should be unixtime as usual.
+        # So, unixtime should be bigger than 2^30 - 1 (= 1073741823) nowadays.
+        # We should check object_id stability to use object_id as optimization for comparing operations.
+        # e.g.)
+        # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i
+        # => 1596187800
+        # irb> Time.parse("2020/07/31 18:30:00+09:00").to_i > 2**30 -1
+        # => true
+        def self.enable_optimize?
+          a1 = 2**30 - 1
+          a2 = 2**30 - 1
+          b1 = 2**62 - 1
+          b2 = 2**62 - 1
+          (a1.object_id == a2.object_id) && (b1.object_id == b2.object_id)
+        end
+
         # This is an optimization code. Current Struct's implementation is comparing all data.
         # https://github.com/ruby/ruby/blob/0623e2b7cc621b1733a760b72af246b06c30cf96/struct.c#L1200-L1203
         # Actually this overhead is very small but this class is generated *per chunk* (and used in hash object).
         # This means that this class is one of the most called object in Fluentd.
         # See https://github.com/fluent/fluentd/pull/2560
-        # But, this optimization has a side effect on Windows due to differing object_id.
+        # But, this optimization has a side effect on Windows and 32bit environment(s) due to differing object_id.
         # This difference causes flood of buffer files.
-        # So, this optimization should be enabled on non-Windows platform.
+        # So, this optimization should be enabled on `enable_optimize?` as true platforms.
         def hash
           timekey.object_id
-        end unless Fluent.windows?
+        end if enable_optimize?
       end
 
       # for tests
@@ -604,7 +620,11 @@ module Fluent
 
             if chunk_size_over?(chunk)
               if format && empty_chunk
-                log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
+                if chunk.bytesize > @chunk_limit_size
+                  log.warn "chunk bytes limit exceeds for an emitted event stream: #{adding_bytesize}bytes"
+                else
+                  log.warn "chunk size limit exceeds for an emitted event stream: #{chunk.size}records"
+                end
               end
               chunk.rollback
 

data/lib/fluent/plugin/in_dummy.rb

@@ -14,126 +14,5 @@
 #    limitations under the License.
 #
 
-require 'json'
-
-require 'fluent/plugin/input'
-require 'fluent/config/error'
-
-module Fluent::Plugin
-  class DummyInput < Input
-    Fluent::Plugin.register_input('dummy', self)
-
-    helpers :thread, :storage
-
-    BIN_NUM = 10
-    DEFAULT_STORAGE_TYPE = 'local'
-
-    desc "The value is the tag assigned to the generated events."
-    config_param :tag, :string
-    desc "The number of events in event stream of each emits."
-    config_param :size, :integer, default: 1
-    desc "It configures how many events to generate per second."
-    config_param :rate, :integer, default: 1
-    desc "If specified, each generated event has an auto-incremented key field."
-    config_param :auto_increment_key, :string, default: nil
-    desc "The boolean to suspend-and-resume incremental value after restart"
-    config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
-    desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
-    config_param :dummy, default: [{"message"=>"dummy"}] do |val|
-      begin
-        parsed = JSON.parse(val)
-      rescue JSON::ParserError => ex
-        # Fluent::ConfigParseError, "got incomplete JSON" will be raised
-        # at literal_parser.rb with --use-v1-config, but I had to
-        # take care at here for the case of --use-v0-config.
-        raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
-      end
-      dummy = parsed.is_a?(Array) ? parsed : [parsed]
-      dummy.each_with_index do |e, i|
-        raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
-      end
-      dummy
-    end
-
-    def initialize
-      super
-      @storage = nil
-    end
-
-    def configure(conf)
-      super
-      @dummy_index = 0
-      config = conf.elements.select{|e| e.name == 'storage' }.first
-      @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
-    end
-
-    def multi_workers_ready?
-      true
-    end
-
-    def start
-      super
-
-      @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
-      @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
-
-      if @auto_increment_key && !@storage.get(:auto_increment_value)
-        @storage.put(:auto_increment_value, -1)
-      end
-
-      thread_create(:dummy_input, &method(:run))
-    end
-
-    def run
-      batch_num    = (@rate / BIN_NUM).to_i
-      residual_num = (@rate % BIN_NUM)
-      while thread_current_running?
-        current_time = Time.now.to_i
-        BIN_NUM.times do
-          break unless (thread_current_running? && Time.now.to_i <= current_time)
-          wait(0.1) { emit(batch_num) }
-        end
-        emit(residual_num) if thread_current_running?
-        # wait for next second
-        while thread_current_running? && Time.now.to_i <= current_time
-          sleep 0.01
-        end
-      end
-    end
-
-    def emit(num)
-      begin
-        if @size > 1
-          num.times do
-            router.emit_array(@tag, Array.new(@size) { [Fluent::Engine.now, generate] })
-          end
-        else
-          num.times { router.emit(@tag, Fluent::Engine.now, generate) }
-        end
-      rescue => _
-        # ignore all errors not to stop emits by emit errors
-      end
-    end
-
-    def generate
-      d = @dummy[@dummy_index]
-      unless d
-        @dummy_index = 0
-        d = @dummy[@dummy_index]
-      end
-      @dummy_index += 1
-      if @auto_increment_key
-        d = d.dup
-        d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
-      end
-      d
-    end
-
-    def wait(time)
-      start_time = Time.now
-      yield
-      sleep_time = time - (Time.now - start_time)
-      sleep sleep_time if sleep_time > 0
-    end
-  end
-end
+# Remove this file in fluentd v2
+require_relative 'in_sample'

data/lib/fluent/plugin/in_forward.rb

@@ -327,7 +327,7 @@ module Fluent::Plugin
                  record = e[1]
                  next if record.nil?
                  time = e[0]
-                 time = Fluent::Engine.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
+                 time = Fluent::EventTime.now if time.nil? || time.to_i == 0 # `to_i == 0` for empty EventTime
                  es.add(time, record)
                }
                es
@@ -347,7 +347,7 @@ module Fluent::Plugin
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
-        time = Fluent::Engine.now if time.to_i == 0
+        time = Fluent::EventTime.now if time.to_i == 0
         if @enable_field_injection
           record[@source_address_key] = conn.remote_addr if @source_address_key
           record[@source_hostname_key] = conn.remote_host if @source_hostname_key

data/lib/fluent/plugin/in_gc_stat.rb

@@ -24,13 +24,22 @@ module Fluent::Plugin
 
     def initialize
       super
+      @key_map = nil
     end
 
     config_param :emit_interval, :time, default: 60
+    config_param :use_symbol_keys, :bool, default: true
     config_param :tag, :string
 
     def configure(conf)
       super
+
+      unless @use_symbol_keys
+        @key_map = {}
+        GC.stat.each_key { |key|
+          @key_map[key] = key.to_s
+        }
+      end
     end
 
     def multi_workers_ready?
@@ -50,6 +59,13 @@ module Fluent::Plugin
     def on_timer
       now = Fluent::EventTime.now
       record = GC.stat
+      unless @use_symbol_keys
+        new_record = {}
+        record.each_pair { |k, v|
+          new_record[@key_map[k]] = v
+        }
+        record = new_record
+      end
       router.emit(@tag, now, record)
     end
   end

data/lib/fluent/plugin/in_http.rb

@@ -27,10 +27,28 @@ require 'json'
 module Fluent::Plugin
   class InHttpParser < Parser
     Fluent::Plugin.register_parser('in_http', self)
+
+    config_set_default :time_key, 'time'
+
+    def configure(conf)
+      super
+
+      # if no time parser related parameters, use in_http's time convert rule
+      @time_parser = if conf.has_key?('time_type') || conf.has_key?('time_format')
+                       time_parser_create
+                     else
+                       nil
+                     end
+    end
+
     def parse(text)
       # this plugin is dummy implementation not to raise error
       yield nil, nil
     end
+
+    def get_time_parser
+      @time_parser
+    end
   end
 
   class HttpInput < Input
@@ -54,12 +72,14 @@ module Fluent::Plugin
     desc 'Add REMOTE_ADDR header to the record.'
     config_param :add_remote_addr, :bool, default: false
     config_param :blocking_timeout, :time, default: 0.5
-    desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
+    desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
     config_param :cors_allow_origins, :array, default: nil
     desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, default: false
     desc 'Respond status code with 204.'
     config_param :use_204_response, :bool, default: false
+    desc 'Dump error log or not'
+    config_param :dump_error_log, :bool, default: true
 
     config_section :parse do
       config_set_default :@type, 'in_http'
@@ -67,6 +87,24 @@ module Fluent::Plugin
 
     EVENT_RECORD_PARAMETER = '_event_record'
 
+    def initialize
+      super
+
+      @km = nil
+      @format_name = nil
+      @parser_time_key = nil
+
+      # default parsers
+      @parser_msgpack = nil
+      @parser_json = nil
+      @default_time_parser = nil
+      @default_keep_time_key = nil
+      @float_time_parser = nil
+
+      # <parse> configured parser
+      @custom_parser = nil
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :parser)
 
@@ -74,20 +112,22 @@ module Fluent::Plugin
 
       m = if @parser_configs.first['@type'] == 'in_http'
             @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
+            @parser_msgpack.time_key = nil
             @parser_msgpack.estimate_current_event = false
             @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
+            @parser_json.time_key = nil
             @parser_json.estimate_current_event = false
+
+            default_parser = parser_create(usage: '')
             @format_name = 'default'
-            @parser_time_key = if parser_config = conf.elements('parse').first
-                                 parser_config['time_key'] || 'time'
-                               else
-                                 'time'
-                               end
+            @parser_time_key = default_parser.time_key
+            @default_time_parser = default_parser.get_time_parser
+            @default_keep_time_key = default_parser.keep_time_key
             method(:parse_params_default)
           else
-            @parser = parser_create
+            @custom_parser = parser_create
             @format_name = @parser_configs.first['@type']
-            @parser_time_key = @parser.time_key
+            @parser_time_key = @custom_parser.time_key
             method(:parse_params_with_parser)
           end
       self.singleton_class.module_eval do
@@ -142,6 +182,13 @@ module Fluent::Plugin
       super
     end
 
+    RES_TEXT_HEADER = {'Content-Type' => 'text/plain'}.freeze
+    RESPONSE_200 = ["200 OK".freeze, RES_TEXT_HEADER, "".freeze].freeze
+    RESPONSE_204 = ["204 No Content".freeze, {}.freeze].freeze
+    RESPONSE_IMG = ["200 OK".freeze, {'Content-Type'=>'image/gif; charset=utf-8'}.freeze, EMPTY_GIF_IMAGE].freeze
+    RES_400_STATUS = "400 Bad Request".freeze
+    RES_500_STATUS = "500 Internal Server Error".freeze
+
     def on_request(path_info, params)
       begin
         path = path_info[1..-1]  # remove /
@@ -152,83 +199,77 @@ module Fluent::Plugin
         if record.nil?
           log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
           if @respond_with_empty_img
-            return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+            return RESPONSE_IMG
           else
             if @use_204_response
-              return  ["204 No Content", {}]
+              return RESPONSE_204
             else
-              return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+              return RESPONSE_200
             end
           end
         end
 
-        unless record.is_a?(Array)
-          if @add_http_headers
-            params.each_pair { |k,v|
-              if k.start_with?("HTTP_")
-                record[k] = v
-              end
-            }
-          end
-          if @add_remote_addr
-            record['REMOTE_ADDR'] = params['REMOTE_ADDR']
-          end
-        end
-        time = if param_time = params['time']
-                 param_time = param_time.to_f
-                 param_time.zero? ? Fluent::Engine.now : @float_time_parser.parse(param_time)
-               else
-                 record_time.nil? ? Fluent::Engine.now : record_time
-               end
-      rescue
-        return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
-      end
-
-      # TODO server error
-      begin
+        mes = nil
         # Support batched requests
         if record.is_a?(Array)
           mes = Fluent::MultiEventStream.new
           record.each do |single_record|
-            if @add_http_headers
-              params.each_pair { |k,v|
-                if k.start_with?("HTTP_")
-                  single_record[k] = v
-                end
-              }
-            end
-            if @add_remote_addr
-              single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
-            end
-
-            if defined? @parser
-              single_time = @parser.parse_time(single_record)
-              single_time, single_record = @parser.convert_values(single_time, single_record)
+            add_params_to_record(single_record, params)
+
+            if param_time = params['time']
+              param_time = param_time.to_f
+              single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
+            elsif @custom_parser
+              single_time = @custom_parser.parse_time(single_record)
+              single_time, single_record = @custom_parser.convert_values(single_time, single_record)
             else
-              single_time = if t = single_record.delete(@parser_time_key)
-                              Fluent::EventTime.from_time(Time.at(t))
-                            else
-                              time
-                            end
+              single_time = convert_time_field(single_record)
             end
 
             mes.add(single_time, single_record)
           end
+        else
+          add_params_to_record(record, params)
+
+          time = if param_time = params['time']
+                   param_time = param_time.to_f
+                   param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
+                 else
+                   if record_time.nil?
+                     convert_time_field(record)
+                   else
+                     record_time
+                   end
+                 end
+        end
+      rescue => e
+        if @dump_error_log
+          log.error "failed to process request", error: e
+        end
+        return [RES_400_STATUS, RES_TEXT_HEADER, "400 Bad Request\n#{e}\n"]
+      end
+
+      # TODO server error
+      begin
+        if mes
           router.emit_stream(tag, mes)
         else
           router.emit(tag, time, record)
         end
-      rescue
-        return ["500 Internal Server Error", {'Content-Type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
+      rescue => e
+        if @dump_error_log
+          log.error "failed to emit data", error: e
+        end
+        return [RES_500_STATUS, RES_TEXT_HEADER, "500 Internal Server Error\n#{e}\n"]
       end
 
       if @respond_with_empty_img
-        return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+        return RESPONSE_IMG
       else
         if @use_204_response
-          return  ["204 No Content", {}]
+          return RESPONSE_204
         else
-          return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+          return RESPONSE_200
         end
       end
     end
@@ -267,7 +308,7 @@ module Fluent::Plugin
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        @parser.parse(content) { |time, record|
+        @custom_parser.parse(content) { |time, record|
           raise "Received event is not #{@format_name}: #{content}" if record.nil?
           return time, record
         }
@@ -276,6 +317,32 @@ module Fluent::Plugin
       end
     end
 
+    def add_params_to_record(record, params)
+      if @add_http_headers
+        params.each_pair { |k, v|
+          if k.start_with?("HTTP_".freeze)
+            record[k] = v
+          end
+        }
+      end
+
+      if @add_remote_addr
+        record['REMOTE_ADDR'] = params['REMOTE_ADDR']
+      end
+    end
+
+    def convert_time_field(record)
+      if t = @default_keep_time_key ? record[@parser_time_key] : record.delete(@parser_time_key)
+        if @default_time_parser
+          @default_time_parser.parse(t)
+        else
+          Fluent::EventTime.from_time(Time.at(t))
+        end
+      else
+        Fluent::EventTime.now
+      end
+    end
+
     class Handler
       attr_reader :content_type
 
@@ -358,7 +425,7 @@ module Fluent::Plugin
           end
         }
         if expect
-          if expect == '100-continue'
+          if expect == '100-continue'.freeze
             if !size || size < @body_size_limit
               send_response_nobody("100 Continue", {})
             else
@@ -380,17 +447,20 @@ module Fluent::Plugin
         @body << chunk
       end
 
+      RES_200_STATUS = "200 OK".freeze
+      RES_403_STATUS = "403 Forbidden".freeze
+
       # Web browsers can send an OPTIONS request before performing POST
       # to check if cross-origin requests are supported.
       def handle_options_request
         # Is CORS enabled in the first place?
         if @cors_allow_origins.nil?
-          return send_response_and_close("403 Forbidden", {}, "")
+          return send_response_and_close(RES_403_STATUS, {}, "")
         end
 
         # in_http does not support HTTP methods except POST
         if @access_control_request_method != 'POST'
-          return send_response_and_close("403 Forbidden", {}, "")
+          return send_response_and_close(RES_403_STATUS, {}, "")
         end
 
         header = {
@@ -401,29 +471,29 @@ module Fluent::Plugin
         # Check the origin and send back a CORS response
         if @cors_allow_origins.include?('*')
           header["Access-Control-Allow-Origin"] = "*"
-          send_response_and_close("200 OK", header, "")
+          send_response_and_close(RES_200_STATUS, header, "")
         elsif include_cors_allow_origin
           header["Access-Control-Allow-Origin"] = @origin
-          send_response_and_close("200 OK", header, "")
+          send_response_and_close(RES_200_STATUS, header, "")
         else
-          send_response_and_close("403 Forbidden", {}, "")
+          send_response_and_close(RES_403_STATUS, {}, "")
         end
       end
 
       def on_message_complete
         return if closing?
 
-        if @parser.http_method == 'OPTIONS'
+        if @parser.http_method == 'OPTIONS'.freeze
           return handle_options_request()
         end
 
         # CORS check
         # ==========
         # For every incoming request, we check if we have some CORS
-        # restrictions and white listed origins through @cors_allow_origins.
+        # restrictions and allow listed origins through @cors_allow_origins.
         unless @cors_allow_origins.nil?
           unless @cors_allow_origins.include?('*') or include_cors_allow_origin
-            send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
+            send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
             return
           end
         end
@@ -433,14 +503,14 @@ module Fluent::Plugin
         # Decode payload according to the "Content-Encoding" header.
         # For now, we only support 'gzip' and 'deflate'.
         begin
-          if @content_encoding == 'gzip'
+          if @content_encoding == 'gzip'.freeze
             @body = Zlib::GzipReader.new(StringIO.new(@body)).read
-          elsif @content_encoding == 'deflate'
+          elsif @content_encoding == 'deflate'.freeze
             @body = Zlib::Inflate.inflate(@body)
           end
         rescue
           @log.warn 'fails to decode payload', error: $!.to_s
-          send_response_and_close("400 Bad Request", {}, "")
+          send_response_and_close(RES_400_STATUS, {}, "")
           return
         end
 
@@ -466,8 +536,9 @@ module Fluent::Plugin
         params.merge!(@env)
         @env.clear
 
-        code, header, body = *@callback.call(path_info, params)
+        code, header, body = @callback.call(path_info, params)
         body = body.to_s
+        header = header.dup if header.frozen?
 
         unless @cors_allow_origins.nil?
           if @cors_allow_origins.include?('*')
@@ -478,7 +549,7 @@ module Fluent::Plugin
         end
 
         if @keep_alive
-          header['Connection'] = 'Keep-Alive'
+          header['Connection'] = 'Keep-Alive'.freeze
           send_response(code, header, body)
         else
           send_response_and_close(code, header, body)
@@ -504,13 +575,13 @@ module Fluent::Plugin
 
       def send_response(code, header, body)
         header['Content-Length'] ||= body.bytesize
-        header['Content-Type'] ||= 'text/plain'
+        header['Content-Type'] ||= 'text/plain'.freeze
 
         data = %[HTTP/1.1 #{code}\r\n]
         header.each_pair {|k,v|
           data << "#{k}: #{v}\r\n"
         }
-        data << "\r\n"
+        data << "\r\n".freeze
         @io.write(data)
 
         @io.write(body)
@@ -521,7 +592,7 @@ module Fluent::Plugin
         header.each_pair {|k,v|
           data << "#{k}: #{v}\r\n"
         }
-        data << "\r\n"
+        data << "\r\n".freeze
         @io.write(data)
       end