fluentd 1.11.0-x86-mingw32 → 1.11.5-x86-mingw32

data/lib/fluent/plugin/in_exec.rb

@@ -25,6 +25,8 @@ module Fluent::Plugin
 
     desc 'The command (program) to execute.'
     config_param :command, :string
+    desc 'Specify connect mode to executed process'
+    config_param :connect_mode, :enum, list: [:read, :read_with_stderr], default: :read
 
     config_section :parse do
       config_set_default :@type, 'tsv'
@@ -72,9 +74,9 @@ module Fluent::Plugin
       super
 
       if @run_interval
-        child_process_execute(:exec_input, @command, interval: @run_interval, mode: [:read], &method(:run))
+        child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
       else
-        child_process_execute(:exec_input, @command, immediate: true, mode: [:read], &method(:run))
+        child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
       end
     end
 

data/lib/fluent/plugin/in_http.rb

@@ -27,10 +27,28 @@ require 'json'
 module Fluent::Plugin
   class InHttpParser < Parser
     Fluent::Plugin.register_parser('in_http', self)
+
+    config_set_default :time_key, 'time'
+
+    def configure(conf)
+      super
+
+      # if no time parser related parameters, use in_http's time convert rule
+      @time_parser = if conf.has_key?('time_type') || conf.has_key?('time_format')
+                       time_parser_create
+                     else
+                       nil
+                     end
+    end
+
     def parse(text)
       # this plugin is dummy implementation not to raise error
       yield nil, nil
     end
+
+    def get_time_parser
+      @time_parser
+    end
   end
 
   class HttpInput < Input
@@ -54,12 +72,14 @@ module Fluent::Plugin
     desc 'Add REMOTE_ADDR header to the record.'
     config_param :add_remote_addr, :bool, default: false
     config_param :blocking_timeout, :time, default: 0.5
-    desc 'Set a white list of domains that can do CORS (Cross-Origin Resource Sharing)'
+    desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
     config_param :cors_allow_origins, :array, default: nil
     desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, default: false
     desc 'Respond status code with 204.'
     config_param :use_204_response, :bool, default: false
+    desc 'Dump error log or not'
+    config_param :dump_error_log, :bool, default: true
 
     config_section :parse do
       config_set_default :@type, 'in_http'
@@ -67,6 +87,24 @@ module Fluent::Plugin
 
     EVENT_RECORD_PARAMETER = '_event_record'
 
+    def initialize
+      super
+
+      @km = nil
+      @format_name = nil
+      @parser_time_key = nil
+
+      # default parsers
+      @parser_msgpack = nil
+      @parser_json = nil
+      @default_time_parser = nil
+      @default_keep_time_key = nil
+      @float_time_parser = nil
+
+      # <parse> configured parser
+      @custom_parser = nil
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :parser)
 
@@ -74,20 +112,22 @@ module Fluent::Plugin
 
       m = if @parser_configs.first['@type'] == 'in_http'
             @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
+            @parser_msgpack.time_key = nil
             @parser_msgpack.estimate_current_event = false
             @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
+            @parser_json.time_key = nil
             @parser_json.estimate_current_event = false
+
+            default_parser = parser_create(usage: '')
             @format_name = 'default'
-            @parser_time_key = if parser_config = conf.elements('parse').first
-                                 parser_config['time_key'] || 'time'
-                               else
-                                 'time'
-                               end
+            @parser_time_key = default_parser.time_key
+            @default_time_parser = default_parser.get_time_parser
+            @default_keep_time_key = default_parser.keep_time_key
             method(:parse_params_default)
           else
-            @parser = parser_create
+            @custom_parser = parser_create
             @format_name = @parser_configs.first['@type']
-            @parser_time_key = @parser.time_key
+            @parser_time_key = @custom_parser.time_key
             method(:parse_params_with_parser)
           end
       self.singleton_class.module_eval do
@@ -142,6 +182,13 @@ module Fluent::Plugin
       super
     end
 
+    RES_TEXT_HEADER = {'Content-Type' => 'text/plain'}.freeze
+    RESPONSE_200 = ["200 OK".freeze, RES_TEXT_HEADER, "".freeze].freeze
+    RESPONSE_204 = ["204 No Content".freeze, {}.freeze].freeze
+    RESPONSE_IMG = ["200 OK".freeze, {'Content-Type'=>'image/gif; charset=utf-8'}.freeze, EMPTY_GIF_IMAGE].freeze
+    RES_400_STATUS = "400 Bad Request".freeze
+    RES_500_STATUS = "500 Internal Server Error".freeze
+
     def on_request(path_info, params)
       begin
         path = path_info[1..-1]  # remove /
@@ -152,83 +199,77 @@ module Fluent::Plugin
         if record.nil?
           log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
           if @respond_with_empty_img
-            return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+            return RESPONSE_IMG
           else
             if @use_204_response
-              return  ["204 No Content", {}]
+              return RESPONSE_204
             else
-              return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+              return RESPONSE_200
             end
           end
         end
 
-        unless record.is_a?(Array)
-          if @add_http_headers
-            params.each_pair { |k,v|
-              if k.start_with?("HTTP_")
-                record[k] = v
-              end
-            }
-          end
-          if @add_remote_addr
-            record['REMOTE_ADDR'] = params['REMOTE_ADDR']
-          end
-        end
-        time = if param_time = params['time']
-                 param_time = param_time.to_f
-                 param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
-               else
-                 record_time.nil? ? Fluent::EventTime.now : record_time
-               end
-      rescue
-        return ["400 Bad Request", {'Content-Type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
-      end
-
-      # TODO server error
-      begin
+        mes = nil
         # Support batched requests
         if record.is_a?(Array)
           mes = Fluent::MultiEventStream.new
           record.each do |single_record|
-            if @add_http_headers
-              params.each_pair { |k,v|
-                if k.start_with?("HTTP_")
-                  single_record[k] = v
-                end
-              }
-            end
-            if @add_remote_addr
-              single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
-            end
-
-            if defined? @parser
-              single_time = @parser.parse_time(single_record)
-              single_time, single_record = @parser.convert_values(single_time, single_record)
+            add_params_to_record(single_record, params)
+
+            if param_time = params['time']
+              param_time = param_time.to_f
+              single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
+            elsif @custom_parser
+              single_time = @custom_parser.parse_time(single_record)
+              single_time, single_record = @custom_parser.convert_values(single_time, single_record)
             else
-              single_time = if t = single_record.delete(@parser_time_key)
-                              Fluent::EventTime.from_time(Time.at(t))
-                            else
-                              time
-                            end
+              single_time = convert_time_field(single_record)
             end
 
             mes.add(single_time, single_record)
           end
+        else
+          add_params_to_record(record, params)
+
+          time = if param_time = params['time']
+                   param_time = param_time.to_f
+                   param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
+                 else
+                   if record_time.nil?
+                     convert_time_field(record)
+                   else
+                     record_time
+                   end
+                 end
+        end
+      rescue => e
+        if @dump_error_log
+          log.error "failed to process request", error: e
+        end
+        return [RES_400_STATUS, RES_TEXT_HEADER, "400 Bad Request\n#{e}\n"]
+      end
+
+      # TODO server error
+      begin
+        if mes
           router.emit_stream(tag, mes)
         else
           router.emit(tag, time, record)
         end
-      rescue
-        return ["500 Internal Server Error", {'Content-Type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
+      rescue => e
+        if @dump_error_log
+          log.error "failed to emit data", error: e
+        end
+        return [RES_500_STATUS, RES_TEXT_HEADER, "500 Internal Server Error\n#{e}\n"]
       end
 
       if @respond_with_empty_img
-        return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
+        return RESPONSE_IMG
       else
         if @use_204_response
-          return  ["204 No Content", {}]
+          return RESPONSE_204
         else
-          return ["200 OK", {'Content-Type'=>'text/plain'}, ""]
+          return RESPONSE_200
         end
       end
     end
@@ -267,7 +308,7 @@ module Fluent::Plugin
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        @parser.parse(content) { |time, record|
+        @custom_parser.parse(content) { |time, record|
           raise "Received event is not #{@format_name}: #{content}" if record.nil?
           return time, record
         }
@@ -276,6 +317,32 @@ module Fluent::Plugin
       end
     end
 
+    def add_params_to_record(record, params)
+      if @add_http_headers
+        params.each_pair { |k, v|
+          if k.start_with?("HTTP_".freeze)
+            record[k] = v
+          end
+        }
+      end
+
+      if @add_remote_addr
+        record['REMOTE_ADDR'] = params['REMOTE_ADDR']
+      end
+    end
+
+    def convert_time_field(record)
+      if t = @default_keep_time_key ? record[@parser_time_key] : record.delete(@parser_time_key)
+        if @default_time_parser
+          @default_time_parser.parse(t)
+        else
+          Fluent::EventTime.from_time(Time.at(t))
+        end
+      else
+        Fluent::EventTime.now
+      end
+    end
+
     class Handler
       attr_reader :content_type
 
@@ -358,7 +425,7 @@ module Fluent::Plugin
           end
         }
         if expect
-          if expect == '100-continue'
+          if expect == '100-continue'.freeze
             if !size || size < @body_size_limit
               send_response_nobody("100 Continue", {})
             else
@@ -380,17 +447,20 @@ module Fluent::Plugin
         @body << chunk
       end
 
+      RES_200_STATUS = "200 OK".freeze
+      RES_403_STATUS = "403 Forbidden".freeze
+
       # Web browsers can send an OPTIONS request before performing POST
       # to check if cross-origin requests are supported.
       def handle_options_request
         # Is CORS enabled in the first place?
         if @cors_allow_origins.nil?
-          return send_response_and_close("403 Forbidden", {}, "")
+          return send_response_and_close(RES_403_STATUS, {}, "")
         end
 
         # in_http does not support HTTP methods except POST
         if @access_control_request_method != 'POST'
-          return send_response_and_close("403 Forbidden", {}, "")
+          return send_response_and_close(RES_403_STATUS, {}, "")
         end
 
         header = {
@@ -401,29 +471,29 @@ module Fluent::Plugin
         # Check the origin and send back a CORS response
         if @cors_allow_origins.include?('*')
           header["Access-Control-Allow-Origin"] = "*"
-          send_response_and_close("200 OK", header, "")
+          send_response_and_close(RES_200_STATUS, header, "")
         elsif include_cors_allow_origin
           header["Access-Control-Allow-Origin"] = @origin
-          send_response_and_close("200 OK", header, "")
+          send_response_and_close(RES_200_STATUS, header, "")
         else
-          send_response_and_close("403 Forbidden", {}, "")
+          send_response_and_close(RES_403_STATUS, {}, "")
         end
       end
 
       def on_message_complete
         return if closing?
 
-        if @parser.http_method == 'OPTIONS'
+        if @parser.http_method == 'OPTIONS'.freeze
           return handle_options_request()
         end
 
         # CORS check
         # ==========
         # For every incoming request, we check if we have some CORS
-        # restrictions and white listed origins through @cors_allow_origins.
+        # restrictions and allow listed origins through @cors_allow_origins.
         unless @cors_allow_origins.nil?
           unless @cors_allow_origins.include?('*') or include_cors_allow_origin
-            send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
+            send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
             return
           end
         end
@@ -433,14 +503,14 @@ module Fluent::Plugin
         # Decode payload according to the "Content-Encoding" header.
         # For now, we only support 'gzip' and 'deflate'.
         begin
-          if @content_encoding == 'gzip'
+          if @content_encoding == 'gzip'.freeze
             @body = Zlib::GzipReader.new(StringIO.new(@body)).read
-          elsif @content_encoding == 'deflate'
+          elsif @content_encoding == 'deflate'.freeze
             @body = Zlib::Inflate.inflate(@body)
           end
         rescue
           @log.warn 'fails to decode payload', error: $!.to_s
-          send_response_and_close("400 Bad Request", {}, "")
+          send_response_and_close(RES_400_STATUS, {}, "")
           return
         end
 
@@ -466,8 +536,9 @@ module Fluent::Plugin
         params.merge!(@env)
         @env.clear
 
-        code, header, body = *@callback.call(path_info, params)
+        code, header, body = @callback.call(path_info, params)
         body = body.to_s
+        header = header.dup if header.frozen?
 
         unless @cors_allow_origins.nil?
           if @cors_allow_origins.include?('*')
@@ -478,7 +549,7 @@ module Fluent::Plugin
         end
 
         if @keep_alive
-          header['Connection'] = 'Keep-Alive'
+          header['Connection'] = 'Keep-Alive'.freeze
           send_response(code, header, body)
         else
           send_response_and_close(code, header, body)
@@ -504,13 +575,13 @@ module Fluent::Plugin
 
       def send_response(code, header, body)
         header['Content-Length'] ||= body.bytesize
-        header['Content-Type'] ||= 'text/plain'
+        header['Content-Type'] ||= 'text/plain'.freeze
 
         data = %[HTTP/1.1 #{code}\r\n]
         header.each_pair {|k,v|
           data << "#{k}: #{v}\r\n"
         }
-        data << "\r\n"
+        data << "\r\n".freeze
         @io.write(data)
 
         @io.write(body)
@@ -521,7 +592,7 @@ module Fluent::Plugin
         header.each_pair {|k,v|
           data << "#{k}: #{v}\r\n"
         }
-        data << "\r\n"
+        data << "\r\n".freeze
         @io.write(data)
       end
 

data/lib/fluent/plugin/in_sample.rb

@@ -0,0 +1,141 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'json'
+
+require 'fluent/plugin/input'
+require 'fluent/config/error'
+
+module Fluent::Plugin
+  class SampleInput < Input
+    Fluent::Plugin.register_input('sample', self)
+    Fluent::Plugin.register_input('dummy', self)
+
+    helpers :thread, :storage
+
+    BIN_NUM = 10
+    DEFAULT_STORAGE_TYPE = 'local'
+
+    desc "The value is the tag assigned to the generated events."
+    config_param :tag, :string
+    desc "The number of events in event stream of each emits."
+    config_param :size, :integer, default: 1
+    desc "It configures how many events to generate per second."
+    config_param :rate, :integer, default: 1
+    desc "If specified, each generated event has an auto-incremented key field."
+    config_param :auto_increment_key, :string, default: nil
+    desc "The boolean to suspend-and-resume incremental value after restart"
+    config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
+    desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
+    config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
+      begin
+        parsed = JSON.parse(val)
+      rescue JSON::ParserError => ex
+        # Fluent::ConfigParseError, "got incomplete JSON" will be raised
+        # at literal_parser.rb with --use-v1-config, but I had to
+        # take care at here for the case of --use-v0-config.
+        raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
+      end
+      sample = parsed.is_a?(Array) ? parsed : [parsed]
+      sample.each_with_index do |e, i|
+        raise Fluent::ConfigError, "#{i}th element of sample, #{e}, is not a hash" unless e.is_a?(Hash)
+      end
+      sample
+    end
+
+    def initialize
+      super
+      @storage = nil
+    end
+
+    def configure(conf)
+      super
+      @sample_index = 0
+      config = conf.elements.select{|e| e.name == 'storage' }.first
+      @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      @storage.put(:increment_value, 0) unless @storage.get(:increment_value)
+      # keep 'dummy' to avoid breaking changes for existing environment. Change it in fluentd v2
+      @storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
+
+      if @auto_increment_key && !@storage.get(:auto_increment_value)
+        @storage.put(:auto_increment_value, -1)
+      end
+
+      thread_create(:sample_input, &method(:run))
+    end
+
+    def run
+      batch_num    = (@rate / BIN_NUM).to_i
+      residual_num = (@rate % BIN_NUM)
+      while thread_current_running?
+        current_time = Time.now.to_i
+        BIN_NUM.times do
+          break unless (thread_current_running? && Time.now.to_i <= current_time)
+          wait(0.1) { emit(batch_num) }
+        end
+        emit(residual_num) if thread_current_running?
+        # wait for next second
+        while thread_current_running? && Time.now.to_i <= current_time
+          sleep 0.01
+        end
+      end
+    end
+
+    def emit(num)
+      begin
+        if @size > 1
+          num.times do
+            router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
+          end
+        else
+          num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
+        end
+      rescue => _
+        # ignore all errors not to stop emits by emit errors
+      end
+    end
+
+    def generate
+      d = @sample[@sample_index]
+      unless d
+        @sample_index = 0
+        d = @sample[@sample_index]
+      end
+      @sample_index += 1
+      if @auto_increment_key
+        d = d.dup
+        d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
+      end
+      d
+    end
+
+    def wait(time)
+      start_time = Time.now
+      yield
+      sleep_time = time - (Time.now - start_time)
+      sleep sleep_time if sleep_time > 0
+    end
+  end
+end