fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/lib/fluent/plugin/filter_grep.rb

@@ -22,54 +22,168 @@ module Fluent::Plugin
   class GrepFilter < Filter
     Fluent::Plugin.register_filter('grep', self)
 
-    REGEXP_MAX_NUM = 20
+    def initialize
+      super
 
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil }
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil }
+      @_regexp_and_conditions = nil
+      @_exclude_and_conditions = nil
+      @_regexp_or_conditions = nil
+      @_exclude_or_conditions = nil
+    end
 
     # for test
-    attr_reader :regexps
-    attr_reader :excludes
+    attr_reader :_regexp_and_conditions, :_exclude_and_conditions, :_regexp_or_conditions, :_exclude_or_conditions
+
+    helpers :record_accessor
+
+    REGEXP_MAX_NUM = 20
+
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil, deprecated: "Use <regexp> section" }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil, deprecated: "Use <exclude> section" }
+
+    config_section :regexp, param_name: :regexps, multi: true do
+      desc "The field name to which the regular expression is applied."
+      config_param :key, :string
+      desc "The regular expression."
+      config_param :pattern, :regexp
+    end
+
+    config_section :exclude, param_name: :excludes, multi: true do
+      desc "The field name to which the regular expression is applied."
+      config_param :key, :string
+      desc "The regular expression."
+      config_param :pattern, :regexp
+    end
+
+    config_section :and, param_name: :and_conditions, multi: true do
+      config_section :regexp, param_name: :regexps, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+      config_section :exclude, param_name: :excludes, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+    end
+
+    config_section :or, param_name: :or_conditions, multi: true do
+      config_section :regexp, param_name: :regexps, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+      config_section :exclude, param_name: :excludes, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+    end
 
     def configure(conf)
       super
 
-      @regexps = {}
+      regexp_and_conditions = {}
+      regexp_or_conditions = {}
+      exclude_and_conditions = {}
+      exclude_or_conditions = {}
+
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["regexp#{i}"]
         key, regexp = conf["regexp#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
-        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
-        @regexps[key] = Regexp.compile(regexp)
+        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if regexp_and_conditions[key]
+        regexp_and_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(regexp))
       end
 
-      @excludes = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["exclude#{i}"]
         key, exclude = conf["exclude#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
-        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
-        @excludes[key] = Regexp.compile(exclude)
+        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if exclude_or_conditions[key]
+        exclude_or_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(exclude))
+      end
+
+      if @regexps.size > 1
+        log.info "Top level multiple <regexp> is intepreted as 'and' condition"
       end
+      @regexps.each do |e|
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if regexp_and_conditions.key?(e.key)
+        regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+      end
+
+      if @excludes.size > 1
+        log.info "Top level multiple <exclude> is intepreted as 'or' condition"
+      end
+      @excludes.each do |e|
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if exclude_or_conditions.key?(e.key)
+        exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+      end
+
+      @and_conditions.each do |and_condition|
+        if !and_condition.regexps.empty? && !and_condition.excludes.empty?
+          raise Fluent::ConfigError, "Do not specify both <regexp> and <exclude> in <and>"
+        end
+        and_condition.regexps.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <and>: #{e.key}" if regexp_and_conditions.key?(e.key)
+          regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+        and_condition.excludes.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <and>: #{e.key}" if exclude_and_conditions.key?(e.key)
+          exclude_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+      end
+
+      @or_conditions.each do |or_condition|
+        if !or_condition.regexps.empty? && !or_condition.excludes.empty?
+          raise Fluent::ConfigError, "Do not specify both <regexp> and <exclude> in <or>"
+        end
+        or_condition.regexps.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <or>: #{e.key}" if regexp_or_conditions.key?(e.key)
+          regexp_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+        or_condition.excludes.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <or>: #{e.key}" if exclude_or_conditions.key?(e.key)
+          exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+      end
+
+      @_regexp_and_conditions = regexp_and_conditions.values unless regexp_and_conditions.empty?
+      @_exclude_and_conditions = exclude_and_conditions.values unless exclude_and_conditions.empty?
+      @_regexp_or_conditions = regexp_or_conditions.values unless regexp_or_conditions.empty?
+      @_exclude_or_conditions = exclude_or_conditions.values unless exclude_or_conditions.empty?
     end
 
     def filter(tag, time, record)
-      result = nil
       begin
-        catch(:break_loop) do
-          @regexps.each do |key, regexp|
-            throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
-          end
-          @excludes.each do |key, exclude|
-            throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
-          end
-          result = record
+        if @_regexp_and_conditions && @_regexp_and_conditions.any? { |expression| !expression.match?(record) }
+          return nil
+        end
+        if @_regexp_or_conditions && @_regexp_or_conditions.none? { |expression| expression.match?(record) }
+          return nil
+        end
+        if @_exclude_and_conditions && @_exclude_and_conditions.all? { |expression| expression.match?(record) }
+          return nil
+        end
+        if @_exclude_or_conditions && @_exclude_or_conditions.any? { |expression| expression.match?(record) }
+          return nil
         end
       rescue => e
         log.warn "failed to grep events", error: e
         log.warn_backtrace
       end
-      result
+      record
+    end
+
+    Expression = Struct.new(:key, :pattern) do
+      def match?(record)
+        ::Fluent::StringUtil.match_regexp(pattern, key.call(record).to_s)
+      end
     end
   end
 end

data/lib/fluent/plugin/filter_parser.rb

@@ -24,14 +24,23 @@ module Fluent::Plugin
   class ParserFilter < Filter
     Fluent::Plugin.register_filter('parser', self)
 
-    helpers :parser, :compat_parameters
+    helpers :parser, :record_accessor, :compat_parameters
 
+    desc 'Specify field name in the record to parse.'
     config_param :key_name, :string
+    desc 'Keep original key-value pair in parsed result.'
     config_param :reserve_data, :bool, default: false
+    desc 'Keep original event time in parsed result.'
     config_param :reserve_time, :bool, default: false
+    desc 'Remove "key_name" field from the record when parsing is succeeded'
+    config_param :remove_key_name_field, :bool, default: false
+    desc 'Store parsed values with specified key name prefix.'
     config_param :inject_key_prefix, :string, default: nil
+    desc 'If true, invalid string is replaced with safe characters and re-parse it.'
     config_param :replace_invalid_sequence, :bool, default: false
+    desc 'Store parsed values as a hash value in a field.'
     config_param :hash_value_field, :string, default: nil
+    desc 'Emit invalid record to @ERROR label'
     config_param :emit_invalid_record_to_error, :bool, default: true
 
     attr_reader :parser
@@ -41,6 +50,7 @@ module Fluent::Plugin
 
       super
 
+      @accessor = record_accessor_create(@key_name)
       @parser = parser_create
     end
 
@@ -48,7 +58,7 @@ module Fluent::Plugin
     REPLACE_CHAR = '?'.freeze
 
     def filter_with_time(tag, time, record)
-      raw_value = record[@key_name]
+      raw_value = @accessor.call(record)
       if raw_value.nil?
         if @emit_invalid_record_to_error
           router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
@@ -67,6 +77,7 @@ module Fluent::Plugin
                 else
                   t.nil? ? time : t
                 end
+            @accessor.delete(record) if @remove_key_name_field
             r = handle_parsed(tag, record, t, values)
             return t, r
           else

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -27,10 +27,12 @@ module Fluent::Plugin
   class RecordTransformerFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('record_transformer', self)
 
+    helpers :record_accessor
+
     desc 'A comma-delimited list of keys to delete.'
-    config_param :remove_keys, :string, default: nil
+    config_param :remove_keys, :array, default: nil
     desc 'A comma-delimited list of keys to keep.'
-    config_param :keep_keys, :string, default: nil
+    config_param :keep_keys, :array, default: nil
     desc 'Create new Hash to transform incoming data'
     config_param :renew_record, :bool, default: false
     desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'
@@ -52,15 +54,14 @@ module Fluent::Plugin
         end
       end
 
-      if @remove_keys
-        @remove_keys = @remove_keys.split(',')
-      end
-
       if @keep_keys
         raise Fluent::ConfigError, "`renew_record` must be true to use `keep_keys`" unless @renew_record
-        @keep_keys = @keep_keys.split(',')
       end
 
+      @key_deleters = if @remove_keys
+                        @remove_keys.map { |k| record_accessor_create(k) }
+                      end
+
       placeholder_expander_params = {
         log: log,
         auto_typecast: @auto_typecast,
@@ -101,8 +102,7 @@ module Fluent::Plugin
           if @renew_time_key && new_record.has_key?(@renew_time_key)
             time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
           end
-          @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
-
+          @key_deleters.each { |deleter| deleter.delete(new_record) } if @key_deleters
           new_es.add(time, new_record)
         rescue => e
           router.emit_error_event(tag, time, record, e)
@@ -129,7 +129,9 @@ module Fluent::Plugin
       placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
 
       new_record = @renew_record ? {} : record.dup
-      @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
+      @keep_keys.each do |k|
+        new_record[k] = record[k] if record.has_key?(k)
+      end if @keep_keys && @renew_record
       new_record.merge!(expand_placeholders(@map, placeholders))
 
       new_record
@@ -218,7 +220,7 @@ module Fluent::Plugin
       # @param [String] str
       # @param [Boolean] force_stringify the value must be string, used for hash key
       def expand(str, placeholders, force_stringify = false)
-        if @auto_typecast and !force_stringify
+        if @auto_typecast && !force_stringify
           single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
           if single_placeholder_matched
             log_if_unknown_placeholder($1, placeholders)
@@ -261,9 +263,9 @@ module Fluent::Plugin
       def preprocess_map(value, force_stringify = false)
         new_value = nil
         if value.is_a?(String)
-          if @auto_typecast and !force_stringify
+          if @auto_typecast && !force_stringify
             num_placeholders = value.scan('${').size
-            if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}')
+            if num_placeholders == 1 && value.start_with?('${') && value.end_with?('}')
               new_value = value[2..-2] # ${..} => ..
             end
           end
@@ -314,7 +316,7 @@ module Fluent::Plugin
         end
 
         (Object.instance_methods).each do |m|
-          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/
         end
       end
     end

data/lib/fluent/plugin/formatter_stdout.rb

@@ -21,13 +21,14 @@ module Fluent
     class StdoutFormatter < Formatter
       Plugin.register_formatter('stdout', self)
 
-      TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
+      TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%N %z'
 
       config_param :output_type, :string, default: 'json'
 
       def configure(conf)
         super
 
+        @time_formatter = Strftime.new(TIME_FORMAT)
         @sub_formatter = Plugin.new_formatter(@output_type, parent: self.owner)
         @sub_formatter.configure(conf)
       end
@@ -38,7 +39,7 @@ module Fluent
       end
 
       def format(tag, time, record)
-        "#{Time.at(time).localtime.strftime(TIME_FORMAT)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\n"
+        "#{@time_formatter.exec(Time.at(time).localtime)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\n"
       end
 
       def stop

data/lib/fluent/plugin/formatter_tsv.rb

@@ -25,9 +25,13 @@ module Fluent
       config_param :keys, :array, value_type: :string
       desc 'The delimiter character (or string) of TSV values'
       config_param :delimiter, :string, default: "\t"
+      desc 'The parameter to enable writing to new lines'
+      config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
-        @keys.map{|k| record[k].to_s }.join(@delimiter)
+        formatted = @keys.map{|k| record[k].to_s }.join(@delimiter)
+        formatted << "\n".freeze if @add_newline
+        formatted
       end
     end
   end

data/lib/fluent/plugin/in_debug_agent.rb

@@ -34,6 +34,9 @@ module Fluent::Plugin
 
     def configure(conf)
       super
+      if system_config.workers > 1
+        @port += fluentd_worker_id
+      end
       if @unix_path
         unless ::Fluent::FileUtil.writable?(@unix_path)
           raise Fluent::ConfigError, "in_debug_agent: `#{@unix_path}` is not writable"
@@ -41,6 +44,10 @@ module Fluent::Plugin
       end
     end
 
+    def multi_workers_ready?
+      @unix_path.nil?
+    end
+
     def start
       super
 
@@ -50,7 +57,7 @@ module Fluent::Plugin
       else
         uri = "druby://#{@bind}:#{@port}"
       end
-      log.info "listening dRuby", uri: uri, object: @object
+      log.info "listening dRuby", uri: uri, object: @object, worker: fluentd_worker_id
       obj = eval(@object)
       @server = DRb::DRbServer.new(uri, obj)
     end

data/lib/fluent/plugin/in_forward.rb

@@ -199,8 +199,8 @@ module Fluent::Plugin
         when :pingpong
           success, reason_or_salt, shared_key = check_ping(msg, conn.remote_addr, user_auth_salt, nonce)
           unless success
+            conn.on(:write_complete) { |c| c.close_after_write_complete }
             send_data.call(serializer, generate_pong(false, reason_or_salt, nonce, shared_key))
-            conn.close
             next
           end
           send_data.call(serializer, generate_pong(true, reason_or_salt, nonce, shared_key))

data/lib/fluent/plugin/in_http.rb

@@ -78,10 +78,16 @@ module Fluent::Plugin
             @parser_json = parser_create(usage: 'parser_in_http_json', type: 'json')
             @parser_json.estimate_current_event = false
             @format_name = 'default'
+            @parser_time_key = if parser_config = conf.elements('parse').first
+                                 parser_config['time_key'] || 'time'
+                               else
+                                 'time'
+                               end
             method(:parse_params_default)
           else
             @parser = parser_create
             @format_name = @parser_configs.first['@type']
+            @parser_time_key = @parser.time_key
             method(:parse_params_with_parser)
           end
       self.singleton_class.module_eval do
@@ -202,7 +208,18 @@ module Fluent::Plugin
             if @add_remote_addr
               single_record['REMOTE_ADDR'] = params['REMOTE_ADDR']
             end
-            single_time = single_record.delete("time") || time
+
+            if defined? @parser
+              single_time = @parser.parse_time(single_record)
+              single_time, single_record = @parser.convert_values(single_time, single_record)
+            else
+              single_time = if t = single_record.delete(@parser_time_key)
+                              Fluent::EventTime.from_time(Time.at(t))
+                            else
+                              time
+                            end
+            end
+
             mes.add(single_time, single_record)
           end
           router.emit_stream(tag, mes)
@@ -301,6 +318,7 @@ module Fluent::Plugin
         end
         @env = {}
         @content_type = ""
+        @content_encoding = ""
         headers.each_pair {|k,v|
           @env["HTTP_#{k.gsub('-','_').upcase}"] = v
           case k
@@ -310,6 +328,8 @@ module Fluent::Plugin
             size = v.to_i
           when /Content-Type/i
             @content_type = v
+          when /Content-Encoding/i
+            @content_encoding = v
           when /Connection/i
             if v =~ /close/i
               @keep_alive = false
@@ -322,6 +342,10 @@ module Fluent::Plugin
             # For multiple X-Forwarded-For headers. Use first header value.
             v = v.first if v.is_a?(Array)
             @remote_addr = v.split(",").first
+          when /Access-Control-Request-Method/i
+            @access_control_request_method = v
+          when /Access-Control-Request-Headers/i
+            @access_control_request_headers = v
           end
         }
         if expect
@@ -347,20 +371,70 @@ module Fluent::Plugin
         @body << chunk
       end
 
+      # Web browsers can send an OPTIONS request before performing POST
+      # to check if cross-origin requests are supported.
+      def handle_options_request
+        # Is CORS enabled in the first place?
+        if @cors_allow_origins.nil?
+          return send_response_and_close("403 Forbidden", {}, "")
+        end
+
+        # in_http does not support HTTP methods except POST
+        if @access_control_request_method != 'POST'
+          return send_response_and_close("403 Forbidden", {}, "")
+        end
+
+        header = {
+          "Access-Control-Allow-Methods" => "POST",
+          "Access-Control-Allow-Headers" => @access_control_request_headers || "",
+        }
+
+        # Check the origin and send back a CORS response
+        if @cors_allow_origins.include?('*')
+          header["Access-Control-Allow-Origin"] = "*"
+          send_response_and_close("200 OK", header, "")
+        elsif @cors_allow_origins.include?(@origin)
+          header["Access-Control-Allow-Origin"] = @origin
+          send_response_and_close("200 OK", header, "")
+        else
+          send_response_and_close("403 Forbidden", {}, "")
+        end
+      end
+
       def on_message_complete
         return if closing?
 
+        if @parser.http_method == 'OPTIONS'
+          return handle_options_request()
+        end
+
         # CORS check
         # ==========
         # For every incoming request, we check if we have some CORS
         # restrictions and white listed origins through @cors_allow_origins.
         unless @cors_allow_origins.nil?
-          unless @cors_allow_origins.include?(@origin)
+          unless @cors_allow_origins.include?('*') or @cors_allow_origins.include?(@origin)
             send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
             return
           end
         end
 
+        # Content Encoding
+        # =================
+        # Decode payload according to the "Content-Encoding" header.
+        # For now, we only support 'gzip' and 'deflate'.
+        begin
+          if @content_encoding == 'gzip'
+            @body = Zlib::GzipReader.new(StringIO.new(@body)).read
+          elsif @content_encoding == 'deflate'
+            @body = Zlib::Inflate.inflate(@body)
+          end
+        rescue
+          @log.warn 'fails to decode payload', error: $!.to_s
+          send_response_and_close("400 Bad Request", {}, "")
+          return
+        end
+
         @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
 
         uri = URI.parse(@parser.request_url)
@@ -386,7 +460,14 @@ module Fluent::Plugin
         code, header, body = *@callback.call(path_info, params)
         body = body.to_s
 
-        header['Access-Control-Allow-Origin'] = @origin if !@cors_allow_origins.nil? && @cors_allow_origins.include?(@origin)
+        unless @cors_allow_origins.nil?
+          if @cors_allow_origins.include?('*')
+            header['Access-Control-Allow-Origin'] = '*'
+          elsif @cors_allow_origins.include?(@origin)
+            header['Access-Control-Allow-Origin'] = @origin
+          end
+        end
+
         if @keep_alive
           header['Connection'] = 'Keep-Alive'
           send_response(code, header, body)