fluentd 1.1.3 → 1.2.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a26c51f262d85284f6ec0144fcab0088682efa9
-  data.tar.gz: 74f6bf4722ca406b51e678b7c0037f8f0c2e0a16
+  metadata.gz: e457fdb0637a4e2ca90d91766e53db7d1a35b44c
+  data.tar.gz: 71aad06bb2784901e8e1517b6433536321fdeeae
 SHA512:
-  metadata.gz: 5de45a297b116281323d7c1d6cba55e46300c77025e4fa5f59de6be27bb21606448c3c2c86142c69e7b5ba4da0cbadb2b3f0acc23af6d3046c55521080559767
-  data.tar.gz: e9b1ddc803810bfd8b98fd2b3d5378bc1f50802b72e7343772b5bef229d8982d6d5e5a2aabf075c1ebe6d8d7a24c7931a3e95833f5e731943cf4d98f2bc66abc
+  metadata.gz: 98f12edb338eb087180087c0d10ed0d23070d06333ee39784faf24c4190ae39cce899e43f0644728a50768743b872954ad90fbd9a962caf67efe0c8ea0b12547
+  data.tar.gz: 7ce0e7b63dcbd094c3c9452851fc91c3660e44ed69fbc3cc7aa67d53cf96ac615f9c172957d0092c054250aa4c12a09837c0f1965e3caa02153a7d092070e449

data/CHANGELOG.md

@@ -31,7 +31,7 @@
 
 * in_tail: Handle records in the correct order on file rotation
   https://github.com/fluent/fluentd/pull/1880
-* out_forward: Fix race condition with <security> on multi thread environment
+* out_forward: Fix race condition with `<security>` on multi thread environment
   https://github.com/fluent/fluentd/pull/1893
 * output: Prevent flushing threads consume too much CPU when retry happens
   https://github.com/fluent/fluentd/pull/1901
@@ -261,7 +261,7 @@ See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
 
 * plugin: Add record_accessor plugin helper
   https://github.com/fluent/fluentd/pull/1637
-* log: Add format and time_format parameters to <system> setting
+* log: Add format and time_format parameters to `<system>` setting
   https://github.com/fluent/fluentd/pull/1644
 
 ### Bug fixes
@@ -355,7 +355,7 @@ See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
 
 ### New features / Enhancements
 
-* Add <worker N> directive
+* Add `<worker N>` directive
   https://github.com/fluent/fluentd/pull/1507
 * in_tail: Do not warn that directories are unreadable in the in_tail plugin
   https://github.com/fluent/fluentd/pull/1540
@@ -410,7 +410,7 @@ See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
   https://github.com/fluent/fluentd/pull/1477
 * Fix Input and Output deadlock when buffer is full during startup
   https://github.com/fluent/fluentd/pull/1502
-* config: Fix log_level handling in <system>
+* config: Fix log_level handling in `<system>`
   https://github.com/fluent/fluentd/pull/1501
 * Fix typo in root agent error log
   https://github.com/fluent/fluentd/pull/1491
@@ -505,7 +505,7 @@ See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
 ## Release v0.14.11 - 2016/12/26
 
 ### New features / Enhancements
-* Add "root_dir" parameter in <system> directive to configure server root directory, used for buffer/storage paths
+* Add "root_dir" parameter in `<system>` directive to configure server root directory, used for buffer/storage paths
   https://github.com/fluent/fluentd/pull/1374
 * Fix not to restart Fluentd processes when unrecoverable errors occur
   https://github.com/fluent/fluentd/pull/1359
@@ -633,7 +633,7 @@ See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
   https://github.com/fluent/fluentd/pull/1207
 * Add a feature to parse/format numeric time (unix time [+ subsecond value])
   https://github.com/fluent/fluentd/pull/1254
-* Raise configuration errors for inconsistent <label> configurations
+* Raise configuration errors for inconsistent `<label>` configurations
   https://github.com/fluent/fluentd/pull/1233
 * Fix to instantiate an unconfigured section even for multi: true
   https://github.com/fluent/fluentd/pull/1210
@@ -959,7 +959,7 @@ This list includes changes of 0.14.0.pre.1 and release candidates.
   https://github.com/fluent/fluentd/pull/838
 * Move TypeConverter mixin to mixin.rb
   https://github.com/fluent/fluentd/pull/842
-* Override default configurations by <system>
+* Override default configurations by `<system>`
   https://github.com/fluent/fluentd/pull/854
 * Suppress Ruby level warnings
   https://github.com/fluent/fluentd/pull/846

data/LICENSE

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2011-2018 Fluentd Authors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

data/README.md

@@ -73,7 +73,7 @@ Many enterprises run Fluentd in production to handle all of their logging needs.
 - Slack / Community: https://slack.fluentd.org
 - Newsletters: https://www.fluentd.org/newsletter_signup
 - Author: Sadayuki Furuhashi
-- Copyright: (c) 2011 FURUHASHI Sadayuki
+- Copyright: 2011-2018 Fluentd Authors
 - License: Apache License, Version 2.0
 
 ## Contributors:

data/lib/fluent/config/types.rb

@@ -70,6 +70,17 @@ module Fluent
       end
     end
 
+    def self.regexp_value(str)
+      return nil unless str
+      return Regexp.compile(str) unless str.start_with?("/")
+      right_slash_position = str.rindex("/")
+      options = str[(right_slash_position + 1)..-1]
+      option = 0
+      option |= Regexp::IGNORECASE if options.include?("i")
+      option |= Regexp::MULTILINE if options.include?("m")
+      Regexp.compile(str[1...right_slash_position], option)
+    end
+
     STRING_TYPE = Proc.new { |val, opts|
       v = val.to_s
       v = v.frozen? ? v.dup : v # config_param can't assume incoming string is mutable
@@ -89,6 +100,7 @@ module Fluent
     SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) }
     BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
     TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
+    REGEXP_TYPE = Proc.new { |val, opts| Config.regexp_value(val) }
 
     REFORMAT_VALUE = ->(type, value) {
       if value.nil?
@@ -101,6 +113,7 @@ module Fluent
         when :size then Config.size_value(value)
         when :bool then Config.bool_value(value)
         when :time then Config.time_value(value)
+        when :regexp then Config.regexp_value(value)
         else
           raise "unknown type in REFORMAT: #{type}"
         end

data/lib/fluent/configurable.rb

@@ -128,6 +128,7 @@ module Fluent
       time: Config::TIME_TYPE,
       hash: Config::HASH_TYPE,
       array: Config::ARRAY_TYPE,
+      regexp: Config::REGEXP_TYPE,
     }.each do |name, type|
       register_type(name, type)
     end

data/lib/fluent/plugin/filter_grep.rb

@@ -25,10 +25,15 @@ module Fluent::Plugin
     def initialize
       super
 
-      @_regexps = {}
-      @_excludes = {}
+      @_regexp_and_conditions = []
+      @_exclude_and_conditions = []
+      @_regexp_or_conditions = []
+      @_exclude_or_conditions = []
     end
 
+    # for test
+    attr_reader :_regexp_and_conditions, :_exclude_and_conditions, :_regexp_or_conditions, :_exclude_or_conditions
+
     helpers :record_accessor
 
     REGEXP_MAX_NUM = 20
@@ -40,79 +45,135 @@ module Fluent::Plugin
       desc "The field name to which the regular expression is applied."
       config_param :key, :string
       desc "The regular expression."
-      config_param :pattern do |value|
-        if value.start_with?("/") and value.end_with?("/")
-          Regexp.compile(value[1..-2])
-        else
-          Regexp.compile(value)
-        end
-      end
+      config_param :pattern, :regexp
     end
 
     config_section :exclude, param_name: :excludes, multi: true do
       desc "The field name to which the regular expression is applied."
       config_param :key, :string
       desc "The regular expression."
-      config_param :pattern do |value|
-        if value.start_with?("/") and value.end_with?("/")
-          Regexp.compile(value[1..-2])
-        else
-          Regexp.compile(value)
-        end
+      config_param :pattern, :regexp
+    end
+
+    config_section :and, param_name: :and_conditions, multi: true do
+      config_section :regexp, param_name: :regexps, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+      config_section :exclude, param_name: :excludes, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
       end
     end
 
-    # for test
-    attr_reader :_regexps
-    attr_reader :_excludes
+    config_section :or, param_name: :or_conditions, multi: true do
+      config_section :regexp, param_name: :regexps, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+      config_section :exclude, param_name: :excludes, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+    end
 
     def configure(conf)
       super
 
-      rs = {}
+      regexp_and_conditions = {}
+      regexp_or_conditions = {}
+      exclude_and_conditions = {}
+      exclude_or_conditions = {}
+
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["regexp#{i}"]
         key, regexp = conf["regexp#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
-        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if rs[key]
-        rs[key] = Regexp.compile(regexp)
+        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if regexp_and_conditions[key]
+        regexp_and_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(regexp))
       end
 
-      es = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["exclude#{i}"]
         key, exclude = conf["exclude#{i}"].split(/ /, 2)
         raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
-        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if es[key]
-        es[key] = Regexp.compile(exclude)
+        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if exclude_or_conditions[key]
+        exclude_or_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(exclude))
       end
 
+      if @regexps.size > 1
+        log.info "Top level multiple <regexp> is intepreted as 'and' condition"
+      end
       @regexps.each do |e|
-        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if rs.key?(e.key)
-        rs[e.key] = e.pattern
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if regexp_and_conditions.key?(e.key)
+        regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+      end
+
+      if @excludes.size > 1
+        log.info "Top level multiple <exclude> is intepreted as 'or' condition"
       end
       @excludes.each do |e|
-        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if es.key?(e.key)
-        es[e.key] = e.pattern
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if exclude_or_conditions.key?(e.key)
+        exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
       end
 
-      rs.each_pair do |k, v|
-        @_regexps[record_accessor_create(k)] = v
+      @and_conditions.each do |and_condition|
+        if !and_condition.regexps.empty? && !and_condition.excludes.empty?
+          raise Fluent::ConfigError, "Do not specify both <regexp> and <exclude> in <and>"
+        end
+        and_condition.regexps.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <and>: #{e.key}" if regexp_and_conditions.key?(e.key)
+          regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+        and_condition.excludes.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <and>: #{e.key}" if exclude_and_conditions.key?(e.key)
+          exclude_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
       end
-      es.each_pair do |k, v|
-        @_excludes[record_accessor_create(k)] = v
+
+      @or_conditions.each do |or_condition|
+        if !or_condition.regexps.empty? && !or_condition.excludes.empty?
+          raise Fluent::ConfigError, "Do not specify both <regexp> and <exclude> in <or>"
+        end
+        or_condition.regexps.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <or>: #{e.key}" if regexp_or_conditions.key?(e.key)
+          regexp_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+        or_condition.excludes.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <or>: #{e.key}" if exclude_or_conditions.key?(e.key)
+          exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
       end
+
+      @_regexp_and_conditions = regexp_and_conditions.values
+      @_exclude_and_conditions = exclude_and_conditions.values
+      @_regexp_or_conditions = regexp_or_conditions.values
+      @_exclude_or_conditions = exclude_or_conditions.values
     end
 
     def filter(tag, time, record)
       result = nil
       begin
         catch(:break_loop) do
-          @_regexps.each do |key, regexp|
-            throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, key.call(record).to_s)
+          @_regexp_and_conditions.each do |expression|
+            throw :break_loop unless expression.match?(record)
+          end
+          if !@_regexp_or_conditions.empty? && @_regexp_or_conditions.none? {|expression| expression.match?(record) }
+            throw :break_loop
+          end
+          if !@_exclude_and_conditions.empty? && @_exclude_and_conditions.all? {|expression| expression.match?(record) }
+            throw :break_loop
           end
-          @_excludes.each do |key, exclude|
-            throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, key.call(record).to_s)
+          @_exclude_or_conditions.each do |expression|
+            throw :break_loop if expression.match?(record)
           end
           result = record
         end
@@ -122,5 +183,11 @@ module Fluent::Plugin
       end
       result
     end
+
+    Expression = Struct.new(:key, :pattern) do
+      def match?(record)
+        ::Fluent::StringUtil.match_regexp(pattern, key.call(record).to_s)
+      end
+    end
   end
 end

data/lib/fluent/plugin/parser_apache.rb

@@ -21,7 +21,7 @@ module Fluent
     class ApacheParser < RegexpParser
       Plugin.register_parser("apache", self)
 
-      config_set_default :expression, %q{/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :expression, /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
       config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
     end
   end

data/lib/fluent/plugin/parser_apache_error.rb

@@ -20,7 +20,7 @@ module Fluent
   module Plugin
     class ApacheErrorParser < RegexpParser
       Plugin.register_parser("apache_error", self)
-      config_set_default :expression, %q{/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/}
+      config_set_default :expression, /^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/
     end
   end
 end

data/lib/fluent/plugin/parser_ltsv.rb

@@ -24,9 +24,7 @@ module Fluent
       desc 'The delimiter character (or string) of TSV values'
       config_param :delimiter, :string, default: "\t"
       desc 'The delimiter pattern of TSV values'
-      config_param :delimiter_pattern, default: nil do |value|
-        Regexp.compile(value[1..-2]) if value
-      end
+      config_param :delimiter_pattern, :regexp, default: nil
       desc 'The delimiter character between field name and value'
       config_param :label_delimiter, :string, default: ":"
 

data/lib/fluent/plugin/parser_multiline.rb

@@ -36,7 +36,7 @@ module Fluent
           if regexp.named_captures.empty?
             raise "No named captures"
           end
-          regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/", "multiline" => true }, [])
+          regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/m" }, [])
           @parser = Fluent::Plugin::RegexpParser.new
           @parser.configure(conf + regexp_conf)
         rescue => e

data/lib/fluent/plugin/parser_nginx.rb

@@ -21,7 +21,7 @@ module Fluent
     class NginxParser < RegexpParser
       Plugin.register_parser("nginx", self)
 
-      config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :expression, /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+(?<http_x_forwarded_for>[^ ]+))?)?$/
       config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
     end
   end

data/lib/fluent/plugin/parser_regexp.rb

@@ -22,30 +22,28 @@ module Fluent
       Plugin.register_parser("regexp", self)
 
       desc 'Regular expression for matching logs'
-      config_param :expression, :string
+      config_param :expression, :regexp
       desc 'Ignore case in matching'
-      config_param :ignorecase, :bool, default: false
+      config_param :ignorecase, :bool, default: false, deprecated: "Use /pattern/i instead, this option is no longer effective"
       desc 'Build regular expression as a multline mode'
-      config_param :multiline, :bool, default: false
+      config_param :multiline, :bool, default: false, deprecated: "Use /pattern/m instead, this option is no longer effective"
 
       config_set_default :time_key, 'time'
 
       def configure(conf)
         super
-
-        expr = if @expression[0] == "/" && @expression[-1] == "/"
-                 @expression[1..-2]
-               else
-                 @expression
-               end
-        regexp_option = 0
-        regexp_option |= Regexp::IGNORECASE if @ignorecase
-        regexp_option |= Regexp::MULTILINE if @multiline
-        @regexp = Regexp.new(expr, regexp_option)
+        # For compat layer
+        if @ignorecase || @multiline
+          options = 0
+          options |= Regexp::IGNORECASE if @ignorecase
+          options |= Regexp::MULTILINE if @multiline
+          @expression = Regexp.compile(@expression.source, options)
+        end
+        @regexp = @expression # For backward compatibility
       end
 
       def parse(text)
-        m = @regexp.match(text)
+        m = @expression.match(text)
         unless m
           yield nil, nil
           return

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.1.3'
+  VERSION = '1.2.0.pre1'
 
 end

data/test/config/test_config_parser.rb

@@ -35,6 +35,7 @@ module Fluent::Config
     config_param :param_time, :time
     config_param :param_hash, :hash
     config_param :param_array, :array
+    config_param :param_regexp, :regexp
   end
 
   class TestV1Parser < ::Test::Unit::TestCase
@@ -451,6 +452,7 @@ module Fluent::Config
           param_time 10m
           param_hash { "key1": "value1", "key2": 2 }
           param_array ["value1", "value2", 100]
+          param_regexp /pattern/
         ])
         target = AllTypes.new.configure(conf)
         assert_equal(conf.to_s, target.config.to_s)
@@ -465,6 +467,7 @@ module Fluent::Config
   param_time 10m
   param_hash {"key1":"value1","key2":2}
   param_array ["value1","value2",100]
+  param_regexp /pattern/
 </ROOT>
 DUMP
         assert_equal(expected, conf.to_s)
@@ -490,6 +493,7 @@ DUMP
           param_time 10m
           param_hash { "key1": "value1", "key2": 2 }
           param_array ["value1", "value2", 100]
+          param_regexp /pattern/
         ])
         target = AllTypes.new.configure(conf)
         assert_equal(conf.to_s, target.config.to_s)
@@ -504,6 +508,7 @@ DUMP
   param_time 10m
   param_hash { "key1": "value1", "key2": 2 }
   param_array ["value1", "value2", 100]
+  param_regexp /pattern/
 </ROOT>
 DUMP
         assert_equal(expected, conf.to_s)

data/test/config/test_configure_proxy.rb

@@ -193,9 +193,10 @@ module Fluent::Config
           assert_nothing_raised{ @proxy.config_param(:p7, :time, **opt) }
           assert_nothing_raised{ @proxy.config_param(:p8, :hash, **opt) }
           assert_nothing_raised{ @proxy.config_param(:p9, :array, **opt) }
+          assert_nothing_raised{ @proxy.config_param(:pa, :regexp, **opt) }
         end
 
-        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, hash: :hash, array: :array)
+        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, hash: :hash, array: :array, regexp: :regexp)
         test 'deny list for non-enum types' do |type|
           assert_raise ArgumentError.new(":list is valid only for :enum type, but #{type}: arg") do
             @proxy.config_argument(:arg, type, list: [:a, :b])
@@ -205,7 +206,7 @@ module Fluent::Config
           end
         end
 
-        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time)
+        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, regexp: :regexp)
         test 'deny value_type for non-hash/array types' do |type|
           assert_raise ArgumentError.new(":value_type is valid only for :hash and :array, but #{type}: arg") do
             @proxy.config_argument(:arg, type, value_type: :string)
@@ -215,7 +216,7 @@ module Fluent::Config
           end
         end
 
-        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, array: :array)
+        data(string: :string, integer: :integer, float: :float, size: :size, bool: :bool, time: :time, array: :array, regexp: :regexp)
         test 'deny symbolize_keys for non-hash types' do |type|
           assert_raise ArgumentError.new(":symbolize_keys is valid only for :hash, but #{type}: arg") do
             @proxy.config_argument(:arg, type, symbolize_keys: true)

data/test/config/test_types.rb

@@ -61,6 +61,26 @@ class TestConfigTypes < ::Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'Config.regexp_value' do
+    data("empty" => [//, "//"],
+         "plain" => [/regexp/, "/regexp/"],
+         "zero width" => [/^$/, "/^$/"],
+         "character classes" => [/[a-z]/, "/[a-z]/"],
+         "meta charactersx" => [/.+.*?\d\w\s\S/, '/.+.*?\d\w\s\S/'])
+    test 'normal case' do |(expected, str)|
+      assert_equal(expected, Config.regexp_value(str))
+    end
+
+    data("empty" => [//, ""],
+         "plain" => [/regexp/, "regexp"],
+         "zero width" => [/^$/, "^$"],
+         "character classes" => [/[a-z]/, "[a-z]"],
+         "meta charactersx" => [/.+.*?\d\w\s\S/, '.+.*?\d\w\s\S'])
+    test 'w/o slashes' do |(expected, str)|
+      assert_equal(expected, Config.regexp_value(str))
+    end
+  end
+
   sub_test_case 'type converters for config_param definitions' do
     test 'string' do
       assert_equal 'test', Config::STRING_TYPE.call('test', {})
@@ -134,6 +154,15 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_equal 86400, Config::TIME_TYPE.call('1d', {})
     end
 
+    data("empty" => [//, "//"],
+         "plain" => [/regexp/, "/regexp/"],
+         "zero width" => [/^$/, "/^$/"],
+         "character classes" => [/[a-z]/, "/[a-z]/"],
+         "meta charactersx" => [/.+.*?\d\w\s\S/, '/.+.*?\d\w\s\S/'])
+    test 'regexp' do |(expected, str)|
+      assert_equal(expected, Config::REGEXP_TYPE.call(str, {}))
+    end
+
     test 'hash' do
       assert_equal({"x"=>"v","k"=>1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {}))
       assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v,k:1', {}))

data/test/plugin/test_filter_grep.rb

@@ -23,15 +23,15 @@ class GrepFilterTest < Test::Unit::TestCase
 
     test "regexpN can contain a space" do
       d = create_driver(%[regexp1 message  foo])
-      d.instance._regexps.each_pair { |key, value|
-        assert_equal(Regexp.compile(/ foo/), value)
+      d.instance._regexp_and_conditions.each { |value|
+        assert_equal(Regexp.compile(/ foo/), value.pattern)
       }
     end
 
     test "excludeN can contain a space" do
       d = create_driver(%[exclude1 message  foo])
-      d.instance._excludes.each_pair { |key, value|
-        assert_equal(Regexp.compile(/ foo/), value)
+      d.instance._exclude_or_conditions.each { |value|
+        assert_equal(Regexp.compile(/ foo/), value.pattern)
       }
     end
 
@@ -72,6 +72,118 @@ class GrepFilterTest < Test::Unit::TestCase
           create_driver(conf)
         end
       end
+
+      test "and/regexp" do
+        conf = %[
+          <and>
+            <regexp>
+              key message
+              pattern test
+            </regexp>
+            <regexp>
+              key message
+              pattern test
+            </regexp>
+          </and>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+
+      test "and/regexp, and/regexp" do
+        conf = %[
+          <and>
+            <regexp>
+              key message
+              pattern test
+            </regexp>
+          </and>
+          <and>
+            <regexp>
+              key message
+              pattern test
+            </regexp>
+          </and>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+
+      test "regexp, and/regexp" do
+        conf = %[
+          <regexp>
+            key message
+            pattern test
+          </regexp>
+          <and>
+            <regexp>
+              key message
+              pattern test
+            </regexp>
+          </and>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+
+      test "and/exclude" do
+        conf = %[
+          <and>
+            <exclude>
+              key message
+              pattern test
+            </exclude>
+            <exclude>
+              key message
+              pattern test
+            </exclude>
+          </and>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+
+      test "and/exclude, and/exclude" do
+        conf = %[
+          <and>
+            <exclude>
+              key message
+              pattern test
+            </exclude>
+          </and>
+          <and>
+            <exclude>
+              key message
+              pattern test
+            </exclude>
+          </and>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+
+      test "exclude, or/exclude" do
+        conf = %[
+          <exclude>
+            key message
+            pattern test
+          </exclude>
+          <or>
+            <exclude>
+              key message
+              pattern test
+            </exclude>
+          </or>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
     end
 
     sub_test_case "pattern with slashes" do
@@ -91,6 +203,44 @@ class GrepFilterTest < Test::Unit::TestCase
         assert_equal(/[A-Z]test/, d.instance.excludes.first.pattern)
       end
     end
+
+    sub_test_case "and/or section" do
+      test "<and> section cannot include both <regexp> and <exclude>" do
+        conf = %[
+          <and>
+            <regexp>
+              key message
+              pattern /test/
+            </regexp>
+            <exclude>
+              key level
+              pattern /debug/
+            </exclude>
+          </and>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+
+      test "<or> section cannot include both <regexp> and <exclude>" do
+        conf = %[
+          <or>
+            <regexp>
+              key message
+              pattern /test/
+            </regexp>
+            <exclude>
+              key level
+              pattern /debug/
+            </exclude>
+          </or>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+    end
   end
 
   sub_test_case 'filter_stream' do
@@ -183,6 +333,289 @@ class GrepFilterTest < Test::Unit::TestCase
         }
       end
     end
+
+    sub_test_case "and/or section" do
+      def records
+        [
+          { "time" => "2013/01/13T07:02:11.124202", "level" => "INFO", "method" => "GET", "path" => "/ping" },
+          { "time" => "2013/01/13T07:02:13.232645", "level" => "WARN", "method" => "POST", "path" => "/auth" },
+          { "time" => "2013/01/13T07:02:21.542145", "level" => "WARN", "method" => "GET", "path" => "/favicon.ico" },
+          { "time" => "2013/01/13T07:02:43.632145", "level" => "WARN", "method" => "POST", "path" => "/login" },
+        ]
+      end
+
+      def filter(conf, records)
+        d = create_driver(conf)
+        d.run do
+          records.each do |record|
+            d.feed("filter.test", @time, record)
+          end
+        end
+        d.filtered_records
+      end
+
+      test "basic and/regexp" do
+        conf = %[
+          <and>
+            <regexp>
+              key level
+              pattern ^INFO$
+            </regexp>
+            <regexp>
+              key method
+              pattern ^GET$
+            </regexp>
+          </and>
+        ]
+        filtered_records = filter(conf, records)
+        assert_equal(records.values_at(0), filtered_records)
+      end
+
+      test "basic or/exclude" do
+        conf = %[
+          <or>
+            <exclude>
+              key level
+              pattern ^INFO$
+            </exclude>
+            <exclude>
+              key method
+              pattern ^GET$
+            </exclude>
+          </or>
+        ]
+        filtered_records = filter(conf, records)
+        assert_equal(records.values_at(1, 3), filtered_records)
+      end
+
+      test "basic or/regexp" do
+        conf = %[
+          <or>
+            <regexp>
+              key level
+              pattern ^INFO$
+            </regexp>
+            <regexp>
+              key method
+              pattern ^GET$
+            </regexp>
+          </or>
+        ]
+        filtered_records = filter(conf, records)
+        assert_equal(records.values_at(0, 2), filtered_records)
+      end
+
+      test "basic and/exclude" do
+        conf = %[
+          <and>
+            <exclude>
+              key level
+              pattern ^INFO$
+            </exclude>
+            <exclude>
+              key method
+              pattern ^GET$
+            </exclude>
+          </and>
+        ]
+        filtered_records = filter(conf, records)
+        assert_equal(records.values_at(1, 2, 3), filtered_records)
+      end
+
+      sub_test_case "and/or combo" do
+        def records
+          [
+            { "time" => "2013/01/13T07:02:11.124202", "level" => "INFO", "method" => "GET", "path" => "/ping" },
+            { "time" => "2013/01/13T07:02:13.232645", "level" => "WARN", "method" => "POST", "path" => "/auth" },
+            { "time" => "2013/01/13T07:02:21.542145", "level" => "WARN", "method" => "GET", "path" => "/favicon.ico" },
+            { "time" => "2013/01/13T07:02:43.632145", "level" => "WARN", "method" => "POST", "path" => "/login" },
+            { "time" => "2013/01/13T07:02:44.959307", "level" => "ERROR", "method" => "POST", "path" => "/login" },
+            { "time" => "2013/01/13T07:02:45.444992", "level" => "ERROR", "method" => "GET", "path" => "/ping" },
+            { "time" => "2013/01/13T07:02:51.247941", "level" => "WARN", "method" => "GET", "path" => "/info" },
+            { "time" => "2013/01/13T07:02:53.108366", "level" => "WARN", "method" => "POST", "path" => "/ban" },
+          ]
+        end
+
+        test "and/regexp, or/exclude" do
+          conf = %[
+            <and>
+              <regexp>
+                key level
+                pattern ^ERROR|WARN$
+              </regexp>
+              <regexp>
+                key method
+                pattern ^GET|POST$
+              </regexp>
+            </and>
+            <or>
+              <exclude>
+                key level
+                pattern ^WARN$
+              </exclude>
+              <exclude>
+                key method
+                pattern ^GET$
+              </exclude>
+            </or>
+          ]
+          filtered_records = filter(conf, records)
+          assert_equal(records.values_at(4), filtered_records)
+        end
+
+        test "and/regexp, and/exclude" do
+          conf = %[
+            <and>
+              <regexp>
+                key level
+                pattern ^ERROR|WARN$
+              </regexp>
+              <regexp>
+                key method
+                pattern ^GET|POST$
+              </regexp>
+            </and>
+            <and>
+              <exclude>
+                key level
+                pattern ^WARN$
+              </exclude>
+              <exclude>
+                key method
+                pattern ^GET$
+              </exclude>
+            </and>
+          ]
+          filtered_records = filter(conf, records)
+          assert_equal(records.values_at(1, 3, 4, 5, 7), filtered_records)
+        end
+
+        test "or/regexp, and/exclude" do
+          conf = %[
+            <or>
+              <regexp>
+                key level
+                pattern ^ERROR|WARN$
+              </regexp>
+              <regexp>
+                key method
+                pattern ^GET|POST$
+              </regexp>
+            </or>
+            <and>
+              <exclude>
+                key level
+                pattern ^WARN$
+              </exclude>
+              <exclude>
+                key method
+                pattern ^GET$
+              </exclude>
+            </and>
+          ]
+          filtered_records = filter(conf, records)
+          assert_equal(records.values_at(0, 1, 3, 4, 5, 7), filtered_records)
+        end
+
+        test "or/regexp, or/exclude" do
+          conf = %[
+            <or>
+              <regexp>
+                key level
+                pattern ^ERROR|WARN$
+              </regexp>
+              <regexp>
+                key method
+                pattern ^GET|POST$
+              </regexp>
+            </or>
+            <or>
+              <exclude>
+                key level
+                pattern ^WARN$
+              </exclude>
+              <exclude>
+                key method
+                pattern ^GET$
+              </exclude>
+            </or>
+          ]
+          filtered_records = filter(conf, records)
+          assert_equal(records.values_at(4), filtered_records)
+        end
+
+        test "regexp, and/regexp" do
+          conf = %[
+            <and>
+              <regexp>
+                key level
+                pattern ^ERROR|WARN$
+              </regexp>
+              <regexp>
+                key method
+                pattern ^GET|POST$
+              </regexp>
+            </and>
+            <regexp>
+              key path
+              pattern ^/login$
+            </regexp>
+          ]
+          filtered_records = filter(conf, records)
+          assert_equal(records.values_at(3, 4), filtered_records)
+        end
+
+        test "regexp, or/exclude" do
+          conf = %[
+            <regexp>
+              key level
+              pattern ^ERROR|WARN$
+            </regexp>
+            <regexp>
+              key method
+              pattern ^GET|POST$
+            </regexp>
+            <or>
+              <exclude>
+                key level
+                pattern ^WARN$
+              </exclude>
+              <exclude>
+                key method
+                pattern ^GET$
+              </exclude>
+            </or>
+          ]
+          filtered_records = filter(conf, records)
+          assert_equal(records.values_at(4), filtered_records)
+        end
+
+        test "regexp, and/exclude" do
+          conf = %[
+            <regexp>
+              key level
+              pattern ^ERROR|WARN$
+            </regexp>
+            <regexp>
+              key method
+              pattern ^GET|POST$
+            </regexp>
+            <and>
+              <exclude>
+                key level
+                pattern ^WARN$
+              </exclude>
+              <exclude>
+                key method
+                pattern ^GET$
+              </exclude>
+            </and>
+          ]
+          filtered_records = filter(conf, records)
+          assert_equal(records.values_at(1, 3, 4, 5, 7), filtered_records)
+        end
+      end
+    end
   end
 
   sub_test_case 'nested keys' do

data/test/plugin/test_in_exec.rb

@@ -131,9 +131,7 @@ EOC
       d = create_driver REGEXP_CONFIG
       assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
       assert_equal "regex_tag", d.instance.tag
-      expression = <<'EXP'.chomp
-(?<time>[^\]]*) (?<message>[^ ]*)
-EXP
+      expression = /(?<time>[^\]]*) (?<message>[^ ]*)/
       assert_equal expression, d.instance.parser.expression
       assert_nil d.instance.extract_config
     end
@@ -194,8 +192,8 @@ EXP
     test 'configure_with_regexp' do
       d = create_driver REGEXP_CONFIG_COMPAT
       assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
-      assert_equal '(?<time>[^\]]*) (?<message>[^ ]*)', d.instance.parser.expression
-      assert_equal 'regex_tag', d.instance.tag
+      assert_equal(/(?<time>[^\]]*) (?<message>[^ ]*)/, d.instance.parser.expression)
+      assert_equal('regex_tag', d.instance.tag)
     end
   end
 

data/test/plugin/test_parser_nginx.rb

@@ -16,6 +16,18 @@ class NginxParserTest < ::Test::Unit::TestCase
       'referer' => '-',
       'agent'   => 'Opera/12.0'
     }
+    @expected_extended = {
+      'remote'               => '127.0.0.1',
+      'host'                 => '192.168.0.1',
+      'user'                 => '-',
+      'method'               => 'GET',
+      'path'                 => '/',
+      'code'                 => '200',
+      'size'                 => '777',
+      'referer'              => '-',
+      'agent'                => 'Opera/12.0',
+      'http_x_forwarded_for' => '-'
+    }
   end
 
   def create_driver
@@ -45,4 +57,12 @@ class NginxParserTest < ::Test::Unit::TestCase
       assert_equal(@expected, record)
     }
   end
+
+  def test_parse_with_http_x_forwarded_for
+    d = create_driver
+    d.instance.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0" -') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal(@expected_extended, record)
+    }
+  end
 end

data/test/plugin/test_parser_regexp.rb

@@ -160,25 +160,24 @@ class RegexpParserTest < ::Test::Unit::TestCase
           'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
         }
         d = create_driver(conf)
-        regexp = d.instance.instance_variable_get(:@regexp)
+        regexp = d.instance.expression
         assert_equal(0, regexp.options)
       end
 
       data(
-        ignorecase: [{ "ignorecase" => true }, Regexp::IGNORECASE],
-        multiline: [{ "multiline" => true }, Regexp::MULTILINE],
-        ignorecase_multiline: [{ "ignorecase" => true, "multiline" => true }, Regexp::IGNORECASE | Regexp::MULTILINE],
+        ignorecase: ["i", Regexp::IGNORECASE],
+        multiline: ["m", Regexp::MULTILINE],
+        ignorecase_multiline: ["im", Regexp::IGNORECASE | Regexp::MULTILINE],
       )
       def test_options(data)
         regexp_option, expected = data
         conf = {
-          'expression' => %q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/!,
+          'expression' => %Q!/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] \[(?<date>[^\]]*)\] "(?<flag>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/#{regexp_option}!,
           'time_format' => "%d/%b/%Y:%H:%M:%S %z",
           'types' => 'user:string,date:time:%d/%b/%Y:%H:%M:%S %z,flag:bool,path:array,code:float,size:integer'
         }
-        conf = conf.merge(regexp_option)
         d = create_driver(conf)
-        regexp = d.instance.instance_variable_get(:@regexp)
+        regexp = d.instance.expression
         assert_equal(expected, regexp.options)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.2.0.pre1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-04 00:00:00.000000000 Z
+date: 2018-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -329,7 +329,6 @@ files:
 - AUTHORS
 - CHANGELOG.md
 - CONTRIBUTING.md
-- COPYING
 - GOVERNANCE.md
 - Gemfile
 - LICENSE
@@ -748,9 +747,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13

data/COPYING

@@ -1,14 +0,0 @@
-Copyright (C) 2011 FURUHASHI Sadayuki
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-