fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/lib/fluent/plugin/parser_apache.rb

@@ -21,7 +21,7 @@ module Fluent
     class ApacheParser < RegexpParser
       Plugin.register_parser("apache", self)
 
-      config_set_default :expression, %q{/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :expression, /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
       config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
     end
   end

data/lib/fluent/plugin/parser_apache2.rb

@@ -26,10 +26,14 @@ module Fluent
 
       def initialize
         super
-        @time_parser = time_parser_create(format: TIME_FORMAT)
         @mutex = Mutex.new
       end
 
+      def configure(conf)
+        super
+        @time_parser = time_parser_create(format: TIME_FORMAT)
+      end
+
       def patterns
         {'format' => REGEXP, 'time_format' => TIME_FORMAT}
       end

data/lib/fluent/plugin/parser_apache_error.rb

@@ -20,7 +20,7 @@ module Fluent
   module Plugin
     class ApacheErrorParser < RegexpParser
       Plugin.register_parser("apache_error", self)
-      config_set_default :expression, %q{/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/}
+      config_set_default :expression, /^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/
     end
   end
 end

data/lib/fluent/plugin/parser_json.rb

@@ -27,6 +27,7 @@ module Fluent
       Plugin.register_parser('json', self)
 
       config_set_default :time_key, 'time'
+      desc 'Set JSON parser'
       config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
 
       config_set_default :time_type, :float
@@ -51,9 +52,15 @@ module Fluent
         else
           raise "BUG: unknown json parser specified: #{name}"
         end
-      rescue LoadError
+      rescue LoadError => ex
         name = :yajl
-        log.info "Oj is not installed, and failing back to Yajl for json parser" if log
+        if log
+          if /\boj\z/ =~ ex.message
+            log.info "Oj is not installed, and failing back to Yajl for json parser"
+          else
+            log.warn ex.message
+          end
+        end
         retry
       end
 
@@ -61,7 +68,7 @@ module Fluent
         r = @load_proc.call(text)
         time, record = convert_values(parse_time(r), r)
         yield time, record
-      rescue @error_class
+      rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
         yield nil, nil
       end
 

data/lib/fluent/plugin/parser_ltsv.rb

@@ -23,11 +23,18 @@ module Fluent
 
       desc 'The delimiter character (or string) of TSV values'
       config_param :delimiter, :string, default: "\t"
+      desc 'The delimiter pattern of TSV values'
+      config_param :delimiter_pattern, :regexp, default: nil
       desc 'The delimiter character between field name and value'
       config_param :label_delimiter, :string, default: ":"
 
       config_set_default :time_key, 'time'
 
+      def configure(conf)
+        super
+        @delimiter = @delimiter_pattern || @delimiter
+      end
+
       def parse(text)
         r = {}
         text.split(@delimiter).each do |pair|

data/lib/fluent/plugin/parser_multiline.rb

@@ -22,6 +22,7 @@ module Fluent
     class MultilineParser < Parser
       Plugin.register_parser('multiline', self)
 
+      desc 'Specify regexp pattern for start line of multiple lines'
       config_param :format_firstline, :string, default: nil
 
       FORMAT_MAX_NUM = 20
@@ -35,7 +36,7 @@ module Fluent
           if regexp.named_captures.empty?
             raise "No named captures"
           end
-          regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/", "multiline" => true }, [])
+          regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/m" }, [])
           @parser = Fluent::Plugin::RegexpParser.new
           @parser.configure(conf + regexp_conf)
         rescue => e

data/lib/fluent/plugin/parser_nginx.rb

@@ -21,7 +21,7 @@ module Fluent
     class NginxParser < RegexpParser
       Plugin.register_parser("nginx", self)
 
-      config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :expression, /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+\"?(?<http_x_forwarded_for>[^\"]*)\"?)?)?$/
       config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
     end
   end

data/lib/fluent/plugin/parser_none.rb

@@ -23,6 +23,7 @@ module Fluent
     class NoneParser < Parser
       Plugin.register_parser('none', self)
 
+      desc 'Field name to contain logs'
       config_param :message_key, :string, default: 'message'
 
       def parse(text)

data/lib/fluent/plugin/parser_regexp.rb

@@ -21,28 +21,29 @@ module Fluent
     class RegexpParser < Parser
       Plugin.register_parser("regexp", self)
 
-      config_param :expression, :string
-      config_param :ignorecase, :bool, default: false
-      config_param :multiline, :bool, default: false
+      desc 'Regular expression for matching logs'
+      config_param :expression, :regexp
+      desc 'Ignore case in matching'
+      config_param :ignorecase, :bool, default: false, deprecated: "Use /pattern/i instead, this option is no longer effective"
+      desc 'Build regular expression as a multline mode'
+      config_param :multiline, :bool, default: false, deprecated: "Use /pattern/m instead, this option is no longer effective"
 
       config_set_default :time_key, 'time'
 
       def configure(conf)
         super
-
-        expr = if @expression[0] == "/" && @expression[-1] == "/"
-                 @expression[1..-2]
-               else
-                 @expression
-               end
-        regexp_option = 0
-        regexp_option |= Regexp::IGNORECASE if @ignorecase
-        regexp_option |= Regexp::MULTILINE if @multiline
-        @regexp = Regexp.new(expr, regexp_option)
+        # For compat layer
+        if @ignorecase || @multiline
+          options = 0
+          options |= Regexp::IGNORECASE if @ignorecase
+          options |= Regexp::MULTILINE if @multiline
+          @expression = Regexp.compile(@expression.source, options)
+        end
+        @regexp = @expression # For backward compatibility
       end
 
       def parse(text)
-        m = @regexp.match(text)
+        m = @expression.match(text)
         unless m
           yield nil, nil
           return

data/lib/fluent/plugin/parser_syslog.rb

@@ -24,15 +24,19 @@ module Fluent
       Plugin.register_parser('syslog', self)
 
       # From existence TextParser pattern
-      REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
-      REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
+      REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|\[(.*)\]))(?: (?<message>.+))?$\z/
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
+      desc 'If the incoming logs have priority prefix, e.g. <9>, set true'
       config_param :with_priority, :bool, default: false
+      desc 'Specify protocol format'
       config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
+      desc 'Specify time format for event time for rfc5424 protocol'
       config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
 
       def initialize
@@ -55,7 +59,7 @@ module Fluent
                       alias_method :parse, :parse_plain
                     end
                     @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
-                    REGEXP_RFC5424
+                    @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
                   when :auto
                     class << self
                       alias_method :parse, :parse_auto
@@ -77,7 +81,7 @@ module Fluent
 
       def parse_auto(text, &block)
         if REGEXP_DETECT_RFC5424.match(text)
-          @regexp = REGEXP_RFC5424
+          @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424
           @time_parser = @time_parser_rfc5424
         else
           @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP

data/lib/fluent/plugin_helper.rb

@@ -26,7 +26,9 @@ require 'fluent/plugin_helper/inject'
 require 'fluent/plugin_helper/extract'
 require 'fluent/plugin_helper/socket'
 require 'fluent/plugin_helper/server'
+require 'fluent/plugin_helper/counter'
 require 'fluent/plugin_helper/retry_state'
+require 'fluent/plugin_helper/record_accessor'
 require 'fluent/plugin_helper/compat_parameters'
 
 module Fluent

data/lib/fluent/plugin_helper/cert_option.rb

@@ -33,6 +33,11 @@ module Fluent
           ctx.ciphers = ciphers
         end
 
+        if conf.client_cert_auth
+            ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+        end
+
+        ctx.ca_file = conf.ca_path
         ctx.cert = cert
         ctx.key = key
         if extra && !extra.empty?
@@ -104,18 +109,28 @@ module Fluent
         cert.not_before = Time.at(0)
         cert.not_after = Time.now + opts[:expiration]
         cert.public_key = key
-        cert.serial = 1
+        cert.version = 2
+        cert.serial = rand(2**(8*10))
         cert.issuer = issuer
         cert.subject  = subject
 
         return cert, key
       end
 
+      def cert_option_add_extensions(cert, extensions)
+        ef = OpenSSL::X509::ExtensionFactory.new
+        extensions.each do |ext|
+          oid, value = ext
+          cert.add_extension ef.create_extension(oid, value)
+        end
+      end
+
       def cert_option_generate_ca_pair_self_signed(generate_opts)
         cert, key = cert_option_generate_pair(generate_opts)
 
-        # basicConstraints: this cert is for CA or not
-        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(true)]))
+        cert_option_add_extensions(cert, [
+          ['basicConstraints', 'CA:TRUE']
+        ])
 
         cert.sign(key, generate_opts[:digest].to_s)
         return cert, key
@@ -127,9 +142,12 @@ module Fluent
         cert, key = cert_option_generate_pair(generate_opts, ca_cert.subject)
         raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
 
-        # basicConstraints: this cert is for CA or not
-        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
-        cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+        cert_option_add_extensions(cert, [
+          ['basicConstraints', 'CA:FALSE'],
+          ['nsCertType', 'server'],
+          ['keyUsage', 'digitalSignature,keyEncipherment'],
+          ['extendedKeyUsage', 'serverAuth']
+        ])
 
         cert.sign(ca_key, generate_opts[:digest].to_s)
         return cert, key, nil
@@ -139,9 +157,10 @@ module Fluent
         cert, key = cert_option_generate_pair(generate_opts)
         raise "BUG: certificate digest algorithm not set" unless generate_opts[:digest]
 
-        # basicConstraints: this cert is for CA or not
-        cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
-        cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+        cert_option_add_extensions(cert, [
+          ['basicConstraints', 'CA:FALSE'],
+          ['nsCertType', 'server']
+        ])
 
         cert.sign(key, generate_opts[:digest].to_s)
         return cert, key, nil

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -65,7 +65,9 @@ module Fluent
         "label_delimiter"  => "label_delimiter", # LabeledTSVParser
         "format_firstline" => "format_firstline", # MultilineParser
         "message_key"      => "message_key", # NoneParser
-        "with_priority"    => "with_priority", # SyslogParser
+        "with_priority"       => "with_priority", # SyslogParser
+        "message_format"      => "message_format", # SyslogParser
+        "rfc5424_time_format" => "rfc5424_time_format", # SyslogParser
         # There has been no parsers which can handle timezone in v0.12
       }
 

data/lib/fluent/plugin_helper/counter.rb

@@ -0,0 +1,51 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/counter/client'
+
+module Fluent
+  module PluginHelper
+    module Counter
+      def counter_client_create(scope:, loop: Coolio::Loop.new)
+        client_conf = system_config.counter_client
+        raise Fluent::ConfigError, '<counter_client> is required in <system>' unless client_conf
+        counter_client = Fluent::Counter::Client.new(loop, port: client_conf.port, host: client_conf.host, log: log, timeout: client_conf.timeout)
+        counter_client.start
+        counter_client.establish(scope)
+        @_counter_client = counter_client
+        counter_client
+      end
+
+      attr_reader :_counter_client
+
+      def initialize
+        super
+        @_counter_client = nil
+      end
+
+      def stop
+        super
+        @_counter_client.stop
+      end
+
+      def terminate
+        @_counter_client = nil
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/event_loop.rb

@@ -43,6 +43,15 @@ module Fluent
         end
       end
 
+      def event_loop_detach(watcher)
+        if watcher.attached?
+          watcher.detach
+        end
+        @_event_loop_mutex.synchronize do
+          @_event_loop_attached_watchers.delete(watcher)
+        end
+      end
+
       def event_loop_wait_until_start
         sleep(0.1) until event_loop_running?
       end

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -0,0 +1,210 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+unless {}.respond_to?(:dig)
+  begin
+    # backport_dig is faster than dig_rb so prefer backport_dig.
+    require 'backport_dig'
+  rescue LoadError
+    require 'dig_rb'
+  end
+end
+
+module Fluent
+  module PluginHelper
+    module RecordAccessor
+      def record_accessor_create(param)
+        Accessor.new(param)
+      end
+
+      class Accessor
+        attr_reader :keys
+
+        def initialize(param)
+          @keys = Accessor.parse_parameter(param)
+
+          if @keys.is_a?(Array)
+            @last_key = @keys.last
+            @dig_keys = @keys[0..-2]
+            if @dig_keys.empty?
+              @keys = @keys.first
+              mcall = method(:call_index)
+              mdelete = method(:delete_top)
+            else
+              mcall = method(:call_dig)
+              mdelete = method(:delete_nest)
+            end
+          else
+            # Call [] for single key to reduce dig overhead
+            mcall = method(:call_index)
+            mdelete = method(:delete_top)
+          end
+
+          singleton_class.module_eval do
+            define_method(:call, mcall)
+            define_method(:delete, mdelete)
+          end
+        end
+
+        def call(r)
+        end
+
+        # To optimize the performance, use class_eval with pre-expanding @keys
+        # See https://gist.github.com/repeatedly/ab553ed260cd080bd01ec71da9427312
+        def call_dig(r)
+          r.dig(*@keys)
+        end
+
+        def call_index(r)
+          r[@keys]
+        end
+
+        def delete(r)
+        end
+
+        def delete_nest(r)
+          if target = r.dig(*@dig_keys)
+            if target.is_a?(Array)
+              target.delete_at(@last_key)
+            else
+              target.delete(@last_key)
+            end
+          end
+        rescue
+          nil
+        end
+
+        def delete_top(r)
+          r.delete(@keys)
+        end
+
+        def self.parse_parameter(param)
+          if param.start_with?('$.')
+            parse_dot_notation(param)
+          elsif param.start_with?('$[')
+            parse_bracket_notation(param)
+          else
+            param
+          end
+        end
+
+        def self.parse_dot_notation(param)
+          result = []
+          keys = param[2..-1].split('.')
+          keys.each { |key|
+            if key.include?('[')
+              result.concat(parse_dot_array_op(key, param))
+            else
+              result << key
+            end
+          }
+
+          raise Fluent::ConfigError, "empty keys in dot notation" if result.empty?
+          validate_dot_keys(result)
+
+          result
+        end
+
+        def self.validate_dot_keys(keys)
+          keys.each { |key|
+            next unless key.is_a?(String)
+            if /\s+/.match(key)
+              raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
+            end
+          }
+        end
+
+        def self.parse_dot_array_op(key, param)
+          start = key.index('[')
+          result = if start.zero?
+                     []
+                   else
+                     [key[0..start - 1]]
+                   end
+          key = key[start + 1..-1]
+          in_bracket = true
+
+          until key.empty?
+            if in_bracket
+              if i = key.index(']')
+                index_value = key[0..i - 1]
+                raise Fluent::ConfigError, "missing array index in '[]'. Invalid syntax: #{param}" if index_value == ']'
+                result << Integer(index_value)
+                key = key[i + 1..-1]
+                in_bracket = false
+              else
+                raise Fluent::ConfigError, "'[' found but ']' not found. Invalid syntax: #{param}"
+              end
+            else
+              if i = key.index('[')
+                key = key[i + 1..-1]
+                in_bracket = true
+              else
+                raise Fluent::ConfigError, "found more characters after ']'. Invalid syntax: #{param}"
+              end
+            end
+          end
+
+          result
+        end
+
+        def self.parse_bracket_notation(param)
+          orig_param = param
+          result = []
+          param = param[1..-1]
+          in_bracket = false
+
+          until param.empty?
+            if in_bracket
+              if param[0] == "'" || param[0] == '"'
+                if i = param.index("']") || param.index('"]')
+                  raise Fluent::ConfigError, "Mismatched quotes. Invalid syntax: #{orig_param}" unless param[0] == param[i]
+                  result << param[1..i - 1]
+                  param = param[i + 2..-1]
+                  in_bracket = false
+                else
+                  raise Fluent::ConfigError, "Incomplete bracket. Invalid syntax: #{orig_param}"
+                end
+              else
+                if i = param.index(']')
+                  index_value = param[0..i - 1]
+                  raise Fluent::ConfigError, "missing array index in '[]'. Invalid syntax: #{param}" if index_value == ']'
+                  result << Integer(index_value)
+                  param = param[i + 1..-1]
+                  in_bracket = false
+                else
+                  raise Fluent::ConfigError, "'[' found but ']' not found. Invalid syntax: #{orig_param}"
+                end
+              end
+            else
+              if i = param.index('[')
+                param = param[i + 1..-1]
+                in_bracket = true
+              else
+                raise Fluent::ConfigError, "found more characters after ']'. Invalid syntax: #{orig_param}"
+              end
+            end
+          end
+
+          raise Fluent::ConfigError, "empty keys in bracket notation" if result.empty?
+
+          result
+        end
+      end
+    end
+  end
+end