dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/plugin/owned_by_mixin.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Plugin
+    module OwnedByMixin
+      def owner=(plugin)
+        @_owner = plugin
+
+        @_plugin_id = plugin.plugin_id
+        @_plugin_id_configured = plugin.plugin_id_configured?
+
+        @log = plugin.log
+      end
+
+      def owner
+        if instance_variable_defined?("@_owner")
+          @_owner
+        end
+      end
+
+      def log
+        if instance_variable_defined?("@log")
+          @log
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser.rb

@@ -0,0 +1,274 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+require 'fluent/error'
+require 'fluent/mixin' # for TypeConverter
+require 'fluent/time'
+require 'fluent/plugin/string_util'
+
+require 'serverengine/blocking_flag'
+
+module Fluent
+  module Plugin
+    class Parser < Base
+      class TimeoutChecker
+        # This implementation now uses mutex because parser is typically used in input.
+        # If this has a performance issue under high concurreny, use concurrent-ruby's map instead.
+        def initialize(timeout)
+          @map = {}
+          @flag = ServerEngine::BlockingFlag.new
+          @mutex = Mutex.new
+          @timeout = timeout
+          @timeout_checker = nil
+        end
+
+        def start
+          @thread = ::Thread.new {
+            until @flag.wait_for_set(0.5)
+              now = Time.now
+              @mutex.synchronize {
+                @map.keys.each { |th|
+                  time = @map[th]
+                  if now - time > @timeout
+                    th.raise UncatchableError, "parsing timed out"
+                    @map.delete(th)
+                  end
+                }
+              }
+            end
+          }
+        end
+
+        def stop
+          @flag.set!
+          @thread.join
+        end
+
+        def execute
+          th = Thread.current
+          @mutex.synchronize { @map[th] = Time.now }
+          yield
+        ensure
+          # Need clean up here because if next event is delayed, incorrect exception will be raised in normal flow.
+          @mutex.synchronize { @map.delete(th) }
+        end
+      end
+
+      include OwnedByMixin
+      include TimeMixin::Parser
+
+      class ParserError < StandardError; end
+
+      configured_in :parse
+
+      ### types can be specified as string-based hash style
+      # field1:type, field2:type, field3:type:option, field4:type:option
+      ### or, JSON format
+      # {"field1":"type", "field2":"type", "field3":"type:option", "field4":"type:option"}
+      config_param :types, :hash, value_type: :string, default: nil
+
+      # available options are:
+      # array: (1st) delimiter
+      # time : type[, format, timezone] -> type should be a valid "time_type"(string/unixtime/float)
+      #      : format[, timezone]
+
+      config_param :time_key, :string, default: nil
+      config_param :null_value_pattern, :regexp, default: nil
+      config_param :null_empty_string, :bool, default: false
+      config_param :estimate_current_event, :bool, default: true
+      config_param :keep_time_key, :bool, default: false
+      config_param :timeout, :time, default: nil
+
+      AVAILABLE_PARSER_VALUE_TYPES = ['string', 'integer', 'float', 'bool', 'time', 'array']
+
+      # for tests
+      attr_reader :type_converters
+
+      PARSER_TYPES = [:text_per_line, :text, :binary]
+      def parser_type
+        :text_per_line
+      end
+
+      def initialize
+        super
+
+        @timeout_checker = nil
+      end
+
+      def configure(conf)
+        super
+
+        @time_parser = time_parser_create
+        @type_converters = build_type_converters(@types)
+        @execute_convert_values = @type_converters || @null_value_pattern || @null_empty_string
+        @timeout_checker = if @timeout
+                             class << self
+                               alias_method :parse_orig, :parse
+                               alias_method :parse, :parse_with_timeout
+                             end
+                             TimeoutChecker.new(@timeout)
+                           else
+                             nil
+                           end
+      end
+
+      def start
+        super
+
+        @timeout_checker.start if @timeout_checker
+      end
+
+      def stop
+        super
+
+        @timeout_checker.stop if @timeout_checker
+      end
+
+      def parse(text, &block)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def parse_with_timeout(text, &block)
+        @timeout_checker.execute {
+          parse_orig(text, &block)
+        }
+      rescue UncatchableError
+        log.warn "parsing timed out with #{self.class}: text = #{text}"
+        # Return nil instead of raising error. in_tail or other plugin can emit broken line.
+        yield nil, nil
+      end
+
+      def call(*a, &b)
+        # Keep backward compatibility for existing plugins
+        # TODO: warn when deprecated
+        parse(*a, &b)
+      end
+
+      def implement?(feature)
+        methods_of_plugin = self.class.instance_methods(false)
+        case feature
+        when :parse_io then methods_of_plugin.include?(:parse_io)
+        when :parse_partial_data then methods_of_plugin.include?(:parse_partial_data)
+        else
+          raise ArgumentError, "Unknown feature for parser plugin: #{feature}"
+        end
+      end
+
+      def parse_io(io, &block)
+        raise NotImplementedError, "Optional API #parse_io is not implemented"
+      end
+
+      def parse_partial_data(data, &block)
+        raise NotImplementedError, "Optional API #parse_partial_data is not implemented"
+      end
+
+      def parse_time(record)
+        if @time_key && record.respond_to?(:has_key?) && record.has_key?(@time_key)
+          src = if @keep_time_key
+                  record[@time_key]
+                else
+                  record.delete(@time_key)
+                end
+          @time_parser.parse(src)
+        elsif @estimate_current_event
+          Fluent::EventTime.now
+        else
+          nil
+        end
+      rescue Fluent::TimeParser::TimeParseError => e
+        raise ParserError, e.message
+      end
+
+      # def parse(text, &block)
+      #   time, record = convert_values(time, record)
+      #   yield time, record
+      # end
+      def convert_values(time, record)
+        return time, record unless @execute_convert_values
+
+        record.each_key do |key|
+          value = record[key]
+          next unless value # nil/null value is always left as-is.
+
+          if value.is_a?(String) && string_like_null(value)
+            record[key] = nil
+            next
+          end
+
+          if @type_converters && @type_converters.has_key?(key)
+            record[key] = @type_converters[key].call(value)
+          end
+        end
+
+        return time, record
+      end
+
+      def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_pattern)
+        null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match(s) }
+      end
+
+      TRUTHY_VALUES = ['true', 'yes', '1']
+
+      def build_type_converters(types)
+        return nil unless types
+
+        converters = {}
+
+        types.each_pair do |field_name, type_definition|
+          type, option = type_definition.split(":", 2)
+          unless AVAILABLE_PARSER_VALUE_TYPES.include?(type)
+            raise Fluent::ConfigError, "unknown value conversion for key:'#{field_name}', type:'#{type}'"
+          end
+
+          conv = case type
+                 when 'string' then ->(v){ v.to_s }
+                 when 'integer' then ->(v){ v.to_i rescue v.to_s.to_i }
+                 when 'float' then ->(v){ v.to_f rescue v.to_s.to_f }
+                 when 'bool' then ->(v){ TRUTHY_VALUES.include?(v.to_s.downcase) }
+                 when 'time'
+                   # comma-separated: time:[timezone:]time_format
+                   # time_format is unixtime/float/string-time-format
+                   timep = if option
+                             time_type = 'string' # estimate
+                             timezone, time_format = option.split(':', 2)
+                             unless Fluent::Timezone.validate(timezone)
+                               timezone, time_format = nil, option
+                             end
+                             if Fluent::TimeMixin::TIME_TYPES.include?(time_format)
+                               time_type, time_format = time_format, nil # unixtime/float
+                             end
+                             time_parser_create(type: time_type.to_sym, format: time_format, timezone: timezone)
+                           else
+                             time_parser_create(type: :string, format: nil, timezone: nil)
+                           end
+                   ->(v){ timep.parse(v) rescue nil }
+                 when 'array'
+                   delimiter = option ? option.to_s : ','
+                   ->(v){ string_safe_encoding(v.to_s){|s| s.split(delimiter) } }
+                 else
+                   raise "BUG: unknown type even after check: #{type}"
+                 end
+          converters[field_name] = conv
+        end
+
+        converters
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class ApacheParser < RegexpParser
+      Plugin.register_parser("apache", self)
+
+      config_set_default :expression, /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache2.rb

@@ -0,0 +1,88 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class Apache2Parser < Parser
+      Plugin.register_parser('apache2', self)
+
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\")*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\")*)" "(?<agent>(?:[^\"]|\\")*)")?$/
+      TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
+
+      def initialize
+        super
+        @mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+        @time_parser = time_parser_create(format: TIME_FORMAT)
+      end
+
+      def patterns
+        {'format' => REGEXP, 'time_format' => TIME_FORMAT}
+      end
+
+      def parse(text)
+        m = REGEXP.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        host = m['host']
+        host = (host == '-') ? nil : host
+
+        user = m['user']
+        user = (user == '-') ? nil : user
+
+        time = m['time']
+        time = @mutex.synchronize { @time_parser.parse(time) }
+
+        method = m['method']
+        path = m['path']
+
+        code = m['code'].to_i
+        code = nil if code == 0
+
+        size = m['size']
+        size = (size == '-') ? nil : size.to_i
+
+        referer = m['referer']
+        referer = (referer == '-') ? nil : referer
+
+        agent = m['agent']
+        agent = (agent == '-') ? nil : agent
+
+        record = {
+          "host" => host,
+          "user" => user,
+          "method" => method,
+          "path" => path,
+          "code" => code,
+          "size" => size,
+          "referer" => referer,
+          "agent" => agent,
+        }
+        record["time"] = m['time'] if @keep_time_key
+
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache_error.rb

@@ -0,0 +1,26 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class ApacheErrorParser < RegexpParser
+      Plugin.register_parser("apache_error", self)
+      config_set_default :expression, /^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/
+    end
+  end
+end

data/lib/fluent/plugin/parser_csv.rb

@@ -0,0 +1,114 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+require 'csv'
+
+module Fluent
+  module Plugin
+    class CSVParser < Parser
+      Plugin.register_parser('csv', self)
+
+      desc 'Names of fields included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of CSV values'
+      config_param :delimiter, :string, default: ','
+      desc 'The parser type used to parse CSV line'
+      config_param :parser_engine, :enum, list: [:normal, :fast], default: :normal, alias: :parser_type
+
+      def configure(conf)
+        super
+
+
+        if @parser_engine == :fast
+          @quote_char = '"'
+          @escape_pattern = Regexp.compile(@quote_char * 2)
+
+          m = method(:parse_fast)
+          self.singleton_class.module_eval do
+            define_method(:parse, m)
+          end
+        end
+      end
+
+      def parse(text, &block)
+        values = CSV.parse_line(text, col_sep: @delimiter)
+        r = Hash[@keys.zip(values)]
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+
+      def parse_fast(text, &block)
+        r = parse_fast_internal(text)
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+
+      # CSV.parse_line is too slow due to initialize lots of object and
+      # CSV module doesn't provide the efficient method for parsing single line.
+      # This method avoids the overhead of CSV.parse_line for typical patterns
+      def parse_fast_internal(text)
+        record = {}
+        text.chomp!
+
+        return record if text.empty?
+
+        # use while because while is now faster than each_with_index
+        columns = text.split(@delimiter, -1)
+        num_columns = columns.size
+        i = 0
+        j = 0
+        while j < num_columns
+          column = columns[j]
+
+          case column.count(@quote_char)
+          when 0
+            if column.empty?
+              column = nil
+            end
+          when 1
+            if column.start_with?(@quote_char)
+              to_merge = [column]
+              j += 1
+              while j < num_columns
+                merged_col = columns[j]
+                to_merge << merged_col
+                break if merged_col.end_with?(@quote_char)
+                j += 1
+              end
+              column = to_merge.join(@delimiter)[1..-2]
+            end
+          when 2
+            if column.start_with?(@quote_char) && column.end_with?(@quote_char)
+              column = column[1..-2]
+            end
+          else
+            if column.start_with?(@quote_char) && column.end_with?(@quote_char)
+              column = column[1..-2]
+            end
+            column.gsub!(@escape_pattern, @quote_char)
+          end
+
+          record[@keys[i]] = column
+          j += 1
+          i += 1
+        end
+        record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_json.rb

@@ -0,0 +1,96 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/time'
+require 'fluent/oj_options'
+
+require 'yajl'
+require 'json'
+
+module Fluent
+  module Plugin
+    class JSONParser < Parser
+      Plugin.register_parser('json', self)
+
+      config_set_default :time_key, 'time'
+      desc 'Set JSON parser'
+      config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
+
+      # The Yajl library defines a default buffer size of 8KiB when parsing
+      # from IO streams, so maintain this for backwards-compatibility.
+      # https://www.rubydoc.info/github/brianmario/yajl-ruby/Yajl%2FParser:parse
+      desc 'Set the buffer size that Yajl will use when parsing streaming input'
+      config_param :stream_buffer_size, :integer, default: 8192
+
+      config_set_default :time_type, :float
+
+      def configure(conf)
+        if conf.has_key?('time_format')
+          conf['time_type'] ||= 'string'
+        end
+
+        super
+        @load_proc, @error_class = configure_json_parser(@json_parser)
+      end
+
+      def configure_json_parser(name)
+        case name
+        when :oj
+          raise LoadError unless Fluent::OjOptions.available?
+          [Oj.method(:load), Oj::ParseError]
+        when :json then [JSON.method(:load), JSON::ParserError]
+        when :yajl then [Yajl.method(:load), Yajl::ParseError]
+        else
+          raise "BUG: unknown json parser specified: #{name}"
+        end
+      rescue LoadError => ex
+        name = :yajl
+        if log
+          if /\boj\z/ =~ ex.message
+            log.info "Oj is not installed, and failing back to Yajl for json parser"
+          else
+            log.warn ex.message
+          end
+        end
+        retry
+      end
+
+      def parse(text)
+        record = @load_proc.call(text)
+        time = parse_time(record)
+        if @execute_convert_values
+          time, record = convert_values(time, record)
+        end
+        yield time, record
+      rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
+        yield nil, nil
+      end
+
+      def parser_type
+        :text
+      end
+
+      def parse_io(io, &block)
+        y = Yajl::Parser.new
+        y.on_parse_complete = ->(record){
+          block.call(parse_time(record), record)
+        }
+        y.parse(io, @stream_buffer_size)
+      end
+    end
+  end
+end