fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/owned_by_mixin.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Plugin
+    module OwnedByMixin
+      def owner=(plugin)
+        @_owner = plugin
+
+        @_plugin_id = plugin.plugin_id
+        @_plugin_id_configured = plugin.plugin_id_configured?
+
+        @log = plugin.log
+      end
+
+      def owner
+        if instance_variable_defined?("@_owner")
+          @_owner
+        end
+      end
+
+      def log
+        if instance_variable_defined?("@log")
+          @log
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser.rb

@@ -0,0 +1,191 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+require 'fluent/mixin' # for TypeConverter
+require 'fluent/time'
+require 'fluent/plugin/string_util'
+
+module Fluent
+  module Plugin
+    class Parser < Base
+      include OwnedByMixin
+      include TimeMixin::Parser
+
+      class ParserError < StandardError; end
+
+      configured_in :parse
+
+      ### types can be specified as string-based hash style
+      # field1:type, field2:type, field3:type:option, field4:type:option
+      ### or, JSON format
+      # {"field1":"type", "field2":"type", "field3":"type:option", "field4":"type:option"}
+      config_param :types, :hash, value_type: :string, default: nil
+
+      # available options are:
+      # array: (1st) delimiter
+      # time : type[, format, timezone] -> type should be a valid "time_type"(string/unixtime/float)
+      #      : format[, timezone]
+
+      config_param :time_key, :string, default: nil
+      config_param :null_value_pattern, :string, default: nil
+      config_param :null_empty_string, :bool, default: false
+      config_param :estimate_current_event, :bool, default: true
+      config_param :keep_time_key, :bool, default: false
+
+      AVAILABLE_PARSER_VALUE_TYPES = ['string', 'integer', 'float', 'bool', 'time', 'array']
+
+      # for tests
+      attr_reader :type_converters
+
+      PARSER_TYPES = [:text_per_line, :text, :binary]
+      def parser_type
+        :text_per_line
+      end
+
+      def configure(conf)
+        super
+
+        @time_parser = time_parser_create
+        @null_value_regexp = @null_value_pattern && Regexp.new(@null_value_pattern)
+        @type_converters = build_type_converters(@types)
+        @execute_convert_values = @type_converters || @null_value_regexp || @null_empty_string
+      end
+
+      def parse(text, &block)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def call(*a, &b)
+        # Keep backward compatibility for existing plugins
+        # TODO: warn when deprecated
+        parse(*a, &b)
+      end
+
+      def implement?(feature)
+        methods_of_plugin = self.class.instance_methods(false)
+        case feature
+        when :parse_io then methods_of_plugin.include?(:parse_io)
+        when :parse_partial_data then methods_of_plugin.include?(:parse_partial_data)
+        else
+          raise ArgumentError, "Unknown feature for parser plugin: #{feature}"
+        end
+      end
+
+      def parse_io(io, &block)
+        raise NotImplementedError, "Optional API #parse_io is not implemented"
+      end
+
+      def parse_partial_data(data, &block)
+        raise NotImplementedError, "Optional API #parse_partial_data is not implemented"
+      end
+
+      def parse_time(record)
+        if @time_key && record.respond_to?(:has_key?) && record.has_key?(@time_key)
+          src = if @keep_time_key
+                  record[@time_key]
+                else
+                  record.delete(@time_key)
+                end
+          @time_parser.parse(src)
+        elsif @estimate_current_event
+          Fluent::EventTime.now
+        else
+          nil
+        end
+      rescue Fluent::TimeParser::TimeParseError => e
+        raise ParserError, e.message
+      end
+
+      # def parse(text, &block)
+      #   time, record = convert_values(time, record)
+      #   yield time, record
+      # end
+      def convert_values(time, record)
+        return time, record unless @execute_convert_values
+
+        record.each_key do |key|
+          value = record[key]
+          next unless value # nil/null value is always left as-is.
+
+          if value.is_a?(String) && string_like_null(value)
+            record[key] = nil
+            next
+          end
+
+          if @type_converters && @type_converters.has_key?(key)
+            record[key] = @type_converters[key].call(value)
+          end
+        end
+
+        return time, record
+      end
+
+      def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_regexp)
+        null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match(s) }
+      end
+
+      TRUTHY_VALUES = ['true', 'yes', '1']
+
+      def build_type_converters(types)
+        return nil unless types
+
+        converters = {}
+
+        types.each_pair do |field_name, type_definition|
+          type, option = type_definition.split(":", 2)
+          unless AVAILABLE_PARSER_VALUE_TYPES.include?(type)
+            raise Fluent::ConfigError, "unknown value conversion for key:'#{field_name}', type:'#{type}'"
+          end
+
+          conv = case type
+                 when 'string' then ->(v){ v.to_s }
+                 when 'integer' then ->(v){ v.to_i rescue v.to_s.to_i }
+                 when 'float' then ->(v){ v.to_f rescue v.to_s.to_f }
+                 when 'bool' then ->(v){ TRUTHY_VALUES.include?(v.to_s.downcase) }
+                 when 'time'
+                   # comma-separated: time:[timezone:]time_format
+                   # time_format is unixtime/float/string-time-format
+                   timep = if option
+                             time_type = 'string' # estimate
+                             timezone, time_format = option.split(':', 2)
+                             unless Fluent::Timezone.validate(timezone)
+                               timezone, time_format = nil, option
+                             end
+                             if Fluent::TimeMixin::TIME_TYPES.include?(time_format)
+                               time_type, time_format = time_format, nil # unixtime/float
+                             end
+                             time_parser_create(type: time_type.to_sym, format: time_format, timezone: timezone)
+                           else
+                             time_parser_create(type: :string, format: nil, timezone: nil)
+                           end
+                   ->(v){ timep.parse(v) rescue nil }
+                 when 'array'
+                   delimiter = option ? option.to_s : ','
+                   ->(v){ string_safe_encoding(v.to_s){|s| s.split(delimiter) } }
+                 else
+                   raise "BUG: unknown type even after check: #{type}"
+                 end
+          converters[field_name] = conv
+        end
+
+        converters
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class ApacheParser < RegexpParser
+      Plugin.register_parser("apache", self)
+
+      config_set_default :expression, /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache2.rb

@@ -0,0 +1,88 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class Apache2Parser < Parser
+      Plugin.register_parser('apache2', self)
+
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
+      TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
+
+      def initialize
+        super
+        @mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+        @time_parser = time_parser_create(format: TIME_FORMAT)
+      end
+
+      def patterns
+        {'format' => REGEXP, 'time_format' => TIME_FORMAT}
+      end
+
+      def parse(text)
+        m = REGEXP.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        host = m['host']
+        host = (host == '-') ? nil : host
+
+        user = m['user']
+        user = (user == '-') ? nil : user
+
+        time = m['time']
+        time = @mutex.synchronize { @time_parser.parse(time) }
+
+        method = m['method']
+        path = m['path']
+
+        code = m['code'].to_i
+        code = nil if code == 0
+
+        size = m['size']
+        size = (size == '-') ? nil : size.to_i
+
+        referer = m['referer']
+        referer = (referer == '-') ? nil : referer
+
+        agent = m['agent']
+        agent = (agent == '-') ? nil : agent
+
+        record = {
+          "host" => host,
+          "user" => user,
+          "method" => method,
+          "path" => path,
+          "code" => code,
+          "size" => size,
+          "referer" => referer,
+          "agent" => agent,
+        }
+        record["time"] = m['time'] if @keep_time_key
+
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache_error.rb

@@ -0,0 +1,26 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class ApacheErrorParser < RegexpParser
+      Plugin.register_parser("apache_error", self)
+      config_set_default :expression, /^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/
+    end
+  end
+end

data/lib/fluent/plugin/parser_csv.rb

@@ -0,0 +1,39 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+require 'csv'
+
+module Fluent
+  module Plugin
+    class CSVParser < Parser
+      Plugin.register_parser('csv', self)
+
+      desc 'Names of fields included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of CSV values'
+      config_param :delimiter, :string, default: ','
+
+      def parse(text, &block)
+        values = CSV.parse_line(text, col_sep: @delimiter)
+        r = Hash[@keys.zip(values)]
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_json.rb

@@ -0,0 +1,94 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/env'
+require 'fluent/time'
+
+require 'yajl'
+require 'json'
+
+module Fluent
+  module Plugin
+    class JSONParser < Parser
+      Plugin.register_parser('json', self)
+
+      config_set_default :time_key, 'time'
+      desc 'Set JSON parser'
+      config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
+
+      # The Yajl library defines a default buffer size of 8KiB when parsing
+      # from IO streams, so maintain this for backwards-compatibility.
+      # https://www.rubydoc.info/github/brianmario/yajl-ruby/Yajl%2FParser:parse
+      desc 'Set the buffer size that Yajl will use when parsing streaming input'
+      config_param :stream_buffer_size, :integer, default: 8192
+
+      config_set_default :time_type, :float
+
+      def configure(conf)
+        if conf.has_key?('time_format')
+          conf['time_type'] ||= 'string'
+        end
+
+        super
+        @load_proc, @error_class = configure_json_parser(@json_parser)
+      end
+
+      def configure_json_parser(name)
+        case name
+        when :oj
+          require 'oj'
+          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+          [Oj.method(:load), Oj::ParseError]
+        when :json then [JSON.method(:load), JSON::ParserError]
+        when :yajl then [Yajl.method(:load), Yajl::ParseError]
+        else
+          raise "BUG: unknown json parser specified: #{name}"
+        end
+      rescue LoadError => ex
+        name = :yajl
+        if log
+          if /\boj\z/ =~ ex.message
+            log.info "Oj is not installed, and failing back to Yajl for json parser"
+          else
+            log.warn ex.message
+          end
+        end
+        retry
+      end
+
+      def parse(text)
+        r = @load_proc.call(text)
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
+        yield nil, nil
+      end
+
+      def parser_type
+        :text
+      end
+
+      def parse_io(io, &block)
+        y = Yajl::Parser.new
+        y.on_parse_complete = ->(record){
+          block.call(parse_time(record), record)
+        }
+        y.parse(io, @stream_buffer_size)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_ltsv.rb

@@ -0,0 +1,49 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class LabeledTSVParser < Parser
+      Plugin.register_parser('ltsv', self)
+
+      desc 'The delimiter character (or string) of TSV values'
+      config_param :delimiter, :string, default: "\t"
+      desc 'The delimiter pattern of TSV values'
+      config_param :delimiter_pattern, :regexp, default: nil
+      desc 'The delimiter character between field name and value'
+      config_param :label_delimiter, :string, default: ":"
+
+      config_set_default :time_key, 'time'
+
+      def configure(conf)
+        super
+        @delimiter = @delimiter_pattern || @delimiter
+      end
+
+      def parse(text)
+        r = {}
+        text.split(@delimiter).each do |pair|
+          key, value = pair.split(@label_delimiter, 2)
+          r[key] = value
+        end
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_msgpack.rb

@@ -0,0 +1,50 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/msgpack_factory'
+
+module Fluent
+  module Plugin
+    class MessagePackParser < Parser
+      Plugin.register_parser('msgpack', self)
+
+      def configure(conf)
+        super
+        @unpacker = Fluent::MessagePackFactory.engine_factory.unpacker
+      end
+
+      def parser_type
+        :binary
+      end
+
+      def parse(data)
+        @unpacker.feed_each(data) do |obj|
+          yield convert_values(parse_time(obj), obj)
+        end
+      end
+      alias parse_partial_data parse
+
+      def parse_io(io, &block)
+        u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
+        u.each do |obj|
+          time, record = convert_values(parse_time(obj), obj)
+          yield time, record
+        end
+      end
+    end
+  end
+end