fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/filter_parser.rb

@@ -1,107 +1,130 @@
-require 'time'
-require 'fluent/parser'
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
 
-class Fluent::ParserFilter < Fluent::Filter
-  Fluent::Plugin.register_filter('parser', self)
+require 'fluent/time'
+require 'fluent/config/error'
+require 'fluent/plugin/filter'
+require 'fluent/plugin_helper/parser'
+require 'fluent/plugin_helper/compat_parameters'
 
-  config_param :key_name, :string
-  config_param :reserve_data, :bool, default: false
-  config_param :inject_key_prefix, :string, default: nil
-  config_param :replace_invalid_sequence, :bool, default: false
-  config_param :hash_value_field, :string, default: nil
-  config_param :suppress_parse_error_log, :bool, default: false
-  config_param :time_parse, :bool, default: true
-  config_param :ignore_key_not_exist, :bool, default: false
-  config_param :emit_invalid_record_to_error, :bool, default: false
+module Fluent::Plugin
+  class ParserFilter < Filter
+    Fluent::Plugin.register_filter('parser', self)
 
-  attr_reader :parser
+    helpers :parser, :record_accessor, :compat_parameters
 
-  def configure(conf)
-    super
+    desc 'Specify field name in the record to parse.'
+    config_param :key_name, :string
+    desc 'Keep original key-value pair in parsed result.'
+    config_param :reserve_data, :bool, default: false
+    desc 'Keep original event time in parsed result.'
+    config_param :reserve_time, :bool, default: false
+    desc 'Remove "key_name" field from the record when parsing is succeeded'
+    config_param :remove_key_name_field, :bool, default: false
+    desc 'Store parsed values with specified key name prefix.'
+    config_param :inject_key_prefix, :string, default: nil
+    desc 'If true, invalid string is replaced with safe characters and re-parse it.'
+    config_param :replace_invalid_sequence, :bool, default: false
+    desc 'Store parsed values as a hash value in a field.'
+    config_param :hash_value_field, :string, default: nil
+    desc 'Emit invalid record to @ERROR label'
+    config_param :emit_invalid_record_to_error, :bool, default: true
 
-    @parser = Fluent::TextParser.new
-    @parser.estimate_current_event = false
-    @parser.configure(conf)
-    if !@time_parse && @parser.parser.respond_to?("time_key=".to_sym)
-      # disable parse time
-      @parser.parser.time_key = nil
+    attr_reader :parser
+
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+
+      super
+
+      @accessor = record_accessor_create(@key_name)
+      @parser = parser_create
     end
 
-    self
-  end
+    FAILED_RESULT = [nil, nil].freeze # reduce allocation cost
+    REPLACE_CHAR = '?'.freeze
 
-  def filter_stream(tag, es)
-    new_es = Fluent::MultiEventStream.new
-    es.each do |time,record|
-      raw_value = record[@key_name]
+    def filter_with_time(tag, time, record)
+      raw_value = @accessor.call(record)
       if raw_value.nil?
         if @emit_invalid_record_to_error
           router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
+        end
+        if @reserve_data
+          return time, handle_parsed(tag, record, time, {})
         else
-          log.warn "#{@key_name} does not exist" unless @ignore_key_not_exist
+          return FAILED_RESULT
         end
-        new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data
-        next
       end
       begin
-        @parser.parse(raw_value) do |t,values|
+        @parser.parse(raw_value) do |t, values|
           if values
-            t ||= time
+            t = if @reserve_time
+                  time
+                else
+                  t.nil? ? time : t
+                end
+            @accessor.delete(record) if @remove_key_name_field
             r = handle_parsed(tag, record, t, values)
-            new_es.add(t, r)
+            return t, r
           else
             if @emit_invalid_record_to_error
-              router.emit_error_event(tag, time, record, ::Fluent::ParserError.new("pattern not match with data '#{raw_value}'"))
-            else
-              log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+              router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
             end
             if @reserve_data
               t = time
               r = handle_parsed(tag, record, time, {})
-              new_es.add(t, r)
+              return t, r
+            else
+              return FAILED_RESULT
             end
           end
         end
-      rescue Fluent::ParserError => e
+      rescue Fluent::Plugin::Parser::ParserError => e
         if @emit_invalid_record_to_error
-          router.emit_error_event(tag, time, record, e)
+          raise e
         else
-          log.warn e.message unless @suppress_parse_error_log
+          return FAILED_RESULT
         end
       rescue ArgumentError => e
         raise unless @replace_invalid_sequence
         raise unless e.message.index("invalid byte sequence in") == 0
 
-        raw_value = replace_invalid_byte(raw_value)
+        raw_value = raw_value.scrub(REPLACE_CHAR)
         retry
       rescue => e
         if @emit_invalid_record_to_error
-          router.emit_error_event(tag, time, record, Fluent::ParserError.new("parse failed #{e.message}"))
+          raise Fluent::Plugin::Parser::ParserError, "parse failed #{e.message}"
         else
-          log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
+          return FAILED_RESULT
         end
       end
     end
-    new_es
-  end
 
-  private
+    private
 
-  def handle_parsed(tag, record, t, values)
-    if values && @inject_key_prefix
-      values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
-    end
-    r = @hash_value_field ? {@hash_value_field => values} : values
-    if @reserve_data
-      r = r ? record.merge(r) : record
+    def handle_parsed(tag, record, t, values)
+      if values && @inject_key_prefix
+        values = Hash[values.map { |k, v| [@inject_key_prefix + k, v] }]
+      end
+      r = @hash_value_field ? {@hash_value_field => values} : values
+      if @reserve_data
+        r = r ? record.merge(r) : record
+      end
+      r
     end
-    r
-  end
-
-  def replace_invalid_byte(string)
-    replace_options = { invalid: :replace, undef: :replace, replace: '?' }
-    original_encoding = string.encoding
-    temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
-    string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
   end
 end

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -18,15 +18,17 @@ require 'socket'
 require 'json'
 require 'ostruct'
 
-require 'fluent/filter'
+require 'fluent/plugin/filter'
 require 'fluent/config/error'
 require 'fluent/event'
 require 'fluent/time'
 
-module Fluent
-  class RecordTransformerFilter < Filter
+module Fluent::Plugin
+  class RecordTransformerFilter < Fluent::Plugin::Filter
     Fluent::Plugin.register_filter('record_transformer', self)
 
+    helpers :record_accessor
+
     desc 'A comma-delimited list of keys to delete.'
     config_param :remove_keys, :array, default: nil
     desc 'A comma-delimited list of keys to keep.'
@@ -38,7 +40,7 @@ module Fluent
     desc 'When set to true, the full Ruby syntax is enabled in the ${...} expression.'
     config_param :enable_ruby, :bool, default: false
     desc 'Use original value type.'
-    config_param :auto_typecast, :bool, default: false # false for lower version compatibility
+    config_param :auto_typecast, :bool, default: true
 
     def configure(conf)
       super
@@ -56,6 +58,10 @@ module Fluent
         raise Fluent::ConfigError, "`renew_record` must be true to use `keep_keys`" unless @renew_record
       end
 
+      @key_deleters = if @remove_keys
+                        @remove_keys.map { |k| record_accessor_create(k) }
+                      end
+
       placeholder_expander_params = {
         log: log,
         auto_typecast: @auto_typecast,
@@ -76,7 +82,7 @@ module Fluent
     end
 
     def filter_stream(tag, es)
-      new_es = MultiEventStream.new
+      new_es = Fluent::MultiEventStream.new
       tag_parts = tag.split('.')
       tag_prefix = tag_prefix(tag_parts)
       tag_suffix = tag_suffix(tag_parts)
@@ -87,23 +93,23 @@ module Fluent
         'tag_suffix' => tag_suffix,
         'hostname'   => @hostname,
       }
-      last_record = nil
       es.each do |time, record|
-        last_record = record # for debug log
-        placeholder_values['time'] = @placeholder_expander.time_value(time)
-        placeholder_values['record'] = record
-        new_record = reform(record, placeholder_values)
-        if @renew_time_key && new_record.has_key?(@renew_time_key)
-          time = new_record[@renew_time_key].to_i
+        begin
+          placeholder_values['time'] = @placeholder_expander.time_value(time)
+          placeholder_values['record'] = record
+
+          new_record = reform(record, placeholder_values)
+          if @renew_time_key && new_record.has_key?(@renew_time_key)
+            time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
+          end
+          @key_deleters.each { |deleter| deleter.delete(new_record) } if @key_deleters
+          new_es.add(time, new_record)
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+          log.debug { "map:#{@map} record:#{record} placeholder_values:#{placeholder_values}" }
         end
-        @remove_keys.each { |k| new_record.delete(k) } if @remove_keys
-        new_es.add(time, new_record)
       end
       new_es
-    rescue => e
-      log.warn "failed to reform records", error_class: e.class, error: e.message
-      log.warn_backtrace
-      log.debug "map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
     end
 
     private
@@ -115,7 +121,7 @@ module Fluent
         value_str
       end
     rescue => e
-      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", error_class: e.class, error: e.message
+      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", error: e
       value_str # emit as string
     end
 
@@ -199,9 +205,6 @@ module Fluent
             end
           elsif value.kind_of?(Hash) # record, etc
             value.each do |k, v|
-              unless placeholder_values.has_key?(k) # prevent overwriting reserved keys such as tag
-                placeholders.store("${#{k}}", v) # foo
-              end
               placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
             end
           else # string, interger, float, and others?
@@ -304,31 +307,16 @@ module Fluent
           placeholders['hostname'],
         )
       rescue => e
-        log.warn "failed to expand `#{str}`", error_class: e.class, error: e.message
-        log.warn_backtrace
-        nil
+        raise "failed to expand `#{str}` : error = #{e}"
       end
 
       class CleanroomExpander
         def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
-          tags = tag_parts # for old version compatibility
-          Thread.current[:record_transformer_record] = record # for old version compatibility
           instance_eval(__str_to_eval__)
         end
 
-        # for old version compatibility
-        def method_missing(name)
-          key = name.to_s
-          record = Thread.current[:record_transformer_record]
-          if record.has_key?(key)
-            record[key]
-          else
-            raise NameError, "undefined local variable or method `#{key}'"
-          end
-        end
-
         (Object.instance_methods).each do |m|
-          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
+          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/
         end
       end
     end

data/lib/fluent/plugin/filter_stdout.rb

@@ -14,31 +14,34 @@
 #    limitations under the License.
 #
 
-require 'fluent/filter'
-require 'fluent/plugin'
+require 'fluent/plugin/filter'
 
-module Fluent
+module Fluent::Plugin
   class StdoutFilter < Filter
-    Plugin.register_filter('stdout', self)
+    Fluent::Plugin.register_filter('stdout', self)
+
+    helpers :formatter, :compat_parameters, :inject
+
+    DEFAULT_FORMAT_TYPE = 'stdout'
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
 
     # for tests
     attr_reader :formatter
 
-    desc 'The format of the output.'
-    config_param :format, :string, default: 'stdout'
-    # config_param :output_type, :string, :default => 'json' (StdoutFormatter defines this)
-
     def configure(conf)
+      compat_parameters_convert(conf, :inject, :formatter)
       super
-
-      @formatter = Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+      @formatter = formatter_create
     end
 
     def filter_stream(tag, es)
       es.each { |time, record|
         begin
-          log.write @formatter.format(tag, time, record)
+          r = inject_values_to_record(tag, time, record)
+          log.write @formatter.format(tag, time, r)
         rescue => e
           router.emit_error_event(tag, time, record, e)
         end

data/lib/fluent/plugin/formatter.rb

@@ -0,0 +1,50 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+require 'fluent/time'
+
+module Fluent
+  module Plugin
+    class Formatter < Base
+      include OwnedByMixin
+      include TimeMixin::Formatter
+
+      configured_in :format
+
+      PARSER_TYPES = [:text_per_line, :text, :binary]
+      def formatter_type
+        :text_per_line
+      end
+
+      def format(tag, time, record)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+    end
+
+    class ProcWrappedFormatter < Formatter
+      def initialize(proc)
+        super()
+        @proc = proc
+      end
+
+      def format(tag, time, record)
+        @proc.call(tag, time, record)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_csv.rb

@@ -0,0 +1,52 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+require 'csv'
+
+module Fluent
+  module Plugin
+    class CsvFormatter < Formatter
+      Plugin.register_formatter('csv', self)
+
+      config_param :delimiter, default: ',' do |val|
+        ['\t', 'TAB'].include?(val) ? "\t" : val
+      end
+      config_param :force_quotes, :bool, default: true
+      # "array" looks good for type of :fields, but this implementation removes tailing comma
+      # TODO: Is it needed to support tailing comma?
+      config_param :fields, :array, value_type: :string
+      config_param :add_newline, :bool, default: true
+
+      def configure(conf)
+        super
+        @fields = fields.select{|f| !f.empty? }
+        raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
+
+        @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes}
+      end
+
+      def format(tag, time, record)
+        row = @fields.map do |key|
+          record[key]
+        end
+        line = CSV.generate_line(row, @generate_opts)
+        line.chomp! unless @add_newline
+        line
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_hash.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class HashFormatter < Formatter
+      Plugin.register_formatter('hash', self)
+
+      config_param :add_newline, :bool, default: true
+
+      def format(tag, time, record)
+        line = record.to_s
+        line << "\n".freeze if @add_newline
+        line
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_json.rb

@@ -0,0 +1,55 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+require 'fluent/env'
+
+module Fluent
+  module Plugin
+    class JSONFormatter < Formatter
+      Plugin.register_formatter('json', self)
+
+      config_param :json_parser, :string, default: 'oj'
+      config_param :add_newline, :bool, default: true
+
+      def configure(conf)
+        super
+
+        begin
+          raise LoadError unless @json_parser == 'oj'
+          require 'oj'
+          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+          @dump_proc = Oj.method(:dump)
+        rescue LoadError
+          @dump_proc = Yajl.method(:dump)
+        end
+
+        # format json is used on various highload environment, so re-define method to skip if check
+        unless @add_newline
+          define_singleton_method(:format, method(:format_without_nl))
+        end
+      end
+
+      def format(tag, time, record)
+        "#{@dump_proc.call(record)}\n"
+      end
+
+      def format_without_nl(tag, time, record)
+        @dump_proc.call(record)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class LabeledTSVFormatter < Formatter
+      Plugin.register_formatter('ltsv', self)
+
+      # http://ltsv.org/
+
+      config_param :delimiter, :string, default: "\t"
+      config_param :label_delimiter, :string, default: ":"
+      config_param :add_newline, :bool, default: true
+
+      # TODO: escaping for \t in values
+      def format(tag, time, record)
+        formatted = ""
+        record.each do |label, value|
+          formatted << @delimiter if formatted.length.nonzero?
+          formatted << "#{label}#{@label_delimiter}#{value}"
+        end
+        formatted << "\n".freeze if @add_newline
+        formatted
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_msgpack.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class MessagePackFormatter < Formatter
+      Plugin.register_formatter('msgpack', self)
+
+      def formatter_type
+        :binary
+      end
+
+      def format(tag, time, record)
+        record.to_msgpack
+      end
+    end
+  end
+end