fluentd 0.14.8 → 0.14.9

data/lib/fluent/config/section.rb

@@ -166,6 +166,13 @@ module Fluent
           varname = subproxy.variable_name
           elements = (conf.respond_to?(:elements) ? conf.elements : []).select{ |e| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
           if elements.empty? && subproxy.init?
+            if subproxy.argument && !subproxy.defaults.has_key?(subproxy.argument.first)
+              raise ArgumentError, "#{name}: init is specified, but default value of argument is missing"
+            end
+            missing_keys = subproxy.params.keys.select{|param_name| !subproxy.defaults.has_key?(param_name)}
+            if !missing_keys.empty?
+              raise ArgumentError, "#{name}: init is specified, but there're parameters without default values:#{missing_keys.join(',')}"
+            end
             elements << Fluent::Config::Element.new(subproxy.name.to_s, '', {}, [])
           end
 

data/lib/fluent/configurable.rb

@@ -71,9 +71,7 @@ module Fluent
       root.instance_eval{ @params.keys }.each do |param_name|
         next if param_name.to_s.start_with?('@')
         varname = "@#{param_name}".to_sym
-        if (! root[param_name].nil?) || (instance_variable_defined?(varname) && instance_variable_get(varname).nil?)
-          instance_variable_set(varname, root[param_name])
-        end
+        instance_variable_set(varname, root[param_name])
       end
 
       self

data/lib/fluent/log.rb

@@ -446,7 +446,7 @@ module Fluent
 
   # This class delegetes some methods which are used in `Fluent::Logger` to a instance variable(`dev`) in `Logger::LogDevice` class
   # https://github.com/ruby/ruby/blob/7b2d47132ff8ee950b0f978ab772dee868d9f1b0/lib/logger.rb#L661
-  class LogDeviceIO < Logger::LogDevice
+  class LogDeviceIO < ::Logger::LogDevice
     def flush
       if @dev.respond_to?(:flush)
         @dev.flush

data/lib/fluent/plugin/base.rb

@@ -31,6 +31,7 @@ module Fluent
       def initialize
         super
         @_state = State.new(false, false, false, false, false, false, false, false, false)
+        @_context_router = nil
         @under_plugin_development = false
       end
 
@@ -45,6 +46,22 @@ module Fluent
         self
       end
 
+      def string_safe_encoding(str)
+        unless str.valid_encoding?
+          log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
+          str = str.scrub('?')
+        end
+        yield str
+      end
+
+      def context_router=(router)
+        @_context_router = router
+      end
+
+      def context_router
+        @_context_router
+      end
+
       def start
         @_state.start = true
         self

data/lib/fluent/plugin/filter_parser.rb

@@ -0,0 +1,108 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/time'
+require 'fluent/config/error'
+require 'fluent/plugin/filter'
+require 'fluent/plugin_helper/parser'
+require 'fluent/plugin_helper/compat_parameters'
+
+module Fluent::Plugin
+  class ParserFilter < Filter
+    Fluent::Plugin.register_filter('parser', self)
+
+    helpers :parser, :compat_parameters
+
+    config_param :key_name, :string
+    config_param :reserve_data, :bool, default: false
+    config_param :reserve_time, :bool, default: false
+    config_param :inject_key_prefix, :string, default: nil
+    config_param :replace_invalid_sequence, :bool, default: false
+    config_param :hash_value_field, :string, default: nil
+
+    attr_reader :parser
+
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+
+      super
+
+      @parser = parser_create
+    end
+
+    FAILED_RESULT = [nil, nil].freeze # reduce allocation cost
+    REPLACE_CHAR = '?'.freeze
+
+    def filter_with_time(tag, time, record)
+      raw_value = record[@key_name]
+      if raw_value.nil?
+        router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
+        if @reserve_data
+          return time, handle_parsed(tag, record, time, {})
+        else
+          return FAILED_RESULT
+        end
+      end
+      begin
+        @parser.parse(raw_value) do |t, values|
+          if values
+            t = if @reserve_time
+                  time
+                else
+                  t.nil? ? time : t
+                end
+            r = handle_parsed(tag, record, t, values)
+            return t, r
+          else
+            router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not match with data '#{raw_value}'"))
+            if @reserve_data
+              t = time
+              r = handle_parsed(tag, record, time, {})
+              return t, r
+            else
+              return FAILED_RESULT
+            end
+          end
+        end
+      rescue Fluent::Plugin::Parser::ParserError => e
+        router.emit_error_event(tag, time, record, e)
+        return FAILED_RESULT
+      rescue ArgumentError => e
+        raise unless @replace_invalid_sequence
+        raise unless e.message.index("invalid byte sequence in") == 0
+
+        raw_value = raw_value.scrub(REPLACE_CHAR)
+        retry
+      rescue => e
+        router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("parse failed #{e.message}"))
+        return FAILED_RESULT
+      end
+    end
+
+    private
+
+    def handle_parsed(tag, record, t, values)
+      if values && @inject_key_prefix
+        values = Hash[values.map { |k, v| [@inject_key_prefix + k, v] }]
+      end
+      r = @hash_value_field ? {@hash_value_field => values} : values
+      if @reserve_data
+        r = r ? record.merge(r) : record
+      end
+      r
+    end
+  end
+end

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -38,7 +38,7 @@ module Fluent::Plugin
     desc 'When set to true, the full Ruby syntax is enabled in the ${...} expression.'
     config_param :enable_ruby, :bool, default: false
     desc 'Use original value type.'
-    config_param :auto_typecast, :bool, default: false # false for lower version compatibility
+    config_param :auto_typecast, :bool, default: true
 
     def configure(conf)
       super
@@ -95,10 +95,9 @@ module Fluent::Plugin
       last_record = nil
       es.each do |time, record|
         last_record = record # for debug log
-        placeholder_values.merge!({
-          'time'     => @placeholder_expander.time_value(time),
-          'record'   => record,
-        })
+        placeholder_values['time'] = @placeholder_expander.time_value(time)
+        placeholder_values['record'] = record
+
         new_record = reform(record, placeholder_values)
         if @renew_time_key && new_record.has_key?(@renew_time_key)
           time = Fluent::EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
@@ -316,8 +315,6 @@ module Fluent::Plugin
 
       class CleanroomExpander
         def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
-          tags = tag_parts # for old version compatibility
-          _ = tags # to suppress "unused variable" warning for tags
           Thread.current[:record_transformer_record] = record # for old version compatibility
           instance_eval(__str_to_eval__)
         end

data/lib/fluent/plugin/filter_stdout.rb

@@ -34,7 +34,7 @@ module Fluent::Plugin
     def configure(conf)
       compat_parameters_convert(conf, :inject, :formatter)
       super
-      @formatter = formatter_create(conf: @config.elements('format').first, default_type: DEFAULT_FORMAT_TYPE)
+      @formatter = formatter_create
     end
 
     def filter_stream(tag, es)

data/lib/fluent/plugin/formatter.rb

@@ -26,6 +26,11 @@ module Fluent
 
       configured_in :format
 
+      PARSER_TYPES = [:text_per_line, :text, :binary]
+      def formatter_type
+        :text_per_line
+      end
+
       def format(tag, time, record)
         raise NotImplementedError, "Implement this method in child class"
       end

data/lib/fluent/plugin/formatter_msgpack.rb

@@ -21,6 +21,10 @@ module Fluent
     class MessagePackFormatter < Formatter
       Plugin.register_formatter('msgpack', self)
 
+      def formatter_type
+        :binary
+      end
+
       def format(tag, time, record)
         record.to_msgpack
       end

data/lib/fluent/plugin/formatter_stdout.rb

@@ -21,6 +21,8 @@ module Fluent
     class StdoutFormatter < Formatter
       Plugin.register_formatter('stdout', self)
 
+      TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
+
       config_param :output_type, :string, default: 'json'
 
       def configure(conf)
@@ -36,8 +38,7 @@ module Fluent
       end
 
       def format(tag, time, record)
-        header = "#{Time.now.localtime} #{tag}: "
-        "#{header}#{@sub_formatter.format(tag, time, record)}"
+        "#{Time.at(time).localtime.strftime(TIME_FORMAT)} #{tag}: #{@sub_formatter.format(tag, time, record).chomp}\n"
       end
 
       def stop

data/lib/fluent/plugin/formatter_tsv.rb

@@ -0,0 +1,34 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class TSVFormatter < Formatter
+      Plugin.register_formatter('tsv', self)
+
+      desc 'Field names included in each lines'
+      config_param :keys, :array, value_type: :string
+      desc 'The delimiter character (or string) of TSV values'
+      config_param :delimiter, :string, default: "\t"
+
+      def format(tag, time, record)
+        @keys.map{|k| record[k].to_s }.join(@delimiter)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_exec.rb

@@ -14,136 +14,91 @@
 #    limitations under the License.
 #
 
-require 'strptime'
-require 'yajl'
-
 require 'fluent/plugin/input'
-require 'fluent/time'
-require 'fluent/timezone'
-require 'fluent/config/error'
+require 'yajl'
 
 module Fluent::Plugin
   class ExecInput < Fluent::Plugin::Input
     Fluent::Plugin.register_input('exec', self)
 
-    helpers :child_process
-
-    def initialize
-      super
-      require 'fluent/plugin/exec_util'
-    end
+    helpers :compat_parameters, :extract, :parser, :child_process
 
     desc 'The command (program) to execute.'
     config_param :command, :string
-    desc 'The format used to map the program output to the incoming event.(tsv,json,msgpack)'
-    config_param :format, :string, default: 'tsv'
-    desc 'Specify the comma-separated keys when using the tsv format.'
-    config_param :keys, default: [] do |val|
-      val.split(',')
+
+    config_section :parse do
+      config_set_default :@type, 'tsv'
+      config_set_default :time_type, :float
+      config_set_default :time_key, nil
+      config_set_default :estimate_current_event, false
+    end
+
+    config_section :extract do
+      config_set_default :time_type, :float
     end
+
     desc 'Tag of the output events.'
     config_param :tag, :string, default: nil
-    desc 'The key to use as the event tag instead of the value in the event record. '
-    config_param :tag_key, :string, default: nil
-    desc 'The key to use as the event time instead of the value in the event record.'
-    config_param :time_key, :string, default: nil
-    desc 'The format of the event time used for the time_key parameter.'
-    config_param :time_format, :string, default: nil
     desc 'The interval time between periodic program runs.'
     config_param :run_interval, :time, default: nil
+    desc 'The default block size to read if parser requires partial read.'
+    config_param :read_block_size, :size, default: 10240 # 10k
 
-    def configure(conf)
-      super
-
-      if conf['localtime']
-        @localtime = true
-      elsif conf['utc']
-        @localtime = false
-      end
-
-      if conf['timezone']
-        @timezone = conf['timezone']
-        Fluent::Timezone.validate!(@timezone)
-      end
-
-      if !@tag && !@tag_key
-        raise Fleunt::ConfigError, "'tag' or 'tag_key' option is required on exec input"
-      end
+    attr_reader :parser
 
-      if @time_key
-        if @time_format
-          f = @time_format
-          @time_parse_proc =
-            begin
-              strptime = Strptime.new(f)
-              Proc.new { |str| Fluent::EventTime.from_time(strptime.exec(str)) }
-            rescue
-              Proc.new {|str| Fluent::EventTime.from_time(Time.strptime(str, f)) }
-            end
-        else
-          @time_parse_proc = Proc.new {|str| Fluent::EventTime.from_time(Time.at(str.to_f)) }
+    def configure(conf)
+      compat_parameters_convert(conf, :extract, :parser)
+      ['parse', 'extract'].each do |subsection_name|
+        if subsection = conf.elements(subsection_name).first
+          if subsection.has_key?('time_format')
+            subsection['time_type'] ||= 'string'
+          end
         end
       end
 
-      @parser = setup_parser(conf)
-    end
+      super
 
-    def setup_parser(conf)
-      case @format
-      when 'tsv'
-        if @keys.empty?
-          raise Fluent::ConfigError, "keys option is required on exec input for tsv format"
-        end
-        Fluent::ExecUtil::TSVParser.new(@keys, method(:on_message))
-      when 'json'
-        Fluent::ExecUtil::JSONParser.new(method(:on_message))
-      when 'msgpack'
-        Fluent::ExecUtil::MessagePackParser.new(method(:on_message))
-      else
-        Fluent::ExecUtil::TextParserWrapperParser.new(conf, method(:on_message))
+      if !@tag && (!@extract_config || !@extract_config.tag_key)
+        raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on exec input"
       end
+      @parser = parser_create
     end
 
     def start
       super
 
       if @run_interval
-        child_process_execute(:exec_input, @command, interval: @run_interval, mode: [:read]) do |io|
-          run(io)
-        end
+        child_process_execute(:exec_input, @command, interval: @run_interval, mode: [:read], &method(:run))
       else
-        child_process_execute(:exec_input, @command, immediate: true, mode: [:read]) do |io|
-          run(io)
-        end
+        child_process_execute(:exec_input, @command, immediate: true, mode: [:read], &method(:run))
       end
     end
 
     def run(io)
-      @parser.call(io)
-    end
-
-    private
-
-    def on_message(record, parsed_time = nil)
-      if val = record.delete(@tag_key)
-        tag = val
-      else
-        tag = @tag
-      end
-
-      if parsed_time
-        time = parsed_time
-      else
-        if val = record.delete(@time_key)
-          time = @time_parse_proc.call(val)
-        else
-          time = Fluent::EventTime.now
+      case
+      when @parser.implement?(:parse_io)
+        @parser.parse_io(io, &method(:on_record))
+      when @parser.implement?(:parse_partial_data)
+        until io.eof?
+          @parser.parse_partial_data(io.readpartial(@read_block_size), &method(:on_record))
+        end
+      when @parser.parser_type == :text_per_line
+        io.each_line do |line|
+          @parser.parse(line.chomp, &method(:on_record))
         end
+      else
+        @parser.parse(io.read, &method(:on_record))
       end
+    end
 
+    def on_record(time, record)
+      tag = extract_tag_from_record(record)
+      tag ||= @tag
+      time ||= extract_time_from_record(record) || Fluent::EventTime.now
       router.emit(tag, time, record)
     rescue => e
-      log.error "exec failed to emit", error: e, tag: tag, record: Yajl.dump(record)
+      log.error "exec failed to emit", tag: tag, record: Yajl.dump(record), error: e
+      router.emit_error_event(tag, time, record, e) if tag && time && record
     end
   end
 end