fluentd 0.12.28 → 0.12.29

data/Rakefile

@@ -15,7 +15,7 @@ Rake::TestTask.new(:base_test) do |t|
   t.libs << "test"
   t.test_files = Dir["test/**/test_*.rb"].sort
   t.verbose = true
-  #t.warning = true
+  t.warning = false
 end
 
 task :parallel_test do

data/lib/fluent/agent.rb

@@ -57,6 +57,7 @@ module Fluent
       conf.elements.select { |e| e.name == 'filter' || e.name == 'match' }.each { |e|
         pattern = e.arg.empty? ? '**' : e.arg
         type = e['@type'] || e['type']
+        raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type
         if e.name == 'filter'
           add_filter(type, pattern, e)
         else

data/lib/fluent/config/configure_proxy.rb

@@ -129,6 +129,12 @@ module Fluent
         merged
       end
 
+      def option_value_type!(name, opts, key, klass)
+        if opts.has_key?(key) && !opts[key].is_a?(klass)
+          raise ArgumentError, "#{name}: #{key} must be a #{klass}, but #{opts[key].class}"
+        end
+      end
+
       def parameter_configuration(name, *args, &block)
         name = name.to_sym
 
@@ -145,7 +151,7 @@ module Fluent
 
         type = opts[:type]
         if block && type
-          raise ArgumentError, "#{self.name}: both of block and type cannot be specified"
+          raise ArgumentError, "#{name}: both of block and type cannot be specified"
         end
 
         begin
@@ -153,8 +159,19 @@ module Fluent
           block ||= Configurable.lookup_type(type)
         rescue ConfigError
           # override error message
-          raise ArgumentError, "#{self.name}: unknown config_argument type `#{type}'"
+          raise ArgumentError, "#{name}: unknown config_argument type `#{type}'"
+        end
+
+        option_value_type!(name, opts, :desc, String)
+        option_value_type!(name, opts, :alias, Symbol)
+        option_value_type!(name, opts, :deprecated, String)
+        option_value_type!(name, opts, :obsoleted, String)
+        if type == :enum
+          if !opts.has_key?(:list) || !opts[:list].all?{|v| v.is_a?(Symbol) }
+            raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
+          end
         end
+        option_value_type!(name, opts, :value_type, Symbol) # hash, array
 
         if opts.has_key?(:default)
           config_set_default(name, opts[:default])
@@ -164,6 +181,10 @@ module Fluent
           config_set_desc(name, opts[:desc])
         end
 
+        if opts[:deprecated] && opts[:obsoleted]
+          raise ArgumentError, "#{name}: both of deprecated and obsoleted cannot be specified at once"
+        end
+
         [name, block, opts]
       end
 
@@ -218,7 +239,7 @@ module Fluent
 
       def config_section(name, *args, &block)
         unless block_given?
-          raise ArgumentError, "#{self.name}: config_section requires block parameter"
+          raise ArgumentError, "#{name}: config_section requires block parameter"
         end
         name = name.to_sym
 

data/lib/fluent/config/error.rb

@@ -20,4 +20,7 @@ module Fluent
 
   class ConfigParseError < ConfigError
   end
+
+  class ObsoletedParameterError < ConfigError
+  end
 end

data/lib/fluent/config/section.rb

@@ -100,6 +100,7 @@ module Fluent
             logger.error "config error in:\n#{conf}"
             raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
           end
+          # argument should NOT be deprecated... (argument always has a value: '')
         end
 
         proxy.params.each_pair do |name, defval|
@@ -112,6 +113,20 @@ module Fluent
                     conf[opts[:alias].to_s]
                   end
             section_params[varname] = self.instance_exec(val, opts, name, &block)
+
+            # Source of definitions of deprecated/obsoleted:
+            # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
+            #
+            # Deprecated: These deprecated features can still be used, but should be used with caution
+            #             because they are expected to be removed entirely sometime in the future.
+            # Obsoleted: These obsolete features have been entirely removed from JavaScript and can no longer be used.
+            if opts[:deprecated]
+              logger.warn "'#{name}' parameter is deprecated: #{opts[:deprecated]}"
+            end
+            if opts[:obsoleted]
+              logger.error "config error in:\n#{conf}" if logger
+              raise ObsoletedParameterError, "'#{name}' parameter is already removed: #{opts[:obsoleted]}" + section_stack
+            end
           end
           unless section_params.has_key?(varname)
             logger.error "config error in:\n#{conf}"

data/lib/fluent/plugin/filter_parser.rb

@@ -0,0 +1,107 @@
+require 'time'
+require 'fluent/parser'
+
+class Fluent::ParserFilter < Fluent::Filter
+  Fluent::Plugin.register_filter('parser', self)
+
+  config_param :key_name, :string
+  config_param :reserve_data, :bool, default: false
+  config_param :inject_key_prefix, :string, default: nil
+  config_param :replace_invalid_sequence, :bool, default: false
+  config_param :hash_value_field, :string, default: nil
+  config_param :suppress_parse_error_log, :bool, default: false
+  config_param :time_parse, :bool, default: true
+  config_param :ignore_key_not_exist, :bool, default: false
+
+  attr_reader :parser
+
+  def configure(conf)
+    super
+
+    @parser = Fluent::TextParser.new
+    @parser.estimate_current_event = false
+    @parser.configure(conf)
+    if !@time_parse && @parser.parser.respond_to?("time_key=".to_sym)
+      # disable parse time
+      @parser.parser.time_key = nil
+    end
+
+    self
+  end
+
+  def filter_stream(tag, es)
+    new_es = Fluent::MultiEventStream.new
+    es.each do |time,record|
+      raw_value = record[@key_name]
+      if raw_value.nil?
+        log.warn "#{@key_name} does not exist" unless @ignore_key_not_exist
+        new_es.add(time, handle_parsed(tag, record, time, {})) if @reserve_data
+        next
+      end
+      begin
+        @parser.parse(raw_value) do |t,values|
+          if values
+            t ||= time
+            r = handle_parsed(tag, record, t, values)
+            new_es.add(t, r)
+          else
+            log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+            if @reserve_data
+              t = time
+              r = handle_parsed(tag, record, time, {})
+              new_es.add(t, r)
+            end
+          end
+        end
+      rescue Fluent::TextParser::ParserError => e
+        log.warn e.message unless @suppress_parse_error_log
+      rescue ArgumentError => e
+        if @replace_invalid_sequence
+          unless e.message.index("invalid byte sequence in") == 0
+            raise
+          end
+          replaced_string = replace_invalid_byte(raw_value)
+          @parser.parse(replaced_string) do |t,values|
+            if values
+              t ||= time
+              r = handle_parsed(tag, record, t, values)
+              new_es.add(t, r)
+            else
+              log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+              if @reserve_data
+                t = time
+                r = handle_parsed(tag, record, time, {})
+                new_es.add(t, r)
+              end
+            end
+          end
+        else
+          raise
+        end
+      rescue => e
+        log.warn "parse failed #{e.message}" unless @suppress_parse_error_log
+      end
+    end
+    new_es
+  end
+
+  private
+
+  def handle_parsed(tag, record, t, values)
+    if values && @inject_key_prefix
+      values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
+    end
+    r = @hash_value_field ? {@hash_value_field => values} : values
+    if @reserve_data
+      r = r ? record.merge(r) : record
+    end
+    r
+  end
+
+  def replace_invalid_byte(string)
+    replace_options = { invalid: :replace, undef: :replace, replace: '?' }
+    original_encoding = string.encoding
+    temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
+    string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
+  end
+end

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -204,7 +204,9 @@ module Fluent
             end
           elsif value.kind_of?(Hash) # record, etc
             value.each do |k, v|
-              placeholders.store("${#{k}}", v) # foo
+              unless placeholder_values.has_key?(k) # prevent overwriting reserved keys such as tag
+                placeholders.store("${#{k}}", v) # foo
+              end
               placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
             end
           else # string, interger, float, and others?

data/lib/fluent/plugin/in_tail.rb

@@ -50,14 +50,10 @@ module Fluent
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
+    desc 'The encoding after conversion of the input.'
+    config_param :encoding, :string, default: nil
     desc 'The encoding of the input.'
-    config_param :encoding, default: nil do |encoding_name|
-      begin
-        Encoding.find(encoding_name)
-      rescue ArgumentError => e
-        raise ConfigError, e.message
-      end
-    end
+    config_param :from_encoding, :string, default: nil
     desc 'Add the log path being tailed to records. Specify the field name to be used.'
     config_param :path_key, :string, default: nil
 
@@ -78,6 +74,7 @@ module Fluent
 
       configure_parser(conf)
       configure_tag
+      configure_encoding
 
       @multiline_mode = conf['format'] =~ /multiline/
       @receive_handler = if @multiline_mode
@@ -102,6 +99,25 @@ module Fluent
       end
     end
 
+    def configure_encoding
+      unless @encoding
+        if @from_encoding
+          raise ConfigError, "tail: 'from_encoding' parameter must be specified with 'encoding' parameter."
+        end
+      end
+
+      @encoding = parse_encoding_param(@encoding) if @encoding
+      @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding
+    end
+
+    def parse_encoding_param(encoding_name)
+      begin
+        Encoding.find(encoding_name) if encoding_name
+      rescue ArgumentError => e
+        raise ConfigError, e.message
+      end
+    end
+
     def start
       if @pos_file
         @pf_file = File.open(@pos_file, File::RDWR|File::CREAT, DEFAULT_FILE_PERMISSION)
@@ -232,7 +248,13 @@ module Fluent
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
-        lb.force_encoding(@encoding) if @encoding
+        if @encoding
+          if @from_encoding
+            lb.encode!(@encoding, @from_encoding)
+          else
+            lb.force_encoding(@encoding)
+          end
+        end
         @parser.parse(lb) { |time, record|
           if time && record
             tag = if @tag_prefix || @tag_suffix
@@ -281,7 +303,13 @@ module Fluent
     def convert_line_to_event(line, es, tail_watcher)
       begin
         line.chomp!  # remove \n
-        line.force_encoding(@encoding) if @encoding
+        if @encoding
+          if @from_encoding
+            line.encode!(@encoding, @from_encoding)
+          else
+            line.force_encoding(@encoding)
+          end
+        end
         @parser.parse(line) { |time, record|
           if time && record
             record[@path_key] ||= tail_watcher.path unless @path_key.nil?

data/lib/fluent/plugin/out_forward.rb

@@ -86,9 +86,8 @@ module Fluent
 
     attr_reader :nodes
 
-    # backward compatibility
-    config_param :port, :integer, default: DEFAULT_LISTEN_PORT
-    config_param :host, :string, default: nil
+    config_param :port, :integer, default: DEFAULT_LISTEN_PORT, deprecated: "User <server> host xxx </server> instead."
+    config_param :host, :string, default: nil, deprecated: "Use <server> port xxx </server> instead."
 
     attr_accessor :extend_internal_protocol
 
@@ -97,7 +96,6 @@ module Fluent
 
       # backward compatibility
       if host = conf['host']
-        log.warn "'host' option in forward output is obsoleted. Use '<server> host xxx </server>' instead."
         port = conf['port']
         port = port ? port.to_i : DEFAULT_LISTEN_PORT
         e = conf.add_element('server')

data/lib/fluent/root_agent.rb

@@ -91,7 +91,7 @@ module Fluent
       else
         conf.elements.select { |e| e.name == 'source' }.each { |e|
           type = e['@type'] || e['type']
-          raise ConfigError, "Missing 'type' parameter on <source> directive" unless type
+          raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
           add_source(type, e)
         }
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.28'
+  VERSION = '0.12.29'
 
 end

data/test/config/test_configurable.rb

@@ -158,6 +158,13 @@ module ConfigurableSpec
       config_param :secret_param2, :string, secret: true
     end
   end
+
+  class UnRecommended
+    include Fluent::Configurable
+    attr_accessor :log
+    config_param :key1, :string, default: 'deprecated', deprecated: "key1 will be removed."
+    config_param :key2, :string, default: 'obsoleted', obsoleted: "key2 has been removed."
+  end
 end
 
 module Fluent::Config
@@ -740,5 +747,87 @@ module Fluent::Config
         end
       end
     end
+    sub_test_case 'non-required options for config_param' do
+      test 'desc must be a string if specified' do
+        assert_raise ArgumentError.new("key: desc must be a String, but Symbol") do
+          class InvalidDescClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', desc: :invalid_description
+          end
+        end
+      end
+      test 'alias must be a symbol if specified' do
+        assert_raise ArgumentError.new("key: alias must be a Symbol, but String") do
+          class InvalidAliasClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', alias: 'yay'
+          end
+        end
+      end
+      test 'deprecated must be a string if specified' do
+        assert_raise ArgumentError.new("key: deprecated must be a String, but TrueClass") do
+          class InvalidDeprecatedClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', deprecated: true
+          end
+        end
+      end
+      test 'obsoleted must be a string if specified' do
+        assert_raise ArgumentError.new("key: obsoleted must be a String, but TrueClass") do
+          class InvalidObsoletedClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', obsoleted: true
+          end
+        end
+      end
+      test 'value_type for hash must be a symbol' do
+        assert_raise ArgumentError.new("key: value_type must be a Symbol, but String") do
+          class InvalidValueTypeOfHashClass
+            include Fluent::Configurable
+            config_param :key, :hash, value_type: 'yay'
+          end
+        end
+      end
+      test 'value_type for array must be a symbol' do
+        assert_raise ArgumentError.new("key: value_type must be a Symbol, but String") do
+          class InvalidValueTypeOfArrayClass
+            include Fluent::Configurable
+            config_param :key, :array, value_type: 'yay'
+          end
+        end
+      end
+    end
+    sub_test_case 'enum parameters' do
+      test 'list must be specified as an array of symbols'
+    end
+    sub_test_case 'deprecated/obsoleted parameters' do
+      test 'both cannot be specified at once' do
+        assert_raise ArgumentError.new("param1: both of deprecated and obsoleted cannot be specified at once") do
+          class Buggy1
+            include Fluent::Configurable
+            config_param :param1, :string, default: '', deprecated: 'yay', obsoleted: 'foo!'
+          end
+        end
+      end
+
+      test 'warned if deprecated parameter is configured' do
+        obj = ConfigurableSpec::UnRecommended.new
+        obj.log = Fluent::Test::TestLogger.new
+        obj.configure(Fluent::Config::Element.new('ROOT', '', {'key1' => 'yay'}, []))
+
+        assert_equal 'yay', obj.key1
+        first_log = obj.log.logs.first
+        assert{ first_log && first_log.include?("[warn]") && first_log.include?("'key1' parameter is deprecated: key1 will be removed.") }
+      end
+
+      test 'error raised if obsoleted parameter is configured' do
+        obj = ConfigurableSpec::UnRecommended.new
+        obj.log = Fluent::Test::TestLogger.new
+
+        assert_raise Fluent::ObsoletedParameterError.new("'key2' parameter is already removed: key2 has been removed.") do
+          obj.configure(Fluent::Config::Element.new('ROOT', '', {'key2' => 'yay'}, []))
+        end
+      end
+    end
   end
 end