fluentd 1.8.0.rc3 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f75be7f2480c2f0fbdddf1999dc3163b91f6bd75fb53b71ff3d7a0ee4f670af
-  data.tar.gz: 5089bdce54388f855748befd4234ceb945f4ef7c6aa0f2a4f1aeea21adb516da
+  metadata.gz: e87e035a4839980ff8d1a31999d731790802d1e92f1c45fe1694622d34ea59c9
+  data.tar.gz: 787e06b6b6b74ee3f93e3e066627f11e02af7d10027f2dea913d22d14b1eab7f
 SHA512:
-  metadata.gz: 6dfd02456cd7a47aaf483759b57075cc0b8c0bbe322a41c20cd6df7f9a163150af07a21201e8514b0ba097f50dc56ae7004c4116f2f4c2b5347eebde303a19a2
-  data.tar.gz: 1ec01c129c043f8fafee99360a3a625907cff7666f9a605093110b4b1cc57691e7fd1eba543e2003568a9aa48d714a853b2644e83d2823f5773f49615bc0d265
+  metadata.gz: a56f21ab62ce38de8eb55d4f5f87340b08c603c46a64553e55475a22df5af40caf75d0768b0d87df99d7581e6a47f5344e8ffa590527d7456ee6334ec5a62d16
+  data.tar.gz: 34c24131c84b70c74201781f1ff87c3e0d5081b56f92c81144cb52dfe3ba89b2c5c2202be0d7e990ccab87945ae0333eb8a8f8a37ea3f02eb3528bcf3d2d3ff8

data/.travis.yml

@@ -49,6 +49,10 @@ matrix:
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3
+    - rvm: 2.5.5
+      os: linux
+      arch: s390x
+      dist: xenial
     - rvm: ruby-head
 
 branches:

data/CHANGELOG.md

@@ -1,9 +1,13 @@
-# Unreleased
+# v1.8
+
+## Release v1.8.0 - 2019/12/11
 
 ### New feature
 
 * Add service discovery plugin and `out_forward` use it
   https://github.com/fluent/fluentd/pull/2541
+* config: Add strict mode and support `default`/`nil` value in ruby embedded mode
+  https://github.com/fluent/fluentd/pull/2685
 
 ### Enhancement
 
@@ -15,6 +19,15 @@
   https://github.com/fluent/fluentd/pull/2673
 * log: Deprecate top-level match for capturing fluentd logs
   https://github.com/fluent/fluentd/pull/2689
+* in_monitor_agent: Expose Fluentd verion in REST API
+  https://github.com/fluent/fluentd/pull/2706
+* time: Accept localtime xor utc
+  https://github.com/fluent/fluentd/pull/2720
+  https://github.com/fluent/fluentd/pull/2731
+* formatter_stdout: Make time_format configurable in stdout format
+  https://github.com/fluent/fluentd/pull/2721
+* supervisor: create log directory when it doesn't exists
+  https://github.com/fluent/fluentd/pull/2732
 * clean up internal classes / methods / code
   https://github.com/fluent/fluentd/pull/2647
   https://github.com/fluent/fluentd/pull/2648
@@ -25,9 +38,14 @@
   https://github.com/fluent/fluentd/pull/2674
   https://github.com/fluent/fluentd/pull/2677
   https://github.com/fluent/fluentd/pull/2680
+  https://github.com/fluent/fluentd/pull/2709
+  https://github.com/fluent/fluentd/pull/2730
 
 ### Bug fixes
 
+* output: Fix warning printed when chunk key placeholder not replaced
+  https://github.com/fluent/fluentd/pull/2523
+  https://github.com/fluent/fluentd/pull/2733
 * Fix dry-run mode
   https://github.com/fluent/fluentd/pull/2651
 * suppress warning
@@ -44,7 +62,11 @@
   https://github.com/fluent/fluentd/pull/2695
 * server helper: Fix IPv6 dual stack mode issue for tcp socket.
   https://github.com/fluent/fluentd/pull/2697
-
+* supervisor: Fix inline config handling
+  https://github.com/fluent/fluentd/pull/2708
+* Fix typo
+  https://github.com/fluent/fluentd/pull/2710
+  https://github.com/fluent/fluentd/pull/2714
 
 # v1.7
 

data/MAINTAINERS.md

@@ -5,3 +5,4 @@
 - [Hiroshi Hatake](https://github.com/cosmo0920), [Clearcode](https://www.clear-code.com/)
 - [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
 - [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
+- [Eduardo Silva](https://github.com/edsiper), [Arm Treasure Data](https://www.treasuredata.com/)

data/lib/fluent/command/fluentd.rb

@@ -131,6 +131,10 @@ op.on('--use-v0-config', "Use v0 configuration format", TrueClass) {|b|
   opts[:use_v1_config] = !b
 }
 
+op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
+  opts[:strict_config_value] = b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

data/lib/fluent/config/element.rb

@@ -142,7 +142,7 @@ module Fluent
         indent = "  " * nest
         nindent = "  " * (nest + 1)
         out = ""
-        if @arg.empty?
+        if @arg.nil? || @arg.empty?
           out << "#{indent}<#{@name}>\n"
         else
           out << "#{indent}<#{@name} #{@arg}>\n"
@@ -194,23 +194,36 @@ module Fluent
         opts[:type]
       end
 
+      def default_value(key)
+        return nil if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        proxy = @corresponding_proxies.detect do |_proxy|
+          _proxy.params.has_key?(param_key)
+        end
+        return nil unless proxy
+        proxy.defaults[param_key]
+      end
+
       def dump_value(k, v, nindent)
-        if secret_param?(k)
-          "#{nindent}#{k} xxxxxx\n"
+        return "#{nindent}#{k} xxxxxx\n" if secret_param?(k)
+        return "#{nindent}#{k} #{v}\n" unless @v1_config
+
+        # for v1 config
+        if v.nil?
+          "#{nindent}#{k} \n"
+        elsif v == :default
+          "#{nindent}#{k} #{default_value(k)}\n"
         else
-          if @v1_config
-            case param_type(k)
-            when :string
-              "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
-            when :enum, :integer, :float, :size, :bool, :time
-              "#{nindent}#{k} #{v}\n"
-            when :hash, :array
-              "#{nindent}#{k} #{v}\n"
-            else
-              # Unknown type
-              "#{nindent}#{k} #{v}\n"
-            end
+          case param_type(k)
+          when :string
+            "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
+          when :enum, :integer, :float, :size, :bool, :time
+            "#{nindent}#{k} #{v}\n"
+          when :hash, :array
+            "#{nindent}#{k} #{v}\n"
           else
+            # Unknown type
             "#{nindent}#{k} #{v}\n"
           end
         end

data/lib/fluent/config/error.rb

@@ -23,4 +23,10 @@ module Fluent
 
   class ObsoletedParameterError < ConfigError
   end
+
+  class SetNil < Exception
+  end
+
+  class SetDefault < Exception
+  end
 end

data/lib/fluent/config/literal_parser.rb

@@ -52,6 +52,16 @@ module Fluent
       def initialize(strscan, eval_context)
         super(strscan)
         @eval_context = eval_context
+        unless @eval_context.respond_to?(:use_nil)
+          def @eval_context.use_nil
+            raise SetNil
+          end
+        end
+        unless @eval_context.respond_to?(:use_default)
+          def @eval_context.use_default
+            raise SetDefault
+          end
+        end
       end
 
       def parse_literal(string_boundary_charset = LINE_END)
@@ -81,7 +91,13 @@ module Fluent
         string = []
         while true
           if skip(/\"/)
-            return string.join
+            if string.include?(nil)
+              return nil
+            elsif string.include?(:default)
+              return :default
+            else
+              return string.join
+            end
           elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
             if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
               string << s
@@ -168,7 +184,13 @@ module Fluent
 hostname = Socket.gethostname
 worker_id = ENV['SERVERENGINE_WORKER_ID'] || ''
 EOM
-        @eval_context.instance_eval(code)
+        begin
+          @eval_context.instance_eval(code)
+        rescue SetNil => e
+          nil
+        rescue SetDefault => e
+          :default
+        end
       end
 
       def eval_escape_char(c)

data/lib/fluent/config/section.rb

@@ -106,7 +106,7 @@ module Fluent
     end
 
     module SectionGenerator
-      def self.generate(proxy, conf, logger, plugin_class, stack = [])
+      def self.generate(proxy, conf, logger, plugin_class, stack = [], strict_config_value = false)
         return nil if conf.nil?
 
         section_stack = ""
@@ -122,9 +122,23 @@ module Fluent
         end
 
         if proxy.argument
-          unless conf.arg.empty?
+          unless conf.arg.nil? || conf.arg.empty?
             key, block, opts = proxy.argument
-            section_params[key] = self.instance_exec(conf.arg, opts, name, &block)
+            opts = opts.merge(strict: true) if strict_config_value
+
+            if conf.arg == :default
+              unless section_params.has_key?(key)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{key}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[key] = self.instance_exec(conf.arg, opts, key, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
           end
           unless section_params.has_key?(proxy.argument.first)
             logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
@@ -136,13 +150,36 @@ module Fluent
         proxy.params.each_pair do |name, defval|
           varname = name.to_sym
           block, opts = defval
+          opts = opts.merge(strict: true) if strict_config_value
+
           if conf.has_key?(name.to_s) || opts[:alias] && conf.has_key?(opts[:alias].to_s)
             val = if conf.has_key?(name.to_s)
                     conf[name.to_s]
                   else
                     conf[opts[:alias].to_s]
                   end
-            section_params[varname] = self.instance_exec(val, opts, name, &block)
+
+            if val == :default
+              # default value is already set if it exists
+              unless section_params.has_key?(varname)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{varname}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[varname] = self.instance_exec(val, opts, name, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
+
+            if section_params[varname].nil?
+              unless proxy.defaults.has_key?(varname) and proxy.defaults[varname].nil?
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{name}' parameter is required but nil is specified"
+              end
+            end
 
             # Source of definitions of deprecated/obsoleted:
             # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
@@ -190,13 +227,13 @@ module Fluent
             raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
           end
           if subproxy.multi?
-            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name]) }
+            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name], strict_config_value) }
           else
             if elements.size > 1
               logger.error "config error in:\n#{conf}" if logger
               raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
             end
-            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name])
+            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name], strict_config_value)
           end
         end
 

data/lib/fluent/config/types.rb

@@ -20,7 +20,9 @@ require 'fluent/config/error'
 
 module Fluent
   module Config
-    def self.size_value(str)
+    def self.size_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
       case str.to_s
       when /([0-9]+)k/i
         $~[1].to_i * 1024
@@ -31,11 +33,13 @@ module Fluent
       when /([0-9]+)t/i
         $~[1].to_i * (1024 ** 4)
       else
-        str.to_i
+        INTEGER_TYPE.call(str, opts, name)
       end
     end
 
-    def self.time_value(str)
+    def self.time_value(str, opts = {}, name = nil)
+      return nil if str.nil?
+
       case str.to_s
       when /([0-9]+)s/
         $~[1].to_i
@@ -46,12 +50,13 @@ module Fluent
       when /([0-9]+)d/
         $~[1].to_i * 24 * 60 * 60
       else
-        str.to_f
+        FLOAT_TYPE.call(str, opts, name)
       end
     end
 
-    def self.bool_value(str)
+    def self.bool_value(str, opts = {}, name = nil)
       return nil if str.nil?
+
       case str.to_s
       when 'true', 'yes'
         true
@@ -64,14 +69,17 @@ module Fluent
         # parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.
         if str.respond_to?('start_with?') && str.start_with?('#')
           true
+        elsif opts[:strict]
+          raise Fluent::ConfigError, "#{name}: invalid bool value: #{str}"
         else
           nil
         end
       end
     end
 
-    def self.regexp_value(str)
+    def self.regexp_value(str, opts = {}, name = nil)
       return nil unless str
+
       return Regexp.compile(str) unless str.start_with?("/")
       right_slash_position = str.rindex("/")
       if right_slash_position < str.size - 3
@@ -84,12 +92,21 @@ module Fluent
       Regexp.compile(str[1...right_slash_position], option)
     end
 
-    STRING_TYPE = Proc.new { |val, opts|
+    def self.string_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
       v = val.to_s
       v = v.frozen? ? v.dup : v # config_param can't assume incoming string is mutable
       v.force_encoding(Encoding::UTF_8)
+    end
+
+    STRING_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.string_value(val, opts, name)
     }
-    ENUM_TYPE = Proc.new { |val, opts|
+
+    def self.enum_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
       s = val.to_sym
       list = opts[:list]
       raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
@@ -97,33 +114,77 @@ module Fluent
         raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
       end
       s
+    end
+
+    ENUM_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.enum_value(val, opts, name)
+    }
+
+    INTEGER_TYPE = Proc.new { |val, opts = {}, name = nil|
+      if val.nil?
+        nil
+      elsif opts[:strict]
+        begin
+          Integer(val)
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "#{name}: #{e.message}"
+        end
+      else
+        val.to_i
+      end
     }
-    INTEGER_TYPE = Proc.new { |val, opts| val.to_i }
-    FLOAT_TYPE = Proc.new { |val, opts| val.to_f }
-    SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) }
-    BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
-    TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
-    REGEXP_TYPE = Proc.new { |val, opts| Config.regexp_value(val) }
-
-    REFORMAT_VALUE = ->(type, value) {
+
+    FLOAT_TYPE = Proc.new { |val, opts = {}, name = nil|
+      if val.nil?
+        nil
+      elsif opts[:strict]
+        begin
+          Float(val)
+        rescue ArgumentError, TypeError => e
+          raise ConfigError, "#{name}: #{e.message}"
+        end
+      else
+        val.to_f
+      end
+    }
+
+    SIZE_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.size_value(val, opts, name)
+    }
+
+    BOOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.bool_value(val, opts, name)
+    }
+
+    TIME_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.time_value(val, opts, name)
+    }
+
+    REGEXP_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.regexp_value(val, opts, name)
+    }
+
+    REFORMAT_VALUE = ->(type, value, opts = {}, name = nil) {
       if value.nil?
         value
       else
         case type
         when :string  then value.to_s.force_encoding(Encoding::UTF_8)
-        when :integer then value.to_i
-        when :float   then value.to_f
-        when :size then Config.size_value(value)
-        when :bool then Config.bool_value(value)
-        when :time then Config.time_value(value)
-        when :regexp then Config.regexp_value(value)
+        when :integer then INTEGER_TYPE.call(value, opts, name)
+        when :float   then FLOAT_TYPE.call(value, opts, name)
+        when :size then Config.size_value(value, opts, name)
+        when :bool then Config.bool_value(value, opts, name)
+        when :time then Config.time_value(value, opts, name)
+        when :regexp then Config.regexp_value(value, opts, name)
         else
           raise "unknown type in REFORMAT: #{type}"
         end
       end
     }
 
-    HASH_TYPE = Proc.new { |val, opts|
+    def self.hash_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
       param = if val.is_a?(String)
                 val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
               else
@@ -138,12 +199,19 @@ module Fluent
         newparam = {}
         param.each_pair do |key, value|
           new_key = opts[:symbolize_keys] ? key.to_sym : key
-          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value) : value
+          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value, opts, new_key) : value
         end
         newparam
       end
+    end
+
+    HASH_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.hash_value(val, opts, name)
     }
-    ARRAY_TYPE = Proc.new { |val, opts|
+
+    def self.array_value(val, opts = {}, name = nil)
+      return nil if val.nil?
+
       param = if val.is_a?(String)
                 val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
               else
@@ -153,10 +221,14 @@ module Fluent
         raise ConfigError, "array required but got #{val.inspect}"
       end
       if opts[:value_type]
-        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v) }
+        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v, opts, nil) }
       else
         param
       end
+    end
+
+    ARRAY_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.array_value(val, opts, name)
     }
   end
 end