fluentd 1.7.4-x64-mingw32 → 1.8.0-x64-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ac4a6f9d933f5ab055685ea55073ed2075774fcfd06e9d6caa83173fa54bf6a
-  data.tar.gz: a6c1591f5668ccfeebbe94f74bed82359cd968011f65c8fae4707f912f09bab7
+  metadata.gz: fc4b25e6ef17858140851741cb1393cea6c9735d70df0ebf3dc5d56b4d09889d
+  data.tar.gz: 173a01d0e1315d960815dedfdd959e95122e76bd63b941f25856e3a06784e336
 SHA512:
-  metadata.gz: 31211269e644bfbfa87535d431c3ee62fe76b67e38c2ed34cc2d76b08849db974b1c940e2dd100f01c18506db63cae8cf6822b7a16191ea019d6f4b408c54ce3
-  data.tar.gz: 9b6671277290439f1711de19b1e5634978b285a9392dbeb739e47f6acee296466defac66e4e5b7b60032703821125c0c094f3b9be181209c2cec34dd01afbdf4
+  metadata.gz: 5be24749486ea83e7de373a04bdb4da3c6d85b87384dc1c2b36c8d506f43f5b6308fb3b9adafca8fd13152220bed2f3bddb366996713abaf7156e9b2a5f412b8
+  data.tar.gz: 85a744dafbd03254f6b889164cc58ffec01380182b9745199e1ed0c7afaeefa160bf7e15eeeda7bacf7d215eff62be0f1802e9d223464a59ee5bf98753a3b546

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,7 +1,6 @@
 ---
 name:  Bug Report
 about: Create a report to help us improve. If you have questions about Fluentd and plugins, please direct these to https://groups.google.com/forum/#!forum/fluentd
-labels: 'bug'
 
 ---
 

data/.travis.yml

@@ -49,6 +49,10 @@ matrix:
     - rvm: 2.4.6
       os: osx
       osx_image: xcode8.3
+    - rvm: 2.5.5
+      os: linux
+      arch: s390x
+      dist: xenial
     - rvm: ruby-head
 
 branches:

data/CHANGELOG.md

@@ -1,3 +1,73 @@
+# v1.8
+
+## Release v1.8.0 - 2019/12/11
+
+### New feature
+
+* Add service discovery plugin and `out_forward` use it
+  https://github.com/fluent/fluentd/pull/2541
+* config: Add strict mode and support `default`/`nil` value in ruby embedded mode
+  https://github.com/fluent/fluentd/pull/2685
+
+### Enhancement
+
+* formatter_csv: Support nested fields
+  https://github.com/fluent/fluentd/pull/2643
+* record_accessor helper: Make code simple and bit faster
+  https://github.com/fluent/fluentd/pull/2660
+* Relax tzinfo dependency to accept v1
+  https://github.com/fluent/fluentd/pull/2673
+* log: Deprecate top-level match for capturing fluentd logs
+  https://github.com/fluent/fluentd/pull/2689
+* in_monitor_agent: Expose Fluentd verion in REST API
+  https://github.com/fluent/fluentd/pull/2706
+* time: Accept localtime xor utc
+  https://github.com/fluent/fluentd/pull/2720
+  https://github.com/fluent/fluentd/pull/2731
+* formatter_stdout: Make time_format configurable in stdout format
+  https://github.com/fluent/fluentd/pull/2721
+* supervisor: create log directory when it doesn't exists
+  https://github.com/fluent/fluentd/pull/2732
+* clean up internal classes / methods / code
+  https://github.com/fluent/fluentd/pull/2647
+  https://github.com/fluent/fluentd/pull/2648
+  https://github.com/fluent/fluentd/pull/2653
+  https://github.com/fluent/fluentd/pull/2654
+  https://github.com/fluent/fluentd/pull/2657
+  https://github.com/fluent/fluentd/pull/2667
+  https://github.com/fluent/fluentd/pull/2674
+  https://github.com/fluent/fluentd/pull/2677
+  https://github.com/fluent/fluentd/pull/2680
+  https://github.com/fluent/fluentd/pull/2709
+  https://github.com/fluent/fluentd/pull/2730
+
+### Bug fixes
+
+* output: Fix warning printed when chunk key placeholder not replaced
+  https://github.com/fluent/fluentd/pull/2523
+  https://github.com/fluent/fluentd/pull/2733
+* Fix dry-run mode
+  https://github.com/fluent/fluentd/pull/2651
+* suppress warning
+  https://github.com/fluent/fluentd/pull/2652
+* suppress keyword argument warning for ruby2.7
+  https://github.com/fluent/fluentd/pull/2664
+* RPC: Fix debug log text
+  https://github.com/fluent/fluentd/pull/2666
+* time: Properly show class names in error message
+  https://github.com/fluent/fluentd/pull/2671
+* Fix a potential bug that ThreadError may occur on SIGUSR1
+  https://github.com/fluent/fluentd/pull/2678
+* server helper: Ignore ECONNREFUSED in TLS accept to avoid fluentd restart
+  https://github.com/fluent/fluentd/pull/2695
+* server helper: Fix IPv6 dual stack mode issue for tcp socket.
+  https://github.com/fluent/fluentd/pull/2697
+* supervisor: Fix inline config handling
+  https://github.com/fluent/fluentd/pull/2708
+* Fix typo
+  https://github.com/fluent/fluentd/pull/2710
+  https://github.com/fluent/fluentd/pull/2714
+
 # v1.7
 
 ## Release v1.7.4 - 2019/10/24

data/MAINTAINERS.md

@@ -5,3 +5,4 @@
 - [Hiroshi Hatake](https://github.com/cosmo0920), [Clearcode](https://www.clear-code.com/)
 - [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
 - [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
+- [Eduardo Silva](https://github.com/edsiper), [Arm Treasure Data](https://www.treasuredata.com/)

data/example/out_forward_sd.conf

@@ -0,0 +1,17 @@
+<source>
+  @type dummy
+  tag test
+</source>
+
+<match test>
+  @type forward
+
+  <service_discovery>
+    @type file
+    path "#{Dir.pwd}/example/sd.yaml"
+  </service_discovery>
+
+  <buffer>
+    flush_interval 1
+  </buffer>
+</match>

data/example/sd.yaml

@@ -0,0 +1,8 @@
+- 'host': 127.0.0.1
+  'port': 24224
+  'weight': 1
+  'name': server1
+- 'host': 127.0.0.1
+  'port': 24225
+  'weight': 1
+  'name': server2

data/fluentd.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
-  gem.add_runtime_dependency("tzinfo", ["~> 2.0"])
+  gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
   gem.add_runtime_dependency("dig_rb", ["~> 1.0.0"])

data/lib/fluent/agent.rb

@@ -62,7 +62,9 @@ module Fluent
 
       # initialize <match> and <filter> elements
       conf.elements('filter', 'match').each { |e|
-        next if e.for_another_worker?
+        if !Fluent::Engine.supervisor_mode && e.for_another_worker?
+          next
+        end
         pattern = e.arg.empty? ? '**' : e.arg
         type = e['@type']
         raise ConfigError, "Missing '@type' parameter on <#{e.name}> directive" unless type

data/lib/fluent/command/cat.rb

@@ -315,7 +315,7 @@ when 'msgpack'
   require 'fluent/engine'
 
   begin
-    u = Fluent::Engine.msgpack_factory.unpacker($stdin)
+    u = Fluent::MessagePackFactory.msgpack_unpacker($stdin)
     u.each {|record|
       w.write(record)
     }
@@ -340,4 +340,3 @@ else
   $stderr.puts "Unknown format '#{format}'"
   exit 1
 end
-

data/lib/fluent/command/fluentd.rb

@@ -131,6 +131,10 @@ op.on('--use-v0-config', "Use v0 configuration format", TrueClass) {|b|
   opts[:use_v1_config] = !b
 }
 
+op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
+  opts[:strict_config_value] = b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max
@@ -162,7 +166,7 @@ op.on('--conf-encoding ENCODING', "specify configuration file encoding") { |s|
 if Fluent.windows?
   require 'windows/library'
   include Windows::Library
-  
+
   opts.merge!(
     :winsvc_name => 'fluentdwinsvc',
     :winsvc_display_name => 'Fluentd Windows Service',
@@ -184,15 +188,15 @@ if Fluent.windows?
   op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
     opts[:fluentdopt] = s
   }
-  
+
   op.on('--winsvc-name NAME', "The Windows Service name to run as (Windows only)") {|s|
     opts[:winsvc_name] = s
   }
-  
+
   op.on('--winsvc-display-name DISPLAY_NAME', "The Windows Service display name (Windows only)") {|s|
     opts[:winsvc_display_name] = s
   }
-  
+
   op.on('--winsvc-desc DESC', "The Windows Service description (Windows only)") {|s|
     opts[:winsvc_desc] = s
   }
@@ -268,7 +272,7 @@ if winsvcinstmode = opts[:regwinsvc]
       start_service = true
     end
 
-    
+
     Service.create(
       service_name: opts[:winsvc_name],
       host: nil,
@@ -309,7 +313,6 @@ end
 
 exit 0 if early_exit
 
-require 'fluent/supervisor'
 if opts[:supervise]
   if Fluent.windows?
     if opts[:log_path] && opts[:log_path] != "-"
@@ -321,11 +324,16 @@ if opts[:supervise]
       end
     end
   end
-  Fluent::Supervisor.new(opts).run_supervisor
+
+  supervisor = Fluent::Supervisor.new(opts)
+  supervisor.configure(supervisor: true)
+  supervisor.run_supervisor(dry_run: opts[:dry_run])
 else
   if opts[:standalone_worker] && opts[:workers] && opts[:workers] > 1
     puts "Error: multi workers is not supported with --no-supervisor"
     exit 2
   end
-  Fluent::Supervisor.new(opts).run_worker
+  worker = Fluent::Supervisor.new(opts)
+  worker.configure
+  worker.run_worker
 end

data/lib/fluent/compat/call_super_mixin.rb

@@ -55,6 +55,15 @@ module Fluent
         end
       end
 
+      def stop
+        klass = self.class
+        @@_super_start.delete(klass)
+        @@_super_before_shutdown.delete(klass)
+        @@_super_shutdown.delete(klass)
+
+        super
+      end
+
       def shutdown
         super
         unless self.shutdown?

data/lib/fluent/compat/exec_util.rb

@@ -85,7 +85,7 @@ module Fluent
 
       class MessagePackParser < Parser
         def call(io)
-          @u = Fluent::Engine.msgpack_factory.unpacker(io)
+          @u = Fluent::MessagePackFactory.msgpack_unpacker(io)
           begin
             @u.each(&@on_message)
           rescue EOFError

data/lib/fluent/config/configure_proxy.rb

@@ -117,9 +117,9 @@ module Fluent
 
         merged = if self.root?
                    options[:root] = true
-                   self.class.new(other.name, options)
+                   self.class.new(other.name, **options)
                  else
-                   self.class.new(@name, options)
+                   self.class.new(@name, **options)
                  end
 
         # configured_in MUST be kept
@@ -172,9 +172,9 @@ module Fluent
 
         merged = if self.root?
                    options[:root] = true
-                   self.class.new(other.name, options)
+                   self.class.new(other.name, **options)
                  else
-                   self.class.new(@name, options)
+                   self.class.new(@name, **options)
                  end
 
         merged.configured_in_section = self.configured_in_section || other.configured_in_section

data/lib/fluent/config/element.rb

@@ -142,7 +142,7 @@ module Fluent
         indent = "  " * nest
         nindent = "  " * (nest + 1)
         out = ""
-        if @arg.empty?
+        if @arg.nil? || @arg.empty?
           out << "#{indent}<#{@name}>\n"
         else
           out << "#{indent}<#{@name} #{@arg}>\n"
@@ -194,23 +194,36 @@ module Fluent
         opts[:type]
       end
 
+      def default_value(key)
+        return nil if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        proxy = @corresponding_proxies.detect do |_proxy|
+          _proxy.params.has_key?(param_key)
+        end
+        return nil unless proxy
+        proxy.defaults[param_key]
+      end
+
       def dump_value(k, v, nindent)
-        if secret_param?(k)
-          "#{nindent}#{k} xxxxxx\n"
+        return "#{nindent}#{k} xxxxxx\n" if secret_param?(k)
+        return "#{nindent}#{k} #{v}\n" unless @v1_config
+
+        # for v1 config
+        if v.nil?
+          "#{nindent}#{k} \n"
+        elsif v == :default
+          "#{nindent}#{k} #{default_value(k)}\n"
         else
-          if @v1_config
-            case param_type(k)
-            when :string
-              "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
-            when :enum, :integer, :float, :size, :bool, :time
-              "#{nindent}#{k} #{v}\n"
-            when :hash, :array
-              "#{nindent}#{k} #{v}\n"
-            else
-              # Unknown type
-              "#{nindent}#{k} #{v}\n"
-            end
+          case param_type(k)
+          when :string
+            "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
+          when :enum, :integer, :float, :size, :bool, :time
+            "#{nindent}#{k} #{v}\n"
+          when :hash, :array
+            "#{nindent}#{k} #{v}\n"
           else
+            # Unknown type
             "#{nindent}#{k} #{v}\n"
           end
         end

data/lib/fluent/config/error.rb

@@ -23,4 +23,10 @@ module Fluent
 
   class ObsoletedParameterError < ConfigError
   end
+
+  class SetNil < Exception
+  end
+
+  class SetDefault < Exception
+  end
 end

data/lib/fluent/config/literal_parser.rb

@@ -52,6 +52,16 @@ module Fluent
       def initialize(strscan, eval_context)
         super(strscan)
         @eval_context = eval_context
+        unless @eval_context.respond_to?(:use_nil)
+          def @eval_context.use_nil
+            raise SetNil
+          end
+        end
+        unless @eval_context.respond_to?(:use_default)
+          def @eval_context.use_default
+            raise SetDefault
+          end
+        end
       end
 
       def parse_literal(string_boundary_charset = LINE_END)
@@ -81,7 +91,13 @@ module Fluent
         string = []
         while true
           if skip(/\"/)
-            return string.join
+            if string.include?(nil)
+              return nil
+            elsif string.include?(:default)
+              return :default
+            else
+              return string.join
+            end
           elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
             if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
               string << s
@@ -168,7 +184,13 @@ module Fluent
 hostname = Socket.gethostname
 worker_id = ENV['SERVERENGINE_WORKER_ID'] || ''
 EOM
-        @eval_context.instance_eval(code)
+        begin
+          @eval_context.instance_eval(code)
+        rescue SetNil => e
+          nil
+        rescue SetDefault => e
+          :default
+        end
       end
 
       def eval_escape_char(c)

data/lib/fluent/config/section.rb

@@ -106,7 +106,7 @@ module Fluent
     end
 
     module SectionGenerator
-      def self.generate(proxy, conf, logger, plugin_class, stack = [])
+      def self.generate(proxy, conf, logger, plugin_class, stack = [], strict_config_value = false)
         return nil if conf.nil?
 
         section_stack = ""
@@ -122,9 +122,23 @@ module Fluent
         end
 
         if proxy.argument
-          unless conf.arg.empty?
+          unless conf.arg.nil? || conf.arg.empty?
             key, block, opts = proxy.argument
-            section_params[key] = self.instance_exec(conf.arg, opts, name, &block)
+            opts = opts.merge(strict: true) if strict_config_value
+
+            if conf.arg == :default
+              unless section_params.has_key?(key)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{key}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[key] = self.instance_exec(conf.arg, opts, key, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
           end
           unless section_params.has_key?(proxy.argument.first)
             logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
@@ -136,13 +150,36 @@ module Fluent
         proxy.params.each_pair do |name, defval|
           varname = name.to_sym
           block, opts = defval
+          opts = opts.merge(strict: true) if strict_config_value
+
           if conf.has_key?(name.to_s) || opts[:alias] && conf.has_key?(opts[:alias].to_s)
             val = if conf.has_key?(name.to_s)
                     conf[name.to_s]
                   else
                     conf[opts[:alias].to_s]
                   end
-            section_params[varname] = self.instance_exec(val, opts, name, &block)
+
+            if val == :default
+              # default value is already set if it exists
+              unless section_params.has_key?(varname)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{varname}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[varname] = self.instance_exec(val, opts, name, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
+
+            if section_params[varname].nil?
+              unless proxy.defaults.has_key?(varname) and proxy.defaults[varname].nil?
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{name}' parameter is required but nil is specified"
+              end
+            end
 
             # Source of definitions of deprecated/obsoleted:
             # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
@@ -190,13 +227,13 @@ module Fluent
             raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
           end
           if subproxy.multi?
-            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name]) }
+            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name], strict_config_value) }
           else
             if elements.size > 1
               logger.error "config error in:\n#{conf}" if logger
               raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
             end
-            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name])
+            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name], strict_config_value)
           end
         end