fluentd 1.3.3 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4e5a7a4c71fe45e0a65fd941bea4093e8a6c5414
-  data.tar.gz: 6e4000e2928c6bfda990ea5566e4e32618e031f7
+  metadata.gz: ca1f6da9527b76b4421c1507ec0e2702687e196d
+  data.tar.gz: 8ae3c1fbac074dbc699bb36cb0975c78e94e3769
 SHA512:
-  metadata.gz: 94194178c70c31f9ae8f20f4c85230f1479f53dfbd25e5e8a13e724a5cf8ca7b38a7658a97153838ff486d554d0f44369bf2f622838af324c585594a3748d76d
-  data.tar.gz: 836807eb180ed4144f8333cb67c11d640b432882497e63d24d760a7a050f4513e137b6ac97bc6503d781e5690e80e26dee25792e6081b99f857f40787256e029
+  metadata.gz: 4a2298e9e76277c4ce41c7b43e239643a0b3ad8a953ab47bc242a9e22645e0d517853f73f8e37699ad5bbc57dc36e9d5cfb0504da155fb575c36eebf1893cfb7
+  data.tar.gz: 783fb04e180adc4711a6ebf83b91d6fa2d318300ca31ba0c49c8d247c2eceed4992b22be98765f7b30e316cd7815cf9aac7e81873b221c480f4df80f51a3fccb

data/.github/ISSUE_TEMPLATE.md

@@ -1,6 +1,8 @@
 Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
 
 - fluentd or td-agent version.
-- Environment information, e.g. OS.
+- Environment information:
+	- Operating system: `cat /etc/os-release`
+	- Kernel version: `uname -r`
 - Your configuration
 - Your problem explanation. If you have an error logs, write it together.

data/.travis.yml

@@ -11,8 +11,6 @@ matrix:
       os: linux
     - rvm: 2.2.10
       os: linux
-    - rvm: 2.3.7
-      os: linux
     - rvm: 2.4.5
       os: linux
     - rvm: 2.5.3
@@ -37,6 +35,8 @@ matrix:
       os: osx
       osx_image: xcode8.3 # OSX 10.12
   allow_failures:
+    - rvm: 2.3.8
+      os: linux
     - rvm: 2.1.10
       os: osx
       osx_image: xcode8.3

data/CHANGELOG.md

@@ -1,6 +1,37 @@
 # v1.3
 
-## Release v1.3.3 - 2018/01/06
+## Release v1.4.0 - 2019/02/24
+
+### New features
+
+* multiprocess: Support <worker N-M> syntax
+  https://github.com/fluent/fluentd/pull/2292
+* output: Work <secondary> and retry_forever together
+  https://github.com/fluent/fluentd/pull/2276
+* out_file: Support placeholders in symlink_path
+  https://github.com/fluent/fluentd/pull/2254
+
+### Enhancements
+
+* output: Add MessagePack unpacker error to unrecoverable error list
+  https://github.com/fluent/fluentd/pull/2301
+* output: Reduce flush deley when large timekey and small timekey_wait are specified
+  https://github.com/fluent/fluentd/pull/2291
+* config: Support embedded ruby code in section argument.
+  https://github.com/fluent/fluentd/pull/2291
+* in_tail: Improve encoding parameter handling
+  https://github.com/fluent/fluentd/pull/2305
+* in_tcp/in_udp: Add <parse> section check
+  https://github.com/fluent/fluentd/pull/2267
+
+### Bug fixes
+
+* server: Ignore IOError and related errors in UDP
+  https://github.com/fluent/fluentd/pull/2310
+* server: Ignore EPIPE in TLS accept to avoid fluentd restart
+  https://github.com/fluent/fluentd/pull/2253
+
+## Release v1.3.3 - 2019/01/06
 
 ### Enhancements
 

data/CONTRIBUTING.md

@@ -27,7 +27,7 @@ submitting an issue to Fluentd. Even better you can submit a Pull Request with a
 * **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs) repository.
 
 If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
-And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent releated issues.
+And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.
 
 Note: Before report the issue, check latest version first. Sometimes users report fixed bug with older version.
 
@@ -37,7 +37,7 @@ Here are some things that would increase a chance that your patch is accepted:
 
 * Write tests.
 * Run tests before send Pull Request by `bundle exec rake test`
-* Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+* Write a [good commit message](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
   * Fluentd repositories needs [DCO](https://github.com/apps/dco) on PR. Please add `Signed-off-by` to the commit(See DCO link for more detail).
 
 There are some patches which are hard to write tests, e.g. process handling, concurrency issue or etc.

data/MAINTAINERS.md

@@ -1,6 +1,6 @@
 # Fluentd Maintainers
 
-- [Naotoshi Seo](https://github.com/sonots), [DeNA](http://dena.com/intl/)
+- [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)
 - [Okkez](https://github.com/okkez), [Clearcode](https://www.clear-code.com/)
 - [Hiroshi Hatake](https://github.com/cosmo0920), [Clearcode](https://www.clear-code.com/)
 - [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)

data/README.md

@@ -72,7 +72,7 @@ Many enterprises run Fluentd in production to handle all of their logging needs.
 - Discussion: https://groups.google.com/group/fluentd
 - Slack / Community: https://slack.fluentd.org
 - Newsletters: https://www.fluentd.org/newsletter_signup
-- Author: Sadayuki Furuhashi
+- Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
 - Copyright: 2011-2018 Fluentd Authors
 - License: Apache License, Version 2.0
 

data/appveyor.yml

@@ -30,14 +30,9 @@ environment:
       devkit: C:\Ruby23\DevKit
     - ruby_version: "22-x64"
       devkit: C:\Ruby23-x64\DevKit
-    - ruby_version: "21-x64"
-      devkit: C:\Ruby23-x64\DevKit
     - ruby_version: "22"
       devkit: C:\Ruby23\DevKit
       WIN_RAPID: true
-    - ruby_version: "21"
-      devkit: C:\Ruby23\DevKit
-      WIN_RAPID: true
 matrix:
   allow_failures:
     - ruby_version: "21"

data/lib/fluent/config/element.rb

@@ -36,12 +36,12 @@ module Fluent
         # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
         @logger = defined?($log) ? $log : nil
 
-        @target_worker_id = nil
+        @target_worker_ids = []
       end
 
       attr_accessor :name, :arg, :unused, :v1_config, :corresponding_proxies, :unused_in
       attr_writer :elements
-      attr_reader :target_worker_id
+      attr_reader :target_worker_ids
 
       RESERVED_PARAMETERS_COMPAT = {
         '@type' => 'type',
@@ -223,22 +223,29 @@ module Fluent
       end
 
       def set_target_worker_id(worker_id)
-        @target_worker_id = worker_id
+        @target_worker_ids = [worker_id]
         @elements.each { |e|
           e.set_target_worker_id(worker_id)
         }
       end
 
+      def set_target_worker_ids(worker_ids)
+        @target_worker_ids = worker_ids.uniq
+        @elements.each { |e|
+          e.set_target_worker_ids(worker_ids.uniq)
+        }
+      end
+
       def for_every_workers?
-        @target_worker_id == nil
+        @target_worker_ids.empty?
       end
 
       def for_this_worker?
-        @target_worker_id == Fluent::Engine.worker_id
+        @target_worker_ids.include?(Fluent::Engine.worker_id)
       end
 
       def for_another_worker?
-        @target_worker_id != nil && @target_worker_id != Fluent::Engine.worker_id
+        !@target_worker_ids.empty? && !@target_worker_ids.include?(Fluent::Engine.worker_id)
       end
     end
   end

data/lib/fluent/config/v1_parser.rb

@@ -82,7 +82,7 @@ module Fluent
           elsif skip(/\</)
             e_name = scan(ELEMENT_NAME)
             spacing
-            e_arg = scan_nonquoted_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
+            e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
             spacing
             unless skip(/\>/)
               parse_error! "expected '>'"

data/lib/fluent/plugin/base.rb

@@ -52,7 +52,12 @@ module Fluent
 
       def configure(conf)
         if conf.respond_to?(:for_this_worker?) && conf.for_this_worker?
-          system_config_override(workers: 1)
+          workers = if conf.target_worker_ids && !conf.target_worker_ids.empty?
+                      conf.target_worker_ids.size
+                    else
+                      1
+                    end
+          system_config_override(workers: workers)
         end
         super
         @_state ||= State.new(false, false, false, false, false, false, false, false, false)

data/lib/fluent/plugin/in_http.rb

@@ -163,6 +163,7 @@ module Fluent::Plugin
 
         # Skip nil record
         if record.nil?
+          log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
           if @respond_with_empty_img
             return ["200 OK", {'Content-Type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
           else

data/lib/fluent/plugin/in_tail.rb

@@ -171,6 +171,9 @@ module Fluent::Plugin
 
       @encoding = parse_encoding_param(@encoding) if @encoding
       @from_encoding = parse_encoding_param(@from_encoding) if @from_encoding
+      if @encoding == @from_encoding
+        log.warn "'encoding' and 'from_encoding' are same encoding. No effect"
+      end
     end
 
     def parse_encoding_param(encoding_name)
@@ -657,6 +660,7 @@ module Fluent::Plugin
         def initialize(from_encoding, encoding)
           @from_encoding = from_encoding
           @encoding = encoding
+          @need_enc = from_encoding != encoding
           @buffer = ''.force_encoding(from_encoding)
           @eol = "\n".encode(from_encoding).freeze
         end
@@ -682,11 +686,13 @@ module Fluent::Plugin
         end
 
         def convert(s)
-          if @from_encoding == @encoding
-            s
+          if @need_enc
+            s.encode!(@encoding, @from_encoding)
           else
-            s.encode(@encoding, @from_encoding)
+            s
           end
+        rescue
+          s.encode!(@encoding, @from_encoding, :invalid => :replace, :undef => :replace)
         end
 
         def next_line

data/lib/fluent/plugin/in_tcp.rb

@@ -41,11 +41,15 @@ module Fluent::Plugin
 
     def configure(conf)
       compat_parameters_convert(conf, :parser)
+      parser_config = conf.elements('parse').first
+      unless parser_config
+        raise Fluent::ConfigError, "<parse> section is required."
+      end
       super
       @_event_loop_blocking_timeout = @blocking_timeout
       @source_hostname_key ||= @source_host_key if @source_host_key
 
-      @parser = parser_create
+      @parser = parser_create(conf: parser_config)
     end
 
     def multi_workers_ready?

data/lib/fluent/plugin/in_udp.rb

@@ -47,12 +47,16 @@ module Fluent::Plugin
 
     def configure(conf)
       compat_parameters_convert(conf, :parser)
+      parser_config = conf.elements('parse').first
+      unless parser_config
+        raise Fluent::ConfigError, "<parse> section is required."
+      end
       super
       @_event_loop_blocking_timeout = @blocking_timeout
       @source_hostname_key ||= @source_host_key if @source_host_key
       @message_length_limit = @body_size_limit if @body_size_limit
 
-      @parser = parser_create
+      @parser = parser_create(conf: parser_config)
     end
 
     def multi_workers_ready?

data/lib/fluent/plugin/out_file.rb

@@ -71,6 +71,10 @@ module Fluent::Plugin
     attr_accessor :last_written_path # for tests
 
     module SymlinkBufferMixin
+      def output_plugin_for_symlink=(output_plugin)
+        @_output_plugin_for_symlink = output_plugin
+      end
+
       def symlink_path=(path)
         @_symlink_path = path
       end
@@ -83,7 +87,7 @@ module Fluent::Plugin
         # These chunks will be enqueued immediately, and will be flushed soon.
         latest_metadata = metadata_list.select{|m| m.timekey }.sort_by(&:timekey).last
         if chunk.metadata == latest_metadata
-          FileUtils.ln_sf(chunk.path, @_symlink_path)
+          FileUtils.ln_sf(chunk.path, @_output_plugin_for_symlink.extract_placeholders(@_symlink_path, chunk))
         end
         chunk
       end
@@ -161,6 +165,7 @@ module Fluent::Plugin
         else
           @buffer.extend SymlinkBufferMixin
           @buffer.symlink_path = @symlink_path
+          @buffer.output_plugin_for_symlink = self
         end
       end
 

data/lib/fluent/plugin/output.rb

@@ -364,7 +364,9 @@ module Fluent
           raise Fluent::ConfigError, "Invalid <secondary> section for non-buffered plugin" unless @buffering
           raise Fluent::ConfigError, "<secondary> section cannot have <buffer> section" if @secondary_config.buffer
           raise Fluent::ConfigError, "<secondary> section cannot have <secondary> section" if @secondary_config.secondary
-          raise Fluent::ConfigError, "<secondary> section and 'retry_forever' are exclusive" if @buffer_config.retry_forever
+          if @buffer_config.retry_forever
+            log.warn "<secondary> with 'retry_forever', only unrecoverable errors are moved to secondary"
+          end
 
           secondary_type = @secondary_config[:@type]
           unless secondary_type
@@ -1081,7 +1083,7 @@ module Fluent
         end
       end
 
-      UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError]
+      UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError, MessagePack::UnpackError]
 
       def try_flush
         chunk = @buffer.dequeue_chunk
@@ -1334,7 +1336,7 @@ module Fluent
         end
         if @chunk_key_time
           if !value_for_interval || @buffer_config.timekey < value_for_interval
-            value_for_interval = @buffer_config.timekey
+            value_for_interval = [@buffer_config.timekey, @buffer_config.timekey_wait].min
           end
         end
         unless value_for_interval

data/lib/fluent/plugin_helper/retry_state.rb

@@ -53,10 +53,6 @@ module Fluent
           @randomize = randomize
           @randomize_width = randomize_width
 
-          if forever && secondary
-            raise "BUG: forever and secondary are exclusive to each other"
-          end
-
           @forever = forever
           @max_steps = max_steps
 
@@ -118,12 +114,12 @@ module Fluent
         end
 
         def secondary?
-          @secondary && (@current == :secondary || current_time >= @secondary_transition_at)
+          !@forever && @secondary && (@current == :secondary || current_time >= @secondary_transition_at)
         end
 
         def step
           @steps += 1
-          if @secondary && @current != :secondary && current_time >= @secondary_transition_at
+          if !@forever && @secondary && @current != :secondary && current_time >= @secondary_transition_at
             @current = :secondary
             @secondary_transition_steps = @steps
           end

data/lib/fluent/plugin_helper/server.rb

@@ -523,7 +523,7 @@ module Fluent
           def on_readable_without_sock
             begin
               data = @sock.recv(@max_bytes, @flags)
-            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET
+            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
             end
             @callback.call(data)
@@ -536,7 +536,7 @@ module Fluent
           def on_readable_with_sock
             begin
               data, addr = @sock.recvfrom(@max_bytes)
-            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET
+            rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
             end
             @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
@@ -722,7 +722,7 @@ module Fluent
 
                 return true
               end
-            rescue Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
+            rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
               @log.trace "unexpected error before accepting TLS connection", error: e
               close rescue nil
             end

data/lib/fluent/root_agent.rb

@@ -64,26 +64,64 @@ module Fluent
     attr_reader :labels
 
     def configure(conf)
+      used_worker_ids = []
+      available_worker_ids = (0..Fluent::Engine.system_config.workers - 1).to_a
       # initialize <worker> elements
       conf.elements(name: 'worker').each do |e|
         target_worker_id_str = e.arg
         if target_worker_id_str.empty?
-          raise ConfigError, "Missing worker id on <worker> directive"
+          raise Fluent::ConfigError, "Missing worker id on <worker> directive"
         end
 
-        target_worker_id = target_worker_id_str.to_i
-        if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
-          raise ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
-        end
+        target_worker_ids = target_worker_id_str.split("-")
+        if target_worker_ids.size == 2
+          first_worker_id = target_worker_ids.first.to_i
+          last_worker_id = target_worker_ids.last.to_i
+          if first_worker_id > last_worker_id
+            raise Fluent::ConfigError, "greater first_worker_id<#{first_worker_id}> than last_worker_id<#{last_worker_id}> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"
+          end
+          target_worker_ids = []
+          first_worker_id.step(last_worker_id, 1) do |worker_id|
+            target_worker_id = worker_id.to_i
+            target_worker_ids << target_worker_id
+
+            if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
+              raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
+            end
+            available_worker_ids.delete(target_worker_id) if available_worker_ids.include?(target_worker_id)
+            if used_worker_ids.include?(target_worker_id) && !Fluent::Engine.dry_run_mode
+              raise Fluent::ConfigError, "specified worker_id<#{worker_id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}"
+            end
+            used_worker_ids << target_worker_id
+
+            e.elements.each do |elem|
+              unless ['source', 'match', 'filter', 'label'].include?(elem.name)
+                raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
+              end
+            end
 
-        ## On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
-        target_worker_id = 0 if Fluent::Engine.dry_run_mode
+            # On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
+            target_worker_ids = [0] if Fluent::Engine.dry_run_mode
 
-        e.elements.each do |elem|
-          unless ['source', 'match', 'filter', 'label'].include?(elem.name)
-            raise ConfigError, "<worker> section cannot have <#{elem.name}> directive"
+            unless target_worker_ids.empty?
+              e.set_target_worker_ids(target_worker_ids.uniq)
+            end
+          end
+        else
+          target_worker_id = target_worker_id_str.to_i
+          if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
+            raise Fluent::ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
+          end
+
+          ## On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
+          target_worker_id = 0 if Fluent::Engine.dry_run_mode
+
+          e.elements.each do |elem|
+            unless ['source', 'match', 'filter', 'label'].include?(elem.name)
+              raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
+            end
+            elem.set_target_worker_id(target_worker_id)
           end
-          elem.set_target_worker_id(target_worker_id)
         end
         conf += e
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.3.3'
+  VERSION = '1.4.0'
 
 end

data/test/command/test_fluentd.rb

@@ -340,7 +340,7 @@ CONF
     end
   end
 
-  sub_test_case 'configured to suppress configration dump' do
+  sub_test_case 'configured to suppress configuration dump' do
     setup do
       @basic_conf = <<CONF
 <source>
@@ -614,7 +614,7 @@ CONF
       )
     end
 
-    test 'failed to start workers when configured plugins as chidren of MultiOutput do not support multi worker configuration' do
+    test 'failed to start workers when configured plugins as children of MultiOutput do not support multi worker configuration' do
       script = <<-EOC
 require 'fluent/plugin/output'
 module Fluent::Plugin

data/test/command/test_plugin_config_formatter.rb

@@ -72,7 +72,7 @@ class TestFluentPluginConfigFormatter < Test::Unit::TestCase
       desc "username"
       config_param :username, :string
       desc "password"
-      config_param :passowrd, :string, secret: true
+      config_param :password, :string, secret: true
     end
 
     config_section :parent do
@@ -162,7 +162,7 @@ slow_flush_log_threshold: float: (20.0)
  <secondary>: optional, single
 <authentication>: required, single
  username: string: (nil)
- passowrd: string: (nil)
+ password: string: (nil)
 <parent>: optional, multiple
  <child>: optional, multiple
   names: array: (nil)
@@ -217,7 +217,7 @@ TEXT
 
 username
 
-#### passowrd (string) (required)
+#### password (string) (required)
 
 password
 

data/test/config/test_config_parser.rb

@@ -329,6 +329,36 @@ module Fluent::Config
       end
     end
 
+    sub_test_case "Embedded Ruby Code in section attributes" do
+      setup do
+        ENV["EMBEDDED_VAR"] = "embedded"
+        ENV["NESTED_EMBEDDED_VAR"] = "nested-embedded"
+        @hostname = Socket.gethostname
+      end
+
+      teardown do
+        ENV["EMBEDDED_VAR"] = nil
+        ENV["NESTED_EMBEDDED_VAR"] = nil
+      end
+
+      test "embedded Ruby code should be expanded" do
+        assert_text_parsed_as(root(
+          e("test", 'embedded', {'key'=>'1'}, [
+            e('nested1', 'nested-embedded'),
+            e('nested2', "#{@hostname}")
+          ])), <<-EOF
+          <test "#{ENV["EMBEDDED_VAR"]}">
+            key 1
+            <nested1 "#{ENV["NESTED_EMBEDDED_VAR"]}">
+            </nested1>
+            <nested2 "#{Socket.gethostname}">
+            </nested2>
+          </test>
+        EOF
+        )
+      end
+    end
+
     # port from test_config.rb
     sub_test_case '@include parsing' do
       TMP_DIR = File.dirname(__FILE__) + "/tmp/v1_config#{ENV['TEST_ENV_NUMBER']}"

data/test/config/test_configurable.rb

@@ -733,7 +733,7 @@ module Fluent::Config
           assert_nothing_raised { b4.configure(config_element('ROOT', '', BASE_ATTRS, [d1.dup, d2.dup, d3.dup, d3.dup])) }
         end
 
-        test 'constructs confuguration object tree for Base3' do
+        test 'constructs configuration object tree for Base3' do
           conf = config_element(
             'ROOT',
             '',

data/test/config/test_element.rb

@@ -157,7 +157,7 @@ class TestConfigElement < ::Test::Unit::TestCase
                              Fluent::Config::Element.new('ROOT', 'mydata', {}, [])],
            "differ keys" => [Fluent::Config::Element.new('ROOT', 'mydata', {}, []),
                              Fluent::Config::Element.new('ROOT', 'mydata', {"k1" => "v1"}, [])],
-           "differ elemnts" =>
+           "differ elements" =>
            [Fluent::Config::Element.new('ROOT', 'mydata', {"k1" => "v1"}, []),
             Fluent::Config::Element.new('ROOT', 'mydata', {"k1" => "v1"}, [
               Fluent::Config::Element.new('test', 'mydata', {'k3' => 'v3'}, [])
@@ -406,11 +406,11 @@ CONF
     test 'set target_worker_id recursively' do
       e = element('label', '@mytest', {}, [ element('filter', '**'), element('match', '**', {}, [ element('store'), element('store') ]) ])
       e.set_target_worker_id(1)
-      assert_equal 1, e.target_worker_id
-      assert_equal 1, e.elements[0].target_worker_id
-      assert_equal 1, e.elements[1].target_worker_id
-      assert_equal 1, e.elements[1].elements[0].target_worker_id
-      assert_equal 1, e.elements[1].elements[1].target_worker_id
+      assert_equal [1], e.target_worker_ids
+      assert_equal [1], e.elements[0].target_worker_ids
+      assert_equal [1], e.elements[1].target_worker_ids
+      assert_equal [1], e.elements[1].elements[0].target_worker_ids
+      assert_equal [1], e.elements[1].elements[1].target_worker_ids
     end
   end
 
@@ -434,12 +434,24 @@ CONF
       assert e.for_this_worker?
     end
 
+    test 'target_worker_ids includes current worker_id' do
+      e = element()
+      e.set_target_worker_ids([0])
+      assert e.for_this_worker?
+    end
+
     test 'target_worker_id != current worker_id' do
       e = element()
       e.set_target_worker_id(1)
       assert_false e.for_this_worker?
     end
 
+    test 'target_worker_ids does not includes current worker_id' do
+      e = element()
+      e.set_target_worker_ids([1, 2])
+      assert_false e.for_this_worker?
+    end
+
     test "doesn't have target_worker_id" do
       e = element()
       assert_false e.for_this_worker?
@@ -453,12 +465,24 @@ CONF
       assert_false e.for_another_worker?
     end
 
+    test 'target_worker_ids contains current worker_id' do
+      e = element()
+      e.set_target_worker_ids([0, 1])
+      assert_false e.for_another_worker?
+    end
+
     test 'target_worker_id != current worker_id' do
       e = element()
       e.set_target_worker_id(1)
       assert e.for_another_worker?
     end
 
+    test 'target_worker_ids does not contains current worker_id' do
+      e = element()
+      e.set_target_worker_ids([1, 2])
+      assert e.for_another_worker?
+    end
+
     test "doesn't have target_worker_id" do
       e = element()
       assert_false e.for_another_worker?

data/test/counter/test_store.rb

@@ -89,7 +89,7 @@ class CounterStoreTest < ::Test::Unit::TestCase
       assert_equal nil, @store.get('unknown_key')
     end
 
-    test "raise a error when when a passed key doesn't exist and raise_error option is true" do
+    test "raise a error when a passed key doesn't exist and raise_error option is true" do
       assert_raise Fluent::Counter::UnknownKey do
         @store.get('unknown_key', raise_error: true)
       end

data/test/plugin/test_buffer.rb

@@ -211,7 +211,7 @@ class BufferTest < Test::Unit::TestCase
       assert_equal 2, @p.queued_num[@dm1]
     end
 
-    test '#close closes all chunks in in dequeued, enqueued and staged' do
+    test '#close closes all chunks in dequeued, enqueued and staged' do
       dmx = create_metadata(Time.parse('2016-04-11 15:50:00 +0000').to_i, nil, nil)
       cx = create_chunk(dmx, ["x" * 1024])
       @p.dequeued[cx.unique_id] = cx
@@ -1027,7 +1027,7 @@ class BufferTest < Test::Unit::TestCase
       ##### 900 + 9500 + 9900 * 4 == 5000 + 45000
     end
 
-    test '#write raises BufferChunkOverflowError if a record is biggar than chunk limit size' do
+    test '#write raises BufferChunkOverflowError if a record is bigger than chunk limit size' do
       assert_equal [@dm0], @p.stage.keys
       assert_equal [], @p.queue.map(&:metadata)
 

data/test/plugin/test_filter.rb

@@ -47,7 +47,7 @@ module FluentPluginFilterTest
     end
   end
   class InvalidPlugin < Fluent::Plugin::Filter
-    # Because of implemnting `filter_with_time` and `filter` methods
+    # Because of implementing `filter_with_time` and `filter` methods
     def filter_with_time(tag, time, record); end
     def filter(tag, time, record); end
   end
@@ -277,7 +277,7 @@ class FilterPluginTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'filter plugins that is implmented `filter_with_time`' do
+  sub_test_case 'filter plugins that is implemented `filter_with_time`' do
     setup do
       Fluent::Test.setup
       @p = FluentPluginFilterTest::NumDoublePluginWithTime.new

data/test/plugin/test_filter_record_transformer.rb

@@ -229,7 +229,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     end
 
     %w[yes no].each do |enable_ruby|
-      test "hostname with enble_ruby #{enable_ruby}" do
+      test "hostname with enable_ruby #{enable_ruby}" do
         config = %[
           enable_ruby #{enable_ruby}
           <record>

data/test/plugin/test_in_tail.rb

@@ -649,6 +649,26 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
   end
 
+  def test_encoding_with_bad_character
+    conf = config_element(
+      "", "", {
+        "format" => "/(?<message>.*)/",
+        "read_from_head" => "true",
+        "from_encoding" => "ASCII-8BIT",
+        "encoding" => "utf-8"
+      })
+    d = create_driver(conf)
+
+    d.run(expect_emits: 1) do
+      File.open("#{TMP_DIR}/tail.txt", "w") { |f|
+        f.write "te\x86st\n"
+      }
+    end
+
+    events = d.events
+    assert_equal("te\uFFFDst", events[0][2]['message'])
+    assert_equal(Encoding::UTF_8, events[0][2]['message'].encoding)
+  end
 
   sub_test_case "multiline" do
     data(flat: MULTILINE_CONFIG,

data/test/plugin/test_in_tcp.rb

@@ -48,6 +48,12 @@ class TcpInputTest < Test::Unit::TestCase
     assert_equal "\n", d.instance.delimiter
   end
 
+  test ' configure w/o parse section' do
+    assert_raise(Fluent::ConfigError.new("<parse> section is required.")) {
+      create_driver(BASE_CONFIG)
+    }
+  end
+
   test_case_data = {
     'none' => {
       'format' => 'none',

data/test/plugin/test_in_udp.rb

@@ -59,6 +59,12 @@ class UdpInputTest < Test::Unit::TestCase
     assert_equal nil, d.instance.receive_buffer_size
   end
 
+  test ' configure w/o parse section' do
+    assert_raise(Fluent::ConfigError.new("<parse> section is required.")) {
+      create_driver(BASE_CONFIG)
+    }
+  end
+
   data(
     'ipv4' => [CONFIG, '127.0.0.1', :ipv4],
     'ipv6' => [IPv6_CONFIG, '::1', :ipv6],

data/test/plugin/test_out_file.rb

@@ -14,7 +14,6 @@ class FileOutputTest < Test::Unit::TestCase
   end
 
   TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file#{ENV['TEST_ENV_NUMBER']}")
-  SYMLINK_PATH = File.expand_path("#{TMP_DIR}/current")
 
   CONFIG = %[
     path #{TMP_DIR}/out_file_test
@@ -663,15 +662,43 @@ class FileOutputTest < Test::Unit::TestCase
     end
   end
 
-  test 'symlink' do
-    omit "Windows doesn't support symlink" if Fluent.windows?
-    conf = CONFIG + %[
-      symlink_path #{SYMLINK_PATH}
-    ]
-    symlink_path = "#{SYMLINK_PATH}"
+  SYMLINK_PATH = File.expand_path("#{TMP_DIR}/current")
+
+  sub_test_case 'symlink' do
+    test 'static symlink' do
+      omit "Windows doesn't support symlink" if Fluent.windows?
+      conf = CONFIG + %[
+        symlink_path #{SYMLINK_PATH}
+      ]
+      symlink_path = "#{SYMLINK_PATH}"
+
+      d = create_driver(conf)
+      begin
+        run_and_check(d, symlink_path)
+      ensure
+        FileUtils.rm_rf(symlink_path)
+      end
+    end
+
+    test 'symlink with placeholders' do
+      omit "Windows doesn't support symlink" if Fluent.windows?
+      conf = %[
+        path #{TMP_DIR}/${tag}/out_file_test
+        symlink_path #{SYMLINK_PATH}-${tag}
+        <buffer tag,time>
+        </buffer>
+      ]
+      symlink_path = "#{SYMLINK_PATH}-tag"
+
+      d = create_driver(conf)
+      begin
+        run_and_check(d, symlink_path)
+      ensure
+        FileUtils.rm_rf(symlink_path)
+      end
+    end
 
-    d = create_driver(conf)
-    begin
+    def run_and_check(d, symlink_path)
       d.run(default_tag: 'tag') do
         es = Fluent::OneEventStream.new(event_time("2011-01-02 13:14:15 UTC"), {"a"=>1})
         d.feed(es)
@@ -688,8 +715,6 @@ class FileOutputTest < Test::Unit::TestCase
         meta = d.instance.metadata('tag', event_time("2011-01-03 14:15:16 UTC"), {})
         assert_equal d.instance.buffer.instance_eval{ @stage[meta].path }, File.readlink(symlink_path)
       end
-    ensure
-      FileUtils.rm_rf(symlink_path)
     end
   end
 

data/test/plugin/test_out_secondary_file.rb

@@ -39,7 +39,7 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
     c.configure(conf)
   end
 
-  sub_test_case 'configture' do
+  sub_test_case 'configure' do
     test 'default configuration' do
       d = create_driver %[directory #{TMP_DIR}]
       assert_equal 'dump.bin', d.instance.basename

data/test/plugin/test_output.rb

@@ -547,7 +547,7 @@ class OutputTest < Test::Unit::TestCase
       assert_equal 86400, s
       assert_equal :day, t
       assert_equal '%d', e
-      s, t, e = @i.get_placeholders_time("my birthiday! at %F")
+      s, t, e = @i.get_placeholders_time("my birthday! at %F")
       assert_equal 86400, s
       assert_equal :day, t
       assert_equal '%d', e

data/test/plugin/test_output_as_buffered.rb

@@ -1074,6 +1074,44 @@ class BufferedOutputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'buffered output with large timekey and small timekey_wait' do
+    setup do
+      chunk_key = 'time'
+      hash = {
+        'timekey' => 86400, # per 1 day
+        'timekey_wait' => 10, # 10 seconds delay for flush
+        'flush_thread_count' => 1,
+        'flush_thread_burst_interval' => 0.01,
+      }
+      @i = create_output(:buffered)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
+      @i.start
+      @i.after_start
+    end
+
+    test '#configure raises config error if timekey is not specified' do
+      Timecop.freeze( Time.parse('2019-02-08 00:01:00 +0900') )
+      ary = []
+      @i.register(:write){|chunk| ary << chunk.read }
+      @i.thread_wait_until_start
+      events = [
+        [event_time('2019-02-08 00:02:00 +0900'), {"message" => "foobar"}]
+      ]
+      @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
+      @i.enqueue_thread_wait
+      assert{ @i.write_count == 0 }
+
+      Timecop.freeze( Time.parse('2019-02-09 00:00:08 +0900') )
+      @i.enqueue_thread_wait
+      assert{ @i.write_count == 0 }
+
+      Timecop.freeze( Time.parse('2019-02-09 00:00:12 +0900') )
+      # wirte should be called in few seconds since
+      # running interval of enque thread is timekey_wait / 11.0.
+      waiting(5){ sleep 0.1 until @i.write_count == 1 }
+    end
+  end
+
   sub_test_case 'buffered output feature with tag key' do
     setup do
       chunk_key = 'tag'

data/test/plugin/test_output_as_buffered_backup.rb

@@ -177,7 +177,12 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
       }
     end
 
-    test 'backup chunk without secondary' do
+    data('unrecoverable error' => Fluent::UnrecoverableError,
+         'type error' => TypeError,
+         'argument error' => ArgumentError,
+         'no method error' => NoMethodError,
+         'msgpack unpack error' => MessagePack::UnpackError)
+    test 'backup chunk without secondary' do |error_class|
       Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
         id = 'backup_test'
         hash = {
@@ -188,7 +193,7 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
         @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
         @i.register(:write) { |chunk|
           chunk_id = chunk.unique_id;
-          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+          raise error_class, "yay, your #write must fail"
         }
 
         flush_chunks

data/test/plugin/test_output_as_buffered_retries.rb

@@ -706,16 +706,20 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
   end
 
   sub_test_case 'buffered output configured as retry_forever' do
-    test 'configuration error will be raised if secondary section is configured' do
+    setup do
+      Fluent::Plugin.register_output('output_retries_secondary_test', FluentPluginOutputAsBufferedRetryTest::DummyFullFeatureOutput2)
+    end
+
+    test 'warning logs are generated if secondary section is configured' do
       chunk_key = 'tag'
       hash = {
         'retry_forever' => true,
         'retry_randomize' => false,
       }
       i = create_output()
-      assert_raise Fluent::ConfigError do
-        i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash),config_element('secondary','')]))
-      end
+      i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash),config_element('secondary','', {'@type' => 'output_retries_secondary_test'})]))
+      logs = i.log.out.logs
+      assert { logs.any? { |l| l.include?("<secondary> with 'retry_forever', only unrecoverable errors are moved to secondary") } }
     end
 
     test 'retry_timeout and retry_max_times will be ignored if retry_forever is true for exponential backoff' do

data/test/plugin/test_parser_syslog.rb

@@ -276,7 +276,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
     def test_auto_with_legacy_syslog_message
       @parser.configure(
                         'time_format' => '%b %d %M:%S:%H',
-                        'mseeage_format' => 'auto',
+                        'message_format' => 'auto',
                         )
       text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|
@@ -290,7 +290,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
       @parser.configure(
                         'time_format' => '%b %d %M:%S:%H',
                         'with_priority' => true,
-                        'mseeage_format' => 'auto',
+                        'message_format' => 'auto',
                         )
       text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
       @parser.instance.parse(text) do |time, record|

data/test/plugin_helper/test_record_accessor.rb

@@ -48,7 +48,7 @@ class RecordAccessorHelperTest < Test::Unit::TestCase
 
     data("missing ']'" => "$['key1'",
          "missing array index with dot" => "$.hello[]",
-         "missing array index with braket" => "$[]",
+         "missing array index with bracket" => "$[]",
          "more chars" => "$.key1[0]foo",
          "whitespace char included key in dot notation" => "$.key[0].ke y",
          "empty keys with dot" => "$.",

data/test/plugin_helper/test_server.rb

@@ -767,7 +767,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
   def write_cert_and_key(cert_path, cert, key_path, key, passphrase)
     File.open(cert_path, "w"){|f| f.write(cert.to_pem) }
-    # Write the secret key (raw or ecnrypted by AES256) in PEM format
+    # Write the secret key (raw or encrypted by AES256) in PEM format
     key_str = passphrase ? key.export(OpenSSL::Cipher.new("AES-256-CBC"), passphrase) : key.export
     File.open(key_path, "w"){|f| f.write(key_str) }
     File.chmod(0600, cert_path, key_path)

data/test/test_output.rb

@@ -129,7 +129,7 @@ module FluentOutputTest
     end
 
     def test_secondary_with_no_warn_log
-      # ObjectBufferedOutput doesn't implemnt `custom_filter`
+      # ObjectBufferedOutput doesn't implement `custom_filter`
       d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)
 
       mock(d.instance.log).warn("secondary type should be same with primary one",

data/test/test_plugin.rb

@@ -97,7 +97,7 @@ class PluginTest < Test::Unit::TestCase
       output1: ['plugin_test_dummy1', Dummy1Output, :new_output],
       output2: ['plugin_test_dummy2', Dummy2Output, :new_output],
     )
-    test 'retruns plugin instances of registered plugin classes' do |(type, klass, m)|
+    test 'returns plugin instances of registered plugin classes' do |(type, klass, m)|
       instance = Fluent::Plugin.__send__(m, type)
       assert_kind_of klass, instance
     end

data/test/test_root_agent.rb

@@ -686,6 +686,54 @@ EOC
       end
     end
 
+    test 'raises configuration error for too big worker id on multi workers syntax' do
+      errmsg = "worker id 4 specified by <worker> directive is not allowed. Available worker id is between 0 and 3"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        conf = <<-EOC
+<worker 1-4>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for worker id collisions on multi workers syntax' do
+      errmsg = "specified worker_id<2> collisions is detected on <worker> directive. Available worker id(s): [3]"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        conf = <<-EOC
+<worker 0-2>
+</worker>
+<worker 2-4>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for worker id collisions on multi workers syntax when multi avaliable worker_ids are left' do
+      errmsg = "specified worker_id<1> collisions is detected on <worker> directive. Available worker id(s): [2, 3]"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        conf = <<-EOC
+<worker 0-1>
+</worker>
+<worker 1-3>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for too big worker id on invalid reversed multi workers syntax' do
+      errmsg = "greater first_worker_id<3> than last_worker_id<0> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        conf = <<-EOC
+<worker 3-0>
+</worker>
+EOC
+        configure_ra(conf)
+      end
+    end
+
     test 'raises configuration error for invalid elements as a child of worker section' do
       errmsg = '<worker> section cannot have <system> directive'
       assert_raise Fluent::ConfigError.new(errmsg) do
@@ -844,5 +892,33 @@ EOC
       assert_equal 0, ra.labels.size
       refute ra.error_collector
     end
+
+    test 'with plugins for workers syntax should match worker_id equals to 2' do
+      conf = <<-EOC
+<worker 0-2>
+  <source>
+    @type forward
+  </source>
+  <filter **>
+    @type test_filter
+    @id test_filter
+  </filter>
+  <match pattern>
+    @type stdout
+  </match>
+  <label @ERROR>
+    <match>
+      @type null
+    </match>
+  </label>
+</worker>
+EOC
+
+      ra = configure_ra(conf)
+      assert_kind_of Fluent::Plugin::ForwardInput, ra.inputs.first
+      assert_kind_of Fluent::Plugin::StdoutOutput, ra.outputs.first
+      assert_kind_of FluentTestFilter, ra.filters.first
+      assert ra.error_collector
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.4.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-07 00:00:00.000000000 Z
+date: 2019-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack