fluentd 1.2.6 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8429bf1a0e35278c81b67e3d517568038a67fb2c
-  data.tar.gz: ba5260ae4f5db3933aebbf2ba598f0c03d9ab878
+  metadata.gz: c973680de36bad1981b1cc6bd3c23c391a04fc66
+  data.tar.gz: eb6b2c87bd06d19c2d90050355059e6030f47c14
 SHA512:
-  metadata.gz: 575ca7960c2f0560d103001eecbbaf43763394c238280a4aebc4b8ab61336a069d29917c8dca25be95a2b78557c80e172e3dc0834ed8d165046bc87c822be426
-  data.tar.gz: be8459ae73c470d11848afc2f5cdc5e4f57a9a74606715e3491a7660e1fd4e362b899c5c7ccbe72504c74bcf1c4d818c6cbb00db37707529e6b30dd4d98c13cf
+  metadata.gz: 390c4766aca79e9e3e235fad17b66e5ab0d979df6d77e91ac7f03b4be9a831b60214bef63a5cf0811548a8020f4954df4f27beda58dedeb5948a0f804b6a1c42
+  data.tar.gz: cef9e1fcf326748d4a72420c5ae4f6b50f3ed8af8178ba0f405ee667a675435121de90099a27170d96ac9592ed421e71655a40b2060a1967f58f62e1c311c047

data/.travis.yml

@@ -9,13 +9,13 @@ matrix:
   include:
     - rvm: 2.1.10
       os: linux
-    - rvm: 2.2.6
+    - rvm: 2.2.10
       os: linux
-    - rvm: 2.3.3
+    - rvm: 2.3.8
       os: linux
-    - rvm: 2.4.3
+    - rvm: 2.4.5
       os: linux
-    - rvm: 2.5.0
+    - rvm: 2.5.3
       os: linux
     - rvm: ruby-head
       os: linux

data/CHANGELOG.md

@@ -1,3 +1,30 @@
+# v1.3
+
+## Release v1.3.0 - 2018/11/10
+
+### New features
+
+* output: Change thread execution control
+  https://github.com/fluent/fluentd/pull/2170
+* in_syslog: Support octet counting frame
+  https://github.com/fluent/fluentd/pull/2147
+* Use `flush_thread_count` value for `queued_chunks_limit_size` when `queued_chunks_limit_size` is not specified
+  https://github.com/fluent/fluentd/pull/2173
+
+### Enhancements
+
+* output: Show backtrace for unrecoverable errors
+  https://github.com/fluent/fluentd/pull/2149
+* in_http: Implement support for CORS preflight requests
+  https://github.com/fluent/fluentd/pull/2144
+
+### Bug fixes
+
+* server: Fix deadlock between on_writable and close in sockets
+  https://github.com/fluent/fluentd/pull/2165
+* output: show correct error when wrong plugin is specified for secondary
+  https://github.com/fluent/fluentd/pull/2169
+
 # v1.2
 
 ## Release v1.2.6 - 2018/10/03

data/CONTRIBUTING.md

@@ -29,6 +29,8 @@ submitting an issue to Fluentd. Even better you can submit a Pull Request with a
 If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
 And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent releated issues.
 
+Note: Before report the issue, check latest version first. Sometimes users report fixed bug with older version.
+
 ## Patch Guidelines
 
 Here are some things that would increase a chance that your patch is accepted:

data/lib/fluent/config/v1_parser.rb

@@ -174,9 +174,8 @@ module Fluent
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
         end
-
       rescue SystemCallError => e
-        cpe = ConfigParseError.new("include error #{uri}")
+        cpe = ConfigParseError.new("include error #{uri} - #{e}")
         cpe.set_backtrace(e.backtrace)
         raise cpe
       end

data/lib/fluent/plugin/buffer.rb

@@ -372,7 +372,7 @@ module Fluent
       end
 
       def queue_full?
-        @queued_chunks_limit_size && (synchronize { @queue.size } >= @queued_chunks_limit_size)
+        synchronize { @queue.size } >= @queued_chunks_limit_size
       end
 
       def queued_records

data/lib/fluent/plugin/in_forward.rb

@@ -199,7 +199,7 @@ module Fluent::Plugin
         when :pingpong
           success, reason_or_salt, shared_key = check_ping(msg, conn.remote_addr, user_auth_salt, nonce)
           unless success
-            conn.on(:write_complete) { |c| c.close }
+            conn.on(:write_complete) { |c| c.close_after_write_complete }
             send_data.call(serializer, generate_pong(false, reason_or_salt, nonce, shared_key))
             next
           end

data/lib/fluent/plugin/in_http.rb

@@ -342,6 +342,10 @@ module Fluent::Plugin
             # For multiple X-Forwarded-For headers. Use first header value.
             v = v.first if v.is_a?(Array)
             @remote_addr = v.split(",").first
+          when /Access-Control-Request-Method/i
+            @access_control_request_method = v
+          when /Access-Control-Request-Headers/i
+            @access_control_request_headers = v
           end
         }
         if expect
@@ -367,9 +371,43 @@ module Fluent::Plugin
         @body << chunk
       end
 
+      # Web browsers can send an OPTIONS request before performing POST
+      # to check if cross-origin requests are supported.
+      def handle_options_request
+        # Is CORS enabled in the first place?
+        if @cors_allow_origins.nil?
+          return send_response_and_close("403 Forbidden", {}, "")
+        end
+
+        # in_http does not support HTTP methods except POST
+        if @access_control_request_method != 'POST'
+          return send_response_and_close("403 Forbidden", {}, "")
+        end
+
+        header = {
+          "Access-Control-Allow-Methods" => "POST",
+          "Access-Control-Allow-Headers" => @access_control_request_headers || "",
+        }
+
+        # Check the origin and send back a CORS response
+        if @cors_allow_origins.include?('*')
+          header["Access-Control-Allow-Origin"] = "*"
+          send_response_and_close("200 OK", header, "")
+        elsif @cors_allow_origins.include?(@origin)
+          header["Access-Control-Allow-Origin"] = @origin
+          send_response_and_close("200 OK", header, "")
+        else
+          send_response_and_close("403 Forbidden", {}, "")
+        end
+      end
+
       def on_message_complete
         return if closing?
 
+        if @parser.http_method == 'OPTIONS'
+          return handle_options_request()
+        end
+
         # CORS check
         # ==========
         # For every incoming request, we check if we have some CORS

data/lib/fluent/plugin/in_syslog.rb

@@ -75,6 +75,8 @@ module Fluent::Plugin
     config_param :tag, :string
     desc 'The transport protocol used to receive logs.(udp, tcp)'
     config_param :protocol_type, :enum, list: [:tcp, :udp], default: :udp
+    desc 'The message frame type.(traditional, octet_count)'
+    config_param :frame_type, :enum, list: [:traditional, :octet_count], default: :traditional
 
     desc 'If true, add source host to event record.'
     config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
@@ -152,18 +154,29 @@ module Fluent::Plugin
     end
 
     def start_tcp_server
-      # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
-      delimiter = "\n"
+      octet_count_frame = @frame_type == :octet_count
+
+      # syslog family adds "\n" to each message when transport is TCP and traditional frame
+      delimiter = octet_count_frame ? " " : "\n"
       delimiter_size = delimiter.size
       server_create_connection(:in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
         conn.data do |data|
           buffer = conn.buffer
           buffer << data
           pos = 0
-          while idx = buffer.index(delimiter, pos)
-            msg = buffer[pos...idx]
-            pos = idx + delimiter_size
-            message_handler(msg, conn)
+          if octet_count_frame
+            while idx = buffer.index(delimiter, pos)
+              num = Integer(buffer[pos..idx])
+              pos = idx + num
+              msg = buffer[idx + 1...pos]
+              message_handler(msg, conn)
+            end
+          else
+            while idx = buffer.index(delimiter, pos)
+              msg = buffer[pos...idx]
+              pos = idx + delimiter_size
+              message_handler(msg, conn)
+            end
           end
           buffer.slice!(0, pos) if pos > 0
         end

data/lib/fluent/plugin/out_forward.rb

@@ -828,7 +828,7 @@ module Fluent::Plugin
         when :pingpong
           succeeded, reason = check_pong(ri, data)
           unless succeeded
-            @log.warn "connection refused to #{@name}: #{reason}"
+            @log.warn "connection refused to #{@name || @host}: #{reason}"
             disable! # shutdown
             return
           end

data/lib/fluent/plugin/output.rb

@@ -154,7 +154,7 @@ module Fluent
       end
 
       # Internal states
-      FlushThreadState = Struct.new(:thread, :next_clock)
+      FlushThreadState = Struct.new(:thread, :next_clock, :mutex, :cond_var)
       DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do
         def expired?
           time + timeout < Time.now
@@ -354,6 +354,10 @@ module Fluent
               log.warn "'flush_interval' is ignored because default 'flush_mode' is not 'interval': '#{@flush_mode}'"
             end
           end
+
+          if @buffer.queued_chunks_limit_size.nil?
+            @buffer.queued_chunks_limit_size = @buffer_config.flush_thread_count
+          end
         end
 
         if @secondary_config
@@ -368,6 +372,9 @@ module Fluent
           end
           secondary_conf = conf.elements(name: 'secondary').first
           @secondary = Plugin.new_output(secondary_type)
+          unless @secondary.respond_to?(:acts_as_secondary)
+            raise Fluent::ConfigError, "Failed to setup secondary plugin in '#{conf['@type']}'. '#{secondary_type}' plugin in not allowed due to non buffered output"
+          end
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
           if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
@@ -440,7 +447,7 @@ module Fluent
 
           @buffer_config.flush_thread_count.times do |i|
             thread_title = "flush_thread_#{i}".to_sym
-            thread_state = FlushThreadState.new(nil, nil)
+            thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
             thread = thread_create(thread_title) do
               flush_thread_run(thread_state)
             end
@@ -506,9 +513,14 @@ module Fluent
           @output_flush_threads_running = false
           if @output_flush_threads && !@output_flush_threads.empty?
             @output_flush_threads.each do |state|
-              state.thread.run if state.thread.alive? # to wakeup thread and make it to stop by itself
-            end
-            @output_flush_threads.each do |state|
+              # to wakeup thread and make it to stop by itself
+              state.mutex.synchronize {
+                if state.thread && state.thread.status
+                  state.next_clock = 0
+                  state.cond_var.signal
+                end
+              }
+              Thread.pass
               state.thread.join
             end
           end
@@ -1119,24 +1131,29 @@ module Fluent
           if @secondary
             if using_secondary
               log.warn "got unrecoverable error in secondary.", error: e
+              log.warn_backtrace
               backup_chunk(chunk, using_secondary, output.delayed_commit)
             else
               if (self.class == @secondary.class)
                 log.warn "got unrecoverable error in primary and secondary type is same as primary. Skip secondary", error: e
+                log.warn_backtrace
                 backup_chunk(chunk, using_secondary, output.delayed_commit)
               else
                 # Call secondary output directly without retry update.
                 # In this case, delayed commit causes inconsistent state in dequeued chunks so async output in secondary is not allowed for now.
                 if @secondary.delayed_commit
                   log.warn "got unrecoverable error in primary and secondary is async output. Skip secondary for backup", error: e
+                  log.warn_backtrace
                   backup_chunk(chunk, using_secondary, output.delayed_commit)
                 else
                   log.warn "got unrecoverable error in primary. Skip retry and flush chunk to secondary", error: e
+                  log.warn_backtrace
                   begin
                     @secondary.write(chunk)
                     commit_write(chunk_id, delayed: output.delayed_commit, secondary: true)
                   rescue => e
                     log.warn "got an error in secondary for unrecoverable error", error: e
+                    log.warn_backtrace
                     backup_chunk(chunk, using_secondary, output.delayed_commit)
                   end
                 end
@@ -1144,6 +1161,7 @@ module Fluent
             end
           else
             log.warn "got unrecoverable error in primary and no secondary", error: e
+            log.warn_backtrace
             backup_chunk(chunk, using_secondary, output.delayed_commit)
           end
         rescue => e
@@ -1254,12 +1272,15 @@ module Fluent
         # Without locks: it is rough but enough to select "next" writer selection
         @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
         state = @output_flush_threads[@output_flush_thread_current_position]
-        state.next_clock = 0
-        if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
-          state.thread.run
-        else
-          log.warn "thread is already dead"
-        end
+        state.mutex.synchronize {
+          if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
+            state.next_clock = 0
+            state.cond_var.signal
+          else
+            log.warn "thread is already dead"
+          end
+        }
+        Thread.pass
       end
 
       def force_flush
@@ -1296,8 +1317,13 @@ module Fluent
       # only for tests of output plugin
       def flush_thread_wakeup
         @output_flush_threads.each do |state|
-          state.next_clock = 0
-          state.thread.run
+          state.mutex.synchronize {
+            if state.thread && state.thread.status
+              state.next_clock = 0
+              state.cond_var.signal
+            end
+          }
+          Thread.pass
         end
       end
 
@@ -1376,6 +1402,7 @@ module Fluent
         end
         log.debug "flush_thread actually running"
 
+        state.mutex.lock
         begin
           # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
           while @output_flush_threads_running
@@ -1391,23 +1418,32 @@ module Fluent
             elsif next_retry_time && next_retry_time > Time.now
               interval = next_retry_time.to_f - Time.now.to_f
             else
-              try_flush
-
-              # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
-              interval = next_flush_time.to_f - Time.now.to_f
-              # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
-              #       because @retry still exists (#commit_write is not called yet in #try_flush)
-              #       @retry should be cleared if delayed commit is enabled? Or any other solution?
-              state.next_clock = Fluent::Clock.now + interval
+              state.mutex.unlock
+              begin
+                try_flush
+                # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
+                interval = next_flush_time.to_f - Time.now.to_f
+                # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
+                #       because @retry still exists (#commit_write is not called yet in #try_flush)
+                #       @retry should be cleared if delayed commit is enabled? Or any other solution?
+                state.next_clock = Fluent::Clock.now + interval
+              ensure
+                state.mutex.lock
+              end
             end
 
             if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? }
               unless @output_flush_interrupted
-                try_rollback_write
+                state.mutex.unlock
+                begin
+                  try_rollback_write
+                ensure
+                  state.mutex.lock
+                end
               end
             end
 
-            sleep interval if interval > 0
+            state.cond_var.wait(state.mutex, interval) if interval > 0
           end
         rescue => e
           # normal errors are rescued by output plugins in #try_flush
@@ -1415,6 +1451,8 @@ module Fluent
           log.error "error on output thread", error: e
           log.error_backtrace
           raise
+        ensure
+          state.mutex.unlock
         end
       end
     end

data/lib/fluent/plugin_helper/server.rb

@@ -408,6 +408,10 @@ module Fluent
           raise "not implemented here"
         end
 
+        def close_after_write_complete
+          @sock.close_after_write_complete = true
+        end
+
         def close
           @sock.close if @close_socket
         end
@@ -492,6 +496,8 @@ module Fluent
 
       module EventHandler
         class UDPServer < Coolio::IO
+          attr_writer :close_after_write_complete # dummy for consistent method call in callbacks
+
           def initialize(sock, max_bytes, flags, close_socket, log, under_plugin_development, &callback)
             raise ArgumentError, "socket must be a UDPSocket: sock = #{sock}" unless sock.is_a?(UDPSocket)
 
@@ -543,6 +549,7 @@ module Fluent
 
         class TCPServer < Coolio::TCPSocket
           attr_reader :closing
+          attr_writer :close_after_write_complete
 
           def initialize(sock, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
             raise ArgumentError, "socket must be a TCPSocket: sock=#{sock}" unless sock.is_a?(TCPSocket)
@@ -560,6 +567,7 @@ module Fluent
             @close_callback = close_callback
 
             @callback_connection = nil
+            @close_after_write_complete = false
             @closing = false
 
             @mutex = Mutex.new # to serialize #write and #close
@@ -587,6 +595,11 @@ module Fluent
             end
           end
 
+          def on_writable
+            super
+            close if @close_after_write_complete
+          end
+
           def on_connect
             @callback_connection = TCPCallbackSocket.new(self)
             @connect_callback.call(@callback_connection)
@@ -625,6 +638,7 @@ module Fluent
 
         class TLSServer < Coolio::Socket
           attr_reader :closing
+          attr_writer :close_after_write_complete
 
           # It can't use Coolio::TCPSocket, because Coolio::TCPSocket checks that underlying socket (1st argument of super) is TCPSocket.
           def initialize(sock, context, socket_option_setter, close_callback, log, under_plugin_development, connect_callback)
@@ -645,6 +659,7 @@ module Fluent
             @close_callback = close_callback
 
             @callback_connection = nil
+            @close_after_write_complete = false
             @closing = false
 
             @mutex = Mutex.new # to serialize #write and #close
@@ -738,6 +753,7 @@ module Fluent
                 @_handler_write_buffer.slice!(0, written_bytes)
                 super
               end
+              close if @close_after_write_complete
             rescue IO::WaitWritable, IO::WaitReadable
               return
             rescue Errno::EINTR

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.2.6'
+  VERSION = '1.3.0'
 
 end

data/test/plugin/test_buffer.rb

@@ -170,7 +170,7 @@ class BufferTest < Test::Unit::TestCase
 
   sub_test_case 'with default configuration and dummy implementation' do
     setup do
-      @p = create_buffer({})
+      @p = create_buffer({'queued_chunks_limit_size' => 100})
       @dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
       @dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
       @dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)

data/test/plugin/test_in_http.rb

@@ -624,6 +624,23 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_cors_preflight
+    d = create_driver(CONFIG + 'cors_allow_origins ["*"]')
+
+    d.run do
+      header = {
+        "Origin" => "http://foo.com",
+        "Access-Control-Request-Method" => "POST",
+        "Access-Control-Request-Headers" => "Content-Type",
+      }
+      res = options("/cors.test", {}, header)
+
+      assert_equal "200", res.code
+      assert_equal "*", res["Access-Control-Allow-Origin"]
+      assert_equal "POST", res["Access-Control-Allow-Methods"]
+    end
+  end
+
   def test_content_encoding_gzip
     d = create_driver
 
@@ -744,6 +761,12 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal $test_in_http_connection_object_ids[0], $test_in_http_connection_object_ids[1]
   end
 
+  def options(path, params, header = {})
+    http = Net::HTTP.new("127.0.0.1", PORT)
+    req = Net::HTTP::Options.new(path, header)
+    http.request(req)
+  end
+
   def post(path, params, header = {}, &block)
     http = Net::HTTP.new("127.0.0.1", PORT)
     req = Net::HTTP::Post.new(path, header)

data/test/plugin/test_in_syslog.rb

@@ -286,4 +286,50 @@ EOS
       assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }
   end
+
+  sub_test_case 'octet counting frame' do
+    def test_msg_size_with_tcp
+      d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
+      tests = create_test_case
+
+      d.run(expect_emits: 2) do
+        tests.each {|test|
+          TCPSocket.open('127.0.0.1', PORT) do |s|
+            s.send(test['msg'], 0)
+          end
+        }
+      end
+
+      assert(d.events.size > 0)
+      compare_test_result(d.events, tests)
+    end
+
+    def test_msg_size_with_same_tcp_connection
+      d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
+      tests = create_test_case
+
+      d.run(expect_emits: 2) do
+        TCPSocket.open('127.0.0.1', PORT) do |s|
+          tests.each {|test|
+            s.send(test['msg'], 0)
+          }
+        end
+      end
+
+      assert(d.events.size > 0)
+      compare_test_result(d.events, tests)
+    end
+
+    def create_test_case(large_message: false)
+      msgs = [
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100, 'expected' => 'x' * 100},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024, 'expected' => 'x' * 1024},
+      ]
+      msgs.each { |msg|
+        m = msg['msg']
+        msg['msg'] = "#{m.size + 1} #{m}"
+      }
+      msgs
+    end
+  end
 end

data/test/plugin/test_output.rb

@@ -828,34 +828,43 @@ class OutputTest < Test::Unit::TestCase
       assert_equal :interval, @i.instance_variable_get(:@flush_mode)
     end
 
-    test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
-      o = create_output(:buffered)
-      mock(o.log).warn("secondary type should be same with primary one",
-                                { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
+    sub_test_case 'configure secondary' do
+      test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
+        o = create_output(:buffered)
+        mock(o.log).warn("secondary type should be same with primary one",
+                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
+
+        o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
+        assert_not_nil o.instance_variable_get(:@secondary)
+      end
 
-      o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
-      assert_not_nil o.instance_variable_get(:@secondary)
-    end
+      test "don't warn if primary type is the same as secondary type" do
+        o = Fluent::Plugin::TestOutput.new
+        mock(o.log).warn("secondary type should be same with primary one",
+                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
 
-    test "don't warn if primary type is the same as secondary type" do
-      o = Fluent::Plugin::TestOutput.new
-      mock(o.log).warn("secondary type should be same with primary one",
-                                { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+        o.configure(config_element('ROOT','',{'name' => "cool2"},
+                                   [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
+                                    config_element('buffer','',{'@type'=>'memory'})]
+                                  ))
+        assert_not_nil o.instance_variable_get(:@secondary)
+      end
 
-      o.configure(config_element('ROOT','',{'name' => "cool2"},
-                                 [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
-                                  config_element('buffer','',{'@type'=>'memory'})]
-                                ))
-      assert_not_nil o.instance_variable_get(:@secondary)
-    end
+      test "don't warn if primary type is different from secondary type and both don't have custom_format" do
+        o = create_output(:standard)
+        mock(o.log).warn("secondary type should be same with primary one",
+                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
 
-    test "don't warn if primary type is different from secondary type and both don't have custom_format" do
-      o = create_output(:standard)
-      mock(o.log).warn("secondary type should be same with primary one",
-                                { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+        o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
+        assert_not_nil o.instance_variable_get(:@secondary)
+      end
 
-      o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
-      assert_not_nil o.instance_variable_get(:@secondary)
+      test "raise configuration error if secondary type specifies non buffered output" do
+        o = create_output(:standard)
+        assert_raise Fluent::ConfigError do
+          o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'copy'})]))
+        end
+      end
     end
   end
 

data/test/plugin/test_output_as_buffered.rb

@@ -219,6 +219,24 @@ class BufferedOutputTest < Test::Unit::TestCase
     Timecop.return
   end
 
+  test 'queued_chunks_limit_size is same as flush_thread_count by default' do
+    hash = {'flush_thread_count' => 4}
+    i = create_output
+    i.register(:prefer_buffered_processing) { true }
+    i.configure(config_element('ROOT', '', {}, [config_element('buffer','tag',hash)]))
+
+    assert_equal 4, i.buffer.queued_chunks_limit_size
+  end
+
+  test 'prefer queued_chunks_limit_size parameter than flush_thread_count' do
+    hash = {'flush_thread_count' => 4, 'queued_chunks_limit_size' => 2}
+    i = create_output
+    i.register(:prefer_buffered_processing) { true }
+    i.configure(config_element('ROOT', '', {}, [config_element('buffer','tag',hash)]))
+
+    assert_equal 2, i.buffer.queued_chunks_limit_size
+  end
+
   sub_test_case 'chunk feature in #write for output plugins' do
     setup do
       @stored_global_logger = $log
@@ -1064,6 +1082,7 @@ class BufferedOutputTest < Test::Unit::TestCase
         'flush_thread_count' => 1,
         'flush_thread_burst_interval' => 0.1,
         'chunk_limit_size' => 1024,
+        'queued_chunks_limit_size' => 100
       }
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))

data/test/plugin/test_output_as_buffered_retries.rb

@@ -123,6 +123,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'flush_interval' => 1,
         'flush_thread_burst_interval' => 0.1,
         'retry_randomize' => false,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -252,6 +253,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'flush_thread_burst_interval' => 0.1,
         'retry_randomize' => false,
         'retry_timeout' => 3600,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -342,6 +344,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'flush_thread_burst_interval' => 0.1,
         'retry_randomize' => false,
         'retry_max_times' => 10,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -485,6 +488,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_type' => :periodic,
         'retry_wait' => 3,
         'retry_randomize' => false,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -531,6 +535,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_wait' => 30,
         'retry_randomize' => false,
         'retry_timeout' => 120,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -626,6 +631,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_wait' => 3,
         'retry_randomize' => false,
         'retry_max_times' => 10,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -724,6 +730,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_randomize' => false,
         'retry_timeout' => 3600,
         'retry_max_times' => 10,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -795,6 +802,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_wait' => 30,
         'retry_timeout' => 360,
         'retry_max_times' => 10,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }

data/test/plugin/test_output_as_buffered_secondary.rb

@@ -544,7 +544,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
     test 'secondary plugin can do delayed commit even if primary does not do it, and non-committed chunks will be rollbacked by primary' do
       written = []
       chunks = []
-      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false})
+      priconf = config_element('buffer','tag',{'flush_interval' => 1, 'retry_type' => :periodic, 'retry_wait' => 3, 'retry_timeout' => 60, 'delayed_commit_timeout' => 2, 'retry_randomize' => false, 'queued_chunks_limit_size' => 10})
       secconf = config_element('secondary','',{'@type' => 'output_secondary_test2'})
       @i.configure(config_element('ROOT','',{},[priconf,secconf]))
       @i.register(:prefer_buffered_processing){ true }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.3.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-04 00:00:00.000000000 Z
+date: 2018-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack