fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

data/lib/fluent/plugin/out_secondary_file.rb

@@ -71,7 +71,7 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      path_without_suffix = extract_placeholders(@path_without_suffix, chunk.metadata)
+      path_without_suffix = extract_placeholders(@path_without_suffix, chunk)
       path = generate_path(path_without_suffix)
       FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
 
@@ -106,7 +106,7 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove tag placeholder, like `${tag}`, from basename or directory"
       end
 
-      vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) }
+      vars = placeholders.reject { |placeholder| placeholder.match(/tag(\[\d+\])?/) || (placeholder == 'chunk_id') }
 
       if ph = vars.find { |v| !@chunk_keys.include?(v) }
         raise Fluent::ConfigError, "out_secondary_file: basename or directory has an incompatible placeholder #{ph}, remove variable placeholder, like `${varname}`, from basename or directory"

data/lib/fluent/plugin/out_stdout.rb

@@ -24,7 +24,6 @@ module Fluent::Plugin
 
     DEFAULT_LINE_FORMAT_TYPE = 'stdout'
     DEFAULT_FORMAT_TYPE = 'json'
-    TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
 
     config_section :buffer do
       config_set_default :chunk_keys, ['tag']

data/lib/fluent/plugin/out_stream.rb

@@ -93,7 +93,7 @@ module Fluent
     def initialize
       super
       $log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
-      $log.warn "see 'forward' section in http://docs.fluentd.org/ for the high-availability configuration."
+      $log.warn "see 'forward' section in https://docs.fluentd.org/ for the high-availability configuration."
     end
 
     config_param :port, :integer, default: LISTEN_PORT

data/lib/fluent/plugin/output.rb

@@ -14,7 +14,9 @@
 #    limitations under the License.
 #
 
+require 'fluent/error'
 require 'fluent/plugin/base'
+require 'fluent/plugin_helper/record_accessor'
 require 'fluent/log'
 require 'fluent/plugin_id'
 require 'fluent/plugin_helper'
@@ -36,8 +38,9 @@ module Fluent
       helpers_internal :thread, :retry_state
 
       CHUNK_KEY_PATTERN = /^[-_.@a-zA-Z0-9]+$/
-      CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@a-zA-Z0-9]+\}/
-      CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[\d+\])?)\}/
+      CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{[-_.@$a-zA-Z0-9]+\}/
+      CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
+      CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
 
       CHUNKING_FIELD_WARN_NUM = 4
 
@@ -89,10 +92,11 @@ module Fluent
         # k: times
         # total retry time: c + c * b^1 + (...) + c*b^k = c*b^(k+1) - 1
         config_param :retry_wait, :time, default: 1, desc: 'Seconds to wait before next retry to flush, or constant factor of exponential backoff.'
-        config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponencial backoff for retries.'
-        config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponencial backoff between retries while failing.'
+        config_param :retry_exponential_backoff_base, :float, default: 2, desc: 'The base number of exponential backoff for retries.'
+        config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
 
         config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
+        config_param :disable_chunk_backup, :bool, default: false, desc: 'If true, chunks are thrown away when unrecoverable error happens'
       end
 
       config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do
@@ -150,7 +154,7 @@ module Fluent
       end
 
       # Internal states
-      FlushThreadState = Struct.new(:thread, :next_clock)
+      FlushThreadState = Struct.new(:thread, :next_clock, :mutex, :cond_var)
       DequeuedChunkInfo = Struct.new(:chunk_id, :time, :timeout) do
         def expired?
           time + timeout < Time.now
@@ -161,7 +165,7 @@ module Fluent
       attr_reader :num_errors, :emit_count, :emit_records, :write_count, :rollback_count
 
       # for tests
-      attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_time, :chunk_key_tag
+      attr_reader :buffer, :retry, :secondary, :chunk_keys, :chunk_key_accessors, :chunk_key_time, :chunk_key_tag
       attr_accessor :output_enqueue_thread_waiting, :dequeued_chunks, :dequeued_chunks_mutex
       # output_enqueue_thread_waiting: for test of output.rb itself
       attr_accessor :retry_for_error_chunk # if true, error flush will be retried even if under_plugin_development is true
@@ -203,7 +207,7 @@ module Fluent
         @output_flush_threads = nil
 
         @simple_chunking = nil
-        @chunk_keys = @chunk_key_time = @chunk_key_tag = nil
+        @chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
         @flush_mode = nil
         @timekey_zone = nil
 
@@ -222,9 +226,9 @@ module Fluent
         end
         self.context_router = primary.context_router
 
-        (class << self; self; end).module_eval do
+        singleton_class.module_eval do
           define_method(:commit_write){ |chunk_id| @primary_instance.commit_write(chunk_id, delayed: delayed_commit, secondary: true) }
-          define_method(:rollback_write){ |chunk_id| @primary_instance.rollback_write(chunk_id) }
+          define_method(:rollback_write){ |chunk_id, update_retry: true| @primary_instance.rollback_write(chunk_id, update_retry) }
         end
       end
 
@@ -276,14 +280,35 @@ module Fluent
           @chunk_keys = @buffer_config.chunk_keys.dup
           @chunk_key_time = !!@chunk_keys.delete('time')
           @chunk_key_tag = !!@chunk_keys.delete('tag')
-          if @chunk_keys.any?{ |key| key !~ CHUNK_KEY_PATTERN }
+          if @chunk_keys.any? { |key|
+              begin
+                k = Fluent::PluginHelper::RecordAccessor::Accessor.parse_parameter(key)
+                if k.is_a?(String)
+                  k !~ CHUNK_KEY_PATTERN
+                else
+                  if key.start_with?('$[')
+                    raise Fluent::ConfigError, "in chunk_keys: bracket notation is not allowed"
+                  else
+                    false
+                  end
+                end
+              rescue => e
+                raise Fluent::ConfigError, "in chunk_keys: #{e.message}"
+              end
+            }
             raise Fluent::ConfigError, "chunk_keys specification includes invalid char"
+          else
+            @chunk_key_accessors = Hash[@chunk_keys.map { |key| [key.to_sym, Fluent::PluginHelper::RecordAccessor::Accessor.new(key)] }]
           end
 
           if @chunk_key_time
             raise Fluent::ConfigError, "<buffer ...> argument includes 'time', but timekey is not configured" unless @buffer_config.timekey
             Fluent::Timezone.validate!(@buffer_config.timekey_zone)
             @timekey_zone = @buffer_config.timekey_use_utc ? '+0000' : @buffer_config.timekey_zone
+            @timekey = @buffer_config.timekey
+            @timekey_use_utc = @buffer_config.timekey_use_utc
+            @offset = Fluent::Timezone.utc_offset(@timekey_zone)
+            @calculate_offset = @offset.respond_to?(:call) ? @offset : nil
             @output_time_formatter_cache = {}
           end
 
@@ -329,6 +354,10 @@ module Fluent
               log.warn "'flush_interval' is ignored because default 'flush_mode' is not 'interval': '#{@flush_mode}'"
             end
           end
+
+          if @buffer.queued_chunks_limit_size.nil?
+            @buffer.queued_chunks_limit_size = @buffer_config.flush_thread_count
+          end
         end
 
         if @secondary_config
@@ -343,6 +372,9 @@ module Fluent
           end
           secondary_conf = conf.elements(name: 'secondary').first
           @secondary = Plugin.new_output(secondary_type)
+          unless @secondary.respond_to?(:acts_as_secondary)
+            raise Fluent::ConfigError, "Failed to setup secondary plugin in '#{conf['@type']}'. '#{secondary_type}' plugin in not allowed due to non buffered output"
+          end
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
           if (self.class != @secondary.class) && (@custom_format || @secondary.implement?(:custom_format))
@@ -370,7 +402,7 @@ module Fluent
 
         if @buffering
           m = method(:emit_buffered)
-          (class << self; self; end).module_eval do
+          singleton_class.module_eval do
             define_method(:emit_events, m)
           end
 
@@ -384,7 +416,7 @@ module Fluent
           @delayed_commit_timeout = @buffer_config.delayed_commit_timeout
         else # !@buffering
           m = method(:emit_sync)
-          (class << self; self; end).module_eval do
+          singleton_class.module_eval do
             define_method(:emit_events, m)
           end
         end
@@ -415,7 +447,7 @@ module Fluent
 
           @buffer_config.flush_thread_count.times do |i|
             thread_title = "flush_thread_#{i}".to_sym
-            thread_state = FlushThreadState.new(nil, nil)
+            thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
             thread = thread_create(thread_title) do
               flush_thread_run(thread_state)
             end
@@ -481,9 +513,14 @@ module Fluent
           @output_flush_threads_running = false
           if @output_flush_threads && !@output_flush_threads.empty?
             @output_flush_threads.each do |state|
-              state.thread.run if state.thread.alive? # to wakeup thread and make it to stop by itself
-            end
-            @output_flush_threads.each do |state|
+              # to wakeup thread and make it to stop by itself
+              state.mutex.synchronize {
+                if state.thread && state.thread.status
+                  state.next_clock = 0
+                  state.cond_var.signal
+                end
+              }
+              Thread.pass
               state.thread.join
             end
           end
@@ -653,7 +690,7 @@ module Fluent
         str.scan(CHUNK_TAG_PLACEHOLDER_PATTERN).map(&:first).each do |ph|
           if ph == "tag"
             parts << -1
-          elsif ph =~ /^tag\[(\d+)\]$/
+          elsif ph =~ /^tag\[(-?\d+)\]$/
             parts << $1.to_i
           end
         end
@@ -661,13 +698,27 @@ module Fluent
       end
 
       def get_placeholders_keys(str)
-        str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map{|ph| ph[2..-2]}.reject{|s| s == "tag"}.sort
+        str.scan(CHUNK_KEY_PLACEHOLDER_PATTERN).map{|ph| ph[2..-2]}.reject{|s| (s == "tag") || (s == 'chunk_id') }.sort
       end
 
       # TODO: optimize this code
-      def extract_placeholders(str, metadata)
+      def extract_placeholders(str, chunk)
+        metadata = if chunk.is_a?(Fluent::Plugin::Buffer::Chunk)
+                     chunk_passed = true
+                     chunk.metadata
+                   else
+                     chunk_passed = false
+                     # For existing plugins. Old plugin passes Chunk.metadata instead of Chunk
+                     chunk
+                   end
         if metadata.empty?
-          str
+          str.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
+            if chunk_passed
+              dump_unique_id_hex(chunk.unique_id)
+            else
+              log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
+            end
+          }
         else
           rvalue = str.dup
           # strftime formatting
@@ -675,15 +726,17 @@ module Fluent
             @output_time_formatter_cache[str] ||= Fluent::Timezone.formatter(@timekey_zone, str)
             rvalue = @output_time_formatter_cache[str].call(metadata.timekey)
           end
-          # ${tag}, ${tag[0]}, ${tag[1]}, ...
+          # ${tag}, ${tag[0]}, ${tag[1]}, ... , ${tag[-2]}, ${tag[-1]}
           if @chunk_key_tag
             if str.include?('${tag}')
               rvalue = rvalue.gsub('${tag}', metadata.tag)
             end
             if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
               hash = {}
-              metadata.tag.split('.').each_with_index do |part, i|
+              tag_parts = metadata.tag.split('.')
+              tag_parts.each_with_index do |part, i|
                 hash["${tag[#{i}]}"] = part
+                hash["${tag[#{i-tag_parts.size}]}"] = part
               end
               rvalue = rvalue.gsub(CHUNK_TAG_PLACEHOLDER_PATTERN, hash)
             end
@@ -700,9 +753,15 @@ module Fluent
             rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN, hash)
           end
           if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
-            log.warn "chunk key placeholder '#{$1}' not replaced. templace:#{str}"
+            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
           end
-          rvalue
+          rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
+            if chunk_passed
+              dump_unique_id_hex(chunk.unique_id)
+            else
+              log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
+            end
+          }
         end
       end
 
@@ -761,28 +820,42 @@ module Fluent
           if !@chunk_key_time && !@chunk_key_tag
             @buffer.metadata()
           elsif @chunk_key_time && @chunk_key_tag
-            time_int = time.to_i
-            timekey = (time_int - (time_int % @buffer_config.timekey)).to_i
+            timekey = calculate_timekey(time)
             @buffer.metadata(timekey: timekey, tag: tag)
           elsif @chunk_key_time
-            time_int = time.to_i
-            timekey = (time_int - (time_int % @buffer_config.timekey)).to_i
+            timekey = calculate_timekey(time)
             @buffer.metadata(timekey: timekey)
           else
             @buffer.metadata(tag: tag)
           end
         else
           timekey = if @chunk_key_time
-                      time_int = time.to_i
-                      (time_int - (time_int % @buffer_config.timekey)).to_i
+                      calculate_timekey(time)
                     else
                       nil
                     end
-          pairs = Hash[@chunk_keys.map{|k| [k.to_sym, record[k]]}]
+          pairs = Hash[@chunk_key_accessors.map { |k, a| [k, a.call(record)] }]
           @buffer.metadata(timekey: timekey, tag: (@chunk_key_tag ? tag : nil), variables: pairs)
         end
       end
 
+      def calculate_timekey(time)
+        time_int = time.to_i
+        if @timekey_use_utc
+          (time_int - (time_int % @timekey)).to_i
+        else
+          offset = @calculate_offset ? @calculate_offset.call(time) : @offset
+          (time_int - ((time_int + offset)% @timekey)).to_i
+        end
+      end
+
+      def chunk_for_test(tag, time, record)
+        require 'fluent/plugin/buffer/memory_chunk'
+
+        m = metadata_for_test(tag, time, record)
+        Fluent::Plugin::Buffer::MemoryChunk.new(m)
+      end
+
       def metadata_for_test(tag, time, record)
         raise "BUG: #metadata_for_test is available only when no actual metadata exists" unless @buffer.metadata_list.empty?
         m = metadata(tag, time, record)
@@ -825,7 +898,7 @@ module Fluent
             begin
               oldest = @buffer.dequeue_chunk
               if oldest
-                log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: oldest.unique_id
+                log.warn "dropping oldest chunk to make space after buffer overflow", chunk_id: dump_unique_id_hex(oldest.unique_id)
                 @buffer.purge_chunk(oldest.unique_id)
               else
                 log.error "no queued chunks to be dropped for drop_oldest_chunk"
@@ -925,7 +998,8 @@ module Fluent
       end
 
       def commit_write(chunk_id, delayed: @delayed_commit, secondary: false)
-        log.trace "committing write operation to a chunk", chunk: dump_unique_id_hex(chunk_id), delayed: delayed
+        log.on_trace { log.trace "committing write operation to a chunk", chunk: dump_unique_id_hex(chunk_id), delayed: delayed }
+
         if delayed
           @dequeued_chunks_mutex.synchronize do
             @dequeued_chunks.delete_if{ |info| info.chunk_id == chunk_id }
@@ -945,7 +1019,9 @@ module Fluent
         end
       end
 
-      def rollback_write(chunk_id)
+      # update_retry parameter is for preventing busy loop by async write
+      # We will remove this parameter by re-design retry_state management between threads.
+      def rollback_write(chunk_id, update_retry: true)
         # This API is to rollback chunks explicitly from plugins.
         # 3rd party plugins can depend it on automatic rollback of #try_rollback_write
         @dequeued_chunks_mutex.synchronize do
@@ -956,8 +1032,10 @@ module Fluent
         # in many cases, false can be just ignored
         if @buffer.takeback_chunk(chunk_id)
           @counters_monitor.synchronize{ @rollback_count += 1 }
-          primary = @as_secondary ? @primary_instance : self
-          primary.update_retry_state(chunk_id, @as_secondary)
+          if update_retry
+            primary = @as_secondary ? @primary_instance : self
+            primary.update_retry_state(chunk_id, @as_secondary)
+          end
           true
         else
           false
@@ -1003,11 +1081,13 @@ module Fluent
         end
       end
 
+      UNRECOVERABLE_ERRORS = [Fluent::UnrecoverableError, TypeError, ArgumentError, NoMethodError]
+
       def try_flush
         chunk = @buffer.dequeue_chunk
         return unless chunk
 
-        log.trace "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id)
+        log.on_trace { log.trace "trying flush for a chunk", chunk: dump_unique_id_hex(chunk.unique_id) }
 
         output = self
         using_secondary = false
@@ -1047,6 +1127,43 @@ module Fluent
             commit_write(chunk_id, delayed: false, secondary: using_secondary)
             log.trace "done to commit a chunk", chunk: dump_chunk_id
           end
+        rescue *UNRECOVERABLE_ERRORS => e
+          if @secondary
+            if using_secondary
+              log.warn "got unrecoverable error in secondary.", error: e
+              log.warn_backtrace
+              backup_chunk(chunk, using_secondary, output.delayed_commit)
+            else
+              if (self.class == @secondary.class)
+                log.warn "got unrecoverable error in primary and secondary type is same as primary. Skip secondary", error: e
+                log.warn_backtrace
+                backup_chunk(chunk, using_secondary, output.delayed_commit)
+              else
+                # Call secondary output directly without retry update.
+                # In this case, delayed commit causes inconsistent state in dequeued chunks so async output in secondary is not allowed for now.
+                if @secondary.delayed_commit
+                  log.warn "got unrecoverable error in primary and secondary is async output. Skip secondary for backup", error: e
+                  log.warn_backtrace
+                  backup_chunk(chunk, using_secondary, output.delayed_commit)
+                else
+                  log.warn "got unrecoverable error in primary. Skip retry and flush chunk to secondary", error: e
+                  log.warn_backtrace
+                  begin
+                    @secondary.write(chunk)
+                    commit_write(chunk_id, delayed: output.delayed_commit, secondary: true)
+                  rescue => e
+                    log.warn "got an error in secondary for unrecoverable error", error: e
+                    log.warn_backtrace
+                    backup_chunk(chunk, using_secondary, output.delayed_commit)
+                  end
+                end
+              end
+            end
+          else
+            log.warn "got unrecoverable error in primary and no secondary", error: e
+            log.warn_backtrace
+            backup_chunk(chunk, using_secondary, output.delayed_commit)
+          end
         rescue => e
           log.debug "taking back chunk for errors.", chunk: dump_unique_id_hex(chunk.unique_id)
           if output.delayed_commit
@@ -1062,6 +1179,25 @@ module Fluent
         end
       end
 
+      def backup_chunk(chunk, using_secondary, delayed_commit)
+        if @buffer_config.disable_chunk_backup
+          log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away"
+        else
+          unique_id = dump_unique_id_hex(chunk.unique_id)
+          safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
+          backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
+          backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
+          backup_dir = File.dirname(backup_file)
+
+          log.warn "bad chunk is moved to #{backup_file}"
+          FileUtils.mkdir_p(backup_dir) unless Dir.exist?(backup_dir)
+          File.open(backup_file, 'ab', system_config.file_permission || 0644) { |f|
+            chunk.write_to(f)
+          }
+        end
+        commit_write(chunk.unique_id, secondary: using_secondary, delayed: delayed_commit)
+      end
+
       def check_slow_flush(start)
         elapsed_time = Fluent::Clock.now - start
         if elapsed_time > @slow_flush_log_threshold
@@ -1136,17 +1272,20 @@ module Fluent
         # Without locks: it is rough but enough to select "next" writer selection
         @output_flush_thread_current_position = (@output_flush_thread_current_position + 1) % @buffer_config.flush_thread_count
         state = @output_flush_threads[@output_flush_thread_current_position]
-        state.next_clock = 0
-        if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
-          state.thread.run
-        else
-          log.warn "thread is already dead"
-        end
+        state.mutex.synchronize {
+          if state.thread && state.thread.status # "run"/"sleep"/"aborting" or false(successfully stop) or nil(killed by exception)
+            state.next_clock = 0
+            state.cond_var.signal
+          else
+            log.warn "thread is already dead"
+          end
+        }
+        Thread.pass
       end
 
       def force_flush
         if @buffering
-          @buffer.enqueue_all
+          @buffer.enqueue_all(true)
           submit_flush_all
         end
       end
@@ -1178,8 +1317,13 @@ module Fluent
       # only for tests of output plugin
       def flush_thread_wakeup
         @output_flush_threads.each do |state|
-          state.next_clock = 0
-          state.thread.run
+          state.mutex.synchronize {
+            if state.thread && state.thread.status
+              state.next_clock = 0
+              state.cond_var.signal
+            end
+          }
+          Thread.pass
         end
       end
 
@@ -1258,30 +1402,48 @@ module Fluent
         end
         log.debug "flush_thread actually running"
 
+        state.mutex.lock
         begin
           # This thread don't use `thread_current_running?` because this thread should run in `before_shutdown` phase
           while @output_flush_threads_running
             current_clock = Fluent::Clock.now
-            interval = state.next_clock - current_clock
+            next_retry_time = nil
 
-            if state.next_clock <= current_clock && (!@retry || @retry_mutex.synchronize{ @retry.next_time } <= Time.now)
-              try_flush
+            @retry_mutex.synchronize do
+              next_retry_time = @retry ? @retry.next_time : nil
+            end
 
-              # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
-              interval = next_flush_time.to_f - Time.now.to_f
-              # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
-              #       because @retry still exists (#commit_write is not called yet in #try_flush)
-              #       @retry should be cleared if delayed commit is enabled? Or any other solution?
-              state.next_clock = Fluent::Clock.now + interval
+            if state.next_clock > current_clock
+              interval = state.next_clock - current_clock
+            elsif next_retry_time && next_retry_time > Time.now
+              interval = next_retry_time.to_f - Time.now.to_f
+            else
+              state.mutex.unlock
+              begin
+                try_flush
+                # next_flush_time uses flush_thread_interval or flush_thread_burst_interval (or retrying)
+                interval = next_flush_time.to_f - Time.now.to_f
+                # TODO: if secondary && delayed-commit, next_flush_time will be much longer than expected
+                #       because @retry still exists (#commit_write is not called yet in #try_flush)
+                #       @retry should be cleared if delayed commit is enabled? Or any other solution?
+                state.next_clock = Fluent::Clock.now + interval
+              ensure
+                state.mutex.lock
+              end
             end
 
             if @dequeued_chunks_mutex.synchronize{ !@dequeued_chunks.empty? && @dequeued_chunks.first.expired? }
               unless @output_flush_interrupted
-                try_rollback_write
+                state.mutex.unlock
+                begin
+                  try_rollback_write
+                ensure
+                  state.mutex.lock
+                end
               end
             end
 
-            sleep interval if interval > 0
+            state.cond_var.wait(state.mutex, interval) if interval > 0
           end
         rescue => e
           # normal errors are rescued by output plugins in #try_flush
@@ -1289,6 +1451,8 @@ module Fluent
           log.error "error on output thread", error: e
           log.error_backtrace
           raise
+        ensure
+          state.mutex.unlock
         end
       end
     end