fluentd 1.11.3-x86-mingw32 → 1.12.2-x86-mingw32

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -22,34 +22,40 @@ module Fluent::Plugin
       UNWATCHED_POSITION = 0xffffffffffffffff
       POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
 
-      def self.load(file, logger:)
-        pf = new(file, logger: logger)
+      def self.load(file, follow_inodes, existing_paths, logger:)
+        pf = new(file, follow_inodes, existing_paths, logger: logger)
         pf.load
         pf
       end
 
-      def initialize(file, logger: nil)
+      def initialize(file, follow_inodes, existing_paths, logger: nil)
         @file = file
         @logger = logger
         @file_mutex = Mutex.new
         @map = {}
+        @follow_inodes = follow_inodes
+        @existing_paths = existing_paths
       end
 
-      def [](path)
-        if m = @map[path]
+      def [](target_info)
+        if m = @map[@follow_inodes ? target_info.ino : target_info.path]
           return m
         end
 
         @file_mutex.synchronize {
           @file.seek(0, IO::SEEK_END)
-          seek = @file.pos + path.bytesize + 1
-          @file.write "#{path}\t0000000000000000\t0000000000000000\n"
-          @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+          seek = @file.pos + target_info.path.bytesize + 1
+          @file.write "#{target_info.path}\t0000000000000000\t0000000000000000\n"
+          if @follow_inodes
+            @map[target_info.ino] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+          else
+            @map[target_info.path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+          end
         }
       end
 
-      def unwatch(path)
-        if (entry = @map.delete(path))
+      def unwatch(target_info)
+        if (entry = @map.delete(@follow_inodes ? target_info.ino : target_info.path))
           entry.update_pos(UNWATCHED_POSITION)
         end
       end
@@ -69,7 +75,11 @@ module Fluent::Plugin
             pos = m[2].to_i(16)
             ino = m[3].to_i(16)
             seek = @file.pos - line.bytesize + path.bytesize + 1
-            map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
+            if @follow_inodes
+              map[ino] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
+            else
+              map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
+            end
           end
         end
 
@@ -94,8 +104,9 @@ module Fluent::Plugin
             @file.truncate(0)
             @file.write(entries.values.map(&:to_entry_fmt).join)
 
-            entries.each do |path, val|
-              if (m = @map[path])
+            # entry contains path/ino key and value.
+            entries.each do |key, val|
+              if (m = @map[key])
                 m.seek = val.seek
               end
             end
@@ -139,13 +150,25 @@ module Fluent::Plugin
               @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
             end
 
-            entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+            if @follow_inodes
+              entries[ino] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+            else
+              entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+            end
             file_pos += line.size
           end
         end
 
+        entries = remove_deleted_files_entries(entries, @existing_paths) if @follow_inodes
         entries
       end
+
+      def remove_deleted_files_entries(existent_entries, existing_paths)
+        filtered_entries = existent_entries.select {|file_entry|
+          existing_paths.key?(file_entry)
+        }
+        filtered_entries
+      end
     end
 
     Entry = Struct.new(:path, :pos, :ino, :seek) do
@@ -224,5 +247,7 @@ module Fluent::Plugin
         @inode
       end
     end
+
+    TargetInfo = Struct.new(:path, :ino)
   end
 end

data/lib/fluent/plugin/in_tcp.rb

@@ -98,6 +98,7 @@ module Fluent::Plugin
     def start
       super
 
+      log.info "listening tcp socket", bind: @bind, port: @port
       del_size = @delimiter.length
       if @_extract_enabled && @_extract_tag_key
         server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|

data/lib/fluent/plugin/out_copy.rb

@@ -27,20 +27,28 @@ module Fluent::Plugin
     desc 'Pass different record to each `store` plugin by specified method'
     config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy
 
-    attr_reader :ignore_errors
+    attr_reader :ignore_errors, :ignore_if_prev_successes
 
     def initialize
       super
       @ignore_errors = []
+      @ignore_if_prev_successes = []
     end
 
     def configure(conf)
       super
 
       @copy_proc = gen_copy_proc
-      @stores.each { |store|
-        @ignore_errors << (store.arg == 'ignore_error')
+      @stores.each_with_index { |store, i|
+        if i == 0 && store.arg.include?('ignore_if_prev_success')
+          raise Fluent::ConfigError, "ignore_if_prev_success must specify 2nd or later <store> directives"
+        end
+        @ignore_errors << (store.arg.include?('ignore_error'))
+        @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
       }
+      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
+        log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
+      end
     end
 
     def multi_workers_ready?
@@ -55,10 +63,15 @@ module Fluent::Plugin
         }
         es = m
       end
-
+      success = Array.new(outputs.size)
       outputs.each_with_index do |output, i|
         begin
-          output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
+          if i > 0 && success[i - 1] && @ignore_if_prev_successes[i]
+            log.debug "ignore copy because prev_success in #{output.plugin_id}", index: i
+          else
+            output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
+            success[i] = true
+          end
         rescue => e
           if @ignore_errors[i]
             log.error "ignore emit error in #{output.plugin_id}", error: e

data/lib/fluent/plugin/out_exec_filter.rb

@@ -159,9 +159,9 @@ module Fluent::Plugin
         @added_prefix_string = @add_prefix + '.'
       end
 
-      @respawns = if @child_respawn.nil? or @child_respawn == 'none' or @child_respawn == '0'
+      @respawns = if @child_respawn.nil? || (@child_respawn == 'none') || (@child_respawn == '0')
                     0
-                  elsif @child_respawn == 'inf' or @child_respawn == '-1'
+                  elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
                     -1
                   elsif @child_respawn =~ /^\d+$/
                     @child_respawn.to_i
@@ -251,7 +251,7 @@ module Fluent::Plugin
 
     def tag_remove_prefix(tag)
       if @remove_prefix
-        if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @removed_prefix_string
+        if ((tag[0, @removed_length] == @removed_prefix_string) && (tag.length > @removed_length)) || (tag == @removed_prefix_string)
           tag = tag[@removed_length..-1] || ''
         end
       end

data/lib/fluent/plugin/out_forward.rb

@@ -166,6 +166,7 @@ module Fluent::Plugin
 
       @usock = nil
       @keep_alive_watcher_interval = 5 # TODO
+      @suspend_flush = false
     end
 
     def configure(conf)
@@ -291,6 +292,15 @@ module Fluent::Plugin
       @require_ack_response
     end
 
+    def overwrite_delayed_commit_timeout
+      # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
+      # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
+      if @delayed_commit_timeout != @ack_response_timeout
+        log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
+        @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
+      end
+    end
+
     def start
       super
 
@@ -303,13 +313,7 @@ module Fluent::Plugin
       end
 
       if @require_ack_response
-        # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
-        # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
-        if @delayed_commit_timeout != @ack_response_timeout
-          log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
-          @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
-        end
-
+        overwrite_delayed_commit_timeout
         thread_create(:out_forward_receiving_ack, &method(:ack_reader))
       end
 
@@ -346,6 +350,26 @@ module Fluent::Plugin
       end
     end
 
+    def before_shutdown
+      super
+      @suspend_flush = true
+    end
+
+    def after_shutdown
+      last_ack if @require_ack_response
+      super
+    end
+
+    def try_flush
+      return if @require_ack_response && @suspend_flush
+      super
+    end
+
+    def last_ack
+      overwrite_delayed_commit_timeout
+      ack_check(ack_select_interval)
+    end
+
     def write(chunk)
       return if chunk.empty?
       tag = chunk.metadata.tag
@@ -361,6 +385,7 @@ module Fluent::Plugin
       end
       tag = chunk.metadata.tag
       discovery_manager.select_service { |node| node.send_data(tag, chunk) }
+      last_ack if @require_ack_response && @suspend_flush
     end
 
     def create_transfer_socket(host, port, hostname, &block)
@@ -481,31 +506,39 @@ module Fluent::Plugin
       @connection_manager.purge_obsolete_socks
     end
 
+    def ack_select_interval
+      if @delayed_commit_timeout > 3
+        1
+      else
+        @delayed_commit_timeout / 3.0
+      end
+    end
+
     def ack_reader
-      select_interval = if @delayed_commit_timeout > 3
-                          1
-                        else
-                          @delayed_commit_timeout / 3.0
-                        end
+      select_interval = ack_select_interval
 
       while thread_current_running?
-        @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
-          @connection_manager.close(sock)
-
-          case result
-          when AckHandler::Result::SUCCESS
-            commit_write(chunk_id)
-          when AckHandler::Result::FAILED
-            node.disable!
-            rollback_write(chunk_id, update_retry: false)
-          when AckHandler::Result::CHUNKID_UNMATCHED
-            rollback_write(chunk_id, update_retry: false)
-          else
-            log.warn("BUG: invalid status #{result} #{chunk_id}")
+        ack_check(select_interval)
+      end
+    end
 
-            if chunk_id
-              rollback_write(chunk_id, update_retry: false)
-            end
+    def ack_check(select_interval)
+      @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
+        @connection_manager.close(sock)
+
+        case result
+        when AckHandler::Result::SUCCESS
+          commit_write(chunk_id)
+        when AckHandler::Result::FAILED
+          node.disable!
+          rollback_write(chunk_id, update_retry: false)
+        when AckHandler::Result::CHUNKID_UNMATCHED
+          rollback_write(chunk_id, update_retry: false)
+        else
+          log.warn("BUG: invalid status #{result} #{chunk_id}")
+
+          if chunk_id
+            rollback_write(chunk_id, update_retry: false)
           end
         end
       end

data/lib/fluent/plugin/out_http.rb

@@ -21,6 +21,16 @@ require 'fluent/tls'
 require 'fluent/plugin/output'
 require 'fluent/plugin_helper/socket'
 
+# patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758.
+# see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712
+unless Net::HTTP::SSL_IVNAMES.include?(:@extra_chain_cert)
+  class Net::HTTP
+    SSL_IVNAMES << :@extra_chain_cert
+    SSL_ATTRIBUTES << :extra_chain_cert
+    attr_accessor :extra_chain_cert
+  end
+end
+
 module Fluent::Plugin
   class HTTPOutput < Output
     Fluent::Plugin.register_output('http', self)
@@ -41,6 +51,8 @@ module Fluent::Plugin
     config_param :json_array, :bool, default: false
     desc 'Additional headers for HTTP request'
     config_param :headers, :hash, default: nil
+    desc 'Additional placeholder based headers for HTTP request'
+    config_param :headers_from_placeholders, :hash, default: nil
 
     desc 'The connection open timeout in seconds'
     config_param :open_timeout, :integer, default: nil
@@ -171,7 +183,15 @@ module Fluent::Plugin
         end
         if @tls_client_cert_path
           raise Fluent::ConfigError, "tls_client_cert_path is wrong: #{@tls_client_cert_path}" unless File.file?(@tls_client_cert_path)
-          opt[:cert] = OpenSSL::X509::Certificate.new(File.read(@tls_client_cert_path))
+
+          bundle = File.read(@tls_client_cert_path)
+          bundle_certs = bundle.scan(/-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/)
+          opt[:cert] = OpenSSL::X509::Certificate.new(bundle_certs[0])
+
+          intermediate_certs = bundle_certs[1..-1]
+          if intermediate_certs
+            opt[:extra_chain_cert] = intermediate_certs.map { |cert| OpenSSL::X509::Certificate.new(cert) }
+          end
         end
         if @tls_private_key_path
           raise Fluent::ConfigError, "tls_private_key_path is wrong: #{@tls_private_key_path}" unless File.file?(@tls_private_key_path)
@@ -195,12 +215,17 @@ module Fluent::Plugin
       URI.parse(endpoint)
     end
 
-    def set_headers(req)
+    def set_headers(req, chunk)
       if @headers
         @headers.each do |k, v|
           req[k] = v
         end
       end
+      if @headers_from_placeholders
+        @headers_from_placeholders.each do |k, v|
+          req[k] = extract_placeholders(v, chunk)
+        end
+      end
       req['Content-Type'] = @content_type
     end
 
@@ -214,8 +239,8 @@ module Fluent::Plugin
       if @auth
         req.basic_auth(@auth.username, @auth.password)
       end
-      set_headers(req)
-      req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read
+      set_headers(req, chunk)
+      req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
       req
     end
 

data/lib/fluent/plugin/output.rb

@@ -769,17 +769,19 @@ module Fluent
             end
           end
 
-          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
-            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
-          end
-
-          rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
+          rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
             if chunk_passed
               dump_unique_id_hex(chunk.unique_id)
             else
               log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
             end
           }
+
+          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
+            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
+          end
+
+          rvalue
         end
       end
 
@@ -1250,7 +1252,13 @@ module Fluent
             log.debug "buffer queue cleared"
             @retry = nil
           else
-            @retry.step
+            # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
+            # @retry.step is called almost as many times as the number of flush threads in a short time.
+            if Time.now >= @retry.next_time
+              @retry.step
+            else
+              @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
+            end
             if error
               if using_secondary
                 msg = "failed to flush the buffer with secondary output."

data/lib/fluent/plugin/storage_local.rb

@@ -87,7 +87,7 @@ module Fluent
           if File.exist?(@path)
             raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
             begin
-              data = open(@path, 'r:utf-8') { |io| io.read }
+              data = File.open(@path, 'r:utf-8') { |io| io.read }
               if data.empty?
                 log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
                 return
@@ -115,7 +115,7 @@ module Fluent
         return if @on_memory
         return unless File.exist?(@path)
         begin
-          json_string = open(@path, 'r:utf-8'){ |io| io.read }
+          json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
           json = Yajl::Parser.parse(json_string)
           unless json.is_a?(Hash)
             log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
@@ -133,7 +133,7 @@ module Fluent
         tmp_path = @path + '.tmp'
         begin
           json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
-          open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
+          File.open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
           File.rename(tmp_path, @path)
         rescue => e
           log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e