fluentd 1.12.0-x64-mingw32 → 1.13.0-x64-mingw32

data/lib/fluent/plugin/out_exec_filter.rb

@@ -159,9 +159,9 @@ module Fluent::Plugin
         @added_prefix_string = @add_prefix + '.'
       end
 
-      @respawns = if @child_respawn.nil? or @child_respawn == 'none' or @child_respawn == '0'
+      @respawns = if @child_respawn.nil? || (@child_respawn == 'none') || (@child_respawn == '0')
                     0
-                  elsif @child_respawn == 'inf' or @child_respawn == '-1'
+                  elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
                     -1
                   elsif @child_respawn =~ /^\d+$/
                     @child_respawn.to_i
@@ -251,7 +251,7 @@ module Fluent::Plugin
 
     def tag_remove_prefix(tag)
       if @remove_prefix
-        if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @removed_prefix_string
+        if ((tag[0, @removed_length] == @removed_prefix_string) && (tag.length > @removed_length)) || (tag == @removed_prefix_string)
           tag = tag[@removed_length..-1] || ''
         end
       end

data/lib/fluent/plugin/out_forward.rb

@@ -166,6 +166,7 @@ module Fluent::Plugin
 
       @usock = nil
       @keep_alive_watcher_interval = 5 # TODO
+      @suspend_flush = false
     end
 
     def configure(conf)
@@ -226,31 +227,14 @@ module Fluent::Plugin
         socket_cache: socket_cache,
       )
 
-      configs = []
-
-      # rewrite for using server as sd_static
-      conf.elements(name: 'server').each do |s|
-        s.name = 'service'
-      end
-
-      unless conf.elements(name: 'service').empty?
-        # To copy `services` element only
-        new_elem = Fluent::Config::Element.new('static_service_discovery', {}, {}, conf.elements(name: 'service'))
-        configs << { type: :static, conf: new_elem }
-      end
-
-      conf.elements(name: 'service_discovery').each_with_index do |c, i|
-        configs << { type: @service_discovery[i][:@type], conf: c }
-      end
-
-      service_discovery_create_manager(
+      service_discovery_configure(
         :out_forward_service_discovery_watcher,
-        configurations: configs,
+        static_default_service_directive: 'server',
         load_balancer: LoadBalancer.new(log),
         custom_build_method: method(:build_node),
       )
 
-      discovery_manager.services.each do |server|
+      service_discovery_services.each do |server|
         # it's only for test
         @nodes << server
         unless @heartbeat_type == :none
@@ -272,7 +256,7 @@ module Fluent::Plugin
         end
       end
 
-      if discovery_manager.services.empty?
+      if service_discovery_services.empty?
         raise Fluent::ConfigError, "forward output plugin requires at least one node is required. Add <server> or <service_discovery>"
       end
 
@@ -291,30 +275,33 @@ module Fluent::Plugin
       @require_ack_response
     end
 
+    def overwrite_delayed_commit_timeout
+      # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
+      # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
+      if @delayed_commit_timeout != @ack_response_timeout
+        log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
+        @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
+      end
+    end
+
     def start
       super
 
       unless @heartbeat_type == :none
         if @heartbeat_type == :udp
-          @usock = socket_create_udp(discovery_manager.services.first.host, discovery_manager.services.first.port, nonblock: true)
+          @usock = socket_create_udp(service_discovery_services.first.host, service_discovery_services.first.port, nonblock: true)
           server_create_udp(:out_forward_heartbeat_receiver, 0, socket: @usock, max_bytes: @read_length, &method(:on_udp_heatbeat_response_recv))
         end
         timer_execute(:out_forward_heartbeat_request, @heartbeat_interval, &method(:on_heartbeat_timer))
       end
 
       if @require_ack_response
-        # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
-        # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
-        if @delayed_commit_timeout != @ack_response_timeout
-          log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
-          @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
-        end
-
+        overwrite_delayed_commit_timeout
         thread_create(:out_forward_receiving_ack, &method(:ack_reader))
       end
 
       if @verify_connection_at_startup
-        discovery_manager.services.each do |node|
+        service_discovery_services.each do |node|
           begin
             node.verify_connection
           rescue StandardError => e
@@ -346,11 +333,31 @@ module Fluent::Plugin
       end
     end
 
+    def before_shutdown
+      super
+      @suspend_flush = true
+    end
+
+    def after_shutdown
+      last_ack if @require_ack_response
+      super
+    end
+
+    def try_flush
+      return if @require_ack_response && @suspend_flush
+      super
+    end
+
+    def last_ack
+      overwrite_delayed_commit_timeout
+      ack_check(ack_select_interval)
+    end
+
     def write(chunk)
       return if chunk.empty?
       tag = chunk.metadata.tag
 
-      discovery_manager.select_service { |node| node.send_data(tag, chunk) }
+      service_discovery_select_service { |node| node.send_data(tag, chunk) }
     end
 
     def try_write(chunk)
@@ -360,7 +367,8 @@ module Fluent::Plugin
         return
       end
       tag = chunk.metadata.tag
-      discovery_manager.select_service { |node| node.send_data(tag, chunk) }
+      service_discovery_select_service { |node| node.send_data(tag, chunk) }
+      last_ack if @require_ack_response && @suspend_flush
     end
 
     def create_transfer_socket(host, port, hostname, &block)
@@ -409,7 +417,7 @@ module Fluent::Plugin
 
     def statistics
       stats = super
-      services = discovery_manager.services
+      services = service_discovery_services
       healthy_nodes_count = 0
       registed_nodes_count = services.size
       services.each do |s|
@@ -446,7 +454,7 @@ module Fluent::Plugin
 
     def on_heartbeat_timer
       need_rebuild = false
-      discovery_manager.services.each do |n|
+      service_discovery_services.each do |n|
         begin
           log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
           n.usock = @usock if @usock
@@ -461,16 +469,16 @@ module Fluent::Plugin
       end
 
       if need_rebuild
-        discovery_manager.rebalance
+        service_discovery_rebalance
       end
     end
 
     def on_udp_heatbeat_response_recv(data, sock)
       sockaddr = Socket.pack_sockaddr_in(sock.remote_port, sock.remote_host)
-      if node = discovery_manager.services.find { |n| n.sockaddr == sockaddr }
+      if node = service_discovery_services.find { |n| n.sockaddr == sockaddr }
         # log.trace "heartbeat arrived", name: node.name, host: node.host, port: node.port
         if node.heartbeat
-          discovery_manager.rebalance
+          service_discovery_rebalance
         end
       else
         log.warn("Unknown heartbeat response received from #{sock.remote_host}:#{sock.remote_port}. It may service out")
@@ -481,31 +489,39 @@ module Fluent::Plugin
       @connection_manager.purge_obsolete_socks
     end
 
+    def ack_select_interval
+      if @delayed_commit_timeout > 3
+        1
+      else
+        @delayed_commit_timeout / 3.0
+      end
+    end
+
     def ack_reader
-      select_interval = if @delayed_commit_timeout > 3
-                          1
-                        else
-                          @delayed_commit_timeout / 3.0
-                        end
+      select_interval = ack_select_interval
 
       while thread_current_running?
-        @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
-          @connection_manager.close(sock)
-
-          case result
-          when AckHandler::Result::SUCCESS
-            commit_write(chunk_id)
-          when AckHandler::Result::FAILED
-            node.disable!
-            rollback_write(chunk_id, update_retry: false)
-          when AckHandler::Result::CHUNKID_UNMATCHED
-            rollback_write(chunk_id, update_retry: false)
-          else
-            log.warn("BUG: invalid status #{result} #{chunk_id}")
+        ack_check(select_interval)
+      end
+    end
 
-            if chunk_id
-              rollback_write(chunk_id, update_retry: false)
-            end
+    def ack_check(select_interval)
+      @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
+        @connection_manager.close(sock)
+
+        case result
+        when AckHandler::Result::SUCCESS
+          commit_write(chunk_id)
+        when AckHandler::Result::FAILED
+          node.disable!
+          rollback_write(chunk_id, update_retry: false)
+        when AckHandler::Result::CHUNKID_UNMATCHED
+          rollback_write(chunk_id, update_retry: false)
+        else
+          log.warn("BUG: invalid status #{result} #{chunk_id}")
+
+          if chunk_id
+            rollback_write(chunk_id, update_retry: false)
           end
         end
       end

data/lib/fluent/plugin/out_http.rb

@@ -51,6 +51,8 @@ module Fluent::Plugin
     config_param :json_array, :bool, default: false
     desc 'Additional headers for HTTP request'
     config_param :headers, :hash, default: nil
+    desc 'Additional placeholder based headers for HTTP request'
+    config_param :headers_from_placeholders, :hash, default: nil
 
     desc 'The connection open timeout in seconds'
     config_param :open_timeout, :integer, default: nil
@@ -213,12 +215,17 @@ module Fluent::Plugin
       URI.parse(endpoint)
     end
 
-    def set_headers(req)
+    def set_headers(req, chunk)
       if @headers
         @headers.each do |k, v|
           req[k] = v
         end
       end
+      if @headers_from_placeholders
+        @headers_from_placeholders.each do |k, v|
+          req[k] = extract_placeholders(v, chunk)
+        end
+      end
       req['Content-Type'] = @content_type
     end
 
@@ -232,7 +239,7 @@ module Fluent::Plugin
       if @auth
         req.basic_auth(@auth.username, @auth.password)
       end
-      set_headers(req)
+      set_headers(req, chunk)
       req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
       req
     end

data/lib/fluent/plugin/output.rb

@@ -754,7 +754,17 @@ module Fluent
               log.warn "tag placeholder '#{$1}' not replaced. tag:#{metadata.tag}, template:#{str}"
             end
           end
-          # ${a_chunk_key}, ...
+
+          # First we replace ${chunk_id} with chunk.unique_id (hexlified).
+          rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
+            if chunk_passed
+              dump_unique_id_hex(chunk.unique_id)
+            else
+              log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
+            end
+          }
+
+          # Then, replace other ${chunk_key}s.
           if !@chunk_keys.empty? && metadata.variables
             hash = {'${tag}' => '${tag}'} # not to erase this wrongly
             @chunk_keys.each do |key|
@@ -769,14 +779,6 @@ module Fluent
             end
           end
 
-          rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
-            if chunk_passed
-              dump_unique_id_hex(chunk.unique_id)
-            else
-              log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
-            end
-          }
-
           if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
             log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
           end

data/lib/fluent/plugin/parser_csv.rb

@@ -28,13 +28,13 @@ module Fluent
       desc 'The delimiter character (or string) of CSV values'
       config_param :delimiter, :string, default: ','
       desc 'The parser type used to parse CSV line'
-      config_param :parser_type, :enum, list: [:normal, :fast], default: :normal
+      config_param :parser_engine, :enum, list: [:normal, :fast], default: :normal, alias: :parser_type
 
       def configure(conf)
         super
 
 
-        if @parser_type == :fast
+        if @parser_engine == :fast
           @quote_char = '"'
           @escape_pattern = Regexp.compile(@quote_char * 2)
 

data/lib/fluent/plugin/parser_syslog.rb

@@ -56,7 +56,7 @@ module Fluent
       desc 'Specify time format for event time for rfc5424 protocol'
       config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
       desc 'The parser type used to parse syslog message'
-      config_param :parser_type, :enum, list: [:regexp, :string], default: :regexp
+      config_param :parser_engine, :enum, list: [:regexp, :string], default: :regexp, alias: :parser_type
       desc 'support colonless ident in string parser'
       config_param :support_colonless_ident, :bool, default: true
 
@@ -79,7 +79,7 @@ module Fluent
       def configure(conf)
         super
 
-        @regexp_parser = @parser_type == :regexp
+        @regexp_parser = @parser_engine == :regexp
         @regexp = case @message_format
                   when :rfc3164
                     if @regexp_parser

data/lib/fluent/plugin/storage_local.rb

@@ -87,7 +87,7 @@ module Fluent
           if File.exist?(@path)
             raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
             begin
-              data = open(@path, 'r:utf-8') { |io| io.read }
+              data = File.open(@path, 'r:utf-8') { |io| io.read }
               if data.empty?
                 log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
                 return
@@ -115,7 +115,7 @@ module Fluent
         return if @on_memory
         return unless File.exist?(@path)
         begin
-          json_string = open(@path, 'r:utf-8'){ |io| io.read }
+          json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
           json = Yajl::Parser.parse(json_string)
           unless json.is_a?(Hash)
             log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
@@ -130,10 +130,10 @@ module Fluent
 
       def save
         return if @on_memory
-        tmp_path = @path + '.tmp'
+        tmp_path = @path + '.tmp.' + Fluent::UniqueId.hex(Fluent::UniqueId.generate)
         begin
           json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
-          open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
+          File.open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
           File.rename(tmp_path, @path)
         rescue => e
           log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e

data/lib/fluent/plugin_helper/server.rb

@@ -709,11 +709,13 @@ module Fluent
                 return true
               end
             rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
-              @log.trace "unexpected error before accepting TLS connection", error: e
+              @log.trace "unexpected error before accepting TLS connection",
+                         host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
               close rescue nil
             rescue OpenSSL::SSL::SSLError => e
               # Use same log level as on_readable
-              @log.warn "unexpected error before accepting TLS connection by OpenSSL", error: e
+              @log.warn "unexpected error before accepting TLS connection by OpenSSL",
+                        host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
               close rescue nil
             end
 

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -22,10 +22,19 @@ module Fluent
     module ServiceDiscovery
       include Fluent::PluginHelper::Timer
 
+      # For the compatibility with older versions without `param_name: :service_discovery_configs`
+      attr_reader :service_discovery
+
       def self.included(mod)
         mod.include ServiceDiscoveryParams
       end
 
+      def configure(conf)
+        super
+        # For the compatibility with older versions without `param_name: :service_discovery_configs`
+        @service_discovery = @service_discovery_configs
+      end
+
       def start
         unless @discovery_manager
           log.warn('There is no discovery_manager. skip start them')
@@ -52,6 +61,35 @@ module Fluent
 
       private
 
+      # @param title [Symbol] the thread name. this value should be unique.
+      # @param static_default_service_directive [String] the directive name of each service when "static" service discovery is enabled in default
+      # @param load_balancer [Object] object which has two methods #rebalance and #select_service
+      # @param custom_build_method [Proc]
+      def service_discovery_configure(title, static_default_service_directive: nil, load_balancer: nil, custom_build_method: nil, interval: 3)
+        configs = @service_discovery_configs.map(&:corresponding_config_element)
+        if static_default_service_directive
+          configs.prepend Fluent::Config::Element.new(
+            'service_discovery',
+            '',
+            {'@type' => 'static'},
+            @config.elements(name: static_default_service_directive.to_s).map{|e| Fluent::Config::Element.new('service', e.arg, e.dup, e.elements, e.unused) }
+          )
+        end
+        service_discovery_create_manager(title, configurations: configs, load_balancer: load_balancer, custom_build_method: custom_build_method, interval: interval)
+      end
+
+      def service_discovery_select_service(&block)
+        @discovery_manager.select_service(&block)
+      end
+
+      def service_discovery_services
+        @discovery_manager.services
+      end
+
+      def service_discovery_rebalance
+        @discovery_manager.rebalance
+      end
+
       # @param title [Symbol] the thread name. this value should be unique.
       # @param configurations [Hash] hash which must has discivery_service type and its configuration like `{ type: :static, conf: <Fluent::Config::Element> }`
       # @param load_balancer [Object] object which has two methods #rebalance and #select_service
@@ -78,7 +116,7 @@ module Fluent
       module ServiceDiscoveryParams
         include Fluent::Configurable
 
-        config_section :service_discovery do
+        config_section :service_discovery, multi: true, param_name: :service_discovery_configs do
           config_param :@type, :string
         end
       end