dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/plugin/out_forward/ack_handler.rb

@@ -0,0 +1,161 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+require 'fluent/plugin_helper/socket'
+require 'fluent/engine'
+require 'fluent/clock'
+
+module Fluent::Plugin
+  class ForwardOutput < Output
+    class AckHandler
+      module Result
+        SUCCESS = :success
+        FAILED = :failed
+        CHUNKID_UNMATCHED = :chunkid_unmatched
+      end
+
+      def initialize(timeout:, log:, read_length:)
+        @mutex = Mutex.new
+        @ack_waitings = []
+        @timeout = timeout
+        @log = log
+        @read_length = read_length
+        @unpacker = Fluent::MessagePackFactory.msgpack_unpacker
+      end
+
+      def collect_response(select_interval)
+        now = Fluent::Clock.now
+        sockets = []
+        results = []
+        begin
+          new_list = []
+          @mutex.synchronize do
+            @ack_waitings.each do |info|
+              if info.expired?(now)
+                # There are 2 types of cases when no response has been received from socket:
+                # (1) the node does not support sending responses
+                # (2) the node does support sending response but responses have not arrived for some reasons.
+                @log.warn 'no response from node. regard it as unavailable.', host: info.node.host, port: info.node.port
+                results << [info, Result::FAILED]
+              else
+                sockets << info.sock
+                new_list << info
+              end
+            end
+            @ack_waitings = new_list
+          end
+
+          readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
+          if readable_sockets
+            readable_sockets.each do |sock|
+              results << read_ack_from_sock(sock)
+            end
+          end
+
+          results.each do |info, ret|
+            if info.nil?
+              yield nil, nil, nil, ret
+            else
+              yield info.chunk_id, info.node, info.sock, ret
+            end
+          end
+        rescue => e
+          @log.error 'unexpected error while receiving ack', error: e
+          @log.error_backtrace
+        end
+      end
+
+      ACKWaitingSockInfo = Struct.new(:sock, :chunk_id, :chunk_id_base64, :node, :expired_time) do
+        def expired?(now)
+          expired_time < now
+        end
+      end
+
+      Ack = Struct.new(:chunk_id, :node, :handler) do
+        def enqueue(sock)
+          handler.enqueue(node, sock, chunk_id)
+        end
+      end
+
+      def create_ack(chunk_id, node)
+        Ack.new(chunk_id, node, self)
+      end
+
+      def enqueue(node, sock, cid)
+        info = ACKWaitingSockInfo.new(sock, cid, Base64.encode64(cid), node, Fluent::Clock.now + @timeout)
+        @mutex.synchronize do
+          @ack_waitings << info
+        end
+      end
+
+      private
+
+      def read_ack_from_sock(sock)
+        begin
+          raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
+        rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
+          raw_data = ''
+        end
+
+        info = find(sock)
+
+        # When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
+        # If this happens we assume the data wasn't delivered and retry it.
+        if raw_data.empty?
+          @log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port
+          # info.node.disable!
+          return info, Result::FAILED
+        else
+          @unpacker.feed(raw_data)
+          res = @unpacker.read
+          @log.trace 'getting response from destination', host: info.node.host, port: info.node.port, chunk_id: dump_unique_id_hex(info.chunk_id), response: res
+          if res['ack'] != info.chunk_id_base64
+            # Some errors may have occurred when ack and chunk id is different, so send the chunk again.
+            @log.warn 'ack in response and chunk id in sent data are different', chunk_id: dump_unique_id_hex(info.chunk_id), ack: res['ack']
+            return info, Result::CHUNKID_UNMATCHED
+          else
+            @log.trace 'got a correct ack response', chunk_id: dump_unique_id_hex(info.chunk_id)
+          end
+
+          return info, Result::SUCCESS
+        end
+      rescue => e
+        @log.error 'unexpected error while receiving ack message', error: e
+        @log.error_backtrace
+        [nil, Result::FAILED]
+      ensure
+        delete(info)
+      end
+
+      def dump_unique_id_hex(unique_id)
+        Fluent::UniqueId.hex(unique_id)
+      end
+
+      def find(sock)
+        @mutex.synchronize do
+          @ack_waitings.find { |info| info.sock == sock }
+        end
+      end
+
+      def delete(info)
+        @mutex.synchronize do
+          @ack_waitings.delete(info)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_forward/connection_manager.rb

@@ -0,0 +1,113 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class ForwardOutput < Output
+    class ConnectionManager
+      RequestInfo = Struct.new(:state, :shared_key_nonce, :auth)
+
+      # @param log [Logger]
+      # @param secure [Boolean]
+      # @param connection_factory [Proc]
+      # @param SocketCache [Fluent::ForwardOutput::SocketCache]
+      def initialize(log:, secure:, connection_factory:, socket_cache:)
+        @log = log
+        @secure = secure
+        @connection_factory = connection_factory
+        @socket_cache = socket_cache
+      end
+
+      def stop
+        @socket_cache && @socket_cache.clear
+      end
+
+      # @param ack [Fluent::Plugin::ForwardOutput::AckHander::Ack|nil]
+      def connect(host:, port:, hostname:, ack: nil, &block)
+        if @socket_cache
+          return connect_keepalive(host: host, port: port, hostname: hostname, ack: ack, &block)
+        end
+
+        @log.debug('connect new socket')
+        socket = @connection_factory.call(host, port, hostname)
+        request_info = RequestInfo.new(@secure ? :helo : :established)
+
+        unless block_given?
+          return [socket, request_info]
+        end
+
+        begin
+          yield(socket, request_info)
+        ensure
+          if ack
+            ack.enqueue(socket)
+          else
+            socket.close_write rescue nil
+            socket.close rescue nil
+          end
+        end
+      end
+
+      def purge_obsolete_socks
+        unless @socket_cache
+          raise "Do not call this method without keepalive option"
+        end
+        @socket_cache.purge_obsolete_socks
+      end
+
+      def close(sock)
+        if @socket_cache
+          @socket_cache.checkin(sock)
+        else
+          sock.close_write rescue nil
+          sock.close rescue nil
+        end
+      end
+
+      private
+
+      def connect_keepalive(host:, port:, hostname:, ack: nil)
+        request_info = RequestInfo.new(:established)
+        socket = @socket_cache.checkout_or([host, port, hostname]) do
+          s = @connection_factory.call(host, port, hostname)
+          request_info = RequestInfo.new(@secure ? :helo : :established) # overwrite if new connection
+          s
+        end
+
+        unless block_given?
+          return [socket, request_info]
+        end
+
+        ret = nil
+        begin
+          ret = yield(socket, request_info)
+        rescue
+          @socket_cache.revoke(socket)
+          raise
+        else
+          if ack
+            ack.enqueue(socket)
+          else
+            @socket_cache.checkin(socket)
+          end
+        end
+
+        ret
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_forward/error.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class ForwardOutput < Output
+    class Error < StandardError; end
+    class NoNodesAvailable < Error; end
+    class ConnectionClosedError < Error; end
+    class HandshakeError < Error; end
+    class HeloError < HandshakeError; end
+    class PingpongError < HandshakeError; end
+  end
+end

data/lib/fluent/plugin/out_forward/failure_detector.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class ForwardOutput < Output
+    class FailureDetector
+      PHI_FACTOR = 1.0 / Math.log(10.0)
+      SAMPLE_SIZE = 1000
+
+      def initialize(heartbeat_interval, hard_timeout, init_last)
+        @heartbeat_interval = heartbeat_interval
+        @last = init_last
+        @hard_timeout = hard_timeout
+
+        # microsec
+        @init_gap = (heartbeat_interval * 1e6).to_i
+        @window = [@init_gap]
+      end
+
+      def hard_timeout?(now)
+        now - @last > @hard_timeout
+      end
+
+      def add(now)
+        if @window.empty?
+          @window << @init_gap
+          @last = now
+        else
+          gap = now - @last
+          @window << (gap * 1e6).to_i
+          @window.shift if @window.length > SAMPLE_SIZE
+          @last = now
+        end
+      end
+
+      def phi(now)
+        size = @window.size
+        return 0.0 if size == 0
+
+        # Calculate weighted moving average
+        mean_usec = 0
+        fact = 0
+        @window.each_with_index {|gap,i|
+          mean_usec += gap * (1+i)
+          fact += (1+i)
+        }
+        mean_usec = mean_usec / fact
+
+        # Normalize arrive intervals into 1sec
+        mean = (mean_usec.to_f / 1e6) - @heartbeat_interval + 1
+
+        # Calculate phi of the phi accrual failure detector
+        t = now - @last - @heartbeat_interval + 1
+        phi = PHI_FACTOR * t / mean
+
+        return phi
+      end
+
+      def sample_size
+        @window.size
+      end
+
+      def clear
+        @window.clear
+        @last = 0
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_forward/handshake_protocol.rb

@@ -0,0 +1,125 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+require 'fluent/plugin/out_forward/error'
+require 'digest'
+
+module Fluent::Plugin
+  class ForwardOutput < Output
+    class HandshakeProtocol
+      def initialize(log:, hostname:, shared_key:, password:, username:)
+        @log = log
+        @hostname = hostname
+        @shared_key = shared_key
+        @password = password
+        @username = username
+        @shared_key_salt = generate_salt
+      end
+
+      def invoke(sock, ri, data)
+        @log.trace __callee__
+
+        case ri.state
+        when :helo
+          unless check_helo(ri, data)
+            raise HeloError, 'received invalid helo message'
+          end
+
+          sock.write(generate_ping(ri).to_msgpack)
+          ri.state = :pingpong
+        when :pingpong
+          succeeded, reason = check_pong(ri, data)
+          unless succeeded
+            raise PingpongError, reason
+          end
+
+          ri.state = :established
+        else
+          raise "BUG: unknown session state: #{ri.state}"
+        end
+      end
+
+      private
+
+      def check_pong(ri, message)
+        @log.debug('checking pong')
+        # ['PONG', bool(authentication result), 'reason if authentication failed',
+        #  self_hostname, sha512\_hex(salt + self_hostname + nonce + sharedkey)]
+        unless message.size == 5 && message[0] == 'PONG'
+          return false, 'invalid format for PONG message'
+        end
+        _pong, auth_result, reason, hostname, shared_key_hexdigest = message
+
+        unless auth_result
+          return false, 'authentication failed: ' + reason
+        end
+
+        if hostname == @hostname
+          return false, 'same hostname between input and output: invalid configuration'
+        end
+
+        clientside = Digest::SHA512.new.update(@shared_key_salt).update(hostname).update(ri.shared_key_nonce).update(@shared_key).hexdigest
+        unless shared_key_hexdigest == clientside
+          return false, 'shared key mismatch'
+        end
+
+        [true, nil]
+      end
+
+      def check_helo(ri, message)
+        @log.debug('checking helo')
+        # ['HELO', options(hash)]
+        unless message.size == 2 && message[0] == 'HELO'
+          return false
+        end
+
+        opts = message[1] || {}
+        # make shared_key_check failed (instead of error) if protocol version mismatch exist
+        ri.shared_key_nonce = opts['nonce'] || ''
+        ri.auth = opts['auth'] || ''
+        true
+      end
+
+      def generate_ping(ri)
+        @log.debug('generating ping')
+        # ['PING', self_hostname, sharedkey\_salt, sha512\_hex(sharedkey\_salt + self_hostname + nonce + shared_key),
+        #  username || '', sha512\_hex(auth\_salt + username + password) || '']
+        shared_key_hexdigest = Digest::SHA512.new.update(@shared_key_salt)
+          .update(@hostname)
+          .update(ri.shared_key_nonce)
+          .update(@shared_key)
+          .hexdigest
+        ping = ['PING', @hostname, @shared_key_salt, shared_key_hexdigest]
+        if !ri.auth.empty?
+          if @username.nil? || @password.nil?
+            raise PingpongError, "username and password are required"
+          end
+
+          password_hexdigest = Digest::SHA512.new.update(ri.auth).update(@username).update(@password).hexdigest
+          ping.push(@username, password_hexdigest)
+        else
+          ping.push('', '')
+        end
+        ping
+      end
+
+      def generate_salt
+        SecureRandom.hex(16)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_forward/load_balancer.rb

@@ -0,0 +1,114 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+require 'fluent/plugin/out_forward/error'
+
+module Fluent::Plugin
+  class ForwardOutput < Output
+    class LoadBalancer
+      def initialize(log)
+        @log = log
+        @weight_array = []
+        @rand_seed = Random.new.seed
+        @rr = 0
+        @mutex = Mutex.new
+      end
+
+      def select_healthy_node
+        error = nil
+
+        # Don't care about the change of @weight_array's size while looping since
+        # it's only used for determining the number of loops and it is not so important.
+        wlen = @weight_array.size
+        wlen.times do
+          node = @mutex.synchronize do
+            r = @rr % @weight_array.size
+            @rr = (r + 1) % @weight_array.size
+            @weight_array[r]
+          end
+          next unless node.available?
+
+          begin
+            ret = yield node
+            return ret, node
+          rescue
+            # for load balancing during detecting crashed servers
+            error = $!  # use the latest error
+          end
+        end
+
+        raise error if error
+        raise NoNodesAvailable, "no nodes are available"
+      end
+
+      def rebuild_weight_array(nodes)
+        standby_nodes, regular_nodes = nodes.select { |e| e.weight > 0 }.partition {|n|
+          n.standby?
+        }
+
+        lost_weight = 0
+        regular_nodes.each {|n|
+          unless n.available?
+            lost_weight += n.weight
+          end
+        }
+        @log.debug("rebuilding weight array", lost_weight: lost_weight)
+
+        if lost_weight > 0
+          standby_nodes.each {|n|
+            if n.available?
+              regular_nodes << n
+              @log.warn "using standby node #{n.host}:#{n.port}", weight: n.weight
+              lost_weight -= n.weight
+              break if lost_weight <= 0
+            end
+          }
+        end
+
+        weight_array = []
+        if regular_nodes.empty?
+          @log.warn('No nodes are available')
+          @mutex.synchronize do
+            @weight_array = weight_array
+          end
+          return @weight_array
+        end
+
+        gcd = regular_nodes.map {|n| n.weight }.inject(0) {|r,w| r.gcd(w) }
+        regular_nodes.each {|n|
+          (n.weight / gcd).times {
+            weight_array << n
+          }
+        }
+
+        # for load balancing during detecting crashed servers
+        coe = (regular_nodes.size * 6) / weight_array.size
+        weight_array *= coe if coe > 1
+
+        r = Random.new(@rand_seed)
+        weight_array.sort_by! { r.rand }
+
+        @mutex.synchronize do
+          @weight_array = weight_array
+        end
+      end
+
+      alias select_service select_healthy_node
+      alias rebalance rebuild_weight_array
+    end
+  end
+end