log-courier 1.10.0 → 2.7.3

data/lib/log-courier/server_zmq.rb

@@ -1,400 +0,0 @@
-# encoding: utf-8
-
-# Copyright 2014-2019 Jason Woods and Contributors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-require 'thread'
-require 'log-courier/zmq_qpoll'
-
-module LogCourier
-  # ZMQ transport implementation for the server
-  class ServerZmq
-    class ZMQError < StandardError; end
-
-    class << self
-      @print_zmq_versions = false
-
-      def print_zmq_versions(logger)
-        return if @print_zmq_versions || logger.nil?
-
-        libversion = LibZMQ.version
-        libversion = "#{libversion[:major]}.#{libversion[:minor]}.#{libversion[:patch]}"
-
-        logger.info 'libzmq', :version => libversion
-        logger.info 'ffi-rzmq-core', :version => LibZMQ::VERSION
-        logger.info 'ffi-rzmq', :version => ZMQ.version
-
-        @print_zmq_versions = true
-      end
-    end
-
-    attr_reader :port
-
-    def initialize(options = {})
-      @options = {
-        logger:           nil,
-        transport:        'zmq',
-        port:             0,
-        address:          '0.0.0.0',
-        curve_secret_key: nil,
-        max_packet_size:  10_485_760,
-        peer_recv_queue:  10,
-      }.merge!(options)
-
-      @logger = @options[:logger]
-
-      self.class.print_zmq_versions @logger
-
-      if @options[:transport] == 'zmq'
-        fail "input/courier: Transport 'zmq' requires libzmq version >= 4" unless LibZMQ.version4?
-
-        fail 'input/courier: \'curve_secret_key\' is required' if @options[:curve_secret_key].nil?
-
-        fail 'input/courier: \'curve_secret_key\' must be a valid 40 character Z85 encoded string' if @options[:curve_secret_key].length != 40 || !z85validate(@options[:curve_secret_key])
-      end
-
-      begin
-        @context = ZMQ::Context.new
-        fail ZMQError, 'context creation error: ' + ZMQ::Util.error_string if @context.nil?
-
-        # Router so we can send multiple responses
-        @socket = @context.socket(ZMQ::ROUTER)
-        fail ZMQError, 'socket creation error: ' + ZMQ::Util.error_string if @socket.nil?
-
-        rc = @socket.setsockopt(ZMQ::LINGER, 0)
-        fail ZMQError, 'setsockopt LINGER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
-
-        if @options[:transport] == 'zmq'
-          rc = @socket.setsockopt(ZMQ::CURVE_SERVER, 1)
-          fail 'setsockopt CURVE_SERVER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
-
-          rc = @socket.setsockopt(ZMQ::CURVE_SECRETKEY, @options[:curve_secret_key])
-          fail 'setsockopt CURVE_SECRETKEY failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
-        end
-
-        bind = 'tcp://' + @options[:address] + (@options[:port] == 0 ? ':*' : ':' + @options[:port].to_s)
-        rc = @socket.bind(bind)
-        fail 'failed to bind at ' + bind + ': ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
-
-        # Lookup port number that was allocated in case it was set to 0
-        endpoint = ''
-        rc = @socket.getsockopt(ZMQ::LAST_ENDPOINT, endpoint)
-        fail 'getsockopt LAST_ENDPOINT failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc) && %r{\Atcp://(?:.*):(?<endpoint_port>\d+)\0\z} =~ endpoint
-        @port = endpoint_port.to_i
-
-        if @options[:port] == 0
-          @logger.warn 'Ephemeral port allocated', :transport => @options[:transport], :port => @port unless @logger.nil?
-        end
-      rescue => e
-        raise "input/courier: Failed to initialise: #{e}"
-      end
-
-      # TODO: Implement workers option by receiving on a ROUTER and proxying to a DEALER, with workers connecting to the DEALER
-
-      # TODO: Make this send queue configurable?
-      @send_queue = EventQueue.new 2
-      @factory = ClientFactoryZmq.new(@options, @send_queue)
-
-      # Setup poller
-      @poller = ZMQPoll::ZMQPoll.new(@context)
-      @poller.register_socket @socket, ZMQ::POLLIN
-      @poller.register_queue_to_socket @send_queue, @socket
-    end
-
-    def run(&block)
-      errors = 0
-      loop do
-        begin
-          @poller.poll(5_000) do |socket, r, w|
-            next if socket != @socket
-            next if !r
-
-            receive &block
-          end
-          errors = 0
-        rescue ZMQPoll::ZMQError => e
-          @logger.warn e, :hint => 'ZMQ poll failure' unless @logger.nil?
-          next
-        rescue ZMQPoll::ZMQTerm
-          # Fall into shutdown signal, context was terminated
-          # This can happen in JRuby - it seems to run finalisers too early
-          fail ShutdownSignal
-        rescue ZMQPoll::TimeoutError
-          # We'll let ZeroMQ manage reconnections and new connections
-          # There is no point in us doing any form of reconnect ourselves
-          next
-        end
-      end
-      return
-    rescue ShutdownSignal
-      # Shutting down
-      @logger.warn 'Server shutting down' unless @logger.nil?
-      return
-    rescue StandardError, NativeException => e
-      # Some other unknown problem
-      @logger.warn e, :hint => 'Unknown error, shutting down' unless @logger.nil?
-      return
-    ensure
-      @poller.shutdown
-      @factory.shutdown
-      @socket.close
-      @context.terminate
-    end
-
-    private
-
-    def z85validate(z85)
-      # ffi-rzmq does not implement decode - but we want to validate during startup
-      decoded = FFI::MemoryPointer.from_string(' ' * (8 * z85.length / 10))
-      ret = LibZMQ.zmq_z85_decode decoded, z85
-      return false if ret.nil?
-      true
-    end
-
-    def receive(&block)
-      # Try to receive a message
-      data = []
-      rc = @socket.recv_strings(data, ZMQ::DONTWAIT)
-      unless ZMQ::Util.resultcode_ok?(rc)
-        fail ZMQError, 'recv_string error: ' + ZMQ::Util.error_string if ZMQ::Util.errno != ZMQ::EAGAIN
-      end
-
-      # Save the source information that appears before the null messages
-      source = []
-      source.push data.shift until data.length == 0 || data[0] == ''
-
-      if data.length == 0
-        @logger.warn 'Invalid message: no data', :source_length => source.length unless @logger.nil?
-        return
-      elsif data.length == 1
-        @logger.warn 'Invalid message: empty data', :source_length => source.length unless @logger.nil?
-        return
-      end
-
-      # Drop the null message separator
-      data.shift
-
-      if data.length != 1
-        @logger.warn 'Invalid message: multipart unexpected', :source_length => source.length, :data_length => data.length unless @logger.nil?
-        if !@logger.nil? && @logger.debug?
-          i = 0
-          parts = {}
-          data.each do |msg|
-            i += 1
-            parts[i] = "#{part.length}:[#{msg[0..31].gsub(/[^[:print:]]/, '.')}]"
-          end
-          @logger.debug 'Data', parts
-        end
-        return
-      end
-
-      @factory.deliver source, data.first, &block
-      return
-    end
-  end
-
-  class ClientFactoryZmq
-    attr_reader :options
-    attr_reader :send_queue
-
-    def initialize(options, send_queue)
-      @options = options
-      @logger = @options[:logger]
-
-      @send_queue = send_queue
-      @index = {}
-      @client_threads = {}
-      @mutex = Mutex.new
-    end
-
-    def shutdown
-      # Stop other threads from try_drop collisions
-      client_threads = @mutex.synchronize do
-        client_threads = @client_threads
-        @client_threads = {}
-        client_threads
-      end
-
-      client_threads.each_value do |thr|
-        thr.raise ShutdownSignal
-      end
-
-      client_threads.each_value(&:join)
-      return
-    end
-
-    def deliver(source, data, &block)
-      # Find the handling thread
-      # We separate each source into threads so that each thread can respond
-      # with partial ACKs if we hit a slow down
-      # If we processed in a single thread, we'd only be able to respond to
-      # a single client with partial ACKs
-      @mutex.synchronize do
-        index = @index
-        source.each do |identity|
-          index[identity] = {} if !index.key?(identity)
-          index = index[identity]
-        end
-
-        if !index.key?('')
-          source_str = source.map do |s|
-            s.each_byte.map do |b|
-              b.to_s(16).rjust(2, '0')
-            end
-          end.join
-
-          @logger.info 'New source', :source => source_str unless @logger.nil?
-
-          # Create the client and associated thread
-          client = ClientZmq.new(self, source, source_str) do
-            try_drop source, source_str
-          end
-
-          thread = Thread.new do
-            client.run &block
-          end
-
-          @client_threads[thread] = thread
-
-          index[''] = {
-            'client' => client,
-            'thread' => thread,
-          }
-        end
-
-        # Existing thread, throw on the queue, if not enough room (timeout) drop the message
-        begin
-          index['']['client'].push data, 0
-        rescue LogCourier::TimeoutError
-          # TODO: Log a warning about this?
-        end
-      end
-      return
-    end
-
-    private
-
-    def try_drop(source, source_str)
-      # This is called when a client goes idle, to cleanup resources
-      # We may tie this into zmq monitor
-      @mutex.synchronize do
-        index = @index
-        parents = []
-        source.each do |identity|
-          if !index.key?(identity)
-            @logger.warn 'Unknown idle source failed to shutdown', :source => source_str unless @logger.nil?
-            break
-          end
-          parents.push [index, identity]
-          index = index[identity]
-        end
-
-        if !index.key?('')
-          @logger.warn 'Unknown idle source failed to shutdown', :source => source_str unless @logger.nil?
-          break
-        end
-
-        # Don't allow drop if we have messages in the queue
-        if index['']['client'].length != 0
-          @logger.warn 'Failed idle source shutdown as message queue is not empty', :source => source_str unless @logger.nil?
-          return false
-        end
-
-        @logger.info 'Idle source shutting down', :source => source_str unless @logger.nil?
-
-        # Delete the entry
-        @client_threads.delete(index['']['thread'])
-        index.delete('')
-
-        # Cleanup orphaned leafs
-        parents.reverse_each do |path|
-          path[0].delete(path[1]) if path[0][path[1]].length == 0
-        end
-      end
-
-      return true
-    end
-  end
-
-  class ClientZmq < EventQueue
-    def initialize(factory, source, source_str, &try_drop)
-      @factory = factory
-      @logger = @factory.options[:logger]
-      @send_queue = @factory.send_queue
-      @source = source
-      @source_str = source_str
-      @try_drop = try_drop
-
-      # Setup the queue for receiving events to process
-      super @factory.options[:peer_recv_queue]
-    end
-
-    def run(&block)
-      loop do
-        begin
-          # TODO: Make timeout configurable?
-          data = self.pop(30)
-          recv(data, &block)
-        rescue TimeoutError
-          # Try to clean up resources - if we fail, new messages have arrived
-          retry if !@try_drop.call(@source)
-          break
-        end
-      end
-      return
-    rescue ShutdownSignal
-      # Shutting down
-      @logger.info 'Source shutting down', :source => @source_str unless @logger.nil?
-      return
-    rescue StandardError, NativeException => e
-      # Some other unknown problem
-      @logger.warn e, :hint => 'Unknown error, connection aborted', :source => @source_str unless @logger.nil?
-      raise e
-    end
-
-    def send(signature, message)
-      data = signature + [message.length].pack('N') + message
-      @send_queue.push @source + ['', data]
-      return
-    end
-
-    def add_fields(event)
-    end
-
-    private
-
-    def recv(data)
-      if data.length < 8
-        @logger.warn 'Invalid message: not enough data', :data_length => data.length, :source => @source_str unless @logger.nil?
-        return
-      end
-
-      # Unpack the header
-      signature, length = data.unpack('A4N')
-
-      # Verify length
-      if data.length - 8 != length
-        @logger.warn 'Invalid message: data has invalid length', :data_length => data.length - 8, :encoded_length => length, :source => @source_str unless @logger.nil?
-        return
-      elsif length > @factory.options[:max_packet_size]
-        @logger.warn 'Invalid message: packet too large', :size => length, :max_packet_size => @options[:max_packet_size], :source => @source_str unless @logger.nil?
-        return
-      end
-
-      # Yield the parts
-      yield signature, data[8, length], self
-      return
-    end
-  end
-end

data/lib/log-courier/zmq_qpoll.rb

@@ -1,324 +0,0 @@
-# encoding: utf-8
-
-# Copyright 2014-2019 Jason Woods and Contributors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-begin
-	require 'ffi-rzmq'
-	require 'ffi-rzmq/version'
-	require 'ffi-rzmq-core/version'
-rescue LoadError => e
-	raise "ZMQPoll could not initialise: #{e}"
-end
-
-module ZMQPoll
-	class ZMQError < StandardError; end
-	class ZMQTerm < StandardError; end
-	class TimeoutError < StandardError; end
-
-	class ZMQPoll
-		def initialize(context, logger=nil)
-			@logger = logger
-			@context = context
-			@poller = ZMQ::Poller.new
-			@sockets = []
-			@socket_to_socket = []
-			@handlers = {}
-			@queues = {}
-		end
-
-		def readables
-			@poller.readables
-		end
-
-		def writables
-			@poller.writables
-		end
-
-		def shutdown
-			@queues.each_key do |queue|
-				deregister_queue queue
-			end
-
-			@socket_to_socket.each do |socket|
-				_close_socket_to_socket socket
-			end
-
-			@sockets.each do |socket|
-				socket.close
-			end
-			return
-		end
-
-		def register_socket(socket, flags)
-			@poller.register socket, flags
-			return
-		end
-
-		def deregister_socket(socket)
-			return if @handlers.key?(socket)
-
-			@poller.delete socket
-			return
-		end
-
-		def register_queue_to_socket(queue, socket)
-			s2s_state = _create_socket_to_socket(socket)
-
-			state = {
-				state:    s2s_state,
-				mutex:    Mutex.new,
-				shutdown: false,
-			}
-
-			state[:thread] = Thread.new do
-				loop do
-					data = queue.pop
-					break if data.nil?
-					begin
-						send s2s_state[:sender], data
-					rescue TimeoutError
-						state[:mutex].synchronize do
-							break if state[:shutdown]
-						end
-						retry
-					end
-				end
-			end
-
-			@queues[queue] = state
-			return
-		end
-
-		def deregister_queue(queue)
-			return if !@queues.key?(queue)
-
-			# Push nil so if we're idle we jump into action and exit
-			# But also set shutdown to try so if we're mid-send and timeout, we exit
-			@queues[queue][:mutex].synchronize do
-				queue.push nil
-				@queues[queue][:shutdown] = true
-			end
-			@queues[queue][:thread].join
-
-			_close_socket_to_socket @queues[queue][:state]
-			@queues.delete queue
-			return
-		end
-
-		def create_socket_to_socket(socket)
-			state = _create_socket_to_socket(socket)
-			@socket_to_socket[state[:sender]] = state
-			state[:sender]
-		end
-
-		def close_socket_to_socket(socket)
-			return if !@socket_to_socket.include?(socket)
-			state = @socket_to_socket[socket]
-			@socket_to_socket.delete socket
-			_close_socket_to_socket(state)
-			return
-		end
-
-		def poll(timeout)
-			if @poller.size == 0
-				fail ZMQError, 'poll run called with zero socket/queues'
-			end
-
-			rc = @poller.poll(timeout)
-			unless ZMQ::Util.resultcode_ok?(rc)
-				# If we get ETERM or ENOTSOCK - we may have hit JRuby bug where finalisers are called too early
-				# We throw it so the caller knows NOT to retry and hit a bad loop
-				# The downside is - this Jruby bug will cause an inevitable deadlock
-				# The finaliser for the context will have run before finalisers for sockets (random order?)
-				# The resulting ZMQ context termination will block, waiting for sockets to be closed
-				# However, when closing a socket, ffi-rzmq will attempt to remove it's finaliser
-				# This removal will deadlock as it attempts to lock the finaliser list, which is locked by the
-				# thread that is running the ZMQ context termination, which is blocked...
-				# TODO: Raise issue in JRuby github and try to track down why finalisers run while code is running
-				# if the exit! call is made
-				fail ZMQTerm, 'poll error: ' + ZMQ::Util.error_string if ZMQ::Util.errno == ZMQ::ETERM || ZMQ::Util.errno == ZMQ::ENOTSOCK
-				fail ZMQError, 'poll error: ' + ZMQ::Util.error_string
-			end
-
-			return if rc == 0
-
-			ready = (@poller.readables|@poller.writables)
-
-			ready.each do |socket|
-				if @handlers.key?(socket)
-					__send__ @handlers[socket][:callback], @handlers[socket]
-				end
-
-				yield socket, @poller.readables.include?(socket), @poller.writables.include?(socket)
-			end
-
-			return
-		end
-
-		private
-
-		def _create_socket_to_socket(socket)
-			receiver = @context.socket(ZMQ::PULL)
-			fail ZMQError, 'socket creation error: ' + ZMQ::Util.error_string if receiver.nil?
-
-			rc = receiver.setsockopt(ZMQ::LINGER, 0)
-			fail ZMQError, 'setsockopt LINGER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
-
-			rc = receiver.bind("inproc://zmqpollreceiver-#{receiver.hash}")
-			fail ZMQError, 'bind error: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
-
-			sender = @context.socket(ZMQ::PUSH)
-			fail ZMQError, 'socket creation error: ' + ZMQ::Util.error_string if sender.nil?
-
-			rc = sender.setsockopt(ZMQ::LINGER, 0)
-			fail ZMQError, 'setsockopt LINGER failure: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
-
-			rc = sender.connect("inproc://zmqpollreceiver-#{receiver.hash}")
-			fail ZMQError, 'bind error: ' + ZMQ::Util.error_string unless ZMQ::Util.resultcode_ok?(rc)
-
-			state = {
-				:callback => :handle_socket_to_socket,
-				:sender   => sender,
-				:receiver => receiver,
-				:socket   => socket,
-				:buffer   => nil,
-				:send_ok  => false,
-				:recv_ok  => false,
-			}
-
-			@poller.register receiver, ZMQ::POLLIN
-			@poller.register socket, ZMQ::POLLOUT
-			@handlers[receiver] = state
-			@handlers[socket] = state
-
-			@sockets.push sender
-
-			state
-		end
-
-		def _close_socket_to_socket(state)
-			@sockets.delete state[:sender]
-
-			@poller.delete state[:receiver]
-			@poller.delete state[:socket]
-
-			state[:sender].close
-			state[:receiver].close
-
-			@handlers.delete state[:receiver]
-			@handlers.delete state[:socket]
-
-			return
-		end
-
-		def handle_socket_to_socket(state)
-			state[:recv_ok] = @poller.readables.include?(state[:receiver]) || state[:recv_ok]
-			state[:send_ok] = @poller.writables.include?(state[:socket]) || state[:send_ok]
-
-			loop do
-				if state[:send_ok] && !state[:buffer].nil?
-					begin
-						send state[:socket], state[:buffer]
-					rescue TimeoutError
-					end
-					state[:buffer] = nil if state[:buffer].length == 0
-					state[:send_ok] = false
-				end
-
-				break if !state[:recv_ok]
-
-				if state[:recv_ok] && state[:buffer].nil?
-					begin
-						state[:buffer] = recv(state[:receiver])
-					rescue TimeoutError
-					end
-					state[:recv_ok] = false
-				end
-
-				break if !state[:send_ok]
-			end
-
-			if state[:recv_ok]
-				@poller.deregister state[:receiver], ZMQ::POLLIN
-			else
-				@poller.register state[:receiver], ZMQ::POLLIN
-			end
-
-			if state[:send_ok]
-				@poller.deregister state[:socket], ZMQ::POLLOUT
-			else
-				@poller.register state[:socket], ZMQ::POLLOUT
-			end
-
-			return
-		end
-
-		def recv(socket)
-			data = []
-
-			poll_eagain(socket, ZMQ::POLLIN, 5) do
-				# recv_strings appears to be safe, ZMQ documents that a client will either
-				# receive 0 parts or all parts
-				socket.recv_strings(data, ZMQ::DONTWAIT)
-			end
-
-			data
-		end
-
-		def send(socket, data)
-			while data.length != 1
-				send_part socket, data.shift, true
-			end
-			send_part socket, data.shift
-			return
-		end
-
-		def send_part(socket, data, more=false)
-			poll_eagain(socket, ZMQ::POLLOUT, 5) do
-				# Try to send a message but never block
-				# We could use send_strings but it is vague on if ZMQ can return an
-				# error midway through sending parts...
-				socket.send_string(data, (more ? ZMQ::SNDMORE : 0) | ZMQ::DONTWAIT)
-			end
-
-			return
-		end
-
-		def poll_eagain(socket, flag, timeout, &block)
-			poller = nil
-			timeout = Time.now.to_i + timeout
-			loop do
-				rc = block.call()
-				break if ZMQ::Util.resultcode_ok?(rc)
-				if ZMQ::Util.errno != ZMQ::EAGAIN
-					fail ZMQError, 'message receive failed: ' + ZMQ::Util.error_string if flag == ZMQ::POLLIN
-					fail ZMQError, 'message send failed: ' + ZMQ::Util.error_string
-				end
-
-				# Wait for send to become available, handling timeouts
-				if poller.nil?
-					poller = ZMQ::Poller.new
-					poller.register socket, flag
-				end
-
-				while poller.poll(1_000) == 0
-					# Using this inner while triggers pollThreadEvents in JRuby which checks for Thread.raise immediately
-					fail TimeoutError while Time.now.to_i >= timeout
-				end
-			end
-			return
-		end
-	end
-end