pulsar-ruby 0.1.0.pre

data/lib/pulsar/internal/broker_error_mapper.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Pulsar
+  module Internal
+    # Converts broker protocol error codes into typed client exceptions.
+    class BrokerErrorMapper
+      ERROR_CLASSES = {
+        AuthenticationError: AuthenticationError,
+        AuthorizationError: AuthorizationError,
+        TopicNotFound: TopicNotFoundError,
+        ProducerBusy: ProducerBusyError,
+        ProducerFenced: ProducerBusyError,
+        ConsumerBusy: ConsumerBusyError
+      }.freeze
+
+      def self.from(server_error, message)
+        error_class = ERROR_CLASSES.fetch(server_error, BrokerError)
+        text = error_class == BrokerError ? "#{server_error}: #{message}" : message
+        error_class.new(text)
+      end
+    end
+  end
+end

data/lib/pulsar/internal/command_factory.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Pulsar
+  module Internal
+    # Builds protobuf commands for the Pulsar binary protocol.
+    class CommandFactory
+      def self.producer(topic:, producer_id:, request_id:)
+        Proto::BaseCommand.new(
+          type: :PRODUCER,
+          producer: Proto::CommandProducer.new(
+            topic: topic,
+            producer_id: producer_id,
+            request_id: request_id
+          )
+        )
+      end
+
+      def self.send_message(producer_id:, sequence_id:, producer_name:, publish_time:, properties: {}, key: nil,
+                            event_time: nil)
+        command = Proto::BaseCommand.new(
+          type: :SEND,
+          send: Proto::CommandSend.new(
+            producer_id: producer_id,
+            sequence_id: sequence_id
+          )
+        )
+        metadata = Proto::MessageMetadata.new(
+          producer_name: producer_name,
+          sequence_id: sequence_id,
+          publish_time: publish_time,
+          properties: properties.map { |key_value, value| Proto::KeyValue.new(key: key_value.to_s, value: value.to_s) }
+        )
+        metadata.partition_key = key if key
+        metadata.event_time = event_time if event_time
+
+        [command, metadata]
+      end
+
+      def self.subscribe(topic:, subscription:, consumer_id:, request_id:, subscription_type: :Exclusive)
+        Proto::BaseCommand.new(
+          type: :SUBSCRIBE,
+          subscribe: Proto::CommandSubscribe.new(
+            topic: topic,
+            subscription: subscription,
+            subType: subscription_type,
+            consumer_id: consumer_id,
+            request_id: request_id
+          )
+        )
+      end
+
+      def self.flow(consumer_id:, permits:)
+        Proto::BaseCommand.new(
+          type: :FLOW,
+          flow: Proto::CommandFlow.new(
+            consumer_id: consumer_id,
+            messagePermits: permits
+          )
+        )
+      end
+
+      def self.ack(consumer_id:, message_id:)
+        Proto::BaseCommand.new(
+          type: :ACK,
+          ack: Proto::CommandAck.new(
+            consumer_id: consumer_id,
+            ack_type: :Individual,
+            message_id: [
+              Proto::MessageIdData.new(
+                ledgerId: message_id.ledger_id,
+                entryId: message_id.entry_id,
+                partition: message_id.partition_index,
+                batch_index: message_id.batch_index
+              )
+            ]
+          )
+        )
+      end
+
+      def self.lookup(topic:, request_id:)
+        Proto::BaseCommand.new(
+          type: :LOOKUP,
+          lookupTopic: Proto::CommandLookupTopic.new(
+            topic: topic,
+            request_id: request_id
+          )
+        )
+      end
+
+      def self.close_producer(producer_id:, request_id:)
+        Proto::BaseCommand.new(
+          type: :CLOSE_PRODUCER,
+          close_producer: Proto::CommandCloseProducer.new(
+            producer_id: producer_id,
+            request_id: request_id
+          )
+        )
+      end
+
+      def self.close_consumer(consumer_id:, request_id:)
+        Proto::BaseCommand.new(
+          type: :CLOSE_CONSUMER,
+          close_consumer: Proto::CommandCloseConsumer.new(
+            consumer_id: consumer_id,
+            request_id: request_id
+          )
+        )
+      end
+
+      def self.pong
+        Proto::BaseCommand.new(
+          type: :PONG,
+          pong: Proto::CommandPong.new
+        )
+      end
+    end
+  end
+end

data/lib/pulsar/internal/connection.rb

@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+module Pulsar
+  module Internal
+    # Owns a broker socket, request correlation, and reader thread.
+    class Connection
+      PROTOCOL_VERSION = 21
+
+      attr_reader :server_version, :protocol_version, :max_message_size
+
+      def self.connect(host:, port:, connection_timeout:, operation_timeout:, client_version:)
+        transport = TcpTransport.connect(host: host, port: port, connection_timeout: connection_timeout)
+        new(
+          transport: transport,
+          operation_timeout: operation_timeout,
+          client_version: client_version
+        ).tap(&:connect)
+      end
+
+      def initialize(transport:, operation_timeout:, client_version:)
+        @transport = transport
+        @operation_timeout = operation_timeout
+        @client_version = client_version
+        @connected = false
+        @closed = false
+        @state_mutex = Mutex.new
+        @write_mutex = Mutex.new
+        @request_id = 0
+        @pending_requests = {}
+        @pending_sends = {}
+        @consumers = {}
+      end
+
+      def connect
+        write_connect_command
+        command = read_command(timeout: @operation_timeout)
+        raise ProtocolError, "expected CONNECTED response, got #{command.type}" unless command.type == :CONNECTED
+
+        @server_version = command.connected.server_version
+        @protocol_version = command.connected.protocol_version
+        @max_message_size = command.connected.max_message_size
+        @connected = true
+        start_reader_thread
+        self
+      rescue Error
+        close
+        raise
+      end
+
+      def connected?
+        @connected && !closed?
+      end
+
+      def close
+        return nil if closed?
+
+        @closed = true
+        @connected = false
+        @transport.close
+        reject_pending(ClosedError.new('connection is closed'))
+        @reader_thread&.join unless Thread.current == @reader_thread
+        nil
+      end
+
+      def closed?
+        @closed
+      end
+
+      def next_request_id
+        @state_mutex.synchronize do
+          @request_id += 1
+        end
+      end
+
+      def request(command, timeout: @operation_timeout)
+        ensure_connected!
+        promise = Promise.new
+        request_id = request_id_for(command)
+        add_pending_request(request_id, promise)
+
+        begin
+          write_frame(FrameCodec.encode_command(command))
+          promise.wait(timeout: timeout)
+        ensure
+          remove_pending_request(request_id)
+        end
+      end
+
+      def send_message(command, metadata, payload, timeout: @operation_timeout)
+        ensure_connected!
+        promise = Promise.new
+        send_key = [command['send'].producer_id, command['send'].sequence_id]
+        add_pending_send(send_key, promise)
+
+        begin
+          write_frame(FrameCodec.encode_message(command, metadata, payload))
+          promise.wait(timeout: timeout)
+        ensure
+          remove_pending_send(send_key)
+        end
+      end
+
+      def write_command(command)
+        ensure_connected!
+
+        write_frame(FrameCodec.encode_command(command))
+      end
+
+      def register_consumer(consumer_id, consumer)
+        @state_mutex.synchronize { @consumers[consumer_id] = consumer }
+        nil
+      end
+
+      def unregister_consumer(consumer_id)
+        @state_mutex.synchronize { @consumers.delete(consumer_id) }
+        nil
+      end
+
+      def read_frame(timeout: @operation_timeout)
+        ensure_connected!
+
+        read_decoded_frame(timeout: timeout)
+      end
+
+      private
+
+      def write_connect_command
+        command = Proto::BaseCommand.new(
+          type: :CONNECT,
+          connect: Proto::CommandConnect.new(
+            client_version: @client_version,
+            protocol_version: PROTOCOL_VERSION
+          )
+        )
+        write_frame(FrameCodec.encode_command(command))
+      end
+
+      def read_command(timeout:)
+        read_decoded_frame(timeout: timeout).command
+      end
+
+      def read_decoded_frame(timeout:)
+        size_prefix = @transport.read_exact(4, timeout: timeout)
+        size = size_prefix.unpack1('N')
+        frame = size_prefix + @transport.read_exact(size, timeout: timeout)
+        FrameCodec.decode_frame(frame)
+      end
+
+      def ensure_connected!
+        raise ClosedError, 'connection is closed' if closed?
+        raise ConnectionError, 'connection is not connected' unless connected?
+      end
+
+      def write_frame(frame)
+        @write_mutex.synchronize { @transport.write(frame) }
+      end
+
+      def start_reader_thread
+        @reader_thread = Thread.new { reader_loop }
+      end
+
+      def reader_loop
+        loop do
+          break if closed?
+
+          route_frame(read_decoded_frame(timeout: @operation_timeout))
+        end
+      rescue ClosedError
+        reject_pending(ClosedError.new('connection is closed')) unless closed?
+      rescue ConnectionError => e
+        if closed?
+          reject_pending(ClosedError.new('connection is closed'))
+        else
+          fail_connection(ConnectionError.new("connection lost: #{e.message}"))
+        end
+      rescue Error => e
+        reject_pending(e)
+      end
+
+      def route_frame(decoded)
+        command = decoded.command
+
+        case command.type
+        when :MESSAGE
+          consumer_for(command.message.consumer_id)&.handle_message(command.message, decoded.headers_and_payload)
+        when :PING
+          write_command(CommandFactory.pong)
+        when :SEND_RECEIPT
+          fulfill_pending_send(command.send_receipt.producer_id, command.send_receipt.sequence_id, command)
+        when :SEND_ERROR
+          reject_pending_send(
+            command.send_error.producer_id,
+            command.send_error.sequence_id,
+            BrokerErrorMapper.from(command.send_error.error, command.send_error.message)
+          )
+        when :ERROR
+          reject_pending_request(command.error.request_id, BrokerErrorMapper.from(command.error.error, command.error.message))
+        else
+          fulfill_pending_request(response_request_id(command), command)
+        end
+      end
+
+      def add_pending_request(request_id, promise)
+        @state_mutex.synchronize { @pending_requests[request_id] = promise }
+      end
+
+      def remove_pending_request(request_id)
+        @state_mutex.synchronize { @pending_requests.delete(request_id) }
+      end
+
+      def fulfill_pending_request(request_id, command)
+        promise = @state_mutex.synchronize { @pending_requests.delete(request_id) }
+        promise&.fulfill(command)
+      end
+
+      def reject_pending_request(request_id, error)
+        promise = @state_mutex.synchronize { @pending_requests.delete(request_id) }
+        promise&.reject(error)
+      end
+
+      def add_pending_send(send_key, promise)
+        @state_mutex.synchronize { @pending_sends[send_key] = promise }
+      end
+
+      def remove_pending_send(send_key)
+        @state_mutex.synchronize { @pending_sends.delete(send_key) }
+      end
+
+      def fulfill_pending_send(producer_id, sequence_id, command)
+        promise = @state_mutex.synchronize { @pending_sends.delete([producer_id, sequence_id]) }
+        promise&.fulfill(command)
+      end
+
+      def reject_pending_send(producer_id, sequence_id, error)
+        promise = @state_mutex.synchronize { @pending_sends.delete([producer_id, sequence_id]) }
+        promise&.reject(error)
+      end
+
+      def consumer_for(consumer_id)
+        @state_mutex.synchronize { @consumers[consumer_id] }
+      end
+
+      def reject_pending(error)
+        requests, sends = @state_mutex.synchronize do
+          [@pending_requests.values.tap { @pending_requests.clear },
+           @pending_sends.values.tap { @pending_sends.clear }]
+        end
+        (requests + sends).each { |promise| promise.reject(error) }
+      end
+
+      def fail_connection(error)
+        @state_mutex.synchronize { @connected = false }
+        reject_pending(error)
+      end
+
+      def request_id_for(command)
+        case command.type
+        when :PRODUCER
+          command.producer.request_id
+        when :SUBSCRIBE
+          command.subscribe.request_id
+        when :LOOKUP
+          command.lookupTopic.request_id
+        when :CLOSE_PRODUCER
+          command.close_producer.request_id
+        when :CLOSE_CONSUMER
+          command.close_consumer.request_id
+        else
+          raise ProtocolError, "command #{command.type} does not have a request id"
+        end
+      end
+
+      def response_request_id(command)
+        case command.type
+        when :PRODUCER_SUCCESS
+          command.producer_success.request_id
+        when :SUCCESS
+          command.success.request_id
+        when :LOOKUP_RESPONSE
+          command.lookupTopicResponse.request_id
+        else
+          raise ProtocolError, "unexpected broker command #{command.type}"
+        end
+      end
+    end
+  end
+end

data/lib/pulsar/internal/consumer_impl.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Pulsar
+  module Internal
+    # Implements broker-side subscription, flow, receive, and ack behavior.
+    class ConsumerImpl
+      attr_reader :topic, :subscription, :consumer_id
+
+      def self.create(topic:, subscription:, consumer_id:, operation_timeout:, receiver_queue_size:,
+                      connection: nil, connection_provider: nil)
+        connection_provider ||= -> { connection }
+        new(
+          connection_provider: connection_provider,
+          topic: topic,
+          subscription: subscription,
+          consumer_id: consumer_id,
+          operation_timeout: operation_timeout,
+          receiver_queue_size: receiver_queue_size
+        ).tap(&:attach)
+      end
+
+      def initialize(connection_provider:, topic:, subscription:, consumer_id:, operation_timeout:, receiver_queue_size:)
+        @connection_provider = connection_provider
+        @connection = nil
+        @topic = topic
+        @subscription = subscription
+        @consumer_id = consumer_id
+        @operation_timeout = operation_timeout
+        @receiver_queue_size = receiver_queue_size
+        @receiver_queue = BoundedQueue.new(capacity: receiver_queue_size)
+        @closed = false
+      end
+
+      def handle_message(command_message, headers_and_payload)
+        raise ClosedError, 'consumer is closed' if closed?
+
+        decoded = FrameCodec.decode_message_data(headers_and_payload)
+        @receiver_queue.push(
+          Message.new(
+            payload: decoded.payload,
+            message_id: message_id_from(command_message.message_id),
+            properties: decoded.metadata.properties.to_h { |property| [property.key, property.value] },
+            key: decoded.metadata.partition_key,
+            publish_time: decoded.metadata.publish_time,
+            event_time: decoded.metadata.event_time
+          ),
+          timeout: @operation_timeout
+        )
+      end
+
+      def receive(timeout: nil)
+        raise ClosedError, 'consumer is closed' if closed?
+
+        attach unless attached?
+        @receiver_queue.pop(timeout: timeout || @operation_timeout).tap do
+          flow(1)
+        end
+      end
+
+      def ack(message_or_message_id)
+        raise ClosedError, 'consumer is closed' if closed?
+
+        attach unless attached?
+        message_id = message_or_message_id.respond_to?(:message_id) ? message_or_message_id.message_id : message_or_message_id
+        @connection.write_command(CommandFactory.ack(consumer_id: consumer_id, message_id: message_id))
+        nil
+      end
+
+      def close
+        return nil if closed?
+
+        if attached?
+          request_id = @connection.next_request_id
+          command = CommandFactory.close_consumer(consumer_id: consumer_id, request_id: request_id)
+          response = @connection.request(command, timeout: @operation_timeout)
+          raise BrokerError, "consumer close failed: #{response.type}" unless response.type == :SUCCESS
+
+          @connection.unregister_consumer(consumer_id)
+        end
+
+        @receiver_queue.close
+        @closed = true
+        nil
+      end
+
+      def closed?
+        @closed
+      end
+
+      def flow(permits)
+        raise ClosedError, 'consumer is closed' if closed?
+
+        attach unless attached?
+        @connection.write_command(CommandFactory.flow(consumer_id: consumer_id, permits: permits))
+      end
+
+      private
+
+      def attach
+        @connection = @connection_provider.call
+        request_id = @connection.next_request_id
+        command = CommandFactory.subscribe(
+          topic: topic,
+          subscription: subscription,
+          consumer_id: consumer_id,
+          request_id: request_id
+        )
+        response = @connection.request(command, timeout: @operation_timeout)
+        raise BrokerError, "subscribe failed: #{response.type}" unless response.type == :SUCCESS
+
+        @connection.register_consumer(consumer_id, self)
+        @connection.write_command(CommandFactory.flow(consumer_id: consumer_id, permits: @receiver_queue_size))
+        nil
+      end
+
+      def attached?
+        @connection&.connected?
+      end
+
+      def message_id_from(data)
+        MessageId.new(
+          ledger_id: data.ledgerId,
+          entry_id: data.entryId,
+          partition_index: data.partition,
+          batch_index: data.batch_index
+        )
+      end
+    end
+  end
+end

data/lib/pulsar/internal/frame_codec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Pulsar
+  module Internal
+    # Encodes and decodes Pulsar binary protocol frames.
+    class FrameCodec
+      DecodedFrame = Struct.new(:command, :headers_and_payload, keyword_init: true)
+      DecodedMessageData = Struct.new(:metadata, :payload, keyword_init: true)
+
+      def self.encode_command(command)
+        encoded_command = Proto::BaseCommand.encode(command)
+        [4 + encoded_command.bytesize, encoded_command.bytesize].pack('NN') + encoded_command
+      end
+
+      def self.encode_message(command, metadata, payload)
+        encoded_command = Proto::BaseCommand.encode(command)
+        encoded_metadata = Proto::MessageMetadata.encode(metadata)
+        payload = String(payload).b
+        total_size = 4 + encoded_command.bytesize + 4 + encoded_metadata.bytesize + payload.bytesize
+
+        [total_size, encoded_command.bytesize].pack('NN') +
+          encoded_command + [encoded_metadata.bytesize].pack('N') + encoded_metadata + payload
+      end
+
+      def self.decode_frame(frame)
+        frame = String(frame).b
+        raise ProtocolError, 'frame size prefix is incomplete' if frame.bytesize < 4
+
+        total_size = frame.byteslice(0, 4).unpack1('N')
+        raise ProtocolError, 'frame is incomplete' if frame.bytesize < 4 + total_size
+        raise ProtocolError, 'command size prefix is incomplete' if total_size < 4
+
+        command_size = frame.byteslice(4, 4).unpack1('N')
+        raise ProtocolError, 'command exceeds frame size' if command_size > total_size - 4
+
+        command_bytes = frame.byteslice(8, command_size)
+        headers_and_payload = frame.byteslice(8 + command_size, total_size - 4 - command_size) || +''
+
+        DecodedFrame.new(
+          command: Proto::BaseCommand.decode(command_bytes),
+          headers_and_payload: headers_and_payload.b
+        )
+      end
+
+      def self.decode_message_data(headers_and_payload)
+        headers_and_payload = String(headers_and_payload).b
+        raise ProtocolError, 'metadata size prefix is incomplete' if headers_and_payload.bytesize < 4
+
+        metadata_size = headers_and_payload.byteslice(0, 4).unpack1('N')
+        raise ProtocolError, 'metadata exceeds message data size' if metadata_size > headers_and_payload.bytesize - 4
+
+        metadata_bytes = headers_and_payload.byteslice(4, metadata_size)
+        payload = headers_and_payload.byteslice(4 + metadata_size, headers_and_payload.bytesize - 4 - metadata_size) || +''
+
+        DecodedMessageData.new(
+          metadata: Proto::MessageMetadata.decode(metadata_bytes),
+          payload: payload.b
+        )
+      end
+    end
+  end
+end

data/lib/pulsar/internal/lookup_service.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Pulsar
+  module Internal
+    # Resolves the broker service URL that owns a topic.
+    class LookupService
+      def initialize(connection:, operation_timeout:)
+        @connection = connection
+        @operation_timeout = operation_timeout
+      end
+
+      def lookup(topic)
+        request_id = @connection.next_request_id
+        response = @connection.request(
+          CommandFactory.lookup(topic: topic, request_id: request_id),
+          timeout: @operation_timeout
+        )
+
+        raise BrokerError, "lookup failed: #{response.type}" unless response.type == :LOOKUP_RESPONSE
+
+        lookup = response.lookupTopicResponse
+        raise BrokerError, "lookup failed: #{lookup.message}" unless lookup.response == :Connect
+
+        lookup.brokerServiceUrl
+      end
+    end
+  end
+end