pulsar_sdk 0.8.8

data/lib/pulsar_sdk/protocol.rb

@@ -0,0 +1,8 @@
+require 'pulsar_sdk/protocol/frame'
+require 'pulsar_sdk/protocol/message'
+require 'pulsar_sdk/protocol/structure'
+require 'pulsar_sdk/protocol/reader'
+require 'pulsar_sdk/protocol/lookup'
+require 'pulsar_sdk/protocol/namespace'
+require 'pulsar_sdk/protocol/topic'
+require 'pulsar_sdk/protocol/partitioned'

data/lib/pulsar_sdk/protocol/frame.rb

@@ -0,0 +1,53 @@
+require 'digest/crc32c'
+
+module PulsarSdk
+  module Protocol
+    class Frame
+      # 预留4byte存放帧长度
+      PREPENDED_SIZE = 4
+      CHECKSUM_SIZE = 4
+      MAGIC_NUMBER = [0x0e, 0x01].pack('C*').freeze
+
+      def self.encode(command, message = nil)
+        raise "command MUST be Pulsar::Proto::BaseCommand but got #{command.class}" unless command.is_a?(Pulsar::Proto::BaseCommand)
+
+        pb_cmd = command.to_proto
+
+        # 非发送消息帧
+        return encode_command(pb_cmd) if message.nil?
+
+        # 消息发送帧
+        # [TOTAL_SIZE] [CMD_SIZE] [CMD] [MAGIC_NUMBER] [CHECKSUM] [METADATA_SIZE] [METADATA] [PAYLOAD]
+        raise "message MUST be PulsarSdk::Producer::Message but got #{message.class}" unless message.is_a?(PulsarSdk::Producer::Message)
+
+        metadata = message.metadata
+        pb_meta = metadata.to_proto
+
+        meta_payload = binary(pb_meta.size) + pb_meta + message.binary_string
+        checksum = crc32(meta_payload)
+
+        total_size = PREPENDED_SIZE + pb_cmd.size + MAGIC_NUMBER.size + CHECKSUM_SIZE + meta_payload.size
+
+        binary(total_size, pb_cmd.size) + pb_cmd + MAGIC_NUMBER + binary(checksum) + meta_payload
+      end
+
+      def self.encode_command(pb_cmd)
+        binary(pb_cmd.size + PREPENDED_SIZE, pb_cmd.size) + pb_cmd
+      end
+
+      def self.decode(byte)
+        Pulsar::Proto::BaseCommand.decode(byte)
+      end
+
+      def self.binary(*obj)
+        obj.map { |x| Array(x).pack('N') }.join
+      end
+
+      def self.crc32(bytes)
+        crc = Digest::CRC32c.new
+        crc << bytes
+        crc.checksum
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/protocol/lookup.rb

@@ -0,0 +1,55 @@
+module PulsarSdk
+  module Protocol
+    class Lookup
+      MAX_LOOKUP_TIMES = 20
+
+      def initialize(client, service_url)
+        @client = client
+        @service_url = service_url
+      end
+
+      # output
+      #   [logical_addr, physical_addr]
+      def lookup(topic)
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::LOOKUP,
+          lookupTopic: Pulsar::Proto::CommandLookupTopic.new(
+            topic: topic,
+            authoritative: false
+          )
+        )
+        resp = @client.request_any_broker(base_cmd).lookupTopicResponse
+
+        # 最多查找这么多次
+        MAX_LOOKUP_TIMES.times do
+          case Pulsar::Proto::CommandLookupTopicResponse::LookupType.resolve(resp.response)
+          when Pulsar::Proto::CommandLookupTopicResponse::LookupType::Failed
+            PulsarSdk.logger.error(__method__){"Failed to lookup topic 「#{topic}」, #{resp.error}"}
+            break
+          when Pulsar::Proto::CommandLookupTopicResponse::LookupType::Redirect
+            logical_addr, physical_addr = extract_addr(resp)
+            base_cmd = Pulsar::Proto::BaseCommand.new(
+              type: Pulsar::Proto::BaseCommand::Type::LOOKUP,
+              lookupTopic: Pulsar::Proto::CommandLookupTopic.new(
+                topic: topic,
+                authoritative: resp.authoritative
+              )
+            )
+            # NOTE 从连接池拿
+            resp = @client.request(logical_addr, physical_addr, base_cmd).lookupTopicResponse
+          when Pulsar::Proto::CommandLookupTopicResponse::LookupType::Connect
+            return extract_addr(resp)
+          end
+        end
+      end
+
+      private
+      def extract_addr(resp)
+        logical_addr = resp.brokerServiceUrl
+        physical_addr = resp.proxy_through_service_url ? @service_url : logical_addr
+
+        [logical_addr, physical_addr]
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/protocol/message.rb

@@ -0,0 +1,55 @@
+module PulsarSdk
+  module Protocol
+    class Message
+      prepend ::PulsarSdk::Tweaks::AssignAttributes
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      attr_accessor :publish_time, :event_time, :partition_key, :payload,
+                    :message_id, :properties, :consumer_id, :topic
+
+      attr_accessor :ack_handler
+
+      # def publish_at
+      # def event_at
+      [:publish, :event].each do |x|
+        define_method "#{x}_at" do
+          v = self.public_send("#{x}_time").to_i
+          return if v.zero?
+          TimeX.at_timestamp(v)
+        end
+      end
+
+      def ack(type = Pulsar::Proto::CommandAck::AckType::Individual)
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::ACK,
+          ack: Pulsar::Proto::CommandAck.new(
+            consumer_id: self.consumer_id,
+            message_id: [self.message_id],
+            ack_type: type
+          )
+        )
+
+        ack_handler.call(base_cmd)
+        @confirmed = true
+      end
+
+      # 检查是否有确认，无论是ack还是nack都算是确认
+      def confirmed?
+        !!@confirmed
+      end
+
+      def nack
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::REDELIVER_UNACKNOWLEDGED_MESSAGES,
+          redeliverUnacknowledgedMessages: Pulsar::Proto::CommandRedeliverUnacknowledgedMessages.new(
+            consumer_id: self.consumer_id,
+            message_ids: [self.message_id]
+          )
+        )
+
+        ack_handler.call(base_cmd)
+        @confirmed = true
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/protocol/namespace.rb

@@ -0,0 +1,22 @@
+module PulsarSdk
+  module Protocol
+    class Namespace
+      def initialize(client)
+        @client = client
+      end
+
+      def topics(namespace)
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::GET_TOPICS_OF_NAMESPACE,
+          getTopicsOfNamespace: Pulsar::Proto::CommandGetTopicsOfNamespace.new(
+            namespace: namespace,
+            mode: Pulsar::Proto::CommandGetTopicsOfNamespace::Mode.resolve(:ALL)
+          )
+        )
+        resp = @client.request_any_broker(base_cmd)
+
+        resp.getTopicsOfNamespaceResponse&.topics
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/protocol/partitioned.rb

@@ -0,0 +1,54 @@
+module PulsarSdk
+  module Protocol
+    class Partitioned
+      def initialize(client, topic)
+        @client = client
+        @tn = ::PulsarSdk::Protocol::Topic.parse(topic)
+      end
+
+      def partitions
+        pmr = topic_metadata
+
+        if !success_response?(pmr)
+          PulsarSdk.logger.error(__method__){"Get topic partitioned metadata failed, #{pmr.error}: #{pmr.message}"}
+          return []
+        end
+
+        return [@tn.to_s] if pmr.partitions.zero?
+
+        tn = @tn.dup
+        (0..pmr.partitions).map do |i|
+          tn.partition = i
+          tn.to_s
+        end
+      end
+
+      # 当前topic是否是分区topic
+      def partitioned?
+        topic_metadata.partitions > 0
+      end
+
+      private
+      def success_response?(pmr)
+        result = false
+        Pulsar::Proto::CommandPartitionedTopicMetadataResponse::LookupType.tap do |x|
+          result = x.resolve(pmr.response) == x.const_get(:Success)
+        end
+
+        result
+      end
+
+      def topic_metadata
+        return @topic_metadata_ unless @topic_metadata_.nil?
+
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::PARTITIONED_METADATA,
+          partitionMetadata: Pulsar::Proto::CommandPartitionedTopicMetadata.new(
+            topic: @tn.to_s
+          )
+        )
+        @topic_metadata_ = @client.request_any_broker(base_cmd).partitionMetadataResponse
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/protocol/reader.rb

@@ -0,0 +1,67 @@
+module PulsarSdk
+  module Protocol
+    class Reader
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      FRAME_SIZE_LEN = 4
+      CMD_SIZE_LEN = 4
+
+      def initialize(io)
+        ensure_interface_implemented!(io)
+        @io = io
+
+        @readed = 0
+      end
+
+      # TODO add timeout?
+      def read_fully
+        frame_szie = read_frame_size
+        raise "IO reader is empty! maybe server error, please check server log for help." if frame_szie.nil?
+
+        base_cmd = read_command
+
+        buffer = read_remaining(frame_szie)
+
+        [base_cmd, buffer]
+      end
+
+      def read_frame_size
+        frame_size = read(FRAME_SIZE_LEN, 'N')
+        # reset cursor! let's read the frame
+        @readed = 0
+        frame_size
+      end
+
+      def read_command
+        cmd_size = read(CMD_SIZE_LEN, 'N')
+        cmd_bytes = read(cmd_size)
+        Pulsar::Proto::BaseCommand.decode(cmd_bytes)
+      end
+
+      def read_remaining(frame_szie)
+        meta_and_payload_size = frame_szie - @readed
+        return if meta_and_payload_size <= 0
+        read(meta_and_payload_size)
+      end
+
+      private
+      def ensure_interface_implemented!(io)
+        [:read, :closed?].each do |x|
+          raise "io must implement method: #{x}" unless io.respond_to?(x)
+        end
+      end
+
+      def read(size, unpack = nil)
+        raise Errno::ECONNRESET if @io.closed?
+        raise Errno::ETIMEDOUT unless IO.select([@io], nil)
+
+        bytes = @io.read(size)
+        @readed = @readed.to_i + size.to_i
+
+        return bytes if unpack.nil? || bytes.nil?
+
+        bytes.unpack(unpack).first
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/protocol/structure.rb

@@ -0,0 +1,93 @@
+module PulsarSdk
+  module Protocol
+    class Structure
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      # [MAGIC_NUMBER] [CHECKSUM] [METADATA_SIZE] [METADATA] [PAYLOAD]
+      MAGIC_NUMBER = [0x0e, 0x01].pack('C*').freeze
+      MAGIC_NUMBER_LEN = MAGIC_NUMBER.size
+      CHECKSUM_LEN = 4
+      METADATA_SIZE_LEN = 4
+
+      def initialize(buff)
+        @buff = buff
+        rewind
+      end
+
+      def decode
+        metadata = nil
+
+        message = PulsarSdk::Protocol::Message.new
+
+        mn_bytes = read_magic_number
+        if mn_bytes == MAGIC_NUMBER
+          _checksum = read_checksum
+          # TODO 可能需要校验一下，防止错误消息
+          metadata = read_metadata
+        else
+          rewind(MAGIC_NUMBER_LEN)
+          metadata = read_metadata
+        end
+
+        msg = read_remaining
+
+        # NOTE 同为Ruby SDK时可以根据Content-Type预先还原
+        # 复杂类型依旧为string，需要特别注意
+        metadata.properties.each do |x|
+          next unless x.key.to_s =~ /Content-Type/i
+          next unless x.value.to_s =~ /json/i
+          PulsarSdk.logger.info("#{self.class}::#{__method__}"){"Found json encode remark, parse JSON mesaage!"}
+          msg = JSON.parse(msg)
+        end
+
+        message.assign_attributes(
+          publish_time: metadata.publish_time,
+          event_time: metadata.event_time,
+          partition_key: metadata.partition_key,
+          properties: metadata.properties,
+          payload: msg
+        )
+
+        message
+      end
+
+      # 回退若干字节，方便处理非连续段
+      def rewind(x = nil)
+        return @readed = 0 if x.nil?
+
+        @readed -= x
+      end
+
+      def read_magic_number
+        read(MAGIC_NUMBER_LEN)
+      end
+
+      # crc32
+      def read_checksum
+        read(CHECKSUM_LEN)
+      end
+
+      def read_metadata
+        metadata_size = read(METADATA_SIZE_LEN, 'N')
+        metadata_bytes = read(metadata_size)
+        Pulsar::Proto::MessageMetadata.decode(metadata_bytes)
+      end
+
+      def read_remaining
+        payload_size = @buff.size - @readed
+        return if payload_size <= 0
+        read(payload_size)
+      end
+
+      private
+      def read(size, unpack = nil)
+        bytes = @buff[@readed..(@readed + size - 1)]
+        @readed += size
+
+        return bytes if unpack.nil? || bytes.nil?
+
+        bytes.unpack(unpack).first
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/protocol/topic.rb

@@ -0,0 +1,74 @@
+module PulsarSdk
+  module Protocol
+    class Topic
+      PUBLIC_TENANT = 'public'.freeze
+      DEFAULT_NAMESPACE = 'default'.freeze
+      PARTITIONED_TOPIC_SUFFIX = '-partition-'.freeze
+
+      prepend ::PulsarSdk::Tweaks::AssignAttributes
+
+      attr_accessor :domain, :namespace, :topic, :partition
+
+      def to_s
+        [
+          mk_domain,
+          self.namespace,
+          mk_topic
+        ].join('/')
+      end
+
+      private
+      def mk_topic
+        return self.topic if self.partition.nil?
+
+        "#{self.topic}#{PARTITIONED_TOPIC_SUFFIX}#{self.partition}"
+      end
+
+      def mk_domain
+        return if domain.nil?
+        "#{self.domain}:/"
+      end
+
+      # new:    persistent://tenant/namespace/topic
+      # legacy: persistent://tenant/cluster/namespace/topic
+      def self.parse(topic)
+        if !topic.include?('://')
+          parts = topic.split('/')
+          if parts.size == 3 || parts.size == 4
+            topic = "persistent://#{topic}"
+          elsif parts.size == 1
+            topic = "persistent://#{PUBLIC_TENANT}/#{DEFAULT_NAMESPACE}/" + parts[0]
+          else
+            raise "Invalid short topic name: #{topic}, it should be in the format of <tenant>/<namespace>/<topic> or <topic>"
+          end
+        end
+
+        domain, rest = topic.split('://', 2)
+        unless ['persistent', 'non-persistent'].include?(domain)
+          raise "Invalid topic domain: #{domain}"
+        end
+
+        tn = new(domain: domain)
+        topic_with_partition = nil
+
+        case rest.count('/')
+        when 2
+          tenant, namespace, topic_with_partition = rest.split('/', 3)
+          tn.namespace = [tenant, namespace].join('/')
+        when 3
+          tenant, cluster, namespace, topic_with_partition = rest.split('/', 4)
+          tn.namespace = [tenant, cluster, namespace].join('/')
+        else
+          raise "Invalid topic name: #{topic}"
+        end
+
+        tn.topic, partition = topic_with_partition.split(PARTITIONED_TOPIC_SUFFIX, 2)
+
+        tn.partition = partition&.to_i
+
+        tn
+      end
+
+    end
+  end
+end