pulsar_sdk 0.8.8

data/lib/pulsar_sdk/client/connection_pool.rb

@@ -0,0 +1,79 @@
+module PulsarSdk
+  module Client
+    class ConnectionPool
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      def initialize(opts)
+        raise "opts expected a PulsarSdk::Options::Connection got #{opts.class}" unless opts.is_a?(PulsarSdk::Options::Connection)
+
+        @mutex = Mutex.new
+        @pool = ::PulsarSdk::Tweaks::WaitMap.new
+
+        @options = opts
+        @keepalive = opts.keepalive
+        @connection_timeout = opts.connection_timeout
+
+        @authentication = opts.auth_provider
+        @tls_options = opts.tls_options
+
+        instance_variables.each do |x|
+          remove_instance_variable(x) if instance_variable_get(x).nil?
+        end
+      end
+
+      def fetch(logical_addr, physical_addr)
+        id = (logical_addr || physical_addr).to_s
+        raise 'logical_addr and physical_addr both empty!' if id.empty?
+
+        conn = nil
+        @mutex.synchronize do
+          conn = @pool.find(id)
+
+          if conn.nil? || conn.closed?
+            # REMOVE closed conncetion from pool
+            @pool.delete(id, 0.01) unless conn.nil?
+
+            opts = @options.dup
+            opts.assign_attributes(
+              logical_addr: logical_addr,
+              physical_addr: physical_addr
+            )
+
+            conn = @pool.add(id, ::PulsarSdk::Client::Connection.establish(opts))
+          end
+        end
+
+        conn
+      end
+
+      def run_checker
+        Thread.new do
+          loop do
+            begin
+              @pool.each do |_k, v|
+                last_ping_at, last_received_at = v.active_status
+
+                case
+                when last_ping_at - last_received_at >= @keepalive * 2
+                  v.close
+                when last_ping_at - last_received_at > @keepalive
+                  v.ping
+                end
+              end
+            rescue => exp
+              PulsarSdk.logger.error(exp)
+            end
+
+            sleep(1)
+          end
+        end
+      end
+
+      def close
+        @pool.clear do |_, v|
+          v.close
+        end
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/client/rpc.rb

@@ -0,0 +1,67 @@
+module PulsarSdk
+  module Client
+    class Rpc
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      def initialize(opts)
+        raise "opts expected a PulsarSdk::Options::Connection got #{opts.class}" unless opts.is_a?(PulsarSdk::Options::Connection)
+
+        @opts = opts
+
+        @cnx = ::PulsarSdk::Client::ConnectionPool.new(opts).tap {|x| x.run_checker}
+
+        @producer_id = 0
+        @consumer_id = 0
+      end
+
+      def connection(logical_addr = nil, physical_addr = nil)
+        logical_addr ||= @opts.logical_addr
+        @cnx.fetch(logical_addr, physical_addr)
+      end
+
+      def lookup(topic)
+        @lookup_service ||= ::PulsarSdk::Protocol::Lookup.new(self, @opts.logical_addr)
+        @lookup_service.lookup(topic)
+      end
+
+      def namespace_topics(namespace)
+        @namespace_service ||= ::PulsarSdk::Protocol::Namespace.new(self)
+        @namespace_service.topics(namespace)
+      end
+
+      def partition_topics(topic)
+        ::PulsarSdk::Protocol::Partitioned.new(self, topic)&.partitions || []
+      end
+
+      def request(physical_addr, logical_addr, cmd)
+        connection(physical_addr, logical_addr).request(cmd, nil, true)
+      end
+
+      def request_any_broker(cmd)
+        connection.request(cmd)
+      end
+
+      def close
+        @cnx.close
+      end
+
+      def create_producer(opts)
+        raise "opts expected a PulsarSdk::Options::Producer got #{opts.class}" unless opts.is_a?(PulsarSdk::Options::Producer)
+        # FIXME check if connection ready
+        ::PulsarSdk::Producer.create(self, opts)
+      end
+
+      def subscribe(opts)
+        raise "opts expected a PulsarSdk::Options::Consumer got #{opts.class}" unless opts.is_a?(PulsarSdk::Options::Consumer)
+        # FIXME check if connection ready
+        consumer = ::PulsarSdk::Consumer.create(self, opts)
+
+        consumer
+      end
+
+      def create_reader(opts = {})
+
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/consumer.rb

@@ -0,0 +1,13 @@
+require 'pulsar_sdk/consumer/base'
+require 'pulsar_sdk/consumer/message_tracker'
+require 'pulsar_sdk/consumer/manager'
+
+module PulsarSdk
+  module Consumer
+    extend self
+
+    def create(client, opts)
+      PulsarSdk::Consumer::Manager.new(client, opts)
+    end
+  end
+end

data/lib/pulsar_sdk/consumer/base.rb

@@ -0,0 +1,148 @@
+module PulsarSdk
+  module Consumer
+    class Base
+      attr_reader :consumer_id, :topic
+
+      def initialize(client, message_tracker, opts)
+        @opts = opts
+        @topic = @opts.topic
+        @message_tracker = message_tracker
+        @client = client
+      end
+
+      def grab_cnx
+        @prefetch = @opts.prefetch
+        @fetched = 0
+        @capacity = 0
+
+        @conn = @client.connection(*@client.lookup(@topic))
+        @established = true
+
+        @seq_generator = SeqGenerator.new(@conn.seq_generator)
+
+        @consumer_id = @seq_generator.new_consumer_id
+        @consumer_name = @opts.name
+        @subscription_name = @opts.subscription_name
+
+        result = init_consumer
+        @consumer_name = result.consumerName unless result.nil?
+      end
+
+      def increase_fetched(n = 1)
+        @fetched += n
+      end
+
+      def flow
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::FLOW,
+          flow: Pulsar::Proto::CommandFlow.new(
+            messagePermits: @prefetch
+          )
+        )
+
+        execute(base_cmd)
+
+        @capacity += @prefetch
+      end
+
+      def subscription
+        @subscription_name
+      end
+
+      def unsubscribe
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::UNSUBSCRIBE,
+          unsubscribe: Pulsar::Proto::CommandUnsubscribe.new
+        )
+        execute_async(base_cmd)
+      end
+
+      def flow_if_need
+        return if @capacity > 0 && [@prefetch / 2, 1].max.ceil < (@capacity - @fetched)
+        flow
+      end
+
+      def close
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::CLOSE_CONSUMER,
+          close_consumer: Pulsar::Proto::CommandCloseConsumer.new(
+            consumer_id: @consumer_id
+          )
+        )
+        execute(base_cmd) unless disconnect?
+
+        remove_handler!
+      end
+
+      def disconnect?
+        !@established
+      end
+
+      def execute(cmd)
+        write(cmd)
+      end
+
+      def execute_async(cmd)
+        write(cmd, nil, true)
+      end
+
+      private
+      def write(cmd, *args)
+        grab_cnx if disconnect?
+        cmd.seq_generator = @seq_generator
+
+        @conn.request(cmd, *args)
+      end
+
+      def bind_handler!
+        handler = Proc.new do |cmd, meta_and_payload|
+          cmd.nil? ? (@established = false) : @message_tracker.receive(cmd, meta_and_payload)
+        end
+        @conn.consumer_handlers.add(@consumer_id, handler)
+      end
+
+      def remove_handler!
+        @conn.consumer_handlers.delete(@consumer_id)
+
+        true
+      end
+
+      def init_consumer
+        bind_handler!
+
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::SUBSCRIBE,
+          subscribe: Pulsar::Proto::CommandSubscribe.new(
+            topic: @opts.topic,
+            subscription: @opts.subscription_name,
+            subType: @opts.subscription_type,
+            consumer_name: @consumer_name,
+            replicate_subscription_state: @opts.replicate_subscription_state,
+            read_compacted: @opts.read_compacted
+          )
+        )
+        result = execute(base_cmd)
+
+        @message_tracker.add_consumer(self)
+
+        result.consumerStatsResponse
+      end
+
+      # NOTE keep consumer_id and sequence_id static
+      class SeqGenerator
+        def initialize(seq_g)
+          @seq_g = seq_g
+          @consumer_id = @seq_g.new_consumer_id
+        end
+
+        def new_consumer_id
+          @consumer_id
+        end
+
+        def method_missing(method)
+          @seq_g.public_send(method)
+        end
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/consumer/manager.rb

@@ -0,0 +1,127 @@
+module PulsarSdk
+  module Consumer
+    class Manager
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      def initialize(client, opts)
+        raise "client expected a PulsarSdk::Client::Rpc got #{client.class}" unless client.is_a?(PulsarSdk::Client::Rpc)
+        raise "opts expected a PulsarSdk::Options::Consumer got #{opts.class}" unless opts.is_a?(PulsarSdk::Options::Consumer)
+
+        @topic = opts.topic
+
+        @listen_wait = opts.listen_wait
+
+        @message_tracker = ::PulsarSdk::Consumer::MessageTracker.new(opts.redelivery_delay)
+
+        @consumers = init_consumer_by(client, opts)
+
+        @stoped = false
+      end
+
+      # NOTE some topic maybe have large permits if there is no message
+      def flow
+        ensure_connection
+        @consumers.each(&:flow_if_need)
+      end
+
+      # NOTE all consumers has same name
+      def subscription
+        ensure_connection
+        @consumers.find(&:subscription)
+      end
+
+      def unsubscribe
+        ensure_connection
+        @consumers.each(&:unsubscribe)
+      end
+
+      # if timeout is nil wait until get message
+      def receive(timeout = nil)
+        ensure_connection
+        @message_tracker.shift(timeout)
+      end
+
+      def listen(autoack = false)
+        raise 'listen require passing a block!!' if !block_given?
+        ensure_connection
+
+        loop do
+          return if @stoped
+
+          flow
+
+          cmd, msg = receive(@listen_wait)
+          return if msg.nil?
+
+          result = yield cmd, msg
+
+          if autoack && result == false
+            msg.nack
+            next
+          end
+
+          msg.ack if autoack
+        end
+      end
+
+      def close
+        PulsarSdk.logger.debug(__method__){"current @stoped #{@stoped} close now!"}
+        return if @stoped
+        @consumers.each(&:close)
+        @stoped = true
+
+        @message_tracker.close
+      end
+
+      private
+      def init_consumer_by(client, opts)
+        topics = []
+
+        case
+        when !opts.topic.nil?
+          PulsarSdk.logger.debug("#{__method__}:single topic"){opts.topic}
+
+          topics << ::PulsarSdk::Protocol::Topic.parse(opts.topic).to_s
+        when !Array(opts.topics).size.zero?
+          PulsarSdk.logger.debug("#{__method__}:multiple topics"){opts.topics}
+
+          opts.topics.each do |topic|
+            topics << ::PulsarSdk::Protocol::Topic.parse(topic).to_s
+          end
+        when !opts.topics_pattern.nil?
+          PulsarSdk.logger.debug("#{__method__}:pattern topic"){opts.topics_pattern}
+
+          tn = ::PulsarSdk::Protocol::Topic.parse(opts.topics_pattern)
+          pattern = Regexp.compile(tn.topic == '*' ? '^*' : tn.topic)
+          client.namespace_topics(tn.namespace).each do |topic|
+            topics << topic if pattern.match(topic)
+          end
+        else
+          raise 'You must provide one topic by 「topic」or「topics」or「topics_pattern」'
+        end
+
+        PulsarSdk.logger.debug("#{__method__}:topics to initialize"){topics}
+
+        topics.flat_map do |topic|
+          partition_topics = client.partition_topics(topic)
+
+          partition_topics.map do |x|
+            opts_ = opts.dup
+
+            opts_.topic = x
+            PulsarSdk::Consumer::Base.new(client, @message_tracker, opts_).tap do |consumer|
+              consumer.grab_cnx
+            end
+          end
+        end
+      end
+
+      def ensure_connection
+        @consumers.each do |consumer|
+          next unless consumer.disconnect?
+          consumer.grab_cnx
+        end
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/consumer/message_tracker.rb

@@ -0,0 +1,86 @@
+module PulsarSdk
+  module Consumer
+    class MessageTracker
+
+      class ReceivedQueue < PulsarSdk::Tweaks::TimeoutQueue; end
+      class NackQueue < PulsarSdk::Tweaks::BinaryHeap; end
+
+      def initialize(redelivery_delay)
+        @redelivery_delay = redelivery_delay
+        @received_message = ReceivedQueue.new
+        @acknowledge_message = NackQueue.new {|parent, child| child[:ack_at] <=> parent[:ack_at] }
+        @consumers = {}
+
+        @tracker = track
+      end
+
+      def add_consumer(consumer)
+        @consumers[consumer.consumer_id] = consumer
+      end
+
+      def receive(*args)
+        @received_message.add(*args)
+      end
+
+      def shift(timeout)
+        cmd, meta_and_payload = @received_message.pop(timeout)
+
+        return if cmd.nil?
+
+        message = PulsarSdk::Protocol::Structure.new(meta_and_payload).decode
+
+        consumer_id = cmd.message&.consumer_id
+        real_consumer = @consumers[consumer_id]
+
+        message.assign_attributes(
+          message_id: cmd.message&.message_id,
+          consumer_id: consumer_id,
+          topic: real_consumer&.topic,
+          ack_handler: ack_handler
+        )
+
+        real_consumer&.increase_fetched
+
+        [cmd, message]
+      end
+
+      def close
+        @received_message.close
+      end
+
+      private
+      def track
+        Thread.new do
+          loop do
+            while item = @acknowledge_message.top
+              break if item[:ack_at] > Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              begin
+                PulsarSdk.logger.debug('acknowledge message'){"#{Process.clock_gettime(Process::CLOCK_MONOTONIC)}: #{item[:cmd].type} --> #{item[:ack_at]}"}
+                execute_async(item[:cmd])
+                @acknowledge_message.shift
+              rescue => exp
+                PulsarSdk.logger.error('Error occur when acknowledge message'){exp}
+                PulsarSdk.logger.error('Error occur when acknowledge message'){item}
+              end
+            end
+            sleep(1)
+          end
+        end
+      end
+
+      def ack_handler
+        Proc.new do |cmd|
+          current_clock = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          ack_at = cmd.typeof_ack? ? (current_clock - 1) : (current_clock + @redelivery_delay.to_i)
+          @acknowledge_message.insert(cmd: cmd, ack_at: ack_at)
+        end
+      end
+
+      def execute_async(cmd)
+        consumer = @consumers[cmd.get_consumer_id]
+
+        consumer.execute_async(cmd)
+      end
+    end
+  end
+end