pulsar_sdk 0.8.8

data/lib/pulsar_sdk/options.rb

@@ -0,0 +1,6 @@
+require 'pulsar_sdk/options/base'
+require 'pulsar_sdk/options/tls'
+require 'pulsar_sdk/options/connection'
+require 'pulsar_sdk/options/consumer'
+require 'pulsar_sdk/options/producer'
+require 'pulsar_sdk/options/reader'

data/lib/pulsar_sdk/options/base.rb

@@ -0,0 +1,10 @@
+require 'securerandom'
+
+module PulsarSdk
+  module Options
+    class Base
+      prepend ::PulsarSdk::Tweaks::AssignAttributes
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+    end
+  end
+end

data/lib/pulsar_sdk/options/connection.rb

@@ -0,0 +1,51 @@
+module PulsarSdk
+  module Options
+    class Connection < Base
+      DEFAULT_PORT = 6650
+
+      attr_accessor :logical_addr, :physical_addr,
+                    :connection_timeout,
+                    :operation_timeout,
+                    :auth_provider,
+                    :tls_options,
+                    :keepalive
+
+      [:logical_addr, :physical_addr].each do |x|
+        define_method "#{x}=" do |v|
+          return instance_variable_set("@#{x}", v) if v.nil?
+          v = v.is_a?(::URI) ? v : ::URI.parse(v)
+          v.port = DEFAULT_PORT if v.port.nil?
+          instance_variable_set("@#{x}", v)
+        end
+      end
+
+      def connecting_through_proxy?
+        logical_addr == physical_addr
+      end
+
+      def proxy_to_broker_url
+        connecting_through_proxy? ? logical_addr : nil
+      end
+
+      def port_and_host_from(name)
+        v = instance_variable_get("@#{name}")
+        return if v.nil?
+        [v.port, v.host]
+      end
+
+      private
+      def set_default
+        self.logical_addr = ENV.fetch("PULSAR_BROKER_URL", nil)
+
+        # 连接5秒超时
+        self.connection_timeout = 5
+
+        # Set the operation timeout (default: 30 seconds)
+        # Producer-create, subscribe and unsubscribe operations will be retried until this interval
+        self.operation_timeout = 30
+
+        self.keepalive = 30
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/options/consumer.rb

@@ -0,0 +1,34 @@
+module PulsarSdk
+  module Options
+    class Consumer < Base
+      attr_accessor :topic, :topics, :topics_pattern,
+                    :name, :subscription_name, :subscription_type,
+                    :prefetch, :redelivery_delay,
+                    :listen_wait, :replicate_subscription_state,
+                    :read_compacted
+
+      def subscription_type
+        sub_type = @subscription_type.to_sym
+        if Pulsar::Proto::CommandSubscribe::SubType.constants.include?(sub_type)
+          return Pulsar::Proto::CommandSubscribe::SubType.resolve(sub_type)
+        end
+
+        raise "subscription_type mismatch! available is #{Pulsar::Proto::CommandSubscribe::SubType.constants}, got 「#{@subscription_type}」"
+      end
+
+      private
+      def set_default
+        self.name = 'ruby-consumer.' + SecureRandom.urlsafe_base64(10)
+        # 相同名字的subscription与订阅模式有关
+        self.subscription_name = 'ruby-subscription'
+        self.subscription_type = :Exclusive
+        # 延迟消息重发，默认60秒
+        self.redelivery_delay = 60
+        # 记录预取数量
+        self.prefetch = 1000
+        self.replicate_subscription_state = true
+        self.read_compacted = false
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/options/producer.rb

@@ -0,0 +1,14 @@
+module PulsarSdk
+  module Options
+    class Producer < Base
+      attr_accessor :topic, :name, :router
+      attr_accessor :schema # TODO
+
+      private
+      def set_default
+        self.name = 'ruby-producer.' + SecureRandom.urlsafe_base64(10)
+        self.router = PulsarSdk::Producer::Router.new
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/options/reader.rb

@@ -0,0 +1,7 @@
+module PulsarSdk
+  module Options
+    class Reader < Base
+
+    end
+  end
+end

data/lib/pulsar_sdk/options/tls.rb

@@ -0,0 +1,8 @@
+module PulsarSdk
+  module Options
+    class Tls < Base
+      attr_accessor :trust_certs_file_path, :allow_insecure_connection,
+                    :validate_hostname
+    end
+  end
+end

data/lib/pulsar_sdk/producer.rb

@@ -0,0 +1,14 @@
+require 'pulsar_sdk/producer/base'
+require 'pulsar_sdk/producer/message'
+require 'pulsar_sdk/producer/router'
+require 'pulsar_sdk/producer/manager'
+
+module PulsarSdk
+  module Producer
+    extend self
+
+    def create(client, opts)
+      PulsarSdk::Producer::Manager.new(client, opts)
+    end
+  end
+end

data/lib/pulsar_sdk/producer/base.rb

@@ -0,0 +1,154 @@
+module PulsarSdk
+  module Producer
+    class Base
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      def initialize(client, opts)
+        @opts = opts
+        @client = client
+      end
+
+      def grab_cnx
+        topic = @opts.topic
+        @conn = @client.connection(*@client.lookup(topic))
+        @established = true
+
+        @seq_generator = SeqGenerator.new(@conn.seq_generator)
+        @producer_id = @seq_generator.new_producer_id
+
+        @producer_name = [@opts.name, @producer_id].join('.')
+
+        @receipt_queue = ReceiptQueue.new
+
+        @stoped = false
+
+        @producer_name = init_producer(topic)
+      end
+
+      def execute(cmd, msg = nil, timeout = nil)
+        write(cmd, msg, false, timeout)
+      end
+
+      def execute_async(cmd, msg = nil)
+        write(cmd, msg, true)
+      end
+
+      # 获取发送回执
+      # TODO get receipt by sequence_id
+      def receipt
+        receipt_ = @receipt_queue.pop.first
+        return if receipt_.nil?
+
+        if block_given?
+          yield receipt_
+        end
+
+        receipt_
+      end
+
+      def close
+        return if @stoped
+
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::CLOSE_PRODUCER,
+          close_producer: Pulsar::Proto::CommandCloseProducer.new
+        )
+        execute(base_cmd) unless disconnect?
+
+        unbind_handler!
+
+        @stoped = true
+
+        @receipt_queue.close
+      end
+
+      def disconnect?
+        !@established
+      end
+
+      private
+      def init_producer(topic)
+        bind_handler!
+
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::PRODUCER,
+          producer: Pulsar::Proto::CommandProducer.new(
+            topic: topic
+          )
+        )
+        result = execute(base_cmd)
+        result.producer_success.producer_name
+      end
+
+      def write(cmd, msg, *args)
+        unless msg.nil? || msg.is_a?(PulsarSdk::Producer::Message)
+          raise "msg expected a PulsarSdk::Producer::Message got #{msg.class}"
+        end
+
+        grab_cnx if disconnect?
+
+        cmd.seq_generator = @seq_generator
+
+        unless msg.nil?
+          msg.producer_name = @producer_name
+          msg.sequence_id = @seq_generator.new_sequence_id
+        end
+
+        result = @conn.request(set_seq_generator(cmd), filling_message(msg), *args)
+
+        # increase sequence_id when success
+        @seq_generator.new_sequence_id(false) unless msg.nil?
+
+        result
+      end
+
+      def set_seq_generator(cmd)
+        cmd.seq_generator = @seq_generator
+        cmd
+      end
+
+      def filling_message(msg)
+        return if msg.nil?
+        msg.producer_name = @producer_name
+        msg
+      end
+
+      def bind_handler!
+        handler = Proc.new do |send_receipt|
+          send_receipt.nil? ? (@established = false) : @receipt_queue.add(send_receipt)
+        end
+        @conn.producer_handlers.add(@producer_id, handler)
+      end
+
+      def unbind_handler!
+        @conn.producer_handlers.delete(@producer_id)
+
+        true
+      end
+    end
+
+    class ReceiptQueue < ::PulsarSdk::Tweaks::TimeoutQueue; end
+
+    # NOTE keep producer_id and sequence_id static
+    class SeqGenerator
+      def initialize(seq_g)
+        @seq_g = seq_g
+        @producer_id = @seq_g.new_producer_id
+        @sequence_id = @seq_g.new_sequence_id
+      end
+
+      def new_producer_id
+        @producer_id
+      end
+
+      def new_sequence_id(cache = true)
+        return @sequence_id if cache
+        @sequence_id = @seq_g.new_sequence_id
+      end
+
+      def method_missing(method)
+        @seq_g.public_send(method)
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/producer/manager.rb

@@ -0,0 +1,67 @@
+module PulsarSdk
+  module Producer
+    class Manager
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      def initialize(client, opts)
+        @topic = opts.topic
+        @producers = init_producer_by(client, opts)
+        @router = opts.router
+      end
+
+      def execute(cmd, msg = nil, timeout = nil)
+        raise "cmd expected a Pulsar::Proto::BaseCommand got #{cmd.class}" unless cmd.is_a?(Pulsar::Proto::BaseCommand)
+        real_producer(msg) do |producer|
+          producer.execute(cmd, msg, timeout)
+        end
+      end
+
+      def execute_async(cmd, msg = nil)
+        raise "cmd expected a Pulsar::Proto::BaseCommand got #{cmd.class}" unless cmd.is_a?(Pulsar::Proto::BaseCommand)
+        real_producer(msg) do |producer|
+          producer.execute_async(cmd, msg)
+        end
+      end
+
+      def real_producer(msg, &block)
+        if @producers.size.zero?
+          PulsarSdk.logger.warn(__method__){"There is no available producer for topic: 「#{@topic}」, skipping action!"}
+          return
+        end
+
+        ensure_connection
+
+        route_index = msg.nil? ? 0 : @router.route(msg.key, @producers.size)
+
+        yield @producers[route_index]
+      end
+
+      def close
+        @producers.each(&:close)
+      end
+
+      private
+      def init_producer_by(client, opts)
+        opts = opts.dup
+
+        topics = client.partition_topics(@topic)
+        topics.map do |topic|
+          opts.topic = topic
+          PulsarSdk::Producer::Base.new(client, opts).tap do |base|
+            base.grab_cnx
+          end
+        end
+      end
+
+      def ensure_connection
+        @producers.each do |producer|
+          next unless producer.disconnect?
+          PulsarSdk.logger.warn('PulsarSdk::Producer::Manager#ensure_connection'){
+            "connection closed! reconnect now! #{producer.inspect}"
+          }
+          producer.grab_cnx
+        end
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/producer/message.rb

@@ -0,0 +1,47 @@
+module PulsarSdk
+  module Producer
+    class Message
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      attr_reader :metadata, :message, :key
+
+      def initialize(msg, metadata = nil, key = nil)
+        # TODO check metadata type
+        @message, @metadata = msg, metadata
+        @metadata ||= Pulsar::Proto::MessageMetadata.new
+
+        # msg must convet to string
+        json_encode! unless @message.is_a?(String)
+
+        publish_time = @metadata.publish_time
+        @metadata.publish_time = publish_time.zero? ? TimeX.now.timestamp : publish_time
+
+        self.key = key
+      end
+
+      def producer_name=(v)
+        @metadata.producer_name = v
+      end
+
+      def sequence_id=(v)
+        @metadata.sequence_id = v
+      end
+
+      def binary_string
+        @message.bytes.pack('C*')
+      end
+
+      def key=(v, b64 = false)
+        @metadata.partition_key = v.to_s
+        @metadata.partition_key_b64_encoded = b64
+      end
+
+      private
+      def json_encode!
+        PulsarSdk.logger.info("#{self.class}::#{__method__}"){"message was 「#{@message.class}」 now encode to json!"}
+        @message = @message.respond_to?(:to_json) ? @message.to_json : JSON.dump(@message)
+        @metadata.properties << Pulsar::Proto::KeyValue.new(key: 'Content-Type', value: 'application/json; charset=utf-8')
+      end
+    end
+  end
+end

data/lib/pulsar_sdk/producer/router.rb

@@ -0,0 +1,100 @@
+module PulsarSdk
+  module Producer
+    class Router
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      def initialize(scheme = :string_hash)
+        case scheme.to_sym
+        when :string_hash
+          @handler = string_hash
+        when :murmur_hash
+          @handler = murmur3_hash
+        else
+          raise "Unknown hash scheme #{scheme}"
+        end
+      end
+
+      def route(key, total, delay = 0)
+        return 0 if total <= 1
+
+        return (@handler.call(key) % total) unless key.to_s.empty?
+
+        Murmur3.int32_hash(TimeX.now.timestamp) % total
+      end
+
+      # 将hash值限制在32位内，防止key过长导致过多内存占用
+      def string_hash
+        max_mod = 1 << 32
+        Proc.new do |key|
+          h = 0
+          key.to_s.each_byte do |x|
+            h = (31*h)%max_mod + x
+          end
+
+          h
+        end
+      end
+
+      def murmur3_hash
+        Proc.new do |key, seed = 31|
+          # Using 0x7fffffff was maintain compatibility with values used in Java client
+          Murmur3.hash(key, seed) & 0x7fffffff
+        end
+      end
+    end
+
+    # cpoy from https://github.com/funny-falcon/murmurhash3-ruby
+    module Murmur3
+      extend self
+
+      MASK32 = 0xffffffff
+
+      def int32_hash(i, seed=0)
+        hash([i].pack("V"), seed)
+      end
+
+      def hash(str, seed=0)
+        h1 = seed
+        numbers = str.unpack('V*C*')
+        tailn = str.bytesize % 4
+        tail = numbers.slice!(numbers.size - tailn, tailn)
+        for k1 in numbers
+          h1 ^= mmix(k1)
+          h1 = rotl(h1, 13)
+          h1 = (h1*5 + 0xe6546b64) & MASK32
+        end
+
+        unless tail.empty?
+          k1 = 0
+          tail.reverse_each do |c1|
+            k1 = (k1 << 8) | c1
+          end
+          h1 ^= mmix(k1)
+        end
+
+        h1 ^= str.bytesize
+        fmix(h1)
+      end
+
+      private
+      def rotl(x, r)
+        ((x << r) | (x >> (32 - r))) & MASK32
+      end
+
+      def fmix(h)
+        h &= MASK32
+        h ^= h >> 16
+        h = (h * 0x85ebca6b) & MASK32
+        h ^= h >> 13
+        h = (h * 0xc2b2ae35) & MASK32
+        h ^ (h >> 16)
+      end
+
+      def mmix(k1)
+        k1 = (k1 * 0xcc9e2d51) & MASK32
+        k1 = rotl(k1, 15)
+        (k1 * 0x1b873593) & MASK32
+      end
+    end
+  end
+end