pulsar_sdk 0.8.8

data/lib/protobuf/validate.rb

@@ -0,0 +1,41 @@
+require 'google/protobuf'
+
+module Google
+  module Protobuf
+    module MessageExts
+      alias_method :orig_to_json, :to_json
+      alias_method :orig_to_proto, :to_proto
+
+      def to_json(options = {})
+        validate_presence!
+        orig_to_json(options)
+      end
+
+      def to_proto
+        validate_presence!
+        orig_to_proto
+      end
+
+      def validate_presence!
+        self.class.descriptor.entries.each do |entry|
+          next if entry.label != :required
+
+          v = self[entry.name]
+
+          validate_fail = case entry.type
+          when :int64, :uint64, :int32, :uint32,  :double, :float
+            v.nil? || v == 0
+          when :string
+            v.nil? || v.empty?
+          when :enum
+            v.nil? || !entry.subtype.entries.map(&:first).include?(v)
+          else
+            v.nil?
+          end
+
+          raise "#{self.class.name}::#{entry.name} was required, but got 「#{v.inspect}」" if validate_fail
+        end
+      end
+    end
+  end
+end

data/lib/pulsar_admin.rb

@@ -0,0 +1,14 @@
+require 'net/http'
+require 'pulsar_admin/api'
+
+module PulsarAdmin
+  extend self
+
+  # options
+  #   endpoint
+  #   tenant
+  #   persistent
+  def create_client(options)
+    PulsarAdmin::Api.new(options)
+  end
+end

data/lib/pulsar_admin/api.rb

@@ -0,0 +1,215 @@
+module PulsarAdmin
+  class Api
+    PublishTimeHeader = /^X-Pulsar-Publish-Time$/i
+    BatchHeader       = /^X-Pulsar-Num-Batch-Message$/i
+    PropertyPrefix    = /^X-Pulsar-PROPERTY-/i
+
+    # opts
+    #   endpoint
+    #   tenant
+    #   persistent
+    def initialize(opts)
+      @endpoint = URI.parse(opts[:endpoint])
+      @tenant = opts[:tenant]
+      @persistent = opts[:persistent] == false ? 'non-persistent' : 'persistent'
+    end
+
+    def list_namespaces
+      get('/admin/v2/namespaces/:tenant').map do |ns|
+        ns.sub("#{@tenant}/", '')
+      end
+    end
+
+    def create_namespace(name)
+      put('/admin/v2/namespaces/:tenant/:namespace', namespace: name)
+    end
+
+    def delete_namespace(name)
+      delete('/admin/v2/namespaces/:tenant/:namespace', namespace: name)
+    end
+
+    def namespace_topics(namespace)
+      result = {}
+      ['', '/partitioned'].flat_map do |pd|
+        resp = get("/admin/v2/:persistent/:tenant/:namespace#{pd}", namespace: namespace)
+        result[pd.empty? ? 'non-partitioned' : 'partitioned'] = resp
+      end
+      result
+    end
+
+    def create_topic(namespace, topic, partitions = 0)
+      put("/admin/v2/:persistent/:tenant/:namespace/:topic#{partitions.zero? ? '' : '/partitions'}",
+            {namespace: namespace, topic: topic}, partitions
+          )
+    end
+
+    def delete_topic(namespace, topic)
+      res1 = delete('/admin/v2/:persistent/:tenant/:namespace/:topic',
+              namespace: namespace, topic: topic
+            )
+
+      res2 = delete('/admin/v2/:persistent/:tenant/:namespace/:topic/partitions',
+        namespace: namespace, topic: topic
+      )
+
+      res1 || res2
+    end
+
+    # options
+    #   namespace
+    #   topic
+    #   sub_name
+    #   message_position
+    #   count
+    def peek_messages(options)
+      (options[:count] || 1).times.map do |x|
+        opts = options.dup
+        opts[:message_position] = (opts[:message_position].to_i + x + 1).to_s
+        peek_message(opts)
+      end.compact
+    end
+
+    private
+    def put(path, payload = {}, body = nil)
+      uri = @endpoint.dup
+      uri.path, payload = handle_restful_path(path, payload)
+
+      req = Net::HTTP::Put.new(uri)
+
+      if payload.empty?
+        req.body = body.to_s
+        req.content_type = body.nil? ? 'application/json' : 'text/plain'
+      else
+        req.body = payload.to_json
+        req.content_type = 'application/json'
+      end
+
+      res = Net::HTTP.start(uri.hostname, uri.port) do |http|
+        http.request(req)
+      end
+
+      case res
+      when Net::HTTPSuccess, Net::HTTPNoContent
+        return true
+      else
+        puts "status: #{res.code} - body: #{res.body} - #{res.inspect}"
+        return false
+      end
+    end
+
+    def raw_get(path, params = {})
+      uri = @endpoint.dup
+      uri.path, params = handle_restful_path(path, params)
+
+      req = Net::HTTP::Get.new(uri)
+      req.set_form_data(params) unless params.empty?
+
+      Net::HTTP.start(uri.hostname, uri.port) do |http|
+        http.request(req)
+      end
+    end
+
+    def get(path, params = {})
+      resp = raw_get(path, params)
+      try_decode_body(resp)
+    end
+
+    def delete(path, payload = {})
+      uri = @endpoint.dup
+      uri.path, payload = handle_restful_path(path, payload)
+
+      req = Net::HTTP::Delete.new(uri)
+      req.body = payload.to_json
+      req.content_type = 'application/json'
+
+      res = Net::HTTP.start(uri.hostname, uri.port) do |http|
+        http.request(req)
+      end
+
+      case res
+      when Net::HTTPSuccess, Net::HTTPNoContent
+        return true
+      else
+        return false
+      end
+    end
+
+    # options
+    #   namespace
+    #   topic
+    #   sub_name
+    #   message_position
+    def peek_message(options)
+      options[:message_position] = options[:message_position].to_s
+      resp = raw_get('/admin/v2/:persistent/:tenant/:namespace/:topic/subscription/:sub_name/position/:message_position', options)
+      unless request_ok?(resp)
+        puts resp.body
+        return
+      end
+
+      payload = resp.body
+
+      msg_id = nil
+      properties = {}
+      resp.each_header do |header|
+        case header
+        when PublishTimeHeader
+          properties['publish-time'] = resp.header[header]
+        when BatchHeader
+          properties['pulsar-num-batch-message'] = resp.header[header]
+        when PropertyPrefix
+          properties[header] = resp.header[header]
+        when /^X-Pulsar-Message-ID$/i
+          msg_id = resp.header[header]
+        end
+      end
+      [
+        msg_id,
+        properties,
+        payload
+      ]
+    end
+
+    def handle_restful_path(path, options)
+      return path unless path.include?(':')
+
+      opts = combine_default_value(options || {})
+      opts.keys.sort.reverse.each do |k|
+        remark = ":#{k}"
+        next unless path.include?(remark)
+        path.gsub!(remark, opts[k])
+        options.delete(k)
+      end
+
+      return [path, options]
+    end
+
+    def combine_default_value(opts)
+      opts.merge(
+        tenant: @tenant,
+        persistent: @persistent
+      )
+    end
+
+    def request_ok?(resp)
+      case resp
+      when Net::HTTPSuccess
+        true
+      when Net::HTTPRedirection
+        false
+      else
+        false
+      end
+    end
+
+    def try_decode_body(resp)
+      unless request_ok?(resp)
+        puts resp.body
+        return
+      end
+      return resp.body unless resp.content_type =~ /application\/json/
+
+      JSON.parse(resp.body) rescue resp.body
+    end
+  end
+end

data/lib/pulsar_sdk.rb

@@ -0,0 +1,55 @@
+require 'logger'
+require 'json'
+require 'uri'
+require 'protobuf/validate'
+require 'protobuf/pulsar_api.pb'
+require "pulsar_sdk/version"
+require 'pulsar_sdk/tweaks'
+require 'pulsar_sdk/protocol'
+require 'pulsar_sdk/options'
+require 'pulsar_sdk/consumer'
+require 'pulsar_sdk/producer'
+require 'pulsar_sdk/client'
+
+module PulsarSdk
+  extend self
+
+  # options Hash see PulsarSdk::Options::Connection for detail
+  def create_client(options)
+    opts = ::PulsarSdk::Options::Connection.new(options)
+    ::PulsarSdk::Client.create(opts)
+  end
+
+  # options Hash see PulsarSdk::Options::Producer for detail
+  def create_producer(client, options)
+    opts = ::PulsarSdk::Options::Producer.new(options)
+    client.create_producer(opts)
+  end
+
+  # options Hash see PulsarSdk::Options::Consumer for detail
+  def create_consumer(client, options)
+    opts = ::PulsarSdk::Options::Consumer.new(options)
+    client.subscribe(opts)
+  end
+
+  def logger
+    @logger ||= Logger.new(STDOUT).tap do |logger|
+                  logger.formatter = Formatter.new
+                end
+  end
+
+  def logger=(v)
+    @logger = v
+  end
+
+  class Formatter < ::Logger::Formatter
+    def call(severity, timestamp, progname, msg)
+      case msg
+      when ::StandardError
+        msg = [msg.message, msg&.backtrace].join(":\n")
+      end
+
+      super
+    end
+  end
+end

data/lib/pulsar_sdk/client.rb

@@ -0,0 +1,13 @@
+require 'pulsar_sdk/client/rpc'
+require 'pulsar_sdk/client/connection'
+require 'pulsar_sdk/client/connection_pool'
+
+module PulsarSdk
+  module Client
+    extend self
+
+    def create(opts)
+      PulsarSdk::Client::Rpc.new(opts)
+    end
+  end
+end

data/lib/pulsar_sdk/client/connection.rb

@@ -0,0 +1,371 @@
+require 'socket'
+
+module PulsarSdk
+  module Client
+    class Connection
+      prepend ::PulsarSdk::Tweaks::CleanInspect
+
+      CLIENT_NAME = "pulsar-client-#{PulsarSdk::VERSION}".freeze
+      PROTOCOL_VER = Pulsar::Proto::ProtocolVersion::V13
+
+      attr_reader :consumer_handlers
+      attr_reader :producer_handlers
+      attr_reader :response_container # 用于处理状态回调
+      attr_reader :seq_generator
+
+      # opts PulsarSdk::Options::Connection
+      def initialize(opts)
+        @conn_options = opts
+
+        @socket = nil
+        @state = Status.new
+
+        @seq_generator = SeqGenerator.new
+
+        @consumer_handlers = ConsumerHandler.new
+        @producer_handlers = ProducerHandler.new
+        @response_container = ResponseContainer.new
+      end
+
+      def start
+        unless connect && do_hand_shake && listen
+          @state.closed!
+        end
+      end
+
+      def self.establish(opts)
+        conn = new(opts).tap do |c|
+          c.start
+        end
+        # TODO check connection ready
+        conn
+      end
+
+      def close
+        @state.closed!
+        consumer_handlers.each{|_k, v| v.call}
+        producer_handlers.each{|_k, v| v.call}
+        Timeout::timeout(2) {@pong&.join} rescue @pong&.kill
+        @pong&.join
+      ensure
+        @socket.close
+      end
+
+      def closed?
+        @state.closed?
+      end
+
+      def ping
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::PING,
+          ping: Pulsar::Proto::CommandPing.new
+        )
+
+        request(base_cmd, nil, true)
+
+        @state.ping!
+      end
+
+      def active_status
+        [@state.last_ping_at, @state.last_received_at]
+      end
+
+      def request(cmd, msg = nil, async = false, timeout = nil)
+        raise 'connection was closed!' if closed?
+
+        cmd.seq_generator ||= @seq_generator
+
+        # NOTE try to auto set *_id
+        cmd.handle_ids
+
+        frame = PulsarSdk::Protocol::Frame.encode(cmd, msg)
+        write(frame)
+        return true if async
+
+        if request_id = cmd.get_request_id
+          return @response_container.delete(request_id, timeout)
+        end
+
+        true
+      end
+
+      private
+      def reader
+        @reader ||= PulsarSdk::Protocol::Reader.new(@socket)
+      end
+
+      def write(bytes)
+        begin
+          @socket.write_nonblock(bytes)
+        rescue IO::WaitWritable
+          IO.select(nil, [@socket], nil, @conn_options.operation_timeout)
+          retry
+        end
+      end
+
+      def listen
+        @pong = Thread.new do
+          loop do
+            break if closed?
+
+            begin
+              @state.ready? ? read_from_connection : @state.wait
+            rescue Errno::ETIMEDOUT
+              # read timeout, do nothing
+            rescue => e
+              PulsarSdk.logger.error("reader error") {e}
+              close
+            end
+          end
+        end
+        @pong.abort_on_exception = false
+
+        true
+      end
+
+      def connect
+        return true if (@socket && !closed?)
+
+        @socket = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+        @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+        @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_KEEPALIVE, true)
+
+        host_port = @conn_options.port_and_host_from(:logical_addr)
+
+        sockaddr = Socket.sockaddr_in(*host_port)
+        begin
+          # Initiate the socket connection in the background. If it doesn't fail
+          # immediately it will raise an IO::WaitWritable (Errno::EINPROGRESS)
+          # indicating the connection is in progress.
+          @socket.connect_nonblock(sockaddr)
+        rescue IO::WaitWritable
+          # IO.select will block until the socket is writable or the timeout
+          # is exceeded, whichever comes first.
+          unless IO.select(nil, [@socket], nil, @conn_options.connection_timeout)
+            # IO.select returns nil when the socket is not ready before timeout
+            # seconds have elapsed
+            @socket.close
+            return false
+          end
+
+          begin
+            # Verify there is now a good connection.
+            @socket.connect_nonblock(sockaddr)
+          rescue Errno::EISCONN
+            # The socket is connected, we're good!
+          end
+        end
+
+        @state.tcp_connected!
+
+        true
+      end
+
+      def do_hand_shake
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::CONNECT,
+          connect: Pulsar::Proto::CommandConnect.new(
+            client_version: CLIENT_NAME,
+            protocol_version: PROTOCOL_VER,
+            proxy_to_broker_url: @conn_options.proxy_to_broker_url
+          )
+        )
+
+        request(base_cmd)
+
+        @state.ready!
+        true
+      end
+
+      def read_from_connection
+        base_cmd, meta_and_payload = reader.read_fully
+        return if base_cmd.nil?
+
+        @state.received!
+
+        handle_base_command(base_cmd, meta_and_payload)
+      end
+
+      def handle_base_command(cmd, payload)
+        PulsarSdk.logger.debug(__method__){cmd.type} unless cmd.typeof_ping?
+
+        case
+        when cmd.typeof_success?
+          handle_response(cmd)
+
+        when cmd.typeof_connected?
+          PulsarSdk.logger.info(__method__){"#{cmd.type}: #{cmd.connected}"}
+
+        when cmd.typeof_producer_success?
+          handle_response(cmd)
+
+        when cmd.typeof_lookup_response?
+          handle_response(cmd)
+
+        when cmd.typeof_get_last_message_id_response?
+          handle_response(cmd)
+
+        when cmd.typeof_consumer_stats_response?
+          handle_response(cmd)
+
+        when cmd.typeof_reached_end_of_topic?
+          # TODO notify consumer no more message
+
+        when cmd.typeof_get_topics_of_namespace_response?
+          handle_response(cmd)
+
+        when cmd.typeof_get_schema_response?
+        when cmd.typeof_partitioned_metadata_response?
+          handle_response(cmd)
+
+        when cmd.typeof_error?
+          PulsarSdk.logger.error(__method__){"#{cmd.error}: #{cmd.message}"}
+
+        when cmd.typeof_close_producer?
+          producer_id = cmd.close_producer.producer_id
+          producer_handlers.find(producer_id)&.call
+
+        when cmd.typeof_close_consumer?
+          consumer_id = cmd.close_consumer.consumer_id
+          consumer_handlers.find(consumer_id)&.call
+
+        when cmd.typeof_active_consumer_change?
+        when cmd.typeof_message?
+          handle_message(cmd, payload)
+
+        when cmd.typeof_send_receipt?
+          handle_send_receipt(cmd)
+
+        when cmd.typeof_ping?
+          handle_ping
+
+        when cmd.typeof_pong?
+
+        else
+          close
+          raise "Received invalid command type: #{cmd.type}"
+        end
+
+        true
+      end
+
+      def handle_response(cmd)
+        request_id = cmd.get_request_id
+        return if request_id.nil?
+        @response_container.add(request_id, cmd)
+      end
+
+      def handle_message(cmd, payload)
+        consumer_id = cmd.get_consumer_id
+        if consumer_id.nil?
+          ::PulsarSdk.logger.warn(__method__){"can not get consumer id from cmd: #{cmd.inspect}"}
+          return
+        end
+        handler = consumer_handlers.find(consumer_id)
+        if handler.nil?
+          ::PulsarSdk.logger.warn(__method__){"can not get consumer_handler from cmd: #{cmd.inspect}"}
+          return
+        end
+        handler.call(cmd, payload)
+      end
+
+      def handle_send_receipt(cmd)
+        send_receipt = cmd.send_receipt
+        producer_id = send_receipt.producer_id
+        handler = producer_handlers.find(producer_id)
+        return if handler.nil?
+        handler.call(send_receipt)
+      end
+
+      def handle_ping
+        base_cmd = Pulsar::Proto::BaseCommand.new(
+          type: Pulsar::Proto::BaseCommand::Type::PONG,
+          pong: Pulsar::Proto::CommandPong.new
+        )
+
+        request(base_cmd, nil, true)
+      end
+
+      class Status
+        attr_reader :last_received_at, :last_ping_at
+
+        STATUS = %w[
+          init
+          connecting
+          tcp_connected
+          ready
+          closed
+        ].freeze
+
+        def initialize
+          @state = 'init'
+          @lock = Mutex.new
+          @signal = ConditionVariable.new
+          @last_received_at = 0
+          @last_ping_at = 0
+        end
+
+        def wait
+          @lock.synchronize do
+            @signal.wait(@lock)
+          end
+        end
+
+        def received!
+          @lock.synchronize do
+            @last_received_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          end
+        end
+
+        def ping!
+          @lock.synchronize do
+            @last_ping_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          end
+        end
+
+        STATUS.each do |x|
+          define_method "#{x.to_s.downcase}?" do
+            @state == x
+          end
+
+          define_method "#{x.to_s.downcase}!" do
+            @lock.synchronize do
+              @state = x
+              @signal.broadcast
+            end
+          end
+        end
+      end
+
+      class SeqGenerator
+        def initialize
+          @mutex = Mutex.new
+          @seq = {}
+        end
+
+        # def new_request_id
+        # def new_producer_id
+        # def new_consumer_id
+        # def new_sequence_id
+        [:request_id, :producer_id, :consumer_id, :sequence_id].each do |k|
+          define_method "new_#{k}" do
+            next!(k)
+          end
+        end
+
+        def next!(key)
+          @mutex.synchronize do
+            @seq[key] ||= 0
+            @seq[key] += 1
+          end
+        end
+      end
+
+      class ConsumerHandler < ::PulsarSdk::Tweaks::WaitMap; end
+
+      class ProducerHandler < ::PulsarSdk::Tweaks::WaitMap; end
+
+      class ResponseContainer < ::PulsarSdk::Tweaks::WaitMap; end
+    end
+  end
+end