pulsar_sdk 0.8.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +51 -0
  3. data/Gemfile +6 -0
  4. data/LICENSE +201 -0
  5. data/README.md +107 -0
  6. data/Rakefile +2 -0
  7. data/bin/console +15 -0
  8. data/bin/setup +8 -0
  9. data/lib/protobuf/pulsar_api.pb.rb +710 -0
  10. data/lib/protobuf/pulsar_api.proto +934 -0
  11. data/lib/protobuf/validate.rb +41 -0
  12. data/lib/pulsar_admin.rb +14 -0
  13. data/lib/pulsar_admin/api.rb +215 -0
  14. data/lib/pulsar_sdk.rb +55 -0
  15. data/lib/pulsar_sdk/client.rb +13 -0
  16. data/lib/pulsar_sdk/client/connection.rb +371 -0
  17. data/lib/pulsar_sdk/client/connection_pool.rb +79 -0
  18. data/lib/pulsar_sdk/client/rpc.rb +67 -0
  19. data/lib/pulsar_sdk/consumer.rb +13 -0
  20. data/lib/pulsar_sdk/consumer/base.rb +148 -0
  21. data/lib/pulsar_sdk/consumer/manager.rb +127 -0
  22. data/lib/pulsar_sdk/consumer/message_tracker.rb +86 -0
  23. data/lib/pulsar_sdk/options.rb +6 -0
  24. data/lib/pulsar_sdk/options/base.rb +10 -0
  25. data/lib/pulsar_sdk/options/connection.rb +51 -0
  26. data/lib/pulsar_sdk/options/consumer.rb +34 -0
  27. data/lib/pulsar_sdk/options/producer.rb +14 -0
  28. data/lib/pulsar_sdk/options/reader.rb +7 -0
  29. data/lib/pulsar_sdk/options/tls.rb +8 -0
  30. data/lib/pulsar_sdk/producer.rb +14 -0
  31. data/lib/pulsar_sdk/producer/base.rb +154 -0
  32. data/lib/pulsar_sdk/producer/manager.rb +67 -0
  33. data/lib/pulsar_sdk/producer/message.rb +47 -0
  34. data/lib/pulsar_sdk/producer/router.rb +100 -0
  35. data/lib/pulsar_sdk/protocol.rb +8 -0
  36. data/lib/pulsar_sdk/protocol/frame.rb +53 -0
  37. data/lib/pulsar_sdk/protocol/lookup.rb +55 -0
  38. data/lib/pulsar_sdk/protocol/message.rb +55 -0
  39. data/lib/pulsar_sdk/protocol/namespace.rb +22 -0
  40. data/lib/pulsar_sdk/protocol/partitioned.rb +54 -0
  41. data/lib/pulsar_sdk/protocol/reader.rb +67 -0
  42. data/lib/pulsar_sdk/protocol/structure.rb +93 -0
  43. data/lib/pulsar_sdk/protocol/topic.rb +74 -0
  44. data/lib/pulsar_sdk/tweaks.rb +10 -0
  45. data/lib/pulsar_sdk/tweaks/assign_attributes.rb +30 -0
  46. data/lib/pulsar_sdk/tweaks/base_command.rb +66 -0
  47. data/lib/pulsar_sdk/tweaks/binary_heap.rb +133 -0
  48. data/lib/pulsar_sdk/tweaks/clean_inspect.rb +15 -0
  49. data/lib/pulsar_sdk/tweaks/time_at_microsecond.rb +27 -0
  50. data/lib/pulsar_sdk/tweaks/timeout_queue.rb +52 -0
  51. data/lib/pulsar_sdk/tweaks/wait_map.rb +81 -0
  52. data/lib/pulsar_sdk/version.rb +3 -0
  53. data/pulsar_sdk.gemspec +31 -0
  54. metadata +151 -0
@@ -0,0 +1,6 @@
1
+ require 'pulsar_sdk/options/base'
2
+ require 'pulsar_sdk/options/tls'
3
+ require 'pulsar_sdk/options/connection'
4
+ require 'pulsar_sdk/options/consumer'
5
+ require 'pulsar_sdk/options/producer'
6
+ require 'pulsar_sdk/options/reader'
@@ -0,0 +1,10 @@
1
+ require 'securerandom'
2
+
3
+ module PulsarSdk
4
+ module Options
5
+ class Base
6
+ prepend ::PulsarSdk::Tweaks::AssignAttributes
7
+ prepend ::PulsarSdk::Tweaks::CleanInspect
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,51 @@
1
+ module PulsarSdk
2
+ module Options
3
+ class Connection < Base
4
+ DEFAULT_PORT = 6650
5
+
6
+ attr_accessor :logical_addr, :physical_addr,
7
+ :connection_timeout,
8
+ :operation_timeout,
9
+ :auth_provider,
10
+ :tls_options,
11
+ :keepalive
12
+
13
+ [:logical_addr, :physical_addr].each do |x|
14
+ define_method "#{x}=" do |v|
15
+ return instance_variable_set("@#{x}", v) if v.nil?
16
+ v = v.is_a?(::URI) ? v : ::URI.parse(v)
17
+ v.port = DEFAULT_PORT if v.port.nil?
18
+ instance_variable_set("@#{x}", v)
19
+ end
20
+ end
21
+
22
+ def connecting_through_proxy?
23
+ logical_addr == physical_addr
24
+ end
25
+
26
+ def proxy_to_broker_url
27
+ connecting_through_proxy? ? logical_addr : nil
28
+ end
29
+
30
+ def port_and_host_from(name)
31
+ v = instance_variable_get("@#{name}")
32
+ return if v.nil?
33
+ [v.port, v.host]
34
+ end
35
+
36
+ private
37
+ def set_default
38
+ self.logical_addr = ENV.fetch("PULSAR_BROKER_URL", nil)
39
+
40
+ # 连接5秒超时
41
+ self.connection_timeout = 5
42
+
43
+ # Set the operation timeout (default: 30 seconds)
44
+ # Producer-create, subscribe and unsubscribe operations will be retried until this interval
45
+ self.operation_timeout = 30
46
+
47
+ self.keepalive = 30
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,34 @@
1
+ module PulsarSdk
2
+ module Options
3
+ class Consumer < Base
4
+ attr_accessor :topic, :topics, :topics_pattern,
5
+ :name, :subscription_name, :subscription_type,
6
+ :prefetch, :redelivery_delay,
7
+ :listen_wait, :replicate_subscription_state,
8
+ :read_compacted
9
+
10
+ def subscription_type
11
+ sub_type = @subscription_type.to_sym
12
+ if Pulsar::Proto::CommandSubscribe::SubType.constants.include?(sub_type)
13
+ return Pulsar::Proto::CommandSubscribe::SubType.resolve(sub_type)
14
+ end
15
+
16
+ raise "subscription_type mismatch! available is #{Pulsar::Proto::CommandSubscribe::SubType.constants}, got 「#{@subscription_type}」"
17
+ end
18
+
19
+ private
20
+ def set_default
21
+ self.name = 'ruby-consumer.' + SecureRandom.urlsafe_base64(10)
22
+ # 相同名字的subscription与订阅模式有关
23
+ self.subscription_name = 'ruby-subscription'
24
+ self.subscription_type = :Exclusive
25
+ # 延迟消息重发,默认60秒
26
+ self.redelivery_delay = 60
27
+ # 记录预取数量
28
+ self.prefetch = 1000
29
+ self.replicate_subscription_state = true
30
+ self.read_compacted = false
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,14 @@
1
+ module PulsarSdk
2
+ module Options
3
+ class Producer < Base
4
+ attr_accessor :topic, :name, :router
5
+ attr_accessor :schema # TODO
6
+
7
+ private
8
+ def set_default
9
+ self.name = 'ruby-producer.' + SecureRandom.urlsafe_base64(10)
10
+ self.router = PulsarSdk::Producer::Router.new
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,7 @@
1
+ module PulsarSdk
2
+ module Options
3
+ class Reader < Base
4
+
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,8 @@
1
+ module PulsarSdk
2
+ module Options
3
+ class Tls < Base
4
+ attr_accessor :trust_certs_file_path, :allow_insecure_connection,
5
+ :validate_hostname
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,14 @@
1
+ require 'pulsar_sdk/producer/base'
2
+ require 'pulsar_sdk/producer/message'
3
+ require 'pulsar_sdk/producer/router'
4
+ require 'pulsar_sdk/producer/manager'
5
+
6
+ module PulsarSdk
7
+ module Producer
8
+ extend self
9
+
10
+ def create(client, opts)
11
+ PulsarSdk::Producer::Manager.new(client, opts)
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,154 @@
1
+ module PulsarSdk
2
+ module Producer
3
+ class Base
4
+ prepend ::PulsarSdk::Tweaks::CleanInspect
5
+
6
+ def initialize(client, opts)
7
+ @opts = opts
8
+ @client = client
9
+ end
10
+
11
+ def grab_cnx
12
+ topic = @opts.topic
13
+ @conn = @client.connection(*@client.lookup(topic))
14
+ @established = true
15
+
16
+ @seq_generator = SeqGenerator.new(@conn.seq_generator)
17
+ @producer_id = @seq_generator.new_producer_id
18
+
19
+ @producer_name = [@opts.name, @producer_id].join('.')
20
+
21
+ @receipt_queue = ReceiptQueue.new
22
+
23
+ @stoped = false
24
+
25
+ @producer_name = init_producer(topic)
26
+ end
27
+
28
+ def execute(cmd, msg = nil, timeout = nil)
29
+ write(cmd, msg, false, timeout)
30
+ end
31
+
32
+ def execute_async(cmd, msg = nil)
33
+ write(cmd, msg, true)
34
+ end
35
+
36
+ # 获取发送回执
37
+ # TODO get receipt by sequence_id
38
+ def receipt
39
+ receipt_ = @receipt_queue.pop.first
40
+ return if receipt_.nil?
41
+
42
+ if block_given?
43
+ yield receipt_
44
+ end
45
+
46
+ receipt_
47
+ end
48
+
49
+ def close
50
+ return if @stoped
51
+
52
+ base_cmd = Pulsar::Proto::BaseCommand.new(
53
+ type: Pulsar::Proto::BaseCommand::Type::CLOSE_PRODUCER,
54
+ close_producer: Pulsar::Proto::CommandCloseProducer.new
55
+ )
56
+ execute(base_cmd) unless disconnect?
57
+
58
+ unbind_handler!
59
+
60
+ @stoped = true
61
+
62
+ @receipt_queue.close
63
+ end
64
+
65
+ def disconnect?
66
+ !@established
67
+ end
68
+
69
+ private
70
+ def init_producer(topic)
71
+ bind_handler!
72
+
73
+ base_cmd = Pulsar::Proto::BaseCommand.new(
74
+ type: Pulsar::Proto::BaseCommand::Type::PRODUCER,
75
+ producer: Pulsar::Proto::CommandProducer.new(
76
+ topic: topic
77
+ )
78
+ )
79
+ result = execute(base_cmd)
80
+ result.producer_success.producer_name
81
+ end
82
+
83
+ def write(cmd, msg, *args)
84
+ unless msg.nil? || msg.is_a?(PulsarSdk::Producer::Message)
85
+ raise "msg expected a PulsarSdk::Producer::Message got #{msg.class}"
86
+ end
87
+
88
+ grab_cnx if disconnect?
89
+
90
+ cmd.seq_generator = @seq_generator
91
+
92
+ unless msg.nil?
93
+ msg.producer_name = @producer_name
94
+ msg.sequence_id = @seq_generator.new_sequence_id
95
+ end
96
+
97
+ result = @conn.request(set_seq_generator(cmd), filling_message(msg), *args)
98
+
99
+ # increase sequence_id when success
100
+ @seq_generator.new_sequence_id(false) unless msg.nil?
101
+
102
+ result
103
+ end
104
+
105
+ def set_seq_generator(cmd)
106
+ cmd.seq_generator = @seq_generator
107
+ cmd
108
+ end
109
+
110
+ def filling_message(msg)
111
+ return if msg.nil?
112
+ msg.producer_name = @producer_name
113
+ msg
114
+ end
115
+
116
+ def bind_handler!
117
+ handler = Proc.new do |send_receipt|
118
+ send_receipt.nil? ? (@established = false) : @receipt_queue.add(send_receipt)
119
+ end
120
+ @conn.producer_handlers.add(@producer_id, handler)
121
+ end
122
+
123
+ def unbind_handler!
124
+ @conn.producer_handlers.delete(@producer_id)
125
+
126
+ true
127
+ end
128
+ end
129
+
130
+ class ReceiptQueue < ::PulsarSdk::Tweaks::TimeoutQueue; end
131
+
132
+ # NOTE keep producer_id and sequence_id static
133
+ class SeqGenerator
134
+ def initialize(seq_g)
135
+ @seq_g = seq_g
136
+ @producer_id = @seq_g.new_producer_id
137
+ @sequence_id = @seq_g.new_sequence_id
138
+ end
139
+
140
+ def new_producer_id
141
+ @producer_id
142
+ end
143
+
144
+ def new_sequence_id(cache = true)
145
+ return @sequence_id if cache
146
+ @sequence_id = @seq_g.new_sequence_id
147
+ end
148
+
149
+ def method_missing(method)
150
+ @seq_g.public_send(method)
151
+ end
152
+ end
153
+ end
154
+ end
@@ -0,0 +1,67 @@
1
+ module PulsarSdk
2
+ module Producer
3
+ class Manager
4
+ prepend ::PulsarSdk::Tweaks::CleanInspect
5
+
6
+ def initialize(client, opts)
7
+ @topic = opts.topic
8
+ @producers = init_producer_by(client, opts)
9
+ @router = opts.router
10
+ end
11
+
12
+ def execute(cmd, msg = nil, timeout = nil)
13
+ raise "cmd expected a Pulsar::Proto::BaseCommand got #{cmd.class}" unless cmd.is_a?(Pulsar::Proto::BaseCommand)
14
+ real_producer(msg) do |producer|
15
+ producer.execute(cmd, msg, timeout)
16
+ end
17
+ end
18
+
19
+ def execute_async(cmd, msg = nil)
20
+ raise "cmd expected a Pulsar::Proto::BaseCommand got #{cmd.class}" unless cmd.is_a?(Pulsar::Proto::BaseCommand)
21
+ real_producer(msg) do |producer|
22
+ producer.execute_async(cmd, msg)
23
+ end
24
+ end
25
+
26
+ def real_producer(msg, &block)
27
+ if @producers.size.zero?
28
+ PulsarSdk.logger.warn(__method__){"There is no available producer for topic: 「#{@topic}」, skipping action!"}
29
+ return
30
+ end
31
+
32
+ ensure_connection
33
+
34
+ route_index = msg.nil? ? 0 : @router.route(msg.key, @producers.size)
35
+
36
+ yield @producers[route_index]
37
+ end
38
+
39
+ def close
40
+ @producers.each(&:close)
41
+ end
42
+
43
+ private
44
+ def init_producer_by(client, opts)
45
+ opts = opts.dup
46
+
47
+ topics = client.partition_topics(@topic)
48
+ topics.map do |topic|
49
+ opts.topic = topic
50
+ PulsarSdk::Producer::Base.new(client, opts).tap do |base|
51
+ base.grab_cnx
52
+ end
53
+ end
54
+ end
55
+
56
+ def ensure_connection
57
+ @producers.each do |producer|
58
+ next unless producer.disconnect?
59
+ PulsarSdk.logger.warn('PulsarSdk::Producer::Manager#ensure_connection'){
60
+ "connection closed! reconnect now! #{producer.inspect}"
61
+ }
62
+ producer.grab_cnx
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,47 @@
1
+ module PulsarSdk
2
+ module Producer
3
+ class Message
4
+ prepend ::PulsarSdk::Tweaks::CleanInspect
5
+
6
+ attr_reader :metadata, :message, :key
7
+
8
+ def initialize(msg, metadata = nil, key = nil)
9
+ # TODO check metadata type
10
+ @message, @metadata = msg, metadata
11
+ @metadata ||= Pulsar::Proto::MessageMetadata.new
12
+
13
+ # msg must convet to string
14
+ json_encode! unless @message.is_a?(String)
15
+
16
+ publish_time = @metadata.publish_time
17
+ @metadata.publish_time = publish_time.zero? ? TimeX.now.timestamp : publish_time
18
+
19
+ self.key = key
20
+ end
21
+
22
+ def producer_name=(v)
23
+ @metadata.producer_name = v
24
+ end
25
+
26
+ def sequence_id=(v)
27
+ @metadata.sequence_id = v
28
+ end
29
+
30
+ def binary_string
31
+ @message.bytes.pack('C*')
32
+ end
33
+
34
+ def key=(v, b64 = false)
35
+ @metadata.partition_key = v.to_s
36
+ @metadata.partition_key_b64_encoded = b64
37
+ end
38
+
39
+ private
40
+ def json_encode!
41
+ PulsarSdk.logger.info("#{self.class}::#{__method__}"){"message was 「#{@message.class}」 now encode to json!"}
42
+ @message = @message.respond_to?(:to_json) ? @message.to_json : JSON.dump(@message)
43
+ @metadata.properties << Pulsar::Proto::KeyValue.new(key: 'Content-Type', value: 'application/json; charset=utf-8')
44
+ end
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,100 @@
1
+ module PulsarSdk
2
+ module Producer
3
+ class Router
4
+ prepend ::PulsarSdk::Tweaks::CleanInspect
5
+
6
+ def initialize(scheme = :string_hash)
7
+ case scheme.to_sym
8
+ when :string_hash
9
+ @handler = string_hash
10
+ when :murmur_hash
11
+ @handler = murmur3_hash
12
+ else
13
+ raise "Unknown hash scheme #{scheme}"
14
+ end
15
+ end
16
+
17
+ def route(key, total, delay = 0)
18
+ return 0 if total <= 1
19
+
20
+ return (@handler.call(key) % total) unless key.to_s.empty?
21
+
22
+ Murmur3.int32_hash(TimeX.now.timestamp) % total
23
+ end
24
+
25
+ # 将hash值限制在32位内,防止key过长导致过多内存占用
26
+ def string_hash
27
+ max_mod = 1 << 32
28
+ Proc.new do |key|
29
+ h = 0
30
+ key.to_s.each_byte do |x|
31
+ h = (31*h)%max_mod + x
32
+ end
33
+
34
+ h
35
+ end
36
+ end
37
+
38
+ def murmur3_hash
39
+ Proc.new do |key, seed = 31|
40
+ # Using 0x7fffffff was maintain compatibility with values used in Java client
41
+ Murmur3.hash(key, seed) & 0x7fffffff
42
+ end
43
+ end
44
+ end
45
+
46
+ # cpoy from https://github.com/funny-falcon/murmurhash3-ruby
47
+ module Murmur3
48
+ extend self
49
+
50
+ MASK32 = 0xffffffff
51
+
52
+ def int32_hash(i, seed=0)
53
+ hash([i].pack("V"), seed)
54
+ end
55
+
56
+ def hash(str, seed=0)
57
+ h1 = seed
58
+ numbers = str.unpack('V*C*')
59
+ tailn = str.bytesize % 4
60
+ tail = numbers.slice!(numbers.size - tailn, tailn)
61
+ for k1 in numbers
62
+ h1 ^= mmix(k1)
63
+ h1 = rotl(h1, 13)
64
+ h1 = (h1*5 + 0xe6546b64) & MASK32
65
+ end
66
+
67
+ unless tail.empty?
68
+ k1 = 0
69
+ tail.reverse_each do |c1|
70
+ k1 = (k1 << 8) | c1
71
+ end
72
+ h1 ^= mmix(k1)
73
+ end
74
+
75
+ h1 ^= str.bytesize
76
+ fmix(h1)
77
+ end
78
+
79
+ private
80
+ def rotl(x, r)
81
+ ((x << r) | (x >> (32 - r))) & MASK32
82
+ end
83
+
84
+ def fmix(h)
85
+ h &= MASK32
86
+ h ^= h >> 16
87
+ h = (h * 0x85ebca6b) & MASK32
88
+ h ^= h >> 13
89
+ h = (h * 0xc2b2ae35) & MASK32
90
+ h ^ (h >> 16)
91
+ end
92
+
93
+ def mmix(k1)
94
+ k1 = (k1 * 0xcc9e2d51) & MASK32
95
+ k1 = rotl(k1, 15)
96
+ (k1 * 0x1b873593) & MASK32
97
+ end
98
+ end
99
+ end
100
+ end