pulsar_sdk 0.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +51 -0
- data/Gemfile +6 -0
- data/LICENSE +201 -0
- data/README.md +107 -0
- data/Rakefile +2 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/lib/protobuf/pulsar_api.pb.rb +710 -0
- data/lib/protobuf/pulsar_api.proto +934 -0
- data/lib/protobuf/validate.rb +41 -0
- data/lib/pulsar_admin.rb +14 -0
- data/lib/pulsar_admin/api.rb +215 -0
- data/lib/pulsar_sdk.rb +55 -0
- data/lib/pulsar_sdk/client.rb +13 -0
- data/lib/pulsar_sdk/client/connection.rb +371 -0
- data/lib/pulsar_sdk/client/connection_pool.rb +79 -0
- data/lib/pulsar_sdk/client/rpc.rb +67 -0
- data/lib/pulsar_sdk/consumer.rb +13 -0
- data/lib/pulsar_sdk/consumer/base.rb +148 -0
- data/lib/pulsar_sdk/consumer/manager.rb +127 -0
- data/lib/pulsar_sdk/consumer/message_tracker.rb +86 -0
- data/lib/pulsar_sdk/options.rb +6 -0
- data/lib/pulsar_sdk/options/base.rb +10 -0
- data/lib/pulsar_sdk/options/connection.rb +51 -0
- data/lib/pulsar_sdk/options/consumer.rb +34 -0
- data/lib/pulsar_sdk/options/producer.rb +14 -0
- data/lib/pulsar_sdk/options/reader.rb +7 -0
- data/lib/pulsar_sdk/options/tls.rb +8 -0
- data/lib/pulsar_sdk/producer.rb +14 -0
- data/lib/pulsar_sdk/producer/base.rb +154 -0
- data/lib/pulsar_sdk/producer/manager.rb +67 -0
- data/lib/pulsar_sdk/producer/message.rb +47 -0
- data/lib/pulsar_sdk/producer/router.rb +100 -0
- data/lib/pulsar_sdk/protocol.rb +8 -0
- data/lib/pulsar_sdk/protocol/frame.rb +53 -0
- data/lib/pulsar_sdk/protocol/lookup.rb +55 -0
- data/lib/pulsar_sdk/protocol/message.rb +55 -0
- data/lib/pulsar_sdk/protocol/namespace.rb +22 -0
- data/lib/pulsar_sdk/protocol/partitioned.rb +54 -0
- data/lib/pulsar_sdk/protocol/reader.rb +67 -0
- data/lib/pulsar_sdk/protocol/structure.rb +93 -0
- data/lib/pulsar_sdk/protocol/topic.rb +74 -0
- data/lib/pulsar_sdk/tweaks.rb +10 -0
- data/lib/pulsar_sdk/tweaks/assign_attributes.rb +30 -0
- data/lib/pulsar_sdk/tweaks/base_command.rb +66 -0
- data/lib/pulsar_sdk/tweaks/binary_heap.rb +133 -0
- data/lib/pulsar_sdk/tweaks/clean_inspect.rb +15 -0
- data/lib/pulsar_sdk/tweaks/time_at_microsecond.rb +27 -0
- data/lib/pulsar_sdk/tweaks/timeout_queue.rb +52 -0
- data/lib/pulsar_sdk/tweaks/wait_map.rb +81 -0
- data/lib/pulsar_sdk/version.rb +3 -0
- data/pulsar_sdk.gemspec +31 -0
- metadata +151 -0
@@ -0,0 +1,51 @@
|
|
1
|
+
module PulsarSdk
|
2
|
+
module Options
|
3
|
+
class Connection < Base
|
4
|
+
DEFAULT_PORT = 6650
|
5
|
+
|
6
|
+
attr_accessor :logical_addr, :physical_addr,
|
7
|
+
:connection_timeout,
|
8
|
+
:operation_timeout,
|
9
|
+
:auth_provider,
|
10
|
+
:tls_options,
|
11
|
+
:keepalive
|
12
|
+
|
13
|
+
[:logical_addr, :physical_addr].each do |x|
|
14
|
+
define_method "#{x}=" do |v|
|
15
|
+
return instance_variable_set("@#{x}", v) if v.nil?
|
16
|
+
v = v.is_a?(::URI) ? v : ::URI.parse(v)
|
17
|
+
v.port = DEFAULT_PORT if v.port.nil?
|
18
|
+
instance_variable_set("@#{x}", v)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def connecting_through_proxy?
|
23
|
+
logical_addr == physical_addr
|
24
|
+
end
|
25
|
+
|
26
|
+
def proxy_to_broker_url
|
27
|
+
connecting_through_proxy? ? logical_addr : nil
|
28
|
+
end
|
29
|
+
|
30
|
+
def port_and_host_from(name)
|
31
|
+
v = instance_variable_get("@#{name}")
|
32
|
+
return if v.nil?
|
33
|
+
[v.port, v.host]
|
34
|
+
end
|
35
|
+
|
36
|
+
private
|
37
|
+
def set_default
|
38
|
+
self.logical_addr = ENV.fetch("PULSAR_BROKER_URL", nil)
|
39
|
+
|
40
|
+
# 连接5秒超时
|
41
|
+
self.connection_timeout = 5
|
42
|
+
|
43
|
+
# Set the operation timeout (default: 30 seconds)
|
44
|
+
# Producer-create, subscribe and unsubscribe operations will be retried until this interval
|
45
|
+
self.operation_timeout = 30
|
46
|
+
|
47
|
+
self.keepalive = 30
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module PulsarSdk
|
2
|
+
module Options
|
3
|
+
class Consumer < Base
|
4
|
+
attr_accessor :topic, :topics, :topics_pattern,
|
5
|
+
:name, :subscription_name, :subscription_type,
|
6
|
+
:prefetch, :redelivery_delay,
|
7
|
+
:listen_wait, :replicate_subscription_state,
|
8
|
+
:read_compacted
|
9
|
+
|
10
|
+
def subscription_type
|
11
|
+
sub_type = @subscription_type.to_sym
|
12
|
+
if Pulsar::Proto::CommandSubscribe::SubType.constants.include?(sub_type)
|
13
|
+
return Pulsar::Proto::CommandSubscribe::SubType.resolve(sub_type)
|
14
|
+
end
|
15
|
+
|
16
|
+
raise "subscription_type mismatch! available is #{Pulsar::Proto::CommandSubscribe::SubType.constants}, got 「#{@subscription_type}」"
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
def set_default
|
21
|
+
self.name = 'ruby-consumer.' + SecureRandom.urlsafe_base64(10)
|
22
|
+
# 相同名字的subscription与订阅模式有关
|
23
|
+
self.subscription_name = 'ruby-subscription'
|
24
|
+
self.subscription_type = :Exclusive
|
25
|
+
# 延迟消息重发,默认60秒
|
26
|
+
self.redelivery_delay = 60
|
27
|
+
# 记录预取数量
|
28
|
+
self.prefetch = 1000
|
29
|
+
self.replicate_subscription_state = true
|
30
|
+
self.read_compacted = false
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module PulsarSdk
|
2
|
+
module Options
|
3
|
+
class Producer < Base
|
4
|
+
attr_accessor :topic, :name, :router
|
5
|
+
attr_accessor :schema # TODO
|
6
|
+
|
7
|
+
private
|
8
|
+
def set_default
|
9
|
+
self.name = 'ruby-producer.' + SecureRandom.urlsafe_base64(10)
|
10
|
+
self.router = PulsarSdk::Producer::Router.new
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'pulsar_sdk/producer/base'
|
2
|
+
require 'pulsar_sdk/producer/message'
|
3
|
+
require 'pulsar_sdk/producer/router'
|
4
|
+
require 'pulsar_sdk/producer/manager'
|
5
|
+
|
6
|
+
module PulsarSdk
|
7
|
+
module Producer
|
8
|
+
extend self
|
9
|
+
|
10
|
+
def create(client, opts)
|
11
|
+
PulsarSdk::Producer::Manager.new(client, opts)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,154 @@
|
|
1
|
+
module PulsarSdk
|
2
|
+
module Producer
|
3
|
+
class Base
|
4
|
+
prepend ::PulsarSdk::Tweaks::CleanInspect
|
5
|
+
|
6
|
+
def initialize(client, opts)
|
7
|
+
@opts = opts
|
8
|
+
@client = client
|
9
|
+
end
|
10
|
+
|
11
|
+
def grab_cnx
|
12
|
+
topic = @opts.topic
|
13
|
+
@conn = @client.connection(*@client.lookup(topic))
|
14
|
+
@established = true
|
15
|
+
|
16
|
+
@seq_generator = SeqGenerator.new(@conn.seq_generator)
|
17
|
+
@producer_id = @seq_generator.new_producer_id
|
18
|
+
|
19
|
+
@producer_name = [@opts.name, @producer_id].join('.')
|
20
|
+
|
21
|
+
@receipt_queue = ReceiptQueue.new
|
22
|
+
|
23
|
+
@stoped = false
|
24
|
+
|
25
|
+
@producer_name = init_producer(topic)
|
26
|
+
end
|
27
|
+
|
28
|
+
def execute(cmd, msg = nil, timeout = nil)
|
29
|
+
write(cmd, msg, false, timeout)
|
30
|
+
end
|
31
|
+
|
32
|
+
def execute_async(cmd, msg = nil)
|
33
|
+
write(cmd, msg, true)
|
34
|
+
end
|
35
|
+
|
36
|
+
# 获取发送回执
|
37
|
+
# TODO get receipt by sequence_id
|
38
|
+
def receipt
|
39
|
+
receipt_ = @receipt_queue.pop.first
|
40
|
+
return if receipt_.nil?
|
41
|
+
|
42
|
+
if block_given?
|
43
|
+
yield receipt_
|
44
|
+
end
|
45
|
+
|
46
|
+
receipt_
|
47
|
+
end
|
48
|
+
|
49
|
+
def close
|
50
|
+
return if @stoped
|
51
|
+
|
52
|
+
base_cmd = Pulsar::Proto::BaseCommand.new(
|
53
|
+
type: Pulsar::Proto::BaseCommand::Type::CLOSE_PRODUCER,
|
54
|
+
close_producer: Pulsar::Proto::CommandCloseProducer.new
|
55
|
+
)
|
56
|
+
execute(base_cmd) unless disconnect?
|
57
|
+
|
58
|
+
unbind_handler!
|
59
|
+
|
60
|
+
@stoped = true
|
61
|
+
|
62
|
+
@receipt_queue.close
|
63
|
+
end
|
64
|
+
|
65
|
+
def disconnect?
|
66
|
+
!@established
|
67
|
+
end
|
68
|
+
|
69
|
+
private
|
70
|
+
def init_producer(topic)
|
71
|
+
bind_handler!
|
72
|
+
|
73
|
+
base_cmd = Pulsar::Proto::BaseCommand.new(
|
74
|
+
type: Pulsar::Proto::BaseCommand::Type::PRODUCER,
|
75
|
+
producer: Pulsar::Proto::CommandProducer.new(
|
76
|
+
topic: topic
|
77
|
+
)
|
78
|
+
)
|
79
|
+
result = execute(base_cmd)
|
80
|
+
result.producer_success.producer_name
|
81
|
+
end
|
82
|
+
|
83
|
+
def write(cmd, msg, *args)
|
84
|
+
unless msg.nil? || msg.is_a?(PulsarSdk::Producer::Message)
|
85
|
+
raise "msg expected a PulsarSdk::Producer::Message got #{msg.class}"
|
86
|
+
end
|
87
|
+
|
88
|
+
grab_cnx if disconnect?
|
89
|
+
|
90
|
+
cmd.seq_generator = @seq_generator
|
91
|
+
|
92
|
+
unless msg.nil?
|
93
|
+
msg.producer_name = @producer_name
|
94
|
+
msg.sequence_id = @seq_generator.new_sequence_id
|
95
|
+
end
|
96
|
+
|
97
|
+
result = @conn.request(set_seq_generator(cmd), filling_message(msg), *args)
|
98
|
+
|
99
|
+
# increase sequence_id when success
|
100
|
+
@seq_generator.new_sequence_id(false) unless msg.nil?
|
101
|
+
|
102
|
+
result
|
103
|
+
end
|
104
|
+
|
105
|
+
def set_seq_generator(cmd)
|
106
|
+
cmd.seq_generator = @seq_generator
|
107
|
+
cmd
|
108
|
+
end
|
109
|
+
|
110
|
+
def filling_message(msg)
|
111
|
+
return if msg.nil?
|
112
|
+
msg.producer_name = @producer_name
|
113
|
+
msg
|
114
|
+
end
|
115
|
+
|
116
|
+
def bind_handler!
|
117
|
+
handler = Proc.new do |send_receipt|
|
118
|
+
send_receipt.nil? ? (@established = false) : @receipt_queue.add(send_receipt)
|
119
|
+
end
|
120
|
+
@conn.producer_handlers.add(@producer_id, handler)
|
121
|
+
end
|
122
|
+
|
123
|
+
def unbind_handler!
|
124
|
+
@conn.producer_handlers.delete(@producer_id)
|
125
|
+
|
126
|
+
true
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
class ReceiptQueue < ::PulsarSdk::Tweaks::TimeoutQueue; end
|
131
|
+
|
132
|
+
# NOTE keep producer_id and sequence_id static
|
133
|
+
class SeqGenerator
|
134
|
+
def initialize(seq_g)
|
135
|
+
@seq_g = seq_g
|
136
|
+
@producer_id = @seq_g.new_producer_id
|
137
|
+
@sequence_id = @seq_g.new_sequence_id
|
138
|
+
end
|
139
|
+
|
140
|
+
def new_producer_id
|
141
|
+
@producer_id
|
142
|
+
end
|
143
|
+
|
144
|
+
def new_sequence_id(cache = true)
|
145
|
+
return @sequence_id if cache
|
146
|
+
@sequence_id = @seq_g.new_sequence_id
|
147
|
+
end
|
148
|
+
|
149
|
+
def method_missing(method)
|
150
|
+
@seq_g.public_send(method)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module PulsarSdk
|
2
|
+
module Producer
|
3
|
+
class Manager
|
4
|
+
prepend ::PulsarSdk::Tweaks::CleanInspect
|
5
|
+
|
6
|
+
def initialize(client, opts)
|
7
|
+
@topic = opts.topic
|
8
|
+
@producers = init_producer_by(client, opts)
|
9
|
+
@router = opts.router
|
10
|
+
end
|
11
|
+
|
12
|
+
def execute(cmd, msg = nil, timeout = nil)
|
13
|
+
raise "cmd expected a Pulsar::Proto::BaseCommand got #{cmd.class}" unless cmd.is_a?(Pulsar::Proto::BaseCommand)
|
14
|
+
real_producer(msg) do |producer|
|
15
|
+
producer.execute(cmd, msg, timeout)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def execute_async(cmd, msg = nil)
|
20
|
+
raise "cmd expected a Pulsar::Proto::BaseCommand got #{cmd.class}" unless cmd.is_a?(Pulsar::Proto::BaseCommand)
|
21
|
+
real_producer(msg) do |producer|
|
22
|
+
producer.execute_async(cmd, msg)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def real_producer(msg, &block)
|
27
|
+
if @producers.size.zero?
|
28
|
+
PulsarSdk.logger.warn(__method__){"There is no available producer for topic: 「#{@topic}」, skipping action!"}
|
29
|
+
return
|
30
|
+
end
|
31
|
+
|
32
|
+
ensure_connection
|
33
|
+
|
34
|
+
route_index = msg.nil? ? 0 : @router.route(msg.key, @producers.size)
|
35
|
+
|
36
|
+
yield @producers[route_index]
|
37
|
+
end
|
38
|
+
|
39
|
+
def close
|
40
|
+
@producers.each(&:close)
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
def init_producer_by(client, opts)
|
45
|
+
opts = opts.dup
|
46
|
+
|
47
|
+
topics = client.partition_topics(@topic)
|
48
|
+
topics.map do |topic|
|
49
|
+
opts.topic = topic
|
50
|
+
PulsarSdk::Producer::Base.new(client, opts).tap do |base|
|
51
|
+
base.grab_cnx
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def ensure_connection
|
57
|
+
@producers.each do |producer|
|
58
|
+
next unless producer.disconnect?
|
59
|
+
PulsarSdk.logger.warn('PulsarSdk::Producer::Manager#ensure_connection'){
|
60
|
+
"connection closed! reconnect now! #{producer.inspect}"
|
61
|
+
}
|
62
|
+
producer.grab_cnx
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module PulsarSdk
|
2
|
+
module Producer
|
3
|
+
class Message
|
4
|
+
prepend ::PulsarSdk::Tweaks::CleanInspect
|
5
|
+
|
6
|
+
attr_reader :metadata, :message, :key
|
7
|
+
|
8
|
+
def initialize(msg, metadata = nil, key = nil)
|
9
|
+
# TODO check metadata type
|
10
|
+
@message, @metadata = msg, metadata
|
11
|
+
@metadata ||= Pulsar::Proto::MessageMetadata.new
|
12
|
+
|
13
|
+
# msg must convet to string
|
14
|
+
json_encode! unless @message.is_a?(String)
|
15
|
+
|
16
|
+
publish_time = @metadata.publish_time
|
17
|
+
@metadata.publish_time = publish_time.zero? ? TimeX.now.timestamp : publish_time
|
18
|
+
|
19
|
+
self.key = key
|
20
|
+
end
|
21
|
+
|
22
|
+
def producer_name=(v)
|
23
|
+
@metadata.producer_name = v
|
24
|
+
end
|
25
|
+
|
26
|
+
def sequence_id=(v)
|
27
|
+
@metadata.sequence_id = v
|
28
|
+
end
|
29
|
+
|
30
|
+
def binary_string
|
31
|
+
@message.bytes.pack('C*')
|
32
|
+
end
|
33
|
+
|
34
|
+
def key=(v, b64 = false)
|
35
|
+
@metadata.partition_key = v.to_s
|
36
|
+
@metadata.partition_key_b64_encoded = b64
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
def json_encode!
|
41
|
+
PulsarSdk.logger.info("#{self.class}::#{__method__}"){"message was 「#{@message.class}」 now encode to json!"}
|
42
|
+
@message = @message.respond_to?(:to_json) ? @message.to_json : JSON.dump(@message)
|
43
|
+
@metadata.properties << Pulsar::Proto::KeyValue.new(key: 'Content-Type', value: 'application/json; charset=utf-8')
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
module PulsarSdk
|
2
|
+
module Producer
|
3
|
+
class Router
|
4
|
+
prepend ::PulsarSdk::Tweaks::CleanInspect
|
5
|
+
|
6
|
+
def initialize(scheme = :string_hash)
|
7
|
+
case scheme.to_sym
|
8
|
+
when :string_hash
|
9
|
+
@handler = string_hash
|
10
|
+
when :murmur_hash
|
11
|
+
@handler = murmur3_hash
|
12
|
+
else
|
13
|
+
raise "Unknown hash scheme #{scheme}"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def route(key, total, delay = 0)
|
18
|
+
return 0 if total <= 1
|
19
|
+
|
20
|
+
return (@handler.call(key) % total) unless key.to_s.empty?
|
21
|
+
|
22
|
+
Murmur3.int32_hash(TimeX.now.timestamp) % total
|
23
|
+
end
|
24
|
+
|
25
|
+
# 将hash值限制在32位内,防止key过长导致过多内存占用
|
26
|
+
def string_hash
|
27
|
+
max_mod = 1 << 32
|
28
|
+
Proc.new do |key|
|
29
|
+
h = 0
|
30
|
+
key.to_s.each_byte do |x|
|
31
|
+
h = (31*h)%max_mod + x
|
32
|
+
end
|
33
|
+
|
34
|
+
h
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def murmur3_hash
|
39
|
+
Proc.new do |key, seed = 31|
|
40
|
+
# Using 0x7fffffff was maintain compatibility with values used in Java client
|
41
|
+
Murmur3.hash(key, seed) & 0x7fffffff
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# cpoy from https://github.com/funny-falcon/murmurhash3-ruby
|
47
|
+
module Murmur3
|
48
|
+
extend self
|
49
|
+
|
50
|
+
MASK32 = 0xffffffff
|
51
|
+
|
52
|
+
def int32_hash(i, seed=0)
|
53
|
+
hash([i].pack("V"), seed)
|
54
|
+
end
|
55
|
+
|
56
|
+
def hash(str, seed=0)
|
57
|
+
h1 = seed
|
58
|
+
numbers = str.unpack('V*C*')
|
59
|
+
tailn = str.bytesize % 4
|
60
|
+
tail = numbers.slice!(numbers.size - tailn, tailn)
|
61
|
+
for k1 in numbers
|
62
|
+
h1 ^= mmix(k1)
|
63
|
+
h1 = rotl(h1, 13)
|
64
|
+
h1 = (h1*5 + 0xe6546b64) & MASK32
|
65
|
+
end
|
66
|
+
|
67
|
+
unless tail.empty?
|
68
|
+
k1 = 0
|
69
|
+
tail.reverse_each do |c1|
|
70
|
+
k1 = (k1 << 8) | c1
|
71
|
+
end
|
72
|
+
h1 ^= mmix(k1)
|
73
|
+
end
|
74
|
+
|
75
|
+
h1 ^= str.bytesize
|
76
|
+
fmix(h1)
|
77
|
+
end
|
78
|
+
|
79
|
+
private
|
80
|
+
def rotl(x, r)
|
81
|
+
((x << r) | (x >> (32 - r))) & MASK32
|
82
|
+
end
|
83
|
+
|
84
|
+
def fmix(h)
|
85
|
+
h &= MASK32
|
86
|
+
h ^= h >> 16
|
87
|
+
h = (h * 0x85ebca6b) & MASK32
|
88
|
+
h ^= h >> 13
|
89
|
+
h = (h * 0xc2b2ae35) & MASK32
|
90
|
+
h ^ (h >> 16)
|
91
|
+
end
|
92
|
+
|
93
|
+
def mmix(k1)
|
94
|
+
k1 = (k1 * 0xcc9e2d51) & MASK32
|
95
|
+
k1 = rotl(k1, 15)
|
96
|
+
(k1 * 0x1b873593) & MASK32
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|