istox 0.1.126 → 0.1.127
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.idea/istox-gem.iml +1691 -2
- data/Gemfile.lock +3 -3
- data/lib/istox/helpers/bunny_boot.rb +24 -26
- data/lib/istox/helpers/publisher.rb +37 -21
- data/lib/istox/helpers/redis.rb +8 -0
- data/lib/istox/helpers/subscriber.rb +5 -3
- data/lib/istox/version.rb +1 -1
- metadata +3 -3
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
istox (0.1.
|
4
|
+
istox (0.1.126)
|
5
5
|
awesome_print
|
6
6
|
binding_of_caller
|
7
7
|
bunny (>= 2.12.0)
|
@@ -97,7 +97,7 @@ GEM
|
|
97
97
|
ffi (1.12.2)
|
98
98
|
globalid (0.4.2)
|
99
99
|
activesupport (>= 4.2.0)
|
100
|
-
google-protobuf (3.11.4)
|
100
|
+
google-protobuf (3.11.4-universal-darwin)
|
101
101
|
googleapis-common-protos-types (1.0.4)
|
102
102
|
google-protobuf (~> 3.0)
|
103
103
|
graphlient (0.3.7)
|
@@ -108,7 +108,7 @@ GEM
|
|
108
108
|
graphql-client (0.16.0)
|
109
109
|
activesupport (>= 3.0)
|
110
110
|
graphql (~> 1.8)
|
111
|
-
grpc (1.27.0)
|
111
|
+
grpc (1.27.0-universal-darwin)
|
112
112
|
google-protobuf (~> 3.11)
|
113
113
|
googleapis-common-protos-types (~> 1.0)
|
114
114
|
grpc-tools (1.27.0)
|
@@ -24,8 +24,8 @@ module Istox
|
|
24
24
|
# Create virtual channel on established connection
|
25
25
|
# Configure pool_size, prefetch, confirm mode according to opts
|
26
26
|
def channel(conn, opts = {})
|
27
|
-
ch = conn.create_channel(nil, opts[
|
28
|
-
ch.prefetch(opts[
|
27
|
+
ch = conn.create_channel(nil, opts[:pool_size] || 1)
|
28
|
+
ch.prefetch(opts[:prefetch]) unless opts[:prefetch].nil?
|
29
29
|
|
30
30
|
# Put channel in confirmation mode
|
31
31
|
ch.confirm_select if opts[:confirm]
|
@@ -218,43 +218,37 @@ module Istox
|
|
218
218
|
end
|
219
219
|
end
|
220
220
|
|
221
|
-
def create_tracker(channel_id, delivery_tag, eid,
|
222
|
-
log.debug "Create track: channel_id: #{channel_id}, delivery_tag: #{delivery_tag}
|
223
|
-
# combination of channel_id
|
221
|
+
def create_tracker(channel_id, delivery_tag, eid, options, payload)
|
222
|
+
log.debug "Create track: channel_id: #{channel_id}, delivery_tag: #{delivery_tag}"
|
223
|
+
# combination of channel_id:delivery_tag can uniquely identify a msg
|
224
|
+
# For each retry of msg, channel_id and delivery_tag is unchanged
|
225
|
+
# But each retry, there is new delivery_tag that should be updated
|
224
226
|
id = "#{channel_id.to_s}:#{delivery_tag.to_s}"
|
225
227
|
|
226
|
-
::Istox::RedisBoot.sets("#{id}:messageid", message_id)
|
227
|
-
::Istox::RedisBoot.sets("#{id}:retry", 0)
|
228
228
|
::Istox::RedisBoot.sets("#{id}:payload", payload.to_s)
|
229
229
|
::Istox::RedisBoot.sets("#{id}:eid", eid.to_s)
|
230
230
|
::Istox::RedisBoot.sets("#{id}:options", options.to_s)
|
231
231
|
end
|
232
232
|
|
233
|
-
def find_tracker_on_channel(channel_id, delivery_tag)
|
234
|
-
|
235
|
-
keys =
|
236
|
-
keys.
|
237
|
-
# payload =
|
238
|
-
end
|
233
|
+
def find_tracker_on_channel(channel_id, delivery_tag, key)
|
234
|
+
pattern = "#{channel_id.to_s}:#{delivery_tag.to_s}:#{key}"
|
235
|
+
keys = find_trackers pattern
|
236
|
+
get_tracker(keys.first)
|
239
237
|
end
|
240
238
|
|
241
239
|
def del_tracker_on_channel(channel_id)
|
242
|
-
trackers =
|
240
|
+
trackers = find_trackers "#{channel_id}*"
|
243
241
|
trackers.each do |k|
|
244
242
|
::Istox::RedisBoot.del k
|
245
243
|
end
|
246
244
|
end
|
247
245
|
|
248
|
-
def
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
::Istox::RedisBoot.
|
254
|
-
end
|
255
|
-
|
256
|
-
def update_tracker_incr_retry(key)
|
257
|
-
::Istox::RedisBoot.incr(key)
|
246
|
+
def rename_tracker(channel_id, old_delivery_tag, new_delivery_tag)
|
247
|
+
old_id = "#{channel_id}:#{old_delivery_tag}"
|
248
|
+
new_id = "#{channel_id}:#{new_delivery_tag}"
|
249
|
+
::Istox::RedisBoot.rename("#{old_id}:payload", "#{new_id}:payload")
|
250
|
+
::Istox::RedisBoot.rename("#{old_id}:eid", "#{new_id}:eid")
|
251
|
+
::Istox::RedisBoot.rename("#{old_id}:options", "#{new_id}:options")
|
258
252
|
end
|
259
253
|
|
260
254
|
def eid(ex)
|
@@ -300,8 +294,12 @@ module Istox
|
|
300
294
|
queue_config_from_consumer_key!(consumer_key)['channel']
|
301
295
|
end
|
302
296
|
|
303
|
-
def
|
304
|
-
::Istox::RedisBoot.keys(
|
297
|
+
def find_trackers(pattern)
|
298
|
+
::Istox::RedisBoot.keys(pattern)
|
299
|
+
end
|
300
|
+
|
301
|
+
def get_tracker(key)
|
302
|
+
::Istox::RedisBoot.get(key)
|
305
303
|
end
|
306
304
|
end
|
307
305
|
end
|
@@ -31,10 +31,6 @@ module Istox
|
|
31
31
|
do_publish ex, options, message
|
32
32
|
end
|
33
33
|
|
34
|
-
def re_publish
|
35
|
-
do_publish ex, options, message
|
36
|
-
end
|
37
|
-
|
38
34
|
private
|
39
35
|
|
40
36
|
# Create physical connection for publisher
|
@@ -78,16 +74,29 @@ module Istox
|
|
78
74
|
# Start new thread on receiving ACK from 'confirm-1' channel on the specified thread
|
79
75
|
Thread.new do
|
80
76
|
loop do
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
77
|
+
begin
|
78
|
+
success = @channel[threadId]['confirm-1'].wait_for_confirms
|
79
|
+
if success
|
80
|
+
::Istox::BunnyBoot.del_tracker_on_channel @channel[threadId]['confirm-1'].id
|
81
|
+
else
|
82
|
+
@channel[threadId]['confirm-1'].nacked_set.each do |n|
|
83
|
+
log.debug "[Async #{threadId}] Confirm mode UnAcked Delivery Tag: #{n}"
|
84
|
+
eid = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, n, 'eid'
|
85
|
+
payload = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, n, 'payload'
|
86
|
+
payload = JSON.dump payload unless payload.nil?
|
87
|
+
options = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, n, 'options'
|
88
|
+
options = JSON.dump options unless options.nil?
|
89
|
+
if options.nil?
|
90
|
+
routing_key = nil
|
91
|
+
else
|
92
|
+
routing_key = options.delete :routing_key
|
93
|
+
options[:delivery_tag] = n
|
94
|
+
end
|
95
|
+
publish(exchange: eid, routing_key: routing_key, message: payload, options: options)
|
96
|
+
end
|
90
97
|
end
|
98
|
+
rescue => e
|
99
|
+
log.debug "wait_for_confirm error happening: #{e}"
|
91
100
|
end
|
92
101
|
|
93
102
|
sleep 2
|
@@ -156,9 +165,9 @@ module Istox
|
|
156
165
|
# when an unroutable message is returned, the BasicReturn is fired first and then an ack is sent, firing the BasicAck second.
|
157
166
|
# So if the current status is unroutable then we need to make sure that we don't overwrite that status with Success (ack).
|
158
167
|
ex.on_return do |return_info, properties, content|
|
159
|
-
log.debug "Got a returned message info: #{return_info}"
|
160
|
-
log.debug "Got a returned message properties: #{properties}"
|
161
|
-
log.debug "Got a returned message content: #{content}"
|
168
|
+
# log.debug "Got a returned message info: #{return_info}"
|
169
|
+
# log.debug "Got a returned message properties: #{properties}"
|
170
|
+
# log.debug "Got a returned message content: #{content}"
|
162
171
|
|
163
172
|
# ::Istox::BunnyBoot.find_trackers("message_id:#{properties[:message_id]}").each do |key|
|
164
173
|
# log.debug("On_return, update key #{key} in redis on status and retry counter")
|
@@ -187,31 +196,38 @@ module Istox
|
|
187
196
|
confirm = channel_confirm? ch
|
188
197
|
mode = confirm_mode eid
|
189
198
|
|
199
|
+
::Istox::BunnyBoot.publish(ex, message, options)
|
190
200
|
if confirm
|
191
201
|
delivery_tag = channel_next_tag ch
|
192
202
|
if options[:message_id].nil?
|
193
203
|
options[:message_id] = SecureRandom.hex
|
194
|
-
|
204
|
+
::Istox::BunnyBoot.create_tracker channel_id, delivery_tag, eid, options, message
|
205
|
+
else
|
206
|
+
log.debug "message_id already existed for re-publish msg #{options[:message_id]}"
|
207
|
+
::Istox::BunnyBoot.rename_tracker channel_id, options[:delivery_tag], delivery_tag
|
195
208
|
end
|
196
209
|
end
|
197
210
|
|
198
|
-
::Istox::BunnyBoot.publish(ex, message, options)
|
199
211
|
if confirm && mode == 0
|
200
212
|
success = ex.channel.wait_for_confirms
|
201
213
|
if success
|
202
214
|
# ::Istox::BunnyBoot.del_tracker_on_channel channel_id
|
203
215
|
0
|
204
216
|
else
|
217
|
+
ret = []
|
205
218
|
ex.channel.nacked_set.each do |n|
|
206
219
|
log.debug("UnAck delivery tag is #{n}")
|
207
|
-
|
208
|
-
|
220
|
+
options = ::Istox::BunnyBoot.find_tracker_on_channel(channel_id, n, 'options')
|
221
|
+
options = JSON.dump options
|
222
|
+
ret << {delivery_tag: n, message_id: options[:message_id]}
|
209
223
|
end
|
210
|
-
|
224
|
+
ret
|
211
225
|
end
|
212
226
|
end
|
213
227
|
rescue Bunny::ConnectionClosedError => e
|
214
228
|
log.debug "Publish fails due to #{e}"
|
229
|
+
sleep 1
|
230
|
+
do_publish(ex,options,message)
|
215
231
|
rescue => e
|
216
232
|
log.debug "Timeout error happens #{e}"
|
217
233
|
end
|
data/lib/istox/helpers/redis.rb
CHANGED
@@ -16,6 +16,10 @@ module Istox
|
|
16
16
|
redis.del(k)
|
17
17
|
end
|
18
18
|
|
19
|
+
def get(k)
|
20
|
+
redis.get(k)
|
21
|
+
end
|
22
|
+
|
19
23
|
def flushdb
|
20
24
|
redis.flushdb
|
21
25
|
end
|
@@ -24,6 +28,10 @@ module Istox
|
|
24
28
|
redis.incr(k)
|
25
29
|
end
|
26
30
|
|
31
|
+
def rename(old, new)
|
32
|
+
redis.rename old, new
|
33
|
+
end
|
34
|
+
|
27
35
|
def lock(key, timeout = 3600)
|
28
36
|
loop do
|
29
37
|
break if set(key, 1, nx: true, px: timeout)
|
@@ -15,6 +15,9 @@ module Istox
|
|
15
15
|
@mutex.synchronize do
|
16
16
|
return if @flag
|
17
17
|
|
18
|
+
# Create connection
|
19
|
+
@conn = ::Istox::BunnyBoot.connection
|
20
|
+
|
18
21
|
subscribing_consumer_keys = ::Istox::BunnyBoot.queues_keys_for_subscribe if subscribing_consumer_keys.empty?
|
19
22
|
subscribing_consumer_keys.each do |key|
|
20
23
|
log.debug "Do subscribe key #{key}"
|
@@ -41,8 +44,7 @@ module Istox
|
|
41
44
|
# Create Channel with specified pool_size and prefetch
|
42
45
|
pool_size = ::Istox::BunnyBoot.channel_pool_size consumer_key
|
43
46
|
prefetch = ::Istox::BunnyBoot.channel_prefetch consumer_key
|
44
|
-
|
45
|
-
active_channel = ::Istox::BunnyBoot.channel(conn, pool_size: pool_size, prefetch: prefetch)
|
47
|
+
active_channel = ::Istox::BunnyBoot.channel(@conn, pool_size: pool_size, prefetch: prefetch)
|
46
48
|
@chs = [] if @chs.nil?
|
47
49
|
@chs << active_channel
|
48
50
|
|
@@ -104,7 +106,7 @@ module Istox
|
|
104
106
|
|
105
107
|
flag = false
|
106
108
|
::Istox::RedisBoot.lock("peatio:#{consumer_key}:locking") do
|
107
|
-
flag = ::Istox::BunnyBoot.queue_ok? conn, queue.name
|
109
|
+
flag = ::Istox::BunnyBoot.queue_ok? @conn, queue.name
|
108
110
|
end
|
109
111
|
break if flag
|
110
112
|
|
data/lib/istox/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: istox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.127
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Siong Leng
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-03-
|
11
|
+
date: 2020-03-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: awesome_print
|
@@ -445,7 +445,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
445
445
|
- !ruby/object:Gem::Version
|
446
446
|
version: '0'
|
447
447
|
requirements: []
|
448
|
-
rubygems_version: 3.0.
|
448
|
+
rubygems_version: 3.0.3
|
449
449
|
signing_key:
|
450
450
|
specification_version: 4
|
451
451
|
summary: istox backend shared gem
|