istox 0.1.126 → 0.1.127

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- istox (0.1.125)
4
+ istox (0.1.126)
5
5
  awesome_print
6
6
  binding_of_caller
7
7
  bunny (>= 2.12.0)
@@ -97,7 +97,7 @@ GEM
97
97
  ffi (1.12.2)
98
98
  globalid (0.4.2)
99
99
  activesupport (>= 4.2.0)
100
- google-protobuf (3.11.4)
100
+ google-protobuf (3.11.4-universal-darwin)
101
101
  googleapis-common-protos-types (1.0.4)
102
102
  google-protobuf (~> 3.0)
103
103
  graphlient (0.3.7)
@@ -108,7 +108,7 @@ GEM
108
108
  graphql-client (0.16.0)
109
109
  activesupport (>= 3.0)
110
110
  graphql (~> 1.8)
111
- grpc (1.27.0)
111
+ grpc (1.27.0-universal-darwin)
112
112
  google-protobuf (~> 3.11)
113
113
  googleapis-common-protos-types (~> 1.0)
114
114
  grpc-tools (1.27.0)
@@ -24,8 +24,8 @@ module Istox
24
24
  # Create virtual channel on established connection
25
25
  # Configure pool_size, prefetch, confirm mode according to opts
26
26
  def channel(conn, opts = {})
27
- ch = conn.create_channel(nil, opts['pool_size'] || 1)
28
- ch.prefetch(opts['prefetch']) unless opts['prefetch'].nil?
27
+ ch = conn.create_channel(nil, opts[:pool_size] || 1)
28
+ ch.prefetch(opts[:prefetch]) unless opts[:prefetch].nil?
29
29
 
30
30
  # Put channel in confirmation mode
31
31
  ch.confirm_select if opts[:confirm]
@@ -218,43 +218,37 @@ module Istox
218
218
  end
219
219
  end
220
220
 
221
- def create_tracker(channel_id, delivery_tag, eid, message_id, options, payload)
222
- log.debug "Create track: channel_id: #{channel_id}, delivery_tag: #{delivery_tag}, message_id: #{message_id}"
223
- # combination of channel_id & delivery_tag can uniquely identify a msg
221
+ def create_tracker(channel_id, delivery_tag, eid, options, payload)
222
+ log.debug "Create track: channel_id: #{channel_id}, delivery_tag: #{delivery_tag}"
223
+ # combination of channel_id:delivery_tag can uniquely identify a msg
224
+ # For each retry of msg, channel_id and delivery_tag is unchanged
225
+ # But each retry, there is new delivery_tag that should be updated
224
226
  id = "#{channel_id.to_s}:#{delivery_tag.to_s}"
225
227
 
226
- ::Istox::RedisBoot.sets("#{id}:messageid", message_id)
227
- ::Istox::RedisBoot.sets("#{id}:retry", 0)
228
228
  ::Istox::RedisBoot.sets("#{id}:payload", payload.to_s)
229
229
  ::Istox::RedisBoot.sets("#{id}:eid", eid.to_s)
230
230
  ::Istox::RedisBoot.sets("#{id}:options", options.to_s)
231
231
  end
232
232
 
233
- def find_tracker_on_channel(channel_id, delivery_tag)
234
- id = "#{channel_id.to_s}:#{delivery_tag.to_s}"
235
- keys = ::Istox::RedisBoot.keys("#{id}:p")
236
- keys.each do |k|
237
- # payload =
238
- end
233
+ def find_tracker_on_channel(channel_id, delivery_tag, key)
234
+ pattern = "#{channel_id.to_s}:#{delivery_tag.to_s}:#{key}"
235
+ keys = find_trackers pattern
236
+ get_tracker(keys.first)
239
237
  end
240
238
 
241
239
  def del_tracker_on_channel(channel_id)
242
- trackers = find_trackers_from_channelId channel_id
240
+ trackers = find_trackers "#{channel_id}*"
243
241
  trackers.each do |k|
244
242
  ::Istox::RedisBoot.del k
245
243
  end
246
244
  end
247
245
 
248
- def find_trackers(key)
249
- a = ::Istox::RedisBoot.keys("#{key}")
250
- end
251
-
252
- def update_tracker_by_status(key, status)
253
- ::Istox::RedisBoot.set(key, status)
254
- end
255
-
256
- def update_tracker_incr_retry(key)
257
- ::Istox::RedisBoot.incr(key)
246
+ def rename_tracker(channel_id, old_delivery_tag, new_delivery_tag)
247
+ old_id = "#{channel_id}:#{old_delivery_tag}"
248
+ new_id = "#{channel_id}:#{new_delivery_tag}"
249
+ ::Istox::RedisBoot.rename("#{old_id}:payload", "#{new_id}:payload")
250
+ ::Istox::RedisBoot.rename("#{old_id}:eid", "#{new_id}:eid")
251
+ ::Istox::RedisBoot.rename("#{old_id}:options", "#{new_id}:options")
258
252
  end
259
253
 
260
254
  def eid(ex)
@@ -300,8 +294,12 @@ module Istox
300
294
  queue_config_from_consumer_key!(consumer_key)['channel']
301
295
  end
302
296
 
303
- def find_trackers_from_channelId(channel_id)
304
- ::Istox::RedisBoot.keys("#{channel_id}*")
297
+ def find_trackers(pattern)
298
+ ::Istox::RedisBoot.keys(pattern)
299
+ end
300
+
301
+ def get_tracker(key)
302
+ ::Istox::RedisBoot.get(key)
305
303
  end
306
304
  end
307
305
  end
@@ -31,10 +31,6 @@ module Istox
31
31
  do_publish ex, options, message
32
32
  end
33
33
 
34
- def re_publish
35
- do_publish ex, options, message
36
- end
37
-
38
34
  private
39
35
 
40
36
  # Create physical connection for publisher
@@ -78,16 +74,29 @@ module Istox
78
74
  # Start new thread on receiving ACK from 'confirm-1' channel on the specified thread
79
75
  Thread.new do
80
76
  loop do
81
- success = @channel[threadId]['confirm-1'].wait_for_confirms
82
- if success
83
- # ::Istox::BunnyBoot.del_tracker_on_channel @channel[threadId]['confirm-1'].id
84
- else
85
- @channel[threadId]['confirm-1'].nacked_set.each do |n|
86
- log.debug "[Async #{threadId}] Confirm mode UnAcked Delivery Tag: #{n}"
87
- # ::Istox::RedisBoot.incr("confirm:#{n.to_s}")
88
- # message = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, delivery_tag
89
- # publish(exchange: ()::Istox::BunnyBoot.eid ex), routing_key: routing_key, message: message)
77
+ begin
78
+ success = @channel[threadId]['confirm-1'].wait_for_confirms
79
+ if success
80
+ ::Istox::BunnyBoot.del_tracker_on_channel @channel[threadId]['confirm-1'].id
81
+ else
82
+ @channel[threadId]['confirm-1'].nacked_set.each do |n|
83
+ log.debug "[Async #{threadId}] Confirm mode UnAcked Delivery Tag: #{n}"
84
+ eid = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, n, 'eid'
85
+ payload = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, n, 'payload'
86
+ payload = JSON.dump payload unless payload.nil?
87
+ options = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, n, 'options'
88
+ options = JSON.dump options unless options.nil?
89
+ if options.nil?
90
+ routing_key = nil
91
+ else
92
+ routing_key = options.delete :routing_key
93
+ options[:delivery_tag] = n
94
+ end
95
+ publish(exchange: eid, routing_key: routing_key, message: payload, options: options)
96
+ end
90
97
  end
98
+ rescue => e
99
+ log.debug "wait_for_confirm error happening: #{e}"
91
100
  end
92
101
 
93
102
  sleep 2
@@ -156,9 +165,9 @@ module Istox
156
165
  # when an unroutable message is returned, the BasicReturn is fired first and then an ack is sent, firing the BasicAck second.
157
166
  # So if the current status is unroutable then we need to make sure that we don't overwrite that status with Success (ack).
158
167
  ex.on_return do |return_info, properties, content|
159
- log.debug "Got a returned message info: #{return_info}"
160
- log.debug "Got a returned message properties: #{properties}"
161
- log.debug "Got a returned message content: #{content}"
168
+ # log.debug "Got a returned message info: #{return_info}"
169
+ # log.debug "Got a returned message properties: #{properties}"
170
+ # log.debug "Got a returned message content: #{content}"
162
171
 
163
172
  # ::Istox::BunnyBoot.find_trackers("message_id:#{properties[:message_id]}").each do |key|
164
173
  # log.debug("On_return, update key #{key} in redis on status and retry counter")
@@ -187,31 +196,38 @@ module Istox
187
196
  confirm = channel_confirm? ch
188
197
  mode = confirm_mode eid
189
198
 
199
+ ::Istox::BunnyBoot.publish(ex, message, options)
190
200
  if confirm
191
201
  delivery_tag = channel_next_tag ch
192
202
  if options[:message_id].nil?
193
203
  options[:message_id] = SecureRandom.hex
194
- # ::Istox::BunnyBoot.create_tracker channel_id, delivery_tag, eid, options[:message_id], options, message
204
+ ::Istox::BunnyBoot.create_tracker channel_id, delivery_tag, eid, options, message
205
+ else
206
+ log.debug "message_id already existed for re-publish msg #{options[:message_id]}"
207
+ ::Istox::BunnyBoot.rename_tracker channel_id, options[:delivery_tag], delivery_tag
195
208
  end
196
209
  end
197
210
 
198
- ::Istox::BunnyBoot.publish(ex, message, options)
199
211
  if confirm && mode == 0
200
212
  success = ex.channel.wait_for_confirms
201
213
  if success
202
214
  # ::Istox::BunnyBoot.del_tracker_on_channel channel_id
203
215
  0
204
216
  else
217
+ ret = []
205
218
  ex.channel.nacked_set.each do |n|
206
219
  log.debug("UnAck delivery tag is #{n}")
207
- #::Istox::RedisBoot.incr("confirm:#{n.to_s}")
208
- # publish(exchange: eid, routing_key: routing_key, message: message, )
220
+ options = ::Istox::BunnyBoot.find_tracker_on_channel(channel_id, n, 'options')
221
+ options = JSON.dump options
222
+ ret << {delivery_tag: n, message_id: options[:message_id]}
209
223
  end
210
- -1
224
+ ret
211
225
  end
212
226
  end
213
227
  rescue Bunny::ConnectionClosedError => e
214
228
  log.debug "Publish fails due to #{e}"
229
+ sleep 1
230
+ do_publish(ex,options,message)
215
231
  rescue => e
216
232
  log.debug "Timeout error happens #{e}"
217
233
  end
@@ -16,6 +16,10 @@ module Istox
16
16
  redis.del(k)
17
17
  end
18
18
 
19
+ def get(k)
20
+ redis.get(k)
21
+ end
22
+
19
23
  def flushdb
20
24
  redis.flushdb
21
25
  end
@@ -24,6 +28,10 @@ module Istox
24
28
  redis.incr(k)
25
29
  end
26
30
 
31
+ def rename(old, new)
32
+ redis.rename old, new
33
+ end
34
+
27
35
  def lock(key, timeout = 3600)
28
36
  loop do
29
37
  break if set(key, 1, nx: true, px: timeout)
@@ -15,6 +15,9 @@ module Istox
15
15
  @mutex.synchronize do
16
16
  return if @flag
17
17
 
18
+ # Create connection
19
+ @conn = ::Istox::BunnyBoot.connection
20
+
18
21
  subscribing_consumer_keys = ::Istox::BunnyBoot.queues_keys_for_subscribe if subscribing_consumer_keys.empty?
19
22
  subscribing_consumer_keys.each do |key|
20
23
  log.debug "Do subscribe key #{key}"
@@ -41,8 +44,7 @@ module Istox
41
44
  # Create Channel with specified pool_size and prefetch
42
45
  pool_size = ::Istox::BunnyBoot.channel_pool_size consumer_key
43
46
  prefetch = ::Istox::BunnyBoot.channel_prefetch consumer_key
44
- conn = ::Istox::BunnyBoot.connection
45
- active_channel = ::Istox::BunnyBoot.channel(conn, pool_size: pool_size, prefetch: prefetch)
47
+ active_channel = ::Istox::BunnyBoot.channel(@conn, pool_size: pool_size, prefetch: prefetch)
46
48
  @chs = [] if @chs.nil?
47
49
  @chs << active_channel
48
50
 
@@ -104,7 +106,7 @@ module Istox
104
106
 
105
107
  flag = false
106
108
  ::Istox::RedisBoot.lock("peatio:#{consumer_key}:locking") do
107
- flag = ::Istox::BunnyBoot.queue_ok? conn, queue.name
109
+ flag = ::Istox::BunnyBoot.queue_ok? @conn, queue.name
108
110
  end
109
111
  break if flag
110
112
 
@@ -1,3 +1,3 @@
1
1
  module Istox
2
- VERSION = '0.1.126'.freeze
2
+ VERSION = '0.1.127'.freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: istox
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.126
4
+ version: 0.1.127
5
5
  platform: ruby
6
6
  authors:
7
7
  - Siong Leng
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-03-02 00:00:00.000000000 Z
11
+ date: 2020-03-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: awesome_print
@@ -445,7 +445,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
445
445
  - !ruby/object:Gem::Version
446
446
  version: '0'
447
447
  requirements: []
448
- rubygems_version: 3.0.6
448
+ rubygems_version: 3.0.3
449
449
  signing_key:
450
450
  specification_version: 4
451
451
  summary: istox backend shared gem