skein 0.3.7 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +5 -5
  2. data/Gemfile +4 -1
  3. data/Gemfile.lock +49 -36
  4. data/LICENSE.md +1 -1
  5. data/README.md +9 -4
  6. data/RELEASES.md +2 -0
  7. data/VERSION +1 -1
  8. data/bin/skein +1 -1
  9. data/examples/echo +66 -0
  10. data/examples/echo-server +77 -0
  11. data/lib/skein/adapter.rb +3 -0
  12. data/lib/skein/client/publisher.rb +10 -2
  13. data/lib/skein/client/rpc.rb +78 -19
  14. data/lib/skein/client/subscriber.rb +27 -4
  15. data/lib/skein/client/worker.rb +221 -63
  16. data/lib/skein/client.rb +20 -11
  17. data/lib/skein/config.rb +3 -1
  18. data/lib/skein/connected.rb +88 -17
  19. data/lib/skein/context.rb +5 -2
  20. data/lib/skein/handler/async.rb +6 -2
  21. data/lib/skein/handler.rb +131 -20
  22. data/lib/skein/rabbitmq.rb +1 -0
  23. data/lib/skein/timeout_queue.rb +43 -0
  24. data/lib/skein.rb +18 -2
  25. data/skein.gemspec +21 -24
  26. data/test/helper.rb +29 -0
  27. data/test/unit/test_skein_client.rb +4 -1
  28. data/test/unit/test_skein_client_publisher.rb +1 -1
  29. data/test/unit/test_skein_client_rpc.rb +37 -0
  30. data/test/unit/test_skein_client_subscriber.rb +29 -12
  31. data/test/unit/test_skein_client_worker.rb +22 -9
  32. data/test/unit/test_skein_connected.rb +21 -0
  33. data/test/unit/test_skein_rpc_timeout.rb +19 -0
  34. data/test/unit/test_skein_worker.rb +4 -0
  35. metadata +41 -16
  36. data/lib/skein/rpc/base.rb +0 -23
  37. data/lib/skein/rpc/error.rb +0 -34
  38. data/lib/skein/rpc/notification.rb +0 -2
  39. data/lib/skein/rpc/request.rb +0 -62
  40. data/lib/skein/rpc/response.rb +0 -38
  41. data/lib/skein/rpc.rb +0 -24
  42. data/test/unit/test_skein_rpc_error.rb +0 -10
  43. data/test/unit/test_skein_rpc_request.rb +0 -93
@@ -19,15 +19,38 @@ class Skein::Client::Subscriber < Skein::Connected
19
19
  def listen(block = true)
20
20
  case (@subscribe_queue.class.to_s.split(/::/)[0])
21
21
  when 'Bunny'
22
- @subscribe_queue.subscribe(block: block) do |delivery_info, properties, payload|
23
- yield(JSON.load(payload), delivery_info, properties)
22
+ begin
23
+ @subscribe_queue.subscribe(block: block) do |delivery_info, properties, payload|
24
+ yield(JSON.load(payload), delivery_info, properties)
25
+ end
24
26
  end
25
27
  when 'MarchHare'
26
- @subscribe_queue.subscribe(block: block) do |metadata, payload|
27
- yield(JSON.load(payload), metadata)
28
+ begin
29
+ @subscribe_queue.subscribe(block: block) do |metadata, payload|
30
+ yield(JSON.load(payload), metadata)
31
+ end
32
+ rescue MarchHare::ChannelAlreadyClosed
33
+ # Connection got killed outside of thread, so shut-down and move on
28
34
  end
29
35
  else
30
36
  raise "Unknown queue type #{@subscribe_queue.class}, cannot listen."
31
37
  end
32
38
  end
39
+
40
+ def close(delete_queue: false)
41
+ if (delete_queue)
42
+ begin
43
+ @subscribe_queue.delete
44
+ rescue => e
45
+ case (e.class.to_s)
46
+ when 'Bunny::ChannelAlreadyClosed', 'MarchHare::ChannelAlreadyClosed'
47
+ # Tried to delete, but this has already been shut down
48
+ else
49
+ raise e
50
+ end
51
+ end
52
+ end
53
+
54
+ super()
55
+ end
33
56
  end
@@ -1,97 +1,121 @@
1
1
  require 'json'
2
2
 
3
3
  class Skein::Client::Worker < Skein::Connected
4
+ # == Properties ===========================================================
5
+
6
+ attr_reader :operations
7
+ attr_reader :queue_name
8
+
9
+ # == Exceptions ===========================================================
10
+
11
+ class RejectMessage < Exception
12
+ end
13
+
14
+ class RetryMessage < Exception
15
+ end
16
+
17
+ # == Class Methods ========================================================
18
+
4
19
  # == Instance Methods =====================================================
5
20
 
6
- def initialize(queue_name, exchange_name: nil, connection: nil, context: nil, concurrency: nil)
7
- super(connection: connection, context: context)
21
+ def initialize(queue_name, exchange_name: nil, connection: nil, context: nil, concurrency: nil, durable: nil, auto_delete: false, routing_key: nil, ident: nil)
22
+ super(connection: connection, context: context, ident: ident)
8
23
 
9
- lock do
10
- @reply_exchange = self.channel.default_exchange
11
- @queue = self.channel.queue(queue_name, durable: !!queue_name.match(/\S/))
24
+ @exchange_name = exchange_name
25
+ @queue_name = queue_name.dup.freeze
26
+ @routing_key = routing_key
27
+ @durable = durable.nil? ? !!@queue_name.match(/\S/) : !!durable
28
+ @operations = [ ]
29
+ @auto_delete = auto_delete
12
30
 
13
- if (exchange_name)
14
- @exchange = self.channel.direct(exchange_name, durable: true)
31
+ concurrency &&= concurrency.to_i
32
+ concurrency ||= 1
15
33
 
16
- @queue.bind(@exchange)
34
+ concurrency.times do |i|
35
+ with_channel_in_thread(name: 'worker-%d' % i) do |channel, meta|
36
+ self.establish_subscriber!(channel, meta)
17
37
  end
38
+ end
18
39
 
19
- @handler = Skein::Handler.for(self)
20
- @received = Queue.new
21
- @replies = Queue.new
22
- @concurrency = concurrency && concurrency.to_i || 1
23
- @threads = [ ]
40
+ self.after_initialize rescue nil
41
+ end
24
42
 
25
- @threads << Thread.new do
26
- Thread.abort_on_exception = true
43
+ # Define in derived classes to implement any desired customization to be
44
+ # performed after initialization.
45
+ def after_initialize
46
+ end
27
47
 
28
- Skein::Adapter.subscribe(@queue) do |payload, delivery_tag, reply_to|
29
- @received << [ payload, delivery_tag, reply_to ]
30
- end
31
- end
48
+ # Define in derived classes. Willl be called immediately after a request is
49
+ # received but before any processing occurs.
50
+ def before_request
51
+ end
32
52
 
33
- @threads << Thread.new do
34
- Thread.abort_on_exception = true
53
+ # Define in derived classes. Will be called immediately prior to executing
54
+ # the worker method.
55
+ def before_execution(method_name)
56
+ end
35
57
 
36
- loop do
37
- payload, delivery_tag, reply_to, reply_json = @replies.pop
58
+ # Define in derived classes. Will be called immediately after executing the
59
+ # worker method.
60
+ def after_execution(method_name)
61
+ end
38
62
 
39
- channel.acknowledge(delivery_tag, true)
63
+ # Define in derived classes. Will be called immediately after handling an
64
+ # RPC call even if an error has occured.
65
+ def after_request
66
+ end
40
67
 
41
- if (reply_to)
42
- @reply_exchange.publish(
43
- reply_json,
44
- routing_key: reply_to,
45
- content_type: 'application/json'
46
- )
47
- end
48
- end
49
- end
68
+ # Declared in derived classes. Will be called immediately after an
69
+ # exception has occurred when processing a request. Any excepions generated
70
+ # in this method call are suppressed and ignored to avoid being caught
71
+ # in a loop.
72
+ def after_exception(e)
73
+ end
50
74
 
51
- @concurrency.times do
52
- @threads << Thread.new do
53
- Thread.abort_on_exception = true
75
+ # Declared in derived classes. Will be called immediately before the worker
76
+ # is closed down.
77
+ def before_close
78
+ end
54
79
 
55
- loop do
56
- payload, delivery_tag, reply_to = @received.pop
57
- thread = Thread.current
80
+ # Declared in derived classes. Will be called immediately after the worker
81
+ # has been closed down.
82
+ def after_close
83
+ end
58
84
 
59
- @handler.handle(payload) do |reply_json|
60
- @replies << [ payload, delivery_tag, reply_to, reply_json ]
85
+ def close(delete_queue: false)
86
+ self.before_close
61
87
 
62
- if (thread == Thread.current)
63
- thread = nil
64
- else
65
- thread.wakeup
66
- end
67
- end
88
+ @operations.each do |meta|
89
+ subscriber = meta[:subscriber]
68
90
 
69
- thread and Thread.stop
70
- end
71
- end
91
+ if (subscriber.respond_to?(:gracefully_shut_down))
92
+ subscriber.gracefully_shut_down
72
93
  end
94
+
95
+ thread = meta[:thread]
96
+
97
+ thread.respond_to?(:terminate!) ? thread.terminate! : thread.kill
98
+ thread.join
73
99
  end
74
100
 
75
- self.after_initialize
76
- end
101
+ if (delete_queue)
102
+ # The connection may have been terminated, so reconnect and delete
103
+ # the queue if necessary.
104
+ channel = @connection.create_channel
77
105
 
78
- # Extend this in derived classes to implement any desired customization to
79
- # be performed after initialization
80
- def after_initialize
81
- end
106
+ channel.queue(@queue_name, durable: @durable).delete
82
107
 
83
- def close
84
- @threads.each do |thread|
85
- thread.kill
86
- thread.join
108
+ channel.close
87
109
  end
88
110
 
89
- super
111
+ super()
112
+
113
+ self.after_close
90
114
  end
91
115
 
92
116
  def join
93
- @threads.each do |thread|
94
- thread.join
117
+ @operations.each do |meta|
118
+ meta[:thread].join
95
119
  end
96
120
  end
97
121
 
@@ -100,4 +124,138 @@ class Skein::Client::Worker < Skein::Connected
100
124
  # callback-style delegation.
101
125
  false
102
126
  end
127
+
128
+ # Signal that the current operation should be abandoned and not retried.
129
+ def reject!
130
+ raise RejectMessage
131
+ end
132
+
133
+ # Signal that the current operation should be abandoned and retried later.
134
+ def retry!
135
+ raise RetryMessage
136
+ end
137
+
138
+ protected
139
+ def state_tracker
140
+ {
141
+ method: nil,
142
+ started: nil,
143
+ finished: nil
144
+ }
145
+ end
146
+
147
+ def metrics_tracker
148
+ Hash.new(0).merge(
149
+ time: 0.0,
150
+ errors: Hash.new(0)
151
+ )
152
+ end
153
+
154
+ def in_thread
155
+ @operations << {
156
+ thread: Thread.new do
157
+ Thread.abort_on_exception = true
158
+
159
+ yield
160
+ end
161
+ }
162
+ end
163
+
164
+ def establish_queue!(channel)
165
+ queue = channel.queue(
166
+ @queue_name,
167
+ durable: @durable,
168
+ auto_delete: @auto_delete
169
+ )
170
+
171
+ if (@exchange_name&.match(/\S/))
172
+ exchange = channel.direct(@exchange_name, durable: true)
173
+
174
+ queue.bind(exchange, routing_key: @routing_key || @queue_name)
175
+ end
176
+
177
+ queue
178
+ end
179
+
180
+ def establish_subscriber!(channel, meta)
181
+ queue = self.establish_queue!(channel)
182
+
183
+ meta[:subscriber] = Skein::Adapter.subscribe(queue) do |payload, delivery_tag, reply_to|
184
+ if (ENV['SKEIN_DEBUG_JSON'] and reply_to)
185
+ $stdout.puts('%s -> %s' % [ reply_to, payload ])
186
+ end
187
+
188
+ self.context.trap do
189
+ self.before_request rescue nil
190
+
191
+ handler.handle(payload, meta[:metrics], meta[:state]) do |reply_json|
192
+ if (ENV['SKEIN_DEBUG_JSON'] and reply_to)
193
+ $stdout.puts('%s <- %s' % [ reply_to, reply_json ])
194
+ end
195
+
196
+ # Secondary (inner) trap required since some handlers are async
197
+ self.context.trap do
198
+ # NOTE: begin...end necessary for rescue in Ruby versions below 2.4
199
+ begin
200
+ channel.acknowledge(delivery_tag, true)
201
+
202
+ if (reply_to)
203
+ channel.default_exchange.publish(
204
+ reply_json,
205
+ routing_key: reply_to,
206
+ content_type: 'application/json'
207
+ )
208
+ end
209
+
210
+ rescue RejectMessage
211
+ # Reject the message
212
+ channel.reject(delivery_tag, false)
213
+ rescue RetryMessage
214
+ # Reject and requeue the message
215
+ channel.reject(delivery_tag, true)
216
+ rescue => e
217
+ self.after_exception(e) rescue nil
218
+ raise e
219
+ ensure
220
+ self.after_request rescue nil
221
+ end
222
+ end
223
+ end
224
+ end
225
+ end
226
+ end
227
+
228
+ def with_channel_in_thread(recover: true, name: nil)
229
+ meta = {
230
+ metrics: metrics_tracker,
231
+ state: state_tracker
232
+ }
233
+
234
+ @operations << meta
235
+
236
+ meta[:thread] = Thread.new do
237
+ Thread.abort_on_exception = true
238
+ Thread.current.name = name
239
+
240
+ begin
241
+ channel = meta[:channel] = self.create_channel
242
+
243
+ yield(channel, meta)
244
+
245
+ channel.close rescue nil
246
+
247
+ redo if (recover)
248
+
249
+ ensure
250
+ # NOTE: The `.close` call may fail for a variety of reasons, but the
251
+ # important thing here is an attempt is made, regardless of
252
+ # outcome.
253
+ channel&.close rescue nil
254
+ end
255
+ end
256
+ end
257
+
258
+ def handler
259
+ @handler ||= Skein::Handler.for(self)
260
+ end
103
261
  end
data/lib/skein/client.rb CHANGED
@@ -24,38 +24,47 @@ class Skein::Client < Skein::Connected
24
24
 
25
25
  # == Instance Methods =====================================================
26
26
 
27
- def initialize(connection: nil, context: nil)
28
- super(connection: connection, context: context)
27
+ def initialize(config: nil, connection: nil, context: nil)
28
+ super(config: config, connection: connection, context: context)
29
29
  end
30
30
 
31
- def rpc(exchange_name = nil, routing_key: nil)
31
+ def rpc(exchange_name = nil, routing_key: nil, ident: nil, expiration: nil, persistent: nil, durable: nil, timeout: nil)
32
32
  Skein::Client::RPC.new(
33
33
  exchange_name,
34
34
  routing_key: routing_key,
35
35
  connection: self.connection,
36
- context: self.context
36
+ context: self.context,
37
+ ident: ident,
38
+ expiration: expiration,
39
+ persistent: persistent,
40
+ durable: durable,
41
+ timeout: timeout
37
42
  )
38
43
  end
39
44
 
40
- def worker(queue_name)
41
- Skein::Client::Worker.new(
45
+ def worker(queue_name, type = nil, ident: nil, durable: nil)
46
+ (type || Skein::Client::Worker).new(
42
47
  queue_name,
43
48
  connection: self.connection,
44
- context: self.context
49
+ context: self.context,
50
+ ident: ident,
51
+ durable: durable
45
52
  )
46
53
  end
47
54
 
48
- def publisher(queue_name)
55
+ def publisher(exchange_name, type: nil, durable: nil)
49
56
  Skein::Client::Publisher.new(
50
- queue_name,
57
+ exchange_name,
58
+ type: type,
59
+ durable: durable,
51
60
  connection: self.connection,
52
61
  context: self.context
53
62
  )
54
63
  end
55
64
 
56
- def subscriber(queue_name, routing_key = nil)
65
+ def subscriber(exchange_name, routing_key = nil)
57
66
  Skein::Client::Subscriber.new(
58
- queue_name,
67
+ exchange_name,
59
68
  routing_key,
60
69
  connection: self.connection,
61
70
  context: self.context
data/lib/skein/config.rb CHANGED
@@ -87,7 +87,9 @@ class Skein::Config < OpenStruct
87
87
  end
88
88
 
89
89
  if (config_path and File.exist?(config_path))
90
- super(DEFAULTS.merge(YAML.load_file(config_path)[self.class.env] || { }))
90
+ super(DEFAULTS.merge(
91
+ YAML.load_file(config_path, aliases: true)[self.class.env] || { }
92
+ ))
91
93
  else
92
94
  super(DEFAULTS)
93
95
  end
@@ -4,19 +4,25 @@ class Skein::Connected
4
4
  attr_reader :context
5
5
  attr_reader :ident
6
6
  attr_reader :connection
7
- attr_reader :channel
8
7
 
9
8
  # == Instance Methods =====================================================
10
9
 
11
- def initialize(connection: nil, context: nil)
10
+ def initialize(config: nil, connection: nil, context: nil, ident: nil)
12
11
  @mutex = Mutex.new
13
- @shared_connection = !!connection
14
12
 
15
- @connection = connection || Skein::RabbitMQ.connect
16
- @channel = @connection.create_channel
13
+ @config = config
14
+ @connection_shared = !connection
15
+ @connection = connection
16
+
17
+ self.connect
18
+ @channels = [ ]
17
19
 
18
20
  @context = context || Skein::Context.new
19
- @ident = @context.ident(self)
21
+ @ident = ident || @context.ident(self)
22
+ end
23
+
24
+ def connection_shared?
25
+ @connection_shared
20
26
  end
21
27
 
22
28
  def lock
@@ -25,26 +31,91 @@ class Skein::Connected
25
31
  end
26
32
  end
27
33
 
34
+ def repeat_until_not_nil(delay: 1.0)
35
+ r = nil
36
+
37
+ loop do
38
+ r = yield
39
+
40
+ break if r
41
+
42
+ sleep(delay)
43
+ end
44
+
45
+ r
46
+ end
47
+
48
+ def connect
49
+ @connection ||= repeat_until_not_nil do
50
+ @connection_shared = false
51
+ Skein::RabbitMQ.connect(@config)
52
+ end
53
+ end
54
+
55
+ def reconnect
56
+ @connection = nil
57
+
58
+ self.connect
59
+ end
60
+
61
+ def create_channel(auto_retry: false)
62
+ channel = begin
63
+ @connection.create_channel
64
+
65
+ rescue RuntimeError
66
+ sleep(1)
67
+
68
+ self.reconnect
69
+
70
+ retry
71
+ end
72
+
73
+ if (channel.respond_to?(:prefetch=))
74
+ channel.prefetch = 1
75
+ else
76
+ channel.prefetch(1)
77
+ end
78
+
79
+ @channels << channel
80
+
81
+ channel
82
+ end
83
+
84
+ def channel
85
+ @channel ||= self.create_channel
86
+ end
87
+
28
88
  def close
29
89
  lock do
30
- begin
31
- @channel and @channel.close
32
-
33
- rescue => e
34
- if (defined?(MarchHare))
35
- case e
36
- when MarchHare::ChannelLevelException, MarchHare::ChannelAlreadyClosed
37
- # Ignored since we're finished with the channel anyway
90
+ @channels.each do |channel|
91
+ begin
92
+ channel.close
93
+
94
+ rescue => e
95
+ if (defined?(MarchHare))
96
+ case (e)
97
+ when MarchHare::ChannelLevelException, MarchHare::ChannelAlreadyClosed
98
+ # Ignored since we're finished with the channel anyway
99
+ else
100
+ raise e
101
+ end
102
+ elsif (defined?(Bunny))
103
+ case (e)
104
+ when Bunny::ChannelAlreadyClosed
105
+ # Ignored since we're finished with the channel anyway
106
+ else
107
+ raise e
108
+ end
38
109
  else
39
110
  raise e
40
111
  end
41
112
  end
42
113
  end
43
114
 
44
- @channel = nil
115
+ @channels = [ ]
45
116
 
46
- unless (@shared_connection)
47
- @connection and @connection.close
117
+ unless (@connection_shared)
118
+ @connection&.close
48
119
  @connection = nil
49
120
  end
50
121
  end