skein 0.3.7 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile +4 -1
- data/Gemfile.lock +49 -36
- data/LICENSE.md +1 -1
- data/README.md +9 -4
- data/RELEASES.md +2 -0
- data/VERSION +1 -1
- data/bin/skein +1 -1
- data/examples/echo +66 -0
- data/examples/echo-server +77 -0
- data/lib/skein/adapter.rb +3 -0
- data/lib/skein/client/publisher.rb +10 -2
- data/lib/skein/client/rpc.rb +78 -19
- data/lib/skein/client/subscriber.rb +27 -4
- data/lib/skein/client/worker.rb +221 -63
- data/lib/skein/client.rb +20 -11
- data/lib/skein/config.rb +3 -1
- data/lib/skein/connected.rb +88 -17
- data/lib/skein/context.rb +5 -2
- data/lib/skein/handler/async.rb +6 -2
- data/lib/skein/handler.rb +131 -20
- data/lib/skein/rabbitmq.rb +1 -0
- data/lib/skein/timeout_queue.rb +43 -0
- data/lib/skein.rb +18 -2
- data/skein.gemspec +21 -24
- data/test/helper.rb +29 -0
- data/test/unit/test_skein_client.rb +4 -1
- data/test/unit/test_skein_client_publisher.rb +1 -1
- data/test/unit/test_skein_client_rpc.rb +37 -0
- data/test/unit/test_skein_client_subscriber.rb +29 -12
- data/test/unit/test_skein_client_worker.rb +22 -9
- data/test/unit/test_skein_connected.rb +21 -0
- data/test/unit/test_skein_rpc_timeout.rb +19 -0
- data/test/unit/test_skein_worker.rb +4 -0
- metadata +41 -16
- data/lib/skein/rpc/base.rb +0 -23
- data/lib/skein/rpc/error.rb +0 -34
- data/lib/skein/rpc/notification.rb +0 -2
- data/lib/skein/rpc/request.rb +0 -62
- data/lib/skein/rpc/response.rb +0 -38
- data/lib/skein/rpc.rb +0 -24
- data/test/unit/test_skein_rpc_error.rb +0 -10
- data/test/unit/test_skein_rpc_request.rb +0 -93
@@ -19,15 +19,38 @@ class Skein::Client::Subscriber < Skein::Connected
|
|
19
19
|
def listen(block = true)
|
20
20
|
case (@subscribe_queue.class.to_s.split(/::/)[0])
|
21
21
|
when 'Bunny'
|
22
|
-
|
23
|
-
|
22
|
+
begin
|
23
|
+
@subscribe_queue.subscribe(block: block) do |delivery_info, properties, payload|
|
24
|
+
yield(JSON.load(payload), delivery_info, properties)
|
25
|
+
end
|
24
26
|
end
|
25
27
|
when 'MarchHare'
|
26
|
-
|
27
|
-
|
28
|
+
begin
|
29
|
+
@subscribe_queue.subscribe(block: block) do |metadata, payload|
|
30
|
+
yield(JSON.load(payload), metadata)
|
31
|
+
end
|
32
|
+
rescue MarchHare::ChannelAlreadyClosed
|
33
|
+
# Connection got killed outside of thread, so shut-down and move on
|
28
34
|
end
|
29
35
|
else
|
30
36
|
raise "Unknown queue type #{@subscribe_queue.class}, cannot listen."
|
31
37
|
end
|
32
38
|
end
|
39
|
+
|
40
|
+
def close(delete_queue: false)
|
41
|
+
if (delete_queue)
|
42
|
+
begin
|
43
|
+
@subscribe_queue.delete
|
44
|
+
rescue => e
|
45
|
+
case (e.class.to_s)
|
46
|
+
when 'Bunny::ChannelAlreadyClosed', 'MarchHare::ChannelAlreadyClosed'
|
47
|
+
# Tried to delete, but this has already been shut down
|
48
|
+
else
|
49
|
+
raise e
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
super()
|
55
|
+
end
|
33
56
|
end
|
data/lib/skein/client/worker.rb
CHANGED
@@ -1,97 +1,121 @@
|
|
1
1
|
require 'json'
|
2
2
|
|
3
3
|
class Skein::Client::Worker < Skein::Connected
|
4
|
+
# == Properties ===========================================================
|
5
|
+
|
6
|
+
attr_reader :operations
|
7
|
+
attr_reader :queue_name
|
8
|
+
|
9
|
+
# == Exceptions ===========================================================
|
10
|
+
|
11
|
+
class RejectMessage < Exception
|
12
|
+
end
|
13
|
+
|
14
|
+
class RetryMessage < Exception
|
15
|
+
end
|
16
|
+
|
17
|
+
# == Class Methods ========================================================
|
18
|
+
|
4
19
|
# == Instance Methods =====================================================
|
5
20
|
|
6
|
-
def initialize(queue_name, exchange_name: nil, connection: nil, context: nil, concurrency: nil)
|
7
|
-
super(connection: connection, context: context)
|
21
|
+
def initialize(queue_name, exchange_name: nil, connection: nil, context: nil, concurrency: nil, durable: nil, auto_delete: false, routing_key: nil, ident: nil)
|
22
|
+
super(connection: connection, context: context, ident: ident)
|
8
23
|
|
9
|
-
|
10
|
-
|
11
|
-
|
24
|
+
@exchange_name = exchange_name
|
25
|
+
@queue_name = queue_name.dup.freeze
|
26
|
+
@routing_key = routing_key
|
27
|
+
@durable = durable.nil? ? !!@queue_name.match(/\S/) : !!durable
|
28
|
+
@operations = [ ]
|
29
|
+
@auto_delete = auto_delete
|
12
30
|
|
13
|
-
|
14
|
-
|
31
|
+
concurrency &&= concurrency.to_i
|
32
|
+
concurrency ||= 1
|
15
33
|
|
16
|
-
|
34
|
+
concurrency.times do |i|
|
35
|
+
with_channel_in_thread(name: 'worker-%d' % i) do |channel, meta|
|
36
|
+
self.establish_subscriber!(channel, meta)
|
17
37
|
end
|
38
|
+
end
|
18
39
|
|
19
|
-
|
20
|
-
|
21
|
-
@replies = Queue.new
|
22
|
-
@concurrency = concurrency && concurrency.to_i || 1
|
23
|
-
@threads = [ ]
|
40
|
+
self.after_initialize rescue nil
|
41
|
+
end
|
24
42
|
|
25
|
-
|
26
|
-
|
43
|
+
# Define in derived classes to implement any desired customization to be
|
44
|
+
# performed after initialization.
|
45
|
+
def after_initialize
|
46
|
+
end
|
27
47
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
48
|
+
# Define in derived classes. Willl be called immediately after a request is
|
49
|
+
# received but before any processing occurs.
|
50
|
+
def before_request
|
51
|
+
end
|
32
52
|
|
33
|
-
|
34
|
-
|
53
|
+
# Define in derived classes. Will be called immediately prior to executing
|
54
|
+
# the worker method.
|
55
|
+
def before_execution(method_name)
|
56
|
+
end
|
35
57
|
|
36
|
-
|
37
|
-
|
58
|
+
# Define in derived classes. Will be called immediately after executing the
|
59
|
+
# worker method.
|
60
|
+
def after_execution(method_name)
|
61
|
+
end
|
38
62
|
|
39
|
-
|
63
|
+
# Define in derived classes. Will be called immediately after handling an
|
64
|
+
# RPC call even if an error has occured.
|
65
|
+
def after_request
|
66
|
+
end
|
40
67
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
68
|
+
# Declared in derived classes. Will be called immediately after an
|
69
|
+
# exception has occurred when processing a request. Any excepions generated
|
70
|
+
# in this method call are suppressed and ignored to avoid being caught
|
71
|
+
# in a loop.
|
72
|
+
def after_exception(e)
|
73
|
+
end
|
50
74
|
|
51
|
-
|
52
|
-
|
53
|
-
|
75
|
+
# Declared in derived classes. Will be called immediately before the worker
|
76
|
+
# is closed down.
|
77
|
+
def before_close
|
78
|
+
end
|
54
79
|
|
55
|
-
|
56
|
-
|
57
|
-
|
80
|
+
# Declared in derived classes. Will be called immediately after the worker
|
81
|
+
# has been closed down.
|
82
|
+
def after_close
|
83
|
+
end
|
58
84
|
|
59
|
-
|
60
|
-
|
85
|
+
def close(delete_queue: false)
|
86
|
+
self.before_close
|
61
87
|
|
62
|
-
|
63
|
-
|
64
|
-
else
|
65
|
-
thread.wakeup
|
66
|
-
end
|
67
|
-
end
|
88
|
+
@operations.each do |meta|
|
89
|
+
subscriber = meta[:subscriber]
|
68
90
|
|
69
|
-
|
70
|
-
|
71
|
-
end
|
91
|
+
if (subscriber.respond_to?(:gracefully_shut_down))
|
92
|
+
subscriber.gracefully_shut_down
|
72
93
|
end
|
94
|
+
|
95
|
+
thread = meta[:thread]
|
96
|
+
|
97
|
+
thread.respond_to?(:terminate!) ? thread.terminate! : thread.kill
|
98
|
+
thread.join
|
73
99
|
end
|
74
100
|
|
75
|
-
|
76
|
-
|
101
|
+
if (delete_queue)
|
102
|
+
# The connection may have been terminated, so reconnect and delete
|
103
|
+
# the queue if necessary.
|
104
|
+
channel = @connection.create_channel
|
77
105
|
|
78
|
-
|
79
|
-
# be performed after initialization
|
80
|
-
def after_initialize
|
81
|
-
end
|
106
|
+
channel.queue(@queue_name, durable: @durable).delete
|
82
107
|
|
83
|
-
|
84
|
-
@threads.each do |thread|
|
85
|
-
thread.kill
|
86
|
-
thread.join
|
108
|
+
channel.close
|
87
109
|
end
|
88
110
|
|
89
|
-
super
|
111
|
+
super()
|
112
|
+
|
113
|
+
self.after_close
|
90
114
|
end
|
91
115
|
|
92
116
|
def join
|
93
|
-
@
|
94
|
-
thread.join
|
117
|
+
@operations.each do |meta|
|
118
|
+
meta[:thread].join
|
95
119
|
end
|
96
120
|
end
|
97
121
|
|
@@ -100,4 +124,138 @@ class Skein::Client::Worker < Skein::Connected
|
|
100
124
|
# callback-style delegation.
|
101
125
|
false
|
102
126
|
end
|
127
|
+
|
128
|
+
# Signal that the current operation should be abandoned and not retried.
|
129
|
+
def reject!
|
130
|
+
raise RejectMessage
|
131
|
+
end
|
132
|
+
|
133
|
+
# Signal that the current operation should be abandoned and retried later.
|
134
|
+
def retry!
|
135
|
+
raise RetryMessage
|
136
|
+
end
|
137
|
+
|
138
|
+
protected
|
139
|
+
def state_tracker
|
140
|
+
{
|
141
|
+
method: nil,
|
142
|
+
started: nil,
|
143
|
+
finished: nil
|
144
|
+
}
|
145
|
+
end
|
146
|
+
|
147
|
+
def metrics_tracker
|
148
|
+
Hash.new(0).merge(
|
149
|
+
time: 0.0,
|
150
|
+
errors: Hash.new(0)
|
151
|
+
)
|
152
|
+
end
|
153
|
+
|
154
|
+
def in_thread
|
155
|
+
@operations << {
|
156
|
+
thread: Thread.new do
|
157
|
+
Thread.abort_on_exception = true
|
158
|
+
|
159
|
+
yield
|
160
|
+
end
|
161
|
+
}
|
162
|
+
end
|
163
|
+
|
164
|
+
def establish_queue!(channel)
|
165
|
+
queue = channel.queue(
|
166
|
+
@queue_name,
|
167
|
+
durable: @durable,
|
168
|
+
auto_delete: @auto_delete
|
169
|
+
)
|
170
|
+
|
171
|
+
if (@exchange_name&.match(/\S/))
|
172
|
+
exchange = channel.direct(@exchange_name, durable: true)
|
173
|
+
|
174
|
+
queue.bind(exchange, routing_key: @routing_key || @queue_name)
|
175
|
+
end
|
176
|
+
|
177
|
+
queue
|
178
|
+
end
|
179
|
+
|
180
|
+
def establish_subscriber!(channel, meta)
|
181
|
+
queue = self.establish_queue!(channel)
|
182
|
+
|
183
|
+
meta[:subscriber] = Skein::Adapter.subscribe(queue) do |payload, delivery_tag, reply_to|
|
184
|
+
if (ENV['SKEIN_DEBUG_JSON'] and reply_to)
|
185
|
+
$stdout.puts('%s -> %s' % [ reply_to, payload ])
|
186
|
+
end
|
187
|
+
|
188
|
+
self.context.trap do
|
189
|
+
self.before_request rescue nil
|
190
|
+
|
191
|
+
handler.handle(payload, meta[:metrics], meta[:state]) do |reply_json|
|
192
|
+
if (ENV['SKEIN_DEBUG_JSON'] and reply_to)
|
193
|
+
$stdout.puts('%s <- %s' % [ reply_to, reply_json ])
|
194
|
+
end
|
195
|
+
|
196
|
+
# Secondary (inner) trap required since some handlers are async
|
197
|
+
self.context.trap do
|
198
|
+
# NOTE: begin...end necessary for rescue in Ruby versions below 2.4
|
199
|
+
begin
|
200
|
+
channel.acknowledge(delivery_tag, true)
|
201
|
+
|
202
|
+
if (reply_to)
|
203
|
+
channel.default_exchange.publish(
|
204
|
+
reply_json,
|
205
|
+
routing_key: reply_to,
|
206
|
+
content_type: 'application/json'
|
207
|
+
)
|
208
|
+
end
|
209
|
+
|
210
|
+
rescue RejectMessage
|
211
|
+
# Reject the message
|
212
|
+
channel.reject(delivery_tag, false)
|
213
|
+
rescue RetryMessage
|
214
|
+
# Reject and requeue the message
|
215
|
+
channel.reject(delivery_tag, true)
|
216
|
+
rescue => e
|
217
|
+
self.after_exception(e) rescue nil
|
218
|
+
raise e
|
219
|
+
ensure
|
220
|
+
self.after_request rescue nil
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
def with_channel_in_thread(recover: true, name: nil)
|
229
|
+
meta = {
|
230
|
+
metrics: metrics_tracker,
|
231
|
+
state: state_tracker
|
232
|
+
}
|
233
|
+
|
234
|
+
@operations << meta
|
235
|
+
|
236
|
+
meta[:thread] = Thread.new do
|
237
|
+
Thread.abort_on_exception = true
|
238
|
+
Thread.current.name = name
|
239
|
+
|
240
|
+
begin
|
241
|
+
channel = meta[:channel] = self.create_channel
|
242
|
+
|
243
|
+
yield(channel, meta)
|
244
|
+
|
245
|
+
channel.close rescue nil
|
246
|
+
|
247
|
+
redo if (recover)
|
248
|
+
|
249
|
+
ensure
|
250
|
+
# NOTE: The `.close` call may fail for a variety of reasons, but the
|
251
|
+
# important thing here is an attempt is made, regardless of
|
252
|
+
# outcome.
|
253
|
+
channel&.close rescue nil
|
254
|
+
end
|
255
|
+
end
|
256
|
+
end
|
257
|
+
|
258
|
+
def handler
|
259
|
+
@handler ||= Skein::Handler.for(self)
|
260
|
+
end
|
103
261
|
end
|
data/lib/skein/client.rb
CHANGED
@@ -24,38 +24,47 @@ class Skein::Client < Skein::Connected
|
|
24
24
|
|
25
25
|
# == Instance Methods =====================================================
|
26
26
|
|
27
|
-
def initialize(connection: nil, context: nil)
|
28
|
-
super(connection: connection, context: context)
|
27
|
+
def initialize(config: nil, connection: nil, context: nil)
|
28
|
+
super(config: config, connection: connection, context: context)
|
29
29
|
end
|
30
30
|
|
31
|
-
def rpc(exchange_name = nil, routing_key: nil)
|
31
|
+
def rpc(exchange_name = nil, routing_key: nil, ident: nil, expiration: nil, persistent: nil, durable: nil, timeout: nil)
|
32
32
|
Skein::Client::RPC.new(
|
33
33
|
exchange_name,
|
34
34
|
routing_key: routing_key,
|
35
35
|
connection: self.connection,
|
36
|
-
context: self.context
|
36
|
+
context: self.context,
|
37
|
+
ident: ident,
|
38
|
+
expiration: expiration,
|
39
|
+
persistent: persistent,
|
40
|
+
durable: durable,
|
41
|
+
timeout: timeout
|
37
42
|
)
|
38
43
|
end
|
39
44
|
|
40
|
-
def worker(queue_name)
|
41
|
-
Skein::Client::Worker.new(
|
45
|
+
def worker(queue_name, type = nil, ident: nil, durable: nil)
|
46
|
+
(type || Skein::Client::Worker).new(
|
42
47
|
queue_name,
|
43
48
|
connection: self.connection,
|
44
|
-
context: self.context
|
49
|
+
context: self.context,
|
50
|
+
ident: ident,
|
51
|
+
durable: durable
|
45
52
|
)
|
46
53
|
end
|
47
54
|
|
48
|
-
def publisher(
|
55
|
+
def publisher(exchange_name, type: nil, durable: nil)
|
49
56
|
Skein::Client::Publisher.new(
|
50
|
-
|
57
|
+
exchange_name,
|
58
|
+
type: type,
|
59
|
+
durable: durable,
|
51
60
|
connection: self.connection,
|
52
61
|
context: self.context
|
53
62
|
)
|
54
63
|
end
|
55
64
|
|
56
|
-
def subscriber(
|
65
|
+
def subscriber(exchange_name, routing_key = nil)
|
57
66
|
Skein::Client::Subscriber.new(
|
58
|
-
|
67
|
+
exchange_name,
|
59
68
|
routing_key,
|
60
69
|
connection: self.connection,
|
61
70
|
context: self.context
|
data/lib/skein/config.rb
CHANGED
@@ -87,7 +87,9 @@ class Skein::Config < OpenStruct
|
|
87
87
|
end
|
88
88
|
|
89
89
|
if (config_path and File.exist?(config_path))
|
90
|
-
super(DEFAULTS.merge(
|
90
|
+
super(DEFAULTS.merge(
|
91
|
+
YAML.load_file(config_path, aliases: true)[self.class.env] || { }
|
92
|
+
))
|
91
93
|
else
|
92
94
|
super(DEFAULTS)
|
93
95
|
end
|
data/lib/skein/connected.rb
CHANGED
@@ -4,19 +4,25 @@ class Skein::Connected
|
|
4
4
|
attr_reader :context
|
5
5
|
attr_reader :ident
|
6
6
|
attr_reader :connection
|
7
|
-
attr_reader :channel
|
8
7
|
|
9
8
|
# == Instance Methods =====================================================
|
10
9
|
|
11
|
-
def initialize(connection: nil, context: nil)
|
10
|
+
def initialize(config: nil, connection: nil, context: nil, ident: nil)
|
12
11
|
@mutex = Mutex.new
|
13
|
-
@shared_connection = !!connection
|
14
12
|
|
15
|
-
@
|
16
|
-
@
|
13
|
+
@config = config
|
14
|
+
@connection_shared = !connection
|
15
|
+
@connection = connection
|
16
|
+
|
17
|
+
self.connect
|
18
|
+
@channels = [ ]
|
17
19
|
|
18
20
|
@context = context || Skein::Context.new
|
19
|
-
@ident = @context.ident(self)
|
21
|
+
@ident = ident || @context.ident(self)
|
22
|
+
end
|
23
|
+
|
24
|
+
def connection_shared?
|
25
|
+
@connection_shared
|
20
26
|
end
|
21
27
|
|
22
28
|
def lock
|
@@ -25,26 +31,91 @@ class Skein::Connected
|
|
25
31
|
end
|
26
32
|
end
|
27
33
|
|
34
|
+
def repeat_until_not_nil(delay: 1.0)
|
35
|
+
r = nil
|
36
|
+
|
37
|
+
loop do
|
38
|
+
r = yield
|
39
|
+
|
40
|
+
break if r
|
41
|
+
|
42
|
+
sleep(delay)
|
43
|
+
end
|
44
|
+
|
45
|
+
r
|
46
|
+
end
|
47
|
+
|
48
|
+
def connect
|
49
|
+
@connection ||= repeat_until_not_nil do
|
50
|
+
@connection_shared = false
|
51
|
+
Skein::RabbitMQ.connect(@config)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def reconnect
|
56
|
+
@connection = nil
|
57
|
+
|
58
|
+
self.connect
|
59
|
+
end
|
60
|
+
|
61
|
+
def create_channel(auto_retry: false)
|
62
|
+
channel = begin
|
63
|
+
@connection.create_channel
|
64
|
+
|
65
|
+
rescue RuntimeError
|
66
|
+
sleep(1)
|
67
|
+
|
68
|
+
self.reconnect
|
69
|
+
|
70
|
+
retry
|
71
|
+
end
|
72
|
+
|
73
|
+
if (channel.respond_to?(:prefetch=))
|
74
|
+
channel.prefetch = 1
|
75
|
+
else
|
76
|
+
channel.prefetch(1)
|
77
|
+
end
|
78
|
+
|
79
|
+
@channels << channel
|
80
|
+
|
81
|
+
channel
|
82
|
+
end
|
83
|
+
|
84
|
+
def channel
|
85
|
+
@channel ||= self.create_channel
|
86
|
+
end
|
87
|
+
|
28
88
|
def close
|
29
89
|
lock do
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
90
|
+
@channels.each do |channel|
|
91
|
+
begin
|
92
|
+
channel.close
|
93
|
+
|
94
|
+
rescue => e
|
95
|
+
if (defined?(MarchHare))
|
96
|
+
case (e)
|
97
|
+
when MarchHare::ChannelLevelException, MarchHare::ChannelAlreadyClosed
|
98
|
+
# Ignored since we're finished with the channel anyway
|
99
|
+
else
|
100
|
+
raise e
|
101
|
+
end
|
102
|
+
elsif (defined?(Bunny))
|
103
|
+
case (e)
|
104
|
+
when Bunny::ChannelAlreadyClosed
|
105
|
+
# Ignored since we're finished with the channel anyway
|
106
|
+
else
|
107
|
+
raise e
|
108
|
+
end
|
38
109
|
else
|
39
110
|
raise e
|
40
111
|
end
|
41
112
|
end
|
42
113
|
end
|
43
114
|
|
44
|
-
@
|
115
|
+
@channels = [ ]
|
45
116
|
|
46
|
-
unless (@
|
47
|
-
@connection
|
117
|
+
unless (@connection_shared)
|
118
|
+
@connection&.close
|
48
119
|
@connection = nil
|
49
120
|
end
|
50
121
|
end
|