queue_classic_plus 4.0.0.alpha17 → 4.0.0.alpha19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +53 -0
- data/lib/queue_classic_plus/base.rb +62 -1
- data/lib/queue_classic_plus/queue_classic/queue.rb +0 -20
- data/lib/queue_classic_plus/version.rb +1 -1
- data/lib/queue_classic_plus.rb +4 -8
- data/spec/base_spec.rb +175 -55
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4ded39f47849c6dacb9e4efe8bc535bf9fae56cf15cc366cdde177e12176ae37
|
4
|
+
data.tar.gz: 228a9d4ed7b5a4541e1d93e9887c3f3ec9193e9464c9766817ffd1690530d864
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 447ed6984427a1649f76ed733c2eafcf6324a5591265cd3f3ec9d29ca8f09552c2a93134b3a481ab06fd1d64b1f78db405fd4ade10e1b372eb62b6e8cbdfe364
|
7
|
+
data.tar.gz: c4b1823ae678c44d3d0892ff2bb44dc6ec844339717e1a8e0ce62ff564aee690e3f5d19c9ab254c6538ce2e3e4466c1789bc5b50beba787e49a8af22d60f892f
|
data/README.md
CHANGED
@@ -124,6 +124,59 @@ class Jobs::NoTransaction < QueueClassicPlus::Base
|
|
124
124
|
end
|
125
125
|
```
|
126
126
|
|
127
|
+
|
128
|
+
#### Callbacks
|
129
|
+
|
130
|
+
Jobs support the following callbacks:
|
131
|
+
```
|
132
|
+
before_enqueue
|
133
|
+
after_enqueue
|
134
|
+
before_perform
|
135
|
+
after_perform
|
136
|
+
```
|
137
|
+
|
138
|
+
- `enqueue` callbacks are called when the job is initially enqueued. Caveats:
|
139
|
+
- not called on retries
|
140
|
+
- not called when scheduled in the future (i.e. `Job.enqueue_perform_in`)
|
141
|
+
- still called if enqueuing fails because a job with `lock!` is already enqueued.
|
142
|
+
- `perform` callbacks are called any time the job is picked up and run by a worker, including retries and jobs scheduled in the future.
|
143
|
+
|
144
|
+
Callbacks can either be implemented by providing a method to be called:
|
145
|
+
|
146
|
+
```ruby
|
147
|
+
class Jobs::WithCallbackMethods < QueueClassicPlus::Base
|
148
|
+
before_enqueue :before_enqueue_method
|
149
|
+
|
150
|
+
@queue = :low
|
151
|
+
|
152
|
+
def self.before_enqueue_method(*args)
|
153
|
+
# ...
|
154
|
+
end
|
155
|
+
|
156
|
+
def self.perform(user_id)
|
157
|
+
# ...
|
158
|
+
end
|
159
|
+
end
|
160
|
+
```
|
161
|
+
|
162
|
+
or by providing a block:
|
163
|
+
|
164
|
+
```ruby
|
165
|
+
class Jobs::WithCallbackBlocks < QueueClassicPlus::Base
|
166
|
+
before_enqueue do |*args|
|
167
|
+
# ...
|
168
|
+
end
|
169
|
+
|
170
|
+
@queue = :low
|
171
|
+
|
172
|
+
def self.perform(user_id)
|
173
|
+
# ...
|
174
|
+
end
|
175
|
+
end
|
176
|
+
```
|
177
|
+
|
178
|
+
The `args` passed to the callback method/block will be the same as the arguments passed to `QueueClassicPlus::Base.do` and `QueueClassicPlus::Base.perform`.
|
179
|
+
|
127
180
|
## Advanced configuration
|
128
181
|
|
129
182
|
If you want to log exceptions in your favorite exception tracker. You can configured it like so:
|
@@ -52,8 +52,32 @@ module QueueClassicPlus
|
|
52
52
|
QueueClassicPlus.logger
|
53
53
|
end
|
54
54
|
|
55
|
+
def self.can_enqueue?(method, *args)
|
56
|
+
if locked?
|
57
|
+
max_lock_time = ENV.fetch("QUEUE_CLASSIC_MAX_LOCK_TIME", 10 * 60).to_i
|
58
|
+
|
59
|
+
q = "SELECT COUNT(1) AS count
|
60
|
+
FROM
|
61
|
+
(
|
62
|
+
SELECT 1
|
63
|
+
FROM queue_classic_jobs
|
64
|
+
WHERE q_name = $1 AND method = $2 AND args::text = $3::text
|
65
|
+
AND (locked_at IS NULL OR locked_at > current_timestamp - interval '#{max_lock_time} seconds')
|
66
|
+
LIMIT 1
|
67
|
+
)
|
68
|
+
AS x"
|
69
|
+
|
70
|
+
result = QC.default_conn_adapter.execute(q, @queue, method, JSON.dump(serialized(args)))
|
71
|
+
result['count'].to_i == 0
|
72
|
+
else
|
73
|
+
true
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
55
77
|
def self.enqueue(method, *args)
|
56
|
-
|
78
|
+
if can_enqueue?(method, *args)
|
79
|
+
queue.enqueue(method, *serialized(args))
|
80
|
+
end
|
57
81
|
end
|
58
82
|
|
59
83
|
def self.enqueue_perform(*args)
|
@@ -112,6 +136,22 @@ module QueueClassicPlus
|
|
112
136
|
end
|
113
137
|
end
|
114
138
|
|
139
|
+
def self.before_enqueue(callback_method = nil, &block)
|
140
|
+
define_callback(:enqueue, :before, callback_method, &block)
|
141
|
+
end
|
142
|
+
|
143
|
+
def self.after_enqueue(callback_method = nil, &block)
|
144
|
+
define_callback(:enqueue, :after, callback_method, &block)
|
145
|
+
end
|
146
|
+
|
147
|
+
def self.before_perform(callback_method = nil, &block)
|
148
|
+
define_callback(:perform, :before, callback_method, &block)
|
149
|
+
end
|
150
|
+
|
151
|
+
def self.after_perform(callback_method = nil, &block)
|
152
|
+
define_callback(:perform, :after, callback_method, &block)
|
153
|
+
end
|
154
|
+
|
115
155
|
# Debugging
|
116
156
|
def self.list
|
117
157
|
q = "SELECT * FROM queue_classic_jobs WHERE q_name = '#{@queue}'"
|
@@ -141,5 +181,26 @@ module QueueClassicPlus
|
|
141
181
|
def self.execute(sql, *args)
|
142
182
|
QC.default_conn_adapter.execute(sql, *args)
|
143
183
|
end
|
184
|
+
|
185
|
+
def self.define_callback(name, type, callback_method, &_block)
|
186
|
+
singleton_class.prepend(
|
187
|
+
Module.new do |callback_module|
|
188
|
+
callback_module.define_method(name) do |*args|
|
189
|
+
callback_args = args
|
190
|
+
if name == :enqueue
|
191
|
+
callback_args = callback_args.drop(1) # Class/method is the first argument. Callbacks don't need that.
|
192
|
+
end
|
193
|
+
|
194
|
+
if type == :before
|
195
|
+
block_given? ? yield(*callback_args) : send(callback_method, *callback_args)
|
196
|
+
end
|
197
|
+
super(*args)
|
198
|
+
if type == :after
|
199
|
+
block_given? ? yield(*callback_args) : send(callback_method, *callback_args)
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
)
|
204
|
+
end
|
144
205
|
end
|
145
206
|
end
|
@@ -50,25 +50,5 @@ module QC
|
|
50
50
|
end
|
51
51
|
end
|
52
52
|
|
53
|
-
def enqueue(method, *args, lock: false)
|
54
|
-
QC.log_yield(:measure => 'queue.enqueue') do
|
55
|
-
insert_sql = <<-EOF
|
56
|
-
INSERT INTO #{QC.table_name} (q_name, method, args, lock)
|
57
|
-
VALUES ($1, $2, $3, $4)
|
58
|
-
ON CONFLICT (q_name, method, args) WHERE lock IS TRUE DO NOTHING
|
59
|
-
RETURNING id
|
60
|
-
EOF
|
61
|
-
begin
|
62
|
-
retries ||= 0
|
63
|
-
conn_adapter.execute(insert_sql, name, method, JSON.dump(args), lock)
|
64
|
-
rescue PG::Error => error
|
65
|
-
if (retries += 1) < 2
|
66
|
-
retry
|
67
|
-
else
|
68
|
-
raise
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
53
|
end
|
74
54
|
end
|
data/lib/queue_classic_plus.rb
CHANGED
@@ -15,18 +15,14 @@ module QueueClassicPlus
|
|
15
15
|
|
16
16
|
def self.migrate(c = QC::default_conn_adapter.connection)
|
17
17
|
conn = QC::ConnAdapter.new(connection: c)
|
18
|
-
conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN
|
19
|
-
conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN
|
20
|
-
conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN IF NOT EXISTS lock BOOLEAN NOT NULL DEFAULT FALSE")
|
21
|
-
conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS index_queue_classic_jobs_enqueue_lock on queue_classic_jobs(q_name, method, args) WHERE lock IS TRUE")
|
18
|
+
conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN last_error TEXT")
|
19
|
+
conn.execute("ALTER TABLE queue_classic_jobs ADD COLUMN remaining_retries INTEGER")
|
22
20
|
end
|
23
21
|
|
24
22
|
def self.demigrate(c = QC::default_conn_adapter.connection)
|
25
23
|
conn = QC::ConnAdapter.new(connection: c)
|
26
|
-
conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN
|
27
|
-
conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN
|
28
|
-
conn.execute("DROP INDEX IF EXISTS index_queue_classic_jobs_enqueue_lock")
|
29
|
-
conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN IF EXISTS lock")
|
24
|
+
conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN last_error")
|
25
|
+
conn.execute("ALTER TABLE queue_classic_jobs DROP COLUMN remaining_retries")
|
30
26
|
end
|
31
27
|
|
32
28
|
def self.exception_handler
|
data/spec/base_spec.rb
CHANGED
@@ -3,24 +3,6 @@ require 'active_record'
|
|
3
3
|
|
4
4
|
describe QueueClassicPlus::Base do
|
5
5
|
context "A child of QueueClassicPlus::Base" do
|
6
|
-
subject do
|
7
|
-
Class.new(QueueClassicPlus::Base) do
|
8
|
-
@queue = :test
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
it "allows multiple enqueues" do
|
13
|
-
threads = []
|
14
|
-
10.times do
|
15
|
-
threads << Thread.new do
|
16
|
-
subject.do
|
17
|
-
end
|
18
|
-
end
|
19
|
-
threads.each(&:join)
|
20
|
-
|
21
|
-
expect(subject).to have_queue_size_of(10)
|
22
|
-
end
|
23
|
-
|
24
6
|
context "that is locked" do
|
25
7
|
subject do
|
26
8
|
Class.new(QueueClassicPlus::Base) do
|
@@ -30,28 +12,9 @@ describe QueueClassicPlus::Base do
|
|
30
12
|
end
|
31
13
|
|
32
14
|
it "does not allow multiple enqueues" do
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
subject.do
|
37
|
-
expect(subject).to have_queue_size_of(1)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
threads.each(&:join)
|
41
|
-
end
|
42
|
-
|
43
|
-
it "allows enqueueing same job with different arguments" do
|
44
|
-
threads = []
|
45
|
-
(1..3).each do |arg|
|
46
|
-
10.times do
|
47
|
-
threads << Thread.new do
|
48
|
-
subject.do(arg)
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
threads.each(&:join)
|
53
|
-
|
54
|
-
expect(subject).to have_queue_size_of(3)
|
15
|
+
subject.do
|
16
|
+
subject.do
|
17
|
+
expect(subject).to have_queue_size_of(1)
|
55
18
|
end
|
56
19
|
|
57
20
|
it "checks for an existing job using the same serializing as job enqueuing" do
|
@@ -65,22 +28,13 @@ describe QueueClassicPlus::Base do
|
|
65
28
|
subject.do(date)
|
66
29
|
expect(subject).to have_queue_size_of(1)
|
67
30
|
end
|
68
|
-
end
|
69
31
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
end
|
77
|
-
|
78
|
-
it "does not create another transaction when enqueueing" do
|
79
|
-
conn = QC.default_conn_adapter.connection
|
80
|
-
expect(conn).to receive(:transaction).exactly(1).times.and_call_original
|
81
|
-
conn.transaction do
|
82
|
-
subject.do
|
83
|
-
end
|
32
|
+
it "does allow multiple enqueues if something got locked for too long" do
|
33
|
+
subject.do
|
34
|
+
one_day_ago = Time.now - 60*60*24
|
35
|
+
execute "UPDATE queue_classic_jobs SET locked_at = '#{one_day_ago}' WHERE q_name = 'test'"
|
36
|
+
subject.do
|
37
|
+
expect(subject).to have_queue_size_of(2)
|
84
38
|
end
|
85
39
|
end
|
86
40
|
|
@@ -230,6 +184,172 @@ describe QueueClassicPlus::Base do
|
|
230
184
|
subject._perform(42, true)
|
231
185
|
end
|
232
186
|
end
|
187
|
+
|
188
|
+
context "with callbacks defined" do
|
189
|
+
subject do
|
190
|
+
Class.new(QueueClassicPlus::Base) do
|
191
|
+
@queue = :test
|
192
|
+
|
193
|
+
before_enqueue :before_enqueue_method
|
194
|
+
after_enqueue :after_enqueue_method
|
195
|
+
|
196
|
+
before_perform :before_perform_method
|
197
|
+
after_perform :after_perform_method
|
198
|
+
|
199
|
+
def self.before_enqueue_method(*_args); end;
|
200
|
+
def self.after_enqueue_method(*_args); end;
|
201
|
+
def self.before_perform_method(*_args); end;
|
202
|
+
def self.after_perform_method(*_args); end;
|
203
|
+
|
204
|
+
def self.perform(*_args); end;
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
it "passes enqueue arguments to callbacks" do
|
209
|
+
expect(subject).to receive(:before_enqueue_method).with("enqueue_argument").once
|
210
|
+
expect(subject).to receive(:after_enqueue_method).with("enqueue_argument").once
|
211
|
+
|
212
|
+
subject.do("enqueue_argument")
|
213
|
+
end
|
214
|
+
|
215
|
+
it "passes perform arguments to callbacks" do
|
216
|
+
expect(subject).to receive(:before_perform_method).with("perform_argument").once
|
217
|
+
expect(subject).to receive(:after_perform_method).with("perform_argument").once
|
218
|
+
|
219
|
+
subject.perform("perform_argument")
|
220
|
+
end
|
221
|
+
|
222
|
+
context "when enqueued" do
|
223
|
+
it "calls the enqueue callback methods" do
|
224
|
+
expect(subject).to receive(:before_enqueue_method).once
|
225
|
+
expect(subject).to receive(:after_enqueue_method).once
|
226
|
+
|
227
|
+
subject.do
|
228
|
+
end
|
229
|
+
|
230
|
+
it "does not call the perform callbacks" do
|
231
|
+
expect(subject).to_not receive(:before_perform_method)
|
232
|
+
expect(subject).to_not receive(:after_perform_method)
|
233
|
+
|
234
|
+
subject.do
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
context "when perform" do
|
239
|
+
it "calls the perform callback methods" do
|
240
|
+
expect(subject).to receive(:before_perform_method).once
|
241
|
+
expect(subject).to receive(:after_perform_method).once
|
242
|
+
|
243
|
+
subject.perform
|
244
|
+
end
|
245
|
+
|
246
|
+
it "does not call the enqueue callbacks" do
|
247
|
+
expect(subject).to_not receive(:before_enqueue_method)
|
248
|
+
expect(subject).to_not receive(:after_enqueue_method)
|
249
|
+
|
250
|
+
subject.perform
|
251
|
+
end
|
252
|
+
|
253
|
+
context "when perform fails" do
|
254
|
+
subject do
|
255
|
+
Class.new(QueueClassicPlus::Base) do
|
256
|
+
@queue = :test
|
257
|
+
|
258
|
+
before_perform :before_perform_method
|
259
|
+
after_perform :after_perform_method
|
260
|
+
|
261
|
+
def self.before_perform_method(*_args); end;
|
262
|
+
def self.after_perform_method(*_args); end;
|
263
|
+
|
264
|
+
def self.perform(*_args)
|
265
|
+
raise StandardError
|
266
|
+
end
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
it "does not call after callbacks" do
|
271
|
+
expect(subject).to receive(:before_perform_method).once
|
272
|
+
expect(subject).to_not receive(:after_perform_method)
|
273
|
+
|
274
|
+
begin
|
275
|
+
subject.perform
|
276
|
+
rescue StandardError
|
277
|
+
end
|
278
|
+
end
|
279
|
+
end
|
280
|
+
end
|
281
|
+
|
282
|
+
context "when callback defined multiple times" do
|
283
|
+
subject do
|
284
|
+
Class.new(QueueClassicPlus::Base) do
|
285
|
+
@queue = :test
|
286
|
+
|
287
|
+
before_enqueue :before_enqueue_method_1
|
288
|
+
before_enqueue :before_enqueue_method_2
|
289
|
+
before_enqueue :before_enqueue_method_3
|
290
|
+
|
291
|
+
def self.before_enqueue_method_1(*_args); end;
|
292
|
+
def self.before_enqueue_method_2(*_args); end;
|
293
|
+
def self.before_enqueue_method_3(*_args); end;
|
294
|
+
|
295
|
+
def self.perform(*_args); end;
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
it "calls each callback" do
|
300
|
+
expect(subject).to receive(:before_enqueue_method_1).once
|
301
|
+
expect(subject).to receive(:before_enqueue_method_2).once
|
302
|
+
expect(subject).to receive(:before_enqueue_method_3).once
|
303
|
+
|
304
|
+
subject.do
|
305
|
+
end
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
context "with callback blocks defined" do
|
310
|
+
subject do
|
311
|
+
Class.new(QueueClassicPlus::Base) do
|
312
|
+
@queue = :test
|
313
|
+
class_variable_set(:@@block_result, [])
|
314
|
+
|
315
|
+
before_enqueue do |*_args|
|
316
|
+
class_variable_get(:@@block_result).append("before_enqueue_block")
|
317
|
+
end
|
318
|
+
after_enqueue do |*_args|
|
319
|
+
class_variable_get(:@@block_result).append("after_enqueue_block")
|
320
|
+
end
|
321
|
+
|
322
|
+
before_perform do |*_args|
|
323
|
+
class_variable_get(:@@block_result).append("before_perform_block")
|
324
|
+
end
|
325
|
+
after_perform do |*_args|
|
326
|
+
class_variable_get(:@@block_result).append("after_perform_block")
|
327
|
+
end
|
328
|
+
|
329
|
+
def self.perform; end;
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
context "when enqueued" do
|
334
|
+
it "calls the enqueue callback blocks" do
|
335
|
+
subject.do
|
336
|
+
|
337
|
+
expect(subject.class_variable_get(:@@block_result)).to eq(
|
338
|
+
%w(before_enqueue_block after_enqueue_block)
|
339
|
+
)
|
340
|
+
end
|
341
|
+
end
|
342
|
+
|
343
|
+
context "when perform" do
|
344
|
+
it "calls the perform callback blocks" do
|
345
|
+
subject.perform
|
346
|
+
|
347
|
+
expect(subject.class_variable_get(:@@block_result)).to eq(
|
348
|
+
%w(before_perform_block after_perform_block)
|
349
|
+
)
|
350
|
+
end
|
351
|
+
end
|
352
|
+
end
|
233
353
|
end
|
234
354
|
|
235
355
|
describe ".librato_key" do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: queue_classic_plus
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0.0.
|
4
|
+
version: 4.0.0.alpha19
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Simon Mathieu
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2023-
|
13
|
+
date: 2023-09-10 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: queue_classic
|