backburner-allq 1.0.36 → 1.0.39
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +21 -0
- data/deploy.sh +1 -1
- data/docker-compose.yml +26 -0
- data/lib/backburner/allq_wrapper.rb +24 -19
- data/lib/backburner/configuration.rb +1 -1
- data/lib/backburner/connection.rb +9 -0
- data/lib/backburner/job.rb +9 -1
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +34 -31
- data/test/allq/back_burner_test.rb +74 -0
- data/test/allq/test_helper.rb +108 -0
- data/test/allq/worker_test.rb +82 -0
- data/test/fixtures/test_jobs.rb +12 -0
- data/test/worker_test.rb +0 -2
- metadata +10 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a737b01e7869afbe98e32059d9f28bd7b9ad4ccb9b763054dccb90f9062544dd
|
4
|
+
data.tar.gz: '00858748b53b6aab0bed26afef647ab3611e90df5a27b41d261c63943a711e60'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3eec6cbbac812b151e230e2f4aedcf6106651225aac47ba0b928a4c38eec4e59bc5f7a186a496e2ce7fb36381b8d369f121b72ee88341b742b9edc0f94570f5e
|
7
|
+
data.tar.gz: 73272508ddf5fa6a6f34685380d03d5b4d17a16d2e3889e60f1bb43d9e3afa492a715f231359404c36ba07bf328868b77ee392b16264c7940968bd0eecef470b
|
data/.gitlab-ci.yml
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
default:
|
2
|
+
image: ruby:3.0
|
3
|
+
|
4
|
+
stages: # List of stages for jobs, and their order of execution
|
5
|
+
- test
|
6
|
+
|
7
|
+
unit-test:
|
8
|
+
tags:
|
9
|
+
- allq
|
10
|
+
stage: test
|
11
|
+
script:
|
12
|
+
- docker-compose down || true
|
13
|
+
- docker-compose up -d || true
|
14
|
+
- sleep 10
|
15
|
+
- ruby -v
|
16
|
+
- bundle install
|
17
|
+
- bundle exec rake test TEST=test/allq/back_burner_test.rb DEBUG=true
|
18
|
+
- bundle exec rake test TEST=test/allq/worker_test.rb DEBUG=true
|
19
|
+
after_script:
|
20
|
+
- docker-compose down || true
|
21
|
+
|
data/deploy.sh
CHANGED
data/docker-compose.yml
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
version: '3'
|
2
|
+
services:
|
3
|
+
allq-client:
|
4
|
+
networks:
|
5
|
+
- local-net
|
6
|
+
image: "blitline/allq_client"
|
7
|
+
environment:
|
8
|
+
- SERVER_STRING=allq-server:5555
|
9
|
+
depends_on:
|
10
|
+
- allq-server
|
11
|
+
ports:
|
12
|
+
- 8090:8090
|
13
|
+
restart: unless-stopped
|
14
|
+
healthcheck:
|
15
|
+
test: ["CMD-SHELL", "echo '{\"action\":\"peek\",\"params\":{\"tube\":\"default\", \"buried\":true}}' | socat - tcp4-connect:127.0.0.1:7768"]
|
16
|
+
interval: 1m
|
17
|
+
timeout: 10s
|
18
|
+
retries: 3
|
19
|
+
allq-server:
|
20
|
+
networks:
|
21
|
+
- local-net
|
22
|
+
image: "blitline/allq_server"
|
23
|
+
restart: unless-stopped
|
24
|
+
networks:
|
25
|
+
local-net:
|
26
|
+
|
@@ -42,13 +42,17 @@ module Backburner
|
|
42
42
|
# Release count
|
43
43
|
attr_accessor :releases
|
44
44
|
|
45
|
-
|
45
|
+
# Release count
|
46
|
+
attr_accessor :special
|
47
|
+
|
48
|
+
def initialize(wrapper, job_response)
|
46
49
|
@client = wrapper
|
47
|
-
@id =
|
48
|
-
@body =
|
49
|
-
@expireds =
|
50
|
-
@releases =
|
51
|
-
@tube =
|
50
|
+
@id = job_response.id
|
51
|
+
@body = job_response.body
|
52
|
+
@expireds = job_response.expireds
|
53
|
+
@releases = job_response.releases
|
54
|
+
@tube = job_response.tube.to_s
|
55
|
+
@special = job_response.special.to_s if job_response.special
|
52
56
|
end
|
53
57
|
|
54
58
|
def done
|
@@ -78,6 +82,10 @@ module Backburner
|
|
78
82
|
def stats
|
79
83
|
{ 'expireds' => expireds, 'releases' => releases }
|
80
84
|
end
|
85
|
+
|
86
|
+
def clear(tube)
|
87
|
+
@client.clear(tube)
|
88
|
+
end
|
81
89
|
end
|
82
90
|
|
83
91
|
class AllQWrapper
|
@@ -93,10 +101,8 @@ module Backburner
|
|
93
101
|
@recent_times = []
|
94
102
|
end
|
95
103
|
|
96
|
-
def
|
97
|
-
|
98
|
-
|
99
|
-
0
|
104
|
+
def clear(tube)
|
105
|
+
@client.tube_delete(tube)
|
100
106
|
end
|
101
107
|
|
102
108
|
def touch(job)
|
@@ -167,12 +173,6 @@ module Backburner
|
|
167
173
|
app_priority > 10 ? 5 : app_priority
|
168
174
|
end
|
169
175
|
|
170
|
-
def log_result(job_result)
|
171
|
-
puts("ALLQ-HTTP-JOB-ID=#{job_result.job_id}")
|
172
|
-
rescue StandardError => e
|
173
|
-
puts(e)
|
174
|
-
end
|
175
|
-
|
176
176
|
def build_new_job(body, options)
|
177
177
|
adjusted_priority = map_priority(options[:pri] || 5)
|
178
178
|
|
@@ -195,7 +195,6 @@ module Backburner
|
|
195
195
|
ttl = options[:ttl] || options[:ttr] || DEFAULT_TIMEOUT
|
196
196
|
tube_name = options[:tube_name] || 'default'
|
197
197
|
delay = options[:delay] || 0
|
198
|
-
parent_id = options[:parent_id]
|
199
198
|
limit = options[:limit]
|
200
199
|
timeout = options[:timeout] || 3_600
|
201
200
|
run_on_timeout = options[:run_on_timeout] || false
|
@@ -254,15 +253,21 @@ module Backburner
|
|
254
253
|
raw_stats = @admin.stats_get
|
255
254
|
final_stats = {}
|
256
255
|
|
257
|
-
raw_stats.
|
256
|
+
raw_stats.each_with_index do |agg, i|
|
258
257
|
agg.stats.each do |tube_ref|
|
259
258
|
name = tube_ref.tube
|
260
|
-
final_stats[name] = {} unless final_stats[name]
|
259
|
+
final_stats[name] = {'avg' => 0, 'tps' => 0} unless final_stats[name]
|
261
260
|
final_stats[name]['ready'] = final_stats[name]['ready'].to_i + tube_ref.ready.to_i
|
262
261
|
final_stats[name]['reserved'] = final_stats[name]['reserved'].to_i + tube_ref.reserved.to_i
|
263
262
|
final_stats[name]['delayed'] = final_stats[name]['delayed'].to_i + tube_ref.delayed.to_i
|
264
263
|
final_stats[name]['buried'] = final_stats[name]['buried'].to_i + tube_ref.buried.to_i
|
265
264
|
final_stats[name]['parents'] = final_stats[name]['parents'].to_i + tube_ref.parents.to_i
|
265
|
+
|
266
|
+
|
267
|
+
# Fancy math to calculate averages on the fly
|
268
|
+
# https://math.stackexchange.com/questions/106313/regular-average-calculated-accumulatively
|
269
|
+
final_stats[name]['avg'] += (tube_ref.avg.to_f - final_stats[name]['avg'].to_f)/(i+1).to_f
|
270
|
+
final_stats[name]['tps'] += (tube_ref.tps.to_f - final_stats[name]['tps'].to_f)/(i+1).to_f
|
266
271
|
end
|
267
272
|
end
|
268
273
|
final_stats
|
@@ -21,7 +21,7 @@ module Backburner
|
|
21
21
|
attr_accessor :job_parser_proc # proc to parse a job body from a string
|
22
22
|
|
23
23
|
def initialize
|
24
|
-
@allq_url = "allq://127.0.0.1:
|
24
|
+
@allq_url = "allq://127.0.0.1:8090"
|
25
25
|
@tube_namespace = "backburner"
|
26
26
|
@namespace_separator = "."
|
27
27
|
@default_priority = 5
|
@@ -23,6 +23,10 @@ module Backburner
|
|
23
23
|
connect!
|
24
24
|
end
|
25
25
|
|
26
|
+
def clear(tube)
|
27
|
+
@allq_wrapper.clear(tube)
|
28
|
+
end
|
29
|
+
|
26
30
|
# Close the connection, if it exists
|
27
31
|
def close
|
28
32
|
@allq_wrapper.close if @allq_wrapper
|
@@ -110,6 +114,11 @@ module Backburner
|
|
110
114
|
}
|
111
115
|
|
112
116
|
options[:shard_key] = opt[:shard_key] if opt[:shard_key]
|
117
|
+
options[:parent_id] = opt[:parent_id] if opt[:parent_id]
|
118
|
+
options[:timeout] = opt[:timeout] if opt[:timeout]
|
119
|
+
options[:run_on_timeout] = opt[:run_on_timeout] if opt[:run_on_timeout]
|
120
|
+
options[:limit] = opt[:limit] if opt[:limit]
|
121
|
+
options[:is_parent] = opt[:is_parent] if opt[:is_parent]
|
113
122
|
|
114
123
|
# Overwrite originals
|
115
124
|
opt.merge!(options)
|
data/lib/backburner/job.rb
CHANGED
@@ -49,7 +49,15 @@ module Backburner
|
|
49
49
|
return false unless res
|
50
50
|
# Execute the job
|
51
51
|
@hooks.around_hook_events(job_name, :around_perform, *args) do
|
52
|
-
|
52
|
+
|
53
|
+
timeout_job_after(@ttr > 1 ? @ttr - 1 : @ttr) do
|
54
|
+
# Decide if we want access to the root task
|
55
|
+
if job_class.methods.include?(:perform_with_task)
|
56
|
+
job_class.perform_with_task(@task, *args)
|
57
|
+
else
|
58
|
+
job_class.perform(*args)
|
59
|
+
end
|
60
|
+
end
|
53
61
|
end
|
54
62
|
task.delete
|
55
63
|
# Invoke after perform hook
|
data/lib/backburner/version.rb
CHANGED
data/lib/backburner/worker.rb
CHANGED
@@ -13,7 +13,10 @@ module Backburner
|
|
13
13
|
# List of known_queue_classes
|
14
14
|
class << self
|
15
15
|
attr_writer :known_queue_classes
|
16
|
-
|
16
|
+
|
17
|
+
def known_queue_classes
|
18
|
+
@known_queue_classes ||= []
|
19
|
+
end
|
17
20
|
end
|
18
21
|
|
19
22
|
# Enqueues a job to be processed later by a worker.
|
@@ -37,8 +40,8 @@ module Backburner
|
|
37
40
|
|
38
41
|
return nil unless res # stop if hook is false
|
39
42
|
|
40
|
-
data = { :
|
41
|
-
queue = opts[:queue] && (
|
43
|
+
data = { class: job_class.name, args: args, ttr: ttr }
|
44
|
+
queue = opts[:queue] && (opts[:queue].is_a?(Proc) ? opts[:queue].call(job_class) : opts[:queue])
|
42
45
|
|
43
46
|
begin
|
44
47
|
response = nil
|
@@ -67,12 +70,10 @@ module Backburner
|
|
67
70
|
# @example
|
68
71
|
# Backburner::Worker.start(["foo.tube.name"])
|
69
72
|
#
|
70
|
-
def self.start(tube_names=nil)
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
# do nothing
|
75
|
-
end
|
73
|
+
def self.start(tube_names = nil)
|
74
|
+
new(tube_names).start
|
75
|
+
rescue SystemExit
|
76
|
+
# do nothing
|
76
77
|
end
|
77
78
|
|
78
79
|
# List of tube names to be watched and processed
|
@@ -84,7 +85,7 @@ module Backburner
|
|
84
85
|
# Worker.new(['test.job'])
|
85
86
|
def initialize(tube_names = nil)
|
86
87
|
@connection = new_connection
|
87
|
-
@tube_names =
|
88
|
+
@tube_names = process_tube_names(tube_names)
|
88
89
|
register_signal_handlers!
|
89
90
|
end
|
90
91
|
|
@@ -140,31 +141,30 @@ module Backburner
|
|
140
141
|
# @raise [Beaneater::NotConnected] If beanstalk fails to connect multiple times.
|
141
142
|
def work_one_job(conn = connection, tube_name = nil)
|
142
143
|
if tube_name.nil?
|
143
|
-
|
144
|
-
tube_name = @tube_names.sample
|
144
|
+
log_error 'Sampling tube, this is bad practice for Allq'
|
145
|
+
tube_name = @tube_names.sample
|
145
146
|
end
|
146
|
-
|
147
|
+
|
147
148
|
begin
|
148
149
|
job = reserve_job(conn, tube_name)
|
149
150
|
rescue Exception => e
|
150
|
-
|
151
|
-
|
152
|
-
sleep(rand*3)
|
151
|
+
log_error "Exception: #{e.full_message}"
|
152
|
+
sleep(rand * 3)
|
153
153
|
return
|
154
154
|
end
|
155
155
|
|
156
156
|
if job && job.body
|
157
157
|
begin
|
158
|
-
|
158
|
+
log_job_begin(job.name, job.args)
|
159
159
|
job.process
|
160
|
-
|
160
|
+
log_job_end(job.name)
|
161
161
|
rescue Backburner::Job::JobFormatInvalid => e
|
162
|
-
|
163
|
-
rescue => e # Error occurred processing job
|
164
|
-
|
162
|
+
log_error exception_message(e)
|
163
|
+
rescue StandardError => e # Error occurred processing job
|
164
|
+
log_error exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
|
165
165
|
|
166
166
|
unless job
|
167
|
-
|
167
|
+
log_error 'Error occurred before we were able to assign a job. Giving up without retrying!'
|
168
168
|
return
|
169
169
|
end
|
170
170
|
|
@@ -172,26 +172,29 @@ module Backburner
|
|
172
172
|
# gone down between the time we reserved / processed the job and here.
|
173
173
|
num_retries = job.releases
|
174
174
|
max_job_retries = resolve_max_job_retries(job.job_class)
|
175
|
-
retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
|
175
|
+
retry_status = "failed: attempt #{num_retries + 1} of #{max_job_retries + 1}"
|
176
176
|
retry_delay = resolve_retry_delay(job.job_class)
|
177
|
-
delay =
|
178
|
-
|
177
|
+
delay = begin
|
178
|
+
resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries)
|
179
|
+
rescue StandardError
|
180
|
+
retry_delay
|
181
|
+
end
|
182
|
+
|
179
183
|
if num_retries + 1 > max_job_retries
|
180
184
|
job.bury
|
181
185
|
else
|
182
186
|
job.release(delay)
|
183
187
|
end
|
184
|
-
|
188
|
+
log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
|
185
189
|
|
186
190
|
handle_error(e, job.name, job.args, job)
|
187
191
|
end
|
188
192
|
else
|
189
|
-
sleep(rand*3)
|
193
|
+
sleep(rand * 3)
|
190
194
|
end
|
191
195
|
job
|
192
196
|
end
|
193
197
|
|
194
|
-
|
195
198
|
protected
|
196
199
|
|
197
200
|
# Return a new connection instance
|
@@ -200,9 +203,10 @@ module Backburner
|
|
200
203
|
end
|
201
204
|
|
202
205
|
# Reserve a job from the watched queues
|
203
|
-
def reserve_job(conn, tube_name,
|
206
|
+
def reserve_job(conn, tube_name, _reserve_timeout = Backburner.configuration.reserve_timeout)
|
204
207
|
job = conn.get(tube_name)
|
205
208
|
return nil if job.nil? || job.body == nil?
|
209
|
+
|
206
210
|
Backburner::Job.new(job)
|
207
211
|
end
|
208
212
|
|
@@ -210,11 +214,10 @@ module Backburner
|
|
210
214
|
# Filtered for tubes that match the known prefix
|
211
215
|
def all_existing_queues
|
212
216
|
known_queues = Backburner::Worker.known_queue_classes.map(&:queue)
|
213
|
-
existing_tubes =
|
217
|
+
existing_tubes = connection.tubes.all.map(&:name).select { |tube| tube =~ /^#{queue_config.tube_namespace}/ }
|
214
218
|
existing_tubes + known_queues + [queue_config.primary_queue]
|
215
219
|
end
|
216
220
|
|
217
|
-
|
218
221
|
# Handles an error according to custom definition
|
219
222
|
# Used when processing a job that errors out
|
220
223
|
def handle_error(e, name, args, job)
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require File.expand_path('test_helper', __dir__)
|
2
|
+
|
3
|
+
$backburner_sum = 0
|
4
|
+
$backburner_numbers = []
|
5
|
+
|
6
|
+
class TestBackburnerJob
|
7
|
+
include Backburner::Queue
|
8
|
+
queue 'test.jobber'
|
9
|
+
|
10
|
+
def self.perform(value, number)
|
11
|
+
$backburner_sum += value
|
12
|
+
$backburner_numbers << number
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
class TestWorker < Backburner::Worker; end
|
17
|
+
|
18
|
+
describe 'Backburner module' do
|
19
|
+
before do
|
20
|
+
Backburner.default_queues.clear
|
21
|
+
clear_jobs!(Backburner.configuration.primary_queue, 'test-plain', 'parent-plain', 'bar.baz.foo')
|
22
|
+
end
|
23
|
+
|
24
|
+
describe 'for work method' do
|
25
|
+
it 'invokes worker simple start' do
|
26
|
+
Backburner::Workers::Simple.expects(:start).with(%w[foo bar])
|
27
|
+
Backburner.work('foo', 'bar')
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'invokes other worker if specified in configuration' do
|
31
|
+
Backburner.configure { |config| config.default_worker = TestWorker }
|
32
|
+
TestWorker.expects(:start).with(%w[foo bar])
|
33
|
+
Backburner.work('foo', 'bar')
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'invokes other worker if specified in work method as options' do
|
37
|
+
TestWorker.expects(:start).with(%w[foo bar])
|
38
|
+
Backburner.work('foo', 'bar', worker: TestWorker)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'invokes worker start with no args' do
|
42
|
+
Backburner::Workers::Simple.expects(:start).with([])
|
43
|
+
Backburner.work
|
44
|
+
end
|
45
|
+
end # work!
|
46
|
+
|
47
|
+
describe 'for configuration' do
|
48
|
+
it 'remembers the tube_namespace' do
|
49
|
+
assert_equal 'demo.test', Backburner.configuration.tube_namespace
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'remembers the namespace_separator' do
|
53
|
+
assert_equal '.', Backburner.configuration.namespace_separator
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'disallows a reserved separator' do
|
57
|
+
assert_raises RuntimeError do
|
58
|
+
Backburner.configuration.namespace_separator = ':'
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end # configuration
|
62
|
+
|
63
|
+
describe 'for default_queues' do
|
64
|
+
it 'supports assignment' do
|
65
|
+
Backburner.default_queues << 'foo'
|
66
|
+
Backburner.default_queues << 'bar'
|
67
|
+
assert_same_elements %w[foo bar], Backburner.default_queues
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
after do
|
72
|
+
Backburner.configure { |config| config.default_worker = Backburner::Workers::Simple }
|
73
|
+
end
|
74
|
+
end # Backburner
|
@@ -0,0 +1,108 @@
|
|
1
|
+
$LOAD_PATH.unshift File.expand_path("lib")
|
2
|
+
require 'backburner'
|
3
|
+
require 'tempfile'
|
4
|
+
require 'minitest/autorun'
|
5
|
+
require_relative '../fixtures/test_jobs'
|
6
|
+
|
7
|
+
begin
|
8
|
+
require 'mocha/setup'
|
9
|
+
rescue LoadError
|
10
|
+
require 'mocha'
|
11
|
+
end
|
12
|
+
|
13
|
+
# Configure Backburner
|
14
|
+
Backburner.configure do |config|
|
15
|
+
config.allq_url = "allq://127.0.0.1:8090"
|
16
|
+
config.tube_namespace = "demo.test"
|
17
|
+
config.on_error = lambda { |e| puts e }
|
18
|
+
end
|
19
|
+
|
20
|
+
def clear_jobs!(*tube_names)
|
21
|
+
connection = allq_connection
|
22
|
+
tube_names.each do |tube_name|
|
23
|
+
expanded_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
|
24
|
+
connection.clear(expanded_name)
|
25
|
+
end
|
26
|
+
ensure
|
27
|
+
connection.close if connection
|
28
|
+
end
|
29
|
+
|
30
|
+
def allq_connection
|
31
|
+
Backburner::Connection.new(Backburner.configuration.allq_url)
|
32
|
+
end
|
33
|
+
|
34
|
+
module Kernel
|
35
|
+
# Redirect standard out, standard error and the buffered logger for sprinkle to StringIO
|
36
|
+
# capture_stdout { any_commands; you_want } => "all output from the commands"
|
37
|
+
def capture_stdout
|
38
|
+
if ENV['DEBUG'] # Skip if debug mode
|
39
|
+
yield
|
40
|
+
return ""
|
41
|
+
end
|
42
|
+
|
43
|
+
out = StringIO.new
|
44
|
+
$stdout = out
|
45
|
+
$stderr = out
|
46
|
+
yield
|
47
|
+
return out.string
|
48
|
+
ensure
|
49
|
+
$stdout = STDOUT
|
50
|
+
$stderr = STDERR
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def sleeper(duration_in_seconds)
|
55
|
+
1.upto(duration_in_seconds) do |i|
|
56
|
+
sleep(1)
|
57
|
+
print '.'
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
# pop_one_job(tube_name)
|
62
|
+
def pop_one_job(tube_name=Backburner.configuration.primary_queue)
|
63
|
+
tube_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
|
64
|
+
connection = allq_connection
|
65
|
+
silenced(3) { @res = connection.get(tube_name)}
|
66
|
+
data = nil
|
67
|
+
data = Backburner.configuration.job_parser_proc.call(@res.body) if @res && @res.body
|
68
|
+
yield @res, data
|
69
|
+
ensure
|
70
|
+
connection.close if connection
|
71
|
+
end
|
72
|
+
|
73
|
+
# def work_one_job(tube_name=Backburner.configuration.primary_queue)
|
74
|
+
# tube_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
|
75
|
+
# connection = allq_connection
|
76
|
+
# job = Backburner::Worker.work_one_job(connection, tube_name)
|
77
|
+
# job.delete
|
78
|
+
# end
|
79
|
+
|
80
|
+
def assert_same_elements(a1, a2, msg = nil)
|
81
|
+
[:select, :inject, :size].each do |m|
|
82
|
+
[a1, a2].each {|a| assert_respond_to(a, m, "Are you sure that #{a.inspect} is an array? It doesn't respond to #{m}.") }
|
83
|
+
end
|
84
|
+
|
85
|
+
assert a1h = a1.inject({}) { |h,e| h[e] ||= a1.select { |i| i == e }.size; h }
|
86
|
+
assert a2h = a2.inject({}) { |h,e| h[e] ||= a2.select { |i| i == e }.size; h }
|
87
|
+
|
88
|
+
assert_equal(a1h, a2h, msg)
|
89
|
+
end
|
90
|
+
|
91
|
+
# assert_contains(['a', '1'], /\d/) => passes
|
92
|
+
# assert_contains(['a', '1'], 'a') => passes
|
93
|
+
# assert_contains(['a', '1'], /not there/) => fails
|
94
|
+
def assert_contains(collection, x, extra_msg = "")
|
95
|
+
collection = [collection] unless collection.is_a?(Array)
|
96
|
+
msg = "#{x.inspect} not found in #{collection.to_a.inspect} #{extra_msg}"
|
97
|
+
case x
|
98
|
+
when Regexp
|
99
|
+
assert(collection.detect { |e| e =~ x }, msg)
|
100
|
+
else
|
101
|
+
assert(collection.include?(x), msg)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
# silenced(5) { ... }
|
106
|
+
def silenced(time=3, &block)
|
107
|
+
Timeout::timeout(time) { capture_stdout(&block) }
|
108
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require File.expand_path('test_helper', __dir__)
|
2
|
+
|
3
|
+
describe 'Backburner::Worker module' do
|
4
|
+
before do
|
5
|
+
Backburner.default_queues.clear
|
6
|
+
clear_jobs!(Backburner.configuration.primary_queue, 'test-plain', 'parent-plain', 'bar.baz.foo')
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should support enqueuing plain job' do
|
10
|
+
Backburner::Worker.enqueue TestPlainJob, [7, 9], { ttr: 144 }
|
11
|
+
pop_one_job('test-plain') do |_job, body|
|
12
|
+
assert_equal 'TestPlainJob', body['class']
|
13
|
+
assert_equal [7, 9], body['args']
|
14
|
+
assert_equal 144, body['ttr'].to_i
|
15
|
+
end
|
16
|
+
end # plain
|
17
|
+
|
18
|
+
it 'should support enqueuing parent job' do
|
19
|
+
options = {
|
20
|
+
is_parent: true,
|
21
|
+
limit: 1,
|
22
|
+
timeout: 3,
|
23
|
+
run_on_timeout: true
|
24
|
+
}
|
25
|
+
# Enqueue parent job
|
26
|
+
response = Backburner::Worker.enqueue TestParentJob, [17, 9], options
|
27
|
+
|
28
|
+
options = {
|
29
|
+
parent_id: response.job_id
|
30
|
+
}
|
31
|
+
# Enqueue child job
|
32
|
+
Backburner::Worker.enqueue TestPlainJob, [7, 9], options
|
33
|
+
pop_one_job('test-plain') do |job, body|
|
34
|
+
assert_equal 'TestPlainJob', body['class']
|
35
|
+
assert_equal [7, 9], body['args']
|
36
|
+
job.delete
|
37
|
+
end
|
38
|
+
|
39
|
+
pop_one_job('parent-plain') do |_job, body|
|
40
|
+
assert_equal [17, 9], body['args']
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'should support enqueuing parent job with timeout' do
|
45
|
+
options = {
|
46
|
+
is_parent: true,
|
47
|
+
limit: 1,
|
48
|
+
timeout: 3,
|
49
|
+
run_on_timeout: true
|
50
|
+
}
|
51
|
+
# Enqueue parent job
|
52
|
+
response = Backburner::Worker.enqueue TestParentJob, [17, 9], options
|
53
|
+
|
54
|
+
options = {
|
55
|
+
parent_id: response.job_id
|
56
|
+
}
|
57
|
+
# Enqueue child job
|
58
|
+
Backburner::Worker.enqueue TestPlainJob, [7, 9], options
|
59
|
+
|
60
|
+
pop_one_job('parent-plain') do |_job, body|
|
61
|
+
assert_equal body, nil
|
62
|
+
sleeper(10)
|
63
|
+
end
|
64
|
+
|
65
|
+
worker = Backburner::Workers::Simple.new('demo.test.parent-plain')
|
66
|
+
worker.prepare
|
67
|
+
|
68
|
+
TestParentJob.expects(:perform_with_task)
|
69
|
+
job = worker.work_one_job(allq_connection, 'demo.test.parent-plain')
|
70
|
+
assert_equal job.special, "true"
|
71
|
+
end
|
72
|
+
|
73
|
+
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
|
78
|
+
# options[:parent_id] = opt[:parent_id] if opt[:parent_id]
|
79
|
+
# options[:timeout] = opt[:timeout] if opt[:timeout]
|
80
|
+
# options[:run_on_timeout] = opt[:run_on_timeout] if opt[:run_on_timeout]
|
81
|
+
# options[:limit] = opt[:limit] if opt[:limit]
|
82
|
+
# options[:is_parent] = opt[:is_parent] if opt[:is_parent]
|
data/test/fixtures/test_jobs.rb
CHANGED
@@ -6,6 +6,18 @@ class TestPlainJob
|
|
6
6
|
def self.perform(x, y); $worker_test_count += x + y + 1; end
|
7
7
|
end
|
8
8
|
|
9
|
+
|
10
|
+
class TestParentJob
|
11
|
+
def self.queue; "parent-plain"; end
|
12
|
+
def self.perform(args)
|
13
|
+
puts "plain perform #{args}"
|
14
|
+
end
|
15
|
+
def self.perform_with_task(task, *args)
|
16
|
+
puts "perform_with_task"
|
17
|
+
puts task.inspect
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
9
21
|
class TestJob
|
10
22
|
include Backburner::Queue
|
11
23
|
queue_priority :medium
|
data/test/worker_test.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: backburner-allq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.39
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Malcolm
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-07-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: allq_rest
|
@@ -109,6 +109,7 @@ extensions: []
|
|
109
109
|
extra_rdoc_files: []
|
110
110
|
files:
|
111
111
|
- ".gitignore"
|
112
|
+
- ".gitlab-ci.yml"
|
112
113
|
- ".travis.yml"
|
113
114
|
- CHANGELOG.md
|
114
115
|
- CONTRIBUTING.md
|
@@ -122,6 +123,7 @@ files:
|
|
122
123
|
- bin/backburner
|
123
124
|
- circle.yml
|
124
125
|
- deploy.sh
|
126
|
+
- docker-compose.yml
|
125
127
|
- examples/custom.rb
|
126
128
|
- examples/demo.rb
|
127
129
|
- examples/god.rb
|
@@ -148,6 +150,9 @@ files:
|
|
148
150
|
- lib/backburner/workers/simple.rb
|
149
151
|
- lib/backburner/workers/threading.rb
|
150
152
|
- lib/backburner/workers/threads_on_fork.rb
|
153
|
+
- test/allq/back_burner_test.rb
|
154
|
+
- test/allq/test_helper.rb
|
155
|
+
- test/allq/worker_test.rb
|
151
156
|
- test/async_proxy_test.rb
|
152
157
|
- test/back_burner_test.rb
|
153
158
|
- test/connection_test.rb
|
@@ -193,6 +198,9 @@ signing_key:
|
|
193
198
|
specification_version: 4
|
194
199
|
summary: Reliable allq background job processing made easy for Ruby and Sinatra
|
195
200
|
test_files:
|
201
|
+
- test/allq/back_burner_test.rb
|
202
|
+
- test/allq/test_helper.rb
|
203
|
+
- test/allq/worker_test.rb
|
196
204
|
- test/async_proxy_test.rb
|
197
205
|
- test/back_burner_test.rb
|
198
206
|
- test/connection_test.rb
|