backburner-allq 1.0.11 → 1.0.17
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/deploy.sh +1 -1
- data/lib/backburner.rb +2 -2
- data/lib/backburner/allq_wrapper.rb +19 -8
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +32 -27
- data/lib/backburner/workers/threads_on_fork.rb +3 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2a73b5d7a8f60e385a3f600bfded5c0579691bbafdd4bc15b0f51e0df7298f4f
|
4
|
+
data.tar.gz: 26000c93ce60b0f53ff666f51f7fcd1549ddde403dd400f7f4c457e056803c92
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a0d2079019d65e529e6b769ec860456bd13b2a29caf6797adcea9c1feca0f3b397d006ac9cd3e8a85e9d78a86af4ef8964d500d5822b001f730123705d80ada3
|
7
|
+
data.tar.gz: 747c8436582124a553424a1e674ae97a44fee865f74720fbe5e84ca5fc6006a05087a7e1c079a6acdb72c2414c7b94db0642fefbba886734d8b575735411ecbb
|
data/deploy.sh
CHANGED
data/lib/backburner.rb
CHANGED
@@ -25,8 +25,8 @@ module Backburner
|
|
25
25
|
# @example
|
26
26
|
# Backburner.enqueue NewsletterSender, self.id, user.id
|
27
27
|
#
|
28
|
-
def enqueue(job_class, args
|
29
|
-
Backburner::Worker.enqueue(job_class, args,
|
28
|
+
def enqueue(job_class, *args)
|
29
|
+
Backburner::Worker.enqueue(job_class, args, {})
|
30
30
|
end
|
31
31
|
|
32
32
|
# Begins working on jobs enqueued with optional tubes specified
|
@@ -13,14 +13,16 @@ module Backburner
|
|
13
13
|
|
14
14
|
def watch
|
15
15
|
Thread.new do
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
16
|
+
loop do
|
17
|
+
ran = false
|
18
|
+
job = @allq_wrapper.get(@tube_name)
|
19
|
+
if job.body
|
20
|
+
perform(job)
|
21
|
+
ran = true
|
22
|
+
end
|
23
|
+
# Wait if nothing returned
|
24
|
+
sleep(rand() * 3) unless ran
|
21
25
|
end
|
22
|
-
# Wait if nothing returned
|
23
|
-
sleep(rand() * 3) unless ran
|
24
26
|
end
|
25
27
|
end
|
26
28
|
end
|
@@ -123,6 +125,10 @@ module Backburner
|
|
123
125
|
stats_hash.keys
|
124
126
|
end
|
125
127
|
|
128
|
+
def tubes
|
129
|
+
tube_names
|
130
|
+
end
|
131
|
+
|
126
132
|
def peek_buried(tube_name = 'default')
|
127
133
|
job = nil
|
128
134
|
job = @client.peek_get(tube_name, buried: true)
|
@@ -208,7 +214,7 @@ module Backburner
|
|
208
214
|
priority: adjusted_priority,
|
209
215
|
timeout: timeout,
|
210
216
|
run_on_timeout: run_on_timeout,
|
211
|
-
shard_key: options[:shard_key]
|
217
|
+
shard_key: options[:shard_key],
|
212
218
|
limit: limit)
|
213
219
|
new_parent_job
|
214
220
|
end
|
@@ -250,6 +256,11 @@ module Backburner
|
|
250
256
|
result
|
251
257
|
end
|
252
258
|
|
259
|
+
def stats(tube)
|
260
|
+
final_stats = stats
|
261
|
+
final_stats[tube]
|
262
|
+
end
|
263
|
+
|
253
264
|
def stats
|
254
265
|
raw_stats = @admin.stats_get
|
255
266
|
final_stats = {}
|
data/lib/backburner/version.rb
CHANGED
data/lib/backburner/worker.rb
CHANGED
@@ -134,35 +134,38 @@ module Backburner
|
|
134
134
|
return
|
135
135
|
end
|
136
136
|
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
137
|
+
if job
|
138
|
+
self.log_job_begin(job.name, job.args)
|
139
|
+
job.process
|
140
|
+
self.log_job_end(job.name)
|
141
|
+
|
142
|
+
rescue Backburner::Job::JobFormatInvalid => e
|
143
|
+
self.log_error self.exception_message(e)
|
144
|
+
rescue => e # Error occurred processing job
|
145
|
+
self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
|
146
|
+
|
147
|
+
unless job
|
148
|
+
self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
|
149
|
+
return
|
150
|
+
end
|
145
151
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
152
|
+
# NB: There's a slight chance here that the connection to allq has
|
153
|
+
# gone down between the time we reserved / processed the job and here.
|
154
|
+
num_retries = job.releases
|
155
|
+
max_job_retries = resolve_max_job_retries(job.job_class)
|
156
|
+
retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
|
157
|
+
retry_delay = resolve_retry_delay(job.job_class)
|
158
|
+
delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
|
159
|
+
if num_retries + 1 > max_job_retries
|
160
|
+
job.bury
|
161
|
+
else
|
162
|
+
job.release(delay)
|
163
|
+
end
|
164
|
+
self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
|
150
165
|
|
151
|
-
|
152
|
-
# gone down between the time we reserved / processed the job and here.
|
153
|
-
num_retries = job.releases
|
154
|
-
max_job_retries = resolve_max_job_retries(job.job_class)
|
155
|
-
retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
|
156
|
-
retry_delay = resolve_retry_delay(job.job_class)
|
157
|
-
delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
|
158
|
-
if num_retries + 1 > max_job_retries
|
159
|
-
job.bury
|
160
|
-
else
|
161
|
-
job.release(delay)
|
166
|
+
handle_error(e, job.name, job.args, job)
|
162
167
|
end
|
163
|
-
|
164
|
-
|
165
|
-
handle_error(e, job.name, job.args, job)
|
168
|
+
job
|
166
169
|
end
|
167
170
|
|
168
171
|
|
@@ -175,7 +178,9 @@ module Backburner
|
|
175
178
|
|
176
179
|
# Reserve a job from the watched queues
|
177
180
|
def reserve_job(conn, reserve_timeout = Backburner.configuration.reserve_timeout)
|
178
|
-
|
181
|
+
job = conn.get(@tube_names.sample)
|
182
|
+
return nil if job.body == nil?
|
183
|
+
Backburner::Job.new(job)
|
179
184
|
end
|
180
185
|
|
181
186
|
# Returns a list of all tubes known within the system
|
@@ -207,7 +207,9 @@ module Backburner
|
|
207
207
|
def run_while_can(conn = connection)
|
208
208
|
while @garbage_after.nil? or @garbage_after > @runs
|
209
209
|
@runs += 1 # FIXME: Likely race condition
|
210
|
-
work_one_job(conn)
|
210
|
+
ran_job = work_one_job(conn)
|
211
|
+
# Wait a second if we didn't find a job
|
212
|
+
sleep(rand() * 3) unless ran_job
|
211
213
|
end
|
212
214
|
end
|
213
215
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: backburner-allq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Malcolm
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-02-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: allq_rest
|