backburner-allq 1.0.37 → 1.0.38
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/deploy.sh +1 -1
- data/lib/backburner/allq_wrapper.rb +10 -19
- data/lib/backburner/job.rb +5 -0
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +34 -31
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1d44569a854f2a26375e77a62b485851cc1970bd7bacce5948178c65e17a1f27
|
4
|
+
data.tar.gz: 29d4a2408a0d7b77c6236a2fd716a07c9ddad32e1566d5d5575c99d557fcb7bf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9ea4a8b0da3978fe8567ecf7edbb4157246ad88da2acf9352038627e665f2240d877511e6f49f3775d42a54d61617c79dbbee2ccb71413a76797f1c77b01218d
|
7
|
+
data.tar.gz: 9e2be902d44f1600e86941c714c8bb09e2e902c636cba0cc34fd26a2fb901e79006e48df357877a3c3cc248966090561b37d4b10d5c6d0f207d5b7ab3be4e0e5
|
data/deploy.sh
CHANGED
@@ -42,13 +42,17 @@ module Backburner
|
|
42
42
|
# Release count
|
43
43
|
attr_accessor :releases
|
44
44
|
|
45
|
-
|
45
|
+
# Release count
|
46
|
+
attr_accessor :special
|
47
|
+
|
48
|
+
def initialize(wrapper, job_response)
|
46
49
|
@client = wrapper
|
47
|
-
@id =
|
48
|
-
@body =
|
49
|
-
@expireds =
|
50
|
-
@releases =
|
51
|
-
@tube =
|
50
|
+
@id = job_response.id
|
51
|
+
@body = job_response.body
|
52
|
+
@expireds = job_response.expireds
|
53
|
+
@releases = job_response.releases
|
54
|
+
@tube = job_response.tube.to_s
|
55
|
+
@special = job_response.special.to_s if job_response.special
|
52
56
|
end
|
53
57
|
|
54
58
|
def done
|
@@ -93,12 +97,6 @@ module Backburner
|
|
93
97
|
@recent_times = []
|
94
98
|
end
|
95
99
|
|
96
|
-
def speed
|
97
|
-
return @recent_times.sum(0.0) / @recent_times.size if @recent_times.size > 0
|
98
|
-
|
99
|
-
0
|
100
|
-
end
|
101
|
-
|
102
100
|
def touch(job)
|
103
101
|
@client.touch_put(job.id)
|
104
102
|
end
|
@@ -167,12 +165,6 @@ module Backburner
|
|
167
165
|
app_priority > 10 ? 5 : app_priority
|
168
166
|
end
|
169
167
|
|
170
|
-
def log_result(job_result)
|
171
|
-
puts("ALLQ-HTTP-JOB-ID=#{job_result.job_id}")
|
172
|
-
rescue StandardError => e
|
173
|
-
puts(e)
|
174
|
-
end
|
175
|
-
|
176
168
|
def build_new_job(body, options)
|
177
169
|
adjusted_priority = map_priority(options[:pri] || 5)
|
178
170
|
|
@@ -195,7 +187,6 @@ module Backburner
|
|
195
187
|
ttl = options[:ttl] || options[:ttr] || DEFAULT_TIMEOUT
|
196
188
|
tube_name = options[:tube_name] || 'default'
|
197
189
|
delay = options[:delay] || 0
|
198
|
-
parent_id = options[:parent_id]
|
199
190
|
limit = options[:limit]
|
200
191
|
timeout = options[:timeout] || 3_600
|
201
192
|
run_on_timeout = options[:run_on_timeout] || false
|
data/lib/backburner/job.rb
CHANGED
@@ -26,6 +26,11 @@ module Backburner
|
|
26
26
|
@name = body["class"] || body[:class]
|
27
27
|
@args = body["args"] || body[:args]
|
28
28
|
@ttr = body["ttr"] || body[:ttr]
|
29
|
+
if @task.special
|
30
|
+
@args ||= {}
|
31
|
+
@args["special"] = @task.special.to_s
|
32
|
+
end
|
33
|
+
|
29
34
|
rescue => ex # Job was not valid format
|
30
35
|
# self.bury
|
31
36
|
# raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
|
data/lib/backburner/version.rb
CHANGED
data/lib/backburner/worker.rb
CHANGED
@@ -13,7 +13,10 @@ module Backburner
|
|
13
13
|
# List of known_queue_classes
|
14
14
|
class << self
|
15
15
|
attr_writer :known_queue_classes
|
16
|
-
|
16
|
+
|
17
|
+
def known_queue_classes
|
18
|
+
@known_queue_classes ||= []
|
19
|
+
end
|
17
20
|
end
|
18
21
|
|
19
22
|
# Enqueues a job to be processed later by a worker.
|
@@ -37,8 +40,8 @@ module Backburner
|
|
37
40
|
|
38
41
|
return nil unless res # stop if hook is false
|
39
42
|
|
40
|
-
data = { :
|
41
|
-
queue = opts[:queue] && (
|
43
|
+
data = { class: job_class.name, args: args, ttr: ttr }
|
44
|
+
queue = opts[:queue] && (opts[:queue].is_a?(Proc) ? opts[:queue].call(job_class) : opts[:queue])
|
42
45
|
|
43
46
|
begin
|
44
47
|
response = nil
|
@@ -67,12 +70,10 @@ module Backburner
|
|
67
70
|
# @example
|
68
71
|
# Backburner::Worker.start(["foo.tube.name"])
|
69
72
|
#
|
70
|
-
def self.start(tube_names=nil)
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
# do nothing
|
75
|
-
end
|
73
|
+
def self.start(tube_names = nil)
|
74
|
+
new(tube_names).start
|
75
|
+
rescue SystemExit
|
76
|
+
# do nothing
|
76
77
|
end
|
77
78
|
|
78
79
|
# List of tube names to be watched and processed
|
@@ -84,7 +85,7 @@ module Backburner
|
|
84
85
|
# Worker.new(['test.job'])
|
85
86
|
def initialize(tube_names = nil)
|
86
87
|
@connection = new_connection
|
87
|
-
@tube_names =
|
88
|
+
@tube_names = process_tube_names(tube_names)
|
88
89
|
register_signal_handlers!
|
89
90
|
end
|
90
91
|
|
@@ -140,31 +141,30 @@ module Backburner
|
|
140
141
|
# @raise [Beaneater::NotConnected] If beanstalk fails to connect multiple times.
|
141
142
|
def work_one_job(conn = connection, tube_name = nil)
|
142
143
|
if tube_name.nil?
|
143
|
-
|
144
|
-
tube_name = @tube_names.sample
|
144
|
+
log_error 'Sampling tube, this is bad practice for Allq'
|
145
|
+
tube_name = @tube_names.sample
|
145
146
|
end
|
146
|
-
|
147
|
+
|
147
148
|
begin
|
148
149
|
job = reserve_job(conn, tube_name)
|
149
150
|
rescue Exception => e
|
150
|
-
|
151
|
-
|
152
|
-
sleep(rand*3)
|
151
|
+
log_error "Exception: #{e.full_message}"
|
152
|
+
sleep(rand * 3)
|
153
153
|
return
|
154
154
|
end
|
155
155
|
|
156
156
|
if job && job.body
|
157
157
|
begin
|
158
|
-
|
158
|
+
log_job_begin(job.name, job.args)
|
159
159
|
job.process
|
160
|
-
|
160
|
+
log_job_end(job.name)
|
161
161
|
rescue Backburner::Job::JobFormatInvalid => e
|
162
|
-
|
163
|
-
rescue => e # Error occurred processing job
|
164
|
-
|
162
|
+
log_error exception_message(e)
|
163
|
+
rescue StandardError => e # Error occurred processing job
|
164
|
+
log_error exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
|
165
165
|
|
166
166
|
unless job
|
167
|
-
|
167
|
+
log_error 'Error occurred before we were able to assign a job. Giving up without retrying!'
|
168
168
|
return
|
169
169
|
end
|
170
170
|
|
@@ -172,26 +172,29 @@ module Backburner
|
|
172
172
|
# gone down between the time we reserved / processed the job and here.
|
173
173
|
num_retries = job.releases
|
174
174
|
max_job_retries = resolve_max_job_retries(job.job_class)
|
175
|
-
retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
|
175
|
+
retry_status = "failed: attempt #{num_retries + 1} of #{max_job_retries + 1}"
|
176
176
|
retry_delay = resolve_retry_delay(job.job_class)
|
177
|
-
delay =
|
178
|
-
|
177
|
+
delay = begin
|
178
|
+
resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries)
|
179
|
+
rescue StandardError
|
180
|
+
retry_delay
|
181
|
+
end
|
182
|
+
|
179
183
|
if num_retries + 1 > max_job_retries
|
180
184
|
job.bury
|
181
185
|
else
|
182
186
|
job.release(delay)
|
183
187
|
end
|
184
|
-
|
188
|
+
log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
|
185
189
|
|
186
190
|
handle_error(e, job.name, job.args, job)
|
187
191
|
end
|
188
192
|
else
|
189
|
-
sleep(rand*3)
|
193
|
+
sleep(rand * 3)
|
190
194
|
end
|
191
195
|
job
|
192
196
|
end
|
193
197
|
|
194
|
-
|
195
198
|
protected
|
196
199
|
|
197
200
|
# Return a new connection instance
|
@@ -200,9 +203,10 @@ module Backburner
|
|
200
203
|
end
|
201
204
|
|
202
205
|
# Reserve a job from the watched queues
|
203
|
-
def reserve_job(conn, tube_name,
|
206
|
+
def reserve_job(conn, tube_name, _reserve_timeout = Backburner.configuration.reserve_timeout)
|
204
207
|
job = conn.get(tube_name)
|
205
208
|
return nil if job.nil? || job.body == nil?
|
209
|
+
|
206
210
|
Backburner::Job.new(job)
|
207
211
|
end
|
208
212
|
|
@@ -210,11 +214,10 @@ module Backburner
|
|
210
214
|
# Filtered for tubes that match the known prefix
|
211
215
|
def all_existing_queues
|
212
216
|
known_queues = Backburner::Worker.known_queue_classes.map(&:queue)
|
213
|
-
existing_tubes =
|
217
|
+
existing_tubes = connection.tubes.all.map(&:name).select { |tube| tube =~ /^#{queue_config.tube_namespace}/ }
|
214
218
|
existing_tubes + known_queues + [queue_config.primary_queue]
|
215
219
|
end
|
216
220
|
|
217
|
-
|
218
221
|
# Handles an error according to custom definition
|
219
222
|
# Used when processing a job that errors out
|
220
223
|
def handle_error(e, name, args, job)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: backburner-allq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.38
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Malcolm
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-07-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: allq_rest
|