backburner-allq 1.0.5 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/deploy.sh +1 -1
- data/lib/backburner/allq_wrapper.rb +23 -31
- data/lib/backburner/configuration.rb +2 -2
- data/lib/backburner/connection.rb +4 -8
- data/lib/backburner/job.rb +1 -6
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +10 -13
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1b73c3b5303a61511d25f26f0d29b7383a530e2f4b6de7aecbf49f7849e1aab1
|
4
|
+
data.tar.gz: 384a76bb81541a15068d7b216bfe1bdba72ff093af3f39119acde1485e5a3719
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d82bbf92e60e38b4a875c7f7d6dd322e143ee239e42122ae83f61a82c5249a9d6306e600951bf9cdd9fa9833a1489235a8b7cb878c253b8fa5bb137d8e45773d
|
7
|
+
data.tar.gz: 98b6537bda44810cd9b312aebe3eb47130ead4a2bc2d6f61f95a844b2742723e96e3c46b64d631515a7812d0b6741a1f0965c1ab2c0dc81f98e4badddc574434
|
data/deploy.sh
CHANGED
@@ -79,7 +79,7 @@ module Backburner
|
|
79
79
|
end
|
80
80
|
|
81
81
|
class AllQWrapper
|
82
|
-
def initialize(url =
|
82
|
+
def initialize(url = 'localhost:8090')
|
83
83
|
allq_conf = Allq::Configuration.new do |config|
|
84
84
|
config.host = url
|
85
85
|
end
|
@@ -127,35 +127,30 @@ module Backburner
|
|
127
127
|
return nil if job.body.nil?
|
128
128
|
|
129
129
|
job.body = Base64.decode64(job.body) if job
|
130
|
-
job_obj =
|
130
|
+
job_obj = Backburner::AllQJob.new(self, job)
|
131
131
|
job_obj
|
132
132
|
end
|
133
133
|
|
134
134
|
def get(tube_name = 'default')
|
135
135
|
job = nil
|
136
|
-
|
137
|
-
job = @client.job_get(tube_name)
|
138
|
-
end
|
139
|
-
puts "Allq http get delta: #{delt} #{tube_name} #{job}" if delt.to_f > 1.5
|
136
|
+
job = @client.job_get(tube_name)
|
140
137
|
|
141
|
-
@recent_times.push(delt.to_f)
|
142
|
-
@recent_times.shift if @recent_times.size > 2
|
143
138
|
# Inplace decode
|
144
139
|
job.body = Base64.decode64(job.body) if job&.body
|
145
140
|
|
146
|
-
job_obj =
|
141
|
+
job_obj = Backburner::AllQJob.new(self, job)
|
147
142
|
job_obj
|
148
143
|
rescue StandardError => ex
|
149
144
|
if ex.message == "Couldn't resolve host name"
|
150
|
-
|
145
|
+
puts("COUDNT RESOLVE HOST NAME------ SHOULD REBOOT")
|
151
146
|
else
|
152
|
-
|
147
|
+
puts(ex)
|
153
148
|
end
|
154
149
|
end
|
155
150
|
|
156
151
|
def close
|
157
152
|
rescue StandardError => ex
|
158
|
-
|
153
|
+
puts(ex)
|
159
154
|
end
|
160
155
|
|
161
156
|
def map_priority(app_priority)
|
@@ -178,9 +173,9 @@ module Backburner
|
|
178
173
|
end
|
179
174
|
|
180
175
|
def log_result(job_result)
|
181
|
-
|
176
|
+
puts("ALLQ-HTTP-JOB-ID=#{job_result.job_id}")
|
182
177
|
rescue StandardError => ex
|
183
|
-
|
178
|
+
puts(ex)
|
184
179
|
end
|
185
180
|
|
186
181
|
def build_new_job(body, options)
|
@@ -241,30 +236,27 @@ module Backburner
|
|
241
236
|
begin
|
242
237
|
Timeout.timeout(10) do
|
243
238
|
if body && body.to_s.include?('["default"]')
|
244
|
-
|
239
|
+
puts "PUTTING DEFAULT! #{caller.inspect}"
|
245
240
|
end
|
246
241
|
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
result = @client.job_post(new_job)
|
254
|
-
end
|
255
|
-
raise 'PUT returned nil' if result.nil? || result.to_s == ''
|
242
|
+
if is_parent
|
243
|
+
new_job = build_new_parent_job(body, options)
|
244
|
+
result = @client.parent_job_post(new_job)
|
245
|
+
else
|
246
|
+
new_job = build_new_job(body, options)
|
247
|
+
result = @client.job_post(new_job)
|
256
248
|
end
|
257
|
-
|
249
|
+
raise 'PUT returned nil' if result.nil? || result.to_s == ''
|
258
250
|
end
|
259
251
|
rescue Timeout::Error
|
260
|
-
|
252
|
+
puts('ALLQ_PUT_TIMEOUT')
|
261
253
|
sleep(5)
|
262
254
|
retry_count += 1
|
263
255
|
retry if retry_count < 4
|
264
256
|
raise 'Failed to put on allq, we are investigating the problem, please try again'
|
265
257
|
rescue StandardError => ex
|
266
|
-
|
267
|
-
|
258
|
+
puts('Failed to ALLQ PUT')
|
259
|
+
puts(ex)
|
268
260
|
retry_count += 1
|
269
261
|
sleep(5)
|
270
262
|
retry if retry_count < 4
|
@@ -290,7 +282,7 @@ module Backburner
|
|
290
282
|
end
|
291
283
|
final_stats
|
292
284
|
rescue StandardError => ex
|
293
|
-
|
285
|
+
puts(ex)
|
294
286
|
{}
|
295
287
|
end
|
296
288
|
|
@@ -300,7 +292,7 @@ module Backburner
|
|
300
292
|
count = tube_stats['ready'].to_i if tube_stats && tube_stats['ready']
|
301
293
|
count
|
302
294
|
rescue StandardError => ex
|
303
|
-
|
295
|
+
puts(ex)
|
304
296
|
-1
|
305
297
|
end
|
306
298
|
|
@@ -308,7 +300,7 @@ module Backburner
|
|
308
300
|
result = get_ready_by_tube('default')
|
309
301
|
result.to_i
|
310
302
|
rescue StandardError => ex
|
311
|
-
|
303
|
+
puts(ex)
|
312
304
|
0
|
313
305
|
end
|
314
306
|
end
|
@@ -21,8 +21,8 @@ module Backburner
|
|
21
21
|
attr_accessor :job_parser_proc # proc to parse a job body from a string
|
22
22
|
|
23
23
|
def initialize
|
24
|
-
@allq_url = "127.0.0.1:
|
25
|
-
@tube_namespace = ""
|
24
|
+
@allq_url = "allq://127.0.0.1:8091"
|
25
|
+
@tube_namespace = "backallq"
|
26
26
|
@namespace_separator = "."
|
27
27
|
@default_priority = 5
|
28
28
|
@respond_timeout = 120
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'delegate'
|
2
2
|
|
3
3
|
module Backburner
|
4
|
-
class Connection
|
4
|
+
class Connection
|
5
5
|
class BadURL < RuntimeError; end
|
6
6
|
|
7
7
|
attr_accessor :url, :allq_wrapper
|
@@ -27,7 +27,6 @@ module Backburner
|
|
27
27
|
def close
|
28
28
|
@allq_wrapper.close if @allq_wrapper
|
29
29
|
@allq_wrapper = nil
|
30
|
-
__setobj__(@allq_wrapper)
|
31
30
|
end
|
32
31
|
|
33
32
|
# Determines if the connection to allq is currently open
|
@@ -67,7 +66,7 @@ module Backburner
|
|
67
66
|
begin
|
68
67
|
yield
|
69
68
|
|
70
|
-
rescue
|
69
|
+
rescue Exception => e
|
71
70
|
if retry_count > 0
|
72
71
|
reconnect!
|
73
72
|
retry_count -= 1
|
@@ -80,8 +79,6 @@ module Backburner
|
|
80
79
|
end
|
81
80
|
end
|
82
81
|
|
83
|
-
protected
|
84
|
-
|
85
82
|
# Attempt to ensure we're connected to allq if the missing method is
|
86
83
|
# present in the delegate and we haven't shut down the connection on purpose
|
87
84
|
# @raise [Beaneater::NotConnected] If allq fails to connect after multiple attempts.
|
@@ -93,8 +90,7 @@ module Backburner
|
|
93
90
|
# Connects to a allq queue
|
94
91
|
# @raise Beaneater::NotConnected if the connection cannot be established
|
95
92
|
def connect!
|
96
|
-
@allq_wrapper = Backburner::
|
97
|
-
__setobj__(@allq_wrapper)
|
93
|
+
@allq_wrapper = Backburner::AllQWrapper.new(allq_addresses)
|
98
94
|
@allq_wrapper
|
99
95
|
end
|
100
96
|
|
@@ -102,7 +98,7 @@ module Backburner
|
|
102
98
|
pri = (opt[:pri] || 5).to_i
|
103
99
|
delay = opt[:delay].to_i
|
104
100
|
ttr = (opt[:ttr] || 600).to_i
|
105
|
-
@allq_wrapper.put2(data, pri, ttr, delay)
|
101
|
+
@allq_wrapper.put2(data, pri, ttr, tube_name, delay)
|
106
102
|
end
|
107
103
|
|
108
104
|
def get(tube_name)
|
data/lib/backburner/job.rb
CHANGED
@@ -48,12 +48,7 @@ module Backburner
|
|
48
48
|
return false unless res
|
49
49
|
# Execute the job
|
50
50
|
@hooks.around_hook_events(job_name, :around_perform, *args) do
|
51
|
-
|
52
|
-
# a) ttr == 0, to support never timing out
|
53
|
-
# b) ttr == 1, so that we don't accidentally set it to never time out
|
54
|
-
# NB: A ttr of 1 will likely result in race conditions between
|
55
|
-
# Backburner and beanstalkd and should probably be avoided
|
56
|
-
timeout_job_after(task.ttr > 1 ? task.ttr - 1 : task.ttr) { job_class.perform(*args) }
|
51
|
+
job_class.perform(*args)
|
57
52
|
end
|
58
53
|
task.delete
|
59
54
|
# Invoke after perform hook
|
data/lib/backburner/version.rb
CHANGED
data/lib/backburner/worker.rb
CHANGED
@@ -40,7 +40,8 @@ module Backburner
|
|
40
40
|
connection.retryable do
|
41
41
|
tube_name = expand_tube_name(queue || job_class)
|
42
42
|
serialized_data = Backburner.configuration.job_serializer_proc.call(data)
|
43
|
-
|
43
|
+
puts "Tube name #{tube_name}"
|
44
|
+
response = connection.put(tube_name, serialized_data, :pri => pri, :delay => delay, :ttr => ttr)
|
44
45
|
end
|
45
46
|
return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
|
46
47
|
ensure
|
@@ -129,7 +130,8 @@ module Backburner
|
|
129
130
|
def work_one_job(conn = connection)
|
130
131
|
begin
|
131
132
|
job = reserve_job(conn)
|
132
|
-
rescue
|
133
|
+
rescue Exception => e
|
134
|
+
sleep(rand*3)
|
133
135
|
return
|
134
136
|
end
|
135
137
|
|
@@ -149,18 +151,13 @@ module Backburner
|
|
149
151
|
|
150
152
|
# NB: There's a slight chance here that the connection to beanstalkd has
|
151
153
|
# gone down between the time we reserved / processed the job and here.
|
152
|
-
num_retries = job.
|
154
|
+
num_retries = job.releases
|
153
155
|
max_job_retries = resolve_max_job_retries(job.job_class)
|
154
156
|
retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
|
160
|
-
else # retries failed, bury
|
161
|
-
job.bury
|
162
|
-
self.log_job_end(job.name, "#{retry_status}, burying") if job_started_at
|
163
|
-
end
|
157
|
+
retry_delay = resolve_retry_delay(job.job_class)
|
158
|
+
delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
|
159
|
+
job.release(delay)
|
160
|
+
self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
|
164
161
|
|
165
162
|
handle_error(e, job.name, job.args, job)
|
166
163
|
end
|
@@ -170,7 +167,7 @@ module Backburner
|
|
170
167
|
|
171
168
|
# Return a new connection instance
|
172
169
|
def new_connection
|
173
|
-
Connection.new(Backburner.configuration.
|
170
|
+
Connection.new(Backburner.configuration.allq_url) { |conn| Backburner::Hooks.invoke_hook_events(self, :on_reconnect, conn) }
|
174
171
|
end
|
175
172
|
|
176
173
|
# Reserve a job from the watched queues
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: backburner-allq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Malcolm
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-09-
|
11
|
+
date: 2020-09-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: allq_rest
|