backburner-allq 1.0.5 → 1.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/deploy.sh +1 -1
- data/lib/backburner/allq_wrapper.rb +23 -31
- data/lib/backburner/configuration.rb +2 -2
- data/lib/backburner/connection.rb +4 -8
- data/lib/backburner/job.rb +1 -6
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +10 -13
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1b73c3b5303a61511d25f26f0d29b7383a530e2f4b6de7aecbf49f7849e1aab1
|
4
|
+
data.tar.gz: 384a76bb81541a15068d7b216bfe1bdba72ff093af3f39119acde1485e5a3719
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d82bbf92e60e38b4a875c7f7d6dd322e143ee239e42122ae83f61a82c5249a9d6306e600951bf9cdd9fa9833a1489235a8b7cb878c253b8fa5bb137d8e45773d
|
7
|
+
data.tar.gz: 98b6537bda44810cd9b312aebe3eb47130ead4a2bc2d6f61f95a844b2742723e96e3c46b64d631515a7812d0b6741a1f0965c1ab2c0dc81f98e4badddc574434
|
data/deploy.sh
CHANGED
@@ -79,7 +79,7 @@ module Backburner
|
|
79
79
|
end
|
80
80
|
|
81
81
|
class AllQWrapper
|
82
|
-
def initialize(url =
|
82
|
+
def initialize(url = 'localhost:8090')
|
83
83
|
allq_conf = Allq::Configuration.new do |config|
|
84
84
|
config.host = url
|
85
85
|
end
|
@@ -127,35 +127,30 @@ module Backburner
|
|
127
127
|
return nil if job.body.nil?
|
128
128
|
|
129
129
|
job.body = Base64.decode64(job.body) if job
|
130
|
-
job_obj =
|
130
|
+
job_obj = Backburner::AllQJob.new(self, job)
|
131
131
|
job_obj
|
132
132
|
end
|
133
133
|
|
134
134
|
def get(tube_name = 'default')
|
135
135
|
job = nil
|
136
|
-
|
137
|
-
job = @client.job_get(tube_name)
|
138
|
-
end
|
139
|
-
puts "Allq http get delta: #{delt} #{tube_name} #{job}" if delt.to_f > 1.5
|
136
|
+
job = @client.job_get(tube_name)
|
140
137
|
|
141
|
-
@recent_times.push(delt.to_f)
|
142
|
-
@recent_times.shift if @recent_times.size > 2
|
143
138
|
# Inplace decode
|
144
139
|
job.body = Base64.decode64(job.body) if job&.body
|
145
140
|
|
146
|
-
job_obj =
|
141
|
+
job_obj = Backburner::AllQJob.new(self, job)
|
147
142
|
job_obj
|
148
143
|
rescue StandardError => ex
|
149
144
|
if ex.message == "Couldn't resolve host name"
|
150
|
-
|
145
|
+
puts("COUDNT RESOLVE HOST NAME------ SHOULD REBOOT")
|
151
146
|
else
|
152
|
-
|
147
|
+
puts(ex)
|
153
148
|
end
|
154
149
|
end
|
155
150
|
|
156
151
|
def close
|
157
152
|
rescue StandardError => ex
|
158
|
-
|
153
|
+
puts(ex)
|
159
154
|
end
|
160
155
|
|
161
156
|
def map_priority(app_priority)
|
@@ -178,9 +173,9 @@ module Backburner
|
|
178
173
|
end
|
179
174
|
|
180
175
|
def log_result(job_result)
|
181
|
-
|
176
|
+
puts("ALLQ-HTTP-JOB-ID=#{job_result.job_id}")
|
182
177
|
rescue StandardError => ex
|
183
|
-
|
178
|
+
puts(ex)
|
184
179
|
end
|
185
180
|
|
186
181
|
def build_new_job(body, options)
|
@@ -241,30 +236,27 @@ module Backburner
|
|
241
236
|
begin
|
242
237
|
Timeout.timeout(10) do
|
243
238
|
if body && body.to_s.include?('["default"]')
|
244
|
-
|
239
|
+
puts "PUTTING DEFAULT! #{caller.inspect}"
|
245
240
|
end
|
246
241
|
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
result = @client.job_post(new_job)
|
254
|
-
end
|
255
|
-
raise 'PUT returned nil' if result.nil? || result.to_s == ''
|
242
|
+
if is_parent
|
243
|
+
new_job = build_new_parent_job(body, options)
|
244
|
+
result = @client.parent_job_post(new_job)
|
245
|
+
else
|
246
|
+
new_job = build_new_job(body, options)
|
247
|
+
result = @client.job_post(new_job)
|
256
248
|
end
|
257
|
-
|
249
|
+
raise 'PUT returned nil' if result.nil? || result.to_s == ''
|
258
250
|
end
|
259
251
|
rescue Timeout::Error
|
260
|
-
|
252
|
+
puts('ALLQ_PUT_TIMEOUT')
|
261
253
|
sleep(5)
|
262
254
|
retry_count += 1
|
263
255
|
retry if retry_count < 4
|
264
256
|
raise 'Failed to put on allq, we are investigating the problem, please try again'
|
265
257
|
rescue StandardError => ex
|
266
|
-
|
267
|
-
|
258
|
+
puts('Failed to ALLQ PUT')
|
259
|
+
puts(ex)
|
268
260
|
retry_count += 1
|
269
261
|
sleep(5)
|
270
262
|
retry if retry_count < 4
|
@@ -290,7 +282,7 @@ module Backburner
|
|
290
282
|
end
|
291
283
|
final_stats
|
292
284
|
rescue StandardError => ex
|
293
|
-
|
285
|
+
puts(ex)
|
294
286
|
{}
|
295
287
|
end
|
296
288
|
|
@@ -300,7 +292,7 @@ module Backburner
|
|
300
292
|
count = tube_stats['ready'].to_i if tube_stats && tube_stats['ready']
|
301
293
|
count
|
302
294
|
rescue StandardError => ex
|
303
|
-
|
295
|
+
puts(ex)
|
304
296
|
-1
|
305
297
|
end
|
306
298
|
|
@@ -308,7 +300,7 @@ module Backburner
|
|
308
300
|
result = get_ready_by_tube('default')
|
309
301
|
result.to_i
|
310
302
|
rescue StandardError => ex
|
311
|
-
|
303
|
+
puts(ex)
|
312
304
|
0
|
313
305
|
end
|
314
306
|
end
|
@@ -21,8 +21,8 @@ module Backburner
|
|
21
21
|
attr_accessor :job_parser_proc # proc to parse a job body from a string
|
22
22
|
|
23
23
|
def initialize
|
24
|
-
@allq_url = "127.0.0.1:
|
25
|
-
@tube_namespace = ""
|
24
|
+
@allq_url = "allq://127.0.0.1:8091"
|
25
|
+
@tube_namespace = "backallq"
|
26
26
|
@namespace_separator = "."
|
27
27
|
@default_priority = 5
|
28
28
|
@respond_timeout = 120
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'delegate'
|
2
2
|
|
3
3
|
module Backburner
|
4
|
-
class Connection
|
4
|
+
class Connection
|
5
5
|
class BadURL < RuntimeError; end
|
6
6
|
|
7
7
|
attr_accessor :url, :allq_wrapper
|
@@ -27,7 +27,6 @@ module Backburner
|
|
27
27
|
def close
|
28
28
|
@allq_wrapper.close if @allq_wrapper
|
29
29
|
@allq_wrapper = nil
|
30
|
-
__setobj__(@allq_wrapper)
|
31
30
|
end
|
32
31
|
|
33
32
|
# Determines if the connection to allq is currently open
|
@@ -67,7 +66,7 @@ module Backburner
|
|
67
66
|
begin
|
68
67
|
yield
|
69
68
|
|
70
|
-
rescue
|
69
|
+
rescue Exception => e
|
71
70
|
if retry_count > 0
|
72
71
|
reconnect!
|
73
72
|
retry_count -= 1
|
@@ -80,8 +79,6 @@ module Backburner
|
|
80
79
|
end
|
81
80
|
end
|
82
81
|
|
83
|
-
protected
|
84
|
-
|
85
82
|
# Attempt to ensure we're connected to allq if the missing method is
|
86
83
|
# present in the delegate and we haven't shut down the connection on purpose
|
87
84
|
# @raise [Beaneater::NotConnected] If allq fails to connect after multiple attempts.
|
@@ -93,8 +90,7 @@ module Backburner
|
|
93
90
|
# Connects to a allq queue
|
94
91
|
# @raise Beaneater::NotConnected if the connection cannot be established
|
95
92
|
def connect!
|
96
|
-
@allq_wrapper = Backburner::
|
97
|
-
__setobj__(@allq_wrapper)
|
93
|
+
@allq_wrapper = Backburner::AllQWrapper.new(allq_addresses)
|
98
94
|
@allq_wrapper
|
99
95
|
end
|
100
96
|
|
@@ -102,7 +98,7 @@ module Backburner
|
|
102
98
|
pri = (opt[:pri] || 5).to_i
|
103
99
|
delay = opt[:delay].to_i
|
104
100
|
ttr = (opt[:ttr] || 600).to_i
|
105
|
-
@allq_wrapper.put2(data, pri, ttr, delay)
|
101
|
+
@allq_wrapper.put2(data, pri, ttr, tube_name, delay)
|
106
102
|
end
|
107
103
|
|
108
104
|
def get(tube_name)
|
data/lib/backburner/job.rb
CHANGED
@@ -48,12 +48,7 @@ module Backburner
|
|
48
48
|
return false unless res
|
49
49
|
# Execute the job
|
50
50
|
@hooks.around_hook_events(job_name, :around_perform, *args) do
|
51
|
-
|
52
|
-
# a) ttr == 0, to support never timing out
|
53
|
-
# b) ttr == 1, so that we don't accidentally set it to never time out
|
54
|
-
# NB: A ttr of 1 will likely result in race conditions between
|
55
|
-
# Backburner and beanstalkd and should probably be avoided
|
56
|
-
timeout_job_after(task.ttr > 1 ? task.ttr - 1 : task.ttr) { job_class.perform(*args) }
|
51
|
+
job_class.perform(*args)
|
57
52
|
end
|
58
53
|
task.delete
|
59
54
|
# Invoke after perform hook
|
data/lib/backburner/version.rb
CHANGED
data/lib/backburner/worker.rb
CHANGED
@@ -40,7 +40,8 @@ module Backburner
|
|
40
40
|
connection.retryable do
|
41
41
|
tube_name = expand_tube_name(queue || job_class)
|
42
42
|
serialized_data = Backburner.configuration.job_serializer_proc.call(data)
|
43
|
-
|
43
|
+
puts "Tube name #{tube_name}"
|
44
|
+
response = connection.put(tube_name, serialized_data, :pri => pri, :delay => delay, :ttr => ttr)
|
44
45
|
end
|
45
46
|
return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
|
46
47
|
ensure
|
@@ -129,7 +130,8 @@ module Backburner
|
|
129
130
|
def work_one_job(conn = connection)
|
130
131
|
begin
|
131
132
|
job = reserve_job(conn)
|
132
|
-
rescue
|
133
|
+
rescue Exception => e
|
134
|
+
sleep(rand*3)
|
133
135
|
return
|
134
136
|
end
|
135
137
|
|
@@ -149,18 +151,13 @@ module Backburner
|
|
149
151
|
|
150
152
|
# NB: There's a slight chance here that the connection to beanstalkd has
|
151
153
|
# gone down between the time we reserved / processed the job and here.
|
152
|
-
num_retries = job.
|
154
|
+
num_retries = job.releases
|
153
155
|
max_job_retries = resolve_max_job_retries(job.job_class)
|
154
156
|
retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
|
160
|
-
else # retries failed, bury
|
161
|
-
job.bury
|
162
|
-
self.log_job_end(job.name, "#{retry_status}, burying") if job_started_at
|
163
|
-
end
|
157
|
+
retry_delay = resolve_retry_delay(job.job_class)
|
158
|
+
delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
|
159
|
+
job.release(delay)
|
160
|
+
self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
|
164
161
|
|
165
162
|
handle_error(e, job.name, job.args, job)
|
166
163
|
end
|
@@ -170,7 +167,7 @@ module Backburner
|
|
170
167
|
|
171
168
|
# Return a new connection instance
|
172
169
|
def new_connection
|
173
|
-
Connection.new(Backburner.configuration.
|
170
|
+
Connection.new(Backburner.configuration.allq_url) { |conn| Backburner::Hooks.invoke_hook_events(self, :on_reconnect, conn) }
|
174
171
|
end
|
175
172
|
|
176
173
|
# Reserve a job from the watched queues
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: backburner-allq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Malcolm
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-09-
|
11
|
+
date: 2020-09-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: allq_rest
|