backburner-allq 1.0.18 → 1.0.19
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/deploy.sh +1 -1
- data/lib/backburner/configuration.rb +1 -1
- data/lib/backburner/connection.rb +4 -0
- data/lib/backburner/job.rb +4 -4
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +9 -2
- data/lib/backburner/workers/forking.rb +1 -1
- data/lib/backburner/workers/simple.rb +1 -1
- data/lib/backburner/workers/threading.rb +2 -2
- data/lib/backburner/workers/threads_on_fork.rb +7 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 961c84a21c68a6bbe287105030dcbcf73e50177d647321dc410786b417ba653c
|
4
|
+
data.tar.gz: bee092eb96d993ee4b34b4c1e9fa671cc92e4a3316179890be130dbe14cc70f1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a09fd305b986a90330434db34e13e961f4f6878bff406348fb246f03ac0a3e0b9b8216862128c62005e5b25a4338371cb341e57133aadaa1e3187066fee8698e
|
7
|
+
data.tar.gz: 12fd13a7430682a5021d16ad32c7f3be1f39767413feeb2a96a7335c465d3639dfbd47905f43249aae27d23030c27a284fce36d098f7b6f396ac789182330efe
|
data/Gemfile
CHANGED
data/deploy.sh
CHANGED
@@ -27,7 +27,7 @@ module Backburner
|
|
27
27
|
@default_priority = 5
|
28
28
|
@respond_timeout = 120
|
29
29
|
@on_error = nil
|
30
|
-
@max_job_retries =
|
30
|
+
@max_job_retries = 1
|
31
31
|
@retry_delay = 5
|
32
32
|
@retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
|
33
33
|
@default_queues = []
|
@@ -38,6 +38,10 @@ module Backburner
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
|
41
|
+
def tubes
|
42
|
+
@allq_wrapper.tube_names if @allq_wrapper
|
43
|
+
end
|
44
|
+
|
41
45
|
# Attempt to reconnect to allq. Note: the connection will not be watching
|
42
46
|
# or using the tubes it was before it was reconnected (as it's actually a
|
43
47
|
# completely new connection)
|
data/lib/backburner/job.rb
CHANGED
@@ -26,8 +26,8 @@ module Backburner
|
|
26
26
|
@name = body["class"] || body[:class]
|
27
27
|
@args = body["args"] || body[:args]
|
28
28
|
rescue => ex # Job was not valid format
|
29
|
-
self.bury
|
30
|
-
raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
|
29
|
+
# self.bury
|
30
|
+
# raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
|
31
31
|
end
|
32
32
|
|
33
33
|
# Sets the delegator object to the underlying beaneater job
|
@@ -60,12 +60,12 @@ module Backburner
|
|
60
60
|
|
61
61
|
def bury
|
62
62
|
@hooks.invoke_hook_events(job_name, :on_bury, *args)
|
63
|
-
task.bury
|
63
|
+
@task.bury
|
64
64
|
end
|
65
65
|
|
66
66
|
def retry(count, delay)
|
67
67
|
@hooks.invoke_hook_events(job_name, :on_retry, count, delay, *args)
|
68
|
-
task.release(delay: delay)
|
68
|
+
@task.release(delay: delay)
|
69
69
|
end
|
70
70
|
|
71
71
|
# Returns the class for the job handler
|
data/lib/backburner/version.rb
CHANGED
data/lib/backburner/worker.rb
CHANGED
@@ -130,11 +130,13 @@ module Backburner
|
|
130
130
|
begin
|
131
131
|
job = reserve_job(conn)
|
132
132
|
rescue Exception => e
|
133
|
+
self.log_error "Sleeping"
|
134
|
+
self.log_error "Exception: #{e.full_message}"
|
133
135
|
sleep(rand*3)
|
134
136
|
return
|
135
137
|
end
|
136
138
|
|
137
|
-
if job
|
139
|
+
if job && job.body
|
138
140
|
begin
|
139
141
|
self.log_job_begin(job.name, job.args)
|
140
142
|
job.process
|
@@ -156,6 +158,9 @@ module Backburner
|
|
156
158
|
retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
|
157
159
|
retry_delay = resolve_retry_delay(job.job_class)
|
158
160
|
delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
|
161
|
+
puts "num_retries = #{num_retries}"
|
162
|
+
puts "max_job_retries = #{max_job_retries}"
|
163
|
+
|
159
164
|
if num_retries + 1 > max_job_retries
|
160
165
|
job.bury
|
161
166
|
else
|
@@ -165,6 +170,8 @@ module Backburner
|
|
165
170
|
|
166
171
|
handle_error(e, job.name, job.args, job)
|
167
172
|
end
|
173
|
+
else
|
174
|
+
sleep(rand*3)
|
168
175
|
end
|
169
176
|
job
|
170
177
|
end
|
@@ -180,7 +187,7 @@ module Backburner
|
|
180
187
|
# Reserve a job from the watched queues
|
181
188
|
def reserve_job(conn, reserve_timeout = Backburner.configuration.reserve_timeout)
|
182
189
|
job = conn.get(@tube_names.sample)
|
183
|
-
return nil if job.body == nil?
|
190
|
+
return nil if job.nil? || job.body == nil?
|
184
191
|
Backburner::Job.new(job)
|
185
192
|
end
|
186
193
|
|
@@ -11,7 +11,7 @@ module Backburner
|
|
11
11
|
def prepare
|
12
12
|
self.tube_names.map! { |name| expand_tube_name(name) }.uniq!
|
13
13
|
log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
|
14
|
-
self.connection.tubes.watch!(*self.tube_names)
|
14
|
+
# self.connection.tubes.watch!(*self.tube_names)
|
15
15
|
end
|
16
16
|
|
17
17
|
# Starts processing new jobs indefinitely.
|
@@ -11,7 +11,7 @@ module Backburner
|
|
11
11
|
def prepare
|
12
12
|
self.tube_names.map! { |name| expand_tube_name(name) }.uniq!
|
13
13
|
log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
|
14
|
-
self.connection.tubes.watch!(*self.tube_names)
|
14
|
+
# self.connection.tubes.watch!(*self.tube_names)
|
15
15
|
end
|
16
16
|
|
17
17
|
# Starts processing new jobs indefinitely.
|
@@ -49,8 +49,8 @@ module Backburner
|
|
49
49
|
@thread_pools.each do |tube_name, pool|
|
50
50
|
pool.max_length.times do
|
51
51
|
# Create a new connection and set it up to listen on this tube name
|
52
|
-
connection = new_connection.tap{ |conn| conn.tubes.watch!(tube_name) }
|
53
|
-
connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
|
52
|
+
# connection = new_connection.tap{ |conn| conn.tubes.watch!(tube_name) }
|
53
|
+
# connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
|
54
54
|
|
55
55
|
# Make it work jobs using its own connection per thread
|
56
56
|
pool.post(connection) do |memo_connection|
|
@@ -181,6 +181,8 @@ module Backburner
|
|
181
181
|
|
182
182
|
@runs = 0
|
183
183
|
|
184
|
+
puts "Threads number = #{@threads_number}"
|
185
|
+
|
184
186
|
if @threads_number == 1
|
185
187
|
watch_tube(name)
|
186
188
|
run_while_can
|
@@ -205,11 +207,15 @@ module Backburner
|
|
205
207
|
|
206
208
|
# Run work_one_job while we can
|
207
209
|
def run_while_can(conn = connection)
|
210
|
+
puts "Run while can"
|
208
211
|
while @garbage_after.nil? or @garbage_after > @runs
|
209
212
|
@runs += 1 # FIXME: Likely race condition
|
210
213
|
ran_job = work_one_job(conn)
|
211
214
|
# Wait a second if we didn't find a job
|
212
|
-
|
215
|
+
unless ran_job
|
216
|
+
puts "sleeping"
|
217
|
+
sleep(rand() * 3)
|
218
|
+
end
|
213
219
|
end
|
214
220
|
end
|
215
221
|
|