backburner-allq 1.0.31 → 1.0.32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/deploy.sh +1 -1
- data/lib/backburner.rb +4 -2
- data/lib/backburner/allq_wrapper.rb +36 -63
- data/lib/backburner/connection.rb +2 -1
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +8 -1
- data/lib/backburner/workers/threads_on_fork.rb +1 -1
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6207b9cc69fea65114989c7ec8d69aceb41fc1f9a68808dfc04c3ecc63600f33
|
4
|
+
data.tar.gz: 3fcf11b39e0b37159ec932ca453d75b16cec3236ea5bd8b5388e2fc9b5d6ab33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 195fb3af0b6e411df1a38c580d5b11e871a0552cb301e8cda1b5ef6ec2148cceac1518dd2ddd727692d1b04440a13a374e4ca0e1bacf386756a0ca0598ea8184
|
7
|
+
data.tar.gz: 1b7761257c2af95899cd6a2a29d53129d5a34d2f52ed97bddf00b936a2064f6f996a194c54c6edabb4ea2826d141260154a501e7a671a2fc445baede9ce1029f
|
data/deploy.sh
CHANGED
data/lib/backburner.rb
CHANGED
@@ -24,10 +24,12 @@ module Backburner
|
|
24
24
|
# @example
|
25
25
|
# Backburner.enqueue NewsletterSender, self.id, user.id
|
26
26
|
#
|
27
|
-
def enqueue(job_class, args,
|
28
|
-
|
27
|
+
def enqueue(job_class, args, opts={})
|
28
|
+
opts[:shard_key] = opts[:shard_key] ? "X" : opts[:shard_key].to_s
|
29
|
+
Backburner::Worker.enqueue(job_class, args, opts)
|
29
30
|
end
|
30
31
|
|
32
|
+
|
31
33
|
# Begins working on jobs enqueued with optional tubes specified
|
32
34
|
#
|
33
35
|
# @example
|
@@ -21,7 +21,7 @@ module Backburner
|
|
21
21
|
ran = true
|
22
22
|
end
|
23
23
|
# Wait if nothing returned
|
24
|
-
sleep(rand
|
24
|
+
sleep(rand * 3) unless ran
|
25
25
|
end
|
26
26
|
end
|
27
27
|
end
|
@@ -81,7 +81,7 @@ module Backburner
|
|
81
81
|
end
|
82
82
|
|
83
83
|
class AllQWrapper
|
84
|
-
DEFAULT_TIMEOUT =
|
84
|
+
DEFAULT_TIMEOUT = 17_800
|
85
85
|
def initialize(url = 'localhost:8090')
|
86
86
|
allq_conf = Allq::Configuration.new do |config|
|
87
87
|
config.host = url
|
@@ -91,14 +91,12 @@ module Backburner
|
|
91
91
|
@client = Allq::ActionsApi.new(raw_client)
|
92
92
|
@admin = Allq::AdminApi.new(raw_client)
|
93
93
|
@recent_times = []
|
94
|
-
|
95
94
|
end
|
96
95
|
|
97
96
|
def speed
|
98
|
-
if @recent_times.size > 0
|
99
|
-
|
100
|
-
|
101
|
-
return 0
|
97
|
+
return @recent_times.sum(0.0) / @recent_times.size if @recent_times.size > 0
|
98
|
+
|
99
|
+
0
|
102
100
|
end
|
103
101
|
|
104
102
|
def touch(job)
|
@@ -118,7 +116,7 @@ module Backburner
|
|
118
116
|
end
|
119
117
|
|
120
118
|
def bury(job)
|
121
|
-
|
119
|
+
@client.bury_put(job.id)
|
122
120
|
end
|
123
121
|
|
124
122
|
def tube_names
|
@@ -131,13 +129,11 @@ module Backburner
|
|
131
129
|
end
|
132
130
|
|
133
131
|
def peek_buried(tube_name = 'default')
|
134
|
-
job = nil
|
135
132
|
job = @client.peek_get(tube_name, buried: true)
|
136
133
|
return nil if job.body.nil?
|
137
134
|
|
138
135
|
job.body = Base64.decode64(job.body) if job
|
139
|
-
|
140
|
-
job_obj
|
136
|
+
Backburner::AllQJob.new(self, job)
|
141
137
|
end
|
142
138
|
|
143
139
|
def get(tube_name = 'default')
|
@@ -147,19 +143,18 @@ module Backburner
|
|
147
143
|
# Inplace decode
|
148
144
|
job.body = Base64.decode64(job.body) if job&.body
|
149
145
|
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
puts("COUDNT RESOLVE HOST NAME------ SHOULD REBOOT")
|
146
|
+
Backburner::AllQJob.new(self, job)
|
147
|
+
rescue StandardError => e
|
148
|
+
if e.message == "Couldn't resolve host name"
|
149
|
+
puts('COUDNT RESOLVE HOST NAME------ SHOULD REBOOT')
|
155
150
|
else
|
156
|
-
puts(
|
151
|
+
puts(e)
|
157
152
|
end
|
158
153
|
end
|
159
154
|
|
160
155
|
def close
|
161
|
-
rescue StandardError =>
|
162
|
-
puts(
|
156
|
+
rescue StandardError => e
|
157
|
+
puts(e)
|
163
158
|
end
|
164
159
|
|
165
160
|
def map_priority(app_priority)
|
@@ -168,16 +163,14 @@ module Backburner
|
|
168
163
|
# IF already using allq-like priority, stick with it
|
169
164
|
return app_priority if app_priority < 11 && app_priority > 0
|
170
165
|
|
171
|
-
default = 5
|
172
|
-
|
173
166
|
# return app_priority unless larger than 10
|
174
167
|
app_priority > 10 ? 5 : app_priority
|
175
168
|
end
|
176
169
|
|
177
170
|
def log_result(job_result)
|
178
171
|
puts("ALLQ-HTTP-JOB-ID=#{job_result.job_id}")
|
179
|
-
rescue StandardError =>
|
180
|
-
puts(
|
172
|
+
rescue StandardError => e
|
173
|
+
puts(e)
|
181
174
|
end
|
182
175
|
|
183
176
|
def build_new_job(body, options)
|
@@ -188,14 +181,13 @@ module Backburner
|
|
188
181
|
delay = options[:delay] || 0
|
189
182
|
parent_id = options[:parent_id]
|
190
183
|
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
new_job
|
184
|
+
Allq::NewJob.new(tube: tube_name,
|
185
|
+
body: Base64.strict_encode64(body),
|
186
|
+
ttl: ttl,
|
187
|
+
delay: delay,
|
188
|
+
priority: adjusted_priority,
|
189
|
+
shard_key: options[:shard_key],
|
190
|
+
parent_id: parent_id)
|
199
191
|
end
|
200
192
|
|
201
193
|
def build_new_parent_job(body, options)
|
@@ -208,16 +200,15 @@ module Backburner
|
|
208
200
|
timeout = options[:timeout] || 3_600
|
209
201
|
run_on_timeout = options[:run_on_timeout] || false
|
210
202
|
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
new_parent_job
|
203
|
+
Allq::NewParentJob.new(tube: tube_name,
|
204
|
+
body: Base64.strict_encode64(body),
|
205
|
+
ttl: ttl,
|
206
|
+
delay: delay,
|
207
|
+
priority: adjusted_priority,
|
208
|
+
timeout: timeout,
|
209
|
+
run_on_timeout: run_on_timeout,
|
210
|
+
shard_key: options[:shard_key],
|
211
|
+
limit: limit)
|
221
212
|
end
|
222
213
|
|
223
214
|
def put(body, options)
|
@@ -246,9 +237,9 @@ module Backburner
|
|
246
237
|
retry_count += 1
|
247
238
|
retry if retry_count < 4
|
248
239
|
raise 'Failed to put on allq, we are investigating the problem, please try again'
|
249
|
-
rescue StandardError =>
|
240
|
+
rescue StandardError => e
|
250
241
|
puts('Failed to ALLQ PUT')
|
251
|
-
puts(
|
242
|
+
puts(e)
|
252
243
|
retry_count += 1
|
253
244
|
sleep(5)
|
254
245
|
retry if retry_count < 4
|
@@ -278,27 +269,9 @@ module Backburner
|
|
278
269
|
end
|
279
270
|
end
|
280
271
|
final_stats
|
281
|
-
rescue StandardError =>
|
282
|
-
puts(
|
272
|
+
rescue StandardError => e
|
273
|
+
puts(e)
|
283
274
|
{}
|
284
275
|
end
|
285
|
-
|
286
|
-
def get_ready_by_tube(name)
|
287
|
-
count = -1
|
288
|
-
tube_stats = stats[name]
|
289
|
-
count = tube_stats['ready'].to_i if tube_stats && tube_stats['ready']
|
290
|
-
count
|
291
|
-
rescue StandardError => ex
|
292
|
-
puts(ex)
|
293
|
-
-1
|
294
|
-
end
|
295
|
-
|
296
|
-
def size
|
297
|
-
result = get_ready_by_tube('default')
|
298
|
-
result.to_i
|
299
|
-
rescue StandardError => ex
|
300
|
-
puts(ex)
|
301
|
-
0
|
302
|
-
end
|
303
276
|
end
|
304
277
|
end
|
data/lib/backburner/version.rb
CHANGED
data/lib/backburner/worker.rb
CHANGED
@@ -40,7 +40,14 @@ module Backburner
|
|
40
40
|
connection.retryable do
|
41
41
|
tube_name = expand_tube_name(queue || job_class)
|
42
42
|
serialized_data = Backburner.configuration.job_serializer_proc.call(data)
|
43
|
-
|
43
|
+
send_data = {
|
44
|
+
pri: pri,
|
45
|
+
delay: delay,
|
46
|
+
ttr: ttr
|
47
|
+
}
|
48
|
+
opts.merge!(send_data)
|
49
|
+
|
50
|
+
response = connection.put(tube_name, serialized_data, opts)
|
44
51
|
end
|
45
52
|
return nil unless Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
|
46
53
|
ensure
|
@@ -207,7 +207,7 @@ module Backburner
|
|
207
207
|
|
208
208
|
# Run work_one_job while we can
|
209
209
|
def run_while_can(conn = connection)
|
210
|
-
|
210
|
+
log_info "Run while can"
|
211
211
|
while @garbage_after.nil? or @garbage_after > @runs
|
212
212
|
@runs += 1 # FIXME: Likely race condition
|
213
213
|
work_one_job(conn, @watched_tube_name)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: backburner-allq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.32
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jason Malcolm
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-08-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: allq_rest
|
@@ -188,8 +188,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
188
188
|
- !ruby/object:Gem::Version
|
189
189
|
version: '0'
|
190
190
|
requirements: []
|
191
|
-
|
192
|
-
rubygems_version: 2.7.9
|
191
|
+
rubygems_version: 3.0.9
|
193
192
|
signing_key:
|
194
193
|
specification_version: 4
|
195
194
|
summary: Reliable allq background job processing made easy for Ruby and Sinatra
|