resque-scheduler 4.7.0 → 4.9.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of resque-scheduler might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4bd2a447d845ab5c056174d73fb88995693d998b06b90a3b565d3aa15981c3bb
4
- data.tar.gz: d3cbbd2b7bdff68726bb9565e5c00531c7eb18397915fe6edfe568a9f0d34c26
3
+ metadata.gz: be1ad7d71c519660fafb8b0ccdae4b2b508e9577a1af5dbb8a57f8c7bce1a02f
4
+ data.tar.gz: 742e298554473ed422e15c420a981f38c446c333636d6cdd09cd511eebda4584
5
5
  SHA512:
6
- metadata.gz: b91aca3627cc71d2082702b3a46be5ae1d38fd6b75961b49fd41c61c78fb3921dbd469a64ed3fb89348d6c00e79dc09c86a68ede59d75ed11d12ef55097bd6bf
7
- data.tar.gz: eae2165cb529f4a38b8321f9c2dfe120ffeae5b13c001819af6bde9901a084a1710724caea469ffbd1b88266621b56e64c7f79b11e75ae55ca45f688497f8d24
6
+ metadata.gz: 91874b66e46123e8ef0476518b98ee6a1923fba44d4feff923bdf1be0b2319da8a60141dbaf91c99e0a62817c8904c223c67f38a6b153b54fc63b8717aa5235d
7
+ data.tar.gz: 55d87dfb5ec01b799f5062354bfa0d80b1194833b7a52eed0e4807f55f2bd50df7e231166ea73847c6195423b12ca76b4732b94409a755bdccf280e3ce2b739b
@@ -26,6 +26,7 @@ jobs:
26
26
  - 2.7
27
27
  - "3.0"
28
28
  - 3.1
29
+ - 3.2
29
30
  resque-version:
30
31
  - "master"
31
32
  - "~> 2.4.0"
@@ -41,6 +42,8 @@ jobs:
41
42
  exclude:
42
43
  - ruby-version: head
43
44
  rufus-scheduler: 3.2
45
+ - ruby-version: 3.2
46
+ rufus-scheduler: 3.2
44
47
 
45
48
  - ruby-version: 2.3
46
49
  resque-version: "~> 1.27"
data/AUTHORS.md CHANGED
@@ -29,6 +29,7 @@ Resque Scheduler authors
29
29
  - Henrik Nyh
30
30
  - Hormoz Kheradmand
31
31
  - Ian Davies
32
+ - Irving Reid
32
33
  - James Le Cuirot
33
34
  - Jarkko Mönkkönen
34
35
  - Jimmy Chao
@@ -87,4 +88,4 @@ Resque Scheduler authors
87
88
  - malomalo
88
89
  - sawanoboly
89
90
  - serek
90
- - iloveitaly
91
+ - iloveitaly
data/CHANGELOG.md CHANGED
@@ -2,6 +2,19 @@
2
2
 
3
3
  **ATTN**: This project uses [semantic versioning](http://semver.org/).
4
4
 
5
+ ## [4.9.0] - 2023-05-31
6
+ ### Changed
7
+ - Adding batching to re-queuing for timestamp by @brennen-stripe in #767
8
+ - Use non-deprecated form of Redis.sadd/srem by @irvingreid in #752
9
+ - Prompt for confirmation on 'Clear Delayed Jobs' by @mishina2228 in #754
10
+ - Address some deprecation warnings in the test suite and with srem/lrem pipelined usage by @PatrickTulskie in #770
11
+
12
+ ### Fixed
13
+ - Update CI matrix and fix tests by @mishina2228 in #766
14
+
15
+ ## [4.8.0] - 2023-27-1
16
+ - Replace deprecated Socket.gethostname with Addrinfo.getaddrinfo to fix deprecation warnings (#753)
17
+
5
18
  ## [4.7.0] - 2022-10-6
6
19
  ### Fixed
7
20
  - Fix tests for redis-rb 5.0 (#757)
data/Gemfile CHANGED
@@ -24,4 +24,6 @@ else
24
24
  gem 'redis', redis_version
25
25
  end
26
26
 
27
+ gem 'sinatra', '> 2.0'
28
+
27
29
  gemspec
@@ -65,6 +65,12 @@ module Resque
65
65
  @app_name ||= environment['APP_NAME']
66
66
  end
67
67
 
68
+ def delayed_requeue_batch_size
69
+ @delayed_requeue_batch_size ||= \
70
+ ENV['DELAYED_REQUEUE_BATCH_SIZE'].to_i if environment['DELAYED_REQUEUE_BATCH_SIZE']
71
+ @delayed_requeue_batch_size ||= 100
72
+ end
73
+
68
74
  # Amount of time in seconds to sleep between polls of the delayed
69
75
  # queue. Defaults to 5
70
76
  attr_writer :poll_sleep_amount
@@ -90,7 +90,7 @@ module Resque
90
90
  redis.rpush("delayed:#{timestamp.to_i}", encode(item))
91
91
 
92
92
  # Store the timestamps at with this item occurs
93
- redis.sadd("timestamps:#{encode(item)}", "delayed:#{timestamp.to_i}")
93
+ redis.sadd("timestamps:#{encode(item)}", ["delayed:#{timestamp.to_i}"])
94
94
 
95
95
  # Now, add this timestamp to the zsets. The score and the value are
96
96
  # the same since we'll be querying by timestamp, and we don't have
@@ -140,7 +140,7 @@ module Resque
140
140
  key = "delayed:#{timestamp.to_i}"
141
141
 
142
142
  encoded_item = redis.lpop(key)
143
- redis.srem("timestamps:#{encoded_item}", key)
143
+ redis.srem("timestamps:#{encoded_item}", [key])
144
144
  item = decode(encoded_item)
145
145
 
146
146
  # If the list is empty, remove it.
@@ -257,7 +257,7 @@ module Resque
257
257
  key = "delayed:#{timestamp.to_i}"
258
258
  encoded_job = encode(job_to_hash(klass, args))
259
259
 
260
- redis.srem("timestamps:#{encoded_job}", key)
260
+ redis.srem("timestamps:#{encoded_job}", [key])
261
261
  count = redis.lrem(key, 0, encoded_job)
262
262
  clean_up_timestamp(key, timestamp)
263
263
 
@@ -297,6 +297,22 @@ module Resque
297
297
  redis.hget('delayed:last_enqueued_at', job_name)
298
298
  end
299
299
 
300
+ def clean_up_timestamp(key, timestamp)
301
+ # Use a watch here to ensure nobody adds jobs to this delayed
302
+ # queue while we're removing it.
303
+ redis.watch(key) do
304
+ if redis.llen(key).to_i == 0
305
+ # If the list is empty, remove it.
306
+ redis.multi do |transaction|
307
+ transaction.del(key)
308
+ transaction.zrem(:delayed_queue_schedule, timestamp.to_i)
309
+ end
310
+ else
311
+ redis.redis.unwatch
312
+ end
313
+ end
314
+ end
315
+
300
316
  private
301
317
 
302
318
  def job_to_hash(klass, args)
@@ -317,7 +333,7 @@ module Resque
317
333
  replies = redis.pipelined do |pipeline|
318
334
  timestamps.each do |key|
319
335
  pipeline.lrem(key, 0, encoded_job)
320
- pipeline.srem("timestamps:#{encoded_job}", key)
336
+ pipeline.srem("timestamps:#{encoded_job}", [key])
321
337
  end
322
338
  end
323
339
 
@@ -328,22 +344,6 @@ module Resque
328
344
  replies.each_slice(2).map(&:first).inject(:+)
329
345
  end
330
346
 
331
- def clean_up_timestamp(key, timestamp)
332
- # Use a watch here to ensure nobody adds jobs to this delayed
333
- # queue while we're removing it.
334
- redis.watch(key) do
335
- if redis.llen(key).to_i == 0
336
- # If the list is empty, remove it.
337
- redis.multi do |transaction|
338
- transaction.del(key)
339
- transaction.zrem(:delayed_queue_schedule, timestamp.to_i)
340
- end
341
- else
342
- redis.redis.unwatch
343
- end
344
- end
345
- end
346
-
347
347
  def search_first_delayed_timestamp_in_range(start_at, stop_at)
348
348
  start_at = start_at.nil? ? '-inf' : start_at.to_i
349
349
  stop_at = stop_at.nil? ? '+inf' : stop_at.to_i
@@ -47,7 +47,7 @@ module Resque
47
47
 
48
48
  def hostname
49
49
  local_hostname = Socket.gethostname
50
- Socket.gethostbyname(local_hostname).first
50
+ Addrinfo.getaddrinfo(local_hostname, 'http').first.getnameinfo.first
51
51
  rescue
52
52
  local_hostname
53
53
  end
@@ -91,7 +91,7 @@ module Resque
91
91
  non_persistent_schedules[name] = decode(encode(config))
92
92
  end
93
93
 
94
- redis.sadd(:schedules_changed, name)
94
+ redis.sadd(:schedules_changed, [name])
95
95
  reload_schedule! if reload
96
96
  end
97
97
 
@@ -105,7 +105,7 @@ module Resque
105
105
  def remove_schedule(name, reload = true)
106
106
  non_persistent_schedules.delete(name)
107
107
  redis.hdel(:persistent_schedules, name)
108
- redis.sadd(:schedules_changed, name)
108
+ redis.sadd(:schedules_changed, [name])
109
109
 
110
110
  reload_schedule! if reload
111
111
  end
@@ -16,6 +16,14 @@
16
16
  Showing <%= start = params[:start].to_i %> to <%= start + 20 %> of <b><%= size %></b> timestamps
17
17
  </p>
18
18
 
19
+ <% if size > 0 %>
20
+ <div style="padding-bottom: 10px">
21
+ <form method="POST" action="<%= u 'delayed/clear' %>" class='clear-delayed confirmSubmission'>
22
+ <input type='submit' name='' value='Clear Delayed Jobs' />
23
+ </form>
24
+ </div>
25
+ <% end %>
26
+
19
27
  <table>
20
28
  <tr>
21
29
  <th></th>
@@ -27,7 +35,7 @@
27
35
  </tr>
28
36
  <% resque.delayed_queue_peek(start, 20).each do |timestamp| %>
29
37
  <tr>
30
- <td>
38
+ <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
31
39
  <form action="<%= u "/delayed/queue_now" %>" method="post">
32
40
  <input type="hidden" name="timestamp" value="<%= timestamp.to_i %>">
33
41
  <input type="submit" value="Queue now">
@@ -53,11 +61,4 @@
53
61
  <% end %>
54
62
  </table>
55
63
 
56
- <% if size > 0 %>
57
- <br>
58
- <form method="POST" action="<%= u 'delayed/clear' %>" class='clear-delayed'>
59
- <input type='submit' name='' value='Clear Delayed Jobs' />
60
- </form>
61
- <% end %>
62
-
63
64
  <%= partial :next_more, :start => start, :size => size %>
@@ -13,13 +13,13 @@
13
13
  </tr>
14
14
  <% delayed.each do |job| %>
15
15
  <tr>
16
- <td>
16
+ <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
17
17
  <form action="<%= u "/delayed/queue_now" %>" method="post">
18
18
  <input type="hidden" name="timestamp" value="<%= job['timestamp'].to_i %>">
19
19
  <input type="submit" value="Queue now">
20
20
  </form>
21
21
  </td>
22
- <td>
22
+ <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
23
23
  <form action="<%= u "/delayed/cancel_now" %>" method="post">
24
24
  <input type="hidden" name="timestamp" value="<%= job['timestamp'].to_i %>">
25
25
  <input type="hidden" name="klass" value="<%= job['class'] %>">
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Resque
4
4
  module Scheduler
5
- VERSION = '4.7.0'.freeze
5
+ VERSION = '4.9.0'.freeze
6
6
  end
7
7
  end
@@ -206,16 +206,80 @@ module Resque
206
206
 
207
207
  # Enqueues all delayed jobs for a timestamp
208
208
  def enqueue_delayed_items_for_timestamp(timestamp)
209
- item = nil
209
+ count = 0
210
+ batch_size = delayed_requeue_batch_size
211
+ actual_batch_size = nil
212
+
213
+ log "Processing delayed items for timestamp #{timestamp}, in batches of #{batch_size}"
214
+
210
215
  loop do
211
216
  handle_shutdown do
212
217
  # Continually check that it is still the master
213
- item = enqueue_next_item(timestamp) if am_master
218
+ if am_master
219
+ actual_batch_size = enqueue_items_in_batch_for_timestamp(timestamp,
220
+ batch_size)
221
+ end
214
222
  end
215
- # continue processing until there are no more ready items in this
216
- # timestamp
217
- break if item.nil?
223
+
224
+ count += actual_batch_size
225
+ log "queued #{count} jobs" if actual_batch_size != -1
226
+
227
+ # continue processing until there are no more items in this
228
+ # timestamp. If we don't have a full batch, this is the last one.
229
+ # This also breaks us in the event of a redis transaction failure
230
+ # i.e. enqueue_items_in_batch_for_timestamp returned -1
231
+ break if actual_batch_size < batch_size
218
232
  end
233
+
234
+ log "finished queueing #{count} total jobs for timestamp #{timestamp}" if count != -1
235
+ end
236
+
237
+ def timestamp_key(timestamp)
238
+ "delayed:#{timestamp.to_i}"
239
+ end
240
+
241
+ def enqueue_items_in_batch_for_timestamp(timestamp, batch_size)
242
+ timestamp_bucket_key = timestamp_key(timestamp)
243
+
244
+ encoded_jobs_to_requeue = Resque.redis.lrange(timestamp_bucket_key, 0, batch_size - 1)
245
+
246
+ # Watch is used to ensure that the timestamp bucket we are operating on
247
+ # is not altered by any other clients between the watch call and when we call exec
248
+ # (to execute the multi block). We should error catch on the redis.exec return value
249
+ # as that will indicate if the entire transaction was aborted or not. Though we should
250
+ # be safe as our ltrim is inside the multi block and therefore also would have been
251
+ # aborted. So nothing would have been queued, but also nothing lost from the bucket.
252
+ watch_result = Resque.redis.watch(timestamp_bucket_key) do
253
+ Resque.redis.multi do |pipeline|
254
+ encoded_jobs_to_requeue.each do |encoded_job|
255
+ pipeline.srem("timestamps:#{encoded_job}", timestamp_bucket_key)
256
+
257
+ decoded_job = Resque.decode(encoded_job)
258
+ enqueue(decoded_job)
259
+ end
260
+
261
+ pipeline.ltrim(timestamp_bucket_key, batch_size, -1)
262
+ end
263
+ end
264
+
265
+ # Did the multi block successfully remove from this timestamp and enqueue the jobs?
266
+ success = !watch_result.nil?
267
+
268
+ # If this was the last batch in this timestamp bucket, clean up
269
+ if success && encoded_jobs_to_requeue.count < batch_size
270
+ Resque.clean_up_timestamp(timestamp_bucket_key, timestamp)
271
+ end
272
+
273
+ unless success
274
+ # Our batched transaction failed in Redis due to the timestamp_bucket_key value
275
+ # being modified while we built our multi block. We return -1 to ensure we break
276
+ # out of the loop iterating on this timestamp so it can be re-processed via the
277
+ # loop in handle_delayed_items.
278
+ return -1
279
+ end
280
+
281
+ # will return 0 if none were left to batch
282
+ encoded_jobs_to_requeue.count
219
283
  end
220
284
 
221
285
  def enqueue(config)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: resque-scheduler
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.7.0
4
+ version: 4.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ben VandenBos
@@ -9,10 +9,10 @@ authors:
9
9
  - Ryan Biesemeyer
10
10
  - Dan Buch
11
11
  - Michael Bianco
12
- autorequire:
12
+ autorequire:
13
13
  bindir: exe
14
14
  cert_chain: []
15
- date: 2022-10-07 00:00:00.000000000 Z
15
+ date: 2023-06-08 00:00:00.000000000 Z
16
16
  dependencies:
17
17
  - !ruby/object:Gem::Dependency
18
18
  name: bundler
@@ -307,7 +307,7 @@ licenses:
307
307
  - MIT
308
308
  metadata:
309
309
  rubygems_mfa_required: 'true'
310
- post_install_message:
310
+ post_install_message:
311
311
  rdoc_options: []
312
312
  require_paths:
313
313
  - lib
@@ -322,8 +322,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
322
322
  - !ruby/object:Gem::Version
323
323
  version: '0'
324
324
  requirements: []
325
- rubygems_version: 3.1.2
326
- signing_key:
325
+ rubygems_version: 3.0.3.1
326
+ signing_key:
327
327
  specification_version: 4
328
328
  summary: Light weight job scheduling on top of Resque
329
329
  test_files: []