sidekiq-batch 0.1.9 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b70659750dde1e5afc337c6d8694055af5e103718a3719e3e12bffd66dbe191f
4
- data.tar.gz: c47488bb9c1a9115e3bc9ee006190d3dd52534006b8bd74184810885e0dbbfcd
3
+ metadata.gz: e0e119457317bdc0bbe60785e41d4fff908f0d818ec760802b96fd9905ad14fc
4
+ data.tar.gz: 46cd957ceb643e946a6bf5ebb49ae93e14e3514384753fa7d910da0476e720b3
5
5
  SHA512:
6
- metadata.gz: afb196a979725830e0576d4d9f1756c7d3afba5298c46d22b575a9cfbab39fcefe19434f27fd80808d5082a92e127784eb543408e7848e50f5f78e962c5e66a6
7
- data.tar.gz: 9cfd132e6beb024c493b6c8acbf7ea375604ed193575abc9ac64b6ee1c10e37992796f80f6441661d24efb9b55963b50e12959b56bb30bf6abb5de6226977125
6
+ metadata.gz: c86dcadbf158fb9a533163ad042e77d18192241ab7985c4204b6526f78d3cb307a08a48fc173dcef7a7697dfb7316a847fe664ec098178b9e244d7d279978fbb
7
+ data.tar.gz: a070c7c91c502c06adc83269c3f28b97789250a80b8518ce80ba48511ddacb80537e7ffe9075c03f45062384a53b50fd03af46cb9f3b8dba39e3f03f0498e8a8
data/.github/FUNDING.yml CHANGED
@@ -1,3 +1,3 @@
1
1
  # These are supported funding model platforms
2
-
2
+ github: breamware
3
3
  open_collective: sidekiq-batch
@@ -1,16 +1,30 @@
1
1
  name: CI
2
2
 
3
- on: [push, pull_request]
3
+ on: [pull_request, workflow_dispatch]
4
4
 
5
5
  jobs:
6
6
  test:
7
+ env:
8
+ REDIS_HOST: 'redis'
7
9
 
8
10
  runs-on: ubuntu-latest
11
+ services:
12
+ redis:
13
+ image: redis
14
+ # Set health checks to wait until redis has started
15
+ options: >-
16
+ --health-cmd "redis-cli ping"
17
+ --health-interval 10s
18
+ --health-timeout 5s
19
+ --health-retries 5
20
+ ports:
21
+ # Maps port 6379 on service container to the host
22
+ - 6379:6379
9
23
 
10
24
  strategy:
11
25
  fail-fast: false
12
26
  matrix:
13
- ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", ruby-head]
27
+ ruby: ["3.0", "3.1", "3.2", ruby-head]
14
28
 
15
29
  steps:
16
30
  - uses: actions/checkout@v2
data/README.md CHANGED
@@ -32,7 +32,41 @@ Or install it yourself as:
32
32
 
33
33
  ## Usage
34
34
 
35
- Sidekiq Batch is drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage.
35
+ Sidekiq Batch is MOSTLY a drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage.
36
+
37
+ ## Caveats/Gotchas
38
+
39
+ Consider the following workflow:
40
+
41
+ * Batch Z created
42
+ * Worker A queued in batch Z
43
+ * Worker A starts Worker B in batch Z
44
+ * Worker B completes *before* worker A does
45
+ * Worker A completes
46
+
47
+ In the standard configuration, the `on(:success)` and `on(:complete)` callbacks will be triggered when Worker B completes.
48
+ This configuration is the default, simply for legacy reasons. This gem adds the following option to the sidekiq.yml options:
49
+
50
+ ```yaml
51
+ :batch_push_interval: 0
52
+ ```
53
+
54
+ When this value is *absent* (aka legacy), Worker A will only push the increment of batch jobs (aka Worker B) *when it completes*
55
+
56
+ When this value is set to `0`, Worker A will increment the count as soon as `WorkerB.perform_async` is called
57
+
58
+ When this value is a positive number, Worker A will wait a maximum of value-seconds before pushing the increment to redis, or until it's done, whichever comes first.
59
+
60
+ This comes into play if Worker A is queueing thousands of WorkerB jobs, or has some other reason for WorkerB to complete beforehand.
61
+
62
+ If you are queueing many WorkerB jobs, it is recommended to set this value to something like `3` to avoid thousands of calls to redis, and call WorkerB like so:
63
+ ```ruby
64
+ WorkerB.perform_in(4.seconds, some, args)
65
+ ```
66
+ this will ensure that the batch callback does not get triggered until WorkerA *and* the last WorkerB job complete.
67
+
68
+ If WorkerA is just slow for whatever reason, setting to `0` will update the batch status immediately so that the callbacks don't fire.
69
+
36
70
 
37
71
  ## Contributing
38
72
 
@@ -1,5 +1,5 @@
1
1
  module Sidekiq
2
2
  class Batch
3
- VERSION = '0.1.9'.freeze
3
+ VERSION = '0.2.0'.freeze
4
4
  end
5
5
  end
data/lib/sidekiq/batch.rb CHANGED
@@ -20,7 +20,11 @@ module Sidekiq
20
20
  @initialized = false
21
21
  @created_at = Time.now.utc.to_f
22
22
  @bidkey = "BID-" + @bid.to_s
23
- @ready_to_queue = []
23
+ @queued_jids = []
24
+ @pending_jids = []
25
+
26
+ @incremental_push = !Sidekiq.default_configuration[:batch_push_interval].nil?
27
+ @batch_push_interval = Sidekiq.default_configuration[:batch_push_interval]
24
28
  end
25
29
 
26
30
  def description=(description)
@@ -64,51 +68,86 @@ module Sidekiq
64
68
  Sidekiq.redis do |r|
65
69
  r.multi do |pipeline|
66
70
  pipeline.hset(@bidkey, "created_at", @created_at)
67
- pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
68
71
  pipeline.expire(@bidkey, BID_EXPIRE_TTL)
72
+ if parent_bid
73
+ pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s)
74
+ pipeline.hincrby("BID-#{parent_bid}", "children", 1)
75
+ end
69
76
  end
70
77
  end
71
78
 
72
79
  @initialized = true
73
80
  end
74
81
 
75
- @ready_to_queue = []
82
+ @queued_jids = []
83
+ @pending_jids = []
76
84
 
77
85
  begin
78
86
  parent = Thread.current[:batch]
79
87
  Thread.current[:batch] = self
88
+ Thread.current[:parent_bid] = parent_bid
80
89
  yield
81
90
  ensure
82
91
  Thread.current[:batch] = parent
92
+ Thread.current[:parent_bid] = nil
83
93
  end
84
94
 
85
- return [] if @ready_to_queue.size == 0
95
+ return [] if @queued_jids.size == 0
96
+ conditional_redis_increment!(true)
86
97
 
87
98
  Sidekiq.redis do |r|
88
99
  r.multi do |pipeline|
89
100
  if parent_bid
90
- pipeline.hincrby("BID-#{parent_bid}", "children", 1)
91
- pipeline.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
92
101
  pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
93
102
  end
94
103
 
95
- pipeline.hincrby(@bidkey, "pending", @ready_to_queue.size)
96
- pipeline.hincrby(@bidkey, "total", @ready_to_queue.size)
97
104
  pipeline.expire(@bidkey, BID_EXPIRE_TTL)
98
105
 
99
- pipeline.sadd(@bidkey + "-jids", [@ready_to_queue])
106
+ pipeline.sadd(@bidkey + "-jids", @queued_jids)
100
107
  pipeline.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
101
108
  end
102
109
  end
103
110
 
104
- @ready_to_queue
111
+ @queued_jids
105
112
  ensure
106
113
  Thread.current[:bid_data] = bid_data
107
114
  end
108
115
  end
109
116
 
110
117
  def increment_job_queue(jid)
111
- @ready_to_queue << jid
118
+ @queued_jids << jid
119
+ @pending_jids << jid
120
+ conditional_redis_increment!
121
+ end
122
+
123
+ def conditional_redis_increment!(force=false)
124
+ if should_increment? || force
125
+ parent_bid = Thread.current[:parent_bid]
126
+ Sidekiq.redis do |r|
127
+ r.multi do |pipeline|
128
+ if parent_bid
129
+ pipeline.hincrby("BID-#{parent_bid}", "total", @pending_jids.length)
130
+ pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
131
+ end
132
+
133
+ pipeline.hincrby(@bidkey, "pending", @pending_jids.length)
134
+ pipeline.hincrby(@bidkey, "total", @pending_jids.length)
135
+ pipeline.expire(@bidkey, BID_EXPIRE_TTL)
136
+ end
137
+ end
138
+ @pending_jids = []
139
+ end
140
+ end
141
+
142
+ def should_increment?
143
+ return false unless @incremental_push
144
+ return true if @batch_push_interval == 0 || @queued_jids.length == 1
145
+ now = Time.now.to_f
146
+ @last_increment ||= now
147
+ if @last_increment + @batch_push_interval > now
148
+ @last_increment = now
149
+ return true
150
+ end
112
151
  end
113
152
 
114
153
  def invalidate_all
@@ -130,7 +169,7 @@ module Sidekiq
130
169
  end
131
170
 
132
171
  def valid?(batch = self)
133
- valid = !Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }
172
+ valid = Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }.zero?
134
173
  batch.parent ? valid && valid?(batch.parent) : valid
135
174
  end
136
175
 
@@ -209,7 +248,7 @@ module Sidekiq
209
248
  already_processed, _, callbacks, queue, parent_bid, callback_batch = Sidekiq.redis do |r|
210
249
  r.multi do |pipeline|
211
250
  pipeline.hget(batch_key, event_name)
212
- pipeline.hset(batch_key, event_name, true)
251
+ pipeline.hset(batch_key, event_name, 'true')
213
252
  pipeline.smembers(callback_key)
214
253
  pipeline.hget(batch_key, "callback_queue")
215
254
  pipeline.hget(batch_key, "parent_bid")
@@ -253,7 +292,7 @@ module Sidekiq
253
292
  else
254
293
  # Otherwise finalize in sub batch complete callback
255
294
  cb_batch = self.new
256
- cb_batch.callback_batch = true
295
+ cb_batch.callback_batch = 'true'
257
296
  Sidekiq.logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
258
297
  cb_batch.on(:complete, "Sidekiq::Batch::Callback::Finalize#dispatch", opts)
259
298
  cb_batch.jobs do
@@ -19,10 +19,9 @@ Gem::Specification.new do |spec|
19
19
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
20
20
  spec.require_paths = ["lib"]
21
21
 
22
- spec.add_dependency "sidekiq", ">= 3"
22
+ spec.add_dependency "sidekiq", ">= 7", "<8"
23
23
 
24
24
  spec.add_development_dependency "bundler", "~> 2.1"
25
25
  spec.add_development_dependency "rake", "~> 13.0"
26
26
  spec.add_development_dependency "rspec", "~> 3.0"
27
- spec.add_development_dependency "fakeredis", "~> 0.8.0"
28
27
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq-batch
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.9
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Marcin Naglik
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-10-03 00:00:00.000000000 Z
11
+ date: 2024-08-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sidekiq
@@ -16,14 +16,20 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '3'
19
+ version: '7'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '8'
20
23
  type: :runtime
21
24
  prerelease: false
22
25
  version_requirements: !ruby/object:Gem::Requirement
23
26
  requirements:
24
27
  - - ">="
25
28
  - !ruby/object:Gem::Version
26
- version: '3'
29
+ version: '7'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '8'
27
33
  - !ruby/object:Gem::Dependency
28
34
  name: bundler
29
35
  requirement: !ruby/object:Gem::Requirement
@@ -66,20 +72,6 @@ dependencies:
66
72
  - - "~>"
67
73
  - !ruby/object:Gem::Version
68
74
  version: '3.0'
69
- - !ruby/object:Gem::Dependency
70
- name: fakeredis
71
- requirement: !ruby/object:Gem::Requirement
72
- requirements:
73
- - - "~>"
74
- - !ruby/object:Gem::Version
75
- version: 0.8.0
76
- type: :development
77
- prerelease: false
78
- version_requirements: !ruby/object:Gem::Requirement
79
- requirements:
80
- - - "~>"
81
- - !ruby/object:Gem::Version
82
- version: 0.8.0
83
75
  description: Sidekiq Batch Jobs Implementation
84
76
  email:
85
77
  - marcin.naglik@gmail.com
@@ -126,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
126
118
  - !ruby/object:Gem::Version
127
119
  version: '0'
128
120
  requirements: []
129
- rubygems_version: 3.3.7
121
+ rubygems_version: 3.5.11
130
122
  signing_key:
131
123
  specification_version: 4
132
124
  summary: Sidekiq Batch Jobs