sidekiq-batch 0.1.9 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b70659750dde1e5afc337c6d8694055af5e103718a3719e3e12bffd66dbe191f
-  data.tar.gz: c47488bb9c1a9115e3bc9ee006190d3dd52534006b8bd74184810885e0dbbfcd
+  metadata.gz: e0e119457317bdc0bbe60785e41d4fff908f0d818ec760802b96fd9905ad14fc
+  data.tar.gz: 46cd957ceb643e946a6bf5ebb49ae93e14e3514384753fa7d910da0476e720b3
 SHA512:
-  metadata.gz: afb196a979725830e0576d4d9f1756c7d3afba5298c46d22b575a9cfbab39fcefe19434f27fd80808d5082a92e127784eb543408e7848e50f5f78e962c5e66a6
-  data.tar.gz: 9cfd132e6beb024c493b6c8acbf7ea375604ed193575abc9ac64b6ee1c10e37992796f80f6441661d24efb9b55963b50e12959b56bb30bf6abb5de6226977125
+  metadata.gz: c86dcadbf158fb9a533163ad042e77d18192241ab7985c4204b6526f78d3cb307a08a48fc173dcef7a7697dfb7316a847fe664ec098178b9e244d7d279978fbb
+  data.tar.gz: a070c7c91c502c06adc83269c3f28b97789250a80b8518ce80ba48511ddacb80537e7ffe9075c03f45062384a53b50fd03af46cb9f3b8dba39e3f03f0498e8a8

data/.github/FUNDING.yml

@@ -1,3 +1,3 @@
 # These are supported funding model platforms
-
+github: breamware
 open_collective: sidekiq-batch

data/.github/workflows/ci.yml

@@ -1,16 +1,30 @@
 name: CI
 
-on: [push, pull_request]
+on: [pull_request, workflow_dispatch]
 
 jobs:
   test:
+    env:
+      REDIS_HOST: 'redis'
 
     runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps port 6379 on service container to the host
+          - 6379:6379
 
     strategy:
       fail-fast: false
       matrix:
-        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", ruby-head]
+        ruby: ["3.0", "3.1", "3.2", ruby-head]
 
     steps:
     - uses: actions/checkout@v2

data/README.md

@@ -32,7 +32,41 @@ Or install it yourself as:
 
 ## Usage
 
-Sidekiq Batch is drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage.
+Sidekiq Batch is MOSTLY a drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage.
+
+## Caveats/Gotchas
+
+Consider the following workflow:
+
+  * Batch Z created
+  * Worker A queued in batch Z
+  * Worker A starts Worker B in batch Z
+  * Worker B completes *before* worker A does
+  * Worker A completes
+
+In the standard configuration, the `on(:success)` and `on(:complete)` callbacks will be triggered when Worker B completes.
+This configuration is the default, simply for legacy reasons. This gem adds the following option to the sidekiq.yml options:
+
+```yaml
+:batch_push_interval: 0
+```
+
+When this value is *absent* (aka legacy), Worker A will only push the increment of batch jobs (aka Worker B) *when it completes*
+
+When this value is set to `0`, Worker A will increment the count as soon as `WorkerB.perform_async` is called
+
+When this value is a positive number, Worker A will wait a maximum of value-seconds before pushing the increment to redis, or until it's done, whichever comes first.
+
+This comes into play if Worker A is queueing thousands of WorkerB jobs, or has some other reason for WorkerB to complete beforehand.
+
+If you are queueing many WorkerB jobs, it is recommended to set this value to something like `3` to avoid thousands of calls to redis, and call WorkerB like so:
+```ruby
+WorkerB.perform_in(4.seconds, some, args)
+```
+this will ensure that the batch callback does not get triggered until WorkerA *and* the last WorkerB job complete.
+
+If WorkerA is just slow for whatever reason, setting to `0` will update the batch status immediately so that the callbacks don't fire.
+
 
 ## Contributing
 

data/lib/sidekiq/batch/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   class Batch
-    VERSION = '0.1.9'.freeze
+    VERSION = '0.2.0'.freeze
   end
 end

data/lib/sidekiq/batch.rb

@@ -20,7 +20,11 @@ module Sidekiq
       @initialized = false
       @created_at = Time.now.utc.to_f
       @bidkey = "BID-" + @bid.to_s
-      @ready_to_queue = []
+      @queued_jids = []
+      @pending_jids = []
+
+      @incremental_push = !Sidekiq.default_configuration[:batch_push_interval].nil?
+      @batch_push_interval = Sidekiq.default_configuration[:batch_push_interval]
     end
 
     def description=(description)
@@ -64,51 +68,86 @@ module Sidekiq
           Sidekiq.redis do |r|
             r.multi do |pipeline|
               pipeline.hset(@bidkey, "created_at", @created_at)
-              pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
               pipeline.expire(@bidkey, BID_EXPIRE_TTL)
+              if parent_bid
+                pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s)
+                pipeline.hincrby("BID-#{parent_bid}", "children", 1)
+              end
             end
           end
 
           @initialized = true
         end
 
-        @ready_to_queue = []
+        @queued_jids = []
+        @pending_jids = []
 
         begin
           parent = Thread.current[:batch]
           Thread.current[:batch] = self
+          Thread.current[:parent_bid] = parent_bid
           yield
         ensure
           Thread.current[:batch] = parent
+          Thread.current[:parent_bid] = nil
         end
 
-        return [] if @ready_to_queue.size == 0
+        return [] if @queued_jids.size == 0
+        conditional_redis_increment!(true)
 
         Sidekiq.redis do |r|
           r.multi do |pipeline|
             if parent_bid
-              pipeline.hincrby("BID-#{parent_bid}", "children", 1)
-              pipeline.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
               pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
             end
 
-            pipeline.hincrby(@bidkey, "pending", @ready_to_queue.size)
-            pipeline.hincrby(@bidkey, "total", @ready_to_queue.size)
             pipeline.expire(@bidkey, BID_EXPIRE_TTL)
 
-            pipeline.sadd(@bidkey + "-jids", [@ready_to_queue])
+            pipeline.sadd(@bidkey + "-jids", @queued_jids)
             pipeline.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
           end
         end
 
-        @ready_to_queue
+        @queued_jids
       ensure
         Thread.current[:bid_data] = bid_data
       end
     end
 
     def increment_job_queue(jid)
-      @ready_to_queue << jid
+      @queued_jids << jid
+      @pending_jids << jid
+      conditional_redis_increment!
+    end
+
+    def conditional_redis_increment!(force=false)
+      if should_increment? || force
+        parent_bid = Thread.current[:parent_bid]
+        Sidekiq.redis do |r|
+          r.multi do |pipeline|
+            if parent_bid
+              pipeline.hincrby("BID-#{parent_bid}", "total", @pending_jids.length)
+              pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            end
+
+            pipeline.hincrby(@bidkey, "pending", @pending_jids.length)
+            pipeline.hincrby(@bidkey, "total", @pending_jids.length)
+            pipeline.expire(@bidkey, BID_EXPIRE_TTL)
+          end
+        end
+        @pending_jids = []
+      end
+    end
+
+    def should_increment?
+      return false unless @incremental_push
+      return true if @batch_push_interval == 0 || @queued_jids.length == 1
+      now = Time.now.to_f
+      @last_increment ||= now
+      if @last_increment + @batch_push_interval > now
+        @last_increment = now
+        return true
+      end
     end
 
     def invalidate_all
@@ -130,7 +169,7 @@ module Sidekiq
     end
 
     def valid?(batch = self)
-      valid = !Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }
+      valid = Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }.zero?
       batch.parent ? valid && valid?(batch.parent) : valid
     end
 
@@ -209,7 +248,7 @@ module Sidekiq
         already_processed, _, callbacks, queue, parent_bid, callback_batch = Sidekiq.redis do |r|
           r.multi do |pipeline|
             pipeline.hget(batch_key, event_name)
-            pipeline.hset(batch_key, event_name, true)
+            pipeline.hset(batch_key, event_name, 'true')
             pipeline.smembers(callback_key)
             pipeline.hget(batch_key, "callback_queue")
             pipeline.hget(batch_key, "parent_bid")
@@ -253,7 +292,7 @@ module Sidekiq
         else
           # Otherwise finalize in sub batch complete callback
           cb_batch = self.new
-          cb_batch.callback_batch = true
+          cb_batch.callback_batch = 'true'
           Sidekiq.logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
           cb_batch.on(:complete, "Sidekiq::Batch::Callback::Finalize#dispatch", opts)
           cb_batch.jobs do

data/sidekiq-batch.gemspec

@@ -19,10 +19,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "sidekiq", ">= 3"
+  spec.add_dependency "sidekiq", ">= 7", "<8"
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "fakeredis", "~> 0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-batch
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.2.0
 platform: ruby
 authors:
 - Marcin Naglik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-10-03 00:00:00.000000000 Z
+date: 2024-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,20 +72,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: fakeredis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.8.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.8.0
 description: Sidekiq Batch Jobs Implementation
 email:
 - marcin.naglik@gmail.com
@@ -126,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: Sidekiq Batch Jobs