sidekiq-batch 0.1.8 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56a9cb0fe2d48eba0959e20ad98e52e62e326bb71168f13f4c48e0ad33079447
-  data.tar.gz: ae145f7d0381f9a83ec2f0ba93be5a290589b1cd77e7544fc20b4f15fc823d0e
+  metadata.gz: e0e119457317bdc0bbe60785e41d4fff908f0d818ec760802b96fd9905ad14fc
+  data.tar.gz: 46cd957ceb643e946a6bf5ebb49ae93e14e3514384753fa7d910da0476e720b3
 SHA512:
-  metadata.gz: 45aee8ac26397847e5738ebf21b34a7505cd367d64596b9e72ff1c51993c622e53850999cb9bcae4fa2152c7e0471c65d3c824af3af06cdefecee199b4f7c8a5
-  data.tar.gz: 294c3bb0b9f908ee2aabf6dd62adcb296ff6409a148b3db77a456f50c728f369583bf15af4d2aa034947fb5175741985549c70d5328acb8ff471332689ad27c7
+  metadata.gz: c86dcadbf158fb9a533163ad042e77d18192241ab7985c4204b6526f78d3cb307a08a48fc173dcef7a7697dfb7316a847fe664ec098178b9e244d7d279978fbb
+  data.tar.gz: a070c7c91c502c06adc83269c3f28b97789250a80b8518ce80ba48511ddacb80537e7ffe9075c03f45062384a53b50fd03af46cb9f3b8dba39e3f03f0498e8a8

data/.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+github: breamware
+open_collective: sidekiq-batch

data/.github/workflows/ci.yml

@@ -1,16 +1,30 @@
 name: CI
 
-on: [push, pull_request]
+on: [pull_request, workflow_dispatch]
 
 jobs:
   test:
+    env:
+      REDIS_HOST: 'redis'
 
     runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps port 6379 on service container to the host
+          - 6379:6379
 
     strategy:
       fail-fast: false
       matrix:
-        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", ruby-head]
+        ruby: ["3.0", "3.1", "3.2", ruby-head]
 
     steps:
     - uses: actions/checkout@v2

data/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '24 3 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'ruby' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

data/README.md

@@ -32,7 +32,41 @@ Or install it yourself as:
 
 ## Usage
 
-Sidekiq Batch is drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage.
+Sidekiq Batch is MOSTLY a drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage.
+
+## Caveats/Gotchas
+
+Consider the following workflow:
+
+  * Batch Z created
+  * Worker A queued in batch Z
+  * Worker A starts Worker B in batch Z
+  * Worker B completes *before* worker A does
+  * Worker A completes
+
+In the standard configuration, the `on(:success)` and `on(:complete)` callbacks will be triggered when Worker B completes.
+This configuration is the default, simply for legacy reasons. This gem adds the following option to the sidekiq.yml options:
+
+```yaml
+:batch_push_interval: 0
+```
+
+When this value is *absent* (aka legacy), Worker A will only push the increment of batch jobs (aka Worker B) *when it completes*
+
+When this value is set to `0`, Worker A will increment the count as soon as `WorkerB.perform_async` is called
+
+When this value is a positive number, Worker A will wait a maximum of value-seconds before pushing the increment to redis, or until it's done, whichever comes first.
+
+This comes into play if Worker A is queueing thousands of WorkerB jobs, or has some other reason for WorkerB to complete beforehand.
+
+If you are queueing many WorkerB jobs, it is recommended to set this value to something like `3` to avoid thousands of calls to redis, and call WorkerB like so:
+```ruby
+WorkerB.perform_in(4.seconds, some, args)
+```
+this will ensure that the batch callback does not get triggered until WorkerA *and* the last WorkerB job complete.
+
+If WorkerA is just slow for whatever reason, setting to `0` will update the batch status immediately so that the callbacks don't fire.
+
 
 ## Contributing
 

data/lib/sidekiq/batch/callback.rb

@@ -40,10 +40,10 @@ module Sidekiq
 
           _, _, success, _, complete, pending, children, failure = Sidekiq.redis do |r|
             r.multi do |pipeline|
-              pipeline.sadd("BID-#{parent_bid}-success", bid)
+              pipeline.sadd("BID-#{parent_bid}-success", [bid])
               pipeline.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL)
               pipeline.scard("BID-#{parent_bid}-success")
-              pipeline.sadd("BID-#{parent_bid}-complete", bid)
+              pipeline.sadd("BID-#{parent_bid}-complete", [bid])
               pipeline.scard("BID-#{parent_bid}-complete")
               pipeline.hincrby("BID-#{parent_bid}", "pending", 0)
               pipeline.hincrby("BID-#{parent_bid}", "children", 0)
@@ -81,7 +81,7 @@ module Sidekiq
             Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
             _, complete, pending, children, failure = Sidekiq.redis do |r|
               r.multi do |pipeline|
-                pipeline.sadd("BID-#{parent_bid}-complete", bid)
+                pipeline.sadd("BID-#{parent_bid}-complete", [bid])
                 pipeline.scard("BID-#{parent_bid}-complete")
                 pipeline.hincrby("BID-#{parent_bid}", "pending", 0)
                 pipeline.hincrby("BID-#{parent_bid}", "children", 0)

data/lib/sidekiq/batch/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   class Batch
-    VERSION = '0.1.8'.freeze
+    VERSION = '0.2.0'.freeze
   end
 end

data/lib/sidekiq/batch.rb

@@ -20,7 +20,11 @@ module Sidekiq
       @initialized = false
       @created_at = Time.now.utc.to_f
       @bidkey = "BID-" + @bid.to_s
-      @ready_to_queue = []
+      @queued_jids = []
+      @pending_jids = []
+
+      @incremental_push = !Sidekiq.default_configuration[:batch_push_interval].nil?
+      @batch_push_interval = Sidekiq.default_configuration[:batch_push_interval]
     end
 
     def description=(description)
@@ -43,10 +47,10 @@ module Sidekiq
       callback_key = "#{@bidkey}-callbacks-#{event}"
       Sidekiq.redis do |r|
         r.multi do |pipeline|
-          pipeline.sadd(callback_key, JSON.unparse({
+          pipeline.sadd(callback_key, [JSON.unparse({
             callback: callback,
             opts: options
-          }))
+          })])
           pipeline.expire(callback_key, BID_EXPIRE_TTL)
         end
       end
@@ -64,51 +68,86 @@ module Sidekiq
           Sidekiq.redis do |r|
             r.multi do |pipeline|
               pipeline.hset(@bidkey, "created_at", @created_at)
-              pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
               pipeline.expire(@bidkey, BID_EXPIRE_TTL)
+              if parent_bid
+                pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s)
+                pipeline.hincrby("BID-#{parent_bid}", "children", 1)
+              end
             end
           end
 
           @initialized = true
         end
 
-        @ready_to_queue = []
+        @queued_jids = []
+        @pending_jids = []
 
         begin
           parent = Thread.current[:batch]
           Thread.current[:batch] = self
+          Thread.current[:parent_bid] = parent_bid
           yield
         ensure
           Thread.current[:batch] = parent
+          Thread.current[:parent_bid] = nil
         end
 
-        return [] if @ready_to_queue.size == 0
+        return [] if @queued_jids.size == 0
+        conditional_redis_increment!(true)
 
         Sidekiq.redis do |r|
           r.multi do |pipeline|
             if parent_bid
-              pipeline.hincrby("BID-#{parent_bid}", "children", 1)
-              pipeline.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
               pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
             end
 
-            pipeline.hincrby(@bidkey, "pending", @ready_to_queue.size)
-            pipeline.hincrby(@bidkey, "total", @ready_to_queue.size)
             pipeline.expire(@bidkey, BID_EXPIRE_TTL)
 
-            pipeline.sadd(@bidkey + "-jids", @ready_to_queue)
+            pipeline.sadd(@bidkey + "-jids", @queued_jids)
             pipeline.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
           end
         end
 
-        @ready_to_queue
+        @queued_jids
       ensure
         Thread.current[:bid_data] = bid_data
       end
     end
 
     def increment_job_queue(jid)
-      @ready_to_queue << jid
+      @queued_jids << jid
+      @pending_jids << jid
+      conditional_redis_increment!
+    end
+
+    def conditional_redis_increment!(force=false)
+      if should_increment? || force
+        parent_bid = Thread.current[:parent_bid]
+        Sidekiq.redis do |r|
+          r.multi do |pipeline|
+            if parent_bid
+              pipeline.hincrby("BID-#{parent_bid}", "total", @pending_jids.length)
+              pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            end
+
+            pipeline.hincrby(@bidkey, "pending", @pending_jids.length)
+            pipeline.hincrby(@bidkey, "total", @pending_jids.length)
+            pipeline.expire(@bidkey, BID_EXPIRE_TTL)
+          end
+        end
+        @pending_jids = []
+      end
+    end
+
+    def should_increment?
+      return false unless @incremental_push
+      return true if @batch_push_interval == 0 || @queued_jids.length == 1
+      now = Time.now.to_f
+      @last_increment ||= now
+      if @last_increment + @batch_push_interval > now
+        @last_increment = now
+        return true
+      end
     end
 
     def invalidate_all
@@ -130,7 +169,7 @@ module Sidekiq
     end
 
     def valid?(batch = self)
-      valid = !Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }
+      valid = Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }.zero?
       batch.parent ? valid && valid?(batch.parent) : valid
     end
 
@@ -149,7 +188,7 @@ module Sidekiq
       def process_failed_job(bid, jid)
         _, pending, failed, children, complete, parent_bid = Sidekiq.redis do |r|
           r.multi do |pipeline|
-            pipeline.sadd("BID-#{bid}-failed", jid)
+            pipeline.sadd("BID-#{bid}-failed", [jid])
 
             pipeline.hincrby("BID-#{bid}", "pending", 0)
             pipeline.scard("BID-#{bid}-failed")
@@ -166,7 +205,7 @@ module Sidekiq
           Sidekiq.redis do |r|
             r.multi do |pipeline|
               pipeline.hincrby("BID-#{parent_bid}", "pending", 1)
-              pipeline.sadd("BID-#{parent_bid}-failed", jid)
+              pipeline.sadd("BID-#{parent_bid}-failed", [jid])
               pipeline.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL)
             end
           end
@@ -188,8 +227,8 @@ module Sidekiq
             pipeline.hget("BID-#{bid}", "total")
             pipeline.hget("BID-#{bid}", "parent_bid")
 
-            pipeline.srem("BID-#{bid}-failed", jid)
-            pipeline.srem("BID-#{bid}-jids", jid)
+            pipeline.srem("BID-#{bid}-failed", [jid])
+            pipeline.srem("BID-#{bid}-jids", [jid])
             pipeline.expire("BID-#{bid}", BID_EXPIRE_TTL)
           end
         end
@@ -209,7 +248,7 @@ module Sidekiq
         already_processed, _, callbacks, queue, parent_bid, callback_batch = Sidekiq.redis do |r|
           r.multi do |pipeline|
             pipeline.hget(batch_key, event_name)
-            pipeline.hset(batch_key, event_name, true)
+            pipeline.hset(batch_key, event_name, 'true')
             pipeline.smembers(callback_key)
             pipeline.hget(batch_key, "callback_queue")
             pipeline.hget(batch_key, "parent_bid")
@@ -253,7 +292,7 @@ module Sidekiq
         else
           # Otherwise finalize in sub batch complete callback
           cb_batch = self.new
-          cb_batch.callback_batch = true
+          cb_batch.callback_batch = 'true'
           Sidekiq.logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
           cb_batch.on(:complete, "Sidekiq::Batch::Callback::Finalize#dispatch", opts)
           cb_batch.jobs do

data/sidekiq-batch.gemspec

@@ -19,10 +19,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "sidekiq", ">= 3"
+  spec.add_dependency "sidekiq", ">= 7", "<8"
 
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "fakeredis", "~> 0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-batch
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.0
 platform: ruby
 authors:
 - Marcin Naglik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-24 00:00:00.000000000 Z
+date: 2024-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,20 +72,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: fakeredis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.8.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.8.0
 description: Sidekiq Batch Jobs Implementation
 email:
 - marcin.naglik@gmail.com
@@ -87,8 +79,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/FUNDING.yml"
 - ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
+- ".github/workflows/codeql-analysis.yml"
 - ".github/workflows/stale.yml"
 - ".gitignore"
 - ".rspec"
@@ -124,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.11
 signing_key:
 specification_version: 4
 summary: Sidekiq Batch Jobs