sidekiq-batchs 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 011c76443fe320ee58137aff6159d3cf4010818da75bb8c7bf1c81d5d26d4493
+  data.tar.gz: 5105e3e07c1cfa271cf308f8b7d9decae96985b628535f24a08295a8981e68c2
+SHA512:
+  metadata.gz: 7c76f4b95624e17a840f2e9863bb24ddccd7fe1bb68b1916fef40261c3fd5c572fb3731aba5e2cc894af93d061e75e13bf25717b3d7cd7d72a1bf96e7e2ca6fe
+  data.tar.gz: fb82c8d808d165684639fd5307135c3505ff706b00438a262e1fd0ba85ee79b52d64093705ea8700bcb73a05a3a5214e312ba6f6fac6ccdab61cd53d9b71cc47

data/.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+github: breamware
+open_collective: sidekiq-batch

data/.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "04:00"
+  open-pull-requests-limit: 10

data/.github/workflows/ci.yml

@@ -0,0 +1,37 @@
+name: CI
+
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  test:
+    env:
+      REDIS_HOST: 'redis'
+
+    runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps port 6379 on service container to the host
+          - 6379:6379
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ["3.0", "3.1", "3.2", ruby-head]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true # 'bundle install' and cache gems
+        ruby-version: ${{ matrix.ruby }}
+    - name: Run tests
+      run: bundle exec rake

data/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '24 3 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'ruby' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

data/.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "0 0 * * *"
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'Stale issue message'
+        stale-pr-message: 'Stale pull request message'
+        stale-issue-label: 'no-issue-activity'
+        stale-pr-label: 'no-pr-activity'

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+dump.rdb
+sidekiq-batch-*.gem
+.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,7 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.12.5
+after_success:
+  - bundle exec codeclimate-test-reporter

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem "simplecov"
+  gem "codeclimate-test-reporter", "~> 1.0.0"
+end

data/Guardfile

@@ -0,0 +1,4 @@
+guard 'rspec', cmd: 'rspec --color' do
+  watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r|^spec/(.*)_spec\.rb|)
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Breamware
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,78 @@
+[gem]: https://rubygems.org/gems/sidekiq-batch
+[travis]: https://travis-ci.org/breamware/sidekiq-batch
+[codeclimate]: https://codeclimate.com/github/breamware/sidekiq-batch
+
+# Sidekiq::Batch
+
+[![Join the chat at https://gitter.im/breamware/sidekiq-batch](https://badges.gitter.im/breamware/sidekiq-batch.svg)](https://gitter.im/breamware/sidekiq-batch?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
+[![Gem Version](https://badge.fury.io/rb/sidekiq-batch.svg)][gem]
+[![Build Status](https://travis-ci.org/breamware/sidekiq-batch.svg?branch=master)][travis]
+[![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/gpa.svg)][codeclimate]
+[![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/coverage.svg)][codeclimate]
+[![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/issue_count.svg)][codeclimate]
+
+Simple Sidekiq Batch Job implementation.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-batch'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-batch
+
+## Usage
+
+Sidekiq Batch is MOSTLY a drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage.
+
+## Caveats/Gotchas
+
+Consider the following workflow:
+
+  * Batch Z created
+  * Worker A queued in batch Z
+  * Worker A starts Worker B in batch Z
+  * Worker B completes *before* worker A does
+  * Worker A completes
+
+In the standard configuration, the `on(:success)` and `on(:complete)` callbacks will be triggered when Worker B completes.
+This configuration is the default, simply for legacy reasons. This gem adds the following option to the sidekiq.yml options:
+
+```yaml
+:batch_push_interval: 0
+```
+
+When this value is *absent* (aka legacy), Worker A will only push the increment of batch jobs (aka Worker B) *when it completes*
+
+When this value is set to `0`, Worker A will increment the count as soon as `WorkerB.perform_async` is called
+
+When this value is a positive number, Worker A will wait a maximum of value-seconds before pushing the increment to redis, or until it's done, whichever comes first.
+
+This comes into play if Worker A is queueing thousands of WorkerB jobs, or has some other reason for WorkerB to complete beforehand.
+
+If you are queueing many WorkerB jobs, it is recommended to set this value to something like `3` to avoid thousands of calls to redis, and call WorkerB like so:
+```ruby
+WorkerB.perform_in(4.seconds, some, args)
+```
+this will ensure that the batch callback does not get triggered until WorkerA *and* the last WorkerB job complete.
+
+If WorkerA is just slow for whatever reason, setting to `0` will update the batch status immediately so that the callbacks don't fire.
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/breamware/sidekiq-batch.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/sidekiq/batch/callback.rb

@@ -0,0 +1,104 @@
+module Sidekiq
+  class Batch
+    module Callback
+      class Worker
+        include Sidekiq::Worker
+
+        def perform(clazz, event, opts, bid, parent_bid)
+          return unless %w(success complete).include?(event)
+          clazz, method = clazz.split("#") if (clazz && clazz.class == String && clazz.include?("#"))
+          method = "on_#{event}" if method.nil?
+          status = Sidekiq::Batch::Status.new(bid)
+
+          if clazz && object = Object.const_get(clazz)
+            instance = object.new
+            instance.send(method, status, opts) if instance.respond_to?(method)
+          end
+        end
+      end
+
+      class Finalize
+        def dispatch status, opts
+          bid = opts["bid"]
+          callback_bid = status.bid
+          event = opts["event"].to_sym
+          callback_batch = bid != callback_bid
+
+          Sidekiq.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
+
+          batch_status = Status.new bid
+          send(event, bid, batch_status, batch_status.parent_bid)
+
+
+          # Different events are run in different callback batches
+          Sidekiq::Batch.cleanup_redis callback_bid if callback_batch
+          Sidekiq::Batch.cleanup_redis bid if event == :success
+        end
+
+        def success(bid, status, parent_bid)
+          return unless parent_bid
+
+          _, _, success, _, complete, pending, children, failure = Sidekiq.redis do |r|
+            r.multi do |pipeline|
+              pipeline.sadd("BID-#{parent_bid}-success", [bid])
+              pipeline.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL)
+              pipeline.scard("BID-#{parent_bid}-success")
+              pipeline.sadd("BID-#{parent_bid}-complete", [bid])
+              pipeline.scard("BID-#{parent_bid}-complete")
+              pipeline.hincrby("BID-#{parent_bid}", "pending", 0)
+              pipeline.hincrby("BID-#{parent_bid}", "children", 0)
+              pipeline.scard("BID-#{parent_bid}-failed")
+            end
+          end
+          # if job finished successfully and parent batch completed call parent complete callback
+          # Success callback is called after complete callback
+          if complete == children && pending == failure
+            Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
+            Batch.enqueue_callbacks(:complete, parent_bid)
+          end
+
+        end
+
+        def complete(bid, status, parent_bid)
+          pending, children, success = Sidekiq.redis do |r|
+            r.multi do |pipeline|
+              pipeline.hincrby("BID-#{bid}", "pending", 0)
+              pipeline.hincrby("BID-#{bid}", "children", 0)
+              pipeline.scard("BID-#{bid}-success")
+            end
+          end
+
+          # if we batch was successful run success callback
+          if pending.to_i.zero? && children == success
+            Batch.enqueue_callbacks(:success, bid)
+
+          elsif parent_bid
+            # if batch was not successfull check and see if its parent is complete
+            # if the parent is complete we trigger the complete callback
+            # We don't want to run this if the batch was successfull because the success
+            # callback may add more jobs to the parent batch
+
+            Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
+            _, complete, pending, children, failure = Sidekiq.redis do |r|
+              r.multi do |pipeline|
+                pipeline.sadd("BID-#{parent_bid}-complete", [bid])
+                pipeline.scard("BID-#{parent_bid}-complete")
+                pipeline.hincrby("BID-#{parent_bid}", "pending", 0)
+                pipeline.hincrby("BID-#{parent_bid}", "children", 0)
+                pipeline.scard("BID-#{parent_bid}-failed")
+              end
+            end
+            if complete == children && pending == failure
+              Batch.enqueue_callbacks(:complete, parent_bid)
+            end
+          end
+        end
+
+        def cleanup_redis bid, callback_bid=nil
+          Sidekiq::Batch.cleanup_redis bid
+          Sidekiq::Batch.cleanup_redis callback_bid if callback_bid
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/batch/extension/worker.rb

@@ -0,0 +1,15 @@
+module Sidekiq::Batch::Extension
+  module Worker
+    def bid
+      Thread.current[:batch].bid
+    end
+
+    def batch
+      Thread.current[:batch]
+    end
+
+    def valid_within_batch?
+      batch.valid?
+    end
+  end
+end

data/lib/sidekiq/batch/middleware.rb

@@ -0,0 +1,55 @@
+require_relative 'extension/worker'
+
+module Sidekiq
+  class Batch
+    module Middleware
+      class ClientMiddleware
+        def call(_worker, msg, _queue, _redis_pool = nil)
+          if (batch = Thread.current[:batch])
+            batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid)
+          end
+          yield
+        end
+      end
+
+      class ServerMiddleware
+        def call(_worker, msg, _queue)
+          if (bid = msg['bid'])
+            begin
+              Thread.current[:batch] = Sidekiq::Batch.new(bid)
+              yield
+              Thread.current[:batch] = nil
+              Batch.process_successful_job(bid, msg['jid'])
+            rescue
+              Batch.process_failed_job(bid, msg['jid'])
+              raise
+            ensure
+              Thread.current[:batch] = nil
+            end
+          else
+            yield
+          end
+        end
+      end
+
+      def self.configure
+        Sidekiq.configure_client do |config|
+          config.client_middleware do |chain|
+            chain.add Sidekiq::Batch::Middleware::ClientMiddleware
+          end
+        end
+        Sidekiq.configure_server do |config|
+          config.client_middleware do |chain|
+            chain.add Sidekiq::Batch::Middleware::ClientMiddleware
+          end
+          config.server_middleware do |chain|
+            chain.add Sidekiq::Batch::Middleware::ServerMiddleware
+          end
+        end
+        Sidekiq::Worker.send(:include, Sidekiq::Batch::Extension::Worker)
+      end
+    end
+  end
+end
+
+Sidekiq::Batch::Middleware.configure

data/lib/sidekiq/batch/status.rb

@@ -0,0 +1,61 @@
+module Sidekiq
+  class Batch
+    class Status
+      attr_reader :bid
+
+      def initialize(bid)
+        @bid = bid
+      end
+
+      def join
+        raise "Not supported"
+      end
+
+      def pending
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i
+      end
+
+      def failures
+        Sidekiq.redis { |r| r.scard("BID-#{bid}-failed") }.to_i
+      end
+
+      def created_at
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", 'created_at') }
+      end
+
+      def total
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i
+      end
+
+      def parent_bid
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
+      end
+
+      def failure_info
+        Sidekiq.redis { |r| r.smembers("BID-#{bid}-failed") } || []
+      end
+
+      def complete?
+        'true' == Sidekiq.redis { |r| r.hget("BID-#{bid}", 'complete') }
+      end
+
+      def child_count
+        Sidekiq.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
+      end
+
+      def data
+        {
+          bid: bid,
+          total: total,
+          failures: failures,
+          pending: pending,
+          created_at: created_at,
+          complete: complete?,
+          failure_info: failure_info,
+          parent_bid: parent_bid,
+          child_count: child_count
+        }
+      end
+    end
+  end
+end

data/lib/sidekiq/batch/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  class Batch
+    VERSION = '0.3.0'.freeze
+  end
+end

data/lib/sidekiq/batch.rb

@@ -0,0 +1,331 @@
+require 'securerandom'
+require 'sidekiq'
+
+require 'sidekiq/batch/callback'
+require 'sidekiq/batch/middleware'
+require 'sidekiq/batch/status'
+require 'sidekiq/batch/version'
+
+module Sidekiq
+  class Batch
+    class NoBlockGivenError < StandardError; end
+
+    BID_EXPIRE_TTL = 2_592_000
+
+    attr_reader :bid, :description, :callback_queue, :created_at
+
+    def initialize(existing_bid = nil)
+      @bid = existing_bid || SecureRandom.urlsafe_base64(10)
+      @existing = !(!existing_bid || existing_bid.empty?)  # Basically existing_bid.present?
+      @initialized = false
+      @created_at = Time.now.utc.to_f
+      @bidkey = "BID-" + @bid.to_s
+      @queued_jids = []
+      @pending_jids = []
+
+      @incremental_push = !Sidekiq.default_configuration[:batch_push_interval].nil?
+      @batch_push_interval = Sidekiq.default_configuration[:batch_push_interval]
+    end
+
+    def description=(description)
+      @description = description
+      persist_bid_attr('description', description)
+    end
+
+    def callback_queue=(callback_queue)
+      @callback_queue = callback_queue
+      persist_bid_attr('callback_queue', callback_queue)
+    end
+
+    def callback_batch=(callback_batch)
+      @callback_batch = callback_batch
+      persist_bid_attr('callback_batch', callback_batch)
+    end
+
+    def on(event, callback, options = {})
+      return unless %w(success complete).include?(event.to_s)
+      callback_key = "#{@bidkey}-callbacks-#{event}"
+      Sidekiq.redis do |r|
+        r.multi do |pipeline|
+          pipeline.sadd(callback_key, [JSON.unparse({
+            callback: callback,
+            opts: options
+          })])
+          pipeline.expire(callback_key, BID_EXPIRE_TTL)
+        end
+      end
+    end
+
+    def jobs
+      raise NoBlockGivenError unless block_given?
+
+      bid_data, Thread.current[:bid_data] = Thread.current[:bid_data], []
+
+      begin
+        if !@existing && !@initialized
+          parent_bid = Thread.current[:batch].bid if Thread.current[:batch]
+
+          Sidekiq.redis do |r|
+            r.multi do |pipeline|
+              pipeline.hset(@bidkey, "created_at", @created_at)
+              pipeline.expire(@bidkey, BID_EXPIRE_TTL)
+              if parent_bid
+                pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s)
+                pipeline.hincrby("BID-#{parent_bid}", "children", 1)
+              end
+            end
+          end
+
+          @initialized = true
+        end
+
+        @queued_jids = []
+        @pending_jids = []
+
+        begin
+          parent = Thread.current[:batch]
+          Thread.current[:batch] = self
+          Thread.current[:parent_bid] = parent_bid
+          yield
+        ensure
+          Thread.current[:batch] = parent
+          Thread.current[:parent_bid] = nil
+        end
+
+        return [] if @queued_jids.size == 0
+        conditional_redis_increment!(true)
+
+        Sidekiq.redis do |r|
+          r.multi do |pipeline|
+            if parent_bid
+              pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            end
+
+            pipeline.expire(@bidkey, BID_EXPIRE_TTL)
+
+            pipeline.sadd(@bidkey + "-jids", @queued_jids)
+            pipeline.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
+          end
+        end
+
+        @queued_jids
+      ensure
+        Thread.current[:bid_data] = bid_data
+      end
+    end
+
+    def increment_job_queue(jid)
+      @queued_jids << jid
+      @pending_jids << jid
+      conditional_redis_increment!
+    end
+
+    def conditional_redis_increment!(force=false)
+      if should_increment? || force
+        parent_bid = Thread.current[:parent_bid]
+        Sidekiq.redis do |r|
+          r.multi do |pipeline|
+            if parent_bid
+              pipeline.hincrby("BID-#{parent_bid}", "total", @pending_jids.length)
+              pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            end
+
+            pipeline.hincrby(@bidkey, "pending", @pending_jids.length)
+            pipeline.hincrby(@bidkey, "total", @pending_jids.length)
+            pipeline.expire(@bidkey, BID_EXPIRE_TTL)
+          end
+        end
+        @pending_jids = []
+      end
+    end
+
+    def should_increment?
+      return false unless @incremental_push
+      return true if @batch_push_interval == 0 || @queued_jids.length == 1
+      now = Time.now.to_f
+      @last_increment ||= now
+      if @last_increment + @batch_push_interval > now
+        @last_increment = now
+        return true
+      end
+    end
+
+    def invalidate_all
+      Sidekiq.redis do |r|
+        r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
+      end
+    end
+
+    def parent_bid
+      Sidekiq.redis do |r|
+        r.hget(@bidkey, "parent_bid")
+      end
+    end
+
+    def parent
+      if parent_bid
+        Sidekiq::Batch.new(parent_bid)
+      end
+    end
+
+    def valid?(batch = self)
+      valid = Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") }.zero?
+      batch.parent ? valid && valid?(batch.parent) : valid
+    end
+
+    private
+
+    def persist_bid_attr(attribute, value)
+      Sidekiq.redis do |r|
+        r.multi do |pipeline|
+          pipeline.hset(@bidkey, attribute, value)
+          pipeline.expire(@bidkey, BID_EXPIRE_TTL)
+        end
+      end
+    end
+
+    class << self
+      def process_failed_job(bid, jid)
+        _, pending, failed, children, complete, parent_bid = Sidekiq.redis do |r|
+          r.multi do |pipeline|
+            pipeline.sadd("BID-#{bid}-failed", [jid])
+
+            pipeline.hincrby("BID-#{bid}", "pending", 0)
+            pipeline.scard("BID-#{bid}-failed")
+            pipeline.hincrby("BID-#{bid}", "children", 0)
+            pipeline.scard("BID-#{bid}-complete")
+            pipeline.hget("BID-#{bid}", "parent_bid")
+
+            pipeline.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
+          end
+        end
+
+        # if the batch failed, and has a parent, update the parent to show one pending and failed job
+        if parent_bid
+          Sidekiq.redis do |r|
+            r.multi do |pipeline|
+              pipeline.hincrby("BID-#{parent_bid}", "pending", 1)
+              pipeline.sadd("BID-#{parent_bid}-failed", [jid])
+              pipeline.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL)
+            end
+          end
+        end
+
+        if pending.to_i == failed.to_i && children == complete
+          enqueue_callbacks(:complete, bid)
+        end
+      end
+
+      def process_successful_job(bid, jid)
+        failed, pending, children, complete, success, total, parent_bid = Sidekiq.redis do |r|
+          r.multi do |pipeline|
+            pipeline.scard("BID-#{bid}-failed")
+            pipeline.hincrby("BID-#{bid}", "pending", -1)
+            pipeline.hincrby("BID-#{bid}", "children", 0)
+            pipeline.scard("BID-#{bid}-complete")
+            pipeline.scard("BID-#{bid}-success")
+            pipeline.hget("BID-#{bid}", "total")
+            pipeline.hget("BID-#{bid}", "parent_bid")
+
+            pipeline.srem("BID-#{bid}-failed", [jid])
+            pipeline.srem("BID-#{bid}-jids", [jid])
+            pipeline.expire("BID-#{bid}", BID_EXPIRE_TTL)
+          end
+        end
+
+        all_success = pending.to_i.zero? && children == success
+        # if complete or successfull call complete callback (the complete callback may then call successful)
+        if (pending.to_i == failed.to_i && children == complete) || all_success
+          enqueue_callbacks(:complete, bid)
+          enqueue_callbacks(:success, bid) if all_success
+        end
+      end
+
+      def enqueue_callbacks(event, bid)
+        event_name = event.to_s
+        batch_key = "BID-#{bid}"
+        callback_key = "#{batch_key}-callbacks-#{event_name}"
+        already_processed, _, callbacks, queue, parent_bid, callback_batch = Sidekiq.redis do |r|
+          r.multi do |pipeline|
+            pipeline.hget(batch_key, event_name)
+            pipeline.hset(batch_key, event_name, 'true')
+            pipeline.smembers(callback_key)
+            pipeline.hget(batch_key, "callback_queue")
+            pipeline.hget(batch_key, "parent_bid")
+            pipeline.hget(batch_key, "callback_batch")
+          end
+        end
+
+        return if already_processed == 'true'
+
+        queue ||= "default"
+        parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid    # Basically parent_bid.blank?
+        callback_args = callbacks.reduce([]) do |memo, jcb|
+          cb = Sidekiq.load_json(jcb)
+          memo << [cb['callback'], event_name, cb['opts'], bid, parent_bid]
+        end
+
+        opts = {"bid" => bid, "event" => event_name}
+
+        # Run callback batch finalize synchronously
+        if callback_batch
+          # Extract opts from cb_args or use current
+          # Pass in stored event as callback finalize is processed on complete event
+          cb_opts = callback_args.first&.at(2) || opts
+
+          Sidekiq.logger.debug {"Run callback batch bid: #{bid} event: #{event_name} args: #{callback_args.inspect}"}
+          # Finalize now
+          finalizer = Sidekiq::Batch::Callback::Finalize.new
+          status = Status.new bid
+          finalizer.dispatch(status, cb_opts)
+
+          return
+        end
+
+        Sidekiq.logger.debug {"Enqueue callback bid: #{bid} event: #{event_name} args: #{callback_args.inspect}"}
+
+        if callback_args.empty?
+          # Finalize now
+          finalizer = Sidekiq::Batch::Callback::Finalize.new
+          status = Status.new bid
+          finalizer.dispatch(status, opts)
+        else
+          # Otherwise finalize in sub batch complete callback
+          cb_batch = self.new
+          cb_batch.callback_batch = 'true'
+          Sidekiq.logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"}
+          cb_batch.on(:complete, "Sidekiq::Batch::Callback::Finalize#dispatch", opts)
+          cb_batch.jobs do
+            push_callbacks callback_args, queue
+          end
+        end
+      end
+
+      def cleanup_redis(bid)
+        Sidekiq.logger.debug {"Cleaning redis of batch #{bid}"}
+        Sidekiq.redis do |r|
+          r.del(
+            "BID-#{bid}",
+            "BID-#{bid}-callbacks-complete",
+            "BID-#{bid}-callbacks-success",
+            "BID-#{bid}-failed",
+
+            "BID-#{bid}-success",
+            "BID-#{bid}-complete",
+            "BID-#{bid}-jids",
+          )
+        end
+      end
+
+    private
+
+      def push_callbacks args, queue
+        Sidekiq::Client.push_bulk(
+          'class' => Sidekiq::Batch::Callback::Worker,
+          'args' => args,
+          'queue' => queue
+        ) unless args.empty?
+      end
+    end
+  end
+end

data/sidekiq-batch.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq/batch/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidekiq-batchs"
+  spec.version       = Sidekiq::Batch::VERSION
+  spec.authors       = ["Marcin Naglik"]
+  spec.email         = ["tianlu1677@gmail.com"]
+
+  spec.summary       = "Sidekiq Batch Jobs"
+  spec.description   = "Sidekiq Batch Jobs Implementation"
+  spec.homepage      = "http://github.com/breamware/sidekiq-batch"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sidekiq", ">= 7", "<8"
+
+  spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-batchs
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Marcin Naglik
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-04-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Sidekiq Batch Jobs Implementation
+email:
+- tianlu1677@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/FUNDING.yml"
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
+- ".github/workflows/codeql-analysis.yml"
+- ".github/workflows/stale.yml"
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sidekiq/batch.rb
+- lib/sidekiq/batch/callback.rb
+- lib/sidekiq/batch/extension/worker.rb
+- lib/sidekiq/batch/middleware.rb
+- lib/sidekiq/batch/status.rb
+- lib/sidekiq/batch/version.rb
+- sidekiq-batch.gemspec
+homepage: http://github.com/breamware/sidekiq-batch
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Sidekiq Batch Jobs
+test_files: []