sidekiq-clutch 1.1.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34898643a173edd20ec4a28e2a2f01f15c05d9e074d67a72e9f06a56580f2367
-  data.tar.gz: 4e39b5cd8a173520db812680c647ba8708f547614986bb8ad13be5d872377117
+  metadata.gz: d471e4ebb14bcae3bcbb03c855ab949b1029b9a2bbb0210927bc6f38b7f8906d
+  data.tar.gz: 5059ca6f29a0a3de77bef0060112b7bd448a64db5c18672a859fa91b70f1b7d7
 SHA512:
-  metadata.gz: fcfd9af62fc2a84546862f12813ffed617eaeee69b10f0136529a70d4307bfb85616b1629223d50e909c967d98462dcf4b04ec0492c56f3dd5fd5ce43bda2946
-  data.tar.gz: 699d22544e893ea1130e0bd4967888781d2ee1856b7a918a919eff639b4dcd40cade5651840ab22b95f9b43a27262078d37b30e32b0ff6aea3a06203d378e897
+  metadata.gz: 9fec12c8483a7f59a466db621b77cf123bd4bb09671181b397926062b4f1c2a058be992581bb1a8b931394921da372ce737b3e365aff3dd420fe2562b818efc2
+  data.tar.gz: a9fc7e8afac8d30c45fa15766cc26eacb524b4f324ded0d6dfda9d314270dabed274831523ed728c8ec08900f9d3f6e540505ff05593daed26badac7cef08c89

data/.circleci/config.yml

@@ -7,5 +7,6 @@ jobs:
     - image: redis:3.2
     steps:
     - checkout
+    - run: gem install bundler
     - run: bundle install
     - run: bundle exec rspec

data/.gitignore

@@ -1,3 +1,4 @@
+/*.gem
 /.bundle/
 .rspec_status
 Gemfile.lock

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+# 2.1.0 - Sep 10, 2021
+
+* PERF: Don't pass step data in success callback to every batch
+
+NOTE: This changes the structure of arguments passed to Batch callbacks, though we
+also handle the old way of doing it (v2.0.2 and before) so it shouldn't affect
+any in-progress batches you may have running.
+
+# 2.0.2 - Sep 24, 2020
+
+* PERF: Delete keys based on known values, instead of a glob
+* PERF: Don't bother enqueing a `:complete` callback if no `on_failure` is specified
+
+# 2.0.1 - Jun 1, 2020
+
+* CHORE: Bump development dependency versions
+* CHORE: Add .ruby-version file for easier development
+
+# 2.0.0 - Feb 5, 2020
+
+* BREAKING: Treat each parallel block as a distinct step
+
 # 1.1.0 - Feb 5, 2020
 
 * FEAT: use Sidekiq's wrapped option for improved logging

data/lib/sidekiq/clutch/job_wrapper.rb

@@ -3,9 +3,6 @@ module Sidekiq
     class JobWrapper
       include Sidekiq::Worker
 
-      # 22 days - how long a Sidekiq job can live with exponential backoff
-      RESULT_KEY_EXPIRATION_DURATION = 22 * 24 * 60 * 60
-
       def perform(bid, job_class, args, last_result_key, current_result_key)
         job = Object.const_get(job_class).new
         assign_previous_results(job, last_result_key)
@@ -14,7 +11,7 @@ module Sidekiq
         Sidekiq.redis do |redis|
           redis.multi do |multi|
             multi.rpush(current_result_key, result.to_json)
-            multi.expire(current_result_key, RESULT_KEY_EXPIRATION_DURATION)
+            multi.expire(current_result_key, TEMPORARY_KEY_EXPIRATION_DURATION)
           end
         end
       end

data/lib/sidekiq/clutch/jobs_collection.rb

@@ -4,7 +4,6 @@ module Sidekiq
       def initialize(service)
         @service = service
         @jobs = []
-        @result_key_prefix = SecureRandom.uuid
         @result_key_index = 0
       end
 
@@ -22,16 +21,25 @@ module Sidekiq
 
       def <<((klass, *params))
         if @service.parallel?
-          @jobs << { 'parallel' => [], 'result_key' => next_result_key } unless @jobs.last && @jobs.last['parallel']
+          @jobs << new_parallel_step unless continue_existing_parallel_step?
           @jobs.last['parallel'] << [klass.name, params]
         else
-          @jobs << { 'series' => [klass.name, params], 'result_key' => next_result_key }
+          @jobs << { 'series' => [klass.name, params], 'result_key_index' => next_result_key_index }
         end
       end
 
-      def next_result_key
+      def next_result_key_index
         @result_key_index += 1
-        "#{@result_key_prefix}-#{@result_key_index}"
+      end
+
+      private
+
+      def new_parallel_step
+        { 'parallel' => [], 'result_key_index' => next_result_key_index, 'parallel_key' => @service.parallel_key }
+      end
+
+      def continue_existing_parallel_step?
+        @jobs.last && @jobs.last['parallel_key'] == @service.parallel_key
       end
     end
   end

data/lib/sidekiq/clutch/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   class Clutch
-    VERSION = "1.1.0"
+    VERSION = '2.1.0'.freeze
   end
 end

data/lib/sidekiq/clutch/worker.rb

@@ -15,7 +15,7 @@ module Sidekiq
         Sidekiq.redis do |redis|
           redis.multi do |multi|
             multi.rpush(current_result_key, result.to_json)
-            multi.expire(current_result_key, RESULT_KEY_EXPIRATION_DURATION)
+            multi.expire(current_result_key, TEMPORARY_KEY_EXPIRATION_DURATION)
           end
         end
       end

data/lib/sidekiq/clutch.rb

@@ -6,22 +6,25 @@ require 'sidekiq/clutch/job_wrapper'
 
 module Sidekiq
   class Clutch
+    # 22 days - how long a Sidekiq job can live with exponential backoff
+    TEMPORARY_KEY_EXPIRATION_DURATION = 22 * 24 * 60 * 60
+
     def initialize(batch = nil)
       @batch = batch || Sidekiq::Batch.new
     end
 
-    attr_reader :batch, :queue
+    attr_reader :batch, :queue, :parallel_key
 
     attr_accessor :current_result_key, :on_failure
 
     def parallel
-      @parallel = true
+      @parallel_key = SecureRandom.uuid
       yield
-      @parallel = false
+      @parallel_key = nil
     end
 
     def parallel?
-      @parallel == true
+      !!@parallel_key
     end
 
     def jobs
@@ -51,10 +54,12 @@ module Sidekiq
     def setup_batch
       jobs_queue = jobs.raw.dup
       step = jobs_queue.shift
+      set_jobs_data_in_redis(jobs_queue)
       return if step.nil?
       batch.callback_queue = queue if queue
-      batch.on(:success, Sidekiq::Clutch, 'jobs' => jobs_queue.dup, 'result_key' => step['result_key'])
-      batch.on(:complete, Sidekiq::Clutch, 'on_failure' => on_failure&.name)
+      batch.on(:success, Sidekiq::Clutch, 'key_base' => key_base, 'result_key_index' => result_key_index(step))
+      on_failure_name = on_failure&.name
+      batch.on(:complete, Sidekiq::Clutch, 'on_failure' => on_failure_name) if on_failure_name
       batch.jobs do
         if step['series']
           series_step(step)
@@ -67,14 +72,23 @@ module Sidekiq
     end
 
     def on_success(status, options)
-      if options['jobs'].empty?
-        clean_up_result_keys(options['result_key'].sub(/-\d+$/, ''))
+      return on_success_legacy(status, options) if options['jobs']
+
+      raise 'invariant: key_base is missing!' unless options['key_base']
+      raise 'invariant: result_key_index is missing!' unless options['result_key_index']
+
+      # NOTE: This is a brand new instance of Sidekiq::Clutch that Sidekiq instantiates,
+      # so we need to set @key_base again.
+      @key_base = options['key_base']
+      remaining_jobs = JSON.parse(Sidekiq.redis { |r| r.get(jobs_key) })
+      if remaining_jobs.empty?
+        clean_up_temporary_keys
         return
       end
       parent_batch = Sidekiq::Batch.new(status.parent_bid)
       service = self.class.new(parent_batch)
-      service.jobs.raw = options['jobs']
-      service.current_result_key = options['result_key']
+      service.jobs.raw = remaining_jobs
+      service.current_result_key = "#{key_base}-#{options['result_key_index']}"
       service.engage
     end
 
@@ -86,19 +100,61 @@ module Sidekiq
 
     private
 
+    # accept old style of passing job data, will be removed in 3.0
+    def on_success_legacy(status, options)
+      @key_base = options['result_key'].sub(/-\d+$/, '')
+      if options['jobs'].empty?
+        clean_up_result_keys
+        return
+      end
+      parent_batch = Sidekiq::Batch.new(status.parent_bid)
+      service = self.class.new(parent_batch)
+      service.jobs.raw = options['jobs']
+      service.current_result_key = options['result_key']
+      service.engage
+    end
+
+    def key_base
+      @key_base ||= SecureRandom.uuid
+    end
+
+    def jobs_key
+      "#{key_base}-jobs"
+    end
+
+    def set_jobs_data_in_redis(data)
+      Sidekiq.redis do |redis|
+        redis.multi do |multi|
+          multi.set(jobs_key, data.to_json)
+          multi.expire(jobs_key, TEMPORARY_KEY_EXPIRATION_DURATION)
+        end
+      end
+    end
+
     def series_step(step)
       (klass, params) = step['series']
-      enqueue_job(klass, params, step['result_key'])
+      enqueue_job(klass, params, result_key_index(step))
     end
 
     def parallel_step(step)
       step['parallel'].each do |(klass, params)|
-        enqueue_job(klass, params, step['result_key'])
+        enqueue_job(klass, params, result_key_index(step))
+      end
+    end
+
+    def result_key_index(step)
+      if step['result_key_index']
+        step['result_key_index']
+      elsif step['result_key'] # legacy style, will be removed in 3.0
+        step['result_key'].split('-').last.to_i
+      else
+        raise "invariant: expected result_key_index passed in step; got: #{step.inspect}"
       end
     end
 
-    def enqueue_job(klass, params, result_key)
+    def enqueue_job(klass, params, result_key_index)
       job_options = Object.const_get(klass).sidekiq_options
+      result_key = "#{key_base}-#{result_key_index}"
       options = {
         'class'     => JobWrapper,
         'queue'     => queue || job_options['queue'],
@@ -110,10 +166,14 @@ module Sidekiq
       Sidekiq::Client.push(options)
     end
 
-    def clean_up_result_keys(key_base)
+    def clean_up_temporary_keys
       Sidekiq.redis do |redis|
-        redis.keys(key_base + '*').each do |key|
-          redis.del(key)
+        redis.del(jobs_key)
+        result_key_index = 1
+        loop do
+          result = redis.del("#{key_base}-#{result_key_index}")
+          result_key_index += 1
+          break if result == 0
         end
       end
     end

data/sidekiq-clutch.gemspec

@@ -15,14 +15,13 @@ Gem::Specification.new do |spec|
   spec.license       = 'MIT'
 
   spec.files         = Dir.chdir(File.expand_path(__dir__)) do
-    `git ls-files`.split("\n").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files`.split("\n").reject { |f| f.match(%r{^(spec|\.ruby-version)}) }
   end
   spec.require_paths = ['lib']
 
   spec.add_dependency 'sidekiq', '>= 5.0.0'
 
-  spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'bundler', '~> 1.16'
-  spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'bundler', '~> 2.2.4'
+  spec.add_development_dependency 'rake', '~> 13.0.1'
+  spec.add_development_dependency 'rspec', '~> 3.9.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-clutch
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Tim Morgan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-05 00:00:00.000000000 Z
+date: 2021-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -24,62 +24,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.0
-- !ruby/object:Gem::Dependency
-  name: pry
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.2.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: 2.2.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 13.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 13.0.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 3.9.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 3.9.0
 description: Sidekiq::Clutch provides an ergonomic wrapper API for Sidekiq Batches
   so you can easily manage serial and parallel jobs.
 email:
@@ -122,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: An ergonomic wrapper API for Sidekiq Batches