sidekiq-clutch 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ec75202216558f8f183a76de69109178e67ba9bb
+  data.tar.gz: c49621ff95d0667e6d9b2394a1a783a740d240e0
+SHA512:
+  metadata.gz: 10eefa778032d53f29c65154057e7ac3f9a7d84c7dfd576da8cc788488f20a744953ea26fedfb2f93812b609b45a382380df77ce901591fc25001c7025219af5
+  data.tar.gz: f1d72de39d77bd3ebc10034620a009bc8982492faf3188917023dbbeead5252fc36d39e62e5d491cdb477f66f48157fd07544016d28d6d357494036d11c41ee9

data/.gitignore

@@ -0,0 +1,3 @@
+/.bundle/
+.rspec_status
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,29 @@
+Metrics/LineLength:
+  Max: 120
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+Layout/EmptyLineAfterGuardClause:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Naming/UncommunicativeMethodParamName:
+  Enabled: false
+
+Style/Next:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 1.0.0 - Nov 5, 2018
+
+Initial release.

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in sidekiq-clutch.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Tim Morgan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,163 @@
+# Sidekiq::Clutch
+
+Sidekiq::Clutch is the API I always wanted when working with Sidekiq Pro [Batches](https://github.com/mperham/sidekiq/wiki/Batches). So I built it!
+
+Features:
+
+* Add jobs to run in series or parallel or mix-and-match however you wish.
+* Pass results from one job onto the next job in series.
+* If running jobs in parallel, pass all the results to the following job in series.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-clutch'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sidekiq-clutch
+
+## Usage
+
+Include the `Sidekiq::Clutch::Worker` mixin in your job class and write your `#perform` method as usual. For example:
+
+```ruby
+class MyJob1
+  include Sidekiq::Worker
+  include Sidekiq::Clutch::Worker
+
+  def perform
+    # do stuff
+  end
+end
+```
+
+Now, enqueue your jobs using a new `Sidekiq::Clutch` instance, like this:
+
+```ruby
+clutch = Sidekiq::Clutch.new
+clutch.jobs << MyJob1
+clutch.jobs << [MyJob2, 'arg1', 'arg2']
+clutch.parallel do
+  clutch.jobs << [MyJob3, 3]
+  clutch.jobs << [MyJob3, 4]
+  clutch.jobs << [MyJob3, 5]
+end
+clutch.jobs << MyFinalizerJob
+clutch.engage
+```
+
+The jobs will run in this order:
+
+* First, `MyJob1` will run.
+* Second, `MyJob2` with args `'arg1'` and `'arg2'` will run.
+* Third, in parallel, three jobs of `MyJob3`, each with args `3`, `4`, and `5` will run.
+* Last, the `MyFinalizerJob` will run.
+
+## Results from Prior Jobs
+
+You can access the results from the previous job in the series using the `previous_results` method:
+
+```ruby
+class MyJob1
+  include Sidekiq::Worker
+  include Sidekiq::Clutch::Worker
+
+  def perform
+    'This string will be passed to the next job in the series'
+  end
+end
+
+class MyJob2
+  include Sidekiq::Worker
+  include Sidekiq::Clutch::Worker
+
+  def perform(arg1, arg2)
+    puts "results of the previous job (MyJob1):"
+    p previous_results # => ["This string will be passed to the next job in the series"]
+  end
+end
+```
+
+If the previous step in a series was a batch of parallel jobs, then `previous_results` will be an array
+containing all the results from all the parallel jobs.
+
+Some notes:
+- Results are always in an array, even if there was only a single prior job run in series.
+- Results are serialized as JSON and temporarily stored in Redis, so only return small-ish values.
+
+## Handling Failure
+
+When jobs fail, Sidekiq will retry them as usual. See [here](https://github.com/mperham/sidekiq/wiki/Error-Handling) for details on how Sidekiq handles job failure and retries.
+
+Jobs next in a series will **not** be enqueued by Clutch as long as a prior job is failing.
+
+If you wish to do something when a job in your batch fails, set `on_failure` like this:
+
+```ruby
+clutch = Sidekiq::Clutch.new
+clutch.on_failure = MyFailureHandlerJob
+# add jobs here...
+clutch.engage
+```
+
+`MyFailureHandlerJob.new.perform(status)` will be called if one of the following scenarios occur:
+
+1. A job in series fails.
+2. One or more jobs in parallel fail and the rest complete.
+
+Note: retries will continue to occur even after your failure handler job is called.
+
+## Nested Batches
+
+You can nest Clutch instances too!
+
+```ruby
+class MyNestedJob
+  include Sidekiq::Worker
+  include Sidekiq::Clutch::Worker
+
+  def perform
+    clutch = Sidekiq::Clutch.new(batch)
+    clutch.jobs << [MyJob2, 'arg1', 'arg2']
+    clutch.jobs << [MyJob3, 3]
+    clutch.engage
+    'result from MyNestedJob'
+  end
+end
+
+clutch = Sidekiq::Clutch.new
+clutch.jobs << MyJob1
+clutch.jobs << MyNestedJob
+clutch.jobs << MyFinalizerJob
+clutch.engage
+```
+
+## Setting the Queue
+
+You can set the queue for jobs (overriding any queue specified in the job class itself) and callbacks by setting it on Clutch, like this:
+
+```ruby
+clutch = Sidekiq::Clutch.new
+clutch.queue = 'critical'
+# add jobs here...
+clutch.engage
+```
+
+## Note about the sidekiq-batch gem
+
+I tested this with the third-party gem [sidekiq-batch](https://github.com/breamware/sidekiq-batch), which
+purports to be a drop-in replacement for Sidekiq Pro batches. However, that gem seems to not support
+[nested callbacks](https://github.com/breamware/sidekiq-batch/issues/11#issuecomment-330625800), which my gem
+relies on. I'm sorry, this gem will **not** work with the sidekiq-batch gem -- it only works with sidekiq-pro.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/sidekiq/clutch.rb

@@ -0,0 +1,120 @@
+require 'sidekiq'
+require 'sidekiq/clutch/version'
+require 'sidekiq/clutch/worker'
+require 'sidekiq/clutch/jobs_collection'
+require 'sidekiq/clutch/job_wrapper'
+
+module Sidekiq
+  class Clutch
+    def initialize(batch = nil)
+      @batch = batch || Sidekiq::Batch.new
+    end
+
+    attr_reader :batch, :queue
+
+    attr_accessor :current_result_key, :on_failure
+
+    def parallel
+      @parallel = true
+      yield
+      @parallel = false
+    end
+
+    def parallel?
+      @parallel == true
+    end
+
+    def jobs
+      @jobs ||= JobsCollection.new(self)
+    end
+
+    def clear
+      @jobs = nil
+    end
+
+    def queue=(q)
+      @queue = q && q.to_s
+    end
+
+    def engage
+      return if jobs.empty?
+      if batch.mutable?
+        setup_batch
+      else
+        batch.jobs do
+          @batch = Sidekiq::Batch.new
+          setup_batch
+        end
+      end
+    end
+
+    def setup_batch
+      jobs_queue = jobs.raw.dup
+      step = jobs_queue.shift
+      return if step.nil?
+      batch.callback_queue = queue if queue
+      batch.on(:success, Sidekiq::Clutch, 'jobs' => jobs_queue.dup, 'result_key' => step['result_key'])
+      batch.on(:complete, Sidekiq::Clutch, 'on_failure' => on_failure&.name)
+      batch.jobs do
+        if step['series']
+          series_step(step)
+        elsif step['parallel']
+          parallel_step(step)
+        else
+          raise "unknown step: #{step.inspect}"
+        end
+      end
+    end
+
+    def on_success(status, options)
+      if options['jobs'].empty?
+        clean_up_result_keys(options['result_key'].sub(/-\d+$/, ''))
+        return
+      end
+      parent_batch = Sidekiq::Batch.new(status.parent_bid)
+      service = self.class.new(parent_batch)
+      service.jobs.raw = options['jobs']
+      service.current_result_key = options['result_key']
+      service.engage
+    end
+
+    def on_complete(status, options)
+      return if status.failures.zero?
+      return if options['on_failure'].nil?
+      Object.const_get(options['on_failure']).new.perform(status)
+    end
+
+    private
+
+    def series_step(step)
+      (klass, params) = step['series']
+      enqueue_job(klass, params, step['result_key'])
+    end
+
+    def parallel_step(step)
+      step['parallel'].each do |(klass, params)|
+        enqueue_job(klass, params, step['result_key'])
+      end
+    end
+
+    def enqueue_job(klass, params, result_key)
+      job_options = Object.const_get(klass).sidekiq_options
+      options = {
+        'class'     => JobWrapper,
+        'queue'     => queue || job_options['queue'],
+        'args'      => [batch.bid, klass, params, current_result_key, result_key],
+        'retry'     => job_options['retry'],
+        'backtrace' => job_options['backtrace']
+      }
+      Sidekiq::Client.push(options)
+    end
+
+    def clean_up_result_keys(key_base)
+      Sidekiq.redis do |redis|
+        redis.keys(key_base + '*').each do |key|
+          redis.del(key)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/clutch/job_wrapper.rb

@@ -0,0 +1,32 @@
+module Sidekiq
+  class Clutch
+    class JobWrapper
+      include Sidekiq::Worker
+
+      # 22 days - how long a Sidekiq job can live with exponential backoff
+      RESULT_KEY_EXPIRATION_DURATION = 22 * 24 * 60 * 60
+
+      def perform(bid, job_class, args, last_result_key, current_result_key)
+        job = Object.const_get(job_class).new
+        assign_previous_results(job, last_result_key)
+        job.define_singleton_method(:batch) { Sidekiq::Batch.new(bid) }
+        result = job.perform(*args)
+        Sidekiq.redis do |redis|
+          redis.multi do |multi|
+            multi.rpush(current_result_key, result.to_json)
+            multi.expire(current_result_key, RESULT_KEY_EXPIRATION_DURATION)
+          end
+        end
+      end
+
+      private
+
+      def assign_previous_results(job, last_result_key)
+        return unless job.respond_to?(:previous_results=)
+        job.previous_results = Sidekiq.redis { |c| c.lrange(last_result_key, 0, -1) }.map do |r|
+          JSON.parse(r, quirks_mode: true) # quirks_mode allows a bare string or number
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/clutch/jobs_collection.rb

@@ -0,0 +1,38 @@
+module Sidekiq
+  class Clutch
+    class JobsCollection
+      def initialize(service)
+        @service = service
+        @jobs = []
+        @result_key_prefix = SecureRandom.uuid
+        @result_key_index = 0
+      end
+
+      def empty?
+        @jobs.empty?
+      end
+
+      def raw
+        @jobs
+      end
+
+      def raw=(jobs)
+        @jobs = jobs
+      end
+
+      def <<((klass, *params))
+        if @service.parallel?
+          @jobs << { 'parallel' => [], 'result_key' => next_result_key } unless @jobs.last && @jobs.last['parallel']
+          @jobs.last['parallel'] << [klass.name, params]
+        else
+          @jobs << { 'series' => [klass.name, params], 'result_key' => next_result_key }
+        end
+      end
+
+      def next_result_key
+        @result_key_index += 1
+        "#{@result_key_prefix}-#{@result_key_index}"
+      end
+    end
+  end
+end

data/lib/sidekiq/clutch/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  class Clutch
+    VERSION = "1.0.0"
+  end
+end

data/lib/sidekiq/clutch/worker.rb

@@ -0,0 +1,24 @@
+module Sidekiq
+  class Clutch
+    module Worker
+      def self.included(base)
+        base.class_eval do
+          attr_accessor :previous_results
+        end
+      end
+
+      def perform(last_result_key, current_result_key, args)
+        self.previous_results = Sidekiq.redis { |c| c.lrange(last_result_key, 0, -1) }.map do |r|
+          JSON.parse(r, quirks_mode: true) # quirks_mode allows a bare string or number
+        end
+        result = perform!(*args)
+        Sidekiq.redis do |redis|
+          redis.multi do |multi|
+            multi.rpush(current_result_key, result.to_json)
+            multi.expire(current_result_key, RESULT_KEY_EXPIRATION_DURATION)
+          end
+        end
+      end
+    end
+  end
+end

data/sidekiq-clutch.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sidekiq/clutch/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'sidekiq-clutch'
+  spec.version       = Sidekiq::Clutch::VERSION
+  spec.authors       = ['Tim Morgan']
+  spec.email         = ['tim@timmorgan.org']
+
+  spec.summary       = 'An ergonomic wrapper API for Sidekiq Batches'
+  spec.description   = 'Sidekiq::Clutch provides an ergonomic wrapper API for Sidekiq Batches ' \
+                       'so you can easily manage serial and parallel jobs.'
+  spec.homepage      = 'https://github.com/seven1m/sidekiq-clutch'
+  spec.license       = 'MIT'
+
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files`.split("\n").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'sidekiq', '>= 5.0.0'
+
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-clutch
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Tim Morgan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-11-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.0
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Sidekiq::Clutch provides an ergonomic wrapper API for Sidekiq Batches
+  so you can easily manage serial and parallel jobs.
+email:
+- tim@timmorgan.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sidekiq/clutch.rb
+- lib/sidekiq/clutch/job_wrapper.rb
+- lib/sidekiq/clutch/jobs_collection.rb
+- lib/sidekiq/clutch/version.rb
+- lib/sidekiq/clutch/worker.rb
+- sidekiq-clutch.gemspec
+homepage: https://github.com/seven1m/sidekiq-clutch
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.3
+signing_key: 
+specification_version: 4
+summary: An ergonomic wrapper API for Sidekiq Batches
+test_files: []