sidekiq-superworker 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 SocialPandas
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,106 @@
+Sidekiq Superworker
+===================
+Chain together Sidekiq workers in parallel and/or serial configurations
+
+Overview
+--------
+
+Sidekiq Superworker lets you create superworkers, which are simple or complex chains of Sidekiq workers.
+
+For example, you can define complex chains of workers, and even use parallel blocks:
+
+[![](https://raw.github.com/socialpandas/sidekiq-superworker/master/doc/diagram-complex.png)](https://raw.github.com/socialpandas/sidekiq-superworker/master/doc/diagram-complex.png)
+
+*(Worker10 will run after Worker5, Worker7, Worker8, and Worker9 have all completed.)*
+
+```ruby
+Superworker.create(:MySuperworker, :user_id, :comment_id) do
+  Worker1 :user_id
+  Worker2 :user_id do
+    parallel do
+      Worker3 :comment_id do
+        Worker4 :comment_id
+        Worker5 :comment_id
+      end
+      Worker6 :user_id do
+        parallel do
+          Worker7 :user_id
+          Worker8 :user_id
+          Worker9 :user_id
+        end
+      end
+    end
+    Worker10 :comment_id
+  end
+end
+```
+
+And you can run it like any other worker:
+
+```ruby
+MySuperworker.perform_async(23, 852)
+```
+
+You can also define simple serial chains of workers:
+
+[![](https://raw.github.com/socialpandas/sidekiq-superworker/master/doc/diagram-simple.png)](https://raw.github.com/socialpandas/sidekiq-superworker/master/doc/diagram-simple.png)
+
+```ruby
+Superworker.create(:MySuperworker, :user_id, :comment_id) do
+  Worker1 :user_id, :comment_id
+  Worker2 :comment_id
+  Worker3 :user_id
+end
+```
+
+If you're also using [sidekiq_monitor](https://github.com/socialpandas/sidekiq_monitor), you can easily monitor when a superworker is running and when it has finished.
+
+Installation
+------------
+
+Include it in your Gemfile:
+
+    gem 'sidekiq-superworker'
+
+Install and run the migration:
+
+    rails g sidekiq:superworker:install
+    rake db:migrate
+
+Usage
+-----
+
+### Arguments
+
+You can define any number of arguments for the superworker and pass them to different subworkers as you see fit:
+
+```ruby
+Superworker.create(:MySuperworker, :user_id, :comment_id) do
+  Worker1 :user_id, :comment_id
+  Worker2 :comment_id
+  Worker3 :user_id
+end
+```
+
+If you want to set any static arguments for the subworkers, you can do that by using any values that are not symbols (e.g. strings, integers, etc):
+
+```ruby
+Superworker.create(:MySuperworker, :user_id, :comment_id) do
+  Worker1 100, :user_id, :comment_id
+  Worker2 'all'
+end
+```
+
+If a subworker doesn't take any arguments, you'll need to include parentheses after it:
+
+```ruby
+Superworker.create(:MySuperworker, :user_id, :comment_id) do
+  Worker1 :user_id, :comment_id
+  Worker2()
+end
+```
+
+License
+-------
+
+Sidekiq Superworker is released under the MIT License. Please see the MIT-LICENSE file for details.

data/Rakefile

@@ -0,0 +1,12 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+task :default => :test
+
+Rake::TestTask.new do |t|
+  t.libs << "lib"
+  t.libs << "test"
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end

data/app/models/sidekiq/superworker/subjob.rb

@@ -0,0 +1,31 @@
+module Sidekiq
+  module Superworker
+    class Subjob < ActiveRecord::Base
+      attr_accessible :jid, :subjob_id, :superjob_id, :parent_id, :children_ids, :next_id,
+        :subworker_class, :superworker_class, :arg_keys, :arg_values, :status
+
+      serialize :children_ids
+      serialize :arg_keys
+      serialize :arg_values
+
+      validates_presence_of :subjob_id, :subworker_class, :superworker_class, :status
+
+      def relatives
+        self.class.where(superjob_id: superjob_id)
+      end
+
+      def parent
+        return nil if parent_id.nil?
+        relatives.where(subjob_id: parent_id).first
+      end
+
+      def children
+        relatives.where(parent_id: subjob_id).order(:subjob_id)
+      end
+
+      def next
+        relatives.where(subjob_id: next_id).first
+      end
+    end
+  end
+end

data/lib/generators/sidekiq/superworker/install/install_generator.rb

@@ -0,0 +1,27 @@
+require 'rails/generators'
+require 'rails/generators/base'
+
+module Sidekiq
+  module Superworker
+    module Generators
+      class InstallGenerator < ::Rails::Generators::Base
+        include ::Rails::Generators::Migration
+        source_root File.expand_path('../templates', __FILE__)
+        desc "Install the migrations"
+
+        def self.next_migration_number(path)
+          unless @prev_migration_nr
+            @prev_migration_nr = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+          else
+            @prev_migration_nr += 1
+          end
+          @prev_migration_nr.to_s
+        end
+
+        def install_migrations
+          migration_template "create_sidekiq_superworker_subjobs.rb", "db/migrate/create_sidekiq_superworker_subjobs.rb"
+        end
+      end 
+    end
+  end
+end

data/lib/generators/sidekiq/superworker/install/templates/create_sidekiq_superworker_subjobs.rb

@@ -0,0 +1,25 @@
+class CreateSidekiqSuperworkerSubjobs < ActiveRecord::Migration
+  def change
+    create_table :sidekiq_superworker_subjobs do |t|
+      t.string :jid
+      t.string :superjob_id, null: false
+      t.integer :subjob_id, null: false
+      t.integer :parent_id
+      t.text :children_ids
+      t.integer :next_id
+      t.string :superworker_class, null: false
+      t.string :subworker_class, null: false
+      t.text :arg_keys
+      t.text :arg_values
+      t.string :status, null: false
+      t.boolean :descendants_are_complete, default: false
+      
+      t.timestamps
+    end
+
+    add_index :sidekiq_superworker_subjobs, :jid
+    add_index :sidekiq_superworker_subjobs, :subjob_id
+    add_index :sidekiq_superworker_subjobs, [:superjob_id, :subjob_id]
+    add_index :sidekiq_superworker_subjobs, [:superjob_id, :parent_id]
+  end
+end

data/lib/sidekiq/superworker/dsl_evaluator.rb

@@ -0,0 +1,9 @@
+module Sidekiq
+  module Superworker
+    class DSLEvaluator
+      def method_missing(method, *args, &block)
+        Fiber.yield([method, args, block])
+      end
+    end
+  end
+end

data/lib/sidekiq/superworker/dsl_parser.rb

@@ -0,0 +1,78 @@
+module Sidekiq
+  module Superworker
+    class DSLParser
+      def self.parse(block)
+        @dsl_evaluator = DSLEvaluator.new
+        set_records_from_block(block)
+        {
+          records: @records,
+          nested_records: @nested_records
+        }
+      end
+
+      def self.set_records_from_block(block)
+        @record_id = 0
+        @records = {}
+        @nested_records = block_to_nested_records(block)
+        set_records_from_nested_records(@nested_records)
+      end
+
+      def self.set_records_from_nested_records(nested_records, parent_id=nil)
+        last_id = nil
+        nested_records.each do |id, value|
+          @records[id] = {
+            subjob_id: id,
+            subworker_class: value[:subworker_class].to_s,
+            arg_keys: value[:arg_keys],
+            parent_id: parent_id,
+            children_ids: value[:children] ? value[:children].keys : nil
+          }
+          @records[last_id][:next_id] = id if @records[last_id]
+          last_id = id
+          set_records_from_nested_records(value[:children], id) if value[:children]
+        end
+      end
+
+      def self.block_to_nested_records(block)
+        fiber = Fiber.new do
+          @dsl_evaluator.instance_eval(&block)
+        end
+        
+        nested_records = {}
+        while (method_result = fiber.resume)
+          method, arg_keys, block = method_result
+          @record_id += 1
+          if block
+            nested_records[@record_id] = { subworker_class: method, arg_keys: arg_keys, children: block_to_nested_records(block) }
+          else
+            nested_records[@record_id] = { subworker_class: method, arg_keys: arg_keys }
+          end
+
+          # For superworkers nested within other superworkers, we'll take the subworkers' nested_records,
+          # adjust their ids to fit in with our current @record_id value, and add them into the tree.
+          if method != :parallel
+            subworker_class = method.to_s.constantize
+            if subworker_class.respond_to?(:is_a_superworker?) && subworker_class.is_a_superworker?
+              parent_record_id = @record_id
+              nested_records[parent_record_id][:children] = rewrite_ids_of_subworker_records(subworker_class.nested_records)
+            end
+          end
+        end
+        nested_records
+      end
+
+      def self.rewrite_ids_of_subworker_records(nested_records)
+        new_hash = {}
+        nested_records.each do |old_record_id, record|
+          @record_id += 1
+          parent_record_id = @record_id
+          new_hash[parent_record_id] = record
+          if record[:children]
+            new_hash[parent_record_id][:children] = rewrite_ids_of_subworker_records(record[:children])
+          end
+        end
+        new_hash
+      end
+    end
+  end
+end

data/lib/sidekiq/superworker/processor.rb

@@ -0,0 +1,31 @@
+module Sidekiq
+  module Superworker
+    class Processor
+      def complete(item, new_thread=true)
+        if new_thread
+          # Run this in a new thread so that its execution isn't considered to be part of the
+          # completed job's execution.
+          Thread.new do
+            complete_item(item)
+          end
+        else
+          complete_item(item)
+        end
+      end
+
+      protected
+
+      def complete_item(item)
+        raise "Job has nil jid: #{item}" if item['jid'].nil?
+        # The job may complete before the Subjob record is created; in case that happens,
+        # we need to sleep briefly and requery.
+        tries = 3
+        while !(subjob = Subjob.find_by_jid(item['jid'])) && tries > 0
+          sleep 1
+          tries -= 1
+        end
+        SubjobProcessor.complete(subjob) if subjob
+      end
+    end
+  end
+end

data/lib/sidekiq/superworker/server/middleware.rb

@@ -0,0 +1,16 @@
+module Sidekiq
+  module Superworker
+    module Server
+      class Middleware
+        def initialize(options=nil)
+          @processor = Sidekiq::Superworker::Processor.new
+        end
+
+        def call(worker, item, queue)
+          yield
+          @processor.complete(item)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/superworker/subjob_processor.rb

@@ -0,0 +1,112 @@
+module Sidekiq
+  module Superworker
+    class SubjobProcessor
+      def self.enqueue(subjob)
+        # Only enqueue subjobs that aren't running, complete, etc
+        return unless subjob.status == 'initialized'
+        
+        # If this is a parallel subjob, enqueue all of its children
+        if subjob.subworker_class == 'parallel'
+          subjob.update_attribute(:status, 'running')
+          jids = subjob.children.collect do |child|
+            enqueue(child)
+          end
+          jid = jids.first
+        else
+          klass = "::#{subjob.subworker_class}".constantize
+
+          # If this is a superworker, mark it as complete, which will queue its children or its next subjob
+          if klass.respond_to?(:is_a_superworker?) && klass.is_a_superworker?
+            complete(subjob)
+          # Otherwise, enqueue it in Sidekiq
+          else
+            jid = enqueue_in_sidekiq(subjob, klass)
+            subjob.update_attributes(
+              jid: jid,
+              status: 'queued'
+            )
+          end
+        end
+        jid
+      end
+
+      def self.enqueue_in_sidekiq(subjob, klass)
+        # If sidekiq-unique-jobs is being used for this worker, a number of issues arise if the subjob isn't
+        # queued, so we'll bypass the unique functionality of the worker while running the subjob.
+        is_unique = klass.respond_to?(:sidekiq_options_hash) && !!klass.sidekiq_options_hash['unique']
+        if is_unique
+          unique_value = klass.sidekiq_options_hash.delete('unique')
+          unique_job_expiration_value = klass.sidekiq_options_hash.delete('unique_job_expiration')
+        end
+
+        arg_values = subjob.arg_values
+        jid = klass.perform_async(*arg_values)
+        warn "Nil JID returned by #{subjob.subworker_class}.perform_async with arguments #{arg_values}" if jid.nil?
+
+        if is_unique
+          klass.sidekiq_options_hash['unique'] = unique_value
+          klass.sidekiq_options_hash['unique_job_expiration'] = unique_job_expiration_value
+        end
+
+        jid
+      end
+
+      def self.complete(subjob)
+        subjob.update_attribute(:status, 'complete')
+
+        # If children are present, enqueue the first one
+        children = subjob.children
+        if children.present?
+          enqueue(children.first)
+          return
+        # Otherwise, set this as having its descendants complete
+        else
+          descendants_are_complete(subjob)
+        end
+      end
+
+      def self.descendants_are_complete(subjob)
+        subjob.update_attribute(:descendants_are_complete, true)
+
+        parent = subjob.parent
+        is_child_of_parallel = parent && parent.subworker_class == 'parallel'
+
+        # If this is a child of a parallel subjob, check to see if the parent's descendants are all complete
+        # and call descendants_are_complete(parent) if so
+        if parent
+          siblings_descendants_are_complete = parent.children.all? { |child| child.descendants_are_complete }
+          if siblings_descendants_are_complete
+            descendants_are_complete(parent)
+            parent.update_attribute(:status, 'complete') if is_child_of_parallel
+          end
+        end
+
+        unless is_child_of_parallel
+          # If a next subjob is present, enqueue it
+          next_subjob = subjob.next
+          if next_subjob
+            enqueue(next_subjob)
+            return
+          end
+
+          # Otherwise, if a parent exists, the parent's descendants are complete
+          if parent
+            descendants_are_complete(parent)
+          # Otherwise, this is the final subjob of the superjob
+          else
+            # Set the superjob Sidekiq::Monitor::Job as being complete
+            if defined?(Sidekiq::Monitor)
+              job = Sidekiq::Monitor::Job.where(queue: :superworker, jid: subjob.superjob_id).first
+              if job
+                job.update_attributes(
+                  status: 'complete',
+                  finished_at: Time.now
+                )
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/superworker/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  module Superworker
+    VERSION = '0.0.1'
+  end
+end

data/lib/sidekiq/superworker/worker.rb

@@ -0,0 +1,72 @@
+module Sidekiq
+  module Superworker
+    class Worker
+      def self.create(*args, &block)
+        class_name = args.shift.to_sym
+        dsl = DSLParser.parse(block)
+        create_class(class_name, args, dsl)
+      end
+
+      protected
+
+      def self.create_class(class_name, arg_keys, dsl)
+        klass = Class.new do
+          @class_name = class_name
+          @arg_keys = arg_keys
+          @records = dsl[:records]
+          @nested_records = dsl[:nested_records]
+
+          class << self
+            attr_reader :nested_records
+
+            def is_a_superworker?
+              true
+            end
+
+            def perform_async(*arg_values)
+              @args = Hash[@arg_keys.zip(arg_values)]
+              subjobs = create_subjobs
+
+              # Create a Sidekiq::Monitor::Job for the superjob
+              if defined?(Sidekiq::Monitor)
+                now = Time.now
+                Sidekiq::Monitor::Job.create(
+                  args: arg_values,
+                  class_name: @class_name,
+                  enqueued_at: now,
+                  jid: @superjob_id,
+                  queue: :superworker,
+                  started_at: now,
+                  status: 'running'
+                )
+              end
+
+              # Enqueue the first root-level subjob
+              first_subjob = subjobs.select{ |subjob| subjob.parent_id.nil? }.first
+              SubjobProcessor.enqueue(first_subjob)
+            end
+
+            protected
+
+            def create_subjobs
+              @superjob_id = SecureRandom.hex(12)
+              @records.collect do |id, record|
+                record[:status] = 'initialized'
+                record[:superjob_id] = @superjob_id
+                record[:superworker_class] = @class_name
+                record[:arg_values] = record[:arg_keys].collect do |arg_key|
+                  # Allow for subjob arg_values to be set within the superworker definition; if a symbol is
+                  # used in the DSL, use @args[arg_key], and otherwise use arg_key as the value
+                  arg_key.is_a?(Symbol) ? @args[arg_key] : arg_key
+                end
+                Sidekiq::Superworker::Subjob.create(record)
+              end
+            end
+          end
+        end
+
+        Object.const_set(class_name, klass)
+      end
+    end
+  end
+end

data/lib/sidekiq/superworker.rb

@@ -0,0 +1,22 @@
+require 'sidekiq'
+
+directory = File.dirname(File.absolute_path(__FILE__))
+Dir.glob("#{directory}/superworker/**/*.rb") { |file| require file }
+Dir.glob("#{directory}/../generators/sidekiq/superworker/**/*.rb") { |file| require file }
+Dir.glob("#{directory}/../../app/models/sidekiq/superworker/*.rb") { |file| require file }
+
+module Sidekiq
+  module Superworker
+    def self.table_name_prefix
+      'sidekiq_superworker_'
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Superworker::Server::Middleware
+  end
+end
+
+Superworker = Sidekiq::Superworker::Worker unless Object.const_defined?('Superworker')

data/lib/sidekiq-superworker.rb

@@ -0,0 +1 @@
+require 'sidekiq/superworker'

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-superworker
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Tom Benner
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Chain together Sidekiq workers in parallel and/or serial configurations
+email:
+- tombenner@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- app/models/sidekiq/superworker/subjob.rb
+- lib/generators/sidekiq/superworker/install/install_generator.rb
+- lib/generators/sidekiq/superworker/install/templates/create_sidekiq_superworker_subjobs.rb
+- lib/sidekiq/superworker/dsl_evaluator.rb
+- lib/sidekiq/superworker/dsl_parser.rb
+- lib/sidekiq/superworker/processor.rb
+- lib/sidekiq/superworker/server/middleware.rb
+- lib/sidekiq/superworker/subjob_processor.rb
+- lib/sidekiq/superworker/version.rb
+- lib/sidekiq/superworker/worker.rb
+- lib/sidekiq/superworker.rb
+- lib/sidekiq-superworker.rb
+- MIT-LICENSE
+- Rakefile
+- README.md
+homepage: https://github.com/socialpandas/sidekiq-superworker
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Chain together Sidekiq workers in parallel and/or serial configurations
+test_files: []