delayed_job_groups_plugin 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2a4de75ba53fbd7b7777b5c65ccdbb29a509cf51
+  data.tar.gz: 611191660f2dcf4b942efe7916006e1f710a8e29
+SHA512:
+  metadata.gz: 0894bd53ffa33e924e1319034f462f7cd2e4b6e2b0ca39bc9cb4209dbdcf89cd5f36cfcca63012639685a05e86aa4402b97608faa672debc4607b6c96b55f106
+  data.tar.gz: b301f22a1c2334058f8d6f3b4c463e5be9cc7aa252d60a713ef79d328a2ea0692e610fd123e1fb367bf8cd4d215c0f0d5e34350fb5df26605fe262ccef6b3ad9

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.iml
+*.rbc
+.bundle
+.config
+.idea/
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+log/
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+env:
+  matrix:
+    - RAILS_VERSION="~> 3.2.0" JRUBY_OPTS="$JRUBY_OPTS --debug"
+    - RAILS_VERSION="~> 4.0.0" JRUBY_OPTS="$JRUBY_OPTS --debug"
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+  - jruby-19mode

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Joel Turkel
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,104 @@
+# Delayed Job Groups
+[![Build Status](https://secure.travis-ci.org/salsify/delayed_job_groups_plugin.png?branch=master)][travis]
+[![Code Climate](https://codeclimate.com/github/salsify/delayed_job_groups_plugin.png)][codeclimate]
+[![Coverage Status](https://coveralls.io/repos/salsify/delayed_job_groups_plugin/badge.png)][coveralls]
+
+[travis]: http://travis-ci.org/salsify/delayed_job_groups_plugin
+[codeclimate]: https://codeclimate.com/github/salsify/delayed_job_groups_plugin
+[coveralls]: https://coveralls.io/r/salsify/delayed_job_groups_plugin
+
+A [Delayed Job](https://github.com/collectiveidea/delayed_job) plugin that adds job groups supporting:
+
+* Canceling all jobs in a job group
+* Canceling the job group when a job in the job group fails
+* Blocking and unblocking all jobs in a job group
+* Running additional processing after all jobs in a job group complete or a job group fails
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'delayed_job_groups_plugin'
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install delayed_job_groups_plugin
+
+Run the required database migrations:
+
+    $ rails generate delayed_job_groups_plugin:install
+    $ rake db:migrate
+
+## Usage
+
+Creating a job group and queueing some jobs:
+
+```ruby
+job_group = Delayed::JobGroups::JobGroup.create!
+
+# JobGroup#enqueue has the same signature as Delayed::Job.enqueue 
+# i.e. it takes a job and an optional hash of options.
+job_group.enqueue(MyJob.new('some arg'), queue: 'general')
+job_group.enqueue(MyJob.new('some other arg'), queue: 'general', priority: 10)
+
+# Tell the JobGroup we're done queueing jobs
+job_group.mark_queueing_complete
+```
+
+Registering a job to run after all jobs in the job group have completed:
+
+```ruby
+# We can optionally pass options that will be used when queueing the on completion job
+job_group = Delayed::JobGroups::JobGroup.create!(on_completion_job: MyCompletionJob.new, 
+                                                 on_completion_job_options: { queue: 'general' })
+```
+
+Registering a job to run if the job group is canceled or fails:
+
+```ruby
+# We can optionally pass options that will be used when queueing the on cancellation job
+job_group = Delayed::JobGroups::JobGroup.create!(on_cancellation_job: MyCancellationJob.new, 
+                                                 on_cancellation_job_options: { queue: 'general' })
+```
+
+Block and unblock jobs in a job group:
+
+```ruby
+# Construct the JobGroup in a blocked state
+job_group = Delayed::JobGroups::JobGroup.create!(blocked: true)
+job_group.enqueue(MyJob.new('some arg'), queue: 'general')
+job_group.mark_queueing_complete
+ 
+# Do more stuff...
+ 
+# Unblock the JobGroup so its jobs can run
+job_group.unblock
+```
+
+Cancel a job group:
+
+```ruby
+job_group = Delayed::JobGroups::JobGroup.create!
+ 
+# Do more stuff...
+ 
+job_group.cancel
+```
+
+## Supported Platforms
+
+* Only the Delayed Job Active Record backend is supported.
+* Tested with Rails 3.2 and 4.0.
+* Tested with MRI 1.9.3, 2.0.0, 2.1.0 and JRuby in 1.9 mode.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/setup'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.verbose = false
+end
+
+task default: :spec

data/delayed_job_groups.gemspec

@@ -0,0 +1,37 @@
+# encoding: UTF-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'delayed/job_groups/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'delayed_job_groups_plugin'
+  spec.version       = Delayed::JobGroups::VERSION
+  spec.authors       = ['Joel Turkel']
+  spec.email         = ['jturkel@salsify.com']
+  spec.description   = %q{Aggregates Delayed::Job jobs into groups with group level management and lifecycle callbacks}
+  spec.summary       = %q{Delayed::Job job groups plugin}
+  spec.homepage      = 'https://github.com/salsify/delayed_job_groups_plugin'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.test_files    = Dir.glob('spec/**/*')
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'delayed_job', '>= 3.0'
+  spec.add_dependency 'delayed_job_active_record', '>= 0.4'
+
+  spec.add_development_dependency 'activerecord', ENV.fetch('RAILS_VERSION', ['>= 3.2', '< 4.1'])
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'database_cleaner', '>= 1.2'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec', '>= 2.14'
+  spec.add_development_dependency 'simplecov', '~> 0.7.1'
+
+  if RUBY_PLATFORM == 'java'
+    spec.add_development_dependency 'jdbc-sqlite3'
+    spec.add_development_dependency 'activerecord-jdbcsqlite3-adapter'
+  else
+    spec.add_development_dependency 'sqlite3'
+  end
+
+end

data/lib/delayed/job_groups/compatibility.rb

@@ -0,0 +1,16 @@
+# encoding: UTF-8
+
+require 'active_support/version'
+require 'active_record/version'
+
+module Delayed
+  module JobGroups
+    module Compatibility
+
+      def self.mass_assignment_security_enabled?
+        ::ActiveRecord::VERSION::MAJOR < 4 || defined?(::ActiveRecord::MassAssignmentSecurity)
+      end
+
+    end
+  end
+end

data/lib/delayed/job_groups/job_extensions.rb

@@ -0,0 +1,30 @@
+# encoding: UTF-8
+
+module Delayed
+  module JobGroups
+    module JobExtensions
+      extend ActiveSupport::Concern
+
+      included do
+        if Delayed::JobGroups::Compatibility.mass_assignment_security_enabled?
+          attr_accessible :job_group_id, :blocked
+        end
+
+        belongs_to :job_group, class_name: 'Delayed::JobGroups::JobGroup'
+
+        class << self
+
+          # Patch ready_to_run to exclude blocked jobs
+          def ready_to_run_with_blocked_filtering(worker_name, max_run_time)
+            ready_to_run_without_blocked_filtering(worker_name, max_run_time).where(blocked: false)
+          end
+          alias_method_chain :ready_to_run, :blocked_filtering
+        end
+      end
+
+      def in_job_group?
+        job_group_id.present?
+      end
+    end
+  end
+end

data/lib/delayed/job_groups/job_group.rb

@@ -0,0 +1,99 @@
+# encoding: UTF-8
+
+module Delayed
+  module JobGroups
+    class JobGroup < ActiveRecord::Base
+
+      self.table_name = "#{ActiveRecord::Base.table_name_prefix}delayed_job_groups"
+
+      if Delayed::JobGroups::Compatibility.mass_assignment_security_enabled?
+        attr_accessible :on_completion_job, :on_completion_job_options, :blocked, :on_cancellation_job,
+                        :on_cancellation_job_options
+      end
+
+      serialize :on_completion_job
+      serialize :on_completion_job_options, Hash
+      serialize :on_cancellation_job
+      serialize :on_cancellation_job_options, Hash
+
+      validates :queueing_complete, :blocked, inclusion: [true, false]
+
+      if ActiveRecord::VERSION::MAJOR >= 4
+        has_many :active_jobs, -> { where(failed_at: nil) }, class_name: Job
+      else
+        has_many :active_jobs, class_name: Job, conditions: {failed_at: nil}
+      end
+
+
+      # Only delete dependent jobs that are unlocked so we can determine if there are in-flight jobs
+      # for canceled job groups
+      if ActiveRecord::VERSION::MAJOR >= 4
+        has_many :queued_jobs, -> { where(failed_at: nil, locked_by: nil) }, class_name: Job,
+                 dependent: :delete_all
+      else
+        has_many :queued_jobs, class_name: Job, conditions: {failed_at: nil, locked_by: nil},
+                 dependent: :delete_all
+      end
+
+      def mark_queueing_complete
+        with_lock do
+          raise 'JobGroup has already completed queueing' if queueing_complete?
+          update_column(:queueing_complete, true)
+          complete if ready_for_completion?
+        end
+      end
+
+      def enqueue(job, options = {})
+        options = options.merge(job_group_id: id)
+        options[:blocked] = blocked?
+        Delayed::Job.enqueue(job, options)
+      end
+
+      def unblock
+        return unless blocked?
+
+        with_lock do
+          update_column(:blocked, false)
+          active_jobs.update_all(blocked: false)
+          complete if ready_for_completion?
+        end
+      end
+
+      def cancel
+        Delayed::Job.enqueue(on_cancellation_job, on_cancellation_job_options || {}) if on_cancellation_job
+        destroy
+      end
+
+      def self.check_for_completion(job_group_id)
+        # Optimization to avoid loading and locking the JobGroup when the group
+        # still has pending jobs
+        return if has_pending_jobs?(job_group_id)
+
+        transaction do
+          # The first completed job to notice the job group's queue count has dropped to
+          # zero will queue the job group's completion job and destroy the job group so
+          # other jobs need to handle the job group having been destroyed already.
+          job_group = where(id: job_group_id).lock(true).first
+          job_group.send(:complete) if job_group && job_group.send(:ready_for_completion?)
+        end
+      end
+
+      def self.has_pending_jobs?(job_group_ids)
+        job_group_ids = Array(job_group_ids)
+        return false if job_group_ids.empty?
+        Delayed::Job.where(job_group_id: job_group_ids, failed_at: nil).exists?
+      end
+
+      private
+
+      def ready_for_completion?
+        queueing_complete? && !JobGroup.has_pending_jobs?(id) && !blocked?
+      end
+
+      def complete
+        Delayed::Job.enqueue(on_completion_job, on_completion_job_options || {}) if on_completion_job
+        destroy
+      end
+    end
+  end
+end

data/lib/delayed/job_groups/plugin.rb

@@ -0,0 +1,65 @@
+# encoding: UTF-8
+
+require 'delayed_job'
+require 'set'
+
+module Delayed
+  module JobGroups
+    class Plugin < Delayed::Plugin
+
+      # Delayed job callbacks will be registered in a global Delayed::Lifecycle every time a
+      # Delayed::Worker is created. This creates problems in test runs that create
+      # multiple workers because we register the callbacks multiple times on the same
+      # global Lifecycle.
+      def self.callbacks(&block)
+        registered_lifecycles = Set.new
+        super do |lifecycle|
+          if registered_lifecycles.add?(lifecycle.object_id)
+            block.call(lifecycle)
+          end
+        end
+      end
+
+      callbacks do |lifecycle|
+        lifecycle.before(:error) do |worker, job|
+          # If the job group has been cancelled then don't let the job be retried
+          if job.in_job_group? && job_group_cancelled?(job.job_group_id)
+            def job.max_attempts
+              1
+            end
+          end
+        end
+
+        lifecycle.before(:failure) do |worker, job|
+          # If a job in the job group fails, then cancel the whole job group.
+          # Need to check that the job group is present since another
+          # job may have concurrently cancelled it.
+          if job.in_job_group? && job.job_group
+            job.job_group.cancel
+          end
+        end
+
+        lifecycle.after(:perform) do |worker, job|
+          # Make sure we only check to see if the job group is complete
+          # if the job succeeded
+          if job.in_job_group? && job_completed?(job)
+            JobGroup.check_for_completion(job.job_group_id)
+          end
+        end
+      end
+
+      private
+
+      def self.job_group_cancelled?(job_group_id)
+        !JobGroup.exists?(job_group_id)
+      end
+
+      def self.job_completed?(job)
+        # Delayed job will already have marked the job for destruction
+        # if it has completed
+        job.destroyed?
+      end
+    end
+  end
+end
+

data/lib/delayed/job_groups/version.rb

@@ -0,0 +1,7 @@
+# encoding: UTF-8
+
+module Delayed
+  module JobGroups
+    VERSION = '0.1.0'
+  end
+end

data/lib/delayed_job_groups_plugin.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+
+require 'active_support'
+require 'active_record'
+require 'delayed_job'
+require 'delayed_job_active_record'
+require 'delayed/job_groups/compatibility'
+require 'delayed/job_groups/job_extensions'
+require 'delayed/job_groups/job_group'
+require 'delayed/job_groups/plugin'
+require 'delayed/job_groups/version'
+
+Delayed::Backend::ActiveRecord::Job.send(:include, Delayed::JobGroups::JobExtensions)
+
+Delayed::Worker.plugins << Delayed::JobGroups::Plugin

data/lib/generators/delayed_job_groups_plugin/install_generator.rb

@@ -0,0 +1,22 @@
+# encoding: UTF-8
+
+require 'rails/generators'
+require 'rails/generators/migration'
+require 'rails/generators/active_record'
+
+module DelayedJobGroupsPlugin
+  class InstallGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    self.source_paths << File.join(File.dirname(__FILE__), 'templates')
+
+    def create_migration_file
+      migration_template('migration.rb', 'db/migrate/create_delayed_job_groups.rb')
+    end
+
+    def self.next_migration_number(dirname)
+      ActiveRecord::Generators::Base.next_migration_number(dirname)
+    end
+
+  end
+end

data/lib/generators/delayed_job_groups_plugin/templates/migration.rb

@@ -0,0 +1,46 @@
+# encoding: UTF-8
+
+class CreateDelayedJobGroups < ActiveRecord::Migration
+
+  def up
+    add_column(:delayed_jobs, :blocked, :boolean, default: false, null: false)
+    add_column(:delayed_jobs, :job_group_id, :integer)
+    add_index(:delayed_jobs, :job_group_id)
+
+    if partial_indexes_supported?
+      remove_index(:delayed_jobs, name: :delayed_jobs_priority)
+      execute <<-SQL
+         CREATE INDEX delayed_jobs_priority
+         ON delayed_jobs(priority, run_at)
+        WHERE failed_at IS NULL AND blocked = FALSE
+      SQL
+    end
+
+    create_table(:delayed_job_groups) do |t|
+      t.text :on_completion_job
+      t.text :on_completion_job_options
+      t.text :on_cancellation_job
+      t.text :on_cancellation_job_options
+      t.boolean :queueing_complete, default: false, null: false
+      t.boolean :blocked, default: false, null: false
+    end
+  end
+
+  def down
+    remove_columns(:delayed_jobs, :blocked, :job_group_id)
+
+    if partial_indexes_supported?
+      execute <<-SQL
+         CREATE INDEX delayed_jobs_priority
+         ON delayed_jobs(priority, run_at)
+         WHERE failed_at IS NULL
+      SQL
+    end
+
+    drop_table(:delayed_job_groups)
+  end
+
+  def partial_indexes_supported?
+    connection.adapter_name == 'PostgreSQL'
+  end
+end

data/spec/db/database.yml

@@ -0,0 +1,3 @@
+sqlite3:
+  adapter: sqlite3
+  database: ":memory:"

data/spec/db/schema.rb

@@ -0,0 +1,29 @@
+# encoding: UTF-8
+
+ActiveRecord::Schema.define(:version => 0) do
+
+  create_table(:delayed_jobs, force: true) do |t|
+    t.integer :priority, default: 0
+    t.integer :attempts, default: 0
+    t.text :handler
+    t.text :last_error
+    t.datetime :run_at
+    t.datetime :locked_at
+    t.datetime :failed_at
+    t.string :locked_by
+    t.string :queue
+    t.timestamps
+    t.boolean :blocked, default: false, null: false
+    t.integer :job_group_id
+  end
+  add_index(:delayed_jobs, :job_group_id)
+
+  create_table(:delayed_job_groups, force: true) do |t|
+    t.text :on_completion_job
+    t.text :on_completion_job_options
+    t.text :on_cancellation_job
+    t.text :on_cancellation_job_options
+    t.boolean :queueing_complete, default: false, null: false
+    t.boolean :blocked, default: false, null: false
+  end
+end

data/spec/delayed/job_groups/job_group_spec.rb

@@ -0,0 +1,223 @@
+# encoding: UTF-8
+
+require 'spec_helper'
+
+describe Delayed::JobGroups::JobGroup do
+
+  let(:blocked) { false }
+  let(:on_completion_job) { 'dummy on completion job' }
+  let(:on_completion_job_options) do
+    { foo: 'bar' }
+  end
+  let(:current_time) { Time.utc(2013) }
+
+  subject(:job_group) do
+    Delayed::JobGroups::JobGroup.create!(on_completion_job: on_completion_job,
+                                         on_completion_job_options: on_completion_job_options,
+                                         blocked: blocked)
+  end
+
+  before do
+    Time.stub(:now).and_return(current_time)
+    Delayed::Job.stub(:enqueue)
+  end
+
+  shared_examples "the job group was completed" do
+    it "queues the completion job" do
+      Delayed::Job.should have_received(:enqueue).with(on_completion_job, on_completion_job_options)
+    end
+
+    it "destroys the job group" do
+      job_group.should have_been_destroyed
+    end
+  end
+
+  shared_examples "the job group was not completed" do
+    it "does not queue the completion job" do
+      Delayed::Job.should_not have_received(:enqueue)
+    end
+
+    it "does not destroy the job group" do
+      job_group.should_not have_been_destroyed
+    end
+  end
+
+  describe "#mark_queueing_complete" do
+
+    context "when no jobs exist" do
+      before { job_group.mark_queueing_complete }
+
+      it { should be_queueing_complete }
+      it_behaves_like "the job group was completed"
+    end
+
+    context "when no jobs exist but the job group is blocked" do
+      let(:blocked) { true }
+      before { job_group.mark_queueing_complete }
+
+      it { should be_queueing_complete }
+      it_behaves_like "the job group was not completed"
+    end
+
+    context "when active jobs exist" do
+      before do
+        Delayed::Job.create!(job_group_id: job_group.id)
+        job_group.mark_queueing_complete
+      end
+
+      it { should be_queueing_complete }
+      it_behaves_like "the job group was not completed"
+    end
+  end
+
+  describe ".check_for_completion" do
+    let!(:job) { Delayed::Job.create!(job_group_id: job_group.id) }
+
+    before do
+      job_group.mark_queueing_complete
+    end
+
+    shared_context "complete job and check job group complete" do
+      before do
+        job.destroy
+        Delayed::JobGroups::JobGroup.check_for_completion(job_group.id)
+      end
+    end
+
+    context "when no jobs exist" do
+      include_context "complete job and check job group complete"
+
+      it_behaves_like "the job group was completed"
+    end
+
+    context "when active jobs exist" do
+      before do
+        Delayed::JobGroups::JobGroup.check_for_completion(job_group.id)
+      end
+
+      it_behaves_like "the job group was not completed"
+    end
+
+    context "when on failed jobs exist" do
+      before do
+        job.update_attributes!(failed_at: Time.now)
+        Delayed::JobGroups::JobGroup.check_for_completion(job_group.id)
+      end
+
+      it_behaves_like "the job group was completed"
+    end
+
+    context "when there are no on_completion_job_options" do
+      let(:on_completion_job_options) { nil }
+
+      include_context "complete job and check job group complete"
+
+      it "queues the completion job with empty options" do
+        Delayed::Job.should have_received(:enqueue).with(on_completion_job, {})
+      end
+
+      it "destroys the job group" do
+        job_group.should have_been_destroyed
+      end
+    end
+
+    context "when there is no on_completion_job" do
+      let(:on_completion_job) { nil }
+
+      include_context "complete job and check job group complete"
+
+      it "doesn't queues the non-existent completion job" do
+        Delayed::Job.should_not have_received(:enqueue)
+      end
+
+      it "destroys the job group" do
+        job_group.should have_been_destroyed
+      end
+    end
+  end
+
+  describe "#enqueue" do
+    let(:job) { 'dummy job' }
+
+    before do
+      job_group.enqueue(job)
+    end
+
+    shared_examples "it enqueues the job in the correct blocked state" do
+      it "enqueues the job in the same blocked state as the job group" do
+        Delayed::Job.should have_received(:enqueue).with(job, job_group_id: job_group.id, blocked: blocked)
+      end
+    end
+
+    it_behaves_like "it enqueues the job in the correct blocked state"
+
+    context "when the job_group is blocked" do
+      let(:blocked) { true }
+
+      it_behaves_like "it enqueues the job in the correct blocked state"
+    end
+  end
+
+  describe "#unblock" do
+
+    context "when the JobGroup isn't blocked" do
+      before do
+        job_group.unblock
+      end
+
+      its(:blocked?) { should be_false }
+    end
+
+    context "when the JobGroup is blocked" do
+      let(:blocked) { true }
+
+      context "when there are pending jobs" do
+        let!(:job) { Delayed::Job.create!(job_group_id: job_group.id, blocked: true) }
+
+        before do
+          job_group.mark_queueing_complete
+          job_group.unblock
+        end
+
+        its(:blocked?) { should be_false }
+
+        it "sets the job's run_at to the current time" do
+          job.reload.run_at.should eq current_time
+        end
+
+        it_behaves_like "the job group was not completed"
+      end
+
+      describe "when there are no pending jobs" do
+        before do
+          job_group.mark_queueing_complete
+          job_group.unblock
+        end
+
+        its(:blocked?) { should be_false }
+        it_behaves_like "the job group was completed"
+      end
+    end
+  end
+
+  describe "#cancel" do
+    let!(:queued_job) { Delayed::Job.create!(job_group_id: job_group.id) }
+    let!(:running_job)  { Delayed::Job.create!(job_group_id: job_group.id, locked_at: Time.now, locked_by: 'test') }
+
+    before do
+      job_group.cancel
+    end
+
+    it "destroys the job group" do
+      job_group.should have_been_destroyed
+    end
+
+    it "destroys queued jobs" do
+      queued_job.should have_been_destroyed
+    end
+
+    it "does not destroy running jobs" do
+      running_job.should_not have_been_destroyed
+    end
+  end
+end

data/spec/delayed/job_groups/plugin_spec.rb

@@ -0,0 +1,255 @@
+# encoding: UTF-8
+
+require 'spec_helper'
+
+describe Delayed::JobGroups::Plugin do
+
+  before(:all) do
+    @old_max_attempts = Delayed::Worker.max_attempts
+    Delayed::Worker.max_attempts = 2
+  end
+
+  after(:all) do
+    Delayed::Worker.max_attempts = @old_max_attempts
+  end
+
+  before(:each) do
+    CompletionJob.invoked = false
+    CancellationJob.invoked = false
+  end
+
+  let!(:job_group) { Delayed::JobGroups::JobGroup.create!(on_completion_job: CompletionJob.new) }
+
+  it "runs the completion job after completing other jobs" do
+    job_group.enqueue(NoOpJob.new)
+    job_group.enqueue(NoOpJob.new)
+    job_group.mark_queueing_complete
+    job_group_count.should eq 1
+    queued_job_count.should eq 2
+
+    # Run our first job
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_false
+    job_group_count.should eq 1
+    queued_job_count.should eq 1
+
+    # Run our second job which should enqueue the completion job
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_false
+    job_group_count.should eq 0
+    queued_job_count.should eq 1
+
+    # Now we should run the completion job
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_true
+    queued_job_count.should eq 0
+  end
+
+  it "only runs the completion job after queueing is completed" do
+    job_group.enqueue(NoOpJob.new)
+    job_group.enqueue(NoOpJob.new)
+    job_group_count.should eq 1
+    queued_job_count.should eq 2
+
+    # Run our first job
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_false
+    job_group_count.should eq 1
+    queued_job_count.should eq 1
+
+    # Run our second job
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_false
+    job_group_count.should eq 1
+    queued_job_count.should eq 0
+
+    # Mark queueing complete which should queue the completion job
+    job_group.mark_queueing_complete
+    job_group_count.should eq 0
+    queued_job_count.should eq 1
+
+    # Now we should run the completion job
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_true
+    queued_job_count.should eq 0
+  end
+
+  it "cancels the job group if a job fails" do
+    Delayed::Worker.max_attempts = 1
+
+    job_group.enqueue(FailingJob.new)
+    job_group.enqueue(NoOpJob.new)
+    job_group.mark_queueing_complete
+    queued_job_count.should eq 2
+    job_group_count.should eq 1
+
+    # Run the job which should fail and cancel the JobGroup
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_false
+    failed_job_count.should eq 1
+    queued_job_count.should eq 0
+    job_group_count.should eq 0
+  end
+
+  it "doesn't retry failed jobs if the job group has been canceled" do
+    job_group.cancel
+    Delayed::Job.enqueue(FailingJob.new, job_group_id: job_group.id)
+    queued_job_count.should eq 1
+
+    # Run the job which should fail and should not queue a retry
+    Delayed::Worker.new.work_off(1)
+    failed_job_count.should eq 1
+    queued_job_count.should eq 0
+  end
+
+  it "doesn't run jobs until they're unblocked" do
+    job_group.blocked = true
+    job_group.save!
+
+    job_group.enqueue(NoOpJob.new)
+    job_group.enqueue(NoOpJob.new)
+    job_group.mark_queueing_complete
+    Delayed::Job.count.should eq 2
+
+    # No jobs should run because they're blocked
+    (successes, failures) = Delayed::Worker.new.work_off
+    successes.should eq 0
+    failures.should eq 0
+    Delayed::Job.count.should eq 2
+
+    job_group.unblock
+
+    # Run our first job
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_false
+    job_group_count.should eq 1
+    Delayed::Job.count.should eq 1
+
+    # Run our second job which should enqueue the completion job
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_false
+    job_group_count.should eq 0
+    Delayed::Job.count.should eq 1
+
+    # Now we should run the completion job
+    Delayed::Worker.new.work_off(1)
+    CompletionJob.invoked.should be_true
+    Delayed::Job.count.should eq 0
+  end
+
+  context "when a cancellation job is provided" do
+    let!(:job_group) do
+      Delayed::JobGroups::JobGroup.create!(on_completion_job: CompletionJob.new,
+                                           on_cancellation_job: CancellationJob.new)
+    end
+
+    it "runs the cancellation job after a job error causes cancellation" do
+      Delayed::Worker.max_attempts = 1
+
+      job_group.enqueue(FailingJob.new)
+      job_group.enqueue(NoOpJob.new)
+      job_group.mark_queueing_complete
+      queued_job_count.should eq 2
+      job_group_count.should eq 1
+
+      # Run the job which should fail and cancel the JobGroup
+      Delayed::Worker.new.work_off(1)
+      CompletionJob.invoked.should be_false
+      CancellationJob.invoked.should be_false
+      failed_job_count.should eq 1
+
+      queued_job_count.should eq 1
+      job_group_count.should eq 0
+
+      # Now we should run the cancellation job
+      Delayed::Worker.new.work_off(1)
+      CompletionJob.invoked.should be_false
+      CancellationJob.invoked.should be_true
+      queued_job_count.should eq 0
+    end
+
+    it "runs the cancellation job after the job group is cancelled" do
+      job_group.enqueue(NoOpJob.new)
+      job_group.enqueue(FailingJob.new)
+      job_group.mark_queueing_complete
+      job_group.cancel
+
+      #cancellation job should be queued
+      queued_job_count.should eq 1
+      CancellationJob.invoked.should be_false
+
+      # Run the cancellation job
+      Delayed::Worker.new.work_off(1)
+      CancellationJob.invoked.should be_true
+      queued_job_count.should eq 0
+    end
+  end
+
+  context "when a no completion job is provided" do
+    let!(:job_group) {  Delayed::JobGroups::JobGroup.create! }
+
+    it "doesn't queue a non-existent completion job" do
+      job_group.enqueue(NoOpJob.new)
+      job_group.enqueue(NoOpJob.new)
+      job_group.mark_queueing_complete
+      job_group_count.should eq 1
+      queued_job_count.should eq 2
+      failed_job_count.should eq 0
+
+      # Run our first job
+      Delayed::Worker.new.work_off(1)
+      job_group_count.should eq 1
+      queued_job_count.should eq 1
+      failed_job_count.should eq 0
+
+      # Run our second job which should delete the job group
+      Delayed::Worker.new.work_off(1)
+      job_group_count.should eq 0
+      queued_job_count.should eq 0
+      failed_job_count.should eq 0
+    end
+  end
+
+  class FailingJob
+
+    def perform
+      raise 'Test failure'
+    end
+
+  end
+
+  class NoOpJob
+
+    def perform
+
+    end
+  end
+
+  class CompletionJob
+    cattr_accessor :invoked
+
+    def perform
+      CompletionJob.invoked = true
+    end
+  end
+
+  class CancellationJob
+    cattr_accessor :invoked
+
+    def perform
+      CancellationJob.invoked = true
+    end
+  end
+
+  def job_group_count
+    Delayed::JobGroups::JobGroup.count
+  end
+
+  def queued_job_count
+    Delayed::Job.where(failed_at: nil).count
+  end
+
+  def failed_job_count
+    Delayed::Job.where('failed_at IS NOT NULL').count
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,55 @@
+# encoding: UTF-8
+
+require 'simplecov'
+require 'coveralls'
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+  SimpleCov::Formatter::HTMLFormatter,
+  Coveralls::SimpleCov::Formatter
+]
+SimpleCov.start do
+ add_filter 'spec'
+end
+
+require 'database_cleaner'
+require 'delayed_job_groups_plugin'
+require 'yaml'
+
+spec_dir = File.dirname(__FILE__)
+Dir["#{spec_dir}/support/**/*.rb"].sort.each { |f| require f }
+
+FileUtils.makedirs('log')
+
+Delayed::Worker.read_ahead = 1
+Delayed::Worker.destroy_failed_jobs = false
+
+Delayed::Worker.logger = Logger.new('log/test.log')
+Delayed::Worker.logger.level = Logger::DEBUG
+ActiveRecord::Base.logger = Delayed::Worker.logger
+ActiveRecord::Migration.verbose = false
+
+db_adapter = ENV.fetch('ADAPTER', 'sqlite3')
+config = YAML.load(File.read('spec/db/database.yml'))
+ActiveRecord::Base.establish_connection(config[db_adapter])
+require 'db/schema'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.order = 'random'
+
+  config.before(:suite) do
+    DatabaseCleaner.clean_with(:truncation)
+  end
+
+  config.before(:each) do
+    DatabaseCleaner.strategy = :transaction
+  end
+
+  config.before(:each) do
+    DatabaseCleaner.start
+  end
+
+  config.after(:each) do
+    DatabaseCleaner.clean
+  end
+end

data/spec/support/destroyed_model.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+
+RSpec::Matchers.define :have_been_destroyed do
+  match do |actual|
+    !actual.class.where(id: actual.id).exists?
+  end
+
+  description do
+    "model should have been destroyed"
+  end
+
+  failure_message_for_should do |actual|
+    "expected #{actual.class}(id: #{actual.id}) to have been destroyed"
+  end
+end

metadata

@@ -0,0 +1,205 @@
+--- !ruby/object:Gem::Specification
+name: delayed_job_groups_plugin
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Joel Turkel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: delayed_job_active_record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Aggregates Delayed::Job jobs into groups with group level management
+  and lifecycle callbacks
+email:
+- jturkel@salsify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- delayed_job_groups.gemspec
+- lib/delayed/job_groups/compatibility.rb
+- lib/delayed/job_groups/job_extensions.rb
+- lib/delayed/job_groups/job_group.rb
+- lib/delayed/job_groups/plugin.rb
+- lib/delayed/job_groups/version.rb
+- lib/delayed_job_groups_plugin.rb
+- lib/generators/delayed_job_groups_plugin/install_generator.rb
+- lib/generators/delayed_job_groups_plugin/templates/migration.rb
+- spec/db/database.yml
+- spec/db/schema.rb
+- spec/delayed/job_groups/job_group_spec.rb
+- spec/delayed/job_groups/plugin_spec.rb
+- spec/spec_helper.rb
+- spec/support/destroyed_model.rb
+homepage: https://github.com/salsify/delayed_job_groups_plugin
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Delayed::Job job groups plugin
+test_files:
+- spec/db/database.yml
+- spec/db/schema.rb
+- spec/delayed/job_groups/job_group_spec.rb
+- spec/delayed/job_groups/plugin_spec.rb
+- spec/spec_helper.rb
+- spec/support/destroyed_model.rb