delayed_job_heartbeat_plugin 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d23c84e35c7097a52dd7c4ff8431ec634adc72b4
+  data.tar.gz: 626b0561368160ea52dd2b7d4cf65240768cd3c7
+SHA512:
+  metadata.gz: fa8a6ff6c94fef094132d2625eb534a2f9b68730fae7696e57001ea513aa28ec7966bfff1b8c9f3e41ac10209ee817c3f495e5288aa4c445999def8c83fa2692
+  data.tar.gz: 5a13041ce00bf26a2dfccd022831f32c1273587e701c9e759423519ae3bbaf4ffe35677bb11ff4c81f2d9bfc6e66114d37bb979d22c33d322a393e03a7b46156

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,14 @@
+sudo: false
+language: ruby
+before_install: gem install bundler -v 1.10.6
+env:
+  matrix:
+    - RAILS_VERSION="~> 3.2.22"
+    - RAILS_VERSION="~> 4.0.13"
+    - RAILS_VERSION="~> 4.1.13"
+    - RAILS_VERSION="~> 4.2.4"
+rvm:
+  - 2.0.0
+  - 2.1.7
+  - 2.2.3
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in delayed_job_heartbeat_plugin.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Joel Turkel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,87 @@
+# Delayed Job Heartbeat Plugin
+
+[![Gem Version](https://badge.fury.io/rb/delayed_job_heartbeat_plugin.png)][gem]
+[![Build Status](https://secure.travis-ci.org/salsify/delayed_job_heartbeat_plugin.png?branch=master)][travis]
+[![Code Climate](https://codeclimate.com/github/salsify/delayed_job_heartbeat_plugin.png)][codeclimate]
+[![Coverage Status](https://coveralls.io/repos/salsify/delayed_job_heartbeat_plugin/badge.png)][coveralls]
+
+[gem]: https://rubygems.org/gems/delayed_job_heartbeat_plugin
+[travis]: http://travis-ci.org/salsify/delayed_job_heartbeat_plugin
+[codeclimate]: https://codeclimate.com/github/salsify/delayed_job_heartbeat_plugin
+[coveralls]: https://coveralls.io/r/salsify/delayed_job_heartbeat_plugin
+
+By default [Delayed Job](https://github.com/collectiveidea/delayed_job) uses the [ClearLocks](https://github.com/collectiveidea/delayed_job/blob/master/lib/delayed/plugins/clear_locks.rb) plugin to unlock jobs when a worker shuts down. Unfortunately this only works if a worker shuts down cleanly. If the worker crashes, the job won't be unlocked until `max_run_time` elapses which can cause unacceptable delays processing background jobs. Enter the Delayed Job Heartbeat Plugin...
+ 
+The Delayed Job Heartbeat Plugin adds a heartbeat to all Delayed Job workers. After a configurable timeout jobs from unresponsive workers can be unlocked either via a Ruby API or a rake task. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'delayed_job_heartbeat_plugin'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install delayed_job_heartbeat_plugin
+
+Run the required database migrations:
+
+    $ rails generate delayed_job_heartbeat_plugin:install
+    $ rake db:migrate
+
+## Usage
+
+There are two parts to using the Delayed Job Heartbeat Plugin:
+
+* Configuring the worker's heartbeat options.
+* Periodically unlocking jobs from unresponsive workers.
+
+### Worker Heartbeat Options
+
+The worker heartbeat can be configured in an initializer (e.g. `config/initializers/delayed_job_heartbeat.rb`) follows:
+
+```ruby
+Delayed::Heartbeat.configure do |configuration|
+  configuration.enabled = Rails.env.production?
+  configuration.heartbeat_interval_seconds = 60
+  configuration.heartbeat_timeout_seconds = 180
+  configuration.worker_termination_enabled = true
+end
+```
+
+The plugin supports the following options (all of which are optional):
+
+* `enabled` - enables/disables the plugin entirely (defaults to true in production and false in other environments)
+* `worker_label` - a label for the worker. Consider setting this to `ENV['DYNO']` if running in Heroku to get the dyno's friendly name (defaults to the Delayed Job worker's name)
+* `heartbeat_interval_seconds` - how often workers should send a heartbeat (defaults to 60 seconds)
+* `heartbeat_timeout_seconds` - theshold after which workers are considered dead if they haven't heartbeated (defaults to 180 seconds)
+* `worker_termination_enabled` - controls whether a worker that detects it has not heartbeated within the timeout period (e.g. due to severe memory swapping) should shut itself down (defaults to true in production and false in other environments)
+* `on_worker_termination` - a callback proc that accepts a `Delayed::Heartbeat::Worker` and an `Exception` if the heartbeat fails. This can be useful for reporting to an error monitoring system. 
+* `worker_version` - a version number of the worker's source code that is only used if you want to cleanup workers from old source code versions (defaults to `nil`)
+
+### Unlocking Unresponsive Workers
+
+Jobs from unresponsive workers can be unlocked by calling `Delayed::Heartbeat.delete_timed_out_workers` or running the `delayed:heartbeat:delete_timed_out_workers` rake task. This should be integrated with a scheduler like cron, [clockwork](https://github.com/tomykaira/clockwork) or the [Heroku Scheduler](https://devcenter.heroku.com/articles/scheduler) (if running in Heroku).
+
+Jobs from workers running an old version of the source code can be unlocked by calling `Delayed::Heartbeat.delete_workers_with_different_version` or running the `delayed:heartbeat:delete_workers_with_different_version` rake task. 
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. 
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/salsify/delayed_job_heartbeat_plugin.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/delayed_job_heartbeat_plugin.gemspec

@@ -0,0 +1,39 @@
+# encoding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'delayed/heartbeat/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'delayed_job_heartbeat_plugin'
+  spec.version       = Delayed::Heartbeat::VERSION
+  spec.authors       = ['Joel Turkel']
+  spec.email         = ['jturkel@salsify.com']
+
+  spec.summary       = 'Delayed::Job plugin to unlock jobs from dead workers'
+  spec.homepage      = 'https://github.com/salsify/delayed_job_heartbeat_plugin'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.test_files    = Dir.glob('spec/**/*')
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 2.0'
+
+  spec.add_dependency 'delayed_job', '>= 4.1.0'
+  spec.add_dependency 'delayed_job_active_record', '>= 4.1.0'
+
+  spec.add_development_dependency 'activerecord', ENV.fetch('RAILS_VERSION', ['>= 3.2', '< 4.3'])
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'database_cleaner', '>= 1.2'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec', '3.3.0'
+  spec.add_development_dependency 'simplecov', '~> 0.7.1'
+  spec.add_development_dependency 'timecop'
+
+  if RUBY_PLATFORM == 'java'
+    spec.add_development_dependency 'jdbc-sqlite3'
+    spec.add_development_dependency 'activerecord-jdbcsqlite3-adapter'
+  else
+    spec.add_development_dependency 'sqlite3'
+  end
+end

data/lib/delayed/heartbeat.rb

@@ -0,0 +1,46 @@
+require 'delayed/heartbeat/compatibility'
+require 'delayed/heartbeat/configuration'
+require 'delayed/heartbeat/delete_worker_results'
+require 'delayed/heartbeat/plugin'
+require 'delayed/heartbeat/version'
+require 'delayed/heartbeat/worker'
+require 'delayed/heartbeat/worker_heartbeat'
+require 'delayed/heartbeat/railtie' if defined?(Rails::Railtie)
+
+module Delayed
+  module Heartbeat
+    @configuration = Delayed::Heartbeat::Configuration.new
+
+    class << self
+      def configure
+        yield(configuration) if block_given?
+      end
+
+      def configuration
+        @configuration
+      end
+
+      def delete_workers_with_different_version(current_version = configuration.worker_version)
+        old_workers = Delayed::Heartbeat::Worker.workers_with_different_version(current_version)
+        cleanup_workers(old_workers, mark_attempt_failed: false)
+      end
+
+      def delete_timed_out_workers(timeout_seconds = configuration.heartbeat_timeout_seconds)
+        dead_workers = Delayed::Heartbeat::Worker.dead_workers(timeout_seconds)
+        cleanup_workers(dead_workers, mark_attempt_failed: true)
+      end
+
+      private
+
+      def cleanup_workers(workers, mark_attempt_failed: true)
+        Delayed::Heartbeat::Worker.transaction do
+          orphaned_jobs = workers.flat_map do |worker|
+            worker.unlock_jobs(mark_attempt_failed: mark_attempt_failed)
+          end
+          Delayed::Heartbeat::Worker.delete_workers(workers)
+          Delayed::Heartbeat::DeleteWorkerResults.new(workers, orphaned_jobs)
+        end
+      end
+    end
+  end
+end

data/lib/delayed/heartbeat/compatibility.rb

@@ -0,0 +1,14 @@
+require 'active_support/version'
+require 'active_record/version'
+
+module Delayed
+  module Heartbeat
+    module Compatibility
+
+      def self.mass_assignment_security_enabled?
+        ::ActiveRecord::VERSION::MAJOR < 4 || defined?(::ActiveRecord::MassAssignmentSecurity)
+      end
+
+    end
+  end
+end

data/lib/delayed/heartbeat/configuration.rb

@@ -0,0 +1,29 @@
+module Delayed
+  module Heartbeat
+    class Configuration
+      attr_accessor :enabled, :worker_label, :worker_version,
+                    :heartbeat_timeout_seconds, :heartbeat_interval_seconds,
+                    :worker_termination_enabled, :on_worker_termination
+      alias_method :enabled?, :enabled
+      alias_method :worker_termination_enabled?, :worker_termination_enabled
+
+      def initialize(options = {})
+        options.each do |key, value|
+          send("#{key}=", value)
+        end
+
+        if enabled.nil?
+          self.enabled = defined?(Rails) ? Rails.env.production? : true
+        end
+
+        if worker_termination_enabled.nil?
+          self.worker_termination_enabled = defined?(Rails) ? Rails.env.production? : true
+        end
+
+        self.heartbeat_timeout_seconds ||= 180
+        self.heartbeat_interval_seconds ||= 60
+        self.on_worker_termination ||= Proc.new { }
+      end
+    end
+  end
+end

data/lib/delayed/heartbeat/delete_worker_results.rb

@@ -0,0 +1,43 @@
+module Delayed
+  module Heartbeat
+    class DeleteWorkerResults
+      attr_reader :workers, :unlocked_jobs
+
+      def initialize(workers, unlocked_jobs)
+        @workers = workers
+        @unlocked_jobs = unlocked_jobs
+      end
+
+      def empty?
+        workers.empty? && unlocked_jobs.empty?
+      end
+
+      def to_s
+        io = StringIO.new
+        workers.each do |worker|
+          io.puts("Deleted worker #{worker_description(worker)}")
+        end
+
+        unlocked_jobs.each do |unlocked_job|
+          worker = worker_map[unlocked_job.locked_by]
+          worker_string = worker ? worker_description(worker) : unlocked_job.locked_by
+          io.puts("Unlocked orphaned job #{unlocked_job.id} from worker #{worker_string}")
+        end
+
+        io.string
+      end
+
+      private
+
+      def worker_map
+        @worker_map ||= workers.each_with_object(Hash.new) do |worker, worker_map|
+          worker_map[worker.name] = worker
+        end
+      end
+
+      def worker_description(worker)
+        "#{worker.label}(#{worker.name})"
+      end
+    end
+  end
+end

data/lib/delayed/heartbeat/plugin.rb

@@ -0,0 +1,22 @@
+require 'set'
+
+module Delayed
+  module Heartbeat
+    class Plugin < Delayed::Plugin
+
+      callbacks do |lifecycle|
+        lifecycle.before(:execute) do |worker|
+          if Delayed::Heartbeat.configuration.enabled?
+            @heartbeat = Delayed::Heartbeat::WorkerHeartbeat.new(worker.name)
+          end
+        end
+
+        lifecycle.after(:execute) do |worker|
+          @heartbeat.stop if @heartbeat
+        end
+      end
+
+    end
+  end
+end
+

data/lib/delayed/heartbeat/railtie.rb

@@ -0,0 +1,12 @@
+require 'delayed_job'
+require 'rails'
+
+module Delayed
+  class Railtie < Rails::Railtie
+
+    rake_tasks do
+      load 'delayed/heartbeat/tasks.rb'
+    end
+
+  end
+end

data/lib/delayed/heartbeat/tasks.rb

@@ -0,0 +1,24 @@
+namespace :delayed do
+  namespace :heartbeat do
+    desc 'Cleans up workers that have not heartbeated recently.'
+    task delete_timed_out_workers: :environment do
+      results = Delayed::Heartbeat.delete_timed_out_workers
+      print_results(results)
+    end
+
+    desc 'Cleans up workers running a different version.'
+    task delete_workers_with_different_version: :environment do
+      results = Delayed::Heartbeat.delete_workers_with_different_version
+      print_results(results)
+    end
+
+    def print_results(results)
+      puts "Deleted #{results.workers.size} and unlocked #{results.unlocked_jobs.size} orphaned jobs"
+      puts results.to_s if verbose? && results.present?
+    end
+
+    def verbose?
+      ENV['VERBOSE'].to_s.downcase == 'true'
+    end
+  end
+end

data/lib/delayed/heartbeat/version.rb

@@ -0,0 +1,7 @@
+# encoding: UTF-8
+
+module Delayed
+  module Heartbeat
+    VERSION = '0.1.0'.freeze
+  end
+end

data/lib/delayed/heartbeat/worker.rb

@@ -0,0 +1,69 @@
+require 'delayed/heartbeat/compatibility'
+
+module Delayed
+  module Heartbeat
+    class Worker < ActiveRecord::Base
+      self.table_name = 'delayed_workers'
+
+      if Delayed::Heartbeat::Compatibility.mass_assignment_security_enabled?
+        attr_accessible :name, :version, :last_heartbeat_at, :host_name, :label
+      end
+
+      before_create do |model|
+        model.last_heartbeat_at ||= Time.now.utc
+        model.host_name ||= Socket.gethostname
+        model.label ||= Delayed::Heartbeat.configuration.worker_label || name
+        model.version ||= Delayed::Heartbeat.configuration.worker_version
+      end
+
+      def jobs
+        Delayed::Job.where(locked_by: name, failed_at: nil)
+      end
+
+      def job
+        jobs.first
+      end
+
+      # Returns the jobs that were unlocked
+      def unlock_jobs(mark_attempt_failed: true)
+        orphaned_jobs = jobs.to_a
+        return orphaned_jobs unless orphaned_jobs.present?
+
+        if mark_attempt_failed
+          mark_job_attempts_failed(orphaned_jobs)
+        else
+          Delayed::Job.where(id: orphaned_jobs.map(&:id)).update_all(locked_at: nil, locked_by: nil)
+        end
+
+        orphaned_jobs
+      end
+
+      def self.dead_workers(timeout_seconds)
+        where('last_heartbeat_at < ?', Time.now.utc - timeout_seconds)
+      end
+
+      def self.workers_with_different_version(current_version)
+        where('version != ?',  current_version)
+      end
+
+      def self.delete_workers(workers)
+        where(id: workers.map(&:id)).delete_all if workers.present?
+      end
+
+      private
+
+      def mark_job_attempts_failed(jobs)
+        dj_worker = Delayed::Worker.new
+        jobs.each do |job|
+          mark_job_attempt_failed(dj_worker, job)
+        end
+      end
+
+      def mark_job_attempt_failed(dj_worker, job)
+        # If there are more attempts this reschedules the job otherwise marks it as failed
+        # and runs appropriate callbacks
+        dj_worker.reschedule(job)
+      end
+    end
+  end
+end

data/lib/delayed/heartbeat/worker_heartbeat.rb

@@ -0,0 +1,91 @@
+module Delayed
+  module Heartbeat
+    class WorkerHeartbeat
+
+      def initialize(worker_name)
+        @worker_model = create_worker_model(worker_name)
+
+        # Use a self-pipe to safely shutdown the heartbeat thread
+        @stop_reader, @stop_writer = IO.pipe
+
+        yield(self) if block_given?
+
+        @heartbeat_thread = Thread.new { run_heartbeat_loop }
+        # Make this a high priority thread to try to ensure it runs
+        @heartbeat_thread.priority = 100
+      end
+
+      def alive?
+        @heartbeat_thread.alive?
+      end
+
+      def stop
+        # Use the self-pipe to tell the heartbeat thread to cleanly
+        # shutdown
+        if @stop_writer
+          @stop_writer.close
+          @stop_writer = nil
+        end
+      end
+
+      private
+
+      def create_worker_model(worker_name)
+        Delayed::Heartbeat::Worker.transaction do
+          Delayed::Heartbeat::Worker.where(name: worker_name).delete_all
+          Delayed::Heartbeat::Worker.create!(name: worker_name)
+        end
+      end
+
+      def run_heartbeat_loop
+        loop do
+          break if sleep_interruptibly(heartbeat_interval)
+          update_heartbeat
+          # Return the connection back to the pool since we won't be needing
+          # it again for a while.
+          Delayed::Backend::ActiveRecord::Job.clear_active_connections!
+        end
+      rescue => e
+        # We don't want the worker to continue running if the heartbeat can't be written.
+        # Don't use Thread.abort_on_exception because that will give Delayed::Job a chance
+        # to mark the job as failed which will unlock it even though the clock
+        # process has likely already unlocked it and another worker may have picked it up.
+        Delayed::Heartbeat.configuration.on_worker_termination.call(@worker_model, e)
+        exit(false)
+      ensure
+        @stop_reader.close
+        @worker_model.delete
+        # Note: The built-in Delayed::Plugins::ClearLocks will unlock the jobs for us
+        Delayed::Backend::ActiveRecord::Job.clear_active_connections!
+      end
+
+      def update_heartbeat
+        now = Time.now.utc
+        heartbeat_delta_seconds = now - @worker_model.last_heartbeat_at
+        if heartbeat_delta_seconds < heartbeat_timeout_seconds || self_termination_disabled?
+          @worker_model.update_column(:last_heartbeat_at, now)
+        else
+          raise Timeout::Error, "Worker heartbeat not updated for #{heartbeat_delta_seconds} seconds which " \
+              "exceeds timeout\n. Current job: #{ @worker_model.job.inspect}"
+        end
+      end
+
+      def self_termination_disabled?
+        !Delayed::Heartbeat.configuration.worker_termination_enabled?
+      end
+
+      def heartbeat_timeout_seconds
+        Delayed::Heartbeat.configuration.heartbeat_timeout_seconds
+      end
+
+      def heartbeat_interval
+        Delayed::Heartbeat.configuration.heartbeat_interval_seconds
+      end
+
+      # Returns a truthy if the sleep was interrupted
+      def sleep_interruptibly(secs)
+        IO.select([@stop_reader], nil, nil, secs)
+      end
+    end
+  end
+end

data/lib/delayed_job_heartbeat_plugin.rb

@@ -0,0 +1,7 @@
+require 'active_support'
+require 'active_record'
+require 'delayed_job'
+require 'delayed_job_active_record'
+require 'delayed/heartbeat'
+
+Delayed::Worker.plugins << Delayed::Heartbeat::Plugin

data/lib/generators/delayed_job_heartbeat_plugin/install_generator.rb

@@ -0,0 +1,20 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+require 'rails/generators/active_record'
+
+module DelayedJobHeartbeatPlugin
+  class InstallGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    self.source_paths << File.join(File.dirname(__FILE__), 'templates')
+
+    def create_migration_file
+      migration_template('migration.rb', 'db/migrate/create_delayed_workers.rb')
+    end
+
+    def self.next_migration_number(dirname)
+      ActiveRecord::Generators::Base.next_migration_number(dirname)
+    end
+
+  end
+end

data/lib/generators/delayed_job_heartbeat_plugin/templates/migration.rb

@@ -0,0 +1,13 @@
+class CreateDelayedWorkers < ActiveRecord::Migration
+
+  def change
+    create_table(:delayed_workers) do |t|
+      t.string :name
+      t.string :version
+      t.datetime :last_heartbeat_at
+      t.string :host_name
+      t.string :label
+    end
+  end
+
+end

data/spec/db/database.yml

@@ -0,0 +1,5 @@
+sqlite3:
+  adapter: sqlite3
+  pool: 5
+  timeout: 15000
+  database: tmp/sqlite3.db

data/spec/db/schema.rb

@@ -0,0 +1,23 @@
+ActiveRecord::Schema.define(version: 0) do
+
+  create_table(:delayed_jobs, force: true) do |t|
+    t.integer :priority, default: 0
+    t.integer :attempts, default: 0
+    t.text :handler
+    t.text :last_error
+    t.datetime :run_at
+    t.datetime :locked_at
+    t.datetime :failed_at
+    t.string :locked_by
+    t.string :queue
+    t.timestamps
+  end
+
+  create_table(:delayed_workers, force: true) do |t|
+    t.string :name
+    t.string :version
+    t.datetime :last_heartbeat_at
+    t.string :host_name
+    t.string :label
+  end
+end

data/spec/delayed/heartbeat/delete_worker_results_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe Delayed::Heartbeat::DeleteWorkerResults do
+  let(:worker) { create_worker(name: 'my-worker') }
+  let(:job) { create_job(locked_by: worker.name) }
+  let(:results) { create_results([worker], [job]) }
+
+  describe "#workers" do
+    specify { expect(results.workers).to eq [worker] }
+  end
+
+  describe "#unlocked_jobs" do
+    specify { expect(results.unlocked_jobs).to eq [job] }
+  end
+
+  describe "#to_s" do
+    it "includes workers" do
+      results = create_results([worker], [])
+      expect(results.to_s).to include(worker.name)
+    end
+
+    it "includes jobs for known workers" do
+      results = create_results([worker], [job])
+      expect(results.to_s).to include(job.id.to_s)
+    end
+
+    it "includes jobs locked by an unknown worker" do
+      job = create_job(locked_by: 'unknown-worker')
+      results = create_results([], [job])
+      expect(results.to_s).to include(job.id.to_s)
+      expect(results.to_s).to include(job.locked_by)
+    end
+  end
+
+  def create_worker(attributes = {})
+    Delayed::Heartbeat::Worker.create!(attributes)
+  end
+
+  def create_job(attributes = {})
+    attributes = attributes.reverse_merge(payload_object: TestJob.new)
+    Delayed::Job.create!(attributes)
+  end
+
+  def create_results(workers, unlocked_jobs)
+    Delayed::Heartbeat::DeleteWorkerResults.new(workers, unlocked_jobs)
+  end
+end

data/spec/delayed/heartbeat/worker_heartbeat_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+describe Delayed::Heartbeat::WorkerHeartbeat, cleaner_strategy: :truncation do
+  let(:worker_name) { 'Test Worker' }
+  let(:worker_model) { find_worker_model(worker_name) }
+  let(:heartbeat_config) do
+    Delayed::Heartbeat::Configuration.new(heartbeat_interval_seconds: 0.001)
+  end
+
+  before do
+    allow(Delayed::Heartbeat).to receive(:configuration).and_return(heartbeat_config)
+
+    Delayed::Job.create!(locked_by: worker_name, payload_object: TestJob.new)
+    @worker_heartbeat = start_heartbeat
+  end
+
+  after do
+    stop_heartbeat
+  end
+
+  it "creates a worker model" do
+    expect(worker_model).to be_present
+  end
+
+  it "updates the worker heartbeat" do
+    original_heartbeat = worker_model.last_heartbeat_at
+    Wait.for('worker heartbeat updated') do
+      worker_model.reload.last_heartbeat_at != original_heartbeat
+    end
+  end
+
+  context "when the heartbeat is stopped" do
+    let!(:job) { Delayed::Backend::ActiveRecord::Job.create!(locked_by: worker_model.name, locked_at: Time.now) }
+
+    before do
+      stop_heartbeat
+    end
+
+    it "destroys the worker model" do
+      expect(find_worker_model(worker_name)).not_to be_present
+    end
+  end
+
+  context "when the heartbeat times out" do
+    let(:heartbeat_config) do
+      # Create a configuration where heartbeat is updated less frequently than the timeout
+      Delayed::Heartbeat::Configuration.new(heartbeat_interval_seconds: 0.001,
+                                            heartbeat_timeout_seconds: 0.00000001,
+                                            worker_termination_enabled: true)
+    end
+
+    before do
+      wait_for_heartbeat_thread_stopped
+    end
+
+    it "aborts the process" do
+      expect(@worker_heartbeat).to have_received(:exit).with(false)
+    end
+  end
+
+  def start_heartbeat
+    Delayed::Heartbeat::WorkerHeartbeat.new(worker_name) do |heartbeat|
+      allow(heartbeat).to receive(:exit)
+    end
+  end
+
+  def stop_heartbeat
+    @worker_heartbeat.stop
+    wait_for_heartbeat_thread_stopped
+  end
+
+  def wait_for_heartbeat_thread_stopped
+    Wait.for('worker thread stopped') do
+      !@worker_heartbeat.alive?
+    end
+  end
+
+  def find_worker_model(worker_name)
+    Delayed::Heartbeat::Worker.where(name: worker_name).first
+  end
+end

data/spec/delayed/heartbeat_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+describe Delayed::Heartbeat do
+  let(:current_version) { 'current version' }
+  let!(:active_worker) { create_worker_model(name: 'active_worker', version: current_version) }
+
+  let!(:active_job) do
+    Delayed::Job.create!(locked_by: active_worker.name,
+                         locked_at: active_worker.last_heartbeat_at,
+                         payload_object: TestJob.new)
+  end
+
+  let!(:orphaned_job) do
+    Delayed::Job.create!(locked_by: dead_worker.name,
+                         locked_at: dead_worker.last_heartbeat_at,
+                         payload_object: TestJob.new)
+  end
+
+  shared_examples "it destroys a worker and unlocks its jobs" do |expected_orphaned_job_attempts: nil|
+    specify { expect(active_worker).not_to have_been_destroyed }
+    specify { expect(active_job.reload.locked_by).not_to be_nil }
+    specify { expect(active_job.reload.locked_at).not_to be_nil }
+    specify { expect(active_job.reload.attempts).to eq(0) }
+    specify { expect(active_job.reload.failed_at).to be_nil }
+
+    specify { expect(dead_worker).to have_been_destroyed }
+    specify { expect(orphaned_job.reload.locked_by).to be_nil }
+    specify { expect(orphaned_job.reload.locked_at).to be_nil }
+    specify { expect(orphaned_job.reload.attempts).to eq(expected_orphaned_job_attempts) }
+    specify { expect(orphaned_job.reload.failed_at).to be_nil }
+  end
+
+  describe ".delete_timed_out_workers" do
+    let!(:dead_worker) do
+      create_worker_model(name: 'dead_worker',
+                          last_heartbeat_at: Time.now - Delayed::Heartbeat.configuration.heartbeat_timeout_seconds - 1)
+    end
+
+    let(:max_attempts) { 5 }
+
+    let!(:failed_orphaned_job) do
+      Delayed::Job.create!(locked_by: dead_worker.name, locked_at: dead_worker.last_heartbeat_at,
+          payload_object: TestJobWithCallbacks.new) do |job|
+        job.attempts = max_attempts - 1
+      end
+    end
+
+    before do
+      TestJobWithCallbacks.clear
+      @old_attempts = Delayed::Worker.max_attempts
+      Delayed::Worker.max_attempts = max_attempts
+
+      Delayed::Heartbeat.delete_timed_out_workers
+    end
+
+    after do
+      Delayed::Worker.max_attempts = @old_attempts
+    end
+
+    specify { expect(failed_orphaned_job.reload.failed_at).to be_present }
+    specify { expect(TestJobWithCallbacks.called_callbacks).to eq [:failure] }
+
+    it_behaves_like "it destroys a worker and unlocks its jobs", expected_orphaned_job_attempts: 1
+  end
+
+  describe ".delete_workers_with_different_version" do
+    let!(:dead_worker) { create_worker_model(name: 'old_worker', version: 'old version') }
+
+    before do
+      Delayed::Heartbeat.delete_workers_with_different_version(current_version)
+    end
+
+    it_behaves_like "it destroys a worker and unlocks its jobs", expected_orphaned_job_attempts: 0
+  end
+
+  def create_worker_model(attributes)
+    Delayed::Heartbeat::Worker.create!(attributes)
+  end
+
+end
+
+

data/spec/spec_helper.rb

@@ -0,0 +1,56 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+
+require 'simplecov'
+require 'coveralls'
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+    SimpleCov::Formatter::HTMLFormatter,
+    Coveralls::SimpleCov::Formatter
+]
+SimpleCov.start do
+  add_filter 'spec'
+end
+
+require 'database_cleaner'
+require 'delayed_job_heartbeat_plugin'
+require 'yaml'
+require 'timecop'
+
+spec_dir = File.dirname(__FILE__)
+Dir["#{spec_dir}/support/**/*.rb"].sort.each { |f| require f }
+
+FileUtils.makedirs('log')
+FileUtils.makedirs('tmp')
+
+Delayed::Worker.read_ahead = 1
+Delayed::Worker.destroy_failed_jobs = false
+
+Delayed::Worker.logger = Logger.new('log/test.log')
+Delayed::Worker.logger.level = Logger::DEBUG
+ActiveRecord::Base.logger = Delayed::Worker.logger
+ActiveRecord::Migration.verbose = false
+
+db_adapter = ENV.fetch('ADAPTER', 'sqlite3')
+config = YAML.load(File.read('spec/db/database.yml'))
+ActiveRecord::Base.establish_connection(config[db_adapter])
+require 'db/schema'
+
+RSpec.configure do |config|
+  config.order = 'random'
+
+  config.before(:suite) do
+    DatabaseCleaner.clean_with(:truncation)
+  end
+
+  config.before(:each) do |example|
+    DatabaseCleaner.strategy = example.metadata.fetch(:cleaner_strategy, :transaction)
+  end
+
+  config.before(:each) do
+    DatabaseCleaner.start
+  end
+
+  config.after(:each) do
+    DatabaseCleaner.clean
+  end
+end

data/spec/support/destroyed_model.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+
+RSpec::Matchers.define :have_been_destroyed do
+  match do |actual|
+    !actual.class.where(id: actual.id).exists?
+  end
+
+  description do
+    "model should have been destroyed"
+  end
+
+  failure_message do |actual|
+    "expected #{actual.class}(id: #{actual.id}) to have been destroyed"
+  end
+end

data/spec/support/test_job.rb

@@ -0,0 +1,5 @@
+class TestJob
+  def perform
+
+  end
+end

data/spec/support/test_job_with_callbacks.rb

@@ -0,0 +1,16 @@
+class TestJobWithCallbacks
+  cattr_accessor :called_callbacks
+  self.called_callbacks = []
+
+  def self.clear
+    called_callbacks.clear
+  end
+
+  def failure(*)
+    self.class.called_callbacks << :failure
+  end
+
+  def perform
+    self.class.called_callbacks << :perform
+  end
+end

data/spec/support/wait.rb

@@ -0,0 +1,13 @@
+module Wait
+  def self.for(condition_name, max_wait_time: 5, polling_interval: 0.001)
+    wait_until = Time.now + max_wait_time.seconds
+    loop do
+      return if yield
+      if Time.now > wait_until
+        raise "Condition not met: #{condition_name}"
+      else
+        sleep(polling_interval)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,232 @@
+--- !ruby/object:Gem::Specification
+name: delayed_job_heartbeat_plugin
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Joel Turkel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+- !ruby/object:Gem::Dependency
+  name: delayed_job_active_record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.3'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- jturkel@salsify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/setup
+- delayed_job_heartbeat_plugin.gemspec
+- lib/delayed/heartbeat.rb
+- lib/delayed/heartbeat/compatibility.rb
+- lib/delayed/heartbeat/configuration.rb
+- lib/delayed/heartbeat/delete_worker_results.rb
+- lib/delayed/heartbeat/plugin.rb
+- lib/delayed/heartbeat/railtie.rb
+- lib/delayed/heartbeat/tasks.rb
+- lib/delayed/heartbeat/version.rb
+- lib/delayed/heartbeat/worker.rb
+- lib/delayed/heartbeat/worker_heartbeat.rb
+- lib/delayed_job_heartbeat_plugin.rb
+- lib/generators/delayed_job_heartbeat_plugin/install_generator.rb
+- lib/generators/delayed_job_heartbeat_plugin/templates/migration.rb
+- spec/db/database.yml
+- spec/db/schema.rb
+- spec/delayed/heartbeat/delete_worker_results_spec.rb
+- spec/delayed/heartbeat/worker_heartbeat_spec.rb
+- spec/delayed/heartbeat_spec.rb
+- spec/spec_helper.rb
+- spec/support/destroyed_model.rb
+- spec/support/test_job.rb
+- spec/support/test_job_with_callbacks.rb
+- spec/support/wait.rb
+homepage: https://github.com/salsify/delayed_job_heartbeat_plugin
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Delayed::Job plugin to unlock jobs from dead workers
+test_files:
+- spec/db/database.yml
+- spec/db/schema.rb
+- spec/delayed/heartbeat/delete_worker_results_spec.rb
+- spec/delayed/heartbeat/worker_heartbeat_spec.rb
+- spec/delayed/heartbeat_spec.rb
+- spec/spec_helper.rb
+- spec/support/destroyed_model.rb
+- spec/support/test_job.rb
+- spec/support/test_job_with_callbacks.rb
+- spec/support/wait.rb