workhorse 1.5.2 → 2.0.0.rc0

data/lib/workhorse.rb

@@ -1,6 +1,7 @@
 require 'active_record'
 require 'active_support/all'
 require 'concurrent'
+require 'fugit'
 require 'socket'
 require 'uri'
 
@@ -84,6 +85,43 @@ module Workhorse
   mattr_accessor :silence_watcher
   self.silence_watcher = false
 
+  # Callback invoked when a job passed its `expires_at` before any worker got
+  # to it. The job is in state `expired` and will not be performed.
+  #
+  # Workhorse logs an expiry at `warn` regardless of this callback. Set the
+  # callback to report it wherever failures belong, e.g.
+  #
+  # ```ruby
+  # config.on_job_expired = proc do |db_job|
+  #   ExceptionNotifier.notify_exception(
+  #     StandardError.new("Job #{db_job.id} (#{db_job.description}) expired")
+  #   )
+  # end
+  # ```
+  #
+  # Called outside the global lock, but on the poller thread: a slow callback
+  # delays this worker's next poll. Anything it raises is passed to
+  # {.on_exception} and does not affect the worker.
+  #
+  # @return [Proc] The expiry callback
+  mattr_accessor :on_job_expired
+  self.on_job_expired = proc do |db_job|
+    # Do something with this job, i.e. notify about it
+  end
+
+  # Callback invoked when a job started later than its `max_lateness` allows.
+  # In contrast to {.on_job_expired} the job does run; it was simply late.
+  #
+  # Called on the worker thread that performs the job, just before it starts,
+  # and receives the job and its lateness in seconds. Anything it raises is
+  # passed to {.on_exception}.
+  #
+  # @return [Proc] The lateness callback
+  mattr_accessor :on_job_late
+  self.on_job_late = proc do |db_job, lateness|
+    # Do something with this job, i.e. notify about it
+  end
+
   # Controls whether jobs are performed within database transactions.
   # Individual job classes can override this with skip_tx?.
   #
@@ -105,6 +143,55 @@ module Workhorse
   mattr_accessor :max_worker_memory_mb
   self.max_worker_memory_mb = 0
 
+  # Channel a {Workhorse::Notifiers::Redis} notifier publishes on. Defaults to
+  # {Workhorse::Notifiers::Redis::DEFAULT_CHANNEL} when nil.
+  #
+  # @return [String, nil] Channel name
+  mattr_accessor :notification_channel
+  self.notification_channel = nil
+
+  # Redis client used by a {Workhorse::Notifiers::Redis} notifier.
+  #
+  # @return [Object, nil] A Redis client
+  mattr_accessor :notification_redis
+  self.notification_redis = nil
+
+  # Path of the file a {Workhorse::Notifiers::FileSystem} notifier touches.
+  # Every process that enqueues jobs and every worker must agree on it.
+  #
+  # Defaults to `tmp/pids/workhorse.wake` below the Rails root, or below the
+  # working directory outside of Rails.
+  #
+  # @return [String] Path of the notification file
+  def self.notification_path
+    return @notification_path if @notification_path
+
+    root = defined?(Rails) ? Rails.root : Dir.pwd
+
+    return ::File.join(root.to_s, 'tmp', 'pids', 'workhorse.wake')
+  end
+
+  # Sets the path of the notification file, see {.notification_path}.
+  #
+  # @param value [String, nil] Path, or nil to restore the default
+  # @return [void]
+  def self.notification_path=(value)
+    @notification_path = value
+  end
+
+  # Seconds the daemon's `stop` waits for a worker to shut down gracefully
+  # before killing it.
+  #
+  # A worker normally goes away as soon as the job it is performing finishes,
+  # so this only takes effect for one that is wedged or performing something
+  # very long. Without a limit `stop` waits forever, and so does whatever is
+  # waiting on it. Set to nil to wait indefinitely, which was the behaviour
+  # before this setting existed.
+  #
+  # @return [Numeric, nil] Timeout in seconds, or nil for no limit
+  mattr_accessor :shutdown_timeout
+  self.shutdown_timeout = 300
+
   # Path to a debug log file for diagnosing log rotation and signal handling issues.
   # When set, Workhorse writes timestamped debug entries to this file at key points
   # (worker startup, HUP signal handling, restart-logging command flow).
@@ -130,6 +217,48 @@ module Workhorse
   rescue Exception # rubocop:disable Lint/SuppressedException
   end
 
+  # Notifier that lets a worker start an enqueued job without waiting for its
+  # next poll. Polling stays the floor, so a notification that is not
+  # delivered costs latency and nothing else.
+  #
+  # Set to `:none` (the default), `:file`, `:redis`, or an instance of a
+  # {Workhorse::Notifiers::Base} subclass.
+  #
+  # @return [Workhorse::Notifiers::Base] The configured notifier
+  def self.notifier
+    return @notifier ||= Workhorse::Notifiers::None.new
+  end
+
+  # Sets the notifier, see {.notifier}.
+  #
+  # @param value [Symbol, Workhorse::Notifiers::Base] The notifier to use
+  # @return [void]
+  def self.notifier=(value)
+    @notifier = case value
+                when :none, nil then Workhorse::Notifiers::None.new
+                when :file      then Workhorse::Notifiers::FileSystem.new
+                when :redis     then Workhorse::Notifiers::Redis.new
+                when Symbol     then fail(ArgumentError, "Unknown notifier #{value.inspect}, use :none, :file or :redis.")
+                else value
+                end
+  end
+
+  # Registers scheduled jobs, see {Workhorse::Schedules::Dsl#schedule}.
+  #
+  # ```ruby
+  # Workhorse.schedules do
+  #   schedule 'cleanup_jobs',
+  #            job:  'Workhorse::Jobs::CleanupSucceededJobs',
+  #            cron: '10 0 * * *'
+  # end
+  # ```
+  #
+  # @yield Block evaluated against {Workhorse::Schedules::Dsl}
+  # @return [void]
+  def self.schedules(&block)
+    Workhorse::Schedules.define(&block)
+  end
+
   # Configuration method for setting up Workhorse options.
   #
   # @yield [self] Configuration block
@@ -142,7 +271,13 @@ module Workhorse
   end
 end
 
+require 'workhorse/notifiers/base'
+require 'workhorse/notifiers/none'
+require 'workhorse/notifiers/file_system'
+require 'workhorse/notifiers/redis'
 require 'workhorse/db_job'
+require 'workhorse/schedules'
+require 'workhorse/schedule'
 require 'workhorse/performer'
 require 'workhorse/poller'
 require 'workhorse/pool'
@@ -151,6 +286,7 @@ require 'workhorse/jobs/run_rails_op'
 require 'workhorse/jobs/run_active_job'
 require 'workhorse/jobs/cleanup_succeeded_jobs'
 require 'workhorse/jobs/detect_stale_jobs_job'
+require 'workhorse/jobs/detect_late_schedules_job'
 
 # Daemon functionality is not available on java platforms
 if RUBY_PLATFORM != 'java'

data/test/lib/db_schema.rb

@@ -17,6 +17,8 @@ ActiveRecord::Schema.define do
 
     t.integer :priority, null: false
     t.datetime :perform_at, null: true
+    t.datetime :expires_at, null: true
+    t.integer :max_lateness, null: true
 
     t.string :description, null: true
 
@@ -24,6 +26,24 @@ ActiveRecord::Schema.define do
   end
 
   add_index :jobs, :queue, length: 191
-  add_index :jobs, :state, length: 191
+  add_index :jobs, %i[state perform_at], length: { state: 191 }, name: 'idx_jobs_state_perform_at'
+  add_index :jobs, %i[state priority created_at], length: { state: 191 }, name: 'idx_jobs_state_prio_created'
+  add_index :jobs, %i[state expires_at], length: { state: 191 }, name: 'idx_jobs_state_expires_at'
   add_index :jobs, :perform_at
+
+  create_table :workhorse_schedules, force: true do |t|
+    t.string :key, null: false
+    t.string :cron, null: false
+    t.string :timezone, null: true
+    t.boolean :enabled, null: false, default: true
+    t.datetime :next_at, null: false
+    t.datetime :last_enqueued_at, null: true
+    t.datetime :last_occurrence, null: true
+    t.integer :last_job_id, null: true
+
+    t.timestamps null: false
+  end
+
+  add_index :workhorse_schedules, :key, unique: true, length: 191, name: 'idx_wh_schedules_key'
+  add_index :workhorse_schedules, %i[enabled next_at], name: 'idx_wh_schedules_due'
 end

data/test/lib/jobs.rb

@@ -67,3 +67,32 @@ class DummyRailsOpsOp
     results << @params
   end
 end
+
+class ScheduledActiveJob < ActiveJob::Base
+  def perform(*); end
+end
+
+# Minimal stand-in for RailsOps, which workhorse only soft-depends on. Its
+# presence is what selects the operation branch of Workhorse::Enqueuer.
+module RailsOps
+  class Operation
+    def self.results
+      return @results ||= Concurrent::Array.new
+    end
+
+    # Workhorse::Jobs::RunRailsOp calls the class method, as RailsOps does.
+    def self.run!(params = {})
+      return new(params).run!
+    end
+
+    def initialize(params = {})
+      @params = params
+    end
+
+    def run!
+      self.class.results << @params
+    end
+  end
+end
+
+class DummyScheduledOp < RailsOps::Operation; end

data/test/lib/test_helper.rb

@@ -69,20 +69,15 @@ class WorkhorseTest < ActiveSupport::TestCase
   end
 
   def clear_locks_and_db_threads!
-    Workhorse::DbJob.connection.execute('SELECT RELEASE_ALL_LOCKS()')
-
-    # Use `select_values` rather than `execute`, as the latter does not return a
-    # result set on every adapter.
-    pids = Workhorse::DbJob.connection.select_values(<<~SQL.squish)
-      SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE ID != CONNECTION_ID()
-    SQL
-
-    begin
-      pids.each { |pid| Workhorse::DbJob.connection.execute("KILL QUERY #{pid}") }
-    rescue ActiveRecord::StatementInvalid
-      # Ignore
-    end
-
+    # Releases the locks held by *this* connection, which is all a fresh run
+    # needs. It used to kill the queries of every other connection as well, to
+    # clear one left behind by a crashed run - but KILL QUERY only aborts a
+    # query and does not release a named lock, which is held by the connection
+    # rather than the statement, so it never achieved that. What it did
+    # achieve was killing queries belonging to the current run, surfacing as
+    # spurious "Lost connection to server during query" failures. Should a
+    # crashed run ever leave a lock behind, kill its *connection*, which does
+    # release it.
     Workhorse::DbJob.connection.execute('SELECT RELEASE_ALL_LOCKS()')
   end
 

data/test/workhorse/daemon_test.rb

@@ -1,6 +1,39 @@
 require 'test_helper'
 
 class Workhorse::DaemonTest < WorkhorseTest
+  # A worker that ignores TERM used to leave `stop` - and whatever is waiting
+  # on it, a deployment usually - looping forever.
+  def test_stop_kills_a_worker_that_ignores_term
+    previous = Workhorse.shutdown_timeout
+    Workhorse.shutdown_timeout = 2
+
+    daemon = Workhorse::Daemon.new(pidfile: 'tmp/pids/stubborn%s.pid') do |d|
+      d.worker 'Stubborn' do
+        Signal.trap('TERM') { nil }
+        Signal.trap('INT') { nil }
+        # A bare sleep returns as soon as a handler runs, so it has to be
+        # re-entered to actually ignore the signal.
+        loop { sleep 1 }
+      end
+    end
+
+    daemon.start(quiet: true)
+    pid = daemon.workers.first.pid
+
+    with_retries(50, interval: 0.1) { assert process?(pid) }
+
+    capture_stderr do
+      Timeout.timeout(30) { daemon.stop(quiet: true) }
+    end
+
+    wait_for_process_exit(pid)
+
+    refute process?(pid)
+  ensure
+    Workhorse.shutdown_timeout = previous
+    FileUtils.rm_f Dir['tmp/pids/stubborn*.pid']
+  end
+
   def setup
     remove_pids!
   end

data/test/workhorse/db_job_test.rb

@@ -26,7 +26,7 @@ class Workhorse::DbJobTest < WorkhorseTest
     err = assert_raises do
       job.reset!
     end
-    assert_equal %(Job #{job.id} is not in state [:succeeded, :failed] but in state "locked".), err.message
+    assert_equal %(Job #{job.id} is not in state [:succeeded, :failed, :expired] but in state "locked".), err.message
   end
 
   def test_forced_reset