workhorse 1.5.2 → 2.0.0.rc1

data/lib/workhorse.rb

@@ -1,6 +1,7 @@
 require 'active_record'
 require 'active_support/all'
 require 'concurrent'
+require 'fugit'
 require 'socket'
 require 'uri'
 
@@ -84,6 +85,43 @@ module Workhorse
   mattr_accessor :silence_watcher
   self.silence_watcher = false
 
+  # Callback invoked when a job passed its `expires_at` before any worker got
+  # to it. The job is in state `expired` and will not be performed.
+  #
+  # Workhorse logs an expiry at `warn` regardless of this callback. Set the
+  # callback to report it wherever failures belong, e.g.
+  #
+  # ```ruby
+  # config.on_job_expired = proc do |db_job|
+  #   ExceptionNotifier.notify_exception(
+  #     StandardError.new("Job #{db_job.id} (#{db_job.description}) expired")
+  #   )
+  # end
+  # ```
+  #
+  # Called outside the global lock, but on the poller thread: a slow callback
+  # delays this worker's next poll. Anything it raises is passed to
+  # {.on_exception} and does not affect the worker.
+  #
+  # @return [Proc] The expiry callback
+  mattr_accessor :on_job_expired
+  self.on_job_expired = proc do |db_job|
+    # Do something with this job, i.e. notify about it
+  end
+
+  # Callback invoked when a job started later than its `max_lateness` allows.
+  # In contrast to {.on_job_expired} the job does run; it was simply late.
+  #
+  # Called on the worker thread that performs the job, just before it starts,
+  # and receives the job and its lateness in seconds. Anything it raises is
+  # passed to {.on_exception}.
+  #
+  # @return [Proc] The lateness callback
+  mattr_accessor :on_job_late
+  self.on_job_late = proc do |db_job, lateness|
+    # Do something with this job, i.e. notify about it
+  end
+
   # Controls whether jobs are performed within database transactions.
   # Individual job classes can override this with skip_tx?.
   #
@@ -105,6 +143,55 @@ module Workhorse
   mattr_accessor :max_worker_memory_mb
   self.max_worker_memory_mb = 0
 
+  # Channel a {Workhorse::Notifiers::Redis} notifier publishes on. Defaults to
+  # {Workhorse::Notifiers::Redis::DEFAULT_CHANNEL} when nil.
+  #
+  # @return [String, nil] Channel name
+  mattr_accessor :notification_channel
+  self.notification_channel = nil
+
+  # Redis client used by a {Workhorse::Notifiers::Redis} notifier.
+  #
+  # @return [Object, nil] A Redis client
+  mattr_accessor :notification_redis
+  self.notification_redis = nil
+
+  # Path of the file a {Workhorse::Notifiers::FileSystem} notifier touches.
+  # Every process that enqueues jobs and every worker must agree on it.
+  #
+  # Defaults to `tmp/pids/workhorse.wake` below the Rails root, or below the
+  # working directory outside of Rails.
+  #
+  # @return [String] Path of the notification file
+  def self.notification_path
+    return @notification_path if @notification_path
+
+    root = defined?(Rails) ? Rails.root : Dir.pwd
+
+    return ::File.join(root.to_s, 'tmp', 'pids', 'workhorse.wake')
+  end
+
+  # Sets the path of the notification file, see {.notification_path}.
+  #
+  # @param value [String, nil] Path, or nil to restore the default
+  # @return [void]
+  def self.notification_path=(value)
+    @notification_path = value
+  end
+
+  # Seconds the daemon's `stop` waits for a worker to shut down gracefully
+  # before killing it.
+  #
+  # A worker normally goes away as soon as the job it is performing finishes,
+  # so this only takes effect for one that is wedged or performing something
+  # very long. Without a limit `stop` waits forever, and so does whatever is
+  # waiting on it. Set to nil to wait indefinitely, which was the behaviour
+  # before this setting existed.
+  #
+  # @return [Numeric, nil] Timeout in seconds, or nil for no limit
+  mattr_accessor :shutdown_timeout
+  self.shutdown_timeout = 300
+
   # Path to a debug log file for diagnosing log rotation and signal handling issues.
   # When set, Workhorse writes timestamped debug entries to this file at key points
   # (worker startup, HUP signal handling, restart-logging command flow).
@@ -130,6 +217,48 @@ module Workhorse
   rescue Exception # rubocop:disable Lint/SuppressedException
   end
 
+  # Notifier that lets a worker start an enqueued job without waiting for its
+  # next poll. Polling stays the floor, so a notification that is not
+  # delivered costs latency and nothing else.
+  #
+  # Set to `:none` (the default), `:file`, `:redis`, or an instance of a
+  # {Workhorse::Notifiers::Base} subclass.
+  #
+  # @return [Workhorse::Notifiers::Base] The configured notifier
+  def self.notifier
+    return @notifier ||= Workhorse::Notifiers::None.new
+  end
+
+  # Sets the notifier, see {.notifier}.
+  #
+  # @param value [Symbol, Workhorse::Notifiers::Base] The notifier to use
+  # @return [void]
+  def self.notifier=(value)
+    @notifier = case value
+                when :none, nil then Workhorse::Notifiers::None.new
+                when :file      then Workhorse::Notifiers::FileSystem.new
+                when :redis     then Workhorse::Notifiers::Redis.new
+                when Symbol     then fail(ArgumentError, "Unknown notifier #{value.inspect}, use :none, :file or :redis.")
+                else value
+                end
+  end
+
+  # Registers scheduled jobs, see {Workhorse::Schedules::Dsl#schedule}.
+  #
+  # ```ruby
+  # Workhorse.schedules do
+  #   schedule 'cleanup_jobs',
+  #            job:  'Workhorse::Jobs::CleanupSucceededJobs',
+  #            cron: '10 0 * * *'
+  # end
+  # ```
+  #
+  # @yield Block evaluated against {Workhorse::Schedules::Dsl}
+  # @return [void]
+  def self.schedules(&block)
+    Workhorse::Schedules.define(&block)
+  end
+
   # Configuration method for setting up Workhorse options.
   #
   # @yield [self] Configuration block
@@ -142,7 +271,13 @@ module Workhorse
   end
 end
 
+require 'workhorse/notifiers/base'
+require 'workhorse/notifiers/none'
+require 'workhorse/notifiers/file_system'
+require 'workhorse/notifiers/redis'
 require 'workhorse/db_job'
+require 'workhorse/schedules'
+require 'workhorse/schedule'
 require 'workhorse/performer'
 require 'workhorse/poller'
 require 'workhorse/pool'
@@ -151,6 +286,7 @@ require 'workhorse/jobs/run_rails_op'
 require 'workhorse/jobs/run_active_job'
 require 'workhorse/jobs/cleanup_succeeded_jobs'
 require 'workhorse/jobs/detect_stale_jobs_job'
+require 'workhorse/jobs/detect_late_schedules_job'
 
 # Daemon functionality is not available on java platforms
 if RUBY_PLATFORM != 'java'

data/test/lib/db_schema.rb

@@ -1,10 +1,21 @@
 ActiveRecord::Schema.define do
   self.verbose = false
 
+  # Prefix lengths for the indexes on string columns. MySQL needs them, as the
+  # default `utf8mb4` charset puts a full `varchar(255)` past the maximum key
+  # length; Oracle indexes the whole column and rejects the option.
+  state_length = DB_ORACLE ? {} : { length: { state: 191 } }
+  queue_length = DB_ORACLE ? {} : { length: 191 }
+  key_length   = DB_ORACLE ? {} : { length: 191 }
+
   create_table :jobs, force: true do |t|
     t.string :state, null: false, default: 'waiting'
     t.string :queue, null: true
-    t.text :handler, null: false, limit: 4_294_967_295
+
+    # Binary rather than text, matching the generated migration: the handler is
+    # a `Marshal.dump`, and a text column is character data - on Oracle a CLOB,
+    # whose character set conversion would corrupt it.
+    t.binary :handler, null: false, limit: 4_294_967_295
 
     t.string :locked_by
     t.datetime :locked_at
@@ -17,13 +28,35 @@ ActiveRecord::Schema.define do
 
     t.integer :priority, null: false
     t.datetime :perform_at, null: true
+    t.datetime :expires_at, null: true
+    t.integer :max_lateness, null: true
 
     t.string :description, null: true
 
     t.timestamps null: false
   end
 
-  add_index :jobs, :queue, length: 191
-  add_index :jobs, :state, length: 191
+  # The index names are given explicitly because the ones Rails would derive
+  # exceed the 30 characters Oracle allows before 12.2.
+  add_index :jobs, :queue, **queue_length
+  add_index :jobs, %i[state perform_at], name: 'idx_jobs_state_perform_at', **state_length
+  add_index :jobs, %i[state priority created_at], name: 'idx_jobs_state_prio_created', **state_length
+  add_index :jobs, %i[state expires_at], name: 'idx_jobs_state_expires_at', **state_length
   add_index :jobs, :perform_at
+
+  create_table :workhorse_schedules, force: true do |t|
+    t.string :key, null: false
+    t.string :cron, null: false
+    t.string :timezone, null: true
+    t.boolean :enabled, null: false, default: true
+    t.datetime :next_at, null: false
+    t.datetime :last_enqueued_at, null: true
+    t.datetime :last_occurrence, null: true
+    t.integer :last_job_id, null: true
+
+    t.timestamps null: false
+  end
+
+  add_index :workhorse_schedules, :key, unique: true, name: 'idx_wh_schedules_key', **key_length
+  add_index :workhorse_schedules, %i[enabled next_at], name: 'idx_wh_schedules_due'
 end

data/test/lib/jobs.rb

@@ -67,3 +67,32 @@ class DummyRailsOpsOp
     results << @params
   end
 end
+
+class ScheduledActiveJob < ActiveJob::Base
+  def perform(*); end
+end
+
+# Minimal stand-in for RailsOps, which workhorse only soft-depends on. Its
+# presence is what selects the operation branch of Workhorse::Enqueuer.
+module RailsOps
+  class Operation
+    def self.results
+      return @results ||= Concurrent::Array.new
+    end
+
+    # Workhorse::Jobs::RunRailsOp calls the class method, as RailsOps does.
+    def self.run!(params = {})
+      return new(params).run!
+    end
+
+    def initialize(params = {})
+      @params = params
+    end
+
+    def run!
+      self.class.results << @params
+    end
+  end
+end
+
+class DummyScheduledOp < RailsOps::Operation; end

data/test/lib/test_helper.rb

@@ -6,8 +6,8 @@ require 'benchmark'
 require 'concurrent'
 require 'jobs'
 
-# The adapter to run the test suite against. Both MySQL / MariaDB adapters are
-# supported, as workhorse has to work with either of them.
+# The adapter to run the test suite against. Workhorse has to work with each of
+# them, so each is covered by CI.
 DB_ADAPTER = ENV.fetch('DB_ADAPTER', 'mysql2')
 
 case DB_ADAPTER
@@ -15,10 +15,16 @@ when 'mysql2'
   require 'mysql2'
 when 'trilogy'
   require 'trilogy'
+when 'oracle_enhanced'
+  require 'active_record/connection_adapters/oracle_enhanced_adapter'
 else
-  fail "Unsupported DB_ADAPTER #{DB_ADAPTER.inspect}, use 'mysql2' or 'trilogy'."
+  fail "Unsupported DB_ADAPTER #{DB_ADAPTER.inspect}, use 'mysql2', 'trilogy' or 'oracle_enhanced'."
 end
 
+# Whether the suite is running against Oracle. Used where the schema or an
+# assertion cannot be written the same way for both families.
+DB_ORACLE = DB_ADAPTER == 'oracle_enhanced'
+
 class MockRailsEnv < String
   def production?
     self == 'production'
@@ -44,6 +50,38 @@ class Rails
 end
 
 class WorkhorseTest < ActiveSupport::TestCase
+  # Seconds a test may run before the backtraces of all its threads are
+  # printed. A hung test otherwise only shows up as a CI attempt killed at its
+  # time limit, with nothing saying where it hung.
+  HANG_REPORT_AFTER = 120
+
+  # Callbacks rather than methods, so that a test class defining its own setup
+  # or teardown does not skip them.
+  setup do
+    test = "#{self.class}##{name}"
+
+    @hang_watchdog = Thread.new do
+      sleep HANG_REPORT_AFTER
+      report_threads "#{test} is still running after #{HANG_REPORT_AFTER}s"
+    end
+  end
+
+  # Prints the backtraces of all threads of this process but the calling one.
+  def report_threads(title)
+    warn "#{title} (PID #{Process.pid}). Its threads:"
+
+    Thread.list.each do |thread|
+      next if thread == Thread.current
+
+      warn "--- #{thread.inspect}\n#{(thread.backtrace || ['(no backtrace)']).join("\n")}"
+    end
+  end
+
+  teardown do
+    @hang_watchdog&.kill
+    restore_termination_traps
+  end
+
   def setup
     remove_pids!
     clear_locks_and_db_threads!
@@ -68,22 +106,56 @@ class WorkhorseTest < ActiveSupport::TestCase
     object.singleton_class.send(:remove_method, method)
   end
 
-  def clear_locks_and_db_threads!
-    Workhorse::DbJob.connection.execute('SELECT RELEASE_ALL_LOCKS()')
+  # Holds workhorse's global lock on a connection of its own, so that the code
+  # under test sees it as taken by another worker.
+  def with_global_lock_held
+    connection = ActiveRecord::Base.connection_pool.checkout
+    connection.select_value(acquire_global_lock_sql)
+    yield
+  ensure
+    if connection
+      connection.select_value(release_global_lock_sql)
+      ActiveRecord::Base.connection_pool.checkin(connection)
+    end
+  end
 
-    # Use `select_values` rather than `execute`, as the latter does not return a
-    # result set on every adapter.
-    pids = Workhorse::DbJob.connection.select_values(<<~SQL.squish)
-      SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE ID != CONNECTION_ID()
+  # Mirrors what Workhorse::Poller#with_global_lock emits, so that the lock the
+  # code under test tries to take is the same one.
+  def acquire_global_lock_sql
+    return <<~SQL.strip if DB_ORACLE
+      SELECT DBMS_LOCK.REQUEST(#{Workhorse::Poller::ORACLE_LOCK_HANDLE}, #{Workhorse::Poller::ORACLE_LOCK_MODE}, 1)
+      FROM DUAL
     SQL
 
-    begin
-      pids.each { |pid| Workhorse::DbJob.connection.execute("KILL QUERY #{pid}") }
-    rescue ActiveRecord::StatementInvalid
-      # Ignore
-    end
+    return "SELECT GET_LOCK(CONCAT(DATABASE(), '_workhorse'), 1)"
+  end
+
+  def release_global_lock_sql
+    return "SELECT DBMS_LOCK.RELEASE(#{Workhorse::Poller::ORACLE_LOCK_HANDLE}) FROM DUAL" if DB_ORACLE
+
+    return "SELECT RELEASE_LOCK(CONCAT(DATABASE(), '_workhorse'))"
+  end
 
-    Workhorse::DbJob.connection.execute('SELECT RELEASE_ALL_LOCKS()')
+  def clear_locks_and_db_threads!
+    # Releases the locks held by *this* connection, which is all a fresh run
+    # needs. It used to kill the queries of every other connection as well, to
+    # clear one left behind by a crashed run - but KILL QUERY only aborts a
+    # query and does not release a named lock, which is held by the connection
+    # rather than the statement, so it never achieved that. What it did
+    # achieve was killing queries belonging to the current run, surfacing as
+    # spurious "Lost connection to server during query" failures. Should a
+    # crashed run ever leave a lock behind, kill its *connection*, which does
+    # release it.
+    if DB_ORACLE
+      # Oracle has no "release everything" call, but workhorse takes exactly
+      # one handle. RELEASE reports a status rather than raising when the lock
+      # is not held, so this needs no guard.
+      Workhorse::DbJob.connection.execute(
+        "SELECT DBMS_LOCK.RELEASE(#{Workhorse::Poller::ORACLE_LOCK_HANDLE}) FROM DUAL"
+      )
+    else
+      Workhorse::DbJob.connection.execute('SELECT RELEASE_ALL_LOCKS()')
+    end
   end
 
   def remove_pids!
@@ -149,8 +221,14 @@ class WorkhorseTest < ActiveSupport::TestCase
     w.shutdown
   end
 
+  # Without auto_terminate unless asked for, as work and work_until have it:
+  # a worker that has it traps TERM and INT for the whole process and never
+  # gives them back, so the test process ignored the TERM meant to stop it for
+  # the rest of the run - and a timed-out CI attempt kept running alongside
+  # the next. Every test hands them back afterwards regardless, see the
+  # teardown above.
   def with_worker(options = {})
-    w = Workhorse::Worker.new(**options)
+    w = Workhorse::Worker.new(auto_terminate: false, **options)
     w.start
     begin
       yield(w)
@@ -176,6 +254,12 @@ class WorkhorseTest < ActiveSupport::TestCase
     daemon.stop(quiet: true)
   end
 
+  # Hands TERM and INT back to the handlers the process started with, which a
+  # worker started with auto_terminate replaced.
+  def restore_termination_traps
+    ORIGINAL_TERMINATION_TRAPS.each { |signal, handler| Signal.trap(signal, handler) }
+  end
+
   def with_retries(max = 50, interval: 0.1, &_block)
     runs = 0
 
@@ -205,15 +289,24 @@ class WorkhorseTest < ActiveSupport::TestCase
   end
 end
 
+# On Oracle the "database" is a service name rather than a schema, and the
+# schema is the user the suite connects as.
 ActiveRecord::Base.establish_connection(
   adapter:  DB_ADAPTER,
-  database: ENV.fetch('DB_NAME', nil) || 'workhorse',
-  username: ENV.fetch('DB_USERNAME', nil) || 'root',
-  password: ENV.fetch('DB_PASSWORD', nil) || '',
+  database: ENV.fetch('DB_NAME', nil) || (DB_ORACLE ? 'FREEPDB1' : 'workhorse'),
+  username: ENV.fetch('DB_USERNAME', nil) || (DB_ORACLE ? 'workhorse' : 'root'),
+  password: ENV.fetch('DB_PASSWORD', nil) || (DB_ORACLE ? 'workhorse' : ''),
   host:     ENV.fetch('DB_HOST', nil) || '127.0.0.1',
-  port:     ENV.fetch('DB_PORT', nil) || 3306,
+  port:     ENV.fetch('DB_PORT', nil) || (DB_ORACLE ? 1521 : 3306),
   pool:     10
 )
 
 require 'db_schema'
 require 'workhorse'
+
+# The handlers the process started with, see #restore_termination_traps.
+ORIGINAL_TERMINATION_TRAPS = Workhorse::Worker::SHUTDOWN_SIGNALS.to_h do |signal|
+  handler = Signal.trap(signal, 'DEFAULT')
+  Signal.trap(signal, handler)
+  [signal, handler]
+end

data/test/workhorse/daemon_test.rb

@@ -1,6 +1,39 @@
 require 'test_helper'
 
 class Workhorse::DaemonTest < WorkhorseTest
+  # A worker that ignores TERM used to leave `stop` - and whatever is waiting
+  # on it, a deployment usually - looping forever.
+  def test_stop_kills_a_worker_that_ignores_term
+    previous = Workhorse.shutdown_timeout
+    Workhorse.shutdown_timeout = 2
+
+    daemon = Workhorse::Daemon.new(pidfile: 'tmp/pids/stubborn%s.pid') do |d|
+      d.worker 'Stubborn' do
+        Signal.trap('TERM') { nil }
+        Signal.trap('INT') { nil }
+        # A bare sleep returns as soon as a handler runs, so it has to be
+        # re-entered to actually ignore the signal.
+        loop { sleep 1 }
+      end
+    end
+
+    daemon.start(quiet: true)
+    pid = daemon.workers.first.pid
+
+    with_retries(50, interval: 0.1) { assert process?(pid) }
+
+    capture_stderr do
+      Timeout.timeout(30) { daemon.stop(quiet: true) }
+    end
+
+    wait_for_process_exit(pid)
+
+    refute process?(pid)
+  ensure
+    Workhorse.shutdown_timeout = previous
+    FileUtils.rm_f Dir['tmp/pids/stubborn*.pid']
+  end
+
   def setup
     remove_pids!
   end

data/test/workhorse/db_job_test.rb

@@ -10,9 +10,7 @@ class Workhorse::DbJobTest < WorkhorseTest
 
   def test_reset_failed
     job = Workhorse.enqueue FailingTestJob.new
-    work 0.5
-    job.reload
-    assert_equal 'failed', job.state
+    work_until(pool_size: 5, polling_interval: 0.2) { assert_equal 'failed', job.reload.state }
 
     job.reset!
 
@@ -26,7 +24,7 @@ class Workhorse::DbJobTest < WorkhorseTest
     err = assert_raises do
       job.reset!
     end
-    assert_equal %(Job #{job.id} is not in state [:succeeded, :failed] but in state "locked".), err.message
+    assert_equal %(Job #{job.id} is not in state [:succeeded, :failed, :expired] but in state "locked".), err.message
   end
 
   def test_forced_reset