workhorse 1.5.2 → 2.0.0.rc0

data/lib/workhorse/poller.rb

@@ -1,7 +1,6 @@
 module Workhorse
   # Database poller that discovers and locks jobs for execution.
   # Handles job querying, global locking, and job distribution to workers.
-  # Supports both MySQL and Oracle databases with database-specific optimizations.
   #
   # @example Basic usage (typically used internally)
   #   poller = Workhorse::Poller.new(worker, proc { true })
@@ -10,8 +9,13 @@ module Workhorse
     MIN_LOCK_TIMEOUT = 0.1 # In seconds
     MAX_LOCK_TIMEOUT = 1.0 # In seconds
 
-    ORACLE_LOCK_MODE   = 6           # X_MODE (exclusive)
-    ORACLE_LOCK_HANDLE = 478_564_848 # Randomly chosen number
+    # Most jobs one poll expires, see {#expire_due_jobs}.
+    MAX_EXPIRIES_PER_POLL = 100
+
+    # Length of one slice of the poller's sleep, in seconds. The poller sleeps
+    # in slices rather than for the whole polling interval so that it stays
+    # responsive to shutdown, to instant repolling and to notifications.
+    SLEEP_SLICE = 0.1
 
     # @return [Workhorse::Worker] The worker this poller serves
     attr_reader :worker
@@ -27,7 +31,6 @@ module Workhorse
       @worker = worker
       @running = false
       @table = Workhorse::DbJob.arel_table
-      @is_oracle = ActiveRecord::Base.connection.adapter_name == 'OracleEnhanced'
       @instant_repoll = Concurrent::AtomicBoolean.new(false)
       @global_lock_fails = 0
       @max_global_lock_fails_reached = false
@@ -51,6 +54,18 @@ module Workhorse
 
       Workhorse.debug_log("[Job worker #{worker.id}] Poller starting")
 
+      begin
+        Workhorse.notifier.start
+      rescue StandardError => e
+        worker.log "Starting the notifier failed, falling back to polling: #{e.class}: #{e.message}", :warn
+      end
+
+      # Only jobs announced from now on concern this worker; anything enqueued
+      # earlier is found by the poll that follows.
+      @last_notification = notifier_token
+
+      reconcile_schedules!
+
       clean_stuck_jobs! if Workhorse.clean_stuck_jobs
 
       @thread = Thread.new do
@@ -91,6 +106,13 @@ module Workhorse
       Workhorse.debug_log("[Job worker #{worker.id}] Poller shutting down")
       @running = false
       wait
+
+      begin
+        Workhorse.notifier.stop
+      rescue StandardError => e
+        worker.log "Stopping the notifier failed: #{e.class}: #{e.message}", :warn
+      end
+
       Workhorse.debug_log("[Job worker #{worker.id}] Poller shut down")
     end
 
@@ -112,6 +134,43 @@ module Workhorse
 
     private
 
+    # Brings the schedules table in line with the registry, see
+    # {Workhorse::Schedule.reconcile!}.
+    #
+    # A worker that cannot reconcile still works through the queue, so this
+    # reports rather than raises.
+    #
+    # @return [void]
+    # @private
+    def reconcile_schedules!
+      @schedules_available = Workhorse::Schedule.table_exists?
+
+      unless @schedules_available
+        if Workhorse::Schedules.any?
+          message = 'Schedules are declared but the workhorse_schedules table does not exist. ' \
+                    'Run the migration that creates it; no scheduled job will run until then.'
+          worker.log message, :error
+
+          begin
+            Workhorse.on_exception.call(StandardError.new(message))
+          rescue Exception => e
+            # Reported through the rescue below would feed the callback its
+            # own failure; escaping leaves the worker half-started.
+            Workhorse.debug_log("on_exception failed: #{e.class}: #{e.message}")
+          end
+        end
+
+        return
+      end
+
+      with_global_lock timeout: MAX_LOCK_TIMEOUT do
+        Workhorse::Schedule.reconcile!
+      end
+    rescue Exception => e
+      worker.log %(Could not reconcile schedules: #{e.message}), :error
+      Workhorse.on_exception.call(e)
+    end
+
     # Cleans up jobs stuck in locked or started states from dead processes.
     # Only cleans jobs from the current hostname.
     #
@@ -174,7 +233,9 @@ module Workhorse
       end
     end
 
-    # Sleeps for the configured polling interval with instant repoll support.
+    # Sleeps for the configured polling interval, returning early for an
+    # instant repoll, for shutdown, or as soon as a notification announces a
+    # newly enqueued job.
     #
     # @return [void]
     # @private
@@ -182,37 +243,91 @@ module Workhorse
       remaining = worker.polling_interval
 
       while running? && remaining > 0 && @instant_repoll.false?
-        Kernel.sleep 0.1
-        remaining -= 0.1
+        Kernel.sleep SLEEP_SLICE
+        remaining -= SLEEP_SLICE
+
+        next unless notified?
+
+        worker.log 'Job was announced, polling ahead of the interval', :debug
+        break
+      end
+
+      # Time left on the clock means the sleep was cut short, which #poll
+      # passes on as `count_failures: false`.
+      @poll_brought_forward = remaining > 0
+    end
+
+    # Returns whether a job has been announced since this poller last looked,
+    # and records what it saw.
+    #
+    # A worker with no idle thread deliberately leaves the token untouched, so
+    # that the notification is still pending once it has capacity again.
+    #
+    # @return [Boolean]
+    # @private
+    def notified?
+      return false unless worker.accepting_jobs?
+      return false if worker.idle.zero?
+
+      token = notifier_token
+
+      return false if token.nil? || token == @last_notification
+
+      @last_notification = token
+
+      return true
+    end
+
+    # Reads the notifier's token, returning nil if it cannot be read.
+    #
+    # A notifier is an accelerator, so a broken one must cost latency rather
+    # than take the worker down: anything raised here would reach the poller's
+    # own rescue, which shuts the worker down. As this runs on every sleep
+    # slice, the failure is reported once rather than many times a second.
+    #
+    # @return [Object, nil]
+    # @private
+    def notifier_token
+      return Workhorse.notifier.token
+    rescue StandardError => e
+      message = "Reading the notifier failed, falling back to polling: #{e.class}: #{e.message}"
+
+      if @notifier_failed
+        worker.log message, :debug
+      else
+        @notifier_failed = true
+        worker.log message, :warn
       end
+
+      return nil
     end
 
     # Executes a block with a global database lock.
-    # Supports both MySQL GET_LOCK and Oracle DBMS_LOCK.
     #
     # @param name [Symbol] Lock name identifier
     # @param timeout [Integer] Lock timeout in seconds
+    # @param count_failures [Boolean] Whether a failure to obtain the lock
+    #   counts towards {Workhorse.max_global_lock_fails}
     # @yield Block to execute while holding the lock
     # @return [void]
     # @private
-    def with_global_lock(name: :workhorse, timeout: 2, &_block)
+    def with_global_lock(name: :workhorse, timeout: 2, count_failures: true, &_block)
       begin # rubocop:disable Style/RedundantBegin
-        if @is_oracle
-          result = Workhorse::DbJob.connection.select_all(
-            "SELECT DBMS_LOCK.REQUEST(#{ORACLE_LOCK_HANDLE}, #{ORACLE_LOCK_MODE}, #{timeout}) FROM DUAL"
-          ).first.values.last
-
-          success = result == 0
-        else
-          result = Workhorse::DbJob.connection.select_all(
-            "SELECT GET_LOCK(CONCAT(DATABASE(), '_#{name}'), #{timeout})"
-          ).first.values.last
-          success = result == 1
-        end
+        result = Workhorse::DbJob.connection.select_all(
+          "SELECT GET_LOCK(CONCAT(DATABASE(), '_#{name}'), #{timeout})"
+        ).first.values.last
+        success = result == 1
 
         if success
           @global_lock_fails = 0
           @max_global_lock_fails_reached = false
+        elsif !count_failures
+          # Losing the race for the lock is the expected outcome when several
+          # workers were woken by the same announcement, and says nothing about
+          # a crashed worker. Counting it would let the alarm below fire within
+          # seconds rather than after the polling intervals it is calibrated
+          # for.
+          worker.log 'Could not obtain global lock for a poll that was brought forward, skipping it.', :debug
         else
           @global_lock_fails += 1
 
@@ -247,11 +362,7 @@ module Workhorse
         yield
       ensure
         if success
-          if @is_oracle
-            Workhorse::DbJob.connection.execute("SELECT DBMS_LOCK.RELEASE(#{ORACLE_LOCK_HANDLE}) FROM DUAL")
-          else
-            Workhorse::DbJob.connection.execute("SELECT RELEASE_LOCK(CONCAT(DATABASE(), '_#{name}'))")
-          end
+          Workhorse::DbJob.connection.execute("SELECT RELEASE_LOCK(CONCAT(DATABASE(), '_#{name}'))")
         end
       end
     end
@@ -265,10 +376,20 @@ module Workhorse
 
       @instant_repoll.make_false
 
+      # A poll the sleep cut short is not the scheduled one the lock-failure
+      # alarm is calibrated against, see #with_global_lock.
+      brought_forward = @poll_brought_forward
+      @poll_brought_forward = false
+
+      expired = []
+
       timeout = worker.polling_interval.clamp(MIN_LOCK_TIMEOUT, MAX_LOCK_TIMEOUT)
-      with_global_lock timeout: timeout do
+      with_global_lock timeout: timeout, count_failures: !brought_forward do
         job_ids = []
 
+        materialize_schedules if Workhorse::Schedules.any? && @schedules_available
+        expired = expire_due_jobs
+
         Workhorse.tx_callback.call do
           # As we are the only thread posting into the worker pool, it is safe to
           # get the number of idle threads without mutex synchronization. The
@@ -303,6 +424,11 @@ module Workhorse
         job_ids.each { |job_id| worker.perform(job_id) } if running? && worker.accepting_jobs?
       end
 
+      # Deliberately outside the global lock: the callback is the application's
+      # and may do something slow, such as sending mail, which would otherwise
+      # block every other worker's poll.
+      notify_expired(expired)
+
       # Record that this worker successfully polled. Done at the very end so it
       # only advances when the poll actually completed (a poll that raises never
       # reaches here). Skipped on the early return above when the worker is no
@@ -310,6 +436,117 @@ module Workhorse
       worker.heartbeat!
     end
 
+    # Materialises the occurrences that have come due into jobs.
+    #
+    # A failing schedule must not take the worker down with it, nor stop the
+    # other schedules, so each is handled on its own.
+    #
+    # @return [void]
+    # @private
+    def materialize_schedules
+      Workhorse::Schedule.due.to_a.each do |schedule|
+        next if schedule.definition.nil?
+
+        occurrences, next_at = schedule.pending_occurrences
+
+        # The claim advances the schedule past these occurrences, so
+        # committing it before the jobs exist would lose them for good if
+        # enqueuing then failed - the schedule would move on and
+        # DetectLateSchedulesJob would see nothing wrong.
+        Workhorse.tx_callback.call do
+          next unless schedule.claim!(next_at)
+
+          occurrences.each do |occurrence|
+            db_job = schedule.enqueue!(occurrence)
+            worker.log "Materialized schedule #{schedule.key.inspect} for #{occurrence} as job #{db_job.id}", :debug
+          end
+        end
+
+        @failed_schedules&.delete(schedule.key)
+      rescue Exception => e
+        report_schedule_failure(schedule, e)
+      end
+
+      return
+    rescue Exception => e
+      # The query itself failed, so no individual schedule can be blamed. This
+      # must not reach the poller's own rescue, which shuts the worker down.
+      worker.log %(Could not query due schedules: #{e.message}), :error
+      Workhorse.on_exception.call(e)
+    end
+
+    # Reports a schedule that could not be materialized.
+    #
+    # Reported once per schedule per worker: a schedule that can never enqueue
+    # - a renamed job class, params its constructor rejects - fails on every
+    # poll, and one typo must not turn into a notification every polling
+    # interval for as long as the worker runs.
+    #
+    # @param schedule [Workhorse::Schedule]
+    # @param exception [Exception]
+    # @return [void]
+    # @private
+    def report_schedule_failure(schedule, exception)
+      message = %(Could not materialize schedule #{schedule.key.inspect}: #{exception.message})
+      @failed_schedules ||= Set.new
+
+      if @failed_schedules.include?(schedule.key)
+        worker.log message, :debug
+        return
+      end
+
+      @failed_schedules << schedule.key
+      worker.log message, :error
+      Workhorse.on_exception.call(exception)
+
+      return
+    end
+
+    # Marks jobs that passed their deadline before any worker got to them.
+    #
+    # @return [Array<Workhorse::DbJob>] The jobs that were expired
+    # @private
+    def expire_due_jobs
+      return [] unless expiry_supported?
+
+      expired = []
+
+      Workhorse.tx_callback.call do
+        rel = Workhorse::DbJob.waiting.where(Workhorse::DbJob.arel_table[:expires_at].lteq(Time.now))
+
+        # Bounded, as this runs while the poller holds the global lock: a
+        # backlog of jobs that all expired at once - workers down over a
+        # weekend, or a bulk enqueue - would otherwise turn one poll into
+        # thousands of updates that block every other worker. The rest is
+        # expired by the following polls, most overdue first.
+        rel.order(:expires_at).limit(MAX_EXPIRIES_PER_POLL).each do |db_job|
+          db_job.mark_expired!
+          expired << db_job
+          worker.log "Job #{db_job.id} passed its deadline of #{db_job.expires_at} and was not run", :warn
+        end
+      end
+
+      return expired
+    end
+
+    # Calls {Workhorse.on_job_expired} for each expired job, keeping a failing
+    # callback away from the poller's own error handling, which would shut the
+    # worker down.
+    #
+    # @param expired [Array<Workhorse::DbJob>]
+    # @return [void]
+    # @private
+    def notify_expired(expired)
+      expired.each do |db_job|
+        Workhorse.on_job_expired.call(db_job)
+      rescue Exception => e
+        worker.log %(on_job_expired failed for job #{db_job.id}: #{e.message}), :error
+        Workhorse.on_exception.call(e)
+      end
+
+      return
+    end
+
     # Returns an array of {Workhorse::DbJob}s that can be started.
     # Uses complex SQL with UNIONs to respect queue ordering and limits.
     #
@@ -335,7 +572,7 @@ module Workhorse
         # any presumptions on the order.
         record_number = queue.nil? ? limit : 1
 
-        union_parts << agnostic_limit(select, record_number)
+        union_parts << select.take(record_number)
       end
 
       return [] if union_parts.empty?
@@ -346,9 +583,6 @@ module Workhorse
       # contained within.
       # Additionally, each of the subselects and the final union select is given
       # an alias to comply with MySQL requirements.
-      # These aliases are added directly instead of using Arel `as`, because it
-      # uses the keyword 'AS' in SQL generated for Oracle, which is invalid for
-      # table aliases.
       union_query_sql = '('
       union_query_sql += "SELECT * FROM (#{union_parts.shift.to_sql}) union_0"
       union_parts.each_with_index do |part, idx|
@@ -366,7 +600,7 @@ module Workhorse
       select = order(select)
 
       # Limit number of records
-      select = agnostic_limit(select, limit)
+      select = select.take(limit)
 
       return Workhorse::DbJob.find_by_sql(select.to_sql).to_a
     end
@@ -375,12 +609,32 @@ module Workhorse
     #
     # @return [Arel::SelectManager] the select manager
     def valid_select_id
+      now = Time.now
+
       select = table.project(table[:id])
       select = select.where(table[:state].eq(:waiting))
-      select = select.where(table[:perform_at].lteq(Time.now).or(table[:perform_at].eq(nil)))
+      select = select.where(table[:perform_at].lteq(now).or(table[:perform_at].eq(nil)))
+
+      # The deadline is enforced here and not only by #expire_due_jobs, which
+      # expires at most MAX_EXPIRIES_PER_POLL jobs per poll: a larger backlog
+      # would otherwise leave the surplus selectable and performed in the very
+      # same poll, past the deadline the caller set.
+      if expiry_supported?
+        select = select.where(table[:expires_at].gt(now).or(table[:expires_at].eq(nil)))
+      end
+
       return select
     end
 
+    # Returns whether this installation has the columns the expiry feature
+    # needs. They are absent until the migration adding them has been run.
+    #
+    # @return [Boolean]
+    # @private
+    def expiry_supported?
+      return Workhorse::DbJob.column_names.include?('expires_at')
+    end
+
     # Returns a fresh Arel select manager containing the id of all waiting jobs,
     # ordered with {#order}.
     #
@@ -398,17 +652,6 @@ module Workhorse
       select.order(Arel.sql('priority').asc).order(Arel.sql('created_at').asc)
     end
 
-    # Limits the number of records
-    #
-    # @param select [Arel::SelectManager] the select manager on which to apply
-    #   the limit
-    # @param number [Integer] the maximum number of records to return
-    # @return [Arel::SelectManager] the resultant select manager
-    def agnostic_limit(select, number)
-      return select.where(Arel.sql('ROWNUM').lteq(number)) if @is_oracle
-      return select.take(number)
-    end
-
     # Returns an Array of queue names for which a job may be posted
     #
     # This is done in multiple steps. First, all queues with jobs that are in
@@ -453,10 +696,9 @@ module Workhorse
       queues = select.project(:queue)
 
       # Note that `select_values` is used here on purpose: `execute` does not
-      # return a result set on every adapter (the Oracle enhanced adapter, for
-      # instance, returns `true` for queries), while `select_values` is
+      # return a result set on every adapter, while `select_values` is
       # implemented in terms of `exec_query` and thus behaves the same on the
-      # mysql2, trilogy and Oracle enhanced adapters.
+      # mysql2 and trilogy adapters.
       return Workhorse::DbJob.connection.select_values(queues.distinct.to_sql)
     end
   end

data/lib/workhorse/pool.rb

@@ -72,17 +72,23 @@ module Workhorse
       @size - @active_threads.value
     end
 
-    # Waits until the pool is shut down. This will wait forever unless you
-    # eventually call {#shutdown} (either before calling `wait` or after it in
-    # another thread).
+    # Waits until the pool is shut down. Without a timeout this waits forever
+    # unless you eventually call {#shutdown} (either before calling `wait` or
+    # after it in another thread).
     #
-    # @return [void]
-    def wait
+    # @param timeout [Numeric, nil] Seconds to wait at most, or nil for no
+    #   limit.
+    # @return [Boolean] Whether the pool is shut down
+    def wait(timeout: nil)
+      deadline = timeout ? Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout : nil
+
       # Here we use a loop-sleep combination instead of using
       # ThreadPoolExecutor's `wait_for_termination`. See issue #21 for more
       # information.
       loop do
-        break if @executor.shutdown?
+        return true if @executor.shutdown?
+        return false if deadline && Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
         sleep 0.1
       end
     end