workhorse 1.5.2 → 2.0.0.rc1

data/lib/workhorse/poller.rb

@@ -10,6 +10,14 @@ module Workhorse
     MIN_LOCK_TIMEOUT = 0.1 # In seconds
     MAX_LOCK_TIMEOUT = 1.0 # In seconds
 
+    # Most jobs one poll expires, see {#expire_due_jobs}.
+    MAX_EXPIRIES_PER_POLL = 100
+
+    # Length of one slice of the poller's sleep, in seconds. The poller sleeps
+    # in slices rather than for the whole polling interval so that it stays
+    # responsive to shutdown, to instant repolling and to notifications.
+    SLEEP_SLICE = 0.1
+
     ORACLE_LOCK_MODE   = 6           # X_MODE (exclusive)
     ORACLE_LOCK_HANDLE = 478_564_848 # Randomly chosen number
 
@@ -51,6 +59,18 @@ module Workhorse
 
       Workhorse.debug_log("[Job worker #{worker.id}] Poller starting")
 
+      begin
+        Workhorse.notifier.start
+      rescue StandardError => e
+        worker.log "Starting the notifier failed, falling back to polling: #{e.class}: #{e.message}", :warn
+      end
+
+      # Only jobs announced from now on concern this worker; anything enqueued
+      # earlier is found by the poll that follows.
+      @last_notification = notifier_token
+
+      reconcile_schedules!
+
       clean_stuck_jobs! if Workhorse.clean_stuck_jobs
 
       @thread = Thread.new do
@@ -91,6 +111,13 @@ module Workhorse
       Workhorse.debug_log("[Job worker #{worker.id}] Poller shutting down")
       @running = false
       wait
+
+      begin
+        Workhorse.notifier.stop
+      rescue StandardError => e
+        worker.log "Stopping the notifier failed: #{e.class}: #{e.message}", :warn
+      end
+
       Workhorse.debug_log("[Job worker #{worker.id}] Poller shut down")
     end
 
@@ -112,6 +139,43 @@ module Workhorse
 
     private
 
+    # Brings the schedules table in line with the registry, see
+    # {Workhorse::Schedule.reconcile!}.
+    #
+    # A worker that cannot reconcile still works through the queue, so this
+    # reports rather than raises.
+    #
+    # @return [void]
+    # @private
+    def reconcile_schedules!
+      @schedules_available = Workhorse::Schedule.table_exists?
+
+      unless @schedules_available
+        if Workhorse::Schedules.any?
+          message = 'Schedules are declared but the workhorse_schedules table does not exist. ' \
+                    'Run the migration that creates it; no scheduled job will run until then.'
+          worker.log message, :error
+
+          begin
+            Workhorse.on_exception.call(StandardError.new(message))
+          rescue Exception => e
+            # Reported through the rescue below would feed the callback its
+            # own failure; escaping leaves the worker half-started.
+            Workhorse.debug_log("on_exception failed: #{e.class}: #{e.message}")
+          end
+        end
+
+        return
+      end
+
+      with_global_lock timeout: MAX_LOCK_TIMEOUT do
+        Workhorse::Schedule.reconcile!
+      end
+    rescue Exception => e
+      worker.log %(Could not reconcile schedules: #{e.message}), :error
+      Workhorse.on_exception.call(e)
+    end
+
     # Cleans up jobs stuck in locked or started states from dead processes.
     # Only cleans jobs from the current hostname.
     #
@@ -174,7 +238,9 @@ module Workhorse
       end
     end
 
-    # Sleeps for the configured polling interval with instant repoll support.
+    # Sleeps for the configured polling interval, returning early for an
+    # instant repoll, for shutdown, or as soon as a notification announces a
+    # newly enqueued job.
     #
     # @return [void]
     # @private
@@ -182,9 +248,63 @@ module Workhorse
       remaining = worker.polling_interval
 
       while running? && remaining > 0 && @instant_repoll.false?
-        Kernel.sleep 0.1
-        remaining -= 0.1
+        Kernel.sleep SLEEP_SLICE
+        remaining -= SLEEP_SLICE
+
+        next unless notified?
+
+        worker.log 'Job was announced, polling ahead of the interval', :debug
+        break
       end
+
+      # Time left on the clock means the sleep was cut short, which #poll
+      # passes on as `count_failures: false`.
+      @poll_brought_forward = remaining > 0
+    end
+
+    # Returns whether a job has been announced since this poller last looked,
+    # and records what it saw.
+    #
+    # A worker with no idle thread deliberately leaves the token untouched, so
+    # that the notification is still pending once it has capacity again.
+    #
+    # @return [Boolean]
+    # @private
+    def notified?
+      return false unless worker.accepting_jobs?
+      return false if worker.idle.zero?
+
+      token = notifier_token
+
+      return false if token.nil? || token == @last_notification
+
+      @last_notification = token
+
+      return true
+    end
+
+    # Reads the notifier's token, returning nil if it cannot be read.
+    #
+    # A notifier is an accelerator, so a broken one must cost latency rather
+    # than take the worker down: anything raised here would reach the poller's
+    # own rescue, which shuts the worker down. As this runs on every sleep
+    # slice, the failure is reported once rather than many times a second.
+    #
+    # @return [Object, nil]
+    # @private
+    def notifier_token
+      return Workhorse.notifier.token
+    rescue StandardError => e
+      message = "Reading the notifier failed, falling back to polling: #{e.class}: #{e.message}"
+
+      if @notifier_failed
+        worker.log message, :debug
+      else
+        @notifier_failed = true
+        worker.log message, :warn
+      end
+
+      return nil
     end
 
     # Executes a block with a global database lock.
@@ -192,11 +312,19 @@ module Workhorse
     #
     # @param name [Symbol] Lock name identifier
     # @param timeout [Integer] Lock timeout in seconds
+    # @param count_failures [Boolean] Whether a failure to obtain the lock
+    #   counts towards {Workhorse.max_global_lock_fails}
     # @yield Block to execute while holding the lock
     # @return [void]
     # @private
-    def with_global_lock(name: :workhorse, timeout: 2, &_block)
-      begin # rubocop:disable Style/RedundantBegin
+    def with_global_lock(name: :workhorse, timeout: 2, count_failures: true, &_block)
+      # Whole seconds, rounded up: MySQL and Oracle both take the timeout as an
+      # integer and round a fraction of a second down to not waiting at all -
+      # only MariaDB honours one. A poll finding the lock taken would then give
+      # up at once, and count towards max_global_lock_fails for mere contention.
+      timeout = timeout.ceil
+
+      begin
         if @is_oracle
           result = Workhorse::DbJob.connection.select_all(
             "SELECT DBMS_LOCK.REQUEST(#{ORACLE_LOCK_HANDLE}, #{ORACLE_LOCK_MODE}, #{timeout}) FROM DUAL"
@@ -213,6 +341,13 @@ module Workhorse
         if success
           @global_lock_fails = 0
           @max_global_lock_fails_reached = false
+        elsif !count_failures
+          # Losing the race for the lock is the expected outcome when several
+          # workers were woken by the same announcement, and says nothing about
+          # a crashed worker. Counting it would let the alarm below fire within
+          # seconds rather than after the polling intervals it is calibrated
+          # for.
+          worker.log 'Could not obtain global lock for a poll that was brought forward, skipping it.', :debug
         else
           @global_lock_fails += 1
 
@@ -265,10 +400,20 @@ module Workhorse
 
       @instant_repoll.make_false
 
+      # A poll the sleep cut short is not the scheduled one the lock-failure
+      # alarm is calibrated against, see #with_global_lock.
+      brought_forward = @poll_brought_forward
+      @poll_brought_forward = false
+
+      expired = []
+
       timeout = worker.polling_interval.clamp(MIN_LOCK_TIMEOUT, MAX_LOCK_TIMEOUT)
-      with_global_lock timeout: timeout do
+      with_global_lock timeout: timeout, count_failures: !brought_forward do
         job_ids = []
 
+        materialize_schedules if Workhorse::Schedules.any? && @schedules_available
+        expired = expire_due_jobs
+
         Workhorse.tx_callback.call do
           # As we are the only thread posting into the worker pool, it is safe to
           # get the number of idle threads without mutex synchronization. The
@@ -303,6 +448,11 @@ module Workhorse
         job_ids.each { |job_id| worker.perform(job_id) } if running? && worker.accepting_jobs?
       end
 
+      # Deliberately outside the global lock: the callback is the application's
+      # and may do something slow, such as sending mail, which would otherwise
+      # block every other worker's poll.
+      notify_expired(expired)
+
       # Record that this worker successfully polled. Done at the very end so it
       # only advances when the poll actually completed (a poll that raises never
       # reaches here). Skipped on the early return above when the worker is no
@@ -310,6 +460,117 @@ module Workhorse
       worker.heartbeat!
     end
 
+    # Materialises the occurrences that have come due into jobs.
+    #
+    # A failing schedule must not take the worker down with it, nor stop the
+    # other schedules, so each is handled on its own.
+    #
+    # @return [void]
+    # @private
+    def materialize_schedules
+      Workhorse::Schedule.due.to_a.each do |schedule|
+        next if schedule.definition.nil?
+
+        occurrences, next_at = schedule.pending_occurrences
+
+        # The claim advances the schedule past these occurrences, so
+        # committing it before the jobs exist would lose them for good if
+        # enqueuing then failed - the schedule would move on and
+        # DetectLateSchedulesJob would see nothing wrong.
+        Workhorse.tx_callback.call do
+          next unless schedule.claim!(next_at)
+
+          occurrences.each do |occurrence|
+            db_job = schedule.enqueue!(occurrence)
+            worker.log "Materialized schedule #{schedule.key.inspect} for #{occurrence} as job #{db_job.id}", :debug
+          end
+        end
+
+        @failed_schedules&.delete(schedule.key)
+      rescue Exception => e
+        report_schedule_failure(schedule, e)
+      end
+
+      return
+    rescue Exception => e
+      # The query itself failed, so no individual schedule can be blamed. This
+      # must not reach the poller's own rescue, which shuts the worker down.
+      worker.log %(Could not query due schedules: #{e.message}), :error
+      Workhorse.on_exception.call(e)
+    end
+
+    # Reports a schedule that could not be materialized.
+    #
+    # Reported once per schedule per worker: a schedule that can never enqueue
+    # - a renamed job class, params its constructor rejects - fails on every
+    # poll, and one typo must not turn into a notification every polling
+    # interval for as long as the worker runs.
+    #
+    # @param schedule [Workhorse::Schedule]
+    # @param exception [Exception]
+    # @return [void]
+    # @private
+    def report_schedule_failure(schedule, exception)
+      message = %(Could not materialize schedule #{schedule.key.inspect}: #{exception.message})
+      @failed_schedules ||= Set.new
+
+      if @failed_schedules.include?(schedule.key)
+        worker.log message, :debug
+        return
+      end
+
+      @failed_schedules << schedule.key
+      worker.log message, :error
+      Workhorse.on_exception.call(exception)
+
+      return
+    end
+
+    # Marks jobs that passed their deadline before any worker got to them.
+    #
+    # @return [Array<Workhorse::DbJob>] The jobs that were expired
+    # @private
+    def expire_due_jobs
+      return [] unless expiry_supported?
+
+      expired = []
+
+      Workhorse.tx_callback.call do
+        rel = Workhorse::DbJob.waiting.where(Workhorse::DbJob.arel_table[:expires_at].lteq(Time.now))
+
+        # Bounded, as this runs while the poller holds the global lock: a
+        # backlog of jobs that all expired at once - workers down over a
+        # weekend, or a bulk enqueue - would otherwise turn one poll into
+        # thousands of updates that block every other worker. The rest is
+        # expired by the following polls, most overdue first.
+        rel.order(:expires_at).limit(MAX_EXPIRIES_PER_POLL).each do |db_job|
+          db_job.mark_expired!
+          expired << db_job
+          worker.log "Job #{db_job.id} passed its deadline of #{db_job.expires_at} and was not run", :warn
+        end
+      end
+
+      return expired
+    end
+
+    # Calls {Workhorse.on_job_expired} for each expired job, keeping a failing
+    # callback away from the poller's own error handling, which would shut the
+    # worker down.
+    #
+    # @param expired [Array<Workhorse::DbJob>]
+    # @return [void]
+    # @private
+    def notify_expired(expired)
+      expired.each do |db_job|
+        Workhorse.on_job_expired.call(db_job)
+      rescue Exception => e
+        worker.log %(on_job_expired failed for job #{db_job.id}: #{e.message}), :error
+        Workhorse.on_exception.call(e)
+      end
+
+      return
+    end
+
     # Returns an array of {Workhorse::DbJob}s that can be started.
     # Uses complex SQL with UNIONs to respect queue ordering and limits.
     #
@@ -335,7 +596,7 @@ module Workhorse
         # any presumptions on the order.
         record_number = queue.nil? ? limit : 1
 
-        union_parts << agnostic_limit(select, record_number)
+        union_parts << limited_sql(select, record_number)
       end
 
       return [] if union_parts.empty?
@@ -350,9 +611,9 @@ module Workhorse
       # uses the keyword 'AS' in SQL generated for Oracle, which is invalid for
       # table aliases.
       union_query_sql = '('
-      union_query_sql += "SELECT * FROM (#{union_parts.shift.to_sql}) union_0"
+      union_query_sql += "SELECT * FROM (#{union_parts.shift}) union_0"
       union_parts.each_with_index do |part, idx|
-        union_query_sql += " UNION SELECT * FROM (#{part.to_sql}) union_#{idx + 1}"
+        union_query_sql += " UNION SELECT * FROM (#{part}) union_#{idx + 1}"
       end
       union_query_sql += ') subselect'
 
@@ -365,22 +626,39 @@ module Workhorse
       select = table.project(Arel.star).where(table[:id].in(select.project(:id)))
       select = order(select)
 
-      # Limit number of records
-      select = agnostic_limit(select, limit)
-
-      return Workhorse::DbJob.find_by_sql(select.to_sql).to_a
+      return Workhorse::DbJob.find_by_sql(limited_sql(select, limit)).to_a
     end
 
     # Returns a fresh Arel select manager containing the id of all waiting jobs.
     #
     # @return [Arel::SelectManager] the select manager
     def valid_select_id
+      now = Time.now
+
       select = table.project(table[:id])
       select = select.where(table[:state].eq(:waiting))
-      select = select.where(table[:perform_at].lteq(Time.now).or(table[:perform_at].eq(nil)))
+      select = select.where(table[:perform_at].lteq(now).or(table[:perform_at].eq(nil)))
+
+      # The deadline is enforced here and not only by #expire_due_jobs, which
+      # expires at most MAX_EXPIRIES_PER_POLL jobs per poll: a larger backlog
+      # would otherwise leave the surplus selectable and performed in the very
+      # same poll, past the deadline the caller set.
+      if expiry_supported?
+        select = select.where(table[:expires_at].gt(now).or(table[:expires_at].eq(nil)))
+      end
+
       return select
     end
 
+    # Returns whether this installation has the columns the expiry feature
+    # needs. They are absent until the migration adding them has been run.
+    #
+    # @return [Boolean]
+    # @private
+    def expiry_supported?
+      return Workhorse::DbJob.column_names.include?('expires_at')
+    end
+
     # Returns a fresh Arel select manager containing the id of all waiting jobs,
     # ordered with {#order}.
     #
@@ -398,15 +676,19 @@ module Workhorse
       select.order(Arel.sql('priority').asc).order(Arel.sql('created_at').asc)
     end
 
-    # Limits the number of records
+    # Returns the SQL of a select, limited to the given number of records.
+    #
+    # On Oracle this is `FETCH FIRST`, which applies after `ORDER BY`. Filtering
+    # on `ROWNUM` instead - the only option before 12c - numbers the rows
+    # before they are sorted, so it returned an arbitrary subset and ignored
+    # the priority order altogether.
     #
-    # @param select [Arel::SelectManager] the select manager on which to apply
-    #   the limit
+    # @param select [Arel::SelectManager] the select manager to limit
     # @param number [Integer] the maximum number of records to return
-    # @return [Arel::SelectManager] the resultant select manager
-    def agnostic_limit(select, number)
-      return select.where(Arel.sql('ROWNUM').lteq(number)) if @is_oracle
-      return select.take(number)
+    # @return [String] the resultant SQL
+    def limited_sql(select, number)
+      return "#{select.to_sql} FETCH FIRST #{Integer(number)} ROWS ONLY" if @is_oracle
+      return select.take(number).to_sql
     end
 
     # Returns an Array of queue names for which a job may be posted

data/lib/workhorse/pool.rb

@@ -72,17 +72,23 @@ module Workhorse
       @size - @active_threads.value
     end
 
-    # Waits until the pool is shut down. This will wait forever unless you
-    # eventually call {#shutdown} (either before calling `wait` or after it in
-    # another thread).
+    # Waits until the pool is shut down. Without a timeout this waits forever
+    # unless you eventually call {#shutdown} (either before calling `wait` or
+    # after it in another thread).
     #
-    # @return [void]
-    def wait
+    # @param timeout [Numeric, nil] Seconds to wait at most, or nil for no
+    #   limit.
+    # @return [Boolean] Whether the pool is shut down
+    def wait(timeout: nil)
+      deadline = timeout ? Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout : nil
+
       # Here we use a loop-sleep combination instead of using
       # ThreadPoolExecutor's `wait_for_termination`. See issue #21 for more
       # information.
       loop do
-        break if @executor.shutdown?
+        return true if @executor.shutdown?
+        return false if deadline && Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
         sleep 0.1
       end
     end