workhorse 1.5.2 → 2.0.0.rc1

data/lib/workhorse/daemon.rb

@@ -3,6 +3,9 @@ module Workhorse
   # Provides functionality to start, stop, restart, and monitor worker processes
   # through a simple Ruby DSL.
   class Daemon
+    # Seconds spent waiting for a process to disappear after KILL, which it
+    # can only survive by being stuck in an uninterruptible syscall.
+    KILL_TIMEOUT = 10
     # Internal representation of a worker process.
     # Stores worker metadata and the block to execute.
     class Worker
@@ -308,9 +311,13 @@ module Workhorse
         $0 = process_name(worker)
         # Close inherited lockfile fd to prevent holding the flock after parent exits
         @lockfile&.close
-        # Reopen pipes to prevent #107576
+        # Reopen pipes to prevent #107576. Not the block form: the descriptor
+        # has to outlive this call, as stdout and stderr keep pointing at it
+        # for the lifetime of the worker.
+        # rubocop:disable Style/FileOpen
         $stdin.reopen File.open(File::NULL, 'r')
-        null_out = File.open(File::NULL, 'w') # rubocop:disable Style/FileOpen
+        null_out = File.open(File::NULL, 'w')
+        # rubocop:enable Style/FileOpen
         $stdout.reopen null_out
         $stderr.reopen null_out
 
@@ -318,7 +325,27 @@ module Workhorse
         # by slot (see Workhorse::Worker#heartbeat!). Same id as the pidfile.
         ENV['WORKHORSE_DAEMON_WORKER_ID'] = worker.id.to_s
 
-        worker.block.call
+        begin
+          worker.block.call
+
+          # Exit without running the at_exit handlers the parent registered:
+          # they belong to whatever started the daemon, not to this worker,
+          # and one that waits on threads the fork did not inherit hangs a
+          # worker that has finished - which then ignores TERM. ShellHandler
+          # skips them for the same reason.
+          exit!(0)
+        rescue Exception => e
+          # Reported here because exit! below denies it to every other route
+          # out: stderr is /dev/null by now, and an error reporter installed
+          # as an at_exit handler never runs. A worker that dies of an
+          # unhandled exception would otherwise do so silently.
+          begin
+            Workhorse.on_exception.call(e)
+          rescue Exception # rubocop:disable Lint/SuppressedException
+          end
+
+          exit!(1)
+        end
       end
       worker.pid = pid
       File.write(pid_file_for(worker), pid)
@@ -337,18 +364,41 @@ module Workhorse
       signals = kill ? %w[KILL] : %w[TERM INT]
 
       Workhorse.debug_log("Daemon: stopping PID #{pid} with signals #{signals.join(', ')}")
+
+      unless signal_until_gone(pid, signals, Workhorse.shutdown_timeout)
+        # A worker that does not go away leaves `stop` - and the deployment
+        # waiting on it - hanging indefinitely, so it is escalated the way an
+        # init system would.
+        warn "Worker #{pid} did not stop within #{Workhorse.shutdown_timeout}s, killing it"
+        Workhorse.debug_log("Daemon: PID #{pid} did not stop in time, sending KILL")
+        signal_until_gone(pid, %w[KILL], KILL_TIMEOUT)
+      end
+
+      Workhorse.debug_log("Daemon: PID #{pid} stopped")
+      FileUtils.rm_f(pid_file)
+    end
+
+    # Sends the given signals once a second until the process is gone.
+    #
+    # @param pid [Integer] The process to signal
+    # @param signals [Array<String>] Signals to send on each attempt
+    # @param timeout [Numeric, nil] Seconds to keep trying, or nil for no limit
+    # @return [Boolean] Whether the process is gone
+    # @private
+    def signal_until_gone(pid, signals, timeout)
+      deadline = timeout ? Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout : nil
+
       loop do
         begin
           signals.each { |signal| Process.kill(signal, pid) }
         rescue Errno::ESRCH
-          break
+          return true
         end
 
+        return false if deadline && Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
         sleep 1
       end
-
-      Workhorse.debug_log("Daemon: PID #{pid} stopped")
-      File.delete(pid_file)
     end
 
     # Sends HUP signal to a worker process.

data/lib/workhorse/db_job.rb

@@ -18,15 +18,22 @@ module Workhorse
     STATE_STARTED   = :started
     STATE_SUCCEEDED = :succeeded
     STATE_FAILED    = :failed
+    STATE_EXPIRED   = :expired
 
     EXP_LOCKED_BY = /^(.*?)\.(\d+?)\.([^.]+)$/
 
     if respond_to?(:attr_accessible)
-      attr_accessible :queue, :priority, :perform_at, :handler, :description
+      attr_accessible :queue, :priority, :perform_at, :handler, :description,
+                      :expires_at, :max_lateness
     end
 
     self.table_name = 'jobs'
 
+    # Notifying before the commit would wake a worker that cannot see the row
+    # yet, sending it back to sleep for a whole polling interval - the very
+    # delay the notification exists to avoid.
+    after_commit :notify_workers, on: :create
+
     # Returns jobs in waiting state.
     #
     # @return [ActiveRecord::Relation] Jobs waiting to be processed
@@ -62,13 +69,57 @@ module Workhorse
       where(state: STATE_FAILED)
     end
 
+    # Returns jobs that passed their deadline and were therefore not run.
+    #
+    # @return [ActiveRecord::Relation] Jobs that expired before being started
+    def self.expired
+      where(state: STATE_EXPIRED)
+    end
+
+    # Marks the job as expired, having passed its deadline before any worker
+    # got to it.
+    #
+    # @raise [RuntimeError] If the job is not in waiting state
+    # @private Only to be used by workhorse
+    def mark_expired!
+      assert_state! STATE_WAITING
+
+      self.state = STATE_EXPIRED
+      save!
+    end
+
+    # Returns how much later than intended this job started, or nil if it has
+    # not started or was not given an intended time.
+    #
+    # @return [Float, nil] Lateness in seconds
+    def lateness
+      return nil unless started_at && perform_at
+
+      return started_at - perform_at
+    end
+
     # Returns a relation with split locked_by field for easier querying.
     # Extracts host, PID, and random string components from locked_by.
     #
+    # `locked_by` is "<host>.<pid>.<random>", and the host may itself contain
+    # dots, so both dialects below work backwards from the last two.
+    #
     # @return [ActiveRecord::Relation] Relation with additional computed columns
     # @private
     def self.with_split_locked_by
-      select(<<~SQL)
+      return select(oracle? ? split_locked_by_sql_oracle : split_locked_by_sql_mysql)
+    end
+
+    # @return [Boolean] Whether the connection speaks Oracle
+    # @private
+    def self.oracle?
+      return connection.adapter_name == 'OracleEnhanced'
+    end
+
+    # @return [String]
+    # @private
+    def self.split_locked_by_sql_mysql
+      return <<~SQL
         #{table_name}.*,
 
         -- random string
@@ -91,12 +142,37 @@ module Workhorse
       SQL
     end
 
+    # Oracle has no `substring_index`. `instr` with a negative position counts
+    # occurrences from the right, which gives the two delimiters directly.
+    #
+    # @return [String]
+    # @private
+    def self.split_locked_by_sql_oracle
+      return <<~SQL
+        #{table_name}.*,
+
+        -- random string
+        substr(locked_by, instr(locked_by, '.', -1, 1) + 1) as locked_by_rnd,
+
+        -- pid
+        substr(
+          locked_by,
+          instr(locked_by, '.', -1, 2) + 1,
+          instr(locked_by, '.', -1, 1) - instr(locked_by, '.', -1, 2) - 1
+        ) as locked_by_pid,
+
+        -- get host
+        substr(locked_by, 1, instr(locked_by, '.', -1, 2) - 1) as locked_by_host
+      SQL
+    end
+
     # Resets job to state "waiting" and clears all meta fields
     # set by workhorse in course of processing this job.
     #
-    # This is only allowed if the job is in a final state ("succeeded" or
-    # "failed"), as only those jobs are safe to modify; workhorse will not touch
-    # these jobs. To reset a job without checking the state it is in, set
+    # This is only allowed if the job is in a final state ("succeeded",
+    # "failed" or "expired"), as only those jobs are safe to modify; workhorse
+    # will not touch these jobs. To reset a job without checking the state it
+    # is in, set
     # "force" to true. Prior to doing so, ensure that the job is not still being
     # processed by a worker. If possible, shut down all workers before
     # performing a forced reset.
@@ -111,7 +187,7 @@ module Workhorse
     # @raise [RuntimeError] If job is not in a final state and force is false
     def reset!(force = false)
       unless force
-        assert_state! STATE_SUCCEEDED, STATE_FAILED
+        assert_state! STATE_SUCCEEDED, STATE_FAILED, STATE_EXPIRED
       end
 
       self.state = STATE_WAITING
@@ -136,8 +212,6 @@ module Workhorse
       end
 
       if locked_at
-        # TODO: Remove this debug output
-        # puts "Already locked. Job: #{self.id} Worker: #{worker_id}"
         fail "Job #{id} is already locked by #{locked_by.inspect}."
       end
 
@@ -185,6 +259,22 @@ module Workhorse
       save!
     end
 
+    # Announces this job to waiting workers, see {Workhorse.notifier}.
+    #
+    # Jobs that are not due yet are not announced: a woken worker would find
+    # nothing to do. They are picked up by the regular poll once their
+    # `perform_at` has passed.
+    #
+    # @return [void]
+    # @private
+    def notify_workers
+      return if perform_at && perform_at > Time.now
+
+      Workhorse.notifier.notify(queue: queue)
+
+      return
+    end
+
     # Asserts that the job is in one of the specified states.
     #
     # @param states [Array<Symbol>] Valid states for the job

data/lib/workhorse/enqueuer.rb

@@ -10,15 +10,28 @@ module Workhorse
     # @param priority [Integer] Job priority (lower numbers = higher priority)
     # @param perform_at [Time] When to perform the job
     # @param description [String, nil] Optional job description
+    # @param expires_at [Time, nil] Deadline after which the job is no longer
+    #   worth running. It is then set to state `expired` instead of being
+    #   performed, and {Workhorse.on_job_expired} is called.
+    # @param max_lateness [Numeric, nil] Seconds the job may start after its
+    #   `perform_at` before {Workhorse.on_job_late} is called.
     # @return [Workhorse::DbJob] The created database job record
-    def enqueue(job, queue: nil, priority: 0, perform_at: Time.now, description: nil)
-      return DbJob.create!(
+    def enqueue(job, queue: nil, priority: 0, perform_at: Time.now, description: nil,
+                expires_at: nil, max_lateness: nil)
+      attributes = {
         queue:       queue,
         priority:    priority,
         perform_at:  perform_at,
         description: description,
         handler:     Marshal.dump(job)
-      )
+      }
+
+      # Only set when used, so that an installation that has not run the
+      # migration adding these columns keeps working as before.
+      attributes[:expires_at] = expires_at if expires_at
+      attributes[:max_lateness] = max_lateness if max_lateness
+
+      return DbJob.create!(**attributes)
     end
 
     # Enqueues an ActiveJob job instance.
@@ -27,16 +40,23 @@ module Workhorse
     # @param perform_at [Time] When to perform the job
     # @param queue [String, Symbol, nil] Optional queue override
     # @param description [String, nil] Optional job description
+    # @param priority [Integer, nil] Priority override. Defaults to the job's
+    #   own priority.
+    # @param expires_at [Time, nil] See {#enqueue}
+    # @param max_lateness [Numeric, nil] See {#enqueue}
     # @return [Workhorse::DbJob] The created database job record
-    def enqueue_active_job(job, perform_at: Time.now, queue: nil, description: nil)
+    def enqueue_active_job(job, perform_at: Time.now, queue: nil, description: nil,
+                           priority: nil, expires_at: nil, max_lateness: nil)
       wrapper_job = Jobs::RunActiveJob.new(job.serialize)
       queue ||= job.queue_name if job.queue_name.present?
       db_job = enqueue(
         wrapper_job,
-        queue:       queue,
-        priority:    job.priority || 0,
-        perform_at:  Time.at(perform_at),
-        description: description
+        queue:        queue,
+        priority:     priority || job.priority || 0,
+        perform_at:   Time.at(perform_at),
+        description:  description,
+        expires_at:   expires_at,
+        max_lateness: max_lateness
       )
       job.provider_job_id = db_job.id
       return db_job
@@ -65,5 +85,28 @@ module Workhorse
       job = Workhorse::Jobs::RunRailsOp.new(cls, op_args)
       enqueue job, **workhorse_args
     end
+
+    # Enqueues a job given by class name, working out how it wants to be
+    # instantiated: as a RailsOps operation, as an ActiveJob job, or as a
+    # plain object responding to `perform`.
+    #
+    # @param job_class [String, Class] The job class
+    # @param params [Hash] Operation params for RailsOps operations, keyword
+    #   arguments to the constructor otherwise
+    # @param options [Hash] Passed on to {#enqueue}
+    # @return [Workhorse::DbJob] The created database job record
+    def enqueue_job_class(job_class, params: {}, **options)
+      cls = job_class.is_a?(String) ? job_class.constantize : job_class
+
+      if defined?(RailsOps::Operation) && cls < RailsOps::Operation
+        return enqueue_op(cls, options, params)
+      elsif defined?(ActiveJob::Base) && cls < ActiveJob::Base
+        job = params.present? ? cls.new(**params) : cls.new
+        return enqueue_active_job(job, **options)
+      else
+        job = params.present? ? cls.new(**params) : cls.new
+        return enqueue(job, **options)
+      end
+    end
   end
 end

data/lib/workhorse/jobs/cleanup_succeeded_jobs.rb

@@ -6,26 +6,44 @@ module Workhorse::Jobs
   # @example Schedule cleanup job
   #   Workhorse.enqueue(CleanupSucceededJobs.new(max_age: 30))
   #
-  # @example Daily cleanup with cron
-  #   # Clean up jobs older than 14 days every day at 2 AM
-  #   Workhorse.enqueue(CleanupSucceededJobs.new, perform_at: 1.day.from_now.beginning_of_day + 2.hours)
+  # @example Daily cleanup on a schedule
+  #   Workhorse.schedules do
+  #     schedule 'cleanup_jobs', job: 'Workhorse::Jobs::CleanupSucceededJobs', cron: '0 2 * * *'
+  #   end
   class CleanupSucceededJobs
+    # States that are cleaned up unless told otherwise. Both are terminal and
+    # will never run again; `expired` is included so that a schedule using
+    # `expires_after` and regularly missing its window - the very case the
+    # option exists for - cannot grow the table without bound.
+    DEFAULT_STATES = [
+      Workhorse::DbJob::STATE_SUCCEEDED,
+      Workhorse::DbJob::STATE_EXPIRED
+    ].freeze
+
     # Instantiates a new job.
     #
     # @param max_age [Integer] The maximal age of jobs to retain, in days. Will
     #   be evaluated at perform time.
-    def initialize(max_age: 14)
+    # @param states [Array<Symbol>] The job states to clean up. Defaults to
+    #   {DEFAULT_STATES}. Pass `[Workhorse::DbJob::STATE_SUCCEEDED]` to keep
+    #   expired jobs around.
+    def initialize(max_age: 14, states: DEFAULT_STATES)
       @max_age = max_age
+      @states = states
     end
 
-    # Executes the cleanup by deleting old succeeded jobs.
+    # Executes the cleanup by deleting old jobs in the configured states.
     #
     # @return [void]
     def perform
       age_limit = seconds_ago(@max_age)
-      Workhorse::DbJob.where(
-        'STATE = ? AND UPDATED_AT <= ?', Workhorse::DbJob::STATE_SUCCEEDED, age_limit
-      ).delete_all
+
+      # An instance of this job enqueued before the upgrade unmarshals without
+      # @states, and `where(state: nil)` would quietly match nothing - leaving
+      # the table growing, which is what this job is for.
+      Workhorse::DbJob.where(state: @states || DEFAULT_STATES)
+                      .where('UPDATED_AT <= ?', age_limit)
+                      .delete_all
     end
 
     private

data/lib/workhorse/jobs/detect_late_schedules_job.rb

@@ -0,0 +1,59 @@
+module Workhorse::Jobs
+  # Job that detects schedules nothing is materialising.
+  #
+  # This is the one check that can catch silence. {Workhorse.on_job_expired}
+  # and {Workhorse.on_job_late} are precise but can only fire for a job that
+  # exists; if no worker is polling, or the global lock is stuck, occurrences
+  # stop being materialised altogether and no callback can report it. A
+  # schedule whose `next_at` lies well in the past is the evidence of that.
+  #
+  # Schedule this regularly, the same way as
+  # {Workhorse::Jobs::DetectStaleJobsJob}. Note that it is itself performed by
+  # a worker, so it reports a stall only while at least one worker still runs
+  # - use it alongside external monitoring, not instead of it.
+  #
+  # @example Schedule with the default threshold
+  #   Workhorse.schedules do
+  #     schedule 'detect_late_schedules',
+  #              job:  'Workhorse::Jobs::DetectLateSchedulesJob',
+  #              cron: '*/30 * * * *'
+  #   end
+  class DetectLateSchedulesJob
+    # Creates a new late schedule detection job.
+    #
+    # @param threshold [Integer] Number of seconds a schedule's next
+    #   occurrence may lie in the past before it is reported.
+    # @param keys [Array<String>, nil] If given, only check these schedules.
+    #   If `nil` (default), all of them are checked.
+    def initialize(threshold: 15 * 60, keys: nil)
+      @threshold = threshold
+      @keys      = keys
+    end
+
+    # Executes the detection.
+    #
+    # @return [void]
+    # @raise [RuntimeError] If schedules are found that are overdue
+    def perform
+      # Only schedules this process declares. A row whose declaration is gone
+      # is waiting to be cleaned up by reconciliation and nothing materializes
+      # it, so reporting it would be a false alarm.
+      keys = @keys || Workhorse::Schedules.definitions.keys
+
+      rel = Workhorse::Schedule.where(enabled: true, key: keys)
+      rel = rel.where(Workhorse::Schedule.arel_table[:next_at].lt(@threshold.seconds.ago))
+
+      overdue = rel.pluck(:key, :next_at)
+
+      return if overdue.empty?
+
+      descriptions = overdue.map do |key, next_at|
+        "#{key.inspect} (due #{next_at}, #{(Time.now - next_at).round}s ago)"
+      end
+
+      fail "The next occurrence of #{overdue.size} schedule(s) is more than #{@threshold}s in the past, " \
+           'which means they are not being materialized. Check that at least one worker is polling and ' \
+           "that the global lock is obtainable. Affected: #{descriptions.join(', ')}."
+    end
+  end
+end

data/lib/workhorse/notifiers/base.rb

@@ -0,0 +1,55 @@
+module Workhorse
+  module Notifiers
+    # Interface for notifiers, which let a worker start a job without waiting
+    # for its next poll.
+    #
+    # Polling remains the floor: a notification that is never delivered costs
+    # latency and nothing else, as the regular poll still happens after
+    # `polling_interval` and picks the job up. Implementations must therefore
+    # be best-effort and must never raise from {#notify}, which runs in the
+    # process that enqueues a job.
+    #
+    # Workers do not consume notifications. Instead, {#token} returns a value
+    # that changes whenever a notification has arrived, and each poller
+    # remembers the last one it saw. This keeps several workers in one process
+    # independent of one another, as none of them can take a notification away
+    # from the others.
+    #
+    # @abstract Subclass and override {#notify} and {#token}.
+    class Base
+      # Announces that a job has been enqueued. Called in the enqueueing
+      # process, after the surrounding transaction has committed.
+      #
+      # Must never raise: an enqueue may not fail because a notification could
+      # not be delivered.
+      #
+      # @param queue [String, Symbol, nil] Queue the job was enqueued into
+      # @return [void]
+      def notify(queue: nil); end
+
+      # Prepares this notifier for use by a worker in this process. Called
+      # when a worker starts, and may be called several times per process.
+      #
+      # @return [void]
+      def start; end
+
+      # Releases what {#start} acquired. Called when a worker shuts down.
+      #
+      # @return [void]
+      def stop; end
+
+      # Returns a value that differs from the previously returned one whenever
+      # at least one notification has arrived in between. Pollers compare it
+      # with `!=` and hold their own last-seen value, so no ordering is implied
+      # and nothing is consumed.
+      #
+      # Returns nil when nothing can be said, for instance because no
+      # notification has ever been sent.
+      #
+      # @return [Object, nil] Comparable token, or nil
+      def token
+        return nil
+      end
+    end
+  end
+end

data/lib/workhorse/notifiers/file_system.rb

@@ -0,0 +1,66 @@
+module Workhorse
+  module Notifiers
+    # Notifier that announces an enqueued job by touching a single file, which
+    # waiting workers stat once per sleep slice.
+    #
+    # This costs no database work at all and roughly a microsecond of local CPU
+    # per check, on a tick the poller performs anyway, so a worker can react
+    # within {Workhorse::Poller::SLEEP_SLICE} rather than within its polling
+    # interval.
+    #
+    # It requires that the enqueueing process and the workers share a
+    # filesystem, which in practice means the same host. Where they do not -
+    # workers on a machine of their own, a console on another host - the touch
+    # simply never reaches those workers and they fall back to polling. Use
+    # {Workhorse::Notifiers::Redis} if that case has to be fast as well.
+    #
+    # One file is used for every queue. A worker that serves a subset of queues
+    # therefore wakes for jobs it cannot run and polls once for nothing; this
+    # is cheaper than having such a worker scan a directory on every tick.
+    #
+    # Detection relies on the file's modification time, so the filesystem has
+    # to keep sub-second mtimes - every current local filesystem does. On one
+    # that does not, two touches within the same second may be seen as one, and
+    # the worker waits for its regular poll instead.
+    class FileSystem < Base
+      # @param path [String, nil] Path of the file to touch. Defaults to
+      #   {Workhorse.notification_path}.
+      def initialize(path: nil)
+        super()
+        @path = path
+      end
+
+      # Resolved when used rather than when constructed, so that
+      # {Workhorse.notification_path} can be set in any order relative to
+      # {Workhorse.notifier=}.
+      #
+      # @return [String] Path of the file that is touched
+      def path
+        return (@path || Workhorse.notification_path).to_s
+      end
+
+      # Touches the notification file, creating it and its directory if
+      # necessary.
+      #
+      # @param queue [String, Symbol, nil] Ignored, see the class description
+      # @return [void]
+      def notify(queue: nil) # rubocop:disable Lint/UnusedMethodArgument
+        FileUtils.mkdir_p(::File.dirname(path))
+        FileUtils.touch(path)
+      rescue StandardError => e
+        # Best-effort by contract: a job must still be enqueued when it cannot
+        # be announced. The worker finds it on its next poll.
+        Workhorse.debug_log("Notification failed: #{e.class}: #{e.message}")
+      end
+
+      # Returns the notification file's modification time.
+      #
+      # @return [Time, nil] mtime, or nil if the file does not exist yet
+      def token
+        return ::File.mtime(path)
+      rescue SystemCallError
+        return nil
+      end
+    end
+  end
+end

data/lib/workhorse/notifiers/none.rb

@@ -0,0 +1,8 @@
+module Workhorse
+  module Notifiers
+    # Notifier that does nothing, leaving workers to discover jobs by polling.
+    # This is the default.
+    class None < Base
+    end
+  end
+end