workhorse 1.5.2 → 2.0.0.rc1

data/lib/workhorse/schedule.rb

@@ -0,0 +1,288 @@
+module Workhorse
+  # A schedule's persisted state: which occurrence is next.
+  #
+  # This is what makes a scheduled job survive a process that is not running
+  # when its time comes. An in-memory scheduler computes the next occurrence
+  # from *now*, so an occurrence that passes while it is down never happens and
+  # leaves no trace. Here the next occurrence is a row, so a worker that comes
+  # back at any later point still sees that it is due and applies the
+  # schedule's catch-up policy to it.
+  #
+  # Rows are reconciled from {Workhorse::Schedules} on worker startup and are
+  # materialised into jobs during {Workhorse::Poller#poll}.
+  class Schedule < ActiveRecord::Base
+    self.table_name = 'workhorse_schedules'
+
+    # How long a schedule that nothing declares any more is kept before its
+    # row is deleted.
+    OBSOLETE_GRACE = 24 * 60 * 60
+
+    # Most candidates {#advance} steps over before giving up. An hour can
+    # repeat only once, so this is only ever reached by a schedule with a very
+    # short period, whose repeats are bounded by the length of that hour.
+    MAX_REPEATED_STEPS = 3600
+
+    # Returns the schedules whose next occurrence has come.
+    #
+    # @param now [Time]
+    # @return [ActiveRecord::Relation]
+    def self.due(now = Time.now)
+      return where(enabled: true).where(arel_table[:next_at].lteq(now))
+    end
+
+    # Brings the table in line with the registry: inserts schedules that are
+    # new, recomputes the next occurrence of those whose cron expression or
+    # timezone changed, and deletes those that are gone.
+    #
+    # Safe to call from several workers at once, as each step is idempotent
+    # and a lost race leaves the table in the same state.
+    #
+    # @param now [Time]
+    # @return [void]
+    def self.reconcile!(now = Time.now)
+      # See Workhorse::Schedule#pending_occurrences for why fugit is never
+      # handed a local time.
+      now = now.getutc
+      definitions = Workhorse::Schedules.definitions
+
+      definitions.each_value do |definition|
+        record = find_by(key: definition.key)
+
+        if record.nil?
+          # A new schedule does not fire immediately: its first occurrence is
+          # the next one the expression produces.
+          create!(
+            key:      definition.key,
+            cron:     definition.cron,
+            timezone: definition.timezone,
+            next_at:  definition.parsed_cron.next_time(now).to_t
+          )
+        elsif record.cron != definition.cron || record.timezone != definition.timezone
+          record.update!(
+            cron:     definition.cron,
+            timezone: definition.timezone,
+            next_at:  definition.parsed_cron.next_time(now).to_t
+          )
+        end
+      rescue ActiveRecord::RecordNotUnique
+        # Another worker inserted the same schedule first, which is the
+        # outcome this would have produced anyway.
+        nil
+      end
+
+      # Mark everything this process knows about as seen, so that a schedule is
+      # only considered gone once no process has claimed it for a while. Were
+      # rows deleted as soon as one process did not declare them, a rolling
+      # deployment would have the old and the new version delete each other's
+      # schedules in turn - and recreating a row resets its next occurrence,
+      # losing exactly the occurrence this whole mechanism exists to keep.
+      where(key: definitions.keys).update_all(updated_at: now) if definitions.any?
+
+      where.not(key: definitions.keys)
+           .where(arel_table[:updated_at].lt(now - OBSOLETE_GRACE))
+           .delete_all
+
+      return
+    end
+
+    # @return [Workhorse::Schedules::Definition, nil] The registry entry
+    def definition
+      return Workhorse::Schedules[key]
+    end
+
+    # Returns the occurrences that are due, according to the schedule's
+    # catch-up policy, together with the occurrence to wait for next.
+    #
+    # @param now [Time]
+    # @return [Array(Array<Time>, Time)] Occurrences to enqueue and the new
+    #   `next_at`.
+    def pending_occurrences(now = Time.now)
+      # EtOrbi resolves a Time's zone by rebuilding it locally and comparing
+      # the abbreviation, which fails for the whole repeated hour of a
+      # fall-back transition ("Cannot determine timezone from \"CEST\"") when
+      # ENV['TZ'] is unset. UTC always resolves, and the cron's own zone still
+      # governs the computation. `getutc` rather than `utc`, which would
+      # convert the caller's own object in place.
+      now = now.getutc
+      cron = definition.parsed_cron
+
+      # Nothing is due, and the schedule must not be rewound: returning the
+      # next occurrence after now could move next_at backwards for a caller
+      # outside the poller.
+      return [[], next_at] if next_at > now
+
+      # Walked backwards from now rather than forwards from next_at, so that
+      # the work is bounded by how many occurrences a policy could use - one,
+      # or `max_catch_up` - instead of by the length of the outage. Walking
+      # forwards costs about 0.45s per 10 000 occurrences, and this runs while
+      # the poller holds the global lock, whose timeout for other workers is
+      # at most a second.
+      occurrences = []
+      cursor = now
+
+      # An occurrence falling on `now` is due as well, and walking backwards
+      # would step straight past it. Truncated to the second, as `match?`
+      # ignores the fraction and perform_at must be the occurrence itself
+      # rather than the moment this poll happened to run.
+      occurrences << Time.at(now.to_i).utc if cron.match?(now)
+
+      while occurrences.size < wanted_occurrences
+        # Normalized on every step, not just the first: fugit hands back a
+        # time in the cron's zone, and feeding an ambiguous one straight back
+        # in is what raises.
+        cursor = cron.previous_time(cursor).to_t.getutc
+        break if cursor < next_at
+
+        occurrences.unshift(cursor)
+      end
+
+      return [apply_catch_up(deduplicate(occurrences), now), advance(cron, now, occurrences)]
+    end
+
+    # Returns how many occurrences the catch-up policy could use at most.
+    #
+    # @return [Integer]
+    # @private
+    def wanted_occurrences
+      return definition.catch_up == :run ? definition.max_catch_up : 1
+    end
+
+    # Claims this schedule by advancing it to `next_at`, and reports whether
+    # the claim succeeded.
+    #
+    # Written as a compare-and-swap rather than a plain update so that exactly
+    # one worker materializes an occurrence even if the claim is ever made
+    # without the global lock the poller currently holds.
+    #
+    # @param new_next_at [Time]
+    # @return [Boolean]
+    # rubocop:disable Naming/PredicateMethod -- reports whether the mutation
+    # took effect, which a predicate name would misrepresent as a question.
+    def claim!(new_next_at)
+      claimed = self.class
+                    .where(id: id, next_at: next_at)
+                    .update_all(next_at: new_next_at, updated_at: Time.now)
+
+      return claimed == 1
+    end
+    # rubocop:enable Naming/PredicateMethod
+
+    # Enqueues the job for the given occurrence.
+    #
+    # `perform_at` is the occurrence's own time rather than the current one,
+    # so that the job carries the moment it was meant to run. The difference
+    # to `started_at` is then the lateness of that occurrence, which is what
+    # {Workhorse.on_job_late} and any reporting on the table can go by.
+    #
+    # @param occurrence [Time]
+    # @return [Workhorse::DbJob]
+    def enqueue!(occurrence)
+      db_job = Workhorse.enqueue_job_class(
+        definition.job,
+        queue:        definition.queue,
+        priority:     definition.priority,
+        perform_at:   occurrence,
+        description:  definition.description || "#{definition.job} (#{key})",
+        params:       definition.params,
+        max_lateness: definition.max_lateness,
+        expires_at:   definition.expires_after ? occurrence + definition.expires_after : nil
+      )
+
+      update_columns(
+        last_enqueued_at: Time.now,
+        last_occurrence:  occurrence,
+        last_job_id:      db_job.id,
+        updated_at:       Time.now
+      )
+
+      return db_job
+    end
+
+    private
+
+    # Returns the occurrence to wait for after the given ones.
+    #
+    # Skips an occurrence repeating the wall-clock time just emitted: the
+    # clocks going back make a local time happen twice, and only
+    # {#deduplicate} would see those two instants when they fall in the same
+    # walk. At the default `catch_up: :run_once` they never do, as only one
+    # occurrence is taken per call.
+    #
+    # @param cron [Fugit::Cron]
+    # @param now [Time]
+    # @param occurrences [Array<Time>]
+    # @return [Time]
+    def advance(cron, now, occurrences)
+      next_at = cron.next_time(now).to_t
+
+      return next_at unless wall_clock_anchored?
+      return next_at if occurrences.empty?
+
+      # Stepped in a loop, not once: an hour holding several occurrences -
+      # `0,30 2 * * *` - replays all of them, and consecutive candidates
+      # never collide pairwise. Formatted times sort chronologically, and the
+      # wall clock advances again once the transition is over, so this
+      # terminates; bounded regardless.
+      last = wall_clock(occurrences.last)
+
+      MAX_REPEATED_STEPS.times do
+        break if wall_clock(next_at) > last
+
+        next_at = cron.next_time(next_at.getutc).to_t
+      end
+
+      return next_at
+    end
+
+    # Drops occurrences that fall on the same wall-clock time, see {#advance}.
+    #
+    # Only for a schedule anchored to one: "at 02:30" means that clock time,
+    # which happens once. An expression with a wildcard hour ticks on every
+    # real instant instead, so both halves of a repeated hour are genuine and
+    # dropping one would silently skip an hour of work once a year.
+    #
+    # @param occurrences [Array<Time>]
+    # @return [Array<Time>]
+    def deduplicate(occurrences)
+      return occurrences unless wall_clock_anchored?
+
+      return occurrences.uniq { |occurrence| wall_clock(occurrence) }
+    end
+
+    # @return [Boolean] Whether the expression names an hour rather than
+    #   ticking within every one
+    def wall_clock_anchored?
+      return !definition.parsed_cron.hours.nil?
+    end
+
+    # @param time [Time]
+    # @return [String] The local time, in the schedule's zone
+    def wall_clock(time)
+      # The zone may be given as an option or as a trailing token of the
+      # expression itself, and fugit reports either.
+      zone = definition.parsed_cron.zone || definition.timezone
+      local = zone ? time.in_time_zone(zone) : time.getlocal
+
+      return local.strftime('%F %T')
+    end
+
+    # Applies the catch-up policy to the occurrences that are due.
+    #
+    # @param occurrences [Array<Time>]
+    # @param now [Time]
+    # @return [Array<Time>]
+    def apply_catch_up(occurrences, now)
+      return occurrences if occurrences.empty?
+
+      case definition.catch_up
+      when :run, :run_once
+        # Already bounded by #wanted_occurrences.
+        return occurrences
+      when :skip
+        return occurrences.select { |occurrence| now - occurrence <= definition.grace }
+      else
+        return []
+      end
+    end
+  end
+end

data/lib/workhorse/schedules.rb

@@ -0,0 +1,197 @@
+module Workhorse
+  # Registry of scheduled jobs, populated through {Workhorse.schedules}.
+  #
+  # A schedule's job class and options live here, in code, and are addressed
+  # from the database by key. Storing the class in the database instead would
+  # turn renaming it into a data migration; keeping it here makes it a
+  # reconciliation, which {Workhorse::Schedule.reconcile!} performs on worker
+  # startup.
+  #
+  # @see Workhorse::Schedule
+  module Schedules
+    # Catch-up policies, deciding what happens to occurrences whose time
+    # passed while nothing was materialising them.
+    CATCH_UP_POLICIES = %i[run run_once skip].freeze
+
+    # Default number of missed occurrences a `:run` schedule materialises at
+    # once, so that a long outage cannot enqueue thousands of jobs.
+    DEFAULT_MAX_CATCH_UP = 10
+
+    # One entry of the registry.
+    class Definition
+      attr_reader :key
+      attr_reader :job
+      attr_reader :cron
+      attr_reader :timezone
+      attr_reader :queue
+      attr_reader :priority
+      attr_reader :description
+      attr_reader :params
+      attr_reader :catch_up
+      attr_reader :grace
+      attr_reader :max_catch_up
+      attr_reader :max_lateness
+      attr_reader :expires_after
+
+      # @see Workhorse::Schedules::Dsl#schedule
+      def initialize(key, job:, cron:, timezone: nil, queue: nil, priority: 0, description: nil,
+                     params: {}, catch_up: :run_once, grace: nil, max_catch_up: DEFAULT_MAX_CATCH_UP,
+                     max_lateness: nil, expires_after: nil)
+        @key = key.to_s
+        @job = job.to_s
+        @cron = cron.to_s
+        @timezone = timezone&.to_s
+        @queue = queue
+        @priority = priority
+        @description = description
+        @params = (params || {}).freeze
+        @catch_up = catch_up.to_sym
+        @grace = grace
+        @max_catch_up = max_catch_up
+        @max_lateness = max_lateness
+        @expires_after = expires_after
+
+        validate!
+        freeze
+      end
+
+      # Returns the parsed cron expression.
+      #
+      # @return [Fugit::Cron]
+      def parsed_cron
+        return self.class.parse_cron(cron, timezone)
+      end
+
+      # Parses a cron expression, applying a timezone if one is given. Fugit
+      # takes the zone as a trailing token of the expression.
+      #
+      # @param cron [String]
+      # @param timezone [String, nil]
+      # @return [Fugit::Cron, nil] nil if the expression cannot be parsed
+      def self.parse_cron(cron, timezone = nil)
+        return Fugit::Cron.parse(timezone ? "#{cron} #{timezone}" : cron)
+      end
+
+      private
+
+      def validate!
+        fail ArgumentError, 'Schedule key must not be blank.' if key.empty?
+        fail ArgumentError, "Schedule #{key.inspect}: job must not be blank." if job.empty?
+
+        unless CATCH_UP_POLICIES.include?(catch_up)
+          fail ArgumentError, "Schedule #{key.inspect}: catch_up must be one of " \
+                              "#{CATCH_UP_POLICIES.inspect}, got #{catch_up.inspect}."
+        end
+
+        # Without a grace period, ":skip" has no way of telling an occurrence
+        # that is merely a moment late from one that is a day late, and would
+        # silently drop every occurrence.
+        if catch_up == :skip && grace.nil?
+          fail ArgumentError, "Schedule #{key.inspect}: catch_up :skip requires a grace period, " \
+                              'which is how late an occurrence may be and still run.'
+        end
+
+        if catch_up == :run && !(max_catch_up.is_a?(Integer) && max_catch_up >= 1)
+          fail ArgumentError, "Schedule #{key.inspect}: max_catch_up must be an Integer >= 1, " \
+                              "got #{max_catch_up.inspect}."
+        end
+
+        if parsed_cron.nil?
+          zone = timezone ? " for timezone #{timezone.inspect}" : nil
+          fail ArgumentError, "Schedule #{key.inspect}: #{cron.inspect} is not a valid cron " \
+                              "expression#{zone}."
+        end
+
+        return
+      end
+    end
+
+    # Evaluator for the {Workhorse.schedules} block.
+    class Dsl
+      # @private
+      def initialize(definitions)
+        @definitions = definitions
+      end
+
+      # Registers a scheduled job.
+      #
+      # @param key [String, Symbol] Stable name of this schedule. Occurrences
+      #   are tracked under it, so renaming it starts a new schedule.
+      # @param job [String, Class] Job class. May be a plain class responding
+      #   to `perform`, an `ActiveJob::Base` subclass, or a
+      #   `RailsOps::Operation`.
+      # @param cron [String] Cron expression in crontab format.
+      # @param timezone [String, nil] Timezone the expression is read in, e.g.
+      #   `'Europe/Zurich'`. Defaults to the process's local time.
+      # @param queue [String, Symbol, nil] Queue to enqueue into.
+      # @param priority [Integer] Job priority, lower runs first.
+      # @param description [String, nil] Description of the enqueued job.
+      # @param params [Hash] Parameters for the job. Passed as operation
+      #   params for RailsOps operations and splatted as keyword arguments
+      #   otherwise.
+      # @param catch_up [Symbol] What to do with occurrences whose time passed
+      #   while nothing was materialising them:
+      #   * `:run_once` (default) collapses them into a single job,
+      #   * `:run` materialises each, up to `max_catch_up`,
+      #   * `:skip` drops those older than `grace`.
+      # @param grace [Numeric, nil] Seconds an occurrence may be late and
+      #   still run. Required for `catch_up: :skip`.
+      # @param max_catch_up [Integer] Most occurrences `catch_up: :run`
+      #   materialises at once.
+      # @param max_lateness [Numeric, nil] Seconds the job may start after its
+      #   occurrence before {Workhorse.on_job_late} is called.
+      # @param expires_after [Numeric, nil] Seconds after its occurrence at
+      #   which the job is no longer worth running. It is then expired instead
+      #   of performed, and {Workhorse.on_job_expired} is called.
+      # @return [void]
+      def schedule(key, **options)
+        definition = Definition.new(key, **options)
+
+        if @definitions.key?(definition.key)
+          fail ArgumentError, "Schedule #{definition.key.inspect} is already defined."
+        end
+
+        @definitions[definition.key] = definition
+
+        return
+      end
+    end
+
+    class << self
+      # @return [Hash{String => Definition}] The registered schedules by key
+      def definitions
+        return @definitions ||= {}
+      end
+
+      # Registers schedules, see {Workhorse::Schedules::Dsl#schedule}.
+      #
+      # @yield Block evaluated against the DSL
+      # @return [void]
+      def define(&block)
+        Dsl.new(definitions).instance_eval(&block)
+
+        return
+      end
+
+      # @param key [String, Symbol]
+      # @return [Definition, nil] The schedule registered under `key`
+      def [](key)
+        return definitions[key.to_s]
+      end
+
+      # @return [Boolean] Whether any schedule is registered
+      def any?
+        return definitions.any?
+      end
+
+      # Empties the registry. Intended for testing.
+      #
+      # @return [void]
+      def reset!
+        @definitions = {}
+
+        return
+      end
+    end
+  end
+end

data/lib/workhorse/worker.rb

@@ -23,6 +23,9 @@ module Workhorse
     LOG_REOPEN_SIGNAL = 'HUP'.freeze
     SOFT_RESTART_SIGNAL = 'USR1'.freeze
 
+    # Seconds a repeated {#shutdown} waits for the one already in progress.
+    REPEAT_SHUTDOWN_TIMEOUT = 60
+
     # @return [Array<Symbol>] The queues this worker processes
     attr_reader :queues
 
@@ -193,30 +196,56 @@ module Workhorse
     end
 
     # Shuts down worker and DB poller. Jobs currently being processed are
-    # properly finished before this method returns. Subsequent calls to this
-    # method are ignored.
+    # properly finished before this method returns, including for a call that
+    # finds another thread already shutting the worker down - such a call
+    # waits rather than returning early. Shutting down a worker that was
+    # never started does nothing.
     #
     # @return [void]
     def shutdown
-      # This is safe to be checked outside of the mutex as 'shutdown' is the
-      # final state this worker can be in.
-      return if @state == :shutdown
-
-      # TODO: There is a race-condition with this shutdown:
-      #  - If the poller is currently locking a job, it may call
-      #    "worker.perform", which in turn tries to synchronize the same mutex.
-      mutex.synchronize do
-        assert_state! :running
+      transitioned = mutex.synchronize do
+        next false unless @state == :running
 
         Workhorse.debug_log("[Job worker #{id}] Shutdown starting")
         log 'Shutting down'
         @state = :shutdown
 
+        next true
+      end
+
+      unless transitioned
+        # Another thread is doing the shutting down, or it is already done.
+        # Wait for the pool rather than returning, so that this call keeps the
+        # promise above that running jobs have finished once it returns. On
+        # a worker that was never started nothing will ever shut the pool
+        # down, and #wait would block forever.
+        #
+        # Bounded, because this runs inside the signal handler for the second
+        # of the TERM and INT the daemon's stop sends, which joins it: were
+        # the shutdown it waits on stuck, an unbounded wait would take the
+        # whole process down with it - including its ability to be killed.
+        if @state == :shutdown && !@pool.wait(timeout: REPEAT_SHUTDOWN_TIMEOUT)
+          log 'Gave up waiting for the concurrent shutdown to finish', :warn
+        end
+
+        return
+      end
+
+      # Deliberately outside the mutex. Stopping the poller waits for the
+      # poller thread, and that thread may be in #perform waiting for this
+      # very mutex, having just committed the lock on a job. Holding the mutex
+      # here would deadlock the two, leaving a worker that ignores TERM.
+      begin
         @poller.shutdown
+      ensure
+        # Even if the poller was already gone - it shuts itself down after an
+        # exception - the pool has to be stopped, or #wait never returns and a
+        # concurrent shutdown waits forever.
         @pool.shutdown
-        log 'Shut down'
-        Workhorse.debug_log("[Job worker #{id}] Shutdown complete")
       end
+
+      log 'Shut down'
+      Workhorse.debug_log("[Job worker #{id}] Shutdown complete")
     end
 
     # Waits until the worker is shut down. This only happens if {#shutdown} gets
@@ -262,7 +291,7 @@ module Workhorse
       return unless daemon_id
 
       path = self.class.heartbeat_file_for(daemon_id)
-      FileUtils.touch(path) if path
+      touch_pid_file(path) if path
     rescue StandardError => e
       Workhorse.debug_log("[Job worker #{id}] Heartbeat touch failed: #{e.class}: #{e.message}")
     end
@@ -272,27 +301,48 @@ module Workhorse
     # @param db_job_id [Integer] The ID of the {Workhorse::DbJob} to perform
     # @return [void]
     def perform(db_job_id)
-      begin # rubocop:disable Style/RedundantBegin
-        mutex.synchronize do
-          assert_state! :running
-          log "Posting job #{db_job_id} to thread pool"
-
-          @pool.post do
-            begin # rubocop:disable Style/RedundantBegin
-              Workhorse::Performer.new(db_job_id, self).perform
-            rescue Exception => e
-              log %(#{e.message}\n#{e.backtrace.join("\n")}), :error
-              Workhorse.on_exception.call(e)
-            end
+      mutex.synchronize do
+        # The poller commits the lock on a job before posting it here, so the
+        # worker may have begun shutting down in between.
+        next release(db_job_id) unless @state == :running
+
+        log "Posting job #{db_job_id} to thread pool"
+
+        @pool.post do
+          begin # rubocop:disable Style/RedundantBegin
+            Workhorse::Performer.new(db_job_id, self).perform
+          rescue Exception => e
+            log %(#{e.message}\n#{e.backtrace.join("\n")}), :error
+            Workhorse.on_exception.call(e)
           end
         end
-      rescue Exception => e
-        Workhorse.on_exception.call(e)
       end
+    rescue Exception => e
+      Workhorse.on_exception.call(e)
     end
 
     private
 
+    # Resets a job this worker locked but will not run back to `waiting`, so
+    # that another worker picks it up.
+    #
+    # Leaving it locked would be worse than the delay: a locked job in a named
+    # queue blocks that queue entirely until somebody resets it by hand.
+    #
+    # @param db_job_id [Integer] The ID of the {Workhorse::DbJob} to release
+    # @return [void]
+    # @private
+    def release(db_job_id)
+      db_job = Workhorse::DbJob.find(db_job_id)
+
+      return unless db_job.state.to_sym == Workhorse::DbJob::STATE_LOCKED
+
+      log "Not performing job #{db_job_id} as the worker is shutting down, resetting it to waiting"
+      db_job.reset!(true)
+
+      return
+    end
+
     # Checks current memory usage and initiates shutdown if limit exceeded.
     #
     # @return [Boolean] True if memory is within limits, false if exceeded
@@ -313,7 +363,7 @@ module Workhorse
       Workhorse.debug_log("[Job worker #{id}] Memory limit exceeded: #{mem}MB > #{max}MB, initiating shutdown")
 
       if defined?(Rails)
-        FileUtils.touch self.class.shutdown_file_for(pid)
+        touch_pid_file(self.class.shutdown_file_for(pid))
       end
 
       log "Worker process #{id.inspect} memory consumption (RSS) of #{mem}MB exceeds " \
@@ -394,8 +444,17 @@ module Workhorse
       # Create shutdown file for watch to detect
       shutdown_file = self.class.shutdown_file_for(pid)
       if shutdown_file
-        FileUtils.touch(shutdown_file)
-        Workhorse.debug_log("[Job worker #{id}] Shutdown file created: #{shutdown_file}")
+        begin
+          touch_pid_file(shutdown_file)
+          Workhorse.debug_log("[Job worker #{id}] Shutdown file created: #{shutdown_file}")
+        rescue StandardError => e
+          # The shutdown has to go ahead regardless: the worker stopped
+          # accepting jobs above, so giving up here would leave it taking none
+          # and never exiting. Without the file, watch sees a worker that is
+          # gone rather than one that asked to be restarted, and starts it all
+          # the same.
+          log "Could not create shutdown file #{shutdown_file}, shutting down regardless: #{e.message}", :warn
+        end
       end
 
       # Monitor in a separate thread to avoid blocking the signal handler
@@ -483,5 +542,17 @@ module Workhorse
         Kernel.sleep 0.2
       end
     end
+
+    # Touches a file in the pids directory, creating the directory first. It
+    # is not guaranteed to exist: `tmp` is rarely checked in, so a fresh
+    # checkout or deployment has none until something writes there.
+    #
+    # @param path [Pathname, String]
+    # @return [void]
+    # @private
+    def touch_pid_file(path)
+      FileUtils.mkdir_p(File.dirname(path))
+      FileUtils.touch(path)
+    end
   end
 end