postburner 1.0.0.pre.11 → 1.0.0.pre.13

data/lib/postburner/scheduler.rb

@@ -0,0 +1,427 @@
+# frozen_string_literal: true
+
+require 'postburner/advisory_lock'
+
+module Postburner
+  # Lightweight scheduler that acts as a safety net for recurring job execution.
+  #
+  # Unlike traditional schedulers that poll for due jobs, Postburner uses **immediate
+  # enqueue** where executions are created and immediately queued to Beanstalkd's
+  # delayed queue. The scheduler acts as a **watchdog safety net** to ensure every
+  # schedule has a future execution queued.
+  #
+  # ## Architecture
+  #
+  # This class is NOT persisted to the database. It's instantiated on-the-fly by
+  # workers when they reserve a watchdog job from the scheduler tube. The watchdog
+  # is ephemeral data in Beanstalkd with this payload:
+  #
+  #   { "scheduler": true, "interval": 300 }
+  #
+  # No dedicated scheduler process is needed - existing workers handle everything.
+  #
+  # ## Watchdog Pattern
+  #
+  # 1. Workers automatically watch the 'scheduler' tube
+  # 2. On reserve timeout, workers check if watchdog exists and create if missing
+  # 3. When worker reserves watchdog, it instantiates Postburner::Scheduler
+  # 4. Scheduler executes with PostgreSQL advisory lock for coordination
+  # 5. After completion, watchdog re-queues itself with delay for next interval
+  #
+  # ## Safety Net Functions
+  #
+  # The watchdog performs three safety net functions:
+  #
+  # 1. **Auto-bootstrap**: Creates first execution for schedules that haven't been started
+  # 2. **Future execution guarantee**: Ensures each schedule has a future execution queued
+  # 3. **Orphan cleanup**: Enqueues any pending executions that weren't properly queued
+  #
+  # Note: For Postburner::Job schedules, a before_attempt callback creates the next
+  # execution when the current job runs, providing immediate pickup without waiting
+  # for the watchdog. The watchdog is just the safety net.
+  #
+  # ## Configuration
+  #
+  # Add to config/postburner.yml:
+  #
+  #   production:
+  #     default_scheduler_interval: 300  # Check every 5 minutes (default)
+  #     default_scheduler_priority: 100  # Watchdog priority (default)
+  #
+  # The interval primarily affects:
+  # - How quickly new schedules are auto-bootstrapped (if you don't call start!)
+  # - Recovery time if an execution somehow fails to enqueue
+  # - How often last_audit_at is updated for monitoring
+  #
+  # Since executions are enqueued immediately to Beanstalkd's delayed queue, the
+  # watchdog interval doesn't affect execution timing - only the safety net checks.
+  #
+  # ## Execution Flow
+  #
+  # 1. Worker reserves watchdog from scheduler tube
+  # 2. Instantiates Scheduler with interval from payload
+  # 3. Scheduler#perform acquires PostgreSQL advisory lock
+  # 4. Processes all enabled schedules:
+  #    - Auto-bootstraps unstarted schedules (creates + enqueues first execution)
+  #    - Ensures future execution exists (creates + enqueues if missing)
+  #    - Enqueues any orphaned pending executions
+  #    - Updates last_audit_at timestamp
+  # 5. Re-queues watchdog with delay for next interval
+  #
+  # ## Observability
+  #
+  # Monitor scheduler health via last_audit_at:
+  #
+  #   stale = Postburner::Schedule.enabled
+  #     .where('last_audit_at < ?', 15.minutes.ago)
+  #     .or(Postburner::Schedule.where(last_audit_at: nil))
+  #
+  # @example Watchdog payload in Beanstalkd
+  #   {
+  #     "scheduler": true,
+  #     "interval": 300
+  #   }
+  #
+  # @example Manual watchdog creation (usually automatic)
+  #   Postburner::Scheduler.enqueue_watchdog(interval: 300, priority: 100)
+  #
+  # @see Postburner::Schedule
+  # @see Postburner::ScheduleExecution
+  #
+  class Scheduler
+    SCHEDULER_TUBE_NAME = 'scheduler'
+    DEFAULT_INTERVAL = 300 # 5 minutes
+    DEFAULT_PRIORITY = 100 # Lower number = higher priority
+
+    attr_reader :interval, :logger
+
+    # Initialize a new scheduler instance.
+    #
+    # The scheduler is instantiated by workers when they reserve a watchdog job
+    # from the scheduler tube. It's not persisted - each run creates a new instance.
+    #
+    # @param interval [Integer] Seconds until next run (default: 300)
+    # @param logger [Logger, nil] Logger instance (default: Rails.logger or stdout)
+    #
+    # @example
+    #   scheduler = Postburner::Scheduler.new(interval: 300)
+    #   scheduler.perform
+    #
+    def initialize(interval: DEFAULT_INTERVAL, logger: nil)
+      @interval = interval
+      @logger = logger || (defined?(Rails) ? Rails.logger : Logger.new($stdout))
+    end
+
+    # Execute the scheduler.
+    #
+    # Processes all enabled schedules with PostgreSQL advisory lock coordination
+    # to prevent concurrent execution across multiple workers. After processing,
+    # automatically re-queues the watchdog for the next run.
+    #
+    # The scheduler performs two main functions:
+    # 1. Auto-bootstrap new schedules (create first execution if not started)
+    # 2. Ensure each schedule has a future execution queued
+    # 3. Enqueue any orphaned pending executions that weren't properly queued
+    #
+    # @return [void]
+    #
+    # @example Called by worker
+    #   # Worker reserves watchdog job with payload:
+    #   # { "scheduler" => true, "interval" => 300 }
+    #   scheduler = Postburner::Scheduler.new(interval: 300)
+    #   scheduler.perform
+    #
+    def perform
+      logger.info "[Postburner::Scheduler] Starting scheduler run"
+
+      # Use advisory lock to coordinate multiple workers
+      acquired = Postburner::AdvisoryLock.with_lock(AdvisoryLock::SCHEDULER_LOCK_KEY, blocking: false) do
+        process_all_schedules
+        true
+      end
+
+      if acquired
+        logger.info "[Postburner::Scheduler] Scheduler run complete"
+      else
+        logger.info "[Postburner::Scheduler] Could not acquire lock, skipping"
+      end
+    ensure
+      # Always re-queue watchdog for next run
+      requeue_watchdog
+    end
+
+    # Class method to enqueue watchdog to Beanstalkd
+    # Mutex for coordinating watchdog checks across threads
+    WATCHDOG_MUTEX = Mutex.new
+    @watchdog_last_checked_at = nil
+    @watchdog_check_failed_at = nil
+
+    class << self
+      attr_accessor :watchdog_last_checked_at, :watchdog_check_failed_at
+    end
+
+    # Ensure a scheduler watchdog exists in the queue.
+    #
+    # Uses process-level mutex coordination so only one thread checks at a time.
+    # Called by workers on reserve timeout to automatically recreate watchdog
+    # if it's missing (e.g., after Beanstalkd restart or watchdog expiration).
+    #
+    # Implements throttling:
+    # - Skips check if successful check within last 60 seconds
+    # - Skips check if failed check within last 60 seconds
+    # - Only one thread can check at a time (mutex)
+    #
+    # @param connection [Postburner::Connection] Existing beanstalkd connection
+    # @return [void]
+    #
+    # @example Called by worker on timeout
+    #   Postburner.connected do |conn|
+    #     Postburner::Scheduler.ensure_watchdog!(connection: conn)
+    #   end
+    #
+    def self.ensure_watchdog!(connection:)
+      # Quick check without lock - skip if recently checked
+      return if watchdog_last_checked_at && Time.current - watchdog_last_checked_at < 60
+      return if watchdog_check_failed_at && Time.current - watchdog_check_failed_at < 60
+
+      # Try to acquire lock, skip if another thread is checking
+      return unless WATCHDOG_MUTEX.try_lock
+
+      begin
+        # Double-check after acquiring lock
+        return if watchdog_last_checked_at && Time.current - watchdog_last_checked_at < 60
+
+        if watchdog_exists?(connection: connection)
+          self.watchdog_last_checked_at = Time.current
+          return
+        end
+
+        enqueue_watchdog
+        self.watchdog_last_checked_at = Time.current
+        self.watchdog_check_failed_at = nil
+      rescue => e
+        self.watchdog_check_failed_at = Time.current
+        Rails.logger.error "[Postburner::Scheduler] Watchdog check failed, will retry in 60s: #{e.message}"
+      ensure
+        WATCHDOG_MUTEX.unlock
+      end
+    end
+
+    # Enqueue a new scheduler watchdog job.
+    #
+    # Creates a watchdog job in the scheduler tube with a delay equal to the interval.
+    # The watchdog will execute after the delay, process all schedules, and re-queue itself.
+    #
+    # Reads interval and priority from configuration if not provided:
+    # - default_scheduler_interval (default: 300 seconds)
+    # - default_scheduler_priority (default: 100)
+    #
+    # @param interval [Integer, nil] Seconds until next run (default: from config)
+    # @param priority [Integer, nil] Beanstalkd priority (default: from config)
+    # @return [Hash] Beanstalkd response with :status and :id keys
+    #
+    # @example Enqueue with defaults
+    #   Postburner::Scheduler.enqueue_watchdog
+    #   # => { status: "INSERTED", id: 12345 }
+    #
+    # @example Enqueue with custom interval
+    #   Postburner::Scheduler.enqueue_watchdog(interval: 60, priority: 0)
+    #
+    def self.enqueue_watchdog(interval: nil, priority: nil)
+      interval ||= Postburner.configuration.default_scheduler_interval || DEFAULT_INTERVAL
+      priority ||= Postburner.configuration.default_scheduler_priority || DEFAULT_PRIORITY
+
+      payload = {
+        'scheduler' => true,
+        'interval' => interval
+      }
+
+      tube_name = Postburner.scheduler_tube_name
+      response = nil
+
+      Postburner.connected do |conn|
+        response = conn.tubes[tube_name].put(
+          JSON.generate(payload),
+          pri: priority,
+          delay: interval,
+          ttr: 120 # 2 minutes to complete
+        )
+      end
+
+      if response[:status] == "INSERTED"
+        runs_at = Time.current + interval
+        Rails.logger.info "[Postburner::Scheduler] Inserted watchdog: #{response[:id]} (#{Time.current.iso8601}, delay: #{interval}s (#{runs_at.iso8601}), tube: #{tube_name})"
+      else
+        Rails.logger.error "[Postburner::Scheduler] Failed to insert watchdog: #{response.inspect} (delay: #{interval}s, tube: #{tube_name})"
+      end
+
+      response
+    end
+
+    # Check if scheduler watchdog exists in queue.
+    #
+    # Peeks both the delayed and ready queues in the scheduler tube.
+    # Returns true if watchdog exists in either state.
+    #
+    # @param connection [Postburner::Connection] Existing beanstalkd connection
+    # @return [Boolean] true if watchdog exists (delayed or ready), false otherwise
+    #
+    # @example Check for watchdog
+    #   Postburner.connected do |conn|
+    #     exists = Postburner::Scheduler.watchdog_exists?(connection: conn)
+    #     puts "Watchdog present" if exists
+    #   end
+    #
+    def self.watchdog_exists?(connection:)
+      tube_name = Postburner.scheduler_tube_name
+      tube = connection.beanstalk.tubes[tube_name]
+
+      # Peek is lighter than stats
+      delayed = tube.peek(:delayed) rescue nil
+      ready = tube.peek(:ready) rescue nil
+
+      delayed.present? || ready.present?
+    rescue Beaneater::NotFoundError
+      false
+    end
+
+    private
+
+    # Process all enabled schedules.
+    #
+    # Iterates through all enabled schedules and calls process_schedule for each.
+    # Logs count of successful and failed schedule processing.
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def process_all_schedules
+      processed_count = 0
+      failed_count = 0
+
+      Postburner::Schedule.enabled.find_each do |schedule|
+        begin
+          process_schedule(schedule)
+          processed_count += 1
+        rescue => e
+          logger.error "[Postburner::Scheduler] Failed to process schedule '#{schedule.name}': #{e.class} - #{e.message}"
+          logger.error e.backtrace.join("\n")
+          failed_count += 1
+        end
+      end
+
+      logger.info "[Postburner::Scheduler] Processed #{processed_count} schedules, #{failed_count} failed"
+    end
+
+    # Process a single schedule.
+    #
+    # The watchdog performs three safety net functions:
+    # 1. Auto-bootstrap: Create first execution if schedule hasn't been started
+    # 2. Ensure there's always a future execution (creates + enqueues if missing)
+    # 3. Enqueue any orphaned pending executions (that somehow weren't enqueued)
+    #
+    # Note: We check for future executions FIRST, then clean up any orphans.
+    # This ensures newly created executions are available for processing in the
+    # same run if they happen to be due.
+    #
+    # @param schedule [Postburner::Schedule] The schedule to process
+    # @return [void]
+    #
+    # @api private
+    #
+    def process_schedule(schedule)
+      # Auto-bootstrap: create first execution if schedule hasn't been started
+      # This will create the execution AND enqueue it to Beanstalkd
+      unless schedule.started?
+        logger.info "[Postburner::Scheduler] Bootstrapping schedule '#{schedule.name}'"
+        schedule.start!
+      end
+
+      # Safety net 1: Ensure schedule has a future execution
+      # If missing, this will create AND enqueue it to Beanstalkd's delayed queue
+      ensure_future_execution!(schedule)
+
+      # Safety net 2: Find any orphaned pending executions and enqueue them
+      # This should rarely happen - only if enqueue! previously failed
+      execution_count = 0
+      schedule.executions.due.find_each do |execution|
+        begin
+          logger.warn "[Postburner::Scheduler] Found orphaned pending execution #{execution.id} for schedule '#{schedule.name}', enqueuing"
+          execution.enqueue!
+          execution_count += 1
+        rescue => e
+          logger.error "[Postburner::Scheduler] Failed to enqueue execution #{execution.id}: #{e.class} - #{e.message}"
+          raise
+        end
+      end
+
+      # Update last_audit_at
+      schedule.update_column(:last_audit_at, Time.current)
+
+      logger.debug "[Postburner::Scheduler] Schedule '#{schedule.name}': enqueued #{execution_count} orphaned executions" if execution_count > 0
+    end
+
+    # Ensure schedule has a future scheduled or pending execution.
+    #
+    # Uses Schedule#create_next_execution! which is idempotent and only
+    # creates an execution if none exists in the future. If no executions
+    # exist at all, bootstraps the schedule first.
+    #
+    # This is the key safety net that ensures every schedule always has
+    # at least one future execution ready to run.
+    #
+    # @param schedule [Postburner::Schedule] The schedule to ensure future execution for
+    # @return [void]
+    #
+    # @api private
+    #
+    def ensure_future_execution!(schedule)
+      # Find the most recent execution to calculate next from
+      last_execution = schedule.executions.order(run_at: :desc).first
+
+      unless last_execution
+        # No executions at all - this shouldn't happen since we bootstrap first,
+        # but handle it as a safety net
+        logger.warn "[Postburner::Scheduler] Schedule '#{schedule.name}' has no executions, bootstrapping"
+        schedule.start!
+        return
+      end
+
+      # Delegate to Schedule - it knows whether a future execution is needed
+      execution = schedule.create_next_execution!(after: last_execution)
+
+      if execution
+        logger.info "[Postburner::Scheduler] Created future execution for '#{schedule.name}'"
+      end
+    end
+
+    # Re-queue the watchdog for the next run (if not already queued).
+    #
+    # Called in the ensure block of perform to guarantee the watchdog
+    # perpetuates itself. Always reads interval from config to pick up
+    # any runtime configuration changes.
+    #
+    # If re-queueing fails, logs error but doesn't raise - workers will
+    # recreate the watchdog on next timeout via ensure_watchdog!
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def requeue_watchdog
+      Postburner.connected do |conn|
+        if self.class.watchdog_exists?(connection: conn)
+          logger.debug "[Postburner::Scheduler] Watchdog already exists, skipping re-queue"
+          return
+        end
+
+        self.class.enqueue_watchdog
+      end
+    rescue => e
+      logger.error "[Postburner::Scheduler] Failed to re-queue watchdog: #{e.class} - #{e.message}"
+      # This is critical - if watchdog isn't re-queued, scheduling stops
+      # Workers will recreate it on next timeout, but log loudly
+    end
+  end
+end

data/lib/postburner/strategies/immediate_test_queue.rb

@@ -105,6 +105,10 @@ module Postburner
       # is invoked. If the job has a future run_at, travels to that time before
       # execution. Otherwise executes immediately.
       #
+      # Uses recursion detection to prevent infinite loops when jobs create and
+      # enqueue other jobs (e.g., schedule callbacks). Jobs enqueued while already
+      # inside a perform! call are queued but not executed immediately.
+      #
       # @param job [Postburner::Job] The job to execute
       # @param options [Hash] Unused in test mode
       #
@@ -115,17 +119,30 @@ module Postburner
       # @api private
       #
       def insert(job, options = {})
-        # If job has a future run_at, travel to that time for execution
-        if job.run_at && job.run_at > Time.zone.now
-          travel_to(job.run_at) do
+        # Detect recursion: if we're already performing a job, don't execute
+        # the newly enqueued job immediately. This prevents infinite loops
+        # when job callbacks (like schedule_next_execution) enqueue new jobs.
+        if Thread.current[:postburner_performing]
+          return { status: 'INLINE_DEFERRED', id: nil }
+        end
+
+        begin
+          Thread.current[:postburner_performing] = true
+
+          # If job has a future run_at, travel to that time for execution
+          if job.run_at && job.run_at > Time.current
+            travel_to(job.run_at) do
+              job.perform!(job.args)
+            end
+          else
+            # No future run_at, execute normally
             job.perform!(job.args)
           end
-        else
-          # No future run_at, execute normally
-          job.perform!(job.args)
+        ensure
+          Thread.current[:postburner_performing] = false
         end
 
-        # Return format matching Backburner response (symbol keys)
+        # Return format matching Beanstalkd response (symbol keys)
         { status: 'INLINE', id: nil }
       end
     end

data/lib/postburner/strategies/nice_queue.rb

@@ -77,7 +77,7 @@ module Postburner
       # @note Logs "PREMATURE; RE-INSERTED" message to job's audit trail
       #
       def handle_premature_perform(job)
-        response = job.insert! delay: job.run_at - Time.zone.now
+        response = job.insert! delay: job.run_at - Time.current
         job.log! "PREMATURE; RE-INSERTED: #{response}"
       end
     end

data/lib/postburner/strategies/null_queue.rb

@@ -88,7 +88,7 @@ module Postburner
       # @api private
       #
       def insert(job, options = {})
-        # Return format matching Backburner response (symbol keys)
+        # Return format matching Beanstalkd response (symbol keys)
         { status: 'NULL', id: nil }
       end
 
@@ -118,7 +118,7 @@ module Postburner
       # @api private
       #
       def handle_perform!(job)
-        if job.run_at && job.run_at > Time.zone.now
+        if job.run_at && job.run_at > Time.current
           travel_to(job.run_at) do
             job.perform!(job.args)
           end

data/lib/postburner/strategies/test_queue.rb

@@ -105,7 +105,7 @@ module Postburner
         # Will raise PrematurePerform if run_at is in the future
         job.perform!(job.args)
 
-        # Return format matching Backburner response (symbol keys)
+        # Return format matching Beanstalkd response (symbol keys)
         { status: 'INLINE', id: nil }
       end
 
@@ -121,7 +121,7 @@ module Postburner
       # @raise [Postburner::Job::PrematurePerform] Always raises with helpful message
       #
       def handle_premature_perform(job)
-        raise Postburner::Job::PrematurePerform, "Job scheduled for #{job.run_at} (#{((job.run_at - Time.zone.now) / 60).round(1)} minutes from now). Use `travel_to(job.run_at)` in your test, or set `Postburner.inline_immediate_test_strategy!` to execute scheduled jobs immediately."
+        raise Postburner::Job::PrematurePerform, "Job scheduled for #{job.run_at} (#{((job.run_at - Time.current) / 60).round(1)} minutes from now). Use `travel_to(job.run_at)` in your test, or set `Postburner.inline_immediate_test_strategy!` to execute scheduled jobs immediately."
       end
     end
   end

data/lib/postburner/time_helpers.rb

@@ -18,7 +18,8 @@ module Postburner
   #
   #   travel_to(2.days.from_now) do
   #     # Code here executes as if it's 2 days in the future
-  #     Time.zone.now  # => 2 days from now
+  #     Time.current   # => 2 days from now (preferred in Rails)
+  #     Time.zone.now  # => 2 days from now (equivalent)
   #   end
   #
   # @example In a class method
@@ -54,7 +55,8 @@ module Postburner
     #
     # @example Travel to specific time
     #   travel_to(Time.zone.parse('2025-12-25 00:00:00')) do
-    #     puts Time.zone.now  # => 2025-12-25 00:00:00
+    #     puts Time.current   # => 2025-12-25 00:00:00 (preferred)
+    #     puts Time.zone.now  # => 2025-12-25 00:00:00 (equivalent)
     #   end
     #
     # @example Travel relative to now

data/lib/postburner/tube.rb

@@ -31,7 +31,11 @@ module Postburner
     # Just pass the last known id to after for the next batch.
     #
     def jobs(count=20, limit: 1000, after: nil)
+      # Access raw hash to avoid beaneater FastStruct method definition issues in Ruby 3.4
       stats = @tube.stats
+      stats_hash = stats.instance_variable_get(:@hash) || {}
+      tube_name = stats_hash['name']
+
       jobs = Array.new
 
       min_known = (
@@ -42,7 +46,11 @@ module Postburner
 
       for i in min..max
         job = @tube.client.jobs.find(i)
-        jobs << job if job && stats[:name] == job.stats[:tube]
+        if job
+          job_stats = job.stats
+          job_stats_hash = job_stats.instance_variable_get(:@hash) || {}
+          jobs << job if job_stats_hash['tube'] == tube_name
+        end
         break if jobs.length >= count
       end
 

data/lib/postburner/version.rb

@@ -1,3 +1,3 @@
 module Postburner
-  VERSION = '1.0.0.pre.11'
+  VERSION = '1.0.0.pre.13'
 end