postburner 1.0.0.pre.11 → 1.0.0.pre.12

data/app/models/postburner/schedule_execution.rb

@@ -0,0 +1,353 @@
+# frozen_string_literal: true
+
+module Postburner
+  # ScheduleExecution represents a single scheduled run of a recurring job.
+  #
+  # Executions are created in advance and immediately enqueued to Beanstalkd's delayed
+  # queue. This ensures all future runs are already queued and ready to execute at the
+  # scheduled time, regardless of scheduler availability.
+  #
+  # ## Lifecycle
+  #
+  # 1. Execution created with `pending` status and immediately enqueued to Beanstalkd
+  # 2. Status changes to `scheduled` once enqueued
+  # 3. At `run_at` time, Beanstalkd releases job to worker
+  # 4. For Postburner::Job schedules: `before_attempt` callback creates next execution
+  # 5. Job completes (tracked in job record, not execution)
+  # 6. Watchdog periodically verifies future executions exist (safety net)
+  #
+  # ## Status Values
+  #
+  # - `pending` - Created but not yet enqueued to Beanstalkd
+  # - `scheduled` - Enqueued to Beanstalkd, waiting for execution
+  # - `skipped` - Cancelled by admin before execution
+  #
+  # Note: We don't track running/completed/failed states here because the
+  # actual job (Postburner::Job or ActiveJob) handles its own lifecycle.
+  # The execution's job is done once it's scheduled.
+  #
+  # ## Database Fields
+  #
+  # - `schedule_id` - Foreign key to parent schedule
+  # - `run_at` - Scheduled execution time
+  # - `next_run_at` - Pre-calculated next run time for validation
+  # - `enqueued_at` - When job was queued to Beanstalkd
+  # - `status` - Current status (pending, scheduled, skipped)
+  # - `beanstalk_job_id` - Beanstalkd job ID (for tracking/cancellation)
+  # - `job_id` - Postburner::Job ID (if using Postburner::Job subclass)
+  # - `cached_schedule` - JSONB snapshot of schedule at creation time
+  #
+  # ## Immediate Enqueue Architecture
+  #
+  # Unlike traditional schedulers that poll for due jobs, Postburner immediately
+  # enqueues executions to Beanstalkd's delayed queue when created. This means:
+  # - Jobs are already queued and will run at run_at regardless of scheduler status
+  # - No polling overhead - Beanstalkd handles delayed delivery
+  # - Watchdog only acts as safety net to ensure future executions exist
+  #
+  # @example Viewing execution details
+  #   execution = schedule.executions.last
+  #   execution.run_at        # => 2025-12-29 09:00:00 UTC
+  #   execution.enqueued_at   # => 2025-12-28 10:00:00 UTC
+  #   execution.status        # => "scheduled"
+  #   execution.beanstalk_job_id  # => 12345
+  #
+  # @example Skipping a future execution
+  #   execution = schedule.executions.future.first
+  #   execution.skip!  # Cancels Beanstalkd job and marks as skipped
+  #
+  # @see Postburner::Schedule
+  # @see Postburner::Scheduler
+  #
+  class ScheduleExecution < ApplicationRecord
+    # Status enum - simplified to only what ScheduleExecution needs to know
+    enum :status, {
+      pending: 0,    # Created, waiting to be enqueued
+      scheduled: 11,  # Enqueued to Beanstalkd
+      skipped: 101     # Cancelled before execution
+    }
+
+    # Associations
+    belongs_to :schedule
+    belongs_to :job, optional: true
+
+    # Validations
+    validates :run_at, presence: true, uniqueness: { scope: :schedule_id }
+    validates :status, presence: true
+    validate :validate_next_run_at!
+
+    # Scopes
+    scope :due, -> { pending.where('run_at <= ?', Time.current).where(enqueued_at: nil) }
+    scope :future, -> { where('run_at > ?', Time.current).order(run_at: :asc) }
+    scope :past, -> { where('run_at <= ?', Time.current).order(run_at: :desc) }
+
+    # Check if execution has been enqueued to Beanstalkd.
+    #
+    # @return [Boolean] true if execution has beanstalk_job_id and enqueued_at set
+    def enqueued?
+      beanstalk_job_id.present? && enqueued_at.present?
+    end
+
+    # Enqueue this execution to Beanstalkd.
+    #
+    # Creates the appropriate job (ActiveJob or Postburner::Job) and queues it
+    # to Beanstalkd at the scheduled run_at time with appropriate delay.
+    #
+    # The job type is determined by the job_class in cached_schedule:
+    # - Postburner::Job subclasses: Creates tracked job with database record
+    # - ActiveJob classes: Enqueues directly without database tracking
+    #
+    # This method is idempotent - calling it multiple times will only enqueue once.
+    #
+    # @return [void]
+    #
+    # @note Creating the next execution is the scheduler's responsibility via
+    #   Schedule#create_next_execution!, not this method's responsibility.
+    #
+    # @example Enqueue an execution
+    #   execution = schedule.executions.build(run_at: 1.hour.from_now)
+    #   execution.save!
+    #   execution.enqueue!  # Creates and queues job to Beanstalkd
+    #
+    def enqueue!
+      return if enqueued?
+
+      # Mark as scheduled BEFORE Beanstalkd operations to prevent duplicates.
+      # If Beanstalkd put fails, we have a scheduled execution with no beanstalk_job_id,
+      # which the watchdog can detect and re-queue. This is safer than having a
+      # pending execution with a job already in Beanstalkd (which causes duplicates).
+      transaction do
+        update_columns(
+          enqueued_at: Time.current,
+          status: self.class.statuses[:scheduled]
+        )
+
+        if tracked_job?
+          enqueue_tracked_job!
+        else
+          enqueue_default_job!
+        end
+      end
+    end
+
+    # Skip this execution.
+    #
+    # Cancels the Beanstalkd job if already enqueued and marks the execution
+    # as skipped. Use this when an admin wants to cancel a scheduled execution.
+    #
+    # This is a destructive operation - skipped executions cannot be unskipped.
+    # A new execution must be created if needed.
+    #
+    # @return [Boolean] true if skipped successfully, false if already skipped
+    #
+    # @example Skip a future execution
+    #   execution = schedule.executions.future.first
+    #   execution.skip!  # Cancels Beanstalkd job and marks as skipped
+    #
+    def skip!
+      return false if skipped?
+
+      transaction do
+        # Cancel the Beanstalkd job if it exists
+        if beanstalk_job_id.present?
+          cancel_beanstalk_job!
+        end
+
+        update!(status: :skipped)
+      end
+
+      true
+    end
+
+    # Validate that the next execution matches the cached next_run_at.
+    #
+    # Similar to Flex::SubscriptionIteration validation. Ensures that the
+    # pre-calculated next_run_at in this execution matches the actual run_at
+    # of the next execution created.
+    #
+    # This validation helps detect calculation bugs or manual tampering with
+    # execution records.
+    #
+    # @param next_execution [ScheduleExecution] The next execution to validate
+    # @return [Boolean] true if validation passes or next_run_at is nil
+    # @raise [NextExecutionRunAtConflict] if times don't match
+    #
+    # @example Validate next execution
+    #   current = schedule.executions.first
+    #   next_exec = schedule.executions.second
+    #   current.validate_next_execution!(next_exec)  # raises if mismatch
+    #
+    def validate_next_execution!(next_execution)
+      return true if next_run_at.nil?
+
+      # Check if the run_at matches down to the second
+      if next_execution.run_at.to_i == Time.parse(next_run_at.to_s).to_i
+        return true
+      end
+
+      raise NextExecutionRunAtConflict,
+        "ScheduleExecution #{id} has next_run_at #{next_run_at} but next execution has run_at #{next_execution.run_at}"
+    end
+
+    private
+
+    # Check if this job is a Postburner::Job subclass (always tracked).
+    #
+    # @return [Boolean] true if job_class is a Postburner::Job subclass
+    #
+    # @api private
+    #
+    def tracked_job?
+      klass = cached_schedule['job_class'].constantize
+      klass < Postburner::Job
+    rescue NameError
+      false
+    end
+
+    # Check if this job is an ActiveJob that includes Postburner::Tracked.
+    #
+    # @return [Boolean] true if job_class includes Postburner::Tracked
+    #
+    # @api private
+    #
+    def tracked_activejob?
+      klass = cached_schedule['job_class'].constantize
+      klass.included_modules.include?(Postburner::Tracked)
+    rescue NameError
+      false
+    end
+
+    # Enqueue a tracked Postburner::Job.
+    #
+    # Creates a Postburner::Job record, configures it with cached schedule
+    # attributes, and enqueues it to Beanstalkd with the scheduled run_at time.
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def enqueue_tracked_job!
+      klass = cached_schedule['job_class'].constantize
+      job = klass.create!(args: cached_schedule['args'] || {})
+
+      # Set optional configuration
+      job.priority = cached_schedule['priority'] if cached_schedule['priority']
+      job.queue_name = cached_schedule['queue'] if cached_schedule['queue']
+
+      # Link execution to job BEFORE queueing so the schedule_next_execution
+      # callback can find the schedule_execution association during perform!
+      update_columns(job_id: job.id)
+
+      # Queue the job
+      bkid = job.queue!(at: run_at)
+
+      # Update beanstalk_job_id after queue
+      update_columns(beanstalk_job_id: bkid)
+    end
+
+    # Enqueue a default ActiveJob.
+    #
+    # Creates and enqueues an ActiveJob instance with the scheduled run_at time.
+    # Handles both Hash args (passed as keyword arguments) and Array args (splatted).
+    #
+    # For tracked ActiveJobs (those including Postburner::Tracked), the adapter
+    # creates a TrackedJob record. We link that record back to this execution
+    # to enable the schedule_next_execution callback.
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def enqueue_default_job!
+      klass = cached_schedule['job_class'].constantize
+      args = cached_schedule['args']
+      is_tracked = tracked_activejob?
+
+      # Build job with configuration
+      # Hash args are passed as keyword arguments, Array args are splatted
+      job = if args.is_a?(Hash) && args.any?
+              klass.new(**args.symbolize_keys)
+            elsif args.is_a?(Array)
+              klass.new(*args)
+            else
+              klass.new
+            end
+      job.queue_name = cached_schedule['queue'] if cached_schedule['queue']
+      job.priority = cached_schedule['priority'] if cached_schedule['priority']
+
+      # Enqueue with scheduled time (use instance method enqueue, not class method perform_later)
+      delay = (run_at - Time.current).to_i
+      delay = 0 if delay < 0
+
+      if delay > 0
+        job.enqueue(wait: delay)
+      else
+        job.enqueue
+      end
+
+      # For tracked ActiveJobs, link to the TrackedJob created by the adapter
+      if is_tracked
+        tracked_job = Postburner::TrackedJob.where(
+          "args->>'job_id' = ?", job.job_id
+        ).order(created_at: :desc).first
+
+        if tracked_job
+          update_columns(
+            job_id: tracked_job.id,
+            beanstalk_job_id: tracked_job.bkid
+          )
+        else
+          update_columns(beanstalk_job_id: nil)
+        end
+      else
+        update_columns(beanstalk_job_id: nil)
+      end
+    end
+
+    # Cancel the Beanstalkd job.
+    #
+    # Uses Postburner::Job's API for tracked jobs, falls back to direct
+    # Beanstalkd deletion for ActiveJobs. Handles NotFoundError gracefully
+    # (job may have already been processed or deleted).
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def cancel_beanstalk_job!
+      if job_id.present? && job.present?
+        # Tracked job - use Job's delete! method
+        job.delete!
+      elsif beanstalk_job_id.present?
+        # ActiveJob - delete directly from Beanstalkd
+        Postburner.connected do |conn|
+          begin
+            bk_job = conn.beanstalk.jobs.find(beanstalk_job_id)
+            bk_job&.delete
+          rescue Beaneater::NotFoundError
+            # Job already processed or deleted - that's fine
+          end
+        end
+      end
+    end
+
+    # Validate next_run_at is after run_at.
+    #
+    # Ensures that if next_run_at is set, it's chronologically after run_at.
+    # This catches calculation bugs where next_run_at is incorrectly set.
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def validate_next_run_at!
+      return if next_run_at.nil?
+
+      if next_run_at <= run_at
+        errors.add(:next_run_at, 'must be after run_at')
+      end
+    end
+
+    class NextExecutionRunAtConflict < StandardError; end
+  end
+end

data/app/views/postburner/jobs/show.html.haml

@@ -1,9 +1,9 @@
 %h1 Job #{@job.id} / #{@job.bkid} / #{@job.sid}
 
-- if @job.run_at && @job.run_at > Time.zone.now
+- if @job.run_at && @job.run_at > Time.current
   Running in
-  = distance_of_time_in_words Time.zone.now, @job.run_at
-  (#{@job.run_at.to_i - Time.zone.now.to_i} seconds)
+  = distance_of_time_in_words Time.current, @job.run_at
+  (#{@job.run_at.to_i - Time.current.to_i} seconds)
 
 %h3 Stats
 - if @job.bk

data/lib/generators/postburner/install/install_generator.rb

@@ -4,6 +4,7 @@ class Postburner::InstallGenerator < Rails::Generators::Base
 
   def install_migrations
     install_migration! 'create_postburner_jobs'
+    install_migration! 'create_postburner_schedules'
   end
 
   def copy_config_file

data/lib/generators/postburner/install/templates/config/postburner.yml

@@ -28,6 +28,13 @@ default: &default
   # Override with ENV['BEANSTALK_URL'] if set
   beanstalk_url: <%= ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300' %>
 
+  # Scheduler configuration
+  # The scheduler runs as a lightweight "watchdog" job that checks for due
+  # schedule executions and enqueues them. All workers automatically watch
+  # the scheduler queue and will process the watchdog when it becomes due.
+  default_scheduler_interval: 300  # Check for due schedules every 5 minutes (300 seconds)
+  default_scheduler_priority: 100  # Scheduler watchdog priority (lower = higher priority)
+
 development:
   <<: *default
 
@@ -136,9 +143,11 @@ production:  # <- environment config, i.e. defaults, NOT worker config
 
 # Env-Level Defaults (can be overridden per worker):
 #
-# default_queue: default           # Default queue name (optional)
-# default_priority: 65536          # Lower = higher priority (optional, 0 is highest)
-# default_ttr: 300                 # Time-to-run in seconds (optional)
-# default_threads: 1               # Thread count per fork (optional, defaults to 1)
-# default_forks: 0                 # Fork count (optional, defaults to 0 = single process)
-# default_gc_limit: nil            # Exit after N jobs for restart (optional, nil = no limit)
+# default_queue: default                  # Default queue name (optional)
+# default_priority: 65536                 # Lower = higher priority (optional, 0 is highest)
+# default_ttr: 300                        # Time-to-run in seconds (optional)
+# default_threads: 1                      # Thread count per fork (optional, defaults to 1)
+# default_forks: 0                        # Fork count (optional, defaults to 0 = single process)
+# default_gc_limit: nil                   # Exit after N jobs for restart (optional, nil = no limit)
+# default_scheduler_interval: 300         # Scheduler check interval in seconds (optional, default: 300)
+# default_scheduler_priority: 100         # Scheduler watchdog priority (optional, default: 100)

data/lib/generators/postburner/install/templates/migrations/create_postburner_schedules.rb.erb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+#
+class CreatePostburnerSchedules < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :postburner_schedules do |t|
+      # Identity
+      t.string :name, null: false
+      t.string :job_class, null: false
+      t.jsonb :args, default: {}
+
+      # Configuration
+      t.string :queue
+      t.integer :priority
+      t.string :timezone, default: 'UTC', null: false
+
+      # Anchor-based scheduling (primary)
+      t.datetime :anchor
+      t.integer :interval
+      t.string :interval_unit
+
+      # Cron-based (power user, optional)
+      t.string :cron
+
+      # Control
+      t.boolean :enabled, default: true, null: false
+      t.boolean :catch_up, default: false, null: false
+      t.datetime :last_audit_at
+      t.text :description
+
+      t.timestamps
+
+      t.index :name, unique: true
+      t.index :job_class
+      t.index :enabled
+      t.index :last_audit_at
+      t.index :args, using: :gin
+    end
+
+    create_table :postburner_schedule_executions do |t|
+      # Relationship
+      t.bigint :schedule_id, null: false
+
+      # Timing
+      t.datetime :run_at, null: false
+      t.datetime :next_run_at
+      t.datetime :enqueued_at
+
+      # Status: 0=pending, 1=scheduled, 2=skipped
+      t.integer :status, default: 0, null: false
+
+      # Job tracking
+      t.bigint :beanstalk_job_id
+      t.bigint :job_id
+
+      # Audit
+      t.jsonb :cached_schedule, default: {}
+
+      t.timestamps
+
+      t.index :schedule_id
+      t.index [:schedule_id, :run_at], unique: true
+      t.index [:status, :run_at], name: 'index_postburner_schedule_executions_on_status_and_run_at'
+      t.index :beanstalk_job_id
+      t.index :job_id
+      t.index :run_at
+
+      t.foreign_key :postburner_schedules, column: :schedule_id, on_delete: :cascade
+      t.foreign_key :postburner_jobs, column: :job_id, on_delete: :nullify
+    end
+  end
+end

data/lib/postburner/active_job/adapter.rb

@@ -121,11 +121,11 @@ module ActiveJob
         tracked_job = Postburner::TrackedJob.create!(
           args: Postburner::ActiveJob::Payload.serialize_for_tracked(job),
           run_at: timestamp ? Time.zone.at(timestamp) : nil,
-          queued_at: Time.zone.now
+          queued_at: Time.current
         )
 
         # Calculate delay for Beanstalkd
-        delay = timestamp ? [(timestamp.to_f - Time.zone.now.to_f).to_i, 0].max : 0
+        delay = timestamp ? [timestamp.to_i - Time.current.to_i, 0].max : 0
 
         # Queue to Beanstalkd with minimal payload
         Postburner.connected do |conn|
@@ -160,7 +160,7 @@ module ActiveJob
       # @return [void]
       #
       def enqueue_default(job, timestamp)
-        delay = timestamp ? [(timestamp.to_f - Time.zone.now.to_f).to_i, 0].max : 0
+        delay = timestamp ? [timestamp.to_i - Time.current.to_i, 0].max : 0
 
         Postburner.connected do |conn|
           tube_name = expand_tube_name(job.queue_name)

data/lib/postburner/active_job/payload.rb

@@ -54,6 +54,10 @@ module Postburner
         # @api private
         #
         def for_job(job, tracked: false, postburner_job_id: nil)
+          # Get TTR from job class if available, otherwise use default
+          ttr = job.class.respond_to?(:postburner_ttr) && job.class.postburner_ttr ||
+                Postburner.configuration.default_ttr
+
           {
             v: VERSION,
             tracked: tracked,
@@ -62,6 +66,7 @@ module Postburner
             job_id: job.job_id,
             queue_name: job.queue_name,
             priority: job.priority,
+            ttr: ttr,
             arguments: ::ActiveJob::Arguments.serialize(job.arguments),
             executions: job.executions,
             exception_executions: job.exception_executions || {},

data/lib/postburner/advisory_lock.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Postburner
+  # PostgreSQL advisory lock support for coordinating distributed processes.
+  #
+  # Advisory locks allow multiple scheduler processes to coordinate without
+  # race conditions. Only one process can hold a specific lock at a time.
+  #
+  # @example Using with a block
+  #   Postburner::AdvisoryLock.with_lock('postburner_scheduler') do
+  #     # Only one process executes this block at a time
+  #     process_due_schedules
+  #   end
+  #
+  # @example Manual lock/unlock
+  #   lock = Postburner::AdvisoryLock.new('postburner_scheduler')
+  #   if lock.acquire
+  #     begin
+  #       process_due_schedules
+  #     ensure
+  #       lock.release
+  #     end
+  #   end
+  #
+  class AdvisoryLock
+    # Lock key for the scheduler process
+    SCHEDULER_LOCK_KEY = 'postburner_scheduler'
+
+    attr_reader :key, :connection
+
+    # Initialize a new advisory lock
+    #
+    # @param key [String] Unique lock identifier
+    # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] Database connection
+    def initialize(key, connection = nil)
+      @key = key
+      @connection = connection || ActiveRecord::Base.connection
+      @acquired = false
+    end
+
+    # Acquire the advisory lock (blocking)
+    #
+    # This will wait until the lock is available.
+    #
+    # @return [Boolean] true if lock acquired
+    def acquire
+      return true if @acquired
+
+      lock_id = generate_lock_id(key)
+      result = connection.execute("SELECT pg_advisory_lock(#{lock_id})")
+      @acquired = true
+    end
+
+    # Try to acquire the advisory lock (non-blocking)
+    #
+    # Returns immediately, indicating whether lock was acquired.
+    #
+    # @return [Boolean] true if lock acquired, false if already held
+    def try_acquire
+      return true if @acquired
+
+      lock_id = generate_lock_id(key)
+      result = connection.execute("SELECT pg_try_advisory_lock(#{lock_id})").first
+      value = result['pg_try_advisory_lock']
+      @acquired = value == true || value == 't'
+    end
+
+    # Release the advisory lock
+    #
+    # @return [Boolean] true if lock was held and released
+    def release
+      return false unless @acquired
+
+      lock_id = generate_lock_id(key)
+      result = connection.execute("SELECT pg_advisory_unlock(#{lock_id})").first
+      @acquired = false
+      value = result['pg_advisory_unlock']
+      value == true || value == 't'
+    end
+
+    # Check if this lock instance has acquired the lock
+    #
+    # @return [Boolean]
+    def acquired?
+      @acquired
+    end
+
+    # Execute a block with an advisory lock
+    #
+    # @param key [String] Lock identifier
+    # @param blocking [Boolean] If true, wait for lock. If false, return immediately if unavailable.
+    # @yield Block to execute while holding lock
+    # @return [Object, nil] Returns block result if lock acquired, nil otherwise
+    def self.with_lock(key, blocking: true)
+      lock = new(key)
+
+      acquired = blocking ? lock.acquire : lock.try_acquire
+      return nil unless acquired
+
+      begin
+        yield
+      ensure
+        lock.release
+      end
+    end
+
+    private
+
+    # Generate a numeric lock ID from a string key
+    #
+    # PostgreSQL advisory locks use bigint IDs. We generate a consistent
+    # numeric ID from the string key using a hash function.
+    #
+    # @param key [String] Lock key
+    # @return [Integer] Numeric lock ID
+    def generate_lock_id(key)
+      # Use CRC32 to generate a consistent numeric ID from the string
+      # This gives us a 32-bit integer which is well within PostgreSQL's bigint range
+      require 'zlib'
+      Zlib.crc32(key)
+    end
+  end
+end