postburner 1.0.0.pre.2 → 1.0.0.pre.4

data/app/concerns/postburner/commands.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Concern providing Beanstalkd command methods for Postburner jobs.
+  #
+  # Provides methods for interacting with Beanstalkd queue operations such as
+  # deleting, kicking, and extending TTR. All methods handle connection retries
+  # and gracefully handle missing bkid (e.g., in test mode).
+  #
+  # @example Basic Beanstalkd operations
+  #   class MyJob < Postburner::Job
+  #     def perform(args)
+  #       # ... work ...
+  #       extend!  # Extend TTR during long operation
+  #     end
+  #   end
+  #
+  #   job.delete!  # Remove from Beanstalkd
+  #   job.kick!    # Retry buried job
+  #
+  module Commands
+    extend ActiveSupport::Concern
+
+    # Kicks a buried job back into the ready queue in Beanstalkd.
+    #
+    # This is a Beanstalkd operation used to retry jobs that were buried due to
+    # repeated failures or explicit burial. Does not modify the ActiveRecord model.
+    #
+    # Automatically retries with fresh connection if Beanstalkd connection is stale.
+    #
+    # @return [void]
+    #
+    # @raise [Beaneater::NotConnected] if Beanstalkd connection fails after retry
+    #
+    # @note Does nothing if job has no bkid (e.g., in test mode)
+    # @note Only works on buried jobs - see Beanstalkd documentation
+    #
+    # @example
+    #   job.kick!  # Moves buried job back to ready queue
+    #
+    # @see #delete!
+    # @see #bk
+    #
+    def kick!
+      return unless self.bk
+      begin
+        self.bk.kick
+      rescue Beaneater::NotConnected => e
+        self.bk!.kick
+      end
+    end
+
+    # Extends the job's time-to-run (TTR) in Beanstalkd.
+    #
+    # Calls touch on the Beanstalkd job, extending the TTR by the original
+    # TTR value. Use this during long-running operations to prevent the job
+    # from timing out.
+    #
+    # @return [void]
+    #
+    # @note Does nothing if job has no bkid (e.g., in test mode)
+    #
+    # @example Process large file line by line
+    #   def perform(args)
+    #     file = File.find(args['file_id'])
+    #     file.each_line do |line|
+    #       # ... process line ...
+    #       extend!  # Extend TTR to prevent timeout
+    #     end
+    #   end
+    #
+    # @see #bk
+    #
+    def extend!
+      return unless self.bk
+      begin
+        self.bk.touch
+      rescue Beaneater::NotConnected => e
+        self.bk!.touch
+      end
+    end
+
+    # Deletes the job from the Beanstalkd queue.
+    #
+    # This is a Beanstalkd operation that removes the job from the queue but
+    # does NOT destroy the ActiveRecord model. Use {#destroy} or {#remove!} to
+    # also update the database record.
+    #
+    # Automatically retries with fresh connection if Beanstalkd connection is stale.
+    # Called automatically by before_destroy callback when using {#destroy}.
+    #
+    # @return [void]
+    #
+    # @raise [Beaneater::NotConnected] if Beanstalkd connection fails after retry
+    #
+    # @note Does nothing if job has no bkid (e.g., in test mode)
+    # @note Does not modify ActiveRecord model - only affects Beanstalkd
+    #
+    # @example
+    #   job.delete!  # Removes from Beanstalkd queue only
+    #   job.reload   # Job still exists in database
+    #
+    # @see #remove!
+    # @see #destroy
+    # @see #bk
+    #
+    def delete!
+      return unless self.bk
+      begin
+        self.bk.delete
+      rescue Beaneater::NotConnected => e
+        self.bk!.delete
+      end
+    end
+
+    # Soft-deletes the job by removing from Beanstalkd and setting removed_at timestamp.
+    #
+    # Unlike {#destroy}, this preserves the job record in the database for audit trails
+    # while removing it from the Beanstalkd queue and marking it as removed.
+    #
+    # @return [void]
+    #
+    # @raise [Beaneater::NotConnected] if Beanstalkd connection fails
+    #
+    # @note Idempotent - does nothing if already removed
+    # @note Does not destroy ActiveRecord model - only soft deletes
+    #
+    # @example
+    #   job.remove!
+    #   job.removed_at  # => 2025-10-31 12:34:56 UTC
+    #   job.persisted?  # => true (still in database)
+    #
+    # @see #delete!
+    # @see #destroy
+    #
+    def remove!
+      return if self.removed_at
+      self.delete!
+      self.update_column(:removed_at, Time.zone.now)
+    end
+
+  end
+end

data/app/concerns/postburner/execution.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Concern providing job execution methods for Postburner jobs.
+  #
+  # Handles the full job execution lifecycle including validation, callbacks,
+  # timing tracking, error handling, and state management. Provides both the
+  # class-level entry point for workers and the instance-level execution logic.
+  #
+  # @example Direct execution
+  #   Postburner::Job.perform(job.id)
+  #
+  # @example Instance execution
+  #   job.perform!(job.args)
+  #
+  module Execution
+    extend ActiveSupport::Concern
+
+    class_methods do
+      # Executes a job by ID, delegating to the current queue strategy.
+      #
+      # Loads the job from database by ID and delegates execution to the current
+      # queue strategy's {handle_perform!} method. This provides a unified API
+      # for job execution regardless of strategy (async, test, or null).
+      #
+      # Called automatically by Backburner workers in production. Can also be
+      # called manually for test/null strategies to trigger execution.
+      #
+      # @param id [Integer] Job ID to execute
+      # @param _ [Hash] Unused Backburner metadata parameter
+      #
+      # @return [void]
+      #
+      # @example Backburner automatic execution (production)
+      #   # Jobs execute in tube: backburner.worker.queue.backburner-jobs
+      #   # Backburner calls: Postburner::Job.perform(job_id)
+      #
+      # @example Manual execution with NullQueue
+      #   Postburner.null_strategy!
+      #   job = MyJob.create!(args: {})
+      #   job.queue!(delay: 1.hour)
+      #   Postburner::Job.perform(job.id)  # Time travels and executes
+      #
+      # @note Strategy-aware: delegates to Postburner.queue_strategy.handle_perform!
+      # @note For NullQueue, automatically handles time travel for scheduled jobs
+      #
+      # @see #perform!
+      # @see Queue.handle_perform!
+      # @see NullQueue.handle_perform!
+      #
+      def perform(id, _={})
+        job = nil
+        begin
+          job = self.find(id)
+        rescue ActiveRecord::RecordNotFound => e
+          Rails.logger.warn <<-MSG
+[Postburner::Job] [#{id}] Not Found.
+          MSG
+        end
+        #job.perform!(job.args)
+        Postburner.queue_strategy.handle_perform!(job)
+      end
+    end
+
+    # Executes the job with full lifecycle management and error handling.
+    #
+    # This is the main execution method called by Backburner workers or test strategies.
+    # Performs validation checks, executes callbacks, calls the subclass {#perform} method,
+    # tracks timing and statistics, and handles errors.
+    #
+    # Execution flow:
+    # 1. Runs attempt callbacks (fires on every retry)
+    # 2. Updates attempting metadata (attempting_at, attempts, lag)
+    # 3. Validates job state (queued, not processed, not removed, not premature)
+    # 4. Runs processing callbacks
+    # 5. Calls subclass {#perform} method
+    # 6. Runs processed callbacks (only on success)
+    # 7. Updates completion metadata (processed_at, duration)
+    # 8. Logs and tracks any exceptions
+    #
+    # @param args [Hash] Arguments to pass to {#perform} method
+    #
+    # @return [void]
+    #
+    # @raise [Exception] Any exception raised by {#perform} is logged and re-raised
+    #
+    # @note Does not execute if queued_at is nil, in the future, already processed, or removed
+    # @note Premature execution (before run_at) is delegated to queue strategy
+    #
+    # @see #perform
+    # @see Postburner.queue_strategy
+    # @see Callbacks
+    #
+    def perform!(args={})
+      run_callbacks :attempt do
+        self.attempting
+
+        self.update_columns(
+          attempting_at:  self.attempting_at,
+          attempts:       self.attempts,
+          attempt_count:  self.attempts.length,
+          lag:            self.lag,
+          processing_at:  Time.zone.now,
+        )
+
+        begin
+          if self.queued_at.nil?
+            self.log! "Not Queued", level: :error
+            return
+          end
+
+          if self.queued_at > Time.zone.now
+            self.log! "Future Queued", level: :error
+            return
+          end
+
+          if self.processed_at.present?
+            self.log! "Already Processed", level: :error
+            self.delete!
+            return
+          end
+
+          if self.removed_at.present?
+            self.log! "Removed", level: :error
+            return
+          end
+
+          if self.run_at && self.run_at.to_i > Time.zone.now.to_i
+            Postburner.queue_strategy.handle_premature_perform(self)
+            return
+          end
+
+          self.log!("START (bkid #{self.bkid})")
+
+          run_callbacks :processing do
+            begin
+              self.perform(args)
+            rescue Exception => exception
+              self.persist_metadata!
+              self.log! '[Postburner] Exception raised during perform prevented completion.'
+              raise exception
+            end
+          end
+
+          self.log!("DONE (bkid #{self.bkid})")
+
+          begin
+            now = Time.zone.now
+            _duration =  (now - self.processing_at) * 1000 rescue nil
+
+            run_callbacks :processed do
+              persist_metadata!(
+                processed_at: now,
+                duration:     _duration,
+              )
+            end
+          rescue Exception => e
+            self.log_exception!(e)
+            self.log! '[Postburner] Could not set data after processing.'
+            # TODO README doesn't retry if Postburner is to blame
+          end
+
+        rescue Exception => exception
+          self.log_exception!(exception)
+          raise exception
+        end
+      end # run_callbacks :attempt
+
+    end
+
+    private
+
+    # Records an attempt and calculates execution lag.
+    #
+    # Appends current time to attempts array, sets attempting_at on first attempt,
+    # and calculates lag (delay between intended and actual execution time).
+    #
+    # @return [Time] Current timestamp
+    #
+    # @api private
+    #
+    def attempting
+      now = Time.zone.now
+      self.attempts << now
+      self.attempting_at ||= now
+      self.lag ||= (self.attempting_at - self.intended_at) * 1000 rescue nil
+      now
+    end
+  end
+end

data/app/concerns/postburner/insertion.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Concern providing queue insertion methods for Postburner jobs.
+  #
+  # Handles enqueuing jobs to Beanstalkd (or test strategies), including
+  # scheduling, re-queueing, and the internal insertion mechanics via
+  # after_save_commit callbacks.
+  #
+  # @example Basic queueing
+  #   class MyJob < Postburner::Job
+  #     def perform(args)
+  #       # ... work ...
+  #     end
+  #   end
+  #
+  #   job = MyJob.create!(args: { foo: 'bar' })
+  #   job.queue!
+  #
+  # @example Scheduled queueing
+  #   job.queue!(delay: 1.hour)
+  #   job.queue!(at: 2.days.from_now)
+  #
+  module Insertion
+    extend ActiveSupport::Concern
+
+    # Enqueues the job to Beanstalkd for processing.
+    #
+    # Sets queued_at timestamp and optionally run_at for scheduled execution.
+    # Triggers enqueue callbacks and inserts job into Beanstalkd via after_save_commit hook.
+    # In test mode, executes immediately instead of queueing to Beanstalkd.
+    #
+    # @param options [Hash] Queue options
+    # @option options [Time, ActiveSupport::Duration] :at Absolute time to run the job
+    # @option options [Integer, ActiveSupport::Duration] :delay Seconds to delay execution
+    # @option options [Integer] :pri Beanstalkd priority (lower = higher priority)
+    # @option options [Integer] :ttr Time-to-run in seconds before job times out
+    #
+    # @return [void]
+    #
+    # @raise [ActiveRecord::RecordInvalid] if job is not valid
+    # @raise [AlreadyProcessed] if job was already processed
+    #
+    # @example Queue immediately
+    #   job.queue!
+    #
+    # @example Queue with delay
+    #   job.queue!(delay: 1.hour)
+    #
+    # @example Queue at specific time
+    #   job.queue!(at: Time.zone.now + 2.days)
+    #
+    # @example Queue with priority
+    #   job.queue!(pri: 0, delay: 30.minutes)
+    #
+    # @see #requeue!
+    # @see Postburner.queue_strategy
+    #
+    def queue!(options={})
+      return if self.queued_at.present? && self.bkid.present?
+      raise ActiveRecord::RecordInvalid, "Can't queue unless valid." unless self.valid?
+      raise AlreadyProcessed, "Processed at #{self.processed_at}" if self.processed_at
+
+      at = options.delete(:at)
+      now = Time.zone.now
+
+      self.queued_at = now
+      self.run_at = case
+                    when at.present?
+                      # this is rudimentary, add error handling
+                      options[:delay] ||= at.to_i - now.to_i
+                      at
+                    when options[:delay].present?
+                      now + options[:delay].seconds
+                    end
+
+      @_insert_options = options
+
+      run_callbacks :enqueue do
+        self.save!
+      end
+    end
+
+    # Re-queues an existing job by removing it from Beanstalkd and queueing again.
+    #
+    # Calls {#delete!} to remove from Beanstalkd, resets queuing metadata,
+    # then calls {#queue!} with new options.
+    #
+    # @param options [Hash] Queue options (same as {#queue!})
+    # @option options [Time, ActiveSupport::Duration] :at Absolute time to run the job
+    # @option options [Integer, ActiveSupport::Duration] :delay Seconds to delay execution
+    # @option options [Integer] :pri Beanstalkd priority
+    # @option options [Integer] :ttr Time-to-run in seconds
+    #
+    # @return [void]
+    #
+    # @raise [ActiveRecord::RecordInvalid] if job is not valid
+    # @raise [Beaneater::NotConnected] if Beanstalkd connection fails
+    #
+    # @example Requeue with different delay
+    #   job.requeue!(delay: 5.minutes)
+    #
+    # @see #queue!
+    # @see #delete!
+    #
+    def requeue!(options={})
+      self.delete!
+      self.bkid, self.queued_at = nil, nil
+
+      self.queue! options
+    end
+
+    # Checks if job is flagged for insertion into Beanstalkd.
+    #
+    # Set internally by {#queue!} to trigger insertion via after_save_commit hook.
+    #
+    # @return [Boolean] true if job will be inserted on save
+    # @api private
+    #
+    def will_insert?
+      @_insert_options.is_a? Hash
+    end
+
+    private
+
+    # After-save-commit callback that inserts job into queue if flagged.
+    #
+    # Checks {#will_insert?} and calls {#insert!} if true. Triggered by
+    # after_save_commit callback after {#queue!} saves the record.
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def insert_if_queued!
+      #debugger
+      return unless self.will_insert?
+      insert!(@_insert_options)
+    end
+
+    # Inserts job into queue via current queue strategy.
+    #
+    # Delegates to Postburner.queue_strategy.insert which handles actual
+    # queueing (Beanstalkd in production, inline execution in test mode).
+    # Updates bkid if the strategy returns a Beanstalkd job ID.
+    #
+    # @param options [Hash] Queue options (delay, pri, ttr, etc.)
+    #
+    # @return [Hash, nil] Queue strategy response
+    #
+    # @api private
+    #
+    def insert!(options={})
+      response = Postburner.queue_strategy.insert(self, options)
+      #debugger
+
+      # Response must be a hash with an :id key (value can be nil)
+      # Backburner returns symbol keys
+      unless response.is_a?(Hash) && response.key?(:id)
+        raise MalformedResponse, "Missing :id key in response: #{response.inspect}"
+      end
+
+      persist_metadata!(bkid: response[:id])
+
+      self.log("QUEUED: #{response}") if response
+
+      response
+    end
+  end
+end

data/app/concerns/postburner/logging.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Concern providing logging and exception tracking for Postburner jobs.
+  #
+  # Provides methods for adding timestamped log entries and tracking exceptions
+  # in the job's PostgreSQL audit trail. All logs are stored in JSONB arrays
+  # with timestamps, levels, and elapsed time tracking.
+  #
+  # @example Basic logging
+  #   class MyJob < Postburner::Job
+  #     def perform(args)
+  #       log "Starting processing"
+  #       # ... work ...
+  #       log! "Critical checkpoint reached"
+  #     end
+  #   end
+  #
+  # @example Exception tracking
+  #   class MyJob < Postburner::Job
+  #     def perform(args)
+  #       process_data(args)
+  #     rescue => e
+  #       log_exception!(e)
+  #       raise
+  #     end
+  #   end
+  #
+  module Logging
+    extend ActiveSupport::Concern
+
+    included do
+      # Valid log levels in order of severity
+      LOG_LEVELS = [
+        :debug,
+        :info,
+        :warning,
+        :error
+      ].freeze
+    end
+
+    # Tracks an exception in the job's errata array.
+    #
+    # Appends exception details to the in-memory errata array with timestamp,
+    # class, message, and backtrace. Does NOT persist to database immediately.
+    # Use {#log_exception!} to persist immediately.
+    #
+    # @param exception [Exception] The exception to track
+    #
+    # @return [Array<Array>] Updated errata array
+    #
+    # @example
+    #   begin
+    #     # ... risky operation ...
+    #   rescue => e
+    #     log_exception(e)
+    #     raise
+    #   end
+    #
+    # @see #log_exception!
+    #
+    def log_exception(exception)
+      self.errata << [
+        Time.zone.now,
+        {
+          bkid:       self.bkid,
+          class:      exception.class,
+          message:    exception.message,
+          backtrace:  exception.backtrace,
+        }
+      ]
+    end
+
+    # Tracks an exception and immediately persists to database.
+    #
+    # Calls {#log_exception} to append exception details, then persists
+    # both errata and error_count to database via {#persist_metadata!}.
+    #
+    # @param exception [Exception] The exception to track
+    #
+    # @return [void]
+    #
+    # @example
+    #   begin
+    #     # ... risky operation ...
+    #   rescue => e
+    #     log_exception!(e)
+    #     raise
+    #   end
+    #
+    # @see #log_exception
+    #
+    def log_exception!(exception)
+      self.log_exception(exception)
+      self.persist_metadata!
+    end
+
+    # Appends a log message to the job's logs array.
+    #
+    # Adds timestamped log entry to in-memory logs array with bkid, level,
+    # message, and elapsed time. Does NOT persist to database immediately.
+    # Use {#log!} to persist immediately.
+    #
+    # @param message [String] Log message to append
+    # @param options [Hash] Log options
+    # @option options [Symbol] :level Log level (:debug, :info, :warning, :error) - defaults to :info
+    #
+    # @return [Array<Array>] Updated logs array
+    #
+    # @note Invalid log levels are coerced to :error
+    #
+    # @example
+    #   log "Processing started"
+    #   log "Warning: rate limit approaching", level: :warning
+    #   log "Critical error occurred", level: :error
+    #
+    # @see #log!
+    # @see #elapsed_ms
+    # @see LOG_LEVELS
+    #
+    def log(message, options={})
+      options[:level] ||= :info
+      options[:level] = :error unless LOG_LEVELS.member?(options[:level])
+
+      self.logs << [
+        Time.zone.now, # time
+        {
+          bkid:     self.bkid,
+          level:    options[:level], # level
+          message:  message, # message
+          elapsed:  self.elapsed_ms, # ms from start
+        }
+      ]
+    end
+
+    # Appends a log message and immediately persists to database.
+    #
+    # Calls {#log} to append message, then persists logs array to database.
+    # Use this for important log messages that should be saved immediately.
+    #
+    # @param message [String] Log message to append
+    # @param options [Hash] Log options
+    # @option options [Symbol] :level Log level (:debug, :info, :warning, :error)
+    #
+    # @return [void]
+    #
+    # @example
+    #   log! "Job started"
+    #   log! "Payment processed successfully", level: :info
+    #   log! "Retrying failed request", level: :warning
+    #
+    # @see #log
+    #
+    def log!(message, options={})
+      self.log(message, options)
+      self.update_column :logs, self.logs
+    end
+
+    private
+
+    # Persists job metadata (errata, logs, and counts) to database.
+    #
+    # Updates errata, error_count, logs, and log_count columns, plus any
+    # additional data passed in. Used internally for atomic metadata updates.
+    #
+    # @param data [Hash] Additional columns to update
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def persist_metadata!(data={})
+      self.update_columns({
+        errata:       self.errata,
+        error_count:  self.errata.length,
+        logs:         self.logs,
+        log_count:    self.logs.length,
+      }.merge(data))
+    end
+  end
+end