postburner 0.8.0 → 1.0.0.pre.1

data/lib/postburner.rb

@@ -1,17 +1,368 @@
 require "postburner/version"
+require "postburner/configuration"
+require "postburner/connection"
+require "postburner/queue_config"
+require "postburner/beanstalkd"
+require "postburner/tracked"
+require "postburner/active_job/payload"
+require "postburner/active_job/execution"
+require "postburner/active_job/adapter"
+require "postburner/workers/base"
+require "postburner/workers/worker"
 require "postburner/engine"
-require "postburner/tube"
+require "postburner/strategies/queue"
+require "postburner/strategies/nice_queue"
+require "postburner/strategies/test_queue"
+require "postburner/strategies/immediate_test_queue"
+require "postburner/strategies/null_queue"
 
+# Postburner - PostgreSQL-backed job queue system built on Backburner and Beanstalkd.
+#
+# Postburner is a Ruby on Rails Engine that provides a database-backed job queue with
+# full audit trails, inspection capabilities, and multiple execution strategies. Every
+# job is stored as an ActiveRecord model, enabling database queries, foreign key
+# relationships, and comprehensive statistics tracking.
+#
+# ## Core Concepts
+#
+# - **Jobs:** Subclass {Postburner::Job} and implement `perform` method
+# - **Queue Strategies:** Control how jobs are executed (async, inline, test modes)
+# - **Beanstalkd Integration:** Production queuing via Backburner
+# - **Database Persistence:** Full audit trail with timestamps, logs, and errors
+# - **Callbacks:** ActiveJob-style lifecycle hooks (enqueue, attempt, processing, processed)
+#
+# ## Queue Strategies
+#
+# Postburner uses a strategy pattern to control job execution:
+#
+# - **{NiceQueue}** (default): Async via Beanstalkd, gracefully handles premature execution
+# - **{Queue}**: Async via Beanstalkd, strict premature execution errors
+# - **{TestQueue}**: Inline/synchronous, requires explicit time travel for scheduled jobs
+# - **{ImmediateTestQueue}**: Inline/synchronous with automatic time travel
+# - **{NullQueue}**: Creates jobs without queueing, manual execution with time travel
+#
+# ## Auto-Detection
+#
+# Postburner automatically detects Rails test mode and switches to {TestQueue} when:
+# - `Rails.env.test?` is true
+# - `ActiveJob::Base.queue_adapter_name == :test`
+#
+# ## Usage
+#
+# @example Creating and queueing a job
+#   class ProcessPayment < Postburner::Job
+#     queue 'critical'
+#     queue_priority 0
+#
+#     def perform(args)
+#       payment = Payment.find(args['payment_id'])
+#       payment.process!
+#       log "Payment #{payment.id} processed"
+#     end
+#   end
+#
+#   job = ProcessPayment.create!(args: { 'payment_id' => 123 })
+#   job.queue!(delay: 1.hour)
+#
+# @example Switching queue strategies
+#   # Production with graceful premature handling (default)
+#   Postburner.nice_async_strategy!
+#
+#   # Production with strict premature errors
+#   Postburner.async_strategy!
+#
+#   # Test mode with explicit time control
+#   Postburner.inline_test_strategy!
+#
+#   # Test mode with automatic time travel
+#   Postburner.inline_immediate_test_strategy!
+#
+# @example Using Beanstalkd connection
+#   Postburner.connected do |conn|
+#     conn.tubes['my.tube'].stats
+#     conn.tubes['my.tube'].kick(3)
+#   end
+#
+# @example Checking if in test mode
+#   if Postburner.testing?
+#     # Jobs execute inline
+#   else
+#     # Jobs queued to Beanstalkd
+#   end
+#
+# @see Postburner::Job
+# @see NiceQueue
+# @see Queue
+# @see TestQueue
+# @see ImmediateTestQueue
+#
 module Postburner
+  # @!attribute [rw] queue_strategy
+  #   The current queue strategy class used for job execution.
+  #
+  #   Defaults to {NiceQueue} for production, auto-switches to {TestQueue}
+  #   in Rails test environment.
+  #
+  #   @return [Class] One of Queue, NiceQueue, TestQueue, or ImmediateTestQueue
+  #
+  #   @example
+  #     Postburner.queue_strategy
+  #     # => Postburner::NiceQueue
+  #
+  #   @see #nice_async_strategy!
+  #   @see #async_strategy!
+  #   @see #inline_test_strategy!
+  #   @see #inline_immediate_test_strategy!
+  #
+  mattr_accessor :queue_strategy
 
+  # Initialize with NiceQueue (production) by default
+  # NiceQueue re-inserts premature jobs with appropriate delay
+  self.queue_strategy = Postburner::NiceQueue
+
+  # Auto-detect Rails test mode and switch to TestQueue
+  if defined?(Rails) && Rails.env.test? && defined?(ActiveJob::Base)
+    if ActiveJob::Base.queue_adapter_name == :test
+      self.queue_strategy = Postburner::TestQueue
+    end
+  end
+
+  # Activates strict test mode with inline job execution.
+  #
+  # Sets queue strategy to {TestQueue}, which executes jobs synchronously
+  # without Beanstalkd. Scheduled jobs (with future run_at) raise
+  # {Postburner::Job::PrematurePerform} exception, forcing explicit time
+  # management with `travel_to`.
+  #
+  # Use this strategy when you want explicit control over time progression
+  # in tests and want to catch scheduling bugs by failing loudly.
+  #
+  # @return [void]
+  #
+  # @example
+  #   Postburner.inline_test_strategy!
+  #   job = MyJob.create!(args: {})
+  #   job.queue!  # Executes immediately
+  #   assert job.reload.processed_at
+  #
+  # @example With scheduled jobs
+  #   Postburner.inline_test_strategy!
+  #   job = MyJob.create!(args: {})
+  #   job.queue!(delay: 1.hour)
+  #   # Raises PrematurePerform - use travel_to instead
+  #
+  # @see TestQueue
+  # @see #inline_immediate_test_strategy!
+  #
+  def self.inline_test_strategy!
+    self.queue_strategy = Postburner::TestQueue
+  end
+
+  # Activates test mode with automatic time travel for scheduled jobs.
+  #
+  # Sets queue strategy to {ImmediateTestQueue}, which executes jobs
+  # synchronously and automatically uses time travel for jobs with future
+  # run_at timestamps.
+  #
+  # Use this strategy for integration/feature tests where you want
+  # convenience over explicit time control.
+  #
+  # @return [void]
+  #
+  # @example
+  #   Postburner.inline_immediate_test_strategy!
+  #   job = MyJob.create!(args: {})
+  #   job.queue!(delay: 1.hour)
+  #   # Automatically travels to scheduled time and executes
+  #   assert job.reload.processed_at
+  #
+  # @note Jobs execute in queue order, not scheduled time order
+  # @note Requires ActiveSupport::Testing::TimeHelpers
+  #
+  # @see ImmediateTestQueue
+  # @see #inline_test_strategy!
+  #
+  def self.inline_immediate_test_strategy!
+    self.queue_strategy = Postburner::ImmediateTestQueue
+  end
+
+  # Activates strict production mode with asynchronous job execution.
+  #
+  # Sets queue strategy to {Queue}, which queues jobs to Beanstalkd and
+  # raises {Postburner::Job::PrematurePerform} if a job is executed before
+  # its scheduled run_at time.
+  #
+  # Use this strategy for production debugging or when you want strict
+  # enforcement of scheduling. For most production use cases, prefer
+  # {#nice_async_strategy!} instead.
+  #
+  # @return [void]
+  #
+  # @example
+  #   Postburner.async_strategy!
+  #   job = MyJob.create!(args: {})
+  #   job.queue!(delay: 1.hour)
+  #   # Queued to Beanstalkd, will raise if picked up early
+  #
+  # @note Requires Beanstalkd server running
+  # @note NOT the default production strategy
+  #
+  # @see Queue
+  # @see #nice_async_strategy!
+  #
+  def self.async_strategy!
+    self.queue_strategy = Postburner::Queue
+  end
+
+  # Activates default production mode with graceful premature handling.
+  #
+  # Sets queue strategy to {NiceQueue} (the default), which queues jobs to
+  # Beanstalkd and gracefully handles premature execution by re-inserting
+  # jobs with appropriate delay.
+  #
+  # This is the recommended production strategy and is set by default on
+  # initialization.
+  #
+  # @return [void]
+  #
+  # @example
+  #   Postburner.nice_async_strategy!
+  #   job = MyJob.create!(args: {})
+  #   job.queue!(delay: 1.hour)
+  #   # Queued to Beanstalkd, automatically re-inserted if picked up early
+  #
+  # @note This is the DEFAULT strategy
+  # @note Requires Beanstalkd server running
+  #
+  # @see NiceQueue
+  # @see #async_strategy!
+  #
+  def self.nice_async_strategy!
+    self.queue_strategy = Postburner::NiceQueue
+  end
+
+  # Activates null mode for creating jobs without queueing to Beanstalkd.
+  #
+  # Sets queue strategy to {NullQueue}, which creates job records in the
+  # database but does NOT queue them to Beanstalkd. Jobs can be executed
+  # later by manually calling {NullQueue.handle_perform!}, which includes
+  # automatic time travel for scheduled jobs.
+  #
+  # Use this strategy for deferred batch processing, conditional execution,
+  # or scenarios where you want to create jobs in advance and execute them
+  # manually on-demand.
+  #
+  # @return [void]
+  #
+  # @example Create job without queueing
+  #   Postburner.null_strategy!
+  #   job = MyJob.create!(args: {})
+  #   job.queue!
+  #   # Job created but NOT queued to Beanstalkd
+  #   job.bkid  # => nil
+  #
+  # @example Manually execute later
+  #   Postburner.null_strategy!
+  #   job = MyJob.create!(args: {})
+  #   job.queue!(delay: 1.hour)
+  #   # Later, manually execute with automatic time travel
+  #   Postburner::Job.perform(job.id)
+  #   # Job executes as if it's 1 hour in the future
+  #
+  # @note Jobs execute when manually triggered via Postburner::Job.perform
+  # @note Does not require Beanstalkd to be running
+  # @note Useful for batch processing and deferred execution patterns
+  #
+  # @see NullQueue
+  # @see Postburner::Job.perform
+  #
+  def self.null_strategy!
+    self.queue_strategy = Postburner::NullQueue
+  end
+
+  # Checks if currently using a test queue strategy.
+  #
+  # Returns true if the current queue strategy is {TestQueue} or
+  # {ImmediateTestQueue}, indicating that jobs execute inline/synchronously
+  # without Beanstalkd.
+  #
+  # @return [Boolean] true if using test strategy, false if using production strategy
+  #
+  # @example
+  #   Postburner.nice_async_strategy!
+  #   Postburner.testing?  # => false
+  #
+  #   Postburner.inline_test_strategy!
+  #   Postburner.testing?  # => true
+  #
+  # @see TestQueue
+  # @see ImmediateTestQueue
+  #
+  def self.testing?
+    queue_strategy.testing
+  end
+
+  # Returns a cached Beanstalkd connection.
+  #
+  # Creates a new {Postburner::Connection} using the configured Beanstalkd
+  # URL and caches it. Automatically reconnects if the connection is stale.
+  #
+  # For most use cases, prefer {#connected} which handles connection cleanup.
+  #
+  # @return [Postburner::Connection] Beanstalkd connection object
+  #
+  # @raise [Beaneater::NotConnected] if connection fails
+  #
+  # @example
+  #   conn = Postburner.connection
+  #   conn.tubes['my.tube'].stats
+  #
+  # @note Connection is cached in @_connection instance variable
+  # @note Automatically reconnects if connection is not active
+  #
+  # @see #connected
+  #
   def self.connection
-    @_connection ||= Backburner::Connection.new(
-      Backburner.configuration.beanstalk_url
-    )
+    @_connection ||= Postburner::Connection.new
     @_connection.reconnect! unless @_connection.connected?
     @_connection
   end
 
+  # Yields a Beanstalkd connection or returns cached connection.
+  #
+  # When called with a block, yields the connection and ensures it's closed
+  # after the block completes. When called without a block, returns the
+  # cached connection.
+  #
+  # @overload connected
+  #   Returns the cached Beanstalkd connection.
+  #   @return [Postburner::Connection] Beanstalkd connection
+  #
+  # @overload connected {|conn| ... }
+  #   Yields connection and ensures cleanup.
+  #   @yieldparam conn [Postburner::Connection] Beanstalkd connection
+  #   @return [void]
+  #
+  # @example With block (recommended)
+  #   Postburner.connected do |conn|
+  #     conn.tubes.to_a.each do |tube|
+  #       puts tube.name
+  #     end
+  #   end
+  #   # Connection automatically closed
+  #
+  # @example Without block
+  #   conn = Postburner.connected
+  #   conn.tubes['my.tube'].stats
+  #
+  # @example Direct tube operations
+  #   Postburner.connected do |conn|
+  #     conn.tubes['critical'].kick(10)
+  #     conn.tubes['background'].stats
+  #   end
+  #
+  # @see #connection
+  #
   def self.connected
     if block_given?
       begin
@@ -24,9 +375,87 @@ module Postburner
     end
   end
 
+  # Removes all jobs from all tubes (not yet implemented).
+  #
+  # This is a destructive operation intended for development/testing cleanup.
+  # Requires confirmation string "CONFIRM" to prevent accidental execution.
+  #
+  # @param confirm [String] Must be exactly "CONFIRM" to execute
+  #
+  # @return [void]
+  #
+  # @example
+  #   Postburner.remove_all!("CONFIRM")
+  #
+  # @note Currently a no-op - implementation pending
+  # @todo Implement job removal from all tubes
+  #
   def self.remove_all!(confirm)
     return unless confirm == "CONFIRM"
 
     # TODO
   end
+
+  # Returns array of watched tube names with environment prefix.
+  #
+  # Expands configured queue names to full tube names and memoizes the result.
+  #
+  # @return [Array<String>] Array of expanded tube names
+  #
+  # @example
+  #   Postburner.watched_tube_names
+  #   # => ['postburner.production.default', 'postburner.production.critical']
+  #
+  def self.watched_tube_names
+    @__watched_tube_names ||= configuration.queue_names.map { |q| configuration.expand_tube_name(q) }
+  end
+
+  # Returns array of watched Beaneater::Tube instances.
+  #
+  # Creates Beaneater tube instances for all configured queues and memoizes the result.
+  #
+  # @return [Array<Beaneater::Tube>] Array of tube instances
+  #
+  # @example
+  #   Postburner.watched_tubes.each { |tube| puts tube.stats }
+  #
+  def self.watched_tubes
+    @__watched_tubes ||= watched_tube_names.map { |tube_name| connection.tubes[tube_name] }
+  end
+
+  # Returns statistics and introspection data about Beanstalkd and configured queues.
+  #
+  # Provides Beaneater tube instances for configured tubes and all tubes that exist
+  # on the Beanstalkd server. Tube instances support introspection methods:
+  #
+  # - tube.name - Tube name
+  # - tube.stats - Tube statistics hash (current-jobs-ready, current-jobs-buried, etc.)
+  # - tube.peek_ready - Next ready job
+  # - tube.peek_delayed - Next delayed job
+  # - tube.peek_buried - Next buried job
+  # - tube.kick(n) - Kick n buried jobs back to ready
+  # - tube.pause(delay) - Pause tube for delay seconds
+  # - tube.clear - Delete all jobs in tube
+  #
+  # @return [Hash] Statistics hash with the following keys:
+  #   - watched_tubes: Array of configured/watched Beaneater::Tube instances
+  #   - tubes: Array of all Beaneater::Tube instances on the server
+  #
+  # @raise [Beaneater::NotConnected] if connection to Beanstalkd fails
+  #
+  # @example
+  #   stats = Postburner.stats
+  #   stats[:watched_tubes].each { |tube| puts "#{tube.name}: #{tube.stats}" }
+  #   stats[:tubes].first.peek_ready
+  #
+  def self.stats
+    connected do |conn|
+
+      {
+        watched_tubes: self.watched_tubes,
+        # Get all tube instances that exist on Beanstalkd
+        tubes: conn.beanstalk.tubes.all
+      }
+    end
+  end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: postburner
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 1.0.0.pre.1
 platform: ruby
 authors:
 - Matt Smith
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-14 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -16,28 +15,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '7.2'
 - !ruby/object:Gem::Dependency
-  name: backburner
+  name: beaneater
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +51,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.3
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: haml-rails
   requirement: !ruby/object:Gem::Requirement
@@ -66,10 +79,27 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.1
-description: Queue background jobs, inspect them, and audit them afterwards.
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: 'Fast Beanstalkd-backed job queue with optional PostgreSQL audit trail.
+  Dual-mode execution: lite jobs (Beanstalkd only) or tracked jobs (full audit with
+  logs, timing, errors).'
 email:
 - matt@nearapogee.com
-executables: []
+executables:
+- postburner
 extensions: []
 extra_rdoc_files: []
 files:
@@ -79,6 +109,7 @@ files:
 - Rakefile
 - app/assets/config/postburner_manifest.js
 - app/assets/stylesheets/postburner/application.css
+- app/concerns/postburner/callbacks.rb
 - app/controllers/postburner/application_controller.rb
 - app/controllers/postburner/jobs_controller.rb
 - app/controllers/postburner/static_controller.rb
@@ -88,18 +119,38 @@ files:
 - app/models/postburner/application_record.rb
 - app/models/postburner/job.rb
 - app/models/postburner/mailer.rb
+- app/models/postburner/tracked_job.rb
 - app/views/layouts/postburner/application.html.haml
 - app/views/postburner/jobs/index.html.haml
 - app/views/postburner/jobs/show.html.haml
+- bin/postburner
+- bin/rails
 - config/environment.rb
+- config/postburner.yml
+- config/postburner.yml.example
 - config/routes.rb
 - lib/generators/postburner/install/USAGE
 - lib/generators/postburner/install/install_generator.rb
 - lib/generators/postburner/install/templates/migrations/create_postburner_jobs.rb.erb
 - lib/postburner.rb
+- lib/postburner/active_job/adapter.rb
+- lib/postburner/active_job/execution.rb
+- lib/postburner/active_job/payload.rb
+- lib/postburner/beanstalkd.rb
+- lib/postburner/configuration.rb
+- lib/postburner/connection.rb
 - lib/postburner/engine.rb
-- lib/postburner/tube.rb
+- lib/postburner/queue_config.rb
+- lib/postburner/strategies/immediate_test_queue.rb
+- lib/postburner/strategies/nice_queue.rb
+- lib/postburner/strategies/null_queue.rb
+- lib/postburner/strategies/queue.rb
+- lib/postburner/strategies/test_queue.rb
+- lib/postburner/time_helpers.rb
+- lib/postburner/tracked.rb
 - lib/postburner/version.rb
+- lib/postburner/workers/base.rb
+- lib/postburner/workers/worker.rb
 - lib/tasks/postburner_tasks.rake
 homepage: https://gitlab.nearapogee.com/opensource/postburner
 licenses:
@@ -108,7 +159,6 @@ metadata:
   homepage_uri: https://gitlab.nearapogee.com/opensource/postburner
   source_code_uri: https://gitlab.nearapogee.com/opensource/postburner
   changelog_uri: https://gitlab.nearapogee.com/opensource/postburner/-/blob/master/CHANGELOG.md
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -123,8 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
-summary: Postgres backed beanstalkd queue via backburner
+summary: PostgreSQL-backed job queue with optional audit trail for ActiveJob
 test_files: []

data/lib/postburner/tube.rb

@@ -1,53 +0,0 @@
-module Postburner
-  class Tube
-    def initialize(tube)
-      @tube = tube
-    end
-
-    # Get all tubes as Postburner::Tube instances.
-    #
-    def self.all
-      Postburner.connection.tubes.all.map { |tube| self.new(tube) }
-    end
-
-    # Get all peeked ids across all known tubes.
-    #
-    def self.peek_ids
-      self.all.map(&:peek_ids).flatten.sort
-    end
-
-    # Get all peeked ids.
-    #
-    def peek_ids
-      [ :buried, :ready, :delayed ].map { |type| @tube.peek(type) }.
-        reject(&:nil?).map(&:id).map(&:to_i)
-    end
-
-    # Get paginated array of jobs.
-    #
-    # Attempts to do this efficiently as possible, by peeking at known
-    # ids and exiting when count has been fulfilled.
-    #
-    # Just pass the last known id to after for the next batch.
-    #
-    def jobs(count=20, limit: 1000, after: nil)
-      stats = @tube.stats
-      jobs = Array.new
-
-      min_known = (
-        peek_ids.any? ? self.peek_ids.min : self.class.peek_ids.min
-      ).to_i
-      min = after ? after + 1 : min_known
-      max = min + limit
-
-      for i in min..max
-        job = @tube.client.jobs.find(i)
-        jobs << job if job && stats[:name] == job.stats[:tube]
-        break if jobs.length >= count
-      end
-
-      return jobs
-    end
-
-  end
-end