postburner 1.0.0.pre.1 → 1.0.0.pre.3

data/app/models/postburner/job.rb

@@ -12,7 +12,7 @@ module Postburner
   # @example Creating and queueing a job
   #   class ProcessDonation < Postburner::Job
   #     queue 'critical'
-  #     queue_priority 0
+  #     priority 0
   #
   #     def perform(args)
   #       donation = Donation.find(args['donation_id'])
@@ -75,7 +75,7 @@ module Postburner
     include Callbacks
 
     # Instance-level queue configuration (overrides class-level defaults)
-    attr_writer :queue_priority, :queue_ttr
+    attr_writer :priority, :ttr
 
     LOG_LEVELS = [
       :debug,
@@ -619,32 +619,32 @@ module Postburner
       self.class.queue
     end
 
-    # Returns the queue priority for this job instance.
+    # Returns the priority for this job instance.
     #
     # Checks instance-level override first, then falls back to class-level configuration.
     #
     # @return [Integer, nil] Priority (lower = higher priority)
     #
     # @example Instance-level override
-    #   job = MyJob.create!(args: {}, queue_priority: 1500)
-    #   job.queue_priority  # => 1500
+    #   job = MyJob.create!(args: {}, priority: 1500)
+    #   job.priority  # => 1500
     #
-    def queue_priority
-      @queue_priority || self.class.queue_priority
+    def priority
+      @priority || self.class.priority
     end
 
-    # Returns the queue TTR (time-to-run) for this job instance.
+    # Returns the TTR (time-to-run) for this job instance.
     #
     # Checks instance-level override first, then falls back to class-level configuration.
     #
     # @return [Integer, nil] TTR in seconds
     #
     # @example Instance-level override
-    #   job = MyJob.create!(args: {}, queue_ttr: 600)
-    #   job.queue_ttr  # => 600
+    #   job = MyJob.create!(args: {}, ttr: 600)
+    #   job.ttr  # => 600
     #
-    def queue_ttr
-      @queue_ttr || self.class.queue_ttr
+    def ttr
+      @ttr || self.class.ttr
     end
 
     def tube_name

data/app/models/postburner/mailer.rb

@@ -27,7 +27,7 @@ module Postburner
     #
     def with(params={})
       self.args.merge!(
-        'params' => ActiveJob::Arguments.serialize(params)
+        'params' => ::ActiveJob::Arguments.serialize(params)
       )
       self
     end
@@ -62,7 +62,7 @@ module Postburner
     # Get the deserialized params.
     #
     def params
-      ActiveJob::Arguments.deserialize(self.args['params']).to_h
+      ::ActiveJob::Arguments.deserialize(self.args['params']).to_h
     end
 
     def perform(args)

data/config/postburner.yml.example

@@ -20,7 +20,7 @@
 # Scale by adjusting forks and threads per worker:
 # - Development: forks=0, threads=1 (single-threaded, easiest debugging)
 # - Staging: forks=0, threads=10 (multi-threaded, moderate load)
-# - Production: forks=4, threads=10 (40 concurrent jobs per queue)
+# - Production: forks=4, threads=10 (40 concurrent jobs per worker)
 #
 
 default: &default
@@ -34,7 +34,7 @@ development:
   workers:
     default:
       # Single-threaded, single process (simplest for debugging)
-      # Defaults: forks=0, threads=1, gc_limit=nil
+      # If not specified, uses env-level defaults
       queues:
         - default
         - mailers
@@ -45,34 +45,39 @@ test:
   workers:
     default:
       # Test mode uses inline strategies automatically
-      # Defaults: forks=0, threads=1, gc_limit=nil
       queues:
         - default
 
-staging:
+staging:  # <- environment config
   <<: *default
 
-  workers:
+  # Env-level defaults (use default_ prefix)
+  default_threads: 10
+  default_gc_limit: 5000
+
+  workers:  # <- worker config overrides
     default:
       # Multi-threaded, single process (moderate concurrency)
-      default_threads: 10
-      default_gc_limit: 5000
+      # Uses env-level defaults: default_threads=10, default_gc_limit=5000
       queues:
         - critical
         - default
         - mailers
 
-production:
+production:  # <- environment config, i.e. defaults, NOT worker config
   <<: *default
 
-  # Example 1: Single worker processing all queues with same settings
+  # Env-level defaults (use default_ prefix)
+  default_forks: 2
+  default_threads: 10
+  default_gc_limit: 5000
+
+  # Example 1: Single worker using env defaults
   # Run: bin/postburner
   #
   # workers:
   #   default:
-  #     default_forks: 4
-  #     default_threads: 10
-  #     default_gc_limit: 5000
+  #     # Uses default_forks=2, default_threads=10
   #     queues:
   #       - critical
   #       - default
@@ -84,21 +89,20 @@ production:
   #   bin/postburner --worker imports    (4 forks, 1 thread each)
   #   bin/postburner --worker general    (2 forks, 100 threads each)
   #
-  workers:
+  workers:  # <- worker config, i.e. overrides, NOT environment config
     # Heavy, memory-intensive jobs - more processes, fewer threads
     imports:
-      default_forks: 4
-      default_threads: 1
-      default_gc_limit: 500
+      forks: 4              # Overrides default_forks
+      threads: 1            # Overrides default_threads
+      gc_limit: 500         # Overrides default_gc_limit
       queues:
         - imports
         - data_processing
 
-    # General jobs - fewer processes, many threads
+    # General jobs - uses env defaults (forks=2, threads=10)
+    # Override threads for higher concurrency
     general:
-      default_forks: 2
-      default_threads: 100
-      default_gc_limit: 5000
+      threads: 100          # Overrides default_threads (forks uses default_forks=2)
       queues:
         - default
         - mailers
@@ -112,27 +116,25 @@ production:
   #
   # workers:
   #   critical:
-  #     default_forks: 1
-  #     default_threads: 1
-  #     default_gc_limit: 100
+  #     forks: 1
+  #     threads: 1
+  #     gc_limit: 100
   #     queues:
   #       - critical
   #
   #   default:
-  #     default_forks: 4
-  #     default_threads: 10
-  #     default_gc_limit: 5000
+  #     forks: 4
+  #     threads: 10
   #     queues:
   #       - default
   #
   #   mailers:
-  #     default_forks: 2
-  #     default_threads: 5
-  #     default_gc_limit: 2000
+  #     forks: 2
+  #     threads: 5
   #     queues:
   #       - mailers
 
-# Global Defaults (can be overridden per worker):
+# Env-Level Defaults (can be overridden per worker):
 #
 # default_queue: default           # Default queue name (optional)
 # default_priority: 65536          # Lower = higher priority (optional, 0 is highest)

data/lib/generators/postburner/install/install_generator.rb

@@ -6,6 +6,16 @@ class Postburner::InstallGenerator < Rails::Generators::Base
     install_migration! 'create_postburner_jobs'
   end
 
+  def copy_config_file
+    destination = "config/postburner.yml"
+
+    if File.exist?(File.join(destination_root, destination))
+      say_status "skip", destination, :yellow
+    else
+      template "config/postburner.yml", destination
+    end
+  end
+
   def self.next_migration_number(dirname)
     timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S")
     stem = timestamp[0..-3]

data/lib/generators/postburner/install/templates/config/postburner.yml

@@ -0,0 +1,144 @@
+# Postburner Configuration Example
+#
+# Copy this file to config/postburner.yml and customize for your environment.
+#
+# ## Named Workers Configuration
+#
+# Postburner uses named worker configurations to support different deployment patterns:
+# - Single worker: bin/postburner (auto-selects the single worker)
+# - Multiple workers: bin/postburner --worker <name> (must specify which worker)
+#
+# Each worker can have different fork/thread settings and process different queues.
+# This enables running different queue groups in separate OS processes with distinct
+# concurrency profiles.
+#
+# ## Puma-Style Architecture
+#
+# - **forks: 0** = Single process with thread pool (development/staging)
+# - **forks: 1+** = Multiple processes with thread pools (production)
+#
+# Scale by adjusting forks and threads per worker:
+# - Development: forks=0, threads=1 (single-threaded, easiest debugging)
+# - Staging: forks=0, threads=10 (multi-threaded, moderate load)
+# - Production: forks=4, threads=10 (40 concurrent jobs per worker)
+#
+
+default: &default
+  # Beanstalkd connection URL
+  # Override with ENV['BEANSTALK_URL'] if set
+  beanstalk_url: <%= ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300' %>
+
+development:
+  <<: *default
+
+  workers:
+    default:
+      # Single-threaded, single process (simplest for debugging)
+      # If not specified, uses env-level defaults
+      queues:
+        - default
+        - mailers
+
+test:
+  <<: *default
+
+  workers:
+    default:
+      # Test mode uses inline strategies automatically
+      queues:
+        - default
+
+staging:  # <- environment config
+  <<: *default
+
+  # Env-level defaults (use default_ prefix)
+  default_threads: 10
+  default_gc_limit: 5000
+
+  workers:  # <- worker config overrides
+    default:
+      # Multi-threaded, single process (moderate concurrency)
+      # Uses env-level defaults: default_threads=10, default_gc_limit=5000
+      queues:
+        - critical
+        - default
+        - mailers
+
+production:  # <- environment config, i.e. defaults, NOT worker config
+  <<: *default
+
+  # Env-level defaults (use default_ prefix)
+  default_forks: 2
+  default_threads: 10
+  default_gc_limit: 5000
+
+  # Example 1: Single worker using env defaults
+  # Run: bin/postburner
+  #
+  # workers:
+  #   default:
+  #     # Uses default_forks=2, default_threads=10
+  #     queues:
+  #       - critical
+  #       - default
+  #       - mailers
+  #       - imports
+
+  # Example 2: Multiple workers with different concurrency profiles
+  # Run separate processes:
+  #   bin/postburner --worker imports    (4 forks, 1 thread each)
+  #   bin/postburner --worker general    (2 forks, 100 threads each)
+  #
+  workers:  # <- worker config, i.e. overrides, NOT environment config
+    # Heavy, memory-intensive jobs - more processes, fewer threads
+    imports:
+      forks: 4              # Overrides default_forks
+      threads: 1            # Overrides default_threads
+      gc_limit: 500         # Overrides default_gc_limit
+      queues:
+        - imports
+        - data_processing
+
+    # General jobs - uses env defaults (forks=2, threads=10)
+    # Override threads for higher concurrency
+    general:
+      threads: 100          # Overrides default_threads (forks uses default_forks=2)
+      queues:
+        - default
+        - mailers
+        - notifications
+
+  # Example 3: Fine-grained control with multiple specialized workers
+  # Run separate processes:
+  #   bin/postburner --worker critical
+  #   bin/postburner --worker default
+  #   bin/postburner --worker mailers
+  #
+  # workers:
+  #   critical:
+  #     forks: 1
+  #     threads: 1
+  #     gc_limit: 100
+  #     queues:
+  #       - critical
+  #
+  #   default:
+  #     forks: 4
+  #     threads: 10
+  #     queues:
+  #       - default
+  #
+  #   mailers:
+  #     forks: 2
+  #     threads: 5
+  #     queues:
+  #       - mailers
+
+# Env-Level Defaults (can be overridden per worker):
+#
+# default_queue: default           # Default queue name (optional)
+# default_priority: 65536          # Lower = higher priority (optional, 0 is highest)
+# default_ttr: 300                 # Time-to-run in seconds (optional)
+# default_threads: 1               # Thread count per fork (optional, defaults to 1)
+# default_forks: 0                 # Fork count (optional, defaults to 0 = single process)
+# default_gc_limit: nil            # Exit after N jobs for restart (optional, nil = no limit)

data/lib/postburner/active_job/adapter.rb

@@ -58,6 +58,19 @@ module ActiveJob
         end
       end
 
+      # Indicates whether the adapter supports enqueuing after transaction commit.
+      #
+      # Returns true because Postburner uses Beanstalkd (external queue),
+      # which means jobs are safely enqueued outside the database transaction.
+      # This allows Rails to automatically defer job enqueuing until after
+      # the current database transaction commits.
+      #
+      # @return [Boolean] Always returns true
+      #
+      def enqueue_after_transaction_commit?
+        true
+      end
+
       private
 
       # Checks if a job should be tracked in PostgreSQL.
@@ -87,7 +100,7 @@ module ActiveJob
         # Create Postburner::TrackedJob record
         tracked_job = Postburner::TrackedJob.create!(
           args: Postburner::ActiveJob::Payload.serialize_for_tracked(job),
-          run_at: timestamp,
+          run_at: timestamp ? Time.zone.at(timestamp) : nil,
           queued_at: Time.zone.now
         )
 
@@ -98,10 +111,10 @@ module ActiveJob
         Postburner.connected do |conn|
           tube_name = expand_tube_name(job.queue_name)
 
-          # Get priority and TTR from class configuration or fall back to defaults
-          pri = job.class.respond_to?(:queue_priority) && job.class.queue_priority ||
-                Postburner.configuration.default_priority
-          ttr = job.class.respond_to?(:queue_ttr) && job.class.queue_ttr ||
+          # Get priority and TTR
+          # Priority order: job.priority (from .set or class.priority) > default
+          pri = job.priority || Postburner.configuration.default_priority
+          ttr = job.class.respond_to?(:postburner_ttr) && job.class.postburner_ttr ||
                 Postburner.configuration.default_ttr
 
           bkid = conn.tubes[tube_name].put(
@@ -132,10 +145,10 @@ module ActiveJob
         Postburner.connected do |conn|
           tube_name = expand_tube_name(job.queue_name)
 
-          # Get priority and TTR from class configuration or fall back to defaults
-          pri = job.class.respond_to?(:queue_priority) && job.class.queue_priority ||
-                Postburner.configuration.default_priority
-          ttr = job.class.respond_to?(:queue_ttr) && job.class.queue_ttr ||
+          # Get priority and TTR
+          # Priority order: job.priority (from .set or class.priority) > default
+          pri = job.priority || Postburner.configuration.default_priority
+          ttr = job.class.respond_to?(:postburner_ttr) && job.class.postburner_ttr ||
                 Postburner.configuration.default_ttr
 
           conn.tubes[tube_name].put(

data/lib/postburner/beanstalkd.rb

@@ -3,9 +3,9 @@
 module Postburner
   # Beanstalkd-specific configuration DSL for ActiveJob classes using Postburner.
   #
-  # Provides consistent API with Postburner::Job for setting Beanstalkd queue
-  # priority and TTR (time-to-run). Include this module in your ActiveJob classes
-  # to use Beanstalkd-specific configuration.
+  # Provides TTR (time-to-run) configuration for Beanstalkd. For priority, use
+  # ActiveJob's built-in `priority` method. Include this module in your ActiveJob
+  # classes to use Beanstalkd-specific configuration.
   #
   # @note Automatically included when you include Postburner::Tracked
   #
@@ -14,8 +14,8 @@ module Postburner
   #     include Postburner::Beanstalkd
   #
   #     queue_as :critical
-  #     queue_priority 0      # Highest priority
-  #     queue_ttr 300         # 5 minutes to complete
+  #     priority 0      # Highest priority (uses ActiveJob's priority)
+  #     ttr 300         # 5 minutes to complete
   #
   #     def perform(payment_id)
   #       # ...
@@ -26,8 +26,8 @@ module Postburner
   #   class ProcessPayment < ApplicationJob
   #     include Postburner::Tracked  # Includes Beanstalkd automatically
   #
-  #     queue_priority 0
-  #     queue_ttr 600
+  #     priority 0
+  #     ttr 600
   #
   #     def perform(payment_id)
   #       log "Processing payment"
@@ -39,54 +39,33 @@ module Postburner
       extend ActiveSupport::Concern
 
       included do
-        class_attribute :postburner_priority, default: nil
         class_attribute :postburner_ttr, default: nil
       end
 
       class_methods do
-        # Sets or returns the queue priority.
-        #
-        # Lower numbers = higher priority in Beanstalkd (0 is highest).
-        # Falls back to Postburner.configuration.default_priority if not set.
-        #
-        # @param pri [Integer, nil] Priority to set (0-4294967295), or nil to get current value
-        #
-        # @return [Integer, nil] Current priority when getting, nil when setting
-        #
-        # @example Set priority
-        #   queue_priority 0  # Highest priority
-        #
-        # @example Get priority
-        #   ProcessPayment.queue_priority  # => 0
-        #
-        def queue_priority(pri = nil)
-          if pri
-            self.postburner_priority = pri
-            nil
-          else
-            postburner_priority
-          end
-        end
-
-        # Sets or returns the queue TTR (time to run).
+        # Sets or returns the TTR (time to run).
         #
         # Number of seconds Beanstalkd will wait for job completion before
         # making it available again. Falls back to Postburner.configuration.default_ttr
         # if not set.
         #
-        # @param ttr [Integer, nil] Timeout in seconds, or nil to get current value
+        # From Beanstalkd:  If the worker does not delete, release, or bury the
+        #   job within <ttr> seconds, the job will time out and the server will
+        #   release the job.
+        #
+        # @param seconds [Integer, nil] Timeout in seconds, or nil to get current value
         #
         # @return [Integer, nil] Current TTR when getting, nil when setting
         #
         # @example Set TTR
-        #   queue_ttr 300  # 5 minutes
+        #   ttr 300  # 5 minutes
         #
         # @example Get TTR
-        #   ProcessPayment.queue_ttr  # => 300
+        #   ProcessPayment.ttr  # => 300
         #
-        def queue_ttr(ttr = nil)
-          if ttr
-            self.postburner_ttr = ttr
+        def ttr(seconds = nil)
+          if seconds
+            self.postburner_ttr = seconds
             nil
           else
             postburner_ttr

data/lib/postburner/configuration.rb

@@ -27,8 +27,8 @@ module Postburner
     # @option options [String] :default_queue Default queue name (default: 'default')
     # @option options [Integer] :default_priority Default job priority (default: 65536, lower = higher priority)
     # @option options [Integer] :default_ttr Default time-to-run in seconds (default: 300)
-    # @option options [Integer] :default_threads Default thread count per queue (default: 1)
-    # @option options [Integer] :default_forks Default fork count per queue (default: 0, single process)
+    # @option options [Integer] :default_threads Default thread count per fork (default: 1)
+    # @option options [Integer] :default_forks Default fork count (default: 0, single process)
     # @option options [Integer] :default_gc_limit Default GC limit for worker restarts (default: nil, no limit)
     #
     def initialize(options = {})
@@ -62,13 +62,36 @@ module Postburner
     # @example Multiple workers (must specify)
     #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production', 'imports')
     #
+    # @example config/postburner.yml example
+    #
+    #   default: &default
+    #     beanstalk_url: beanstalk://localhost:11300
+    #     default_priority: 131072 # change default priority from 65536 to 131072
+    #
+    #   production: # <- environment config, i.e. defaults, NOT worker config
+    #     <<: *default
+    #     default_forks: 2
+    #     default_threads: 10
+    #     default_gc_limit: 5000
+    #     default_ttr: 300
+    #     workers: # <- worker config, i.e. overrides, NOT environment config
+    #       imports: # <- worker "group" name
+    #         forks: 4         # Overrides default_forks
+    #         threads: 1       # Overrides default_threads
+    #         gc_limit: 500    # Overrides default_gc_limit
+    #         # ttr: 60        # Use default from production, i.e. 300 because not set
+    #         queues:
+    #           - imports
+    #           - data_processing
+    #
     def self.load_yaml(path, env = 'development', worker_name = nil)
-      yaml = YAML.load_file(path)
-      env_config = yaml[env.to_s] || yaml[env.to_sym]
+      yaml = YAML.load_file(path, aliases: true)
+      # env_defaults = top-level environment config (development:, production:, etc.)
+      env_defaults = yaml[env.to_s] || yaml[env.to_sym]
 
-      raise ArgumentError, "Environment '#{env}' not found in #{path}" unless env_config
+      raise ArgumentError, "Environment '#{env}' not found in #{path}" unless env_defaults
 
-      workers = env_config['workers']
+      workers = env_defaults['workers']
       raise ArgumentError, "No 'workers:' section found in #{path} for environment '#{env}'" unless workers
 
       # Auto-select single worker or validate worker_name
@@ -84,24 +107,28 @@ module Postburner
         end
       end
 
+      # worker_config = specific worker configuration (workers: imports:)
       worker_config = workers[worker_name]
 
-      # Convert queue array to hash format expected by rest of system
+      # Convert queue array to hash format (queues no longer have per-queue config)
       queue_list = worker_config['queues'] || []
       queues_hash = {}
       queue_list.each do |queue_name|
-        queues_hash[queue_name] = {}
+        queues_hash[queue_name] = {}  # Empty hash - queues run in worker pool
       end
 
+      # Cascade: worker-level overrides env-level defaults
+      # Worker uses: forks, threads, gc_limit, ttr, priority (NO default_ prefix)
+      # Env uses: default_forks, default_threads, etc. (WITH default_ prefix)
       options = {
-        beanstalk_url: env_config['beanstalk_url'],
+        beanstalk_url: env_defaults['beanstalk_url'],
         queues: queues_hash,
-        default_queue: worker_config['default_queue'] || env_config['default_queue'],
-        default_priority: worker_config['default_priority'] || env_config['default_priority'],
-        default_ttr: worker_config['default_ttr'] || env_config['default_ttr'],
-        default_threads: worker_config['default_threads'] || env_config['default_threads'],
-        default_forks: worker_config['default_forks'] || env_config['default_forks'],
-        default_gc_limit: worker_config['default_gc_limit'] || env_config['default_gc_limit']
+        default_queue: worker_config['default_queue'] || env_defaults['default_queue'],
+        default_priority: worker_config['priority'] || env_defaults['default_priority'],
+        default_ttr: worker_config['ttr'] || env_defaults['default_ttr'],
+        default_threads: worker_config['threads'] || env_defaults['default_threads'],
+        default_forks: worker_config['forks'] || env_defaults['default_forks'],
+        default_gc_limit: worker_config['gc_limit'] || env_defaults['default_gc_limit']
       }
 
       new(options)