postburner 1.0.0.pre.2 → 1.0.0.pre.4

data/app/models/postburner/mailer.rb

@@ -27,7 +27,7 @@ module Postburner
     #
     def with(params={})
       self.args.merge!(
-        'params' => ActiveJob::Arguments.serialize(params)
+        'params' => ::ActiveJob::Arguments.serialize(params)
       )
       self
     end
@@ -62,7 +62,7 @@ module Postburner
     # Get the deserialized params.
     #
     def params
-      ActiveJob::Arguments.deserialize(self.args['params']).to_h
+      ::ActiveJob::Arguments.deserialize(self.args['params']).to_h
     end
 
     def perform(args)

data/app/views/postburner/jobs/show.html.haml

@@ -6,8 +6,8 @@
   (#{@job.run_at.to_i - Time.zone.now.to_i} seconds)
 
 %h3 Stats
-- if @job.beanstalk_job
-  %pre= JSON.pretty_generate @job.beanstalk_job.stats.as_json
+- if @job.bk
+  %pre= JSON.pretty_generate @job.bk.stats.as_json
 
 %h3 Job
 %pre= JSON.pretty_generate @job.as_json

data/config/postburner.yml.example

@@ -20,7 +20,7 @@
 # Scale by adjusting forks and threads per worker:
 # - Development: forks=0, threads=1 (single-threaded, easiest debugging)
 # - Staging: forks=0, threads=10 (multi-threaded, moderate load)
-# - Production: forks=4, threads=10 (40 concurrent jobs per queue)
+# - Production: forks=4, threads=10 (40 concurrent jobs per worker)
 #
 
 default: &default
@@ -34,7 +34,7 @@ development:
   workers:
     default:
       # Single-threaded, single process (simplest for debugging)
-      # Defaults: forks=0, threads=1, gc_limit=nil
+      # If not specified, uses env-level defaults
       queues:
         - default
         - mailers
@@ -45,34 +45,39 @@ test:
   workers:
     default:
       # Test mode uses inline strategies automatically
-      # Defaults: forks=0, threads=1, gc_limit=nil
       queues:
         - default
 
-staging:
+staging:  # <- environment config
   <<: *default
 
-  workers:
+  # Env-level defaults (use default_ prefix)
+  default_threads: 10
+  default_gc_limit: 5000
+
+  workers:  # <- worker config overrides
     default:
       # Multi-threaded, single process (moderate concurrency)
-      default_threads: 10
-      default_gc_limit: 5000
+      # Uses env-level defaults: default_threads=10, default_gc_limit=5000
       queues:
         - critical
         - default
         - mailers
 
-production:
+production:  # <- environment config, i.e. defaults, NOT worker config
   <<: *default
 
-  # Example 1: Single worker processing all queues with same settings
+  # Env-level defaults (use default_ prefix)
+  default_forks: 2
+  default_threads: 10
+  default_gc_limit: 5000
+
+  # Example 1: Single worker using env defaults
   # Run: bin/postburner
   #
   # workers:
   #   default:
-  #     default_forks: 4
-  #     default_threads: 10
-  #     default_gc_limit: 5000
+  #     # Uses default_forks=2, default_threads=10
   #     queues:
   #       - critical
   #       - default
@@ -84,21 +89,20 @@ production:
   #   bin/postburner --worker imports    (4 forks, 1 thread each)
   #   bin/postburner --worker general    (2 forks, 100 threads each)
   #
-  workers:
+  workers:  # <- worker config, i.e. overrides, NOT environment config
     # Heavy, memory-intensive jobs - more processes, fewer threads
     imports:
-      default_forks: 4
-      default_threads: 1
-      default_gc_limit: 500
+      forks: 4              # Overrides default_forks
+      threads: 1            # Overrides default_threads
+      gc_limit: 500         # Overrides default_gc_limit
       queues:
         - imports
         - data_processing
 
-    # General jobs - fewer processes, many threads
+    # General jobs - uses env defaults (forks=2, threads=10)
+    # Override threads for higher concurrency
     general:
-      default_forks: 2
-      default_threads: 100
-      default_gc_limit: 5000
+      threads: 100          # Overrides default_threads (forks uses default_forks=2)
       queues:
         - default
         - mailers
@@ -112,27 +116,25 @@ production:
   #
   # workers:
   #   critical:
-  #     default_forks: 1
-  #     default_threads: 1
-  #     default_gc_limit: 100
+  #     forks: 1
+  #     threads: 1
+  #     gc_limit: 100
   #     queues:
   #       - critical
   #
   #   default:
-  #     default_forks: 4
-  #     default_threads: 10
-  #     default_gc_limit: 5000
+  #     forks: 4
+  #     threads: 10
   #     queues:
   #       - default
   #
   #   mailers:
-  #     default_forks: 2
-  #     default_threads: 5
-  #     default_gc_limit: 2000
+  #     forks: 2
+  #     threads: 5
   #     queues:
   #       - mailers
 
-# Global Defaults (can be overridden per worker):
+# Env-Level Defaults (can be overridden per worker):
 #
 # default_queue: default           # Default queue name (optional)
 # default_priority: 65536          # Lower = higher priority (optional, 0 is highest)

data/lib/generators/postburner/install/templates/config/postburner.yml

@@ -20,7 +20,7 @@
 # Scale by adjusting forks and threads per worker:
 # - Development: forks=0, threads=1 (single-threaded, easiest debugging)
 # - Staging: forks=0, threads=10 (multi-threaded, moderate load)
-# - Production: forks=4, threads=10 (40 concurrent jobs per queue)
+# - Production: forks=4, threads=10 (40 concurrent jobs per worker)
 #
 
 default: &default
@@ -34,7 +34,7 @@ development:
   workers:
     default:
       # Single-threaded, single process (simplest for debugging)
-      # Defaults: forks=0, threads=1, gc_limit=nil
+      # If not specified, uses env-level defaults
       queues:
         - default
         - mailers
@@ -45,34 +45,39 @@ test:
   workers:
     default:
       # Test mode uses inline strategies automatically
-      # Defaults: forks=0, threads=1, gc_limit=nil
       queues:
         - default
 
-staging:
+staging:  # <- environment config
   <<: *default
 
-  workers:
+  # Env-level defaults (use default_ prefix)
+  default_threads: 10
+  default_gc_limit: 5000
+
+  workers:  # <- worker config overrides
     default:
       # Multi-threaded, single process (moderate concurrency)
-      default_threads: 10
-      default_gc_limit: 5000
+      # Uses env-level defaults: default_threads=10, default_gc_limit=5000
       queues:
         - critical
         - default
         - mailers
 
-production:
+production:  # <- environment config, i.e. defaults, NOT worker config
   <<: *default
 
-  # Example 1: Single worker processing all queues with same settings
+  # Env-level defaults (use default_ prefix)
+  default_forks: 2
+  default_threads: 10
+  default_gc_limit: 5000
+
+  # Example 1: Single worker using env defaults
   # Run: bin/postburner
   #
   # workers:
   #   default:
-  #     default_forks: 4
-  #     default_threads: 10
-  #     default_gc_limit: 5000
+  #     # Uses default_forks=2, default_threads=10
   #     queues:
   #       - critical
   #       - default
@@ -84,21 +89,20 @@ production:
   #   bin/postburner --worker imports    (4 forks, 1 thread each)
   #   bin/postburner --worker general    (2 forks, 100 threads each)
   #
-  workers:
+  workers:  # <- worker config, i.e. overrides, NOT environment config
     # Heavy, memory-intensive jobs - more processes, fewer threads
     imports:
-      default_forks: 4
-      default_threads: 1
-      default_gc_limit: 500
+      forks: 4              # Overrides default_forks
+      threads: 1            # Overrides default_threads
+      gc_limit: 500         # Overrides default_gc_limit
       queues:
         - imports
         - data_processing
 
-    # General jobs - fewer processes, many threads
+    # General jobs - uses env defaults (forks=2, threads=10)
+    # Override threads for higher concurrency
     general:
-      default_forks: 2
-      default_threads: 100
-      default_gc_limit: 5000
+      threads: 100          # Overrides default_threads (forks uses default_forks=2)
       queues:
         - default
         - mailers
@@ -112,27 +116,25 @@ production:
   #
   # workers:
   #   critical:
-  #     default_forks: 1
-  #     default_threads: 1
-  #     default_gc_limit: 100
+  #     forks: 1
+  #     threads: 1
+  #     gc_limit: 100
   #     queues:
   #       - critical
   #
   #   default:
-  #     default_forks: 4
-  #     default_threads: 10
-  #     default_gc_limit: 5000
+  #     forks: 4
+  #     threads: 10
   #     queues:
   #       - default
   #
   #   mailers:
-  #     default_forks: 2
-  #     default_threads: 5
-  #     default_gc_limit: 2000
+  #     forks: 2
+  #     threads: 5
   #     queues:
   #       - mailers
 
-# Global Defaults (can be overridden per worker):
+# Env-Level Defaults (can be overridden per worker):
 #
 # default_queue: default           # Default queue name (optional)
 # default_priority: 65536          # Lower = higher priority (optional, 0 is highest)

data/lib/postburner/active_job/adapter.rb

@@ -100,7 +100,7 @@ module ActiveJob
         # Create Postburner::TrackedJob record
         tracked_job = Postburner::TrackedJob.create!(
           args: Postburner::ActiveJob::Payload.serialize_for_tracked(job),
-          run_at: timestamp,
+          run_at: timestamp ? Time.zone.at(timestamp) : nil,
           queued_at: Time.zone.now
         )
 
@@ -111,10 +111,10 @@ module ActiveJob
         Postburner.connected do |conn|
           tube_name = expand_tube_name(job.queue_name)
 
-          # Get priority and TTR from class configuration or fall back to defaults
-          pri = job.class.respond_to?(:queue_priority) && job.class.queue_priority ||
-                Postburner.configuration.default_priority
-          ttr = job.class.respond_to?(:queue_ttr) && job.class.queue_ttr ||
+          # Get priority and TTR
+          # Priority order: job.priority (from .set or class.priority) > default
+          pri = job.priority || Postburner.configuration.default_priority
+          ttr = job.class.respond_to?(:postburner_ttr) && job.class.postburner_ttr ||
                 Postburner.configuration.default_ttr
 
           bkid = conn.tubes[tube_name].put(
@@ -145,10 +145,10 @@ module ActiveJob
         Postburner.connected do |conn|
           tube_name = expand_tube_name(job.queue_name)
 
-          # Get priority and TTR from class configuration or fall back to defaults
-          pri = job.class.respond_to?(:queue_priority) && job.class.queue_priority ||
-                Postburner.configuration.default_priority
-          ttr = job.class.respond_to?(:queue_ttr) && job.class.queue_ttr ||
+          # Get priority and TTR
+          # Priority order: job.priority (from .set or class.priority) > default
+          pri = job.priority || Postburner.configuration.default_priority
+          ttr = job.class.respond_to?(:postburner_ttr) && job.class.postburner_ttr ||
                 Postburner.configuration.default_ttr
 
           conn.tubes[tube_name].put(

data/lib/postburner/beanstalkd.rb

@@ -3,9 +3,9 @@
 module Postburner
   # Beanstalkd-specific configuration DSL for ActiveJob classes using Postburner.
   #
-  # Provides consistent API with Postburner::Job for setting Beanstalkd queue
-  # priority and TTR (time-to-run). Include this module in your ActiveJob classes
-  # to use Beanstalkd-specific configuration.
+  # Provides TTR (time-to-run) configuration for Beanstalkd. For priority, use
+  # ActiveJob's built-in `priority` method. Include this module in your ActiveJob
+  # classes to use Beanstalkd-specific configuration.
   #
   # @note Automatically included when you include Postburner::Tracked
   #
@@ -14,8 +14,8 @@ module Postburner
   #     include Postburner::Beanstalkd
   #
   #     queue_as :critical
-  #     queue_priority 0      # Highest priority
-  #     queue_ttr 300         # 5 minutes to complete
+  #     priority 0      # Highest priority (uses ActiveJob's priority)
+  #     ttr 300         # 5 minutes to complete
   #
   #     def perform(payment_id)
   #       # ...
@@ -26,8 +26,8 @@ module Postburner
   #   class ProcessPayment < ApplicationJob
   #     include Postburner::Tracked  # Includes Beanstalkd automatically
   #
-  #     queue_priority 0
-  #     queue_ttr 600
+  #     priority 0
+  #     ttr 600
   #
   #     def perform(payment_id)
   #       log "Processing payment"
@@ -39,54 +39,33 @@ module Postburner
       extend ActiveSupport::Concern
 
       included do
-        class_attribute :postburner_priority, default: nil
         class_attribute :postburner_ttr, default: nil
       end
 
       class_methods do
-        # Sets or returns the queue priority.
-        #
-        # Lower numbers = higher priority in Beanstalkd (0 is highest).
-        # Falls back to Postburner.configuration.default_priority if not set.
-        #
-        # @param pri [Integer, nil] Priority to set (0-4294967295), or nil to get current value
-        #
-        # @return [Integer, nil] Current priority when getting, nil when setting
-        #
-        # @example Set priority
-        #   queue_priority 0  # Highest priority
-        #
-        # @example Get priority
-        #   ProcessPayment.queue_priority  # => 0
-        #
-        def queue_priority(pri = nil)
-          if pri
-            self.postburner_priority = pri
-            nil
-          else
-            postburner_priority
-          end
-        end
-
-        # Sets or returns the queue TTR (time to run).
+        # Sets or returns the TTR (time to run).
         #
         # Number of seconds Beanstalkd will wait for job completion before
         # making it available again. Falls back to Postburner.configuration.default_ttr
         # if not set.
         #
-        # @param ttr [Integer, nil] Timeout in seconds, or nil to get current value
+        # From Beanstalkd:  If the worker does not delete, release, or bury the
+        #   job within <ttr> seconds, the job will time out and the server will
+        #   release the job.
+        #
+        # @param seconds [Integer, nil] Timeout in seconds, or nil to get current value
         #
         # @return [Integer, nil] Current TTR when getting, nil when setting
         #
         # @example Set TTR
-        #   queue_ttr 300  # 5 minutes
+        #   ttr 300  # 5 minutes
         #
         # @example Get TTR
-        #   ProcessPayment.queue_ttr  # => 300
+        #   ProcessPayment.ttr  # => 300
         #
-        def queue_ttr(ttr = nil)
-          if ttr
-            self.postburner_ttr = ttr
+        def ttr(seconds = nil)
+          if seconds
+            self.postburner_ttr = seconds
             nil
           else
             postburner_ttr

data/lib/postburner/configuration.rb

@@ -27,8 +27,8 @@ module Postburner
     # @option options [String] :default_queue Default queue name (default: 'default')
     # @option options [Integer] :default_priority Default job priority (default: 65536, lower = higher priority)
     # @option options [Integer] :default_ttr Default time-to-run in seconds (default: 300)
-    # @option options [Integer] :default_threads Default thread count per queue (default: 1)
-    # @option options [Integer] :default_forks Default fork count per queue (default: 0, single process)
+    # @option options [Integer] :default_threads Default thread count per fork (default: 1)
+    # @option options [Integer] :default_forks Default fork count (default: 0, single process)
     # @option options [Integer] :default_gc_limit Default GC limit for worker restarts (default: nil, no limit)
     #
     def initialize(options = {})
@@ -62,13 +62,36 @@ module Postburner
     # @example Multiple workers (must specify)
     #   config = Postburner::Configuration.load_yaml('config/postburner.yml', 'production', 'imports')
     #
+    # @example config/postburner.yml example
+    #
+    #   default: &default
+    #     beanstalk_url: beanstalk://localhost:11300
+    #     default_priority: 131072 # change default priority from 65536 to 131072
+    #
+    #   production: # <- environment config, i.e. defaults, NOT worker config
+    #     <<: *default
+    #     default_forks: 2
+    #     default_threads: 10
+    #     default_gc_limit: 5000
+    #     default_ttr: 300
+    #     workers: # <- worker config, i.e. overrides, NOT environment config
+    #       imports: # <- worker "group" name
+    #         forks: 4         # Overrides default_forks
+    #         threads: 1       # Overrides default_threads
+    #         gc_limit: 500    # Overrides default_gc_limit
+    #         # ttr: 60        # Use default from production, i.e. 300 because not set
+    #         queues:
+    #           - imports
+    #           - data_processing
+    #
     def self.load_yaml(path, env = 'development', worker_name = nil)
       yaml = YAML.load_file(path, aliases: true)
-      env_config = yaml[env.to_s] || yaml[env.to_sym]
+      # env_defaults = top-level environment config (development:, production:, etc.)
+      env_defaults = yaml[env.to_s] || yaml[env.to_sym]
 
-      raise ArgumentError, "Environment '#{env}' not found in #{path}" unless env_config
+      raise ArgumentError, "Environment '#{env}' not found in #{path}" unless env_defaults
 
-      workers = env_config['workers']
+      workers = env_defaults['workers']
       raise ArgumentError, "No 'workers:' section found in #{path} for environment '#{env}'" unless workers
 
       # Auto-select single worker or validate worker_name
@@ -84,24 +107,28 @@ module Postburner
         end
       end
 
+      # worker_config = specific worker configuration (workers: imports:)
       worker_config = workers[worker_name]
 
-      # Convert queue array to hash format expected by rest of system
+      # Convert queue array to hash format (queues no longer have per-queue config)
       queue_list = worker_config['queues'] || []
       queues_hash = {}
       queue_list.each do |queue_name|
-        queues_hash[queue_name] = {}
+        queues_hash[queue_name] = {}  # Empty hash - queues run in worker pool
       end
 
+      # Cascade: worker-level overrides env-level defaults
+      # Worker uses: forks, threads, gc_limit, ttr, priority (NO default_ prefix)
+      # Env uses: default_forks, default_threads, etc. (WITH default_ prefix)
       options = {
-        beanstalk_url: env_config['beanstalk_url'],
+        beanstalk_url: env_defaults['beanstalk_url'],
         queues: queues_hash,
-        default_queue: worker_config['default_queue'] || env_config['default_queue'],
-        default_priority: worker_config['default_priority'] || env_config['default_priority'],
-        default_ttr: worker_config['default_ttr'] || env_config['default_ttr'],
-        default_threads: worker_config['default_threads'] || env_config['default_threads'],
-        default_forks: worker_config['default_forks'] || env_config['default_forks'],
-        default_gc_limit: worker_config['default_gc_limit'] || env_config['default_gc_limit']
+        default_queue: worker_config['default_queue'] || env_defaults['default_queue'],
+        default_priority: worker_config['priority'] || env_defaults['default_priority'],
+        default_ttr: worker_config['ttr'] || env_defaults['default_ttr'],
+        default_threads: worker_config['threads'] || env_defaults['default_threads'],
+        default_forks: worker_config['forks'] || env_defaults['default_forks'],
+        default_gc_limit: worker_config['gc_limit'] || env_defaults['default_gc_limit']
       }
 
       new(options)

data/lib/postburner/strategies/queue.rb

@@ -76,20 +76,30 @@ module Postburner
         #debugger
         Postburner::Job.transaction do
           Postburner.connected do |conn|
-            tube_name = job.tube_name
+            tube_name = job.expanded_tube_name
             data = { class: job.class.name, args: [job.id] }
 
             # Get priority, TTR from job instance (respects instance overrides) or options
-            pri = options[:pri] || job.queue_priority || Postburner.configuration.default_priority
+            pri = options[:pri] || job.priority || Postburner.configuration.default_priority
             delay = options[:delay] || 0
-            ttr = options[:ttr] || job.queue_ttr || Postburner.configuration.default_ttr
+            ttr = options[:ttr] || job.ttr || Postburner.configuration.default_ttr
 
-            response = conn.tubes[tube_name].put(
-              JSON.generate(data),
-              pri: pri,
-              delay: delay,
-              ttr: ttr
-            )
+            begin
+              response = conn.tubes[tube_name].put(
+                JSON.generate(data),
+                pri: pri,
+                delay: delay,
+                ttr: ttr
+              )
+            rescue Beaneater::BadFormatError => e
+              Rails.logger.error "Beanstalkd BAD_FORMAT error from:"
+              Rails.logger.error "  tube: #{tube_name}"
+              Rails.logger.error "  data: #{JSON.generate(data)}"
+              Rails.logger.error "  pri: #{pri.inspect} (#{pri.class})"
+              Rails.logger.error "  delay: #{delay.inspect} (#{delay.class})"
+              Rails.logger.error "  ttr: #{ttr.inspect} (#{ttr.class})"
+              raise
+            end
 
             response
           end

data/lib/postburner/version.rb

@@ -1,3 +1,3 @@
 module Postburner
-  VERSION = '1.0.0.pre.2'
+  VERSION = '1.0.0.pre.4'
 end

data/lib/postburner/workers/worker.rb

@@ -43,63 +43,62 @@ module Postburner
     #
     # **Development:**
     # ```yaml
-    # default_forks: 0
-    # default_threads: 1
+    # forks: 0
+    # threads: 1
     # ```
     # Single-threaded, single-process (simplest debugging)
     #
     # **Staging:**
     # ```yaml
-    # default_forks: 0
-    # default_threads: 10
+    # forks: 0
+    # threads: 10
     # ```
     # Multi-threaded, single-process (moderate concurrency)
     #
     # **Production:**
     # ```yaml
-    # default_forks: 4
-    # default_threads: 10
+    # forks: 4
+    # threads: 10
     # ```
     # 4 processes × 10 threads = 40 concurrent jobs per queue
     #
     # ## Configuration
     #
     # @example Development (single-threaded)
-    #   development:
+    #   development:  # <- environment config, i.e. defaults
     #     default_forks: 0
     #     default_threads: 1
-    #     queues:
-    #       default: {}
-    #       mailers: {}
+    #     workers:  # <- worker config, i.e. overrides
+    #       default:
+    #         queues:
+    #           - default
+    #           - mailers
     #
     # @example Staging (multi-threaded, single process)
-    #   staging:
+    #   staging:  # <- environment config, i.e. defaults
     #     default_forks: 0
     #     default_threads: 10
     #     default_gc_limit: 5000
-    #     queues:
-    #       critical:
-    #         threads: 1      # Single-threaded for critical jobs
+    #     workers:  # <- worker config, i.e. overrides
     #       default:
-    #         threads: 10     # 10 concurrent threads
-    #       mailers:
-    #         threads: 5      # 5 concurrent emails
+    #         queues:
+    #           - critical
+    #           - default
+    #           - mailers
     #
-    # @example Production (Puma-style: forks × threads)
-    #   production:
+    # @example Production (Puma-style: forks × threads with worker overrides)
+    #   production:  # <- environment config, i.e. defaults
     #     default_forks: 2
     #     default_threads: 10
     #     default_gc_limit: 5000
-    #     queues:
-    #       critical:
-    #         forks: 1        # Single fork
-    #         threads: 1      # Single thread = 1 concurrent job
+    #     workers:  # <- worker config, i.e. overrides
     #       default:
-    #         forks: 4        # 4 forks (Puma-style)
-    #         threads: 10     # 10 threads per fork = 40 total concurrent jobs
-    #       mailers:
-    #         forks: 2
-    #         threads: 5      # 2×5 = 10 total email senders
+    #         forks: 4        # Overrides default_forks
+    #         threads: 10     # Overrides default_threads
+    #         queues:
+    #           - critical
+    #           - default
+    #           - mailers
     #
     class Worker < Base
       # Starts the worker.

data/lib/postburner.rb

@@ -1,7 +1,6 @@
 require "postburner/version"
 require "postburner/configuration"
 require "postburner/connection"
-require "postburner/queue_config"
 require "postburner/beanstalkd"
 require "postburner/tracked"
 require "postburner/active_job/payload"