postburner 1.0.0.pre.12 → 1.0.0.pre.14

data/lib/generators/postburner/install/templates/config/postburner.yml

@@ -1,153 +1,38 @@
-# Postburner Configuration Example
+# Postburner Configuration
 #
-# Copy this file to config/postburner.yml and customize for your environment.
+# See docs/configuration.md for advanced options including:
+# - Multiple workers with different concurrency profiles
+# - Fork/thread tuning for production deployments
+# - Scheduler configuration
 #
-# ## Named Workers Configuration
-#
-# Postburner uses named worker configurations to support different deployment patterns:
-# - Single worker: bin/postburner (auto-selects the single worker)
-# - Multiple workers: bin/postburner --worker <name> (must specify which worker)
-#
-# Each worker can have different fork/thread settings and process different queues.
-# This enables running different queue groups in separate OS processes with distinct
-# concurrency profiles.
-#
-# ## Puma-Style Architecture
-#
-# - **forks: 0** = Single process with thread pool (development/staging)
-# - **forks: 1+** = Multiple processes with thread pools (production)
-#
-# Scale by adjusting forks and threads per worker:
-# - Development: forks=0, threads=1 (single-threaded, easiest debugging)
-# - Staging: forks=0, threads=10 (multi-threaded, moderate load)
-# - Production: forks=4, threads=10 (40 concurrent jobs per worker)
+# Start the worker with:
+#   bin/postburner
+#   bin/postburner --worker default
+#   rake postburner:work
+#   rake postburner:work WORKER=default
 #
+# With a single worker defined, it's auto-selected. Add more workers under
+# `workers:` for different queue sets (see docs/configuration.md).
 
-default: &default
-  # Beanstalkd connection URL
-  # Override with ENV['BEANSTALK_URL'] if set
+shared: &shared
   beanstalk_url: <%= ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300' %>
-
-  # Scheduler configuration
-  # The scheduler runs as a lightweight "watchdog" job that checks for due
-  # schedule executions and enqueues them. All workers automatically watch
-  # the scheduler queue and will process the watchdog when it becomes due.
-  default_scheduler_interval: 300  # Check for due schedules every 5 minutes (300 seconds)
-  default_scheduler_priority: 100  # Scheduler watchdog priority (lower = higher priority)
-
-development:
-  <<: *default
-
   workers:
     default:
-      # Single-threaded, single process (simplest for debugging)
-      # If not specified, uses env-level defaults
       queues:
         - default
         - mailers
 
-test:
-  <<: *default
-
-  workers:
-    default:
-      # Test mode uses inline strategies automatically
-      queues:
-        - default
-
-staging:  # <- environment config
-  <<: *default
-
-  # Env-level defaults (use default_ prefix)
-  default_threads: 10
-  default_gc_limit: 5000
-
-  workers:  # <- worker config overrides
-    default:
-      # Multi-threaded, single process (moderate concurrency)
-      # Uses env-level defaults: default_threads=10, default_gc_limit=5000
-      queues:
-        - critical
-        - default
-        - mailers
+# Single process, single thread (forks: 0, threads: 1)
+development:
+  <<: *shared
 
-production:  # <- environment config, i.e. defaults, NOT worker config
-  <<: *default
+# Inline execution (jobs run immediately in-process)
+test:
+  <<: *shared
 
-  # Env-level defaults (use default_ prefix)
+# Multi-process, multi-thread (forks: 2, threads: 4)
+production:
+  <<: *shared
   default_forks: 2
-  default_threads: 10
-  default_gc_limit: 5000
-
-  # Example 1: Single worker using env defaults
-  # Run: bin/postburner
-  #
-  # workers:
-  #   default:
-  #     # Uses default_forks=2, default_threads=10
-  #     queues:
-  #       - critical
-  #       - default
-  #       - mailers
-  #       - imports
-
-  # Example 2: Multiple workers with different concurrency profiles
-  # Run separate processes:
-  #   bin/postburner --worker imports    (4 forks, 1 thread each)
-  #   bin/postburner --worker general    (2 forks, 100 threads each)
-  #
-  workers:  # <- worker config, i.e. overrides, NOT environment config
-    # Heavy, memory-intensive jobs - more processes, fewer threads
-    imports:
-      forks: 4              # Overrides default_forks
-      threads: 1            # Overrides default_threads
-      gc_limit: 500         # Overrides default_gc_limit
-      queues:
-        - imports
-        - data_processing
-
-    # General jobs - uses env defaults (forks=2, threads=10)
-    # Override threads for higher concurrency
-    general:
-      threads: 100          # Overrides default_threads (forks uses default_forks=2)
-      queues:
-        - default
-        - mailers
-        - notifications
-
-  # Example 3: Fine-grained control with multiple specialized workers
-  # Run separate processes:
-  #   bin/postburner --worker critical
-  #   bin/postburner --worker default
-  #   bin/postburner --worker mailers
-  #
-  # workers:
-  #   critical:
-  #     forks: 1
-  #     threads: 1
-  #     gc_limit: 100
-  #     queues:
-  #       - critical
-  #
-  #   default:
-  #     forks: 4
-  #     threads: 10
-  #     queues:
-  #       - default
-  #
-  #   mailers:
-  #     forks: 2
-  #     threads: 5
-  #     queues:
-  #       - mailers
-
-# Env-Level Defaults (can be overridden per worker):
-#
-# default_queue: default                  # Default queue name (optional)
-# default_priority: 65536                 # Lower = higher priority (optional, 0 is highest)
-# default_ttr: 300                        # Time-to-run in seconds (optional)
-# default_threads: 1                      # Thread count per fork (optional, defaults to 1)
-# default_forks: 0                        # Fork count (optional, defaults to 0 = single process)
-# default_gc_limit: nil                   # Exit after N jobs for restart (optional, nil = no limit)
-# default_scheduler_interval: 300         # Scheduler check interval in seconds (optional, default: 300)
-# default_scheduler_priority: 100         # Scheduler watchdog priority (optional, default: 100)
+  default_threads: 4
+  default_gc_limit: 64

data/lib/postburner/active_job/adapter.rb

@@ -106,11 +106,56 @@ module ActiveJob
         job.class.included_modules.include?(Postburner::Tracked)
       end
 
+      # Resolves Beanstalkd options (priority and TTR) for a job.
+      #
+      # Uses a priority cascade:
+      # 1. job.priority (from .set(priority: n))
+      # 2. enqueue_options hook (for third-party jobs like Turbo)
+      # 3. Class-level configuration (postburner_ttr, priority DSL)
+      # 4. Global defaults from configuration
+      #
+      # @param job [ActiveJob::Base] The job instance
+      #
+      # @return [Hash] Hash with :pri and :ttr keys
+      #
+      def job_options(job)
+        hook_options = resolve_enqueue_options(job)
+
+        # Priority cascade: job.priority > hook > class-level > default
+        pri = job.priority ||
+              hook_options[:priority] ||
+              Postburner.configuration.default_priority
+
+        # TTR cascade: class-level > hook > default
+        ttr = (job.class.respond_to?(:postburner_ttr) && job.class.postburner_ttr) ||
+              hook_options[:ttr] ||
+              Postburner.configuration.default_ttr
+
+        { pri: pri, ttr: ttr }
+      end
+
+      # Calls the enqueue_options hook if configured.
+      #
+      # @param job [ActiveJob::Base] The job instance
+      #
+      # @return [Hash] Options hash with :priority and/or :ttr, or empty hash
+      #
+      def resolve_enqueue_options(job)
+        hook = Postburner.configuration.enqueue_options
+        return {} unless hook.respond_to?(:call)
+
+        hook.call(job) || {}
+      end
+
       # Enqueues a tracked job (with PostgreSQL audit trail).
       #
       # Creates a Postburner::TrackedJob record, then queues minimal payload
       # to Beanstalkd with just the job ID reference.
       #
+      # Instruments with ActiveSupport::Notifications:
+      # - enqueue.job.postburner: When job is queued immediately
+      # - enqueue_at.job.postburner: When job is queued with delay
+      #
       # @param job [ActiveJob::Base] The job instance
       # @param timestamp [Time, nil] When to execute the job
       #
@@ -128,25 +173,41 @@ module ActiveJob
         delay = timestamp ? [timestamp.to_i - Time.current.to_i, 0].max : 0
 
         # Queue to Beanstalkd with minimal payload
+        bkid = nil
         Postburner.connected do |conn|
           tube_name = expand_tube_name(job.queue_name)
-
-          # Get priority and TTR
-          # Priority order: job.priority (from .set or class.priority) > default
-          pri = job.priority || Postburner.configuration.default_priority
-          ttr = job.class.respond_to?(:postburner_ttr) && job.class.postburner_ttr ||
-                Postburner.configuration.default_ttr
+          opts = job_options(job)
 
           bkid = conn.tubes[tube_name].put(
             Postburner::ActiveJob::Payload.tracked_payload(job, tracked_job.id),
-            pri: pri,
+            pri: opts[:pri],
             delay: delay,
-            ttr: ttr
+            ttr: opts[:ttr]
           )
 
           # Update tracked_job with Beanstalkd ID
           tracked_job.update_column(:bkid, bkid)
         end
+
+        # Instrument enqueue event
+        job_payload = Postburner::Instrumentation.job_payload_from_activejob(
+          job,
+          tracked: true,
+          postburner_job_id: tracked_job.id,
+          beanstalk_job_id: bkid
+        )
+        scheduled_at = timestamp ? Time.zone.at(timestamp) : nil
+
+        if scheduled_at && scheduled_at > Time.current
+          ActiveSupport::Notifications.instrument('enqueue_at.job.postburner', {
+            job: job_payload,
+            scheduled_at: scheduled_at
+          })
+        else
+          ActiveSupport::Notifications.instrument('enqueue.job.postburner', {
+            job: job_payload
+          })
+        end
       end
 
       # Enqueues a default job (Beanstalkd only, no PostgreSQL).
@@ -154,6 +215,10 @@ module ActiveJob
       # Queues full job data to Beanstalkd for fast execution without
       # PostgreSQL overhead.
       #
+      # Instruments with ActiveSupport::Notifications:
+      # - enqueue.job.postburner: When job is queued immediately
+      # - enqueue_at.job.postburner: When job is queued with delay
+      #
       # @param job [ActiveJob::Base] The job instance
       # @param timestamp [Time, nil] When to execute the job
       #
@@ -161,23 +226,39 @@ module ActiveJob
       #
       def enqueue_default(job, timestamp)
         delay = timestamp ? [timestamp.to_i - Time.current.to_i, 0].max : 0
+        bkid = nil
 
         Postburner.connected do |conn|
           tube_name = expand_tube_name(job.queue_name)
+          opts = job_options(job)
 
-          # Get priority and TTR
-          # Priority order: job.priority (from .set or class.priority) > default
-          pri = job.priority || Postburner.configuration.default_priority
-          ttr = job.class.respond_to?(:postburner_ttr) && job.class.postburner_ttr ||
-                Postburner.configuration.default_ttr
-
-          conn.tubes[tube_name].put(
+          bkid = conn.tubes[tube_name].put(
             Postburner::ActiveJob::Payload.default_payload(job),
-            pri: pri,
+            pri: opts[:pri],
             delay: delay,
-            ttr: ttr
+            ttr: opts[:ttr]
           )
         end
+
+        # Instrument enqueue event
+        job_payload = Postburner::Instrumentation.job_payload_from_activejob(
+          job,
+          tracked: false,
+          postburner_job_id: nil,
+          beanstalk_job_id: bkid
+        )
+        scheduled_at = timestamp ? Time.zone.at(timestamp) : nil
+
+        if scheduled_at && scheduled_at > Time.current
+          ActiveSupport::Notifications.instrument('enqueue_at.job.postburner', {
+            job: job_payload,
+            scheduled_at: scheduled_at
+          })
+        else
+          ActiveSupport::Notifications.instrument('enqueue.job.postburner', {
+            job: job_payload
+          })
+        end
       end
 
       # Expands queue name to full tube name with environment prefix.

data/lib/postburner/configuration.rb

@@ -23,6 +23,7 @@ module Postburner
     # Global settings
     attr_accessor :beanstalk_url, :logger, :default_queue, :default_priority, :default_ttr
     attr_accessor :default_scheduler_interval, :default_scheduler_priority
+    attr_accessor :enqueue_options
 
     # Worker-specific settings (loaded for a single worker)
     attr_accessor :worker_config
@@ -35,6 +36,9 @@ module Postburner
     # @option options [Integer] :default_ttr Default time-to-run in seconds (default: 300)
     # @option options [Integer] :default_scheduler_interval Scheduler check interval in seconds (default: 300)
     # @option options [Integer] :default_scheduler_priority Scheduler job priority (default: 100)
+    # @option options [Proc] :enqueue_options Proc that receives job and returns options hash
+    #   with :priority and/or :ttr keys. Called during enqueue to customize job options.
+    #   Priority cascade: job.priority > hook > class-level > default
     # @option options [Hash] :worker_config Worker configuration hash with keys:
     #   - :name [String] Worker name
     #   - :queues [Array<String>] Queue/tube names to process
@@ -52,6 +56,7 @@ module Postburner
       @default_ttr = options[:default_ttr] || 300
       @default_scheduler_interval = options[:default_scheduler_interval] || 300
       @default_scheduler_priority = options[:default_scheduler_priority] || 100
+      @enqueue_options = options[:enqueue_options]
       @worker_config = options[:worker_config] || {
         name: 'default',
         queues: ['default'],

data/lib/postburner/instrumentation.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Instrumentation helpers for ActiveSupport::Notifications events.
+  #
+  # Provides standardized payload builders for all Postburner instrumentation events.
+  # Event naming follows the pattern: `verb.noun.postburner`
+  #
+  # ## Event Categories
+  #
+  # ### Job Events
+  # - `perform_start.job.postburner` - Before job execution
+  # - `perform.job.postburner` - Around job execution (includes duration)
+  # - `retry.job.postburner` - Default job retried
+  # - `discard.job.postburner` - Default job exhausts retries
+  # - `enqueue.job.postburner` - Job queued (immediate)
+  # - `enqueue_at.job.postburner` - Job queued with delay
+  # - `retry_stopped.job.postburner` - Tracked job buried after failures
+  #
+  # ### Schedule Events
+  # - `create.schedule.postburner` - Schedule created
+  # - `update.schedule.postburner` - Schedule updated
+  # - `audit.schedule.postburner` - Scheduler audits a schedule
+  #
+  # ### Schedule Execution Events
+  # - `create.schedule_execution.postburner` - Execution created
+  # - `enqueue.schedule_execution.postburner` - Execution enqueued to Beanstalkd
+  # - `skip.schedule_execution.postburner` - Execution skipped
+  #
+  # ### Scheduler Watchdog Events
+  # - `perform_start.scheduler.postburner` - Watchdog run begins
+  # - `perform.scheduler.postburner` - Around watchdog run (summary)
+  #
+  # @example Subscribing to job events
+  #   ActiveSupport::Notifications.subscribe('perform.job.postburner') do |name, start, finish, id, payload|
+  #     duration = (finish - start) * 1000
+  #     Rails.logger.info "[Postburner] #{payload[:job][:class]} completed in #{duration.round(2)}ms"
+  #   end
+  #
+  # @example Subscribing to schedule events
+  #   ActiveSupport::Notifications.subscribe('create.schedule.postburner') do |*args|
+  #     payload = args.last
+  #     schedule = payload[:schedule]
+  #     Rails.logger.info "[Postburner] Schedule '#{schedule[:name]}' created"
+  #   end
+  #
+  module Instrumentation
+    module_function
+
+    # Build a job payload hash for instrumentation.
+    #
+    # @param job [Postburner::Job, Hash] Job instance or parsed payload hash
+    # @param beanstalk_job_id [Integer, nil] Beanstalkd job ID
+    # @return [Hash] Standardized job payload
+    #
+    def job_payload(job, beanstalk_job_id: nil)
+      if job.is_a?(Postburner::Job)
+        job_payload_from_model(job, beanstalk_job_id: beanstalk_job_id)
+      elsif job.is_a?(Hash)
+        job_payload_from_hash(job, beanstalk_job_id: beanstalk_job_id)
+      else
+        raise ArgumentError, "Expected Postburner::Job or Hash, got #{job.class}"
+      end
+    end
+
+    # Build job payload from a Postburner::Job model.
+    #
+    # @param job [Postburner::Job] Job model instance
+    # @param beanstalk_job_id [Integer, nil] Beanstalkd job ID
+    # @return [Hash] Standardized job payload
+    #
+    def job_payload_from_model(job, beanstalk_job_id: nil)
+      {
+        class: job.class.name,
+        id: job.id,
+        job_id: job.respond_to?(:args) && job.args.is_a?(Hash) ? job.args['job_id'] : nil,
+        arguments: job.args,
+        queue_name: job.respond_to?(:queue_name) ? job.queue_name : job.class.try(:postburner_queue),
+        beanstalk_job_id: beanstalk_job_id || job.bkid,
+        tracked: true
+      }
+    end
+
+    # Build job payload from a parsed Beanstalkd payload hash.
+    #
+    # Handles both ActiveJob format and legacy Postburner::Job format.
+    #
+    # @param payload [Hash] Parsed JSON payload from Beanstalkd
+    # @param beanstalk_job_id [Integer, nil] Beanstalkd job ID
+    # @return [Hash] Standardized job payload
+    #
+    def job_payload_from_hash(payload, beanstalk_job_id: nil)
+      if Postburner::ActiveJob::Payload.legacy_format?(payload)
+        # Legacy format: { "class" => "JobClass", "args" => [id] }
+        {
+          class: payload['class'],
+          id: payload['args']&.first,
+          job_id: nil,
+          arguments: payload['args'],
+          queue_name: nil,
+          beanstalk_job_id: beanstalk_job_id,
+          tracked: true
+        }
+      else
+        # ActiveJob format
+        tracked = payload['tracked'] == true
+        {
+          class: payload['job_class'],
+          id: tracked ? payload['postburner_job_id'] : nil,
+          job_id: payload['job_id'],
+          arguments: payload['arguments'],
+          queue_name: payload['queue_name'],
+          beanstalk_job_id: beanstalk_job_id,
+          tracked: tracked
+        }
+      end
+    end
+
+    # Build job payload from an ActiveJob instance.
+    #
+    # @param job [ActiveJob::Base] ActiveJob instance
+    # @param tracked [Boolean] Whether job is tracked in PostgreSQL
+    # @param postburner_job_id [Integer, nil] Postburner::TrackedJob ID if tracked
+    # @param beanstalk_job_id [Integer, nil] Beanstalkd job ID
+    # @return [Hash] Standardized job payload
+    #
+    def job_payload_from_activejob(job, tracked:, postburner_job_id: nil, beanstalk_job_id: nil)
+      {
+        class: job.class.name,
+        id: postburner_job_id,
+        job_id: job.job_id,
+        arguments: job.arguments,
+        queue_name: job.queue_name,
+        beanstalk_job_id: beanstalk_job_id,
+        tracked: tracked
+      }
+    end
+
+    # Build a schedule payload hash for instrumentation.
+    #
+    # @param schedule [Postburner::Schedule] Schedule model instance
+    # @return [Hash] Standardized schedule payload
+    #
+    def schedule_payload(schedule)
+      {
+        id: schedule.id,
+        name: schedule.name,
+        job_class: schedule.job_class,
+        enabled: schedule.enabled,
+        interval: schedule.interval,
+        interval_unit: schedule.interval_unit,
+        cron: schedule.cron,
+        timezone: schedule.timezone
+      }
+    end
+
+    # Build a schedule execution payload hash for instrumentation.
+    #
+    # @param execution [Postburner::ScheduleExecution] Execution model instance
+    # @return [Hash] Standardized execution payload
+    #
+    def execution_payload(execution)
+      {
+        id: execution.id,
+        schedule_id: execution.schedule_id,
+        run_at: execution.run_at,
+        next_run_at: execution.next_run_at,
+        status: execution.status,
+        beanstalk_job_id: execution.beanstalk_job_id,
+        job_id: execution.job_id
+      }
+    end
+
+    # Build changes hash from ActiveRecord model, excluding specified attributes.
+    #
+    # @param model [ActiveRecord::Base] Model with changes
+    # @param exclude [Array<String, Symbol>] Attributes to exclude
+    # @return [Hash] Changes hash in { attribute: [old, new] } format
+    #
+    def changes_payload(model, exclude: [])
+      exclude = exclude.map(&:to_s)
+      model.saved_changes.except(*exclude, 'updated_at', 'created_at')
+    end
+
+    # Instrument an event with ActiveSupport::Notifications.
+    #
+    # @param event [String] Event name (e.g., 'perform.job.postburner')
+    # @param payload [Hash] Event payload
+    # @yield Block to execute within instrumentation (for timing)
+    # @return [Object] Result of block if given
+    #
+    def instrument(event, payload = {}, &block)
+      ActiveSupport::Notifications.instrument(event, payload, &block)
+    end
+  end
+end