postburner 1.0.0.pre.13 → 1.0.0.pre.14

data/lib/generators/postburner/install/templates/config/postburner.yml

@@ -1,153 +1,38 @@
-# Postburner Configuration Example
+# Postburner Configuration
 #
-# Copy this file to config/postburner.yml and customize for your environment.
+# See docs/configuration.md for advanced options including:
+# - Multiple workers with different concurrency profiles
+# - Fork/thread tuning for production deployments
+# - Scheduler configuration
 #
-# ## Named Workers Configuration
-#
-# Postburner uses named worker configurations to support different deployment patterns:
-# - Single worker: bin/postburner (auto-selects the single worker)
-# - Multiple workers: bin/postburner --worker <name> (must specify which worker)
-#
-# Each worker can have different fork/thread settings and process different queues.
-# This enables running different queue groups in separate OS processes with distinct
-# concurrency profiles.
-#
-# ## Puma-Style Architecture
-#
-# - **forks: 0** = Single process with thread pool (development/staging)
-# - **forks: 1+** = Multiple processes with thread pools (production)
-#
-# Scale by adjusting forks and threads per worker:
-# - Development: forks=0, threads=1 (single-threaded, easiest debugging)
-# - Staging: forks=0, threads=10 (multi-threaded, moderate load)
-# - Production: forks=4, threads=10 (40 concurrent jobs per worker)
+# Start the worker with:
+#   bin/postburner
+#   bin/postburner --worker default
+#   rake postburner:work
+#   rake postburner:work WORKER=default
 #
+# With a single worker defined, it's auto-selected. Add more workers under
+# `workers:` for different queue sets (see docs/configuration.md).
 
-default: &default
-  # Beanstalkd connection URL
-  # Override with ENV['BEANSTALK_URL'] if set
+shared: &shared
   beanstalk_url: <%= ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300' %>
-
-  # Scheduler configuration
-  # The scheduler runs as a lightweight "watchdog" job that checks for due
-  # schedule executions and enqueues them. All workers automatically watch
-  # the scheduler queue and will process the watchdog when it becomes due.
-  default_scheduler_interval: 300  # Check for due schedules every 5 minutes (300 seconds)
-  default_scheduler_priority: 100  # Scheduler watchdog priority (lower = higher priority)
-
-development:
-  <<: *default
-
   workers:
     default:
-      # Single-threaded, single process (simplest for debugging)
-      # If not specified, uses env-level defaults
       queues:
         - default
         - mailers
 
-test:
-  <<: *default
-
-  workers:
-    default:
-      # Test mode uses inline strategies automatically
-      queues:
-        - default
-
-staging:  # <- environment config
-  <<: *default
-
-  # Env-level defaults (use default_ prefix)
-  default_threads: 10
-  default_gc_limit: 5000
-
-  workers:  # <- worker config overrides
-    default:
-      # Multi-threaded, single process (moderate concurrency)
-      # Uses env-level defaults: default_threads=10, default_gc_limit=5000
-      queues:
-        - critical
-        - default
-        - mailers
+# Single process, single thread (forks: 0, threads: 1)
+development:
+  <<: *shared
 
-production:  # <- environment config, i.e. defaults, NOT worker config
-  <<: *default
+# Inline execution (jobs run immediately in-process)
+test:
+  <<: *shared
 
-  # Env-level defaults (use default_ prefix)
+# Multi-process, multi-thread (forks: 2, threads: 4)
+production:
+  <<: *shared
   default_forks: 2
-  default_threads: 10
-  default_gc_limit: 5000
-
-  # Example 1: Single worker using env defaults
-  # Run: bin/postburner
-  #
-  # workers:
-  #   default:
-  #     # Uses default_forks=2, default_threads=10
-  #     queues:
-  #       - critical
-  #       - default
-  #       - mailers
-  #       - imports
-
-  # Example 2: Multiple workers with different concurrency profiles
-  # Run separate processes:
-  #   bin/postburner --worker imports    (4 forks, 1 thread each)
-  #   bin/postburner --worker general    (2 forks, 100 threads each)
-  #
-  workers:  # <- worker config, i.e. overrides, NOT environment config
-    # Heavy, memory-intensive jobs - more processes, fewer threads
-    imports:
-      forks: 4              # Overrides default_forks
-      threads: 1            # Overrides default_threads
-      gc_limit: 500         # Overrides default_gc_limit
-      queues:
-        - imports
-        - data_processing
-
-    # General jobs - uses env defaults (forks=2, threads=10)
-    # Override threads for higher concurrency
-    general:
-      threads: 100          # Overrides default_threads (forks uses default_forks=2)
-      queues:
-        - default
-        - mailers
-        - notifications
-
-  # Example 3: Fine-grained control with multiple specialized workers
-  # Run separate processes:
-  #   bin/postburner --worker critical
-  #   bin/postburner --worker default
-  #   bin/postburner --worker mailers
-  #
-  # workers:
-  #   critical:
-  #     forks: 1
-  #     threads: 1
-  #     gc_limit: 100
-  #     queues:
-  #       - critical
-  #
-  #   default:
-  #     forks: 4
-  #     threads: 10
-  #     queues:
-  #       - default
-  #
-  #   mailers:
-  #     forks: 2
-  #     threads: 5
-  #     queues:
-  #       - mailers
-
-# Env-Level Defaults (can be overridden per worker):
-#
-# default_queue: default                  # Default queue name (optional)
-# default_priority: 65536                 # Lower = higher priority (optional, 0 is highest)
-# default_ttr: 300                        # Time-to-run in seconds (optional)
-# default_threads: 1                      # Thread count per fork (optional, defaults to 1)
-# default_forks: 0                        # Fork count (optional, defaults to 0 = single process)
-# default_gc_limit: nil                   # Exit after N jobs for restart (optional, nil = no limit)
-# default_scheduler_interval: 300         # Scheduler check interval in seconds (optional, default: 300)
-# default_scheduler_priority: 100         # Scheduler watchdog priority (optional, default: 100)
+  default_threads: 4
+  default_gc_limit: 64

data/lib/postburner/active_job/adapter.rb

@@ -152,6 +152,10 @@ module ActiveJob
       # Creates a Postburner::TrackedJob record, then queues minimal payload
       # to Beanstalkd with just the job ID reference.
       #
+      # Instruments with ActiveSupport::Notifications:
+      # - enqueue.job.postburner: When job is queued immediately
+      # - enqueue_at.job.postburner: When job is queued with delay
+      #
       # @param job [ActiveJob::Base] The job instance
       # @param timestamp [Time, nil] When to execute the job
       #
@@ -169,6 +173,7 @@ module ActiveJob
         delay = timestamp ? [timestamp.to_i - Time.current.to_i, 0].max : 0
 
         # Queue to Beanstalkd with minimal payload
+        bkid = nil
         Postburner.connected do |conn|
           tube_name = expand_tube_name(job.queue_name)
           opts = job_options(job)
@@ -183,6 +188,26 @@ module ActiveJob
           # Update tracked_job with Beanstalkd ID
           tracked_job.update_column(:bkid, bkid)
         end
+
+        # Instrument enqueue event
+        job_payload = Postburner::Instrumentation.job_payload_from_activejob(
+          job,
+          tracked: true,
+          postburner_job_id: tracked_job.id,
+          beanstalk_job_id: bkid
+        )
+        scheduled_at = timestamp ? Time.zone.at(timestamp) : nil
+
+        if scheduled_at && scheduled_at > Time.current
+          ActiveSupport::Notifications.instrument('enqueue_at.job.postburner', {
+            job: job_payload,
+            scheduled_at: scheduled_at
+          })
+        else
+          ActiveSupport::Notifications.instrument('enqueue.job.postburner', {
+            job: job_payload
+          })
+        end
       end
 
       # Enqueues a default job (Beanstalkd only, no PostgreSQL).
@@ -190,6 +215,10 @@ module ActiveJob
       # Queues full job data to Beanstalkd for fast execution without
       # PostgreSQL overhead.
       #
+      # Instruments with ActiveSupport::Notifications:
+      # - enqueue.job.postburner: When job is queued immediately
+      # - enqueue_at.job.postburner: When job is queued with delay
+      #
       # @param job [ActiveJob::Base] The job instance
       # @param timestamp [Time, nil] When to execute the job
       #
@@ -197,18 +226,39 @@ module ActiveJob
       #
       def enqueue_default(job, timestamp)
         delay = timestamp ? [timestamp.to_i - Time.current.to_i, 0].max : 0
+        bkid = nil
 
         Postburner.connected do |conn|
           tube_name = expand_tube_name(job.queue_name)
           opts = job_options(job)
 
-          conn.tubes[tube_name].put(
+          bkid = conn.tubes[tube_name].put(
             Postburner::ActiveJob::Payload.default_payload(job),
             pri: opts[:pri],
             delay: delay,
             ttr: opts[:ttr]
           )
         end
+
+        # Instrument enqueue event
+        job_payload = Postburner::Instrumentation.job_payload_from_activejob(
+          job,
+          tracked: false,
+          postburner_job_id: nil,
+          beanstalk_job_id: bkid
+        )
+        scheduled_at = timestamp ? Time.zone.at(timestamp) : nil
+
+        if scheduled_at && scheduled_at > Time.current
+          ActiveSupport::Notifications.instrument('enqueue_at.job.postburner', {
+            job: job_payload,
+            scheduled_at: scheduled_at
+          })
+        else
+          ActiveSupport::Notifications.instrument('enqueue.job.postburner', {
+            job: job_payload
+          })
+        end
       end
 
       # Expands queue name to full tube name with environment prefix.

data/lib/postburner/instrumentation.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Postburner
+  # Instrumentation helpers for ActiveSupport::Notifications events.
+  #
+  # Provides standardized payload builders for all Postburner instrumentation events.
+  # Event naming follows the pattern: `verb.noun.postburner`
+  #
+  # ## Event Categories
+  #
+  # ### Job Events
+  # - `perform_start.job.postburner` - Before job execution
+  # - `perform.job.postburner` - Around job execution (includes duration)
+  # - `retry.job.postburner` - Default job retried
+  # - `discard.job.postburner` - Default job exhausts retries
+  # - `enqueue.job.postburner` - Job queued (immediate)
+  # - `enqueue_at.job.postburner` - Job queued with delay
+  # - `retry_stopped.job.postburner` - Tracked job buried after failures
+  #
+  # ### Schedule Events
+  # - `create.schedule.postburner` - Schedule created
+  # - `update.schedule.postburner` - Schedule updated
+  # - `audit.schedule.postburner` - Scheduler audits a schedule
+  #
+  # ### Schedule Execution Events
+  # - `create.schedule_execution.postburner` - Execution created
+  # - `enqueue.schedule_execution.postburner` - Execution enqueued to Beanstalkd
+  # - `skip.schedule_execution.postburner` - Execution skipped
+  #
+  # ### Scheduler Watchdog Events
+  # - `perform_start.scheduler.postburner` - Watchdog run begins
+  # - `perform.scheduler.postburner` - Around watchdog run (summary)
+  #
+  # @example Subscribing to job events
+  #   ActiveSupport::Notifications.subscribe('perform.job.postburner') do |name, start, finish, id, payload|
+  #     duration = (finish - start) * 1000
+  #     Rails.logger.info "[Postburner] #{payload[:job][:class]} completed in #{duration.round(2)}ms"
+  #   end
+  #
+  # @example Subscribing to schedule events
+  #   ActiveSupport::Notifications.subscribe('create.schedule.postburner') do |*args|
+  #     payload = args.last
+  #     schedule = payload[:schedule]
+  #     Rails.logger.info "[Postburner] Schedule '#{schedule[:name]}' created"
+  #   end
+  #
+  module Instrumentation
+    module_function
+
+    # Build a job payload hash for instrumentation.
+    #
+    # @param job [Postburner::Job, Hash] Job instance or parsed payload hash
+    # @param beanstalk_job_id [Integer, nil] Beanstalkd job ID
+    # @return [Hash] Standardized job payload
+    #
+    def job_payload(job, beanstalk_job_id: nil)
+      if job.is_a?(Postburner::Job)
+        job_payload_from_model(job, beanstalk_job_id: beanstalk_job_id)
+      elsif job.is_a?(Hash)
+        job_payload_from_hash(job, beanstalk_job_id: beanstalk_job_id)
+      else
+        raise ArgumentError, "Expected Postburner::Job or Hash, got #{job.class}"
+      end
+    end
+
+    # Build job payload from a Postburner::Job model.
+    #
+    # @param job [Postburner::Job] Job model instance
+    # @param beanstalk_job_id [Integer, nil] Beanstalkd job ID
+    # @return [Hash] Standardized job payload
+    #
+    def job_payload_from_model(job, beanstalk_job_id: nil)
+      {
+        class: job.class.name,
+        id: job.id,
+        job_id: job.respond_to?(:args) && job.args.is_a?(Hash) ? job.args['job_id'] : nil,
+        arguments: job.args,
+        queue_name: job.respond_to?(:queue_name) ? job.queue_name : job.class.try(:postburner_queue),
+        beanstalk_job_id: beanstalk_job_id || job.bkid,
+        tracked: true
+      }
+    end
+
+    # Build job payload from a parsed Beanstalkd payload hash.
+    #
+    # Handles both ActiveJob format and legacy Postburner::Job format.
+    #
+    # @param payload [Hash] Parsed JSON payload from Beanstalkd
+    # @param beanstalk_job_id [Integer, nil] Beanstalkd job ID
+    # @return [Hash] Standardized job payload
+    #
+    def job_payload_from_hash(payload, beanstalk_job_id: nil)
+      if Postburner::ActiveJob::Payload.legacy_format?(payload)
+        # Legacy format: { "class" => "JobClass", "args" => [id] }
+        {
+          class: payload['class'],
+          id: payload['args']&.first,
+          job_id: nil,
+          arguments: payload['args'],
+          queue_name: nil,
+          beanstalk_job_id: beanstalk_job_id,
+          tracked: true
+        }
+      else
+        # ActiveJob format
+        tracked = payload['tracked'] == true
+        {
+          class: payload['job_class'],
+          id: tracked ? payload['postburner_job_id'] : nil,
+          job_id: payload['job_id'],
+          arguments: payload['arguments'],
+          queue_name: payload['queue_name'],
+          beanstalk_job_id: beanstalk_job_id,
+          tracked: tracked
+        }
+      end
+    end
+
+    # Build job payload from an ActiveJob instance.
+    #
+    # @param job [ActiveJob::Base] ActiveJob instance
+    # @param tracked [Boolean] Whether job is tracked in PostgreSQL
+    # @param postburner_job_id [Integer, nil] Postburner::TrackedJob ID if tracked
+    # @param beanstalk_job_id [Integer, nil] Beanstalkd job ID
+    # @return [Hash] Standardized job payload
+    #
+    def job_payload_from_activejob(job, tracked:, postburner_job_id: nil, beanstalk_job_id: nil)
+      {
+        class: job.class.name,
+        id: postburner_job_id,
+        job_id: job.job_id,
+        arguments: job.arguments,
+        queue_name: job.queue_name,
+        beanstalk_job_id: beanstalk_job_id,
+        tracked: tracked
+      }
+    end
+
+    # Build a schedule payload hash for instrumentation.
+    #
+    # @param schedule [Postburner::Schedule] Schedule model instance
+    # @return [Hash] Standardized schedule payload
+    #
+    def schedule_payload(schedule)
+      {
+        id: schedule.id,
+        name: schedule.name,
+        job_class: schedule.job_class,
+        enabled: schedule.enabled,
+        interval: schedule.interval,
+        interval_unit: schedule.interval_unit,
+        cron: schedule.cron,
+        timezone: schedule.timezone
+      }
+    end
+
+    # Build a schedule execution payload hash for instrumentation.
+    #
+    # @param execution [Postburner::ScheduleExecution] Execution model instance
+    # @return [Hash] Standardized execution payload
+    #
+    def execution_payload(execution)
+      {
+        id: execution.id,
+        schedule_id: execution.schedule_id,
+        run_at: execution.run_at,
+        next_run_at: execution.next_run_at,
+        status: execution.status,
+        beanstalk_job_id: execution.beanstalk_job_id,
+        job_id: execution.job_id
+      }
+    end
+
+    # Build changes hash from ActiveRecord model, excluding specified attributes.
+    #
+    # @param model [ActiveRecord::Base] Model with changes
+    # @param exclude [Array<String, Symbol>] Attributes to exclude
+    # @return [Hash] Changes hash in { attribute: [old, new] } format
+    #
+    def changes_payload(model, exclude: [])
+      exclude = exclude.map(&:to_s)
+      model.saved_changes.except(*exclude, 'updated_at', 'created_at')
+    end
+
+    # Instrument an event with ActiveSupport::Notifications.
+    #
+    # @param event [String] Event name (e.g., 'perform.job.postburner')
+    # @param payload [Hash] Event payload
+    # @yield Block to execute within instrumentation (for timing)
+    # @return [Object] Result of block if given
+    #
+    def instrument(event, payload = {}, &block)
+      ActiveSupport::Notifications.instrument(event, payload, &block)
+    end
+  end
+end

data/lib/postburner/scheduler.rb

@@ -123,6 +123,10 @@ module Postburner
     # 2. Ensure each schedule has a future execution queued
     # 3. Enqueue any orphaned pending executions that weren't properly queued
     #
+    # Instruments with ActiveSupport::Notifications:
+    # - perform_start.scheduler.postburner: When scheduler run begins
+    # - perform.scheduler.postburner: Around scheduler run (summary stats)
+    #
     # @return [void]
     #
     # @example Called by worker
@@ -134,16 +138,46 @@ module Postburner
     def perform
       logger.info "[Postburner::Scheduler] Starting scheduler run"
 
+      ActiveSupport::Notifications.instrument('perform_start.scheduler.postburner', {
+        interval: interval
+      })
+
+      # Track stats for summary event
+      @schedules_processed = 0
+      @schedules_failed = 0
+      @executions_created = 0
+      @orphans_enqueued = 0
+      lock_acquired = false
+
+      # Build payload hash that will be mutated with final stats
+      payload = {
+        interval: interval,
+        lock_acquired: nil,
+        schedules_processed: nil,
+        schedules_failed: nil,
+        executions_created: nil,
+        orphans_enqueued: nil
+      }
+
       # Use advisory lock to coordinate multiple workers
-      acquired = Postburner::AdvisoryLock.with_lock(AdvisoryLock::SCHEDULER_LOCK_KEY, blocking: false) do
-        process_all_schedules
-        true
-      end
+      ActiveSupport::Notifications.instrument('perform.scheduler.postburner', payload) do
+        lock_acquired = Postburner::AdvisoryLock.with_lock(AdvisoryLock::SCHEDULER_LOCK_KEY, blocking: false) do
+          process_all_schedules
+          true
+        end
 
-      if acquired
-        logger.info "[Postburner::Scheduler] Scheduler run complete"
-      else
-        logger.info "[Postburner::Scheduler] Could not acquire lock, skipping"
+        if lock_acquired
+          logger.info "[Postburner::Scheduler] Scheduler run complete"
+        else
+          logger.info "[Postburner::Scheduler] Could not acquire lock, skipping"
+        end
+
+        # Update payload with final stats (mutates the hash subscribers receive)
+        payload[:lock_acquired] = lock_acquired
+        payload[:schedules_processed] = @schedules_processed
+        payload[:schedules_failed] = @schedules_failed
+        payload[:executions_created] = @executions_created
+        payload[:orphans_enqueued] = @orphans_enqueued
       end
     ensure
       # Always re-queue watchdog for next run
@@ -297,21 +331,18 @@ module Postburner
     # @api private
     #
     def process_all_schedules
-      processed_count = 0
-      failed_count = 0
-
       Postburner::Schedule.enabled.find_each do |schedule|
         begin
           process_schedule(schedule)
-          processed_count += 1
+          @schedules_processed += 1
         rescue => e
           logger.error "[Postburner::Scheduler] Failed to process schedule '#{schedule.name}': #{e.class} - #{e.message}"
           logger.error e.backtrace.join("\n")
-          failed_count += 1
+          @schedules_failed += 1
         end
       end
 
-      logger.info "[Postburner::Scheduler] Processed #{processed_count} schedules, #{failed_count} failed"
+      logger.info "[Postburner::Scheduler] Processed #{@schedules_processed} schedules, #{@schedules_failed} failed"
     end
 
     # Process a single schedule.
@@ -325,31 +356,43 @@ module Postburner
     # This ensures newly created executions are available for processing in the
     # same run if they happen to be due.
     #
+    # Instruments with ActiveSupport::Notifications:
+    # - audit.schedule.postburner: When schedule is audited
+    #
     # @param schedule [Postburner::Schedule] The schedule to process
     # @return [void]
     #
     # @api private
     #
     def process_schedule(schedule)
+      bootstrapped = false
+      execution_created = false
+      orphans_count = 0
+
       # Auto-bootstrap: create first execution if schedule hasn't been started
       # This will create the execution AND enqueue it to Beanstalkd
       unless schedule.started?
         logger.info "[Postburner::Scheduler] Bootstrapping schedule '#{schedule.name}'"
         schedule.start!
+        bootstrapped = true
+        @executions_created += 1
       end
 
       # Safety net 1: Ensure schedule has a future execution
       # If missing, this will create AND enqueue it to Beanstalkd's delayed queue
-      ensure_future_execution!(schedule)
+      if ensure_future_execution!(schedule)
+        execution_created = true
+        @executions_created += 1
+      end
 
       # Safety net 2: Find any orphaned pending executions and enqueue them
       # This should rarely happen - only if enqueue! previously failed
-      execution_count = 0
       schedule.executions.due.find_each do |execution|
         begin
           logger.warn "[Postburner::Scheduler] Found orphaned pending execution #{execution.id} for schedule '#{schedule.name}', enqueuing"
           execution.enqueue!
-          execution_count += 1
+          orphans_count += 1
+          @orphans_enqueued += 1
         rescue => e
           logger.error "[Postburner::Scheduler] Failed to enqueue execution #{execution.id}: #{e.class} - #{e.message}"
           raise
@@ -359,7 +402,15 @@ module Postburner
       # Update last_audit_at
       schedule.update_column(:last_audit_at, Time.current)
 
-      logger.debug "[Postburner::Scheduler] Schedule '#{schedule.name}': enqueued #{execution_count} orphaned executions" if execution_count > 0
+      # Instrument audit event
+      ActiveSupport::Notifications.instrument('audit.schedule.postburner', {
+        schedule: Postburner::Instrumentation.schedule_payload(schedule),
+        bootstrapped: bootstrapped,
+        execution_created: execution_created,
+        orphans_enqueued: orphans_count
+      })
+
+      logger.debug "[Postburner::Scheduler] Schedule '#{schedule.name}': enqueued #{orphans_count} orphaned executions" if orphans_count > 0
     end
 
     # Ensure schedule has a future scheduled or pending execution.
@@ -372,7 +423,7 @@ module Postburner
     # at least one future execution ready to run.
     #
     # @param schedule [Postburner::Schedule] The schedule to ensure future execution for
-    # @return [void]
+    # @return [Boolean] true if a new execution was created, false otherwise
     #
     # @api private
     #
@@ -385,7 +436,7 @@ module Postburner
         # but handle it as a safety net
         logger.warn "[Postburner::Scheduler] Schedule '#{schedule.name}' has no executions, bootstrapping"
         schedule.start!
-        return
+        return true
       end
 
       # Delegate to Schedule - it knows whether a future execution is needed
@@ -393,6 +444,9 @@ module Postburner
 
       if execution
         logger.info "[Postburner::Scheduler] Created future execution for '#{schedule.name}'"
+        true
+      else
+        false
       end
     end