postburner 1.0.0.pre.13 → 1.0.0.pre.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61fa262497356b4b08226b9cb7138184c9a0b46958cb7a81d4c4ebd622be5487
-  data.tar.gz: 93bd5ef944fa62392546d96d607d02c076d94a2dc6e7d8f18fc32cfc90cdd4ec
+  metadata.gz: 71c6a81826b32021b0f0083c1771b27423e1b25198d7c00669cbab1150c431e7
+  data.tar.gz: 26403dba984c15f4bad94d81f1f5833d6a6bb16c0bb216a236f6bcaf07c0d019
 SHA512:
-  metadata.gz: e2129e509d590c58c9d46192f6a60ea4576af7e4e56f94fceceb3d39c0d8db8fbda7f4bb1e65f140b294d9b4378ea965ec728088517384034c07881cb186c6fe
-  data.tar.gz: 9807a50fd61a2c24e8fe0620cafbc4fd10d903cde1a7d2b5a21118452c709fd69578e211a152bd10752fef9ab82561524d3d5040ae896394f8bab07c0dbdf6df
+  metadata.gz: bc088b5d6e5b36b514562635ee3ed75955dfe809efe981e3ddbdc47c5f5387715233b7eb0eec8b4e78f97cf715202beabb7f8b90df553c605eec058eee462a9c
+  data.tar.gz: 40dbfc6f0979226d5c67780ffe31c73700a6a7b7faf97bf8e7fd127a2c889e708db91835bc9389032191ae2fae6460f3cc1b21f28ee17a6a3f00dbc84c31fbea

data/README.md

@@ -110,7 +110,6 @@ bundle exec rake postburner:work WORKER=default
 - [Why Beanstalkd?](#why-beanstalkd)
 - [Beanstalkd Integration](#beanstalkd-integration)
 - [Installation](#installation)
-- [Deployment](#deployment)
 - [Web UI - v2 Coming Soon](#web-ui)
 
 ## Why
@@ -787,36 +786,42 @@ Postburner uses different strategies to control job execution. These affect `Pos
 
 | Strategy | When to Use | Behavior | Requires Beanstalkd |
 |----------|-------------|----------|---------------------|
-| **NiceQueue** (default) | Production | Async via Beanstalkd, gracefully re-queues premature jobs | Yes |
-| **Queue** | Production (strict) | Async via Beanstalkd, raises error on premature execution | Yes |
-| **TestQueue** | Testing with explicit time control | Inline execution, raises error for scheduled jobs | No |
-| **ImmediateTestQueue** | Testing with automatic time travel | Inline execution, auto time-travels for scheduled jobs | No |
-| **NullQueue** | Batch processing / deferred execution | Jobs created but not queued, manual execution | No |
+| **DefaultQueue** | Production | Async, gracefully requeues premature jobs | Yes |
+| **StrictQueue** | Production | Async via Beanstalkd, raises error on premature execution | Yes |
+| **NullQueue** | Test (usually) | Jobs created but not inserted into Beanstalkd, manual execution | No |
+| **InlineTestQueue** | Test | Inline execution (error raised for delayed jobs) | No |
+| **TimeTravelTestQueue** | Test | Inline execution, (auto time-travels for delayed jobs) | No |
 
 ```ruby
 # Switch strategies
-Postburner.nice_async_strategy!           # Default production (NiceQueue)
-Postburner.async_strategy!                # Strict production (Queue)
-Postburner.inline_test_strategy!          # Testing (TestQueue)
-Postburner.inline_immediate_test_strategy! # Testing with time travel
-Postburner.null_strategy!                 # Deferred execution
+Postburner.default_strategy!              # Default production (DefaultQueue)
+Postburner.strict_strategy!               # Strict production (StrictQueue)
+Postburner.null_strategy!                 # Deferred, or manual execution
+Postburner.inline_test_strategy!          # Testing (InlineTestQueue)
+Postburner.time_travel_test_strategy!     # Testing inline but also with time travel
 ```
 
-**Note:** These strategies only affect `Postburner::Job` subclasses. ActiveJob classes execute according to the ActiveJob adapter configuration.
+### Adding a Queue Strategy
 
-## Testing
+There are 4 hook methods that are used with inheritance:
+- `insert` - Inserts the job into the queue
+- `handle_perform!` - Handles the job being performed
+- `handle_premature_perform` - Handles jobs executed before their scheduled run_at time
+- `testing` - Returns true if the strategy is for testing
 
-Postburner provides test-friendly execution modes that don't require Beanstalkd.
+## Testing
 
 ### Automatic Test Mode
 
-In Rails test environments, Postburner automatically uses inline execution:
+In Rails test environments, Postburner automatically uses inline execution via the `InlineTestQueue` strategy:
 
 ```ruby
 # test/test_helper.rb - automatic!
 Postburner.testing?  # => true in tests
 ```
 
+Postburner provides test-friendly execution modes that don't require Beanstalkd.
+
 ### Testing Default Jobs (ActiveJob)
 
 Use standard ActiveJob test helpers:
@@ -850,26 +855,47 @@ test "tracked job logs execution" do
 end
 ```
 
-### Testing Legacy Postburner::Job
+### Switching Queue Strategies
+
+For tests requiring specific queue behaviors, use `switch_queue_strategy!` and `restore_queue_strategy!`:
 
 ```ruby
-test "processes immediately" do
-  job = ProcessPayment.create!(args: { 'payment_id' => 123 })
-  job.queue!
+class ScheduleTest < ActiveSupport::TestCase
+  def setup
+    switch_queue_strategy! Postburner::TimeTravelTestQueue
+  end
+
+  def teardown
+    restore_queue_strategy!
+  end
 
-  assert job.reload.processed_at
+  test "scheduled job executes" do
+    job = MyJob.create!(args: {})
+    job.queue!(delay: 1.hour)
+    assert job.reload.processed_at  # Auto time-travels
+  end
 end
+```
 
-test "scheduled job with time travel" do
-  job = ProcessPayment.create!(args: { 'payment_id' => 123 })
+**Block form for isolated tests:**
 
-  travel_to(2.hours.from_now) do
-    job.queue!(delay: 2.hours)
-    assert job.reload.processed_at
+```ruby
+test "specific strategy for one test" do
+  use_queue_strategy Postburner::NullQueue do
+    job = MyJob.create!(args: {})
+    job.queue!
+    assert_nil job.bkid  # Not queued to Beanstalkd
   end
 end
 ```
 
+**Available strategies:**
+- `Postburner::InlineTestQueue` - Default test mode, raises on delayed jobs
+- `Postburner::TimeTravelTestQueue` - Auto time-travel for delayed jobs
+- `Postburner::NullQueue` - Create jobs without queueing
+- `Postburner::StrictQueue` - Production mode with error on premature execution
+- `Postburner::DefaultQueue` - Production mode with graceful requeue
+
 ## Workers
 
 Postburner uses named worker configurations to support different deployment patterns. Each worker can have different fork/thread settings and process different queues, enabling flexible production deployments.
@@ -989,6 +1015,8 @@ bin/postburner --worker general    # Run the 'general' worker
 bin/postburner --worker general --queues default,mailers  # Only process specific queues
 ```
 
+**Note:** Prefer defining separate workers in `config/postburner.yml` rather than using `--queues` overrides. Named workers make your deployment configuration explicit and version-controlled. Use `--queues` only for debugging or temporary overrides.
+
 **Rake task:**
 ```bash
 bundle exec rake postburner:work                          # Auto-select worker
@@ -1240,6 +1268,15 @@ production:  # <- environment config, i.e. defaults
         - mailers
 ```
 
+**Start workers with:**
+
+```bash
+bin/postburner                    # Auto-select single worker
+bin/postburner --worker default   # Specify worker
+rake postburner:work              # Rake task (auto-select)
+rake postburner:work WORKER=default
+```
+
 ### Queue Names
 
 Postburner automatically prefixes all queue names with `postburner.{env}.` to create Beanstalkd tube names. This namespacing prevents collisions when multiple applications share the same Beanstalkd server.
@@ -1372,16 +1409,86 @@ end
 
 ## Instrumentation
 
-Postburner emits ActiveSupport::Notifications events following ActiveJob conventions. Use these for monitoring, logging, or alerting.
+Postburner emits ActiveSupport::Notifications events following Rails conventions. Use these for monitoring, logging, or alerting.
+
+### Job Events
+
+| Event | When | Payload Keys |
+|-------|------|--------------|
+| `perform_start.job.postburner` | Before job execution begins | `:job`, `:beanstalk_job_id` |
+| `perform.job.postburner` | Around job execution (includes duration) | `:job`, `:beanstalk_job_id` |
+| `retry.job.postburner` | When job is retried after error | `:job`, `:beanstalk_job_id`, `:error`, `:wait`, `:attempt` |
+| `retry_stopped.job.postburner` | When tracked job exhausts retries | `:job`, `:beanstalk_job_id`, `:error` |
+| `discard.job.postburner` | When default job exhausts retries | `:job`, `:beanstalk_job_id`, `:error` |
+| `enqueue.job.postburner` | When job is enqueued for immediate execution | `:job` |
+| `enqueue_at.job.postburner` | When job is enqueued with delay | `:job`, `:scheduled_at` |
+
+**Job Payload Structure:**
+
+```ruby
+{
+  class: "ProcessPayment",           # Job class name
+  id: 123,                           # Postburner job ID (tracked jobs only)
+  job_id: "abc-123",                 # ActiveJob UUID
+  arguments: { payment_id: 456 },    # Job arguments
+  queue_name: "critical",            # Queue name
+  beanstalk_job_id: 789,             # Beanstalkd job ID
+  tracked: true                      # Whether job is tracked in PostgreSQL
+}
+```
 
-### Available Events
+### Schedule Events
 
 | Event | When | Payload Keys |
 |-------|------|--------------|
-| `perform_start.postburner` | Before job execution | `:payload`, `:beanstalk_job_id` |
-| `perform.postburner` | Around job execution (includes duration) | `:payload`, `:beanstalk_job_id` |
-| `retry.postburner` | When default job is retried | `:payload`, `:beanstalk_job_id`, `:error`, `:wait`, `:attempt` |
-| `discard.postburner` | When default job exhausts retries | `:payload`, `:beanstalk_job_id`, `:error` |
+| `create.schedule.postburner` | When schedule is created | `:schedule` |
+| `update.schedule.postburner` | When schedule is updated | `:schedule`, `:changes` |
+| `audit.schedule.postburner` | When scheduler audits a schedule | `:schedule` |
+
+**Schedule Payload Structure:**
+
+```ruby
+{
+  id: 1,
+  name: "daily_cleanup",
+  job_class: "CleanupJob",
+  enabled: true,
+  cron: nil,
+  anchor: "2025-01-01T09:00:00Z",
+  interval: 1,
+  interval_unit: "days"
+}
+```
+
+### Schedule Execution Events
+
+| Event | When | Payload Keys |
+|-------|------|--------------|
+| `create.schedule_execution.postburner` | When execution is created | `:schedule`, `:execution` |
+| `enqueue.schedule_execution.postburner` | When execution is enqueued to Beanstalkd | `:schedule`, `:execution`, `:beanstalk_job_id` |
+| `skip.schedule_execution.postburner` | When execution is skipped | `:schedule`, `:execution` |
+
+**Execution Payload Structure:**
+
+```ruby
+{
+  id: 42,
+  schedule_id: 1,
+  status: "scheduled",
+  run_at: "2025-01-15T09:00:00Z",
+  next_run_at: "2025-01-16T09:00:00Z",
+  enqueued_at: "2025-01-14T10:00:00Z",
+  beanstalk_job_id: 789,
+  job_id: 123
+}
+```
+
+### Scheduler Events
+
+| Event | When | Payload Keys |
+|-------|------|--------------|
+| `perform_start.scheduler.postburner` | Before scheduler watchdog runs | `:interval` |
+| `perform.scheduler.postburner` | Around scheduler watchdog (includes summary) | `:interval`, `:lock_acquired`, `:schedules_processed`, `:schedules_failed`, `:executions_created`, `:orphans_enqueued` |
 
 ### Subscribing to Events
 
@@ -1389,32 +1496,39 @@ Postburner emits ActiveSupport::Notifications events following ActiveJob convent
 # config/initializers/postburner_instrumentation.rb
 
 # Log all job executions
-ActiveSupport::Notifications.subscribe('perform.postburner') do |name, start, finish, id, payload|
+ActiveSupport::Notifications.subscribe('perform.job.postburner') do |name, start, finish, id, payload|
   duration = (finish - start) * 1000
-  Rails.logger.info "[Postburner] #{payload[:payload]['job_class']} completed in #{duration.round(2)}ms"
+  Rails.logger.info "[Postburner] #{payload[:job][:class]} completed in #{duration.round(2)}ms"
 end
 
 # Alert on discarded jobs
-ActiveSupport::Notifications.subscribe('discard.postburner') do |*args|
+ActiveSupport::Notifications.subscribe('discard.job.postburner') do |*args|
   payload = args.last
   Alerting.notify(
     "Job discarded after max retries",
-    job_class: payload[:payload]['job_class'],
+    job_class: payload[:job][:class],
     error: payload[:error].message
   )
 end
 
 # Track retry metrics
-ActiveSupport::Notifications.subscribe('retry.postburner') do |*args|
+ActiveSupport::Notifications.subscribe('retry.job.postburner') do |*args|
   payload = args.last
   StatsD.increment('postburner.retry', tags: [
-    "job:#{payload[:payload]['job_class']}",
+    "job:#{payload[:job][:class]}",
     "attempt:#{payload[:attempt]}"
   ])
 end
+
+# Monitor scheduler performance
+ActiveSupport::Notifications.subscribe('perform.scheduler.postburner') do |name, start, finish, id, payload|
+  duration = (finish - start) * 1000
+  Rails.logger.info "[Scheduler] Processed #{payload[:schedules_processed]} schedules, " \
+                    "created #{payload[:executions_created]} executions in #{duration.round(2)}ms"
+end
 ```
 
-**Note:** These events are emitted by the worker for jobs processed through Beanstalkd. They complement (don't replace) ActiveJob's built-in instrumentation events.
+**Note:** These events complement (don't replace) ActiveJob's built-in instrumentation events like `enqueue.active_job` and `perform.active_job`.
 
 ## Why Beanstalkd?
 
@@ -1960,49 +2074,6 @@ cp config/postburner.yml.example config/postburner.yml
 
 Edit `config/postburner.yml` for your environment (see [Configuration](#configuration)).
 
-## Deployment
-
-Some recipies that might be useful or instructive:
-
-### Docker
-
-```dockerfile
-# Dockerfile.worker
-FROM ruby:3.3
-
-WORKDIR /app
-COPY . .
-RUN bundle install
-
-CMD ["bundle", "exec", "postburner", "--config", "config/postburner.yml", "--env", "production"]
-```
-
-```yaml
-# docker-compose.yml
-services:
-  beanstalkd:
-    image: schickling/beanstalkd
-    command: ["-l", "0.0.0.0", "-p", "11300", "-b", "/var/lib/beanstalkd"]
-    ports:
-      - "11300:11300"
-    volumes:
-      - beanstalkd_data:/var/lib/beanstalkd
-
-  worker:
-    build:
-      context: .
-      dockerfile: Dockerfile.worker
-    depends_on:
-      - beanstalkd
-      - postgres
-    environment:
-      BEANSTALK_URL: beanstalk://beanstalkd:11300
-      DATABASE_URL: postgres://postgres@postgres/myapp_production
-
-volumes:
-  beanstalkd_data:
-```
-
 ## Migration from v0.x
 
 Key changes in v1.0:

data/app/concerns/postburner/insertion.rb

@@ -151,6 +151,10 @@ module Postburner
     # queueing (Beanstalkd in production, inline execution in test mode).
     # Updates bkid if the strategy returns a Beanstalkd job ID.
     #
+    # Instruments with ActiveSupport::Notifications:
+    # - enqueue.job.postburner: When job is queued immediately
+    # - enqueue_at.job.postburner: When job is queued with delay/at
+    #
     # @param options [Hash] Queue options (delay, pri, ttr, etc.)
     #
     # @return [Hash, nil] Queue strategy response
@@ -159,7 +163,6 @@ module Postburner
     #
     def insert!(options={})
       response = Postburner.queue_strategy.insert(self, options)
-      #debugger
 
       # Response must be a hash with an :id key (value can be nil)
       unless response.is_a?(Hash) && response.key?(:id)
@@ -168,6 +171,19 @@ module Postburner
 
       persist_metadata!(bkid: response[:id])
 
+      # Instrument enqueue event
+      job_payload = Postburner::Instrumentation.job_payload_from_model(self, beanstalk_job_id: response[:id])
+      if self.run_at.present? && self.run_at > Time.current
+        ActiveSupport::Notifications.instrument('enqueue_at.job.postburner', {
+          job: job_payload,
+          scheduled_at: self.run_at
+        })
+      else
+        ActiveSupport::Notifications.instrument('enqueue.job.postburner', {
+          job: job_payload
+        })
+      end
+
       self.log("QUEUED: #{response}") if response
 
       response

data/app/models/postburner/schedule.rb

@@ -114,6 +114,10 @@ module Postburner
     validate :validate_job_class_exists!
     validate :validate_cron_expression!, if: :cron?
 
+    # Instrumentation callbacks
+    after_create_commit :instrument_create
+    after_update_commit :instrument_update
+
     # Scopes
     scope :enabled, -> { where(enabled: true) }
     scope :disabled, -> { where(enabled: false) }
@@ -348,6 +352,9 @@ module Postburner
     # delayed queue will hold it until run_at. This ensures all future executions
     # are already queued and ready to go.
     #
+    # Instruments with ActiveSupport::Notifications:
+    # - create.schedule_execution.postburner: When execution is created
+    #
     # @param after [Time, ScheduleExecution, nil] Calculate after this time/execution
     # @return [ScheduleExecution, nil] The created execution, or nil if no more runs
     #
@@ -358,6 +365,13 @@ module Postburner
       return nil if execution.nil?
 
       execution.save!
+
+      # Instrument execution creation
+      ActiveSupport::Notifications.instrument('create.schedule_execution.postburner', {
+        schedule: Postburner::Instrumentation.schedule_payload(self),
+        execution: Postburner::Instrumentation.execution_payload(execution)
+      })
+
       execution.enqueue!
       execution
     end
@@ -718,5 +732,41 @@ module Postburner
         n += 1
       end
     end
+
+    # Instrument schedule creation.
+    #
+    # Emits create.schedule.postburner event with schedule payload and changes.
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def instrument_create
+      ActiveSupport::Notifications.instrument('create.schedule.postburner', {
+        schedule: Postburner::Instrumentation.schedule_payload(self),
+        changes: Postburner::Instrumentation.changes_payload(self, exclude: ['last_audit_at'])
+      })
+    end
+
+    # Instrument schedule update.
+    #
+    # Emits update.schedule.postburner event with schedule payload and changes.
+    # Excludes last_audit_at from changes to avoid noise from scheduler audits.
+    #
+    # @return [void]
+    #
+    # @api private
+    #
+    def instrument_update
+      changes = Postburner::Instrumentation.changes_payload(self, exclude: ['last_audit_at'])
+
+      # Skip instrumentation if only excluded attributes changed
+      return if changes.empty?
+
+      ActiveSupport::Notifications.instrument('update.schedule.postburner', {
+        schedule: Postburner::Instrumentation.schedule_payload(self),
+        changes: changes
+      })
+    end
   end
 end

data/app/models/postburner/schedule_execution.rb

@@ -99,6 +99,9 @@ module Postburner
     #
     # This method is idempotent - calling it multiple times will only enqueue once.
     #
+    # Instruments with ActiveSupport::Notifications:
+    # - enqueue.schedule_execution.postburner: When execution is enqueued
+    #
     # @return [void]
     #
     # @note Creating the next execution is the scheduler's responsibility via
@@ -128,6 +131,13 @@ module Postburner
           enqueue_default_job!
         end
       end
+
+      # Instrument enqueue event (after transaction commits)
+      ActiveSupport::Notifications.instrument('enqueue.schedule_execution.postburner', {
+        schedule: Postburner::Instrumentation.schedule_payload(schedule),
+        execution: Postburner::Instrumentation.execution_payload(self.reload),
+        beanstalk_job_id: beanstalk_job_id
+      })
     end
 
     # Skip this execution.
@@ -138,6 +148,9 @@ module Postburner
     # This is a destructive operation - skipped executions cannot be unskipped.
     # A new execution must be created if needed.
     #
+    # Instruments with ActiveSupport::Notifications:
+    # - skip.schedule_execution.postburner: When execution is skipped
+    #
     # @return [Boolean] true if skipped successfully, false if already skipped
     #
     # @example Skip a future execution
@@ -156,6 +169,12 @@ module Postburner
         update!(status: :skipped)
       end
 
+      # Instrument skip event (after transaction commits)
+      ActiveSupport::Notifications.instrument('skip.schedule_execution.postburner', {
+        schedule: Postburner::Instrumentation.schedule_payload(schedule),
+        execution: Postburner::Instrumentation.execution_payload(self)
+      })
+
       true
     end
 

data/bin/postburner

@@ -3,14 +3,22 @@
 
 # Postburner worker executable
 #
-# Loads configuration from YAML and starts the appropriate worker type.
+# Loads configuration from YAML and starts the appropriate worker.
 #
 # Usage:
-#   bin/postburner [--config PATH] [--env ENVIRONMENT]
+#   bin/postburner [options]
+#
+# Options:
+#   -c, --config PATH      Path to YAML config (default: config/postburner.yml)
+#   -e, --env ENVIRONMENT  Environment (default: RAILS_ENV, RACK_ENV, or development)
+#   -w, --worker WORKER    Worker name (required if multiple workers defined)
+#   -q, --queues QUEUES    Override configured queues (prefer defining workers)
 #
 # Examples:
 #   bin/postburner
-#   bin/postburner --config config/postburner.yml --env production
+#   bin/postburner --worker default
+#   bin/postburner --env production --worker imports
+#   bin/postburner --worker default --queues default,mailers
 #
 
 require 'optparse'
@@ -30,7 +38,7 @@ OptionParser.new do |opts|
     options[:config] = path
   end
 
-  opts.on('-e', '--env ENVIRONMENT', 'Environment (default: RAILS_ENV or development)') do |env|
+  opts.on('-e', '--env ENVIRONMENT', 'Environment (default: RAILS_ENV, RACK_ENV, or development)') do |env|
     options[:env] = env
   end
 
@@ -38,7 +46,7 @@ OptionParser.new do |opts|
     options[:worker] = worker
   end
 
-  opts.on('-q', '--queues QUEUES', 'Comma-separated list of queues to process (default: all configured queues)') do |queues|
+  opts.on('-q', '--queues QUEUES', 'Override configured queues, e.g. --queues default,mailers (prefer defining in yaml file, and selecting sets with --worker)') do |queues|
     options[:queues] = queues.split(',').map(&:strip)
   end