postburner 1.0.0.pre.19 → 1.0.0.pre.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bbfdd062766071e8ab4069cbf651b0ebaf946a89fc87620c73f284a0cf4c539
-  data.tar.gz: ea5d14000cab0fa2493fd810076ca31a9b6843c97efbdaf295933e5dae411df4
+  metadata.gz: a8b142c8e0e5a1f5ed9235d5514f262b2ca899900a359d4a0372a7dc484031cc
+  data.tar.gz: dcaa400529791fef277deba18df58fb1bbf92f306d3b08c1b0ccc8f48f86e927
 SHA512:
-  metadata.gz: 67d55a00a1cdc5e56aaca8087dde099f1e32f96ad3d4086f06eae5f3c694a27a013588e57f031c133ea779b6e8740bc5c0fd0267a7ad73af72628a0f173dde32
-  data.tar.gz: 5d5cd736f414f2fb24ed56ddbe46f4ff271e996ef5209d6d15a10b1cb2cefd61202aa3ec5e9355607e5751cca336a97bfa5414e7c231c69633a4398cdf49410e
+  metadata.gz: 3668f883407fb9671ab430c99a0d1dadda83e89c24bfa6ed336dd9e180ae3c4d32178f985c9d95ee244c380dc74d131990383e294af4ce5d5794a5fa180a5453
+  data.tar.gz: 676d4fd685ddf5cdd7be6ab1693231eb1bf4311d7707dcd85fc968b756eeda0b448bc6c4e6899c7032c17f684d2f4678503113cf50a52173851864ecd0e187d8

data/README.md

@@ -934,6 +934,7 @@ For tests requiring specific queue behaviors, use `switch_queue_strategy!` and `
 ```ruby
 class ScheduleTest < ActiveSupport::TestCase
   def setup
+    # Use time travel test strategy for inline execution with auto time travel
     switch_queue_strategy! Postburner::TimeTravelTestQueue
   end
 
@@ -953,10 +954,15 @@ end
 
 ```ruby
 test "specific strategy for one test" do
+  # Use the NullQueue to not push the job to Beanstalkd, only create the db record.
   use_queue_strategy Postburner::NullQueue do
     job = MyJob.create!(args: {})
     job.queue!
     assert_nil job.bkid  # Not queued to Beanstalkd
+
+    # The manually execute the job, you must use `travel_to` if it is scheduled for the future.
+    Postburner::Job.perform(job.id)
+    assert job.reload.processed_at # completed time
   end
 end
 ```
@@ -1489,8 +1495,8 @@ Postburner emits ActiveSupport::Notifications events following Rails conventions
 
 | Event | When | Payload Keys |
 |-------|------|--------------|
-| `perform_start.job.postburner` | Before job execution begins | `:job`, `:beanstalk_job_id` |
-| `perform.job.postburner` | Around job execution (includes duration) | `:job`, `:beanstalk_job_id` |
+| `perform_start.job.postburner` | Before job execution begins | `:job`, `:beanstalk_job_id`, `:gc_count`, `:gc_limit` |
+| `perform.job.postburner` | Around job execution (includes duration) | `:job`, `:beanstalk_job_id`, `:gc_count`, `:gc_limit` |
 | `retry.job.postburner` | When Postburner::Job is retried | `:job`, `:beanstalk_job_id`, `:error`, `:wait`, `:attempt` |
 | `retry_stopped.job.postburner` | When Postburner::Job exhausts retries (buried) | `:job`, `:beanstalk_job_id`, `:error` |
 | `discard.job.postburner` | When default ActiveJob fails (discarded) | `:job`, `:beanstalk_job_id`, `:error` |
@@ -1507,7 +1513,9 @@ Postburner emits ActiveSupport::Notifications events following Rails conventions
   arguments: { payment_id: 456 },    # Job arguments
   queue_name: "critical",            # Queue name
   beanstalk_job_id: 789,             # Beanstalkd job ID
-  tracked: true                      # Whether job is tracked in PostgreSQL
+  tracked: true,                     # Whether job is tracked in PostgreSQL
+  gc_count: 42,                      # Jobs processed in this thread (nil if unavailable)
+  gc_limit: 5000                     # GC limit for this worker (nil if unlimited)
 }
 ```
 
@@ -1557,12 +1565,48 @@ Postburner emits ActiveSupport::Notifications events following Rails conventions
 }
 ```
 
-### Scheduler Events
+### Watchdog Events (Worker Level)
+
+Emitted by the worker when executing the watchdog job from Beanstalkd:
+
+| Event | When | Payload Keys |
+|-------|------|--------------|
+| `perform_start.watchdog.postburner` | Before watchdog job execution begins | `:beanstalk_job_id`, `:interval`, `:watchdog`, `:gc_count`, `:gc_limit` |
+| `perform.watchdog.postburner` | Around watchdog job execution (includes duration) | `:beanstalk_job_id`, `:interval`, `:watchdog`, `:gc_count`, `:gc_limit` |
+
+**Watchdog Payload Structure:**
+
+```ruby
+{
+  beanstalk_job_id: 789,       # Beanstalkd job ID
+  interval: 300,               # Scheduler interval in seconds
+  watchdog: true,              # Identifier flag
+  gc_count: 42,                # Jobs processed (nil if unavailable)
+  gc_limit: 5000               # GC limit for worker (nil if unlimited)
+}
+```
+
+### Scheduler Events (Scheduler Level)
+
+Emitted by the Scheduler class when processing schedules (nested within watchdog execution):
 
 | Event | When | Payload Keys |
 |-------|------|--------------|
-| `perform_start.scheduler.postburner` | Before scheduler watchdog runs | `:interval` |
-| `perform.scheduler.postburner` | Around scheduler watchdog (includes summary) | `:interval`, `:lock_acquired`, `:schedules_processed`, `:schedules_failed`, `:executions_created`, `:orphans_enqueued` |
+| `perform_start.scheduler.postburner` | Before scheduler processes schedules | `:interval` |
+| `perform.scheduler.postburner` | Around scheduler run (includes summary) | `:interval`, `:lock_acquired`, `:schedules_processed`, `:schedules_failed`, `:executions_created`, `:orphans_enqueued` |
+
+**Scheduler Payload Structure:**
+
+```ruby
+{
+  interval: 300,               # Scheduler interval in seconds
+  lock_acquired: true,         # Whether advisory lock was acquired
+  schedules_processed: 5,      # Number of schedules processed
+  schedules_failed: 0,         # Number of schedules that failed
+  executions_created: 3,       # Number of executions created
+  orphans_enqueued: 1          # Number of orphaned executions enqueued
+}
+```
 
 ### Subscribing to Events
 
@@ -1594,7 +1638,14 @@ ActiveSupport::Notifications.subscribe('retry.job.postburner') do |*args|
   ])
 end
 
-# Monitor scheduler performance
+# Monitor watchdog job execution (worker level)
+ActiveSupport::Notifications.subscribe('perform.watchdog.postburner') do |name, start, finish, id, payload|
+  duration = (finish - start) * 1000
+  Rails.logger.info "[Watchdog] Job completed in #{duration.round(2)}ms " \
+                    "(interval: #{payload[:interval]}s, gc_count: #{payload[:gc_count]}/#{payload[:gc_limit]})"
+end
+
+# Monitor scheduler processing (scheduler level, nested within watchdog)
 ActiveSupport::Notifications.subscribe('perform.scheduler.postburner') do |name, start, finish, id, payload|
   duration = (finish - start) * 1000
   Rails.logger.info "[Scheduler] Processed #{payload[:schedules_processed]} schedules, " \

data/lib/postburner/instrumentation.rb

@@ -27,9 +27,13 @@ module Postburner
   # - `enqueue.schedule_execution.postburner` - Execution enqueued to Beanstalkd
   # - `skip.schedule_execution.postburner` - Execution skipped
   #
-  # ### Scheduler Watchdog Events
-  # - `perform_start.scheduler.postburner` - Watchdog run begins
-  # - `perform.scheduler.postburner` - Around watchdog run (summary)
+  # ### Watchdog Events (Worker Level)
+  # - `perform_start.watchdog.postburner` - Watchdog job execution begins
+  # - `perform.watchdog.postburner` - Around watchdog job execution
+  #
+  # ### Scheduler Events (Scheduler Level)
+  # - `perform_start.scheduler.postburner` - Scheduler processing begins
+  # - `perform.scheduler.postburner` - Around scheduler processing (summary)
   #
   # @example Subscribing to job events
   #   ActiveSupport::Notifications.subscribe('perform.job.postburner') do |name, start, finish, id, payload|

data/lib/postburner/scheduler.rb

@@ -303,9 +303,9 @@ module Postburner
 
       if response[:status] == "INSERTED"
         runs_at = Time.current + interval
-        Rails.logger.info "[Postburner::Scheduler] Inserted watchdog: #{response[:id]} (#{Time.current.iso8601}, delay: #{interval}s (#{runs_at.iso8601}), tube: #{tube_name})"
+        Rails.logger.info "[Postburner::Scheduler] Inserted watchdog (bkid: #{response[:id]}, interval: #{interval}s, runs_at: #{runs_at.iso8601}, tube: #{tube_name})"
       else
-        Rails.logger.error "[Postburner::Scheduler] Failed to insert watchdog: #{response.inspect} (delay: #{interval}s, tube: #{tube_name})"
+        Rails.logger.error "[Postburner::Scheduler] Failed to insert watchdog: #{response.inspect} (interval: #{interval}s, tube: #{tube_name})"
       end
 
       response

data/lib/postburner/version.rb

@@ -1,3 +1,3 @@
 module Postburner
-  VERSION = '1.0.0.pre.19'
+  VERSION = '1.0.0.pre.20'
 end

data/lib/postburner/worker.rb

@@ -112,6 +112,7 @@ module Postburner
       logger.info "[Postburner] #{config.beanstalk_url} known tubes: #{all_tubes.join(', ')}"
       log_next_scheduler_watchdog
       ensure_watchdog_on_startup!
+      log_tube_stats("Startup")
 
       if worker_config[:forks] > 0
         start_forked_mode
@@ -234,9 +235,9 @@ module Postburner
           time_left = stats.time_left.to_i
           at = Time.current + time_left
 
-          logger.info "[Postburner::Worker] Next scheduler watchdog at #{at.iso8601} (job #{job.id}, time-left: #{time_left}s); config: #{config.default_scheduler_interval}s"
+          logger.info "[Postburner::Worker] Watchdog found (bkid: #{job.id}, runs_at: #{at.iso8601}, time_left: #{time_left}s, config: #{config.default_scheduler_interval}s)"
         else
-          logger.info "[Postburner::Worker] Schedule watchdog not found (tube: #{tube_name})"
+          logger.info "[Postburner::Worker] Watchdog not found (tube: #{tube_name})"
         end
       end
     rescue => e
@@ -254,7 +255,7 @@ module Postburner
     def start_single_process_mode
       logger.info "[Postburner::Worker] Mode: Single process (forks: 0)"
 
-      @jobs_processed = Concurrent::AtomicFixnum.new(0)
+      @gc_count = Concurrent::AtomicFixnum.new(0)
       @gc_limit = worker_config[:gc_limit]
 
       thread_count = worker_config[:threads]
@@ -264,7 +265,7 @@ module Postburner
         @pool.post { process_jobs }
       end
 
-      until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
+      until shutdown? || (@gc_limit && @gc_count.value >= @gc_limit)
         sleep 0.5
       end
 
@@ -272,10 +273,12 @@ module Postburner
       @pool.shutdown
       @pool.wait_for_termination(worker_config[:shutdown_timeout])
 
-      if @gc_limit && @jobs_processed.value >= @gc_limit
-        logger.debug "[Postburner::Worker] Reached GC limit (#{@jobs_processed.value} jobs), exiting for restart..."
+      if @gc_limit && @gc_count.value >= @gc_limit
+        logger.debug "[Postburner::Worker] Reached GC limit (#{@gc_count.value} jobs), exiting for restart..."
+        log_tube_stats("Shutdown (GC limit)")
         exit 99
       else
+        log_tube_stats("Shutdown")
         logger.info "[Postburner::Worker] Shutdown complete"
       end
     end
@@ -293,16 +296,21 @@ module Postburner
       timeout = worker_config[:timeout]
       reconnect_attempts = 0
 
+      # Set thread-local gc_limit for execution methods
+      Thread.current[:gc_limit] = @gc_limit
+
       watch_queues(connection, config.queue_names)
 
-      until shutdown? || (@gc_limit && @jobs_processed.value >= @gc_limit)
+      until shutdown? || (@gc_limit && @gc_count.value >= @gc_limit)
         begin
           job = connection.beanstalk.tubes.reserve(timeout)
 
           if job
             logger.debug "[Postburner::Worker] Thread #{Thread.current.object_id} reserved job #{job.id}"
+            # Set current count before execution so it's available in execute_job
+            Thread.current[:gc_count] = @gc_count.value
             execute_job(job)
-            @jobs_processed.increment
+            @gc_count.increment
             reconnect_attempts = 0  # Reset backoff on successful job execution
           else
             ensure_scheduler_watchdog!(connection)
@@ -372,6 +380,7 @@ module Postburner
 
       logger.info "[Postburner::Worker] Shutting down, waiting for children..."
       shutdown_children
+      log_tube_stats("Shutdown")
       logger.info "[Postburner::Worker] Shutdown complete"
     end
 
@@ -401,14 +410,14 @@ module Postburner
 
       logger.info "[Postburner::Worker] Fork #{fork_num}: #{thread_count} threads, GC limit #{gc_limit || 'unlimited'}"
 
-      jobs_processed = Concurrent::AtomicFixnum.new(0)
+      gc_count = Concurrent::AtomicFixnum.new(0)
       pool = Concurrent::FixedThreadPool.new(thread_count)
 
       thread_count.times do
-        pool.post { process_jobs_in_fork(fork_num, jobs_processed, gc_limit) }
+        pool.post { process_jobs_in_fork(fork_num, gc_count, gc_limit) }
       end
 
-      until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
+      until shutdown? || (gc_limit && gc_count.value >= gc_limit)
         if orphaned?
           logger.error "[Postburner::Worker] Fork #{fork_num} detected parent died (orphaned), initiating shutdown"
           shutdown
@@ -420,8 +429,8 @@ module Postburner
       pool.shutdown
       pool.wait_for_termination(worker_config[:shutdown_timeout])
 
-      if gc_limit && jobs_processed.value >= gc_limit
-        logger.debug "[Postburner::Worker] Fork #{fork_num} reached GC limit (#{jobs_processed.value} jobs), exiting for restart..."
+      if gc_limit && gc_count.value >= gc_limit
+        logger.debug "[Postburner::Worker] Fork #{fork_num} reached GC limit (#{gc_count.value} jobs), exiting for restart..."
         exit 99
       else
         logger.info "[Postburner::Worker] Fork #{fork_num} shutting down gracefully..."
@@ -439,25 +448,30 @@ module Postburner
     # tracking jobs processed across all threads in the fork.
     #
     # @param fork_num [Integer] Fork identifier for logging
-    # @param jobs_processed [Concurrent::AtomicFixnum] Shared job counter
+    # @param gc_count [Concurrent::AtomicFixnum] Shared job counter
     # @param gc_limit [Integer, nil] Maximum jobs before exit (nil = unlimited)
     # @return [void]
     # @api private
-    def process_jobs_in_fork(fork_num, jobs_processed, gc_limit)
+    def process_jobs_in_fork(fork_num, gc_count, gc_limit)
       connection = Postburner::Connection.new
       timeout = worker_config[:timeout]
       reconnect_attempts = 0
 
+      # Set thread-local gc_limit for execution methods
+      Thread.current[:gc_limit] = gc_limit
+
       watch_queues(connection, config.queue_names)
 
-      until shutdown? || (gc_limit && jobs_processed.value >= gc_limit)
+      until shutdown? || (gc_limit && gc_count.value >= gc_limit)
         begin
           job = connection.beanstalk.tubes.reserve(timeout)
 
           if job
             logger.debug "[Postburner::Worker] Fork #{fork_num} thread #{Thread.current.object_id} reserved job #{job.id}"
+            # Set current count before execution so it's available in execute_job
+            Thread.current[:gc_count] = gc_count.value
             execute_job(job)
-            jobs_processed.increment
+            gc_count.increment
             reconnect_attempts = 0  # Reset backoff on successful job execution
           else
             ensure_scheduler_watchdog!(connection)
@@ -538,22 +552,39 @@ module Postburner
     # Instantiates the Scheduler and runs it to process due schedules.
     # Deletes the job from Beanstalkd after completion.
     #
+    # Instruments with ActiveSupport::Notifications:
+    # - perform_start.watchdog.postburner: Before watchdog execution
+    # - perform.watchdog.postburner: Around watchdog execution (includes duration)
+    #
     # @param beanstalk_job [Beaneater::Job] Reserved scheduler job
     # @param payload [Hash] Parsed job body with 'scheduler' and 'interval' keys
     # @return [void]
     # @see Postburner::Scheduler#perform
     # @api private
     def execute_scheduler_job(beanstalk_job, payload)
-      logger.info "[Postburner] Executing scheduler watchdog #{beanstalk_job.id}"
-
       interval = payload['interval'] || 300
-      scheduler = Postburner::Scheduler.new(interval: interval, logger: logger)
 
-      Rails.application.reloader.wrap do
-        scheduler.perform
+      # Build instrumentation payload
+      worker_stats = thread_worker_stats
+      instrument_payload = {
+        beanstalk_job_id: beanstalk_job.id,
+        interval: interval,
+        watchdog: true
+      }.merge(worker_stats)
+
+      logger.info "[Postburner] Executing scheduler watchdog (bkid: #{beanstalk_job.id})"
+
+      ActiveSupport::Notifications.instrument('perform_start.watchdog.postburner', instrument_payload)
+
+      ActiveSupport::Notifications.instrument('perform.watchdog.postburner', instrument_payload) do
+        scheduler = Postburner::Scheduler.new(interval: interval, logger: logger)
+
+        Rails.application.reloader.wrap do
+          scheduler.perform
+        end
       end
 
-      logger.info "[Postburner] Completed scheduler watchdog #{beanstalk_job.id}"
+      logger.info "[Postburner] Completed watchdog (bkid: #{beanstalk_job.id}, #{formatted_thread_worker_stats})"
       delete_job!(beanstalk_job)
     end
 
@@ -575,7 +606,13 @@ module Postburner
     def execute_regular_job(beanstalk_job, payload)
       job_description = format_job_description(payload)
       job_payload = Postburner::Instrumentation.job_payload_from_hash(payload, beanstalk_job_id: beanstalk_job.id)
-      instrument_payload = { job: job_payload, beanstalk_job_id: beanstalk_job.id }
+
+      # Add worker stats to instrumentation payload
+      worker_stats = thread_worker_stats
+      instrument_payload = {
+        job: job_payload,
+        beanstalk_job_id: beanstalk_job.id
+      }.merge(worker_stats)
 
       logger.info "[Postburner] Executing #{job_description} (bkid: #{beanstalk_job.id})"
 
@@ -587,7 +624,7 @@ module Postburner
         end
       end
 
-      logger.info "[Postburner] Completed #{job_description} (bkid: #{beanstalk_job.id})"
+      logger.info "[Postburner] Completed #{job_description} (bkid: #{beanstalk_job.id}, #{formatted_thread_worker_stats})"
       delete_job!(beanstalk_job)
     end
 
@@ -799,5 +836,67 @@ module Postburner
       sleep 1
       beanstalk_job.delete rescue nil
     end
+
+    # Returns worker stats from thread-local variables as a hash.
+    #
+    # Used for instrumentation payload to make stats available to
+    # ActiveSupport::Notifications subscribers. Adds 1 to gc_count
+    # to reflect that the current job is being executed.
+    #
+    # @return [Hash] Hash with :gc_count and :gc_limit keys
+    # @api private
+    def thread_worker_stats
+      gc_count = Thread.current[:gc_count]
+      {
+        gc_count: gc_count ? gc_count + 1 : nil,
+        gc_limit: Thread.current[:gc_limit]
+      }
+    end
+
+    # Formats worker stats for logging.
+    #
+    # Produces a string like "(jobs: 42/5000)" showing the current
+    # job count and gc_limit. Returns empty string if no stats available.
+    # Delegates to thread_worker_stats for the actual counts.
+    #
+    # @return [String] Formatted stats string or empty string
+    # @api private
+    def formatted_thread_worker_stats
+      stats = thread_worker_stats
+      gc_count = stats[:gc_count]
+      gc_limit = stats[:gc_limit]
+
+      return "gc_count: unlimited" unless gc_count
+
+      if gc_limit
+        "gc_count: #{gc_count}, gc_limit: #{gc_limit}"
+      else
+        "gc_count: #{gc_count}"
+      end
+    end
+
+    # Logs Beanstalkd tube statistics with formatted breakdown.
+    #
+    # Shows aggregate totals on first line, followed by per-tube breakdown.
+    # Used at worker startup and shutdown to track job backlog.
+    #
+    # @param label [String] Label for the stats (e.g., "Startup", "Shutdown")
+    # @return [void]
+    # @api private
+    def log_tube_stats(label)
+      tube_names = config.expanded_tube_names + [config.scheduler_tube_name]
+      stats = Postburner.stats(tube_names)
+
+      # Log totals on first line
+      totals = stats[:totals]
+      logger.info "[Postburner::Worker] #{label} stats: ready=#{totals[:ready]} delayed=#{totals[:delayed]} buried=#{totals[:buried]} reserved=#{totals[:reserved]} total=#{totals[:total]}"
+
+      # Log per-tube breakdown
+      stats[:tubes].each do |tube|
+        logger.info "  #{tube[:name]}: ready=#{tube[:ready]} delayed=#{tube[:delayed]} buried=#{tube[:buried]} reserved=#{tube[:reserved]} total=#{tube[:total]}"
+      end
+    rescue => e
+      logger.warn "[Postburner::Worker] Failed to retrieve tube stats: #{e.message}"
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: postburner
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.19
+  version: 1.0.0.pre.20
 platform: ruby
 authors:
 - Matt Smith