postburner 1.0.0.pre.18 → 1.0.0.pre.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +68 -10
- data/lib/postburner/instrumentation.rb +7 -3
- data/lib/postburner/scheduler.rb +2 -2
- data/lib/postburner/version.rb +1 -1
- data/lib/postburner/worker.rb +149 -25
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a8b142c8e0e5a1f5ed9235d5514f262b2ca899900a359d4a0372a7dc484031cc
|
|
4
|
+
data.tar.gz: dcaa400529791fef277deba18df58fb1bbf92f306d3b08c1b0ccc8f48f86e927
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3668f883407fb9671ab430c99a0d1dadda83e89c24bfa6ed336dd9e180ae3c4d32178f985c9d95ee244c380dc74d131990383e294af4ce5d5794a5fa180a5453
|
|
7
|
+
data.tar.gz: 676d4fd685ddf5cdd7be6ab1693231eb1bf4311d7707dcd85fc968b756eeda0b448bc6c4e6899c7032c17f684d2f4678503113cf50a52173851864ecd0e187d8
|
data/README.md
CHANGED
|
@@ -126,7 +126,7 @@ Postburner [beanstalkd](https://beanstalkd.github.io/) is used with PostgreSQL t
|
|
|
126
126
|
|
|
127
127
|
```ruby
|
|
128
128
|
# Gemfile
|
|
129
|
-
gem 'postburner', '~> 1.0.0.pre.
|
|
129
|
+
gem 'postburner', '~> 1.0.0.pre.18'
|
|
130
130
|
|
|
131
131
|
# config/application.rb
|
|
132
132
|
config.active_job.queue_adapter = :postburner
|
|
@@ -934,6 +934,7 @@ For tests requiring specific queue behaviors, use `switch_queue_strategy!` and `
|
|
|
934
934
|
```ruby
|
|
935
935
|
class ScheduleTest < ActiveSupport::TestCase
|
|
936
936
|
def setup
|
|
937
|
+
# Use time travel test strategy for inline execution with auto time travel
|
|
937
938
|
switch_queue_strategy! Postburner::TimeTravelTestQueue
|
|
938
939
|
end
|
|
939
940
|
|
|
@@ -953,10 +954,15 @@ end
|
|
|
953
954
|
|
|
954
955
|
```ruby
|
|
955
956
|
test "specific strategy for one test" do
|
|
957
|
+
# Use the NullQueue to not push the job to Beanstalkd, only create the db record.
|
|
956
958
|
use_queue_strategy Postburner::NullQueue do
|
|
957
959
|
job = MyJob.create!(args: {})
|
|
958
960
|
job.queue!
|
|
959
961
|
assert_nil job.bkid # Not queued to Beanstalkd
|
|
962
|
+
|
|
963
|
+
# The manually execute the job, you must use `travel_to` if it is scheduled for the future.
|
|
964
|
+
Postburner::Job.perform(job.id)
|
|
965
|
+
assert job.reload.processed_at # completed time
|
|
960
966
|
end
|
|
961
967
|
end
|
|
962
968
|
```
|
|
@@ -1489,8 +1495,8 @@ Postburner emits ActiveSupport::Notifications events following Rails conventions
|
|
|
1489
1495
|
|
|
1490
1496
|
| Event | When | Payload Keys |
|
|
1491
1497
|
|-------|------|--------------|
|
|
1492
|
-
| `perform_start.job.postburner` | Before job execution begins | `:job`, `:beanstalk_job_id` |
|
|
1493
|
-
| `perform.job.postburner` | Around job execution (includes duration) | `:job`, `:beanstalk_job_id` |
|
|
1498
|
+
| `perform_start.job.postburner` | Before job execution begins | `:job`, `:beanstalk_job_id`, `:gc_count`, `:gc_limit` |
|
|
1499
|
+
| `perform.job.postburner` | Around job execution (includes duration) | `:job`, `:beanstalk_job_id`, `:gc_count`, `:gc_limit` |
|
|
1494
1500
|
| `retry.job.postburner` | When Postburner::Job is retried | `:job`, `:beanstalk_job_id`, `:error`, `:wait`, `:attempt` |
|
|
1495
1501
|
| `retry_stopped.job.postburner` | When Postburner::Job exhausts retries (buried) | `:job`, `:beanstalk_job_id`, `:error` |
|
|
1496
1502
|
| `discard.job.postburner` | When default ActiveJob fails (discarded) | `:job`, `:beanstalk_job_id`, `:error` |
|
|
@@ -1507,7 +1513,9 @@ Postburner emits ActiveSupport::Notifications events following Rails conventions
|
|
|
1507
1513
|
arguments: { payment_id: 456 }, # Job arguments
|
|
1508
1514
|
queue_name: "critical", # Queue name
|
|
1509
1515
|
beanstalk_job_id: 789, # Beanstalkd job ID
|
|
1510
|
-
tracked: true
|
|
1516
|
+
tracked: true, # Whether job is tracked in PostgreSQL
|
|
1517
|
+
gc_count: 42, # Jobs processed in this thread (nil if unavailable)
|
|
1518
|
+
gc_limit: 5000 # GC limit for this worker (nil if unlimited)
|
|
1511
1519
|
}
|
|
1512
1520
|
```
|
|
1513
1521
|
|
|
@@ -1557,12 +1565,48 @@ Postburner emits ActiveSupport::Notifications events following Rails conventions
|
|
|
1557
1565
|
}
|
|
1558
1566
|
```
|
|
1559
1567
|
|
|
1560
|
-
###
|
|
1568
|
+
### Watchdog Events (Worker Level)
|
|
1569
|
+
|
|
1570
|
+
Emitted by the worker when executing the watchdog job from Beanstalkd:
|
|
1561
1571
|
|
|
1562
1572
|
| Event | When | Payload Keys |
|
|
1563
1573
|
|-------|------|--------------|
|
|
1564
|
-
| `perform_start.
|
|
1565
|
-
| `perform.
|
|
1574
|
+
| `perform_start.watchdog.postburner` | Before watchdog job execution begins | `:beanstalk_job_id`, `:interval`, `:watchdog`, `:gc_count`, `:gc_limit` |
|
|
1575
|
+
| `perform.watchdog.postburner` | Around watchdog job execution (includes duration) | `:beanstalk_job_id`, `:interval`, `:watchdog`, `:gc_count`, `:gc_limit` |
|
|
1576
|
+
|
|
1577
|
+
**Watchdog Payload Structure:**
|
|
1578
|
+
|
|
1579
|
+
```ruby
|
|
1580
|
+
{
|
|
1581
|
+
beanstalk_job_id: 789, # Beanstalkd job ID
|
|
1582
|
+
interval: 300, # Scheduler interval in seconds
|
|
1583
|
+
watchdog: true, # Identifier flag
|
|
1584
|
+
gc_count: 42, # Jobs processed (nil if unavailable)
|
|
1585
|
+
gc_limit: 5000 # GC limit for worker (nil if unlimited)
|
|
1586
|
+
}
|
|
1587
|
+
```
|
|
1588
|
+
|
|
1589
|
+
### Scheduler Events (Scheduler Level)
|
|
1590
|
+
|
|
1591
|
+
Emitted by the Scheduler class when processing schedules (nested within watchdog execution):
|
|
1592
|
+
|
|
1593
|
+
| Event | When | Payload Keys |
|
|
1594
|
+
|-------|------|--------------|
|
|
1595
|
+
| `perform_start.scheduler.postburner` | Before scheduler processes schedules | `:interval` |
|
|
1596
|
+
| `perform.scheduler.postburner` | Around scheduler run (includes summary) | `:interval`, `:lock_acquired`, `:schedules_processed`, `:schedules_failed`, `:executions_created`, `:orphans_enqueued` |
|
|
1597
|
+
|
|
1598
|
+
**Scheduler Payload Structure:**
|
|
1599
|
+
|
|
1600
|
+
```ruby
|
|
1601
|
+
{
|
|
1602
|
+
interval: 300, # Scheduler interval in seconds
|
|
1603
|
+
lock_acquired: true, # Whether advisory lock was acquired
|
|
1604
|
+
schedules_processed: 5, # Number of schedules processed
|
|
1605
|
+
schedules_failed: 0, # Number of schedules that failed
|
|
1606
|
+
executions_created: 3, # Number of executions created
|
|
1607
|
+
orphans_enqueued: 1 # Number of orphaned executions enqueued
|
|
1608
|
+
}
|
|
1609
|
+
```
|
|
1566
1610
|
|
|
1567
1611
|
### Subscribing to Events
|
|
1568
1612
|
|
|
@@ -1594,7 +1638,14 @@ ActiveSupport::Notifications.subscribe('retry.job.postburner') do |*args|
|
|
|
1594
1638
|
])
|
|
1595
1639
|
end
|
|
1596
1640
|
|
|
1597
|
-
# Monitor
|
|
1641
|
+
# Monitor watchdog job execution (worker level)
|
|
1642
|
+
ActiveSupport::Notifications.subscribe('perform.watchdog.postburner') do |name, start, finish, id, payload|
|
|
1643
|
+
duration = (finish - start) * 1000
|
|
1644
|
+
Rails.logger.info "[Watchdog] Job completed in #{duration.round(2)}ms " \
|
|
1645
|
+
"(interval: #{payload[:interval]}s, gc_count: #{payload[:gc_count]}/#{payload[:gc_limit]})"
|
|
1646
|
+
end
|
|
1647
|
+
|
|
1648
|
+
# Monitor scheduler processing (scheduler level, nested within watchdog)
|
|
1598
1649
|
ActiveSupport::Notifications.subscribe('perform.scheduler.postburner') do |name, start, finish, id, payload|
|
|
1599
1650
|
duration = (finish - start) * 1000
|
|
1600
1651
|
Rails.logger.info "[Scheduler] Processed #{payload[:schedules_processed]} schedules, " \
|
|
@@ -2159,7 +2210,7 @@ beanstalkd -l 127.0.0.1 -p 11300 -b /var/lib/beanstalkd
|
|
|
2159
2210
|
|
|
2160
2211
|
```ruby
|
|
2161
2212
|
# Gemfile
|
|
2162
|
-
gem 'postburner', '~> 1.0.0.pre.
|
|
2213
|
+
gem 'postburner', '~> 1.0.0.pre.18'
|
|
2163
2214
|
```
|
|
2164
2215
|
|
|
2165
2216
|
```bash
|
|
@@ -2194,6 +2245,13 @@ Edit `config/postburner.yml` for your environment (see [Configuration](#configur
|
|
|
2194
2245
|
|
|
2195
2246
|
Key changes in v1.0:
|
|
2196
2247
|
|
|
2248
|
+
### Changed
|
|
2249
|
+
- `Postburner::Job#backburner_job` -> `Postburner::Job#bk`
|
|
2250
|
+
- `queue_priority` → `priority`
|
|
2251
|
+
- `queue_ttr` → `ttr`
|
|
2252
|
+
- `queue_max_job_retries` → `max_retries`
|
|
2253
|
+
- `queue_retry_delay` → `retry_delay`
|
|
2254
|
+
|
|
2197
2255
|
### Removed
|
|
2198
2256
|
- Backburner dependency
|
|
2199
2257
|
- `config/initializers/backburner.rb`
|
|
@@ -2210,7 +2268,7 @@ Key changes in v1.0:
|
|
|
2210
2268
|
|
|
2211
2269
|
1. **Update Gemfile:**
|
|
2212
2270
|
```ruby
|
|
2213
|
-
gem 'postburner', '~> 1.0.0.pre.
|
|
2271
|
+
gem 'postburner', '~> 1.0.0.pre.18'
|
|
2214
2272
|
```
|
|
2215
2273
|
|
|
2216
2274
|
2. **Remove Backburner config:**
|
|
@@ -27,9 +27,13 @@ module Postburner
|
|
|
27
27
|
# - `enqueue.schedule_execution.postburner` - Execution enqueued to Beanstalkd
|
|
28
28
|
# - `skip.schedule_execution.postburner` - Execution skipped
|
|
29
29
|
#
|
|
30
|
-
# ###
|
|
31
|
-
# - `perform_start.
|
|
32
|
-
# - `perform.
|
|
30
|
+
# ### Watchdog Events (Worker Level)
|
|
31
|
+
# - `perform_start.watchdog.postburner` - Watchdog job execution begins
|
|
32
|
+
# - `perform.watchdog.postburner` - Around watchdog job execution
|
|
33
|
+
#
|
|
34
|
+
# ### Scheduler Events (Scheduler Level)
|
|
35
|
+
# - `perform_start.scheduler.postburner` - Scheduler processing begins
|
|
36
|
+
# - `perform.scheduler.postburner` - Around scheduler processing (summary)
|
|
33
37
|
#
|
|
34
38
|
# @example Subscribing to job events
|
|
35
39
|
# ActiveSupport::Notifications.subscribe('perform.job.postburner') do |name, start, finish, id, payload|
|
data/lib/postburner/scheduler.rb
CHANGED
|
@@ -303,9 +303,9 @@ module Postburner
|
|
|
303
303
|
|
|
304
304
|
if response[:status] == "INSERTED"
|
|
305
305
|
runs_at = Time.current + interval
|
|
306
|
-
Rails.logger.info "[Postburner::Scheduler] Inserted watchdog: #{response[:id]}
|
|
306
|
+
Rails.logger.info "[Postburner::Scheduler] Inserted watchdog (bkid: #{response[:id]}, interval: #{interval}s, runs_at: #{runs_at.iso8601}, tube: #{tube_name})"
|
|
307
307
|
else
|
|
308
|
-
Rails.logger.error "[Postburner::Scheduler] Failed to insert watchdog: #{response.inspect} (
|
|
308
|
+
Rails.logger.error "[Postburner::Scheduler] Failed to insert watchdog: #{response.inspect} (interval: #{interval}s, tube: #{tube_name})"
|
|
309
309
|
end
|
|
310
310
|
|
|
311
311
|
response
|
data/lib/postburner/version.rb
CHANGED
data/lib/postburner/worker.rb
CHANGED
|
@@ -107,10 +107,12 @@ module Postburner
|
|
|
107
107
|
logger.info "[Postburner::Worker] Starting worker '#{worker_config[:name]}'..."
|
|
108
108
|
logger.info "[Postburner::Worker] Queues: #{config.queue_names.join(', ')}"
|
|
109
109
|
logger.info "[Postburner::Worker] Config: #{worker_config[:forks]} forks, #{worker_config[:threads]} threads, gc_limit: #{worker_config[:gc_limit] || 'unlimited'}, timeout: #{worker_config[:timeout]}s"
|
|
110
|
+
log_reloading_status
|
|
110
111
|
all_tubes = config.expanded_tube_names + [config.scheduler_tube_name]
|
|
111
112
|
logger.info "[Postburner] #{config.beanstalk_url} known tubes: #{all_tubes.join(', ')}"
|
|
112
113
|
log_next_scheduler_watchdog
|
|
113
114
|
ensure_watchdog_on_startup!
|
|
115
|
+
log_tube_stats("Startup")
|
|
114
116
|
|
|
115
117
|
if worker_config[:forks] > 0
|
|
116
118
|
start_forked_mode
|
|
@@ -192,6 +194,25 @@ module Postburner
|
|
|
192
194
|
config.expand_tube_name(queue_name)
|
|
193
195
|
end
|
|
194
196
|
|
|
197
|
+
# Logs whether Rails code reloading is enabled.
|
|
198
|
+
#
|
|
199
|
+
# In development, Rails reloads code between job executions, allowing
|
|
200
|
+
# code changes to take effect without restarting the worker.
|
|
201
|
+
#
|
|
202
|
+
# @return [void]
|
|
203
|
+
# @api private
|
|
204
|
+
def log_reloading_status
|
|
205
|
+
reloading_enabled = if Rails.application.config.respond_to?(:enable_reloading)
|
|
206
|
+
Rails.application.config.enable_reloading
|
|
207
|
+
else
|
|
208
|
+
!Rails.application.config.cache_classes
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
if reloading_enabled
|
|
212
|
+
logger.info "[Postburner::Worker] Code reloading enabled (development mode)"
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
|
|
195
216
|
# Logs info about the next scheduler watchdog job.
|
|
196
217
|
#
|
|
197
218
|
# Peeks at the scheduler tube to find the next watchdog job and logs
|
|
@@ -214,9 +235,9 @@ module Postburner
|
|
|
214
235
|
time_left = stats.time_left.to_i
|
|
215
236
|
at = Time.current + time_left
|
|
216
237
|
|
|
217
|
-
logger.info "[Postburner::Worker]
|
|
238
|
+
logger.info "[Postburner::Worker] Watchdog found (bkid: #{job.id}, runs_at: #{at.iso8601}, time_left: #{time_left}s, config: #{config.default_scheduler_interval}s)"
|
|
218
239
|
else
|
|
219
|
-
logger.info "[Postburner::Worker]
|
|
240
|
+
logger.info "[Postburner::Worker] Watchdog not found (tube: #{tube_name})"
|
|
220
241
|
end
|
|
221
242
|
end
|
|
222
243
|
rescue => e
|
|
@@ -234,7 +255,7 @@ module Postburner
|
|
|
234
255
|
def start_single_process_mode
|
|
235
256
|
logger.info "[Postburner::Worker] Mode: Single process (forks: 0)"
|
|
236
257
|
|
|
237
|
-
@
|
|
258
|
+
@gc_count = Concurrent::AtomicFixnum.new(0)
|
|
238
259
|
@gc_limit = worker_config[:gc_limit]
|
|
239
260
|
|
|
240
261
|
thread_count = worker_config[:threads]
|
|
@@ -244,7 +265,7 @@ module Postburner
|
|
|
244
265
|
@pool.post { process_jobs }
|
|
245
266
|
end
|
|
246
267
|
|
|
247
|
-
until shutdown? || (@gc_limit && @
|
|
268
|
+
until shutdown? || (@gc_limit && @gc_count.value >= @gc_limit)
|
|
248
269
|
sleep 0.5
|
|
249
270
|
end
|
|
250
271
|
|
|
@@ -252,10 +273,12 @@ module Postburner
|
|
|
252
273
|
@pool.shutdown
|
|
253
274
|
@pool.wait_for_termination(worker_config[:shutdown_timeout])
|
|
254
275
|
|
|
255
|
-
if @gc_limit && @
|
|
256
|
-
logger.debug "[Postburner::Worker] Reached GC limit (#{@
|
|
276
|
+
if @gc_limit && @gc_count.value >= @gc_limit
|
|
277
|
+
logger.debug "[Postburner::Worker] Reached GC limit (#{@gc_count.value} jobs), exiting for restart..."
|
|
278
|
+
log_tube_stats("Shutdown (GC limit)")
|
|
257
279
|
exit 99
|
|
258
280
|
else
|
|
281
|
+
log_tube_stats("Shutdown")
|
|
259
282
|
logger.info "[Postburner::Worker] Shutdown complete"
|
|
260
283
|
end
|
|
261
284
|
end
|
|
@@ -273,16 +296,21 @@ module Postburner
|
|
|
273
296
|
timeout = worker_config[:timeout]
|
|
274
297
|
reconnect_attempts = 0
|
|
275
298
|
|
|
299
|
+
# Set thread-local gc_limit for execution methods
|
|
300
|
+
Thread.current[:gc_limit] = @gc_limit
|
|
301
|
+
|
|
276
302
|
watch_queues(connection, config.queue_names)
|
|
277
303
|
|
|
278
|
-
until shutdown? || (@gc_limit && @
|
|
304
|
+
until shutdown? || (@gc_limit && @gc_count.value >= @gc_limit)
|
|
279
305
|
begin
|
|
280
306
|
job = connection.beanstalk.tubes.reserve(timeout)
|
|
281
307
|
|
|
282
308
|
if job
|
|
283
309
|
logger.debug "[Postburner::Worker] Thread #{Thread.current.object_id} reserved job #{job.id}"
|
|
310
|
+
# Set current count before execution so it's available in execute_job
|
|
311
|
+
Thread.current[:gc_count] = @gc_count.value
|
|
284
312
|
execute_job(job)
|
|
285
|
-
@
|
|
313
|
+
@gc_count.increment
|
|
286
314
|
reconnect_attempts = 0 # Reset backoff on successful job execution
|
|
287
315
|
else
|
|
288
316
|
ensure_scheduler_watchdog!(connection)
|
|
@@ -352,6 +380,7 @@ module Postburner
|
|
|
352
380
|
|
|
353
381
|
logger.info "[Postburner::Worker] Shutting down, waiting for children..."
|
|
354
382
|
shutdown_children
|
|
383
|
+
log_tube_stats("Shutdown")
|
|
355
384
|
logger.info "[Postburner::Worker] Shutdown complete"
|
|
356
385
|
end
|
|
357
386
|
|
|
@@ -381,14 +410,14 @@ module Postburner
|
|
|
381
410
|
|
|
382
411
|
logger.info "[Postburner::Worker] Fork #{fork_num}: #{thread_count} threads, GC limit #{gc_limit || 'unlimited'}"
|
|
383
412
|
|
|
384
|
-
|
|
413
|
+
gc_count = Concurrent::AtomicFixnum.new(0)
|
|
385
414
|
pool = Concurrent::FixedThreadPool.new(thread_count)
|
|
386
415
|
|
|
387
416
|
thread_count.times do
|
|
388
|
-
pool.post { process_jobs_in_fork(fork_num,
|
|
417
|
+
pool.post { process_jobs_in_fork(fork_num, gc_count, gc_limit) }
|
|
389
418
|
end
|
|
390
419
|
|
|
391
|
-
until shutdown? || (gc_limit &&
|
|
420
|
+
until shutdown? || (gc_limit && gc_count.value >= gc_limit)
|
|
392
421
|
if orphaned?
|
|
393
422
|
logger.error "[Postburner::Worker] Fork #{fork_num} detected parent died (orphaned), initiating shutdown"
|
|
394
423
|
shutdown
|
|
@@ -400,8 +429,8 @@ module Postburner
|
|
|
400
429
|
pool.shutdown
|
|
401
430
|
pool.wait_for_termination(worker_config[:shutdown_timeout])
|
|
402
431
|
|
|
403
|
-
if gc_limit &&
|
|
404
|
-
logger.debug "[Postburner::Worker] Fork #{fork_num} reached GC limit (#{
|
|
432
|
+
if gc_limit && gc_count.value >= gc_limit
|
|
433
|
+
logger.debug "[Postburner::Worker] Fork #{fork_num} reached GC limit (#{gc_count.value} jobs), exiting for restart..."
|
|
405
434
|
exit 99
|
|
406
435
|
else
|
|
407
436
|
logger.info "[Postburner::Worker] Fork #{fork_num} shutting down gracefully..."
|
|
@@ -419,25 +448,30 @@ module Postburner
|
|
|
419
448
|
# tracking jobs processed across all threads in the fork.
|
|
420
449
|
#
|
|
421
450
|
# @param fork_num [Integer] Fork identifier for logging
|
|
422
|
-
# @param
|
|
451
|
+
# @param gc_count [Concurrent::AtomicFixnum] Shared job counter
|
|
423
452
|
# @param gc_limit [Integer, nil] Maximum jobs before exit (nil = unlimited)
|
|
424
453
|
# @return [void]
|
|
425
454
|
# @api private
|
|
426
|
-
def process_jobs_in_fork(fork_num,
|
|
455
|
+
def process_jobs_in_fork(fork_num, gc_count, gc_limit)
|
|
427
456
|
connection = Postburner::Connection.new
|
|
428
457
|
timeout = worker_config[:timeout]
|
|
429
458
|
reconnect_attempts = 0
|
|
430
459
|
|
|
460
|
+
# Set thread-local gc_limit for execution methods
|
|
461
|
+
Thread.current[:gc_limit] = gc_limit
|
|
462
|
+
|
|
431
463
|
watch_queues(connection, config.queue_names)
|
|
432
464
|
|
|
433
|
-
until shutdown? || (gc_limit &&
|
|
465
|
+
until shutdown? || (gc_limit && gc_count.value >= gc_limit)
|
|
434
466
|
begin
|
|
435
467
|
job = connection.beanstalk.tubes.reserve(timeout)
|
|
436
468
|
|
|
437
469
|
if job
|
|
438
470
|
logger.debug "[Postburner::Worker] Fork #{fork_num} thread #{Thread.current.object_id} reserved job #{job.id}"
|
|
471
|
+
# Set current count before execution so it's available in execute_job
|
|
472
|
+
Thread.current[:gc_count] = gc_count.value
|
|
439
473
|
execute_job(job)
|
|
440
|
-
|
|
474
|
+
gc_count.increment
|
|
441
475
|
reconnect_attempts = 0 # Reset backoff on successful job execution
|
|
442
476
|
else
|
|
443
477
|
ensure_scheduler_watchdog!(connection)
|
|
@@ -518,19 +552,39 @@ module Postburner
|
|
|
518
552
|
# Instantiates the Scheduler and runs it to process due schedules.
|
|
519
553
|
# Deletes the job from Beanstalkd after completion.
|
|
520
554
|
#
|
|
555
|
+
# Instruments with ActiveSupport::Notifications:
|
|
556
|
+
# - perform_start.watchdog.postburner: Before watchdog execution
|
|
557
|
+
# - perform.watchdog.postburner: Around watchdog execution (includes duration)
|
|
558
|
+
#
|
|
521
559
|
# @param beanstalk_job [Beaneater::Job] Reserved scheduler job
|
|
522
560
|
# @param payload [Hash] Parsed job body with 'scheduler' and 'interval' keys
|
|
523
561
|
# @return [void]
|
|
524
562
|
# @see Postburner::Scheduler#perform
|
|
525
563
|
# @api private
|
|
526
564
|
def execute_scheduler_job(beanstalk_job, payload)
|
|
527
|
-
logger.info "[Postburner] Executing scheduler watchdog #{beanstalk_job.id}"
|
|
528
|
-
|
|
529
565
|
interval = payload['interval'] || 300
|
|
530
|
-
scheduler = Postburner::Scheduler.new(interval: interval, logger: logger)
|
|
531
|
-
scheduler.perform
|
|
532
566
|
|
|
533
|
-
|
|
567
|
+
# Build instrumentation payload
|
|
568
|
+
worker_stats = thread_worker_stats
|
|
569
|
+
instrument_payload = {
|
|
570
|
+
beanstalk_job_id: beanstalk_job.id,
|
|
571
|
+
interval: interval,
|
|
572
|
+
watchdog: true
|
|
573
|
+
}.merge(worker_stats)
|
|
574
|
+
|
|
575
|
+
logger.info "[Postburner] Executing scheduler watchdog (bkid: #{beanstalk_job.id})"
|
|
576
|
+
|
|
577
|
+
ActiveSupport::Notifications.instrument('perform_start.watchdog.postburner', instrument_payload)
|
|
578
|
+
|
|
579
|
+
ActiveSupport::Notifications.instrument('perform.watchdog.postburner', instrument_payload) do
|
|
580
|
+
scheduler = Postburner::Scheduler.new(interval: interval, logger: logger)
|
|
581
|
+
|
|
582
|
+
Rails.application.reloader.wrap do
|
|
583
|
+
scheduler.perform
|
|
584
|
+
end
|
|
585
|
+
end
|
|
586
|
+
|
|
587
|
+
logger.info "[Postburner] Completed watchdog (bkid: #{beanstalk_job.id}, #{formatted_thread_worker_stats})"
|
|
534
588
|
delete_job!(beanstalk_job)
|
|
535
589
|
end
|
|
536
590
|
|
|
@@ -552,17 +606,25 @@ module Postburner
|
|
|
552
606
|
def execute_regular_job(beanstalk_job, payload)
|
|
553
607
|
job_description = format_job_description(payload)
|
|
554
608
|
job_payload = Postburner::Instrumentation.job_payload_from_hash(payload, beanstalk_job_id: beanstalk_job.id)
|
|
555
|
-
|
|
609
|
+
|
|
610
|
+
# Add worker stats to instrumentation payload
|
|
611
|
+
worker_stats = thread_worker_stats
|
|
612
|
+
instrument_payload = {
|
|
613
|
+
job: job_payload,
|
|
614
|
+
beanstalk_job_id: beanstalk_job.id
|
|
615
|
+
}.merge(worker_stats)
|
|
556
616
|
|
|
557
617
|
logger.info "[Postburner] Executing #{job_description} (bkid: #{beanstalk_job.id})"
|
|
558
618
|
|
|
559
619
|
ActiveSupport::Notifications.instrument('perform_start.job.postburner', instrument_payload)
|
|
560
620
|
|
|
561
621
|
ActiveSupport::Notifications.instrument('perform.job.postburner', instrument_payload) do
|
|
562
|
-
|
|
622
|
+
Rails.application.reloader.wrap do
|
|
623
|
+
Postburner::ActiveJob::Execution.execute(beanstalk_job.body)
|
|
624
|
+
end
|
|
563
625
|
end
|
|
564
626
|
|
|
565
|
-
logger.info "[Postburner] Completed #{job_description} (bkid: #{beanstalk_job.id})"
|
|
627
|
+
logger.info "[Postburner] Completed #{job_description} (bkid: #{beanstalk_job.id}, #{formatted_thread_worker_stats})"
|
|
566
628
|
delete_job!(beanstalk_job)
|
|
567
629
|
end
|
|
568
630
|
|
|
@@ -774,5 +836,67 @@ module Postburner
|
|
|
774
836
|
sleep 1
|
|
775
837
|
beanstalk_job.delete rescue nil
|
|
776
838
|
end
|
|
839
|
+
|
|
840
|
+
# Returns worker stats from thread-local variables as a hash.
|
|
841
|
+
#
|
|
842
|
+
# Used for instrumentation payload to make stats available to
|
|
843
|
+
# ActiveSupport::Notifications subscribers. Adds 1 to gc_count
|
|
844
|
+
# to reflect that the current job is being executed.
|
|
845
|
+
#
|
|
846
|
+
# @return [Hash] Hash with :gc_count and :gc_limit keys
|
|
847
|
+
# @api private
|
|
848
|
+
def thread_worker_stats
|
|
849
|
+
gc_count = Thread.current[:gc_count]
|
|
850
|
+
{
|
|
851
|
+
gc_count: gc_count ? gc_count + 1 : nil,
|
|
852
|
+
gc_limit: Thread.current[:gc_limit]
|
|
853
|
+
}
|
|
854
|
+
end
|
|
855
|
+
|
|
856
|
+
# Formats worker stats for logging.
|
|
857
|
+
#
|
|
858
|
+
# Produces a string like "(jobs: 42/5000)" showing the current
|
|
859
|
+
# job count and gc_limit. Returns empty string if no stats available.
|
|
860
|
+
# Delegates to thread_worker_stats for the actual counts.
|
|
861
|
+
#
|
|
862
|
+
# @return [String] Formatted stats string or empty string
|
|
863
|
+
# @api private
|
|
864
|
+
def formatted_thread_worker_stats
|
|
865
|
+
stats = thread_worker_stats
|
|
866
|
+
gc_count = stats[:gc_count]
|
|
867
|
+
gc_limit = stats[:gc_limit]
|
|
868
|
+
|
|
869
|
+
return "gc_count: unlimited" unless gc_count
|
|
870
|
+
|
|
871
|
+
if gc_limit
|
|
872
|
+
"gc_count: #{gc_count}, gc_limit: #{gc_limit}"
|
|
873
|
+
else
|
|
874
|
+
"gc_count: #{gc_count}"
|
|
875
|
+
end
|
|
876
|
+
end
|
|
877
|
+
|
|
878
|
+
# Logs Beanstalkd tube statistics with formatted breakdown.
|
|
879
|
+
#
|
|
880
|
+
# Shows aggregate totals on first line, followed by per-tube breakdown.
|
|
881
|
+
# Used at worker startup and shutdown to track job backlog.
|
|
882
|
+
#
|
|
883
|
+
# @param label [String] Label for the stats (e.g., "Startup", "Shutdown")
|
|
884
|
+
# @return [void]
|
|
885
|
+
# @api private
|
|
886
|
+
def log_tube_stats(label)
|
|
887
|
+
tube_names = config.expanded_tube_names + [config.scheduler_tube_name]
|
|
888
|
+
stats = Postburner.stats(tube_names)
|
|
889
|
+
|
|
890
|
+
# Log totals on first line
|
|
891
|
+
totals = stats[:totals]
|
|
892
|
+
logger.info "[Postburner::Worker] #{label} stats: ready=#{totals[:ready]} delayed=#{totals[:delayed]} buried=#{totals[:buried]} reserved=#{totals[:reserved]} total=#{totals[:total]}"
|
|
893
|
+
|
|
894
|
+
# Log per-tube breakdown
|
|
895
|
+
stats[:tubes].each do |tube|
|
|
896
|
+
logger.info " #{tube[:name]}: ready=#{tube[:ready]} delayed=#{tube[:delayed]} buried=#{tube[:buried]} reserved=#{tube[:reserved]} total=#{tube[:total]}"
|
|
897
|
+
end
|
|
898
|
+
rescue => e
|
|
899
|
+
logger.warn "[Postburner::Worker] Failed to retrieve tube stats: #{e.message}"
|
|
900
|
+
end
|
|
777
901
|
end
|
|
778
902
|
end
|