postburner 1.0.0.pre.14 → 1.0.0.pre.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +149 -107
- data/app/concerns/postburner/execution.rb +1 -1
- data/app/concerns/postburner/insertion.rb +22 -9
- data/app/concerns/postburner/properties.rb +7 -2
- data/app/controllers/postburner/application_controller.rb +1 -0
- data/app/models/postburner/job.rb +12 -3
- data/bin/postburner +1 -1
- data/lib/generators/postburner/install/templates/config/postburner.yml +3 -3
- data/lib/postburner/configuration.rb +12 -18
- data/lib/postburner/scheduler.rb +64 -15
- data/lib/postburner/tube.rb +3 -5
- data/lib/postburner/version.rb +1 -1
- data/lib/postburner/worker.rb +55 -5
- data/lib/postburner.rb +33 -10
- data/lib/tasks/postburner.rake +104 -0
- metadata +1 -3
- data/config/postburner.yml +0 -22
- data/config/postburner.yml.example +0 -144
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8e032f7d37f1680fe34a1402f8eb9f0b3ca61aa9231f6d540c35c3a919e3f58d
|
|
4
|
+
data.tar.gz: bd35338dd14bc499c4c7d9f57d30eff3902615cd72876f766f1e379f12baba77
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f207c1d4dd840bee6b00ad71de9cc1b774d9b2376f7f34125fcc3a164220a340badc6b1ef17931d3dc09a0bb78a09c66b1f1f9ffdf1ead3e9737c44a39fe9a29
|
|
7
|
+
data.tar.gz: e0212499c62223a87720462226660045585ac4bde21258eb54aeed9d85683c0c9652477a6cf5824bdffa7e30314efb1e1cbb9d146eae8ef3ade2202af73f2abe
|
data/README.md
CHANGED
|
@@ -2,24 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
Fast Beanstalkd-backed job queue with **optional PostgreSQL records**.
|
|
4
4
|
|
|
5
|
-
Depends on Beanstalkd, Postgres, ActiveRecord, and ActiveJob (1).
|
|
6
|
-
|
|
7
5
|
Postburner provides dual-mode job execution:
|
|
8
6
|
- **Fast jobs**: Fast execution via Beanstalkd
|
|
9
7
|
- **Tracked jobs**: Audited jobs logs, timing, errors, and statistics
|
|
10
8
|
|
|
11
9
|
Built for production environments where you want fast background processing for most jobs, but comprehensive auditing for critical operations.
|
|
12
10
|
|
|
13
|
-
- **ActiveJob
|
|
14
|
-
- **Dual-mode execution** -
|
|
15
|
-
- **Rich audit trail** - Logs, timing, errors, retry tracking (tracked jobs only)
|
|
16
|
-
- **ActiveRecord** - Query jobs with ActiveRecord (
|
|
11
|
+
- **ActiveJob Adapter** - To use with Rails, ActionMailer, ActiveStorage
|
|
12
|
+
- **Dual-mode execution** - Beanstalkd only or tracked (database backed)
|
|
13
|
+
- **Rich audit trail** - Logs, timing, errors, instrumentation, retry tracking (tracked jobs only)
|
|
14
|
+
- **ActiveRecord** - Query jobs with ActiveRecord (on opt in)
|
|
17
15
|
- **Scheduler** - Schedule jobs at fixed intervals, cron expressions, and calendar-aware anchor points.
|
|
18
|
-
- **
|
|
16
|
+
- **Test-friendly** - Testing jobs can be tricky, so we go beyond just inline.
|
|
19
17
|
- **Process isolation** - Forking workers with optional threading for throughput
|
|
20
|
-
- **
|
|
21
|
-
|
|
22
|
-
(1) An ActiveJob adapter is provided for seamless integration with the default Rails stack, you can use Postburner without ActiveJob by using the `Postburner::Job` class directly.
|
|
18
|
+
- **Beanstalkd** - Fast, reliable queue separate from your database, persistent storage
|
|
23
19
|
|
|
24
20
|
```ruby
|
|
25
21
|
# Default job (fast, no PostgreSQL overhead)
|
|
@@ -75,7 +71,7 @@ end
|
|
|
75
71
|
# Scheduled job (fixed interval, anchor based)
|
|
76
72
|
Postburner::Schedule.create!(
|
|
77
73
|
name: 'daily_event_retention',
|
|
78
|
-
job_class: EventRetentionJob, # either Postburner::Job or ActiveJob
|
|
74
|
+
job_class: EventRetentionJob, # either Postburner::Job or ActiveJob ancestor
|
|
79
75
|
anchor: Time.zone.parse('2025-01-01 09:30:00'),
|
|
80
76
|
interval: 1,
|
|
81
77
|
interval_unit: 'days',
|
|
@@ -101,6 +97,7 @@ bundle exec rake postburner:work WORKER=default
|
|
|
101
97
|
- [Postburner::Job](#postburnerjob)
|
|
102
98
|
- [Scheduler](#scheduler)
|
|
103
99
|
- [Job Management](#job-management)
|
|
100
|
+
- [Writing Jobs](#writing-jobs)
|
|
104
101
|
- [Queue Strategies](#queue-strategies)
|
|
105
102
|
- [Testing](#testing)
|
|
106
103
|
- [Workers](#workers)
|
|
@@ -132,7 +129,7 @@ Postburner [beanstalkd](https://beanstalkd.github.io/) is used with PostgreSQL t
|
|
|
132
129
|
|
|
133
130
|
```ruby
|
|
134
131
|
# Gemfile
|
|
135
|
-
gem 'postburner', '~> 1.0.0.pre.
|
|
132
|
+
gem 'postburner', '~> 1.0.0.pre.14'
|
|
136
133
|
|
|
137
134
|
# config/application.rb
|
|
138
135
|
config.active_job.queue_adapter = :postburner
|
|
@@ -140,28 +137,28 @@ config.active_job.queue_adapter = :postburner
|
|
|
140
137
|
|
|
141
138
|
```yaml
|
|
142
139
|
# config/postburner.yml
|
|
143
|
-
development: # <- environment config
|
|
140
|
+
development: # <- environment config
|
|
144
141
|
beanstalk_url: <%= ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300' %>
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
142
|
+
forks: 2
|
|
143
|
+
threads: 10
|
|
144
|
+
gc_limit: 500
|
|
148
145
|
|
|
149
|
-
workers: # <- worker config
|
|
146
|
+
workers: # <- worker config (overrides env-level)
|
|
150
147
|
default:
|
|
151
|
-
threads: 16 # Overrides
|
|
148
|
+
threads: 16 # Overrides env-level threads
|
|
152
149
|
queues:
|
|
153
150
|
- default
|
|
154
151
|
- mailers
|
|
155
152
|
critical:
|
|
156
|
-
forks: 4 # Overrides
|
|
157
|
-
threads: 8 # Overrides
|
|
158
|
-
gc_limit: 24 # Overrides
|
|
153
|
+
forks: 4 # Overrides env-level forks
|
|
154
|
+
threads: 8 # Overrides env-level threads
|
|
155
|
+
gc_limit: 24 # Overrides env-level gc_limit
|
|
159
156
|
queues:
|
|
160
157
|
- payments
|
|
161
158
|
slow:
|
|
162
|
-
forks: 4 # Overrides
|
|
163
|
-
threads: 1 # Overrides
|
|
164
|
-
gc_limit: 1 # Overrides
|
|
159
|
+
forks: 4 # Overrides env-level forks
|
|
160
|
+
threads: 1 # Overrides env-level threads
|
|
161
|
+
gc_limit: 1 # Overrides env-level gc_limit
|
|
165
162
|
queues:
|
|
166
163
|
- imports
|
|
167
164
|
- video
|
|
@@ -181,16 +178,45 @@ config.active_job.queue_adapter = :postburner
|
|
|
181
178
|
#config.active_job.queue_name_prefix = Postburner.tube_prefix(Rails.env) # i.e. "postburner.#{Rails.env}"
|
|
182
179
|
config.action_mailer.deliver_later_queue_name = 'mailers' # gets prefixed by config.active_job.queue_name_prefix
|
|
183
180
|
|
|
184
|
-
|
|
185
181
|
bundle exec postburner # start with bin/postburner
|
|
186
182
|
bundle exec rake postburner:work # or with rake task
|
|
187
183
|
```
|
|
188
184
|
|
|
189
|
-
|
|
185
|
+
### Enqueueing Jobs
|
|
190
186
|
|
|
191
|
-
|
|
187
|
+
```ruby
|
|
188
|
+
# ActiveJob (standard Rails API)
|
|
189
|
+
SendEmailJob.perform_later(user_id) # Enqueue immediately
|
|
190
|
+
SendEmailJob.set(wait: 1.hour).perform_later(user_id) # Delay by duration
|
|
191
|
+
SendEmailJob.set(wait_until: Date.tomorrow.noon).perform_later(user_id) # Run at specific time
|
|
192
|
+
SendEmailJob.set(queue: 'critical').perform_later(user_id) # Override queue
|
|
193
|
+
SendEmailJob.set(priority: 0).perform_later(user_id) # Override priority
|
|
194
|
+
|
|
195
|
+
# Postburner::Job (always tracked, full API)
|
|
196
|
+
job = ProcessPayment.create!(args: { 'payment_id' => 123 })
|
|
197
|
+
job.queue! # Enqueue immediately
|
|
198
|
+
job.queue!(delay: 1.hour) # Delay by duration
|
|
199
|
+
job.queue!(at: Date.tomorrow.noon) # Run at specific time
|
|
200
|
+
job.queue!(queue: 'critical') # Override queue
|
|
201
|
+
job.queue!(priority: 0, ttr: 600) # Set priority and TTR
|
|
202
|
+
```
|
|
192
203
|
|
|
193
|
-
|
|
204
|
+
### ActiveJob vs Postburner::Job TL;DR
|
|
205
|
+
|
|
206
|
+
`Postburner::Job` as simple subclass of `ActiveRecord`, so the normal
|
|
207
|
+
`ActiveRecord` API applies! Thus the workflow is to create an instance,
|
|
208
|
+
then queue it!
|
|
209
|
+
|
|
210
|
+
| Operation | ActiveJob | Postburner::Job |
|
|
211
|
+
|-----------|-----------|-----------------|
|
|
212
|
+
| **Enqueue immediately** | `MyJob.perform_later(args)` | `MyJob.create!(args: {}).queue!` |
|
|
213
|
+
| **Delay** | `.set(wait: 1.hour)` | `job.queue!(delay: 1.hour)` |
|
|
214
|
+
| **Run at** | `.set(wait_until: time)` | `job.queue!(at: time)` |
|
|
215
|
+
| **Set queue** | `.set(queue: 'critical')` | `job.queue!(queue: 'critical')` |
|
|
216
|
+
| **Set priority** | `.set(priority: 0)` | `job.queue!(priority: 0)` |
|
|
217
|
+
| **Set TTR** | `.set(ttr: 300)` | `job.queue!(ttr: 300)` |
|
|
218
|
+
|
|
219
|
+
## Usage
|
|
194
220
|
|
|
195
221
|
### Default Jobs
|
|
196
222
|
|
|
@@ -241,8 +267,8 @@ end
|
|
|
241
267
|
```
|
|
242
268
|
|
|
243
269
|
**Configuration options:**
|
|
244
|
-
- `
|
|
245
|
-
- `
|
|
270
|
+
- `priority` - Beanstalkd priority (0-4294967295, lower = higher priority)
|
|
271
|
+
- `ttr` - Time-to-run in seconds before job times out
|
|
246
272
|
|
|
247
273
|
Jobs without `Postburner::Beanstalkd` use defaults from `config/postburner.yml`:
|
|
248
274
|
- `default_priority: 65536`
|
|
@@ -276,7 +302,7 @@ end
|
|
|
276
302
|
|
|
277
303
|
Tracked jobs store full execution details in PostgreSQL, providing comprehensive audit trails (i.e. logging, timing, errors, retry tracking) for critical operations.
|
|
278
304
|
|
|
279
|
-
**Note:** `Postburner::Tracked` automatically includes `Postburner::Beanstalkd`, giving you access to `
|
|
305
|
+
**Note:** `Postburner::Tracked` automatically includes `Postburner::Beanstalkd`, giving you access to `priority`, `ttr`, `bk`, and `extend!`.
|
|
280
306
|
|
|
281
307
|
```ruby
|
|
282
308
|
class ProcessPayment < ApplicationJob
|
|
@@ -339,7 +365,7 @@ job.duration # Execution time in milliseconds
|
|
|
339
365
|
job.lag # Queue lag in milliseconds
|
|
340
366
|
```
|
|
341
367
|
|
|
342
|
-
###
|
|
368
|
+
### Postburner::Job Usage
|
|
343
369
|
|
|
344
370
|
Direct `Postburner::Job` subclasses are **always tracked**:
|
|
345
371
|
|
|
@@ -367,6 +393,15 @@ job.queue!(delay: 1.hour)
|
|
|
367
393
|
job.queue!(at: 2.days.from_now)
|
|
368
394
|
```
|
|
369
395
|
|
|
396
|
+
> **Note:** The `args` parameter in `perform(args)` is optional. It's a convenience accessor to `self.args`, which is stored in a JSONB column on the job record. You can omit the parameter and access args directly:
|
|
397
|
+
>
|
|
398
|
+
> ```ruby
|
|
399
|
+
> def perform
|
|
400
|
+
> payment = Payment.find(self.args['payment_id'])
|
|
401
|
+
> # ...
|
|
402
|
+
> end
|
|
403
|
+
> ```
|
|
404
|
+
|
|
370
405
|
#### Instance-Level Queue Configuration
|
|
371
406
|
|
|
372
407
|
Override queue priority and TTR per job instance for dynamic behavior:
|
|
@@ -375,8 +410,8 @@ Override queue priority and TTR per job instance for dynamic behavior:
|
|
|
375
410
|
# Set priority during creation
|
|
376
411
|
job = ProcessPayment.create!(
|
|
377
412
|
args: { 'payment_id' => 123 },
|
|
378
|
-
|
|
379
|
-
|
|
413
|
+
priority: 1500, # Override class-level priority
|
|
414
|
+
ttr: 300 # Override class-level TTR
|
|
380
415
|
)
|
|
381
416
|
job.queue!
|
|
382
417
|
|
|
@@ -427,7 +462,7 @@ Postburner includes a lightweight, fixed-rate scheduler for recurring jobs. Perf
|
|
|
427
462
|
The scheduler uses **immediate enqueue** combined with a **watchdog safety net**:
|
|
428
463
|
|
|
429
464
|
1. When an execution is created, it's immediately enqueued to Beanstalkd's delayed queue with the appropriate delay until `run_at`
|
|
430
|
-
2. For `Postburner::Job` and `ActiveJob` with `Postburner::Tracked` schedules the next execution when the current job runs - providing immediate pickup without waiting for the watchdog. Normal `ActiveJob` schedules need to rely on the watchdog to create the next execution, so set the `
|
|
465
|
+
2. For `Postburner::Job` and `ActiveJob` with `Postburner::Tracked` schedules the next execution when the current job runs - providing immediate pickup without waiting for the watchdog. Normal `ActiveJob` schedules need to rely on the watchdog to create the next execution, so set the `scheduler_interval` to pick up executions appropriately.
|
|
431
466
|
3. A lightweight watchdog job in the `scheduler` tube acts as a safety net:
|
|
432
467
|
```json
|
|
433
468
|
{ "scheduler": true, "interval": 300 }
|
|
@@ -464,8 +499,8 @@ Add scheduler settings to `config/postburner.yml`:
|
|
|
464
499
|
```yaml
|
|
465
500
|
production:
|
|
466
501
|
beanstalk_url: <%= ENV['BEANSTALK_URL'] %>
|
|
467
|
-
|
|
468
|
-
|
|
502
|
+
scheduler_interval: 300 # Pickup new schedules every 5 minutes
|
|
503
|
+
scheduler_priority: 100 # Scheduler jobs run at priority 100
|
|
469
504
|
|
|
470
505
|
workers:
|
|
471
506
|
default:
|
|
@@ -475,8 +510,8 @@ production:
|
|
|
475
510
|
```
|
|
476
511
|
|
|
477
512
|
**Configuration options:**
|
|
478
|
-
- `
|
|
479
|
-
- `
|
|
513
|
+
- `scheduler_interval` - How often (in seconds) to check for due schedules (default: 300)
|
|
514
|
+
- `scheduler_priority` - Beanstalkd priority for watchdog jobs (default: 100)
|
|
480
515
|
|
|
481
516
|
**Choosing an interval:** Since executions are enqueued immediately to Beanstalkd's delayed queue, the watchdog interval primarily affects:
|
|
482
517
|
- How quickly new schedules are auto-bootstrapped (if you don't call `start!`)
|
|
@@ -636,8 +671,8 @@ schedule.next_run_at_times(5) # Next 5 run times
|
|
|
636
671
|
|
|
637
672
|
# View executions
|
|
638
673
|
schedule.executions.pending
|
|
639
|
-
schedule.executions.
|
|
640
|
-
schedule.executions.
|
|
674
|
+
schedule.executions.scheduled
|
|
675
|
+
schedule.executions.skipped
|
|
641
676
|
```
|
|
642
677
|
|
|
643
678
|
#### Starting Schedules
|
|
@@ -664,7 +699,7 @@ The job is immediately in Beanstalkd's delayed queue and will run at the schedul
|
|
|
664
699
|
|
|
665
700
|
**Option 2: Auto-bootstrap (eventual pickup)**
|
|
666
701
|
|
|
667
|
-
If you don't call `start!`, the scheduler watchdog will automatically bootstrap the schedule on its next run. This adds up to one `
|
|
702
|
+
If you don't call `start!`, the scheduler watchdog will automatically bootstrap the schedule on its next run. This adds up to one `scheduler_interval` of delay before the first execution is enqueued:
|
|
668
703
|
|
|
669
704
|
```ruby
|
|
670
705
|
schedule = Postburner::Schedule.create!(...)
|
|
@@ -683,10 +718,9 @@ Each scheduled run creates an execution record for tracking:
|
|
|
683
718
|
```ruby
|
|
684
719
|
execution = Postburner::ScheduleExecution.find(123)
|
|
685
720
|
|
|
686
|
-
execution.status # pending,
|
|
721
|
+
execution.status # pending, scheduled, skipped
|
|
687
722
|
execution.run_at # Scheduled time
|
|
688
723
|
execution.enqueued_at # When job was queued
|
|
689
|
-
execution.completed_at # When job finished
|
|
690
724
|
execution.beanstalk_job_id # Beanstalkd job ID
|
|
691
725
|
execution.job_id # Postburner::Job ID (if using Postburner::Job)
|
|
692
726
|
```
|
|
@@ -780,6 +814,14 @@ job.errata # Array of exceptions with backtraces
|
|
|
780
814
|
job.attempts # Array of attempt timestamps
|
|
781
815
|
```
|
|
782
816
|
|
|
817
|
+
## Writing Jobs
|
|
818
|
+
|
|
819
|
+
Pay attention to the following when writing jobs:
|
|
820
|
+
|
|
821
|
+
**Job Idempotency:** Jobs should be designed to be idempotent and safely re-runnable. Like all job queues, Postburner provides at-least-once delivery—in rare errant cases outside of Postburner's control, a job may be executed more than once, i.e. network issues, etc.
|
|
822
|
+
|
|
823
|
+
**TTR (Time-to-Run):** If a job exceeds its TTR without completion, Beanstalkd releases it back to the queue while still running—causing duplicate execution. For long-running jobs, call `extend!` periodically to reset the TTR, or set a sufficiently high TTR value. You must include the `Postburner::Beanstalkd` or `Postburner::Tracked` module with `ActiveJob` to use `extend!`.
|
|
824
|
+
|
|
783
825
|
## Queue Strategies
|
|
784
826
|
|
|
785
827
|
Postburner uses different strategies to control job execution. These affect `Postburner::Job` subclasses (not ActiveJob classes).
|
|
@@ -905,25 +947,25 @@ Postburner uses named worker configurations to support different deployment patt
|
|
|
905
947
|
Configure multiple named workers with different concurrency profiles:
|
|
906
948
|
|
|
907
949
|
```yaml
|
|
908
|
-
production: # <- environment config
|
|
950
|
+
production: # <- environment config
|
|
909
951
|
beanstalk_url: <%= ENV['BEANSTALK_URL'] %>
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
952
|
+
forks: 2
|
|
953
|
+
threads: 10
|
|
954
|
+
gc_limit: 5000
|
|
913
955
|
|
|
914
|
-
workers: # <- worker config
|
|
956
|
+
workers: # <- worker config (overrides env-level)
|
|
915
957
|
# Heavy, memory-intensive jobs - more processes, fewer threads
|
|
916
958
|
imports:
|
|
917
|
-
forks: 4 # Overrides
|
|
918
|
-
threads: 1 # Overrides
|
|
919
|
-
gc_limit: 500 # Overrides
|
|
959
|
+
forks: 4 # Overrides env-level forks
|
|
960
|
+
threads: 1 # Overrides env-level threads
|
|
961
|
+
gc_limit: 500 # Overrides env-level gc_limit
|
|
920
962
|
queues:
|
|
921
963
|
- imports
|
|
922
964
|
- data_processing
|
|
923
965
|
|
|
924
966
|
# General jobs - fewer processes, many threads
|
|
925
967
|
general:
|
|
926
|
-
threads: 100 # Overrides
|
|
968
|
+
threads: 100 # Overrides env-level threads (forks uses env-level forks=2)
|
|
927
969
|
queues:
|
|
928
970
|
- default
|
|
929
971
|
- mailers
|
|
@@ -957,12 +999,12 @@ development: # <- environment config
|
|
|
957
999
|
```yaml
|
|
958
1000
|
staging: # <- environment config
|
|
959
1001
|
beanstalk_url: beanstalk://localhost:11300
|
|
960
|
-
|
|
961
|
-
|
|
1002
|
+
threads: 10
|
|
1003
|
+
gc_limit: 5000
|
|
962
1004
|
|
|
963
1005
|
workers: # <- worker config
|
|
964
1006
|
default:
|
|
965
|
-
# Uses env
|
|
1007
|
+
# Uses env-level: threads=10, gc_limit=5000
|
|
966
1008
|
queues:
|
|
967
1009
|
- critical
|
|
968
1010
|
- default
|
|
@@ -971,25 +1013,25 @@ staging: # <- environment config
|
|
|
971
1013
|
|
|
972
1014
|
**Production (multiple workers with different profiles):**
|
|
973
1015
|
```yaml
|
|
974
|
-
production: # <- environment config
|
|
1016
|
+
production: # <- environment config
|
|
975
1017
|
beanstalk_url: <%= ENV['BEANSTALK_URL'] %>
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
1018
|
+
forks: 2
|
|
1019
|
+
threads: 10
|
|
1020
|
+
gc_limit: 5000
|
|
979
1021
|
|
|
980
|
-
workers: # <- worker config
|
|
1022
|
+
workers: # <- worker config (overrides env-level)
|
|
981
1023
|
imports:
|
|
982
|
-
forks: 4 # Overrides
|
|
983
|
-
threads: 1 # Overrides
|
|
984
|
-
gc_limit: 500 # Overrides
|
|
1024
|
+
forks: 4 # Overrides env-level forks (4 processes)
|
|
1025
|
+
threads: 1 # Overrides env-level threads (1 thread per process = 4 concurrent jobs)
|
|
1026
|
+
gc_limit: 500 # Overrides env-level gc_limit
|
|
985
1027
|
queues:
|
|
986
1028
|
- imports
|
|
987
1029
|
- data_processing
|
|
988
1030
|
|
|
989
1031
|
general:
|
|
990
|
-
# forks uses
|
|
991
|
-
threads: 100 # Overrides
|
|
992
|
-
# gc_limit uses
|
|
1032
|
+
# forks uses env-level forks=2 (2 processes)
|
|
1033
|
+
threads: 100 # Overrides env-level threads (100 threads per process = 200 concurrent jobs)
|
|
1034
|
+
# gc_limit uses env-level gc_limit=5000
|
|
993
1035
|
queues:
|
|
994
1036
|
- default
|
|
995
1037
|
- mailers
|
|
@@ -1148,8 +1190,8 @@ The `shutdown_timeout` controls how long workers wait for in-flight jobs to comp
|
|
|
1148
1190
|
|
|
1149
1191
|
```yaml
|
|
1150
1192
|
production:
|
|
1151
|
-
default_ttr: 300
|
|
1152
|
-
|
|
1193
|
+
default_ttr: 300 # Default TTR for jobs
|
|
1194
|
+
shutdown_timeout: 300 # Defaults to default_ttr if not specified
|
|
1153
1195
|
|
|
1154
1196
|
workers:
|
|
1155
1197
|
imports:
|
|
@@ -1164,7 +1206,7 @@ production:
|
|
|
1164
1206
|
|
|
1165
1207
|
### GC Limits
|
|
1166
1208
|
|
|
1167
|
-
Set `
|
|
1209
|
+
Set `gc_limit` at environment level or per worker to automatically restart after processing N jobs.
|
|
1168
1210
|
|
|
1169
1211
|
- Worker processes N jobs
|
|
1170
1212
|
- Worker exits with code 99
|
|
@@ -1173,22 +1215,22 @@ Set `default_gc_limit` at environment level or `gc_limit` per worker to automati
|
|
|
1173
1215
|
|
|
1174
1216
|
```yaml
|
|
1175
1217
|
production: # <- environment config
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1218
|
+
forks: 2
|
|
1219
|
+
threads: 10
|
|
1220
|
+
gc_limit: 5000
|
|
1179
1221
|
|
|
1180
1222
|
workers: # <- worker config
|
|
1181
1223
|
imports:
|
|
1182
|
-
forks: 4 # Overrides
|
|
1183
|
-
threads: 1 # Overrides
|
|
1184
|
-
gc_limit: 500 # Overrides
|
|
1224
|
+
forks: 4 # Overrides env-level forks
|
|
1225
|
+
threads: 1 # Overrides env-level threads
|
|
1226
|
+
gc_limit: 500 # Overrides env-level gc_limit (restart after 500 jobs, memory-intensive)
|
|
1185
1227
|
queues:
|
|
1186
1228
|
- imports
|
|
1187
1229
|
- data_processing
|
|
1188
1230
|
|
|
1189
1231
|
general:
|
|
1190
|
-
# Uses
|
|
1191
|
-
threads: 100 # Overrides
|
|
1232
|
+
# Uses env-level forks=2, threads=100, gc_limit=5000 (restart after 5000 jobs)
|
|
1233
|
+
threads: 100 # Overrides env-level threads
|
|
1192
1234
|
queues:
|
|
1193
1235
|
- default
|
|
1194
1236
|
- mailers
|
|
@@ -1229,41 +1271,41 @@ test: # <- environment config
|
|
|
1229
1271
|
|
|
1230
1272
|
staging: # <- environment config
|
|
1231
1273
|
<<: *default
|
|
1232
|
-
|
|
1233
|
-
|
|
1274
|
+
threads: 10 # Multi-threaded, single process
|
|
1275
|
+
gc_limit: 5000
|
|
1234
1276
|
|
|
1235
1277
|
workers: # <- worker config
|
|
1236
1278
|
default:
|
|
1237
|
-
# Uses env
|
|
1279
|
+
# Uses env-level: threads=10, gc_limit=5000
|
|
1238
1280
|
queues:
|
|
1239
1281
|
- default
|
|
1240
1282
|
- mailers
|
|
1241
1283
|
|
|
1242
|
-
production: # <- environment config
|
|
1284
|
+
production: # <- environment config
|
|
1243
1285
|
<<: *default
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1286
|
+
forks: 2 # Inherited by workers
|
|
1287
|
+
threads: 10 # Inherited by workers
|
|
1288
|
+
gc_limit: 5000 # Inherited by workers
|
|
1247
1289
|
|
|
1248
|
-
workers: # <- worker config
|
|
1290
|
+
workers: # <- worker config (overrides env-level)
|
|
1249
1291
|
critical:
|
|
1250
|
-
forks: 1 # Overrides
|
|
1251
|
-
threads: 1 # Overrides
|
|
1252
|
-
gc_limit: 100 # Overrides
|
|
1292
|
+
forks: 1 # Overrides env-level forks
|
|
1293
|
+
threads: 1 # Overrides env-level threads (1 concurrent job)
|
|
1294
|
+
gc_limit: 100 # Overrides env-level gc_limit
|
|
1253
1295
|
queues:
|
|
1254
1296
|
- critical
|
|
1255
1297
|
|
|
1256
1298
|
default:
|
|
1257
|
-
forks: 4 # Overrides
|
|
1258
|
-
threads: 10 # Overrides
|
|
1259
|
-
gc_limit: 1000 # Overrides
|
|
1299
|
+
forks: 4 # Overrides env-level forks
|
|
1300
|
+
threads: 10 # Overrides env-level threads (40 total concurrent jobs: 4 × 10)
|
|
1301
|
+
gc_limit: 1000 # Overrides env-level gc_limit
|
|
1260
1302
|
queues:
|
|
1261
1303
|
- default
|
|
1262
1304
|
|
|
1263
1305
|
mailers:
|
|
1264
|
-
# forks uses
|
|
1265
|
-
threads: 5 # Overrides
|
|
1266
|
-
gc_limit: 500 # Overrides
|
|
1306
|
+
# forks uses env-level forks=2
|
|
1307
|
+
threads: 5 # Overrides env-level threads (10 total email senders: 2 × 5)
|
|
1308
|
+
gc_limit: 500 # Overrides env-level gc_limit
|
|
1267
1309
|
queues:
|
|
1268
1310
|
- mailers
|
|
1269
1311
|
```
|
|
@@ -1311,7 +1353,7 @@ Both refer to the Beanstalkd tube `postburner.production.mailers`.
|
|
|
1311
1353
|
|
|
1312
1354
|
**Naming convention:** Use underscores for multi-word queue names (e.g., `background_jobs`, `high_priority`). Avoid hyphens as they can cause issues with some Beanstalkd client libraries.
|
|
1313
1355
|
|
|
1314
|
-
**Important:** No need to set `config.active_job.queue_name_prefix` - Postburner handles prefixing automatically
|
|
1356
|
+
**Important:** No need to set `config.active_job.queue_name_prefix` - Postburner handles prefixing automatically when jobs are enqueued with postburner.
|
|
1315
1357
|
|
|
1316
1358
|
### Queue Configuration Methods
|
|
1317
1359
|
|
|
@@ -1534,16 +1576,16 @@ end
|
|
|
1534
1576
|
|
|
1535
1577
|
Beanstalkd is a simple, fast, and reliable queue system. It is a good choice for production environments where you want fast background processing for most jobs, but comprehensive auditing for critical operations.
|
|
1536
1578
|
|
|
1537
|
-
The [protocol](https://github.com/beanstalkd/beanstalkd/blob/master/doc/protocol.txt) reads more like a README than a protocol. Check it out and you will instantly understand how it works.
|
|
1579
|
+
The [protocol](https://github.com/beanstalkd/beanstalkd/blob/master/doc/protocol.txt) reads more like a README than a protocol. Check it out and you will instantly understand how it works.
|
|
1538
1580
|
|
|
1539
|
-
|
|
1581
|
+
Diagram of the typical job lifecycle:
|
|
1540
1582
|
|
|
1541
1583
|
```
|
|
1542
1584
|
put reserve delete
|
|
1543
1585
|
-----> [READY] ---------> [RESERVED] --------> *poof*`
|
|
1544
1586
|
```
|
|
1545
1587
|
|
|
1546
|
-
|
|
1588
|
+
Diagram with all lifecycle:
|
|
1547
1589
|
|
|
1548
1590
|
```
|
|
1549
1591
|
put with delay release with delay
|
|
@@ -1748,7 +1790,7 @@ end
|
|
|
1748
1790
|
```ruby
|
|
1749
1791
|
class DataImport < Postburner::Job
|
|
1750
1792
|
queue 'imports'
|
|
1751
|
-
ttr 1800 # 30 minutes
|
|
1793
|
+
ttr 1800 # 30 minutes
|
|
1752
1794
|
|
|
1753
1795
|
def perform(args)
|
|
1754
1796
|
# Long-running import logic
|
|
@@ -1761,8 +1803,8 @@ end
|
|
|
1761
1803
|
For jobs that may take longer than expected, you can extend the TTR dynamically using `extend!`. This calls Beanstalkd's `touch` command, which resets the TTR countdown.
|
|
1762
1804
|
|
|
1763
1805
|
**Available for:**
|
|
1764
|
-
- `Postburner::Job` subclasses (
|
|
1765
|
-
- `Postburner::Tracked` ActiveJob classes (
|
|
1806
|
+
- `Postburner::Job` subclasses (via `extend!` method)
|
|
1807
|
+
- `Postburner::Tracked` ActiveJob classes (via `extend!` method)
|
|
1766
1808
|
|
|
1767
1809
|
```ruby
|
|
1768
1810
|
class ProcessImport < ApplicationJob
|
|
@@ -1799,7 +1841,7 @@ class LargeDataProcessor < Postburner::Job
|
|
|
1799
1841
|
dataset.each_chunk do |chunk|
|
|
1800
1842
|
process_chunk(chunk)
|
|
1801
1843
|
|
|
1802
|
-
|
|
1844
|
+
extend! # Reset TTR (calls bk.touch internally)
|
|
1803
1845
|
log "Chunk processed, TTR extended"
|
|
1804
1846
|
end
|
|
1805
1847
|
end
|
|
@@ -2000,7 +2042,7 @@ Postburner.connected do |conn|
|
|
|
2000
2042
|
end
|
|
2001
2043
|
```
|
|
2002
2044
|
|
|
2003
|
-
## Web UI
|
|
2045
|
+
## Web UI - v2 Coming Soon
|
|
2004
2046
|
|
|
2005
2047
|
Mount the inspection interface:
|
|
2006
2048
|
|
|
@@ -2043,7 +2085,7 @@ beanstalkd -l 127.0.0.1 -p 11300 -b /var/lib/beanstalkd
|
|
|
2043
2085
|
|
|
2044
2086
|
```ruby
|
|
2045
2087
|
# Gemfile
|
|
2046
|
-
gem 'postburner', '~> 1.0.0.pre.
|
|
2088
|
+
gem 'postburner', '~> 1.0.0.pre.14'
|
|
2047
2089
|
```
|
|
2048
2090
|
|
|
2049
2091
|
```bash
|
|
@@ -2094,7 +2136,7 @@ Key changes in v1.0:
|
|
|
2094
2136
|
|
|
2095
2137
|
1. **Update Gemfile:**
|
|
2096
2138
|
```ruby
|
|
2097
|
-
gem 'postburner', '~> 1.0.0.pre.
|
|
2139
|
+
gem 'postburner', '~> 1.0.0.pre.14'
|
|
2098
2140
|
```
|
|
2099
2141
|
|
|
2100
2142
|
2. **Remove Backburner config:**
|
|
@@ -2138,7 +2180,7 @@ We encourage AI tools, but do not vibe, as the code must look like it was writte
|
|
|
2138
2180
|
```bash
|
|
2139
2181
|
bundle install
|
|
2140
2182
|
bundle exec rails test # must have beanstalkd on 11300 by default
|
|
2141
|
-
bundle exec rails app:postburner:work #
|
|
2183
|
+
bundle exec rails app:postburner:work # run worker from engine root (uses test/dummy app)
|
|
2142
2184
|
```
|
|
2143
2185
|
|
|
2144
2186
|
## License
|
|
@@ -134,7 +134,7 @@ module Postburner
|
|
|
134
134
|
|
|
135
135
|
run_callbacks :processing do
|
|
136
136
|
begin
|
|
137
|
-
self.perform(args)
|
|
137
|
+
method(:perform).arity == 0 ? self.perform : self.perform(args)
|
|
138
138
|
rescue Exception => exception
|
|
139
139
|
self.persist_metadata!
|
|
140
140
|
self.log! '[Postburner] Exception raised during perform prevented completion.'
|
|
@@ -37,10 +37,12 @@ module Postburner
|
|
|
37
37
|
# @param options [Hash] Queue options
|
|
38
38
|
# @option options [Time, ActiveSupport::Duration] :at Absolute time to run the job
|
|
39
39
|
# @option options [Integer, ActiveSupport::Duration] :delay Seconds to delay execution
|
|
40
|
-
# @option options [Integer] :
|
|
41
|
-
# @option options [Integer] :
|
|
40
|
+
# @option options [Integer] :priority Priority (0-4294967295, 0 = HIGHEST), sets instance attribute
|
|
41
|
+
# @option options [Integer] :pri Beanstalkd priority (pass-through, for backwards compatibility)
|
|
42
|
+
# @option options [Integer] :ttr Time-to-run in seconds (1-4294967295, 0 is silently changed to 1)
|
|
43
|
+
# @option options [String] :queue Queue name override
|
|
42
44
|
#
|
|
43
|
-
# @return [
|
|
45
|
+
# @return [true] on success (including if already queued)
|
|
44
46
|
#
|
|
45
47
|
# @raise [ActiveRecord::RecordInvalid] if job is not valid
|
|
46
48
|
# @raise [AlreadyProcessed] if job was already processed
|
|
@@ -57,17 +59,25 @@ module Postburner
|
|
|
57
59
|
# job.queue!(at: '2025-01-15 09:00:00'.in_time_zone)
|
|
58
60
|
# job.queue!(at: Time.parse('2025-01-15 09:00:00 EST'))
|
|
59
61
|
#
|
|
60
|
-
# @example Queue with priority
|
|
61
|
-
# job.queue!(
|
|
62
|
+
# @example Queue with priority and TTR
|
|
63
|
+
# job.queue!(priority: 0, ttr: 600)
|
|
64
|
+
#
|
|
65
|
+
# @example Queue to specific queue
|
|
66
|
+
# job.queue!(queue: 'critical', delay: 30.minutes)
|
|
62
67
|
#
|
|
63
68
|
# @see #requeue!
|
|
64
69
|
# @see Postburner.queue_strategy
|
|
65
70
|
#
|
|
66
71
|
def queue!(options={})
|
|
67
|
-
return if self.queued_at.present? && self.bkid.present?
|
|
72
|
+
return true if self.queued_at.present? && self.bkid.present?
|
|
68
73
|
raise ActiveRecord::RecordInvalid, "Can't queue unless valid." unless self.valid?
|
|
69
74
|
raise AlreadyProcessed, "Processed at #{self.processed_at}" if self.processed_at
|
|
70
75
|
|
|
76
|
+
# Extract and set instance-level overrides
|
|
77
|
+
self.priority = options.delete(:priority) if options.key?(:priority)
|
|
78
|
+
self.ttr = options.delete(:ttr) if options.key?(:ttr)
|
|
79
|
+
self.queue_name = options.delete(:queue) if options.key?(:queue)
|
|
80
|
+
|
|
71
81
|
at = options.delete(:at)
|
|
72
82
|
now = Time.current
|
|
73
83
|
|
|
@@ -86,6 +96,8 @@ module Postburner
|
|
|
86
96
|
run_callbacks :enqueue do
|
|
87
97
|
self.save!
|
|
88
98
|
end
|
|
99
|
+
|
|
100
|
+
true
|
|
89
101
|
end
|
|
90
102
|
|
|
91
103
|
# Re-queues an existing job by removing it from Beanstalkd and queueing again.
|
|
@@ -96,10 +108,11 @@ module Postburner
|
|
|
96
108
|
# @param options [Hash] Queue options (same as {#queue!})
|
|
97
109
|
# @option options [Time, ActiveSupport::Duration] :at Absolute time to run the job
|
|
98
110
|
# @option options [Integer, ActiveSupport::Duration] :delay Seconds to delay execution
|
|
99
|
-
# @option options [Integer] :
|
|
100
|
-
# @option options [Integer] :ttr Time-to-run in seconds
|
|
111
|
+
# @option options [Integer] :priority Priority (0-4294967295, lower = higher priority)
|
|
112
|
+
# @option options [Integer] :ttr Time-to-run in seconds (1-4294967295)
|
|
113
|
+
# @option options [String] :queue Queue name override
|
|
101
114
|
#
|
|
102
|
-
# @return [
|
|
115
|
+
# @return [true] on success
|
|
103
116
|
#
|
|
104
117
|
# @raise [ActiveRecord::RecordInvalid] if job is not valid
|
|
105
118
|
# @raise [Beaneater::NotConnected] if Beanstalkd connection fails
|