postburner 1.0.0.pre.4 → 1.0.0.pre.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +137 -2
- data/app/concerns/postburner/insertion.rb +5 -2
- data/app/concerns/postburner/properties.rb +19 -13
- data/app/models/postburner/job.rb +34 -1
- data/bin/postburner +3 -34
- data/lib/postburner/configuration.rb +9 -2
- data/lib/postburner/connection.rb +62 -0
- data/lib/postburner/runner.rb +126 -0
- data/lib/postburner/strategies/queue.rb +6 -9
- data/lib/postburner/time_helpers.rb +4 -2
- data/lib/postburner/version.rb +1 -1
- data/lib/postburner.rb +103 -34
- data/lib/tasks/postburner.rake +18 -0
- metadata +3 -2
- data/MIT-LICENSE +0 -20
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a480ddab26850f1ecacc0f3c1ec75de82d8ac1f0acd2b4b7f0997dc5b3cc044c
|
|
4
|
+
data.tar.gz: 6d7ed3b8dd92df4d1064750c7eead44b8fa8d3c60887cb5c98771844a14f014d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 92f61d683537fa34a3aec8c7a052eb2313476fa2b647760d909022b28d0171db89b90c572b73af7eef6438bb1fc462f457d4b44bad1d249064993f68269de7e1
|
|
7
|
+
data.tar.gz: 495c9b4fba775507dd54666ab964bfa4d41fcc6608c3499ab5f4cc24b4f0af9a2cb4265b18331b566d61fce1318b1b1bc6a7ed9bc05d73619c5d5dd5dbc1711b
|
data/README.md
CHANGED
|
@@ -48,8 +48,11 @@ class ProcessPayment < ApplicationJob
|
|
|
48
48
|
end
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
-
# Run worker
|
|
51
|
+
# Run worker (bin/postburner)
|
|
52
52
|
bundle exec postburner --worker default
|
|
53
|
+
|
|
54
|
+
# Or with rake task
|
|
55
|
+
bundle exec rake postburner:work WORKER=default
|
|
53
56
|
```
|
|
54
57
|
|
|
55
58
|
## Why
|
|
@@ -124,7 +127,8 @@ beanstalkd -l 127.0.0.1 -p 11300 # Start beanstalkd
|
|
|
124
127
|
bundle exec rails generate postburner:install
|
|
125
128
|
bundle exec rails db:migrate
|
|
126
129
|
|
|
127
|
-
bundle exec postburner
|
|
130
|
+
bundle exec postburner # start with bin/postburner
|
|
131
|
+
bundle exec rake postburner:work # or with rake task
|
|
128
132
|
```
|
|
129
133
|
|
|
130
134
|
## Table of Contents
|
|
@@ -849,6 +853,52 @@ bin/postburner --worker general # Run the 'general' worker
|
|
|
849
853
|
bin/postburner --worker general --queues default,mailers # Only process specific queues
|
|
850
854
|
```
|
|
851
855
|
|
|
856
|
+
**Rake task:**
|
|
857
|
+
```bash
|
|
858
|
+
bundle exec rake postburner:work # Auto-select worker
|
|
859
|
+
bundle exec rake postburner:work WORKER=general # Specific worker
|
|
860
|
+
bundle exec rake postburner:work WORKER=general QUEUES=default,mailers # Filter queues
|
|
861
|
+
bundle exec rake postburner:work CONFIG=config/custom.yml # Custom config
|
|
862
|
+
```
|
|
863
|
+
|
|
864
|
+
### Programmatic Worker Control
|
|
865
|
+
|
|
866
|
+
Use `Postburner::Runner` to start workers programmatically from Ruby code:
|
|
867
|
+
|
|
868
|
+
```ruby
|
|
869
|
+
# Basic usage
|
|
870
|
+
runner = Postburner::Runner.new(
|
|
871
|
+
config: 'config/postburner.yml',
|
|
872
|
+
env: 'production',
|
|
873
|
+
worker: 'default'
|
|
874
|
+
)
|
|
875
|
+
runner.run
|
|
876
|
+
|
|
877
|
+
# With queue filtering
|
|
878
|
+
runner = Postburner::Runner.new(
|
|
879
|
+
worker: 'general',
|
|
880
|
+
queues: ['default', 'mailers']
|
|
881
|
+
)
|
|
882
|
+
runner.run
|
|
883
|
+
|
|
884
|
+
# From environment variables (Rake task pattern)
|
|
885
|
+
runner = Postburner::Runner.new(
|
|
886
|
+
config: ENV['CONFIG'] || 'config/postburner.yml',
|
|
887
|
+
env: Rails.env,
|
|
888
|
+
worker: ENV['WORKER'],
|
|
889
|
+
queues: ENV['QUEUES']&.split(',')
|
|
890
|
+
)
|
|
891
|
+
runner.run
|
|
892
|
+
```
|
|
893
|
+
|
|
894
|
+
**Options:**
|
|
895
|
+
- `:config` - Path to YAML configuration file (default: `'config/postburner.yml'`)
|
|
896
|
+
- `:env` - Environment name (default: `RAILS_ENV` or `'development'`)
|
|
897
|
+
- `:worker` - Worker name from config (required if multiple workers defined)
|
|
898
|
+
- `:queues` - Array of queue names to filter (overrides config queues)
|
|
899
|
+
|
|
900
|
+
This provides a unified interface used by both `bin/postburner` and `rake postburner:work`, making it easy to integrate Postburner workers into custom deployment scripts or process managers.
|
|
901
|
+
|
|
852
902
|
### Running Workers in Separate Processes
|
|
853
903
|
|
|
854
904
|
For production deployments, run different workers in separate OS processes for isolation and resource allocation:
|
|
@@ -1302,6 +1352,91 @@ end
|
|
|
1302
1352
|
```
|
|
1303
1353
|
|
|
1304
1354
|
|
|
1355
|
+
### Tube Statistics and Management
|
|
1356
|
+
|
|
1357
|
+
Postburner provides methods to inspect and manage Beanstalkd tubes:
|
|
1358
|
+
|
|
1359
|
+
**View tube statistics:**
|
|
1360
|
+
|
|
1361
|
+
```ruby
|
|
1362
|
+
# View all tubes on the Beanstalkd server
|
|
1363
|
+
stats = Postburner.stats
|
|
1364
|
+
# => {
|
|
1365
|
+
# tubes: [
|
|
1366
|
+
# { name: "postburner.production.default", ready: 10, delayed: 5, buried: 0, reserved: 2, total: 17 },
|
|
1367
|
+
# { name: "postburner.production.critical", ready: 0, delayed: 0, buried: 0, reserved: 1, total: 1 }
|
|
1368
|
+
# ],
|
|
1369
|
+
# totals: { ready: 10, delayed: 5, buried: 0, reserved: 3, total: 18 }
|
|
1370
|
+
# }
|
|
1371
|
+
|
|
1372
|
+
# View specific tubes only
|
|
1373
|
+
stats = Postburner.stats(['postburner.production.critical'])
|
|
1374
|
+
# => { tubes: [...], totals: {...} }
|
|
1375
|
+
```
|
|
1376
|
+
|
|
1377
|
+
**Clear jobs from tubes:**
|
|
1378
|
+
|
|
1379
|
+
For safety, `clear_jobs!` requires you to explicitly specify which tubes to clear. This prevents accidentally clearing tubes from other applications sharing the same Beanstalkd server.
|
|
1380
|
+
|
|
1381
|
+
```ruby
|
|
1382
|
+
# Collect stats only (no clearing)
|
|
1383
|
+
result = Postburner.clear_jobs!
|
|
1384
|
+
# => { tubes: [...], totals: {...}, cleared: false }
|
|
1385
|
+
|
|
1386
|
+
# Clear specific tubes (must be in config/postburner.yml)
|
|
1387
|
+
result = Postburner.clear_jobs!(['postburner.production.default'])
|
|
1388
|
+
# => { tubes: [...], totals: {...}, cleared: true }
|
|
1389
|
+
|
|
1390
|
+
# Pretty-print JSON output
|
|
1391
|
+
Postburner.clear_jobs!(['postburner.production.default'], silent: false)
|
|
1392
|
+
# Outputs formatted JSON to stdout
|
|
1393
|
+
|
|
1394
|
+
# Silent mode (no output, just return data)
|
|
1395
|
+
result = Postburner.clear_jobs!(['postburner.production.default'], silent: true)
|
|
1396
|
+
```
|
|
1397
|
+
|
|
1398
|
+
**Safety validation:**
|
|
1399
|
+
|
|
1400
|
+
Only tubes defined in your loaded configuration can be cleared. This prevents mistakes in multi-tenant Beanstalkd environments:
|
|
1401
|
+
|
|
1402
|
+
```ruby
|
|
1403
|
+
# Error: trying to clear tube not in config
|
|
1404
|
+
Postburner.clear_jobs!(['postburner.production.other-app'])
|
|
1405
|
+
# => ArgumentError: Cannot clear tubes not in configuration.
|
|
1406
|
+
# Invalid tubes: postburner.production.other-app
|
|
1407
|
+
# Configured tubes: postburner.production.default, postburner.production.critical
|
|
1408
|
+
```
|
|
1409
|
+
|
|
1410
|
+
**Shortcut using watched_tube_names:**
|
|
1411
|
+
|
|
1412
|
+
Clear all configured tubes at once:
|
|
1413
|
+
|
|
1414
|
+
```ruby
|
|
1415
|
+
# Get all tubes from current configuration
|
|
1416
|
+
watched_tubes = Postburner.watched_tube_names
|
|
1417
|
+
# => ["postburner.production.default", "postburner.production.critical", "postburner.production.mailers"]
|
|
1418
|
+
|
|
1419
|
+
# Clear all configured tubes
|
|
1420
|
+
Postburner.clear_jobs!(watched_tubes, silent: true)
|
|
1421
|
+
# or
|
|
1422
|
+
Postburner.clear_jobs!(Postburner.watched_tube_names, silent: true)
|
|
1423
|
+
```
|
|
1424
|
+
|
|
1425
|
+
**Low-level Connection API:**
|
|
1426
|
+
|
|
1427
|
+
For programmatic use without output formatting, use `Connection#clear_tubes!`:
|
|
1428
|
+
|
|
1429
|
+
```ruby
|
|
1430
|
+
Postburner.connected do |conn|
|
|
1431
|
+
# Returns data only (no puts)
|
|
1432
|
+
result = conn.clear_tubes!(Postburner.watched_tube_names)
|
|
1433
|
+
# => { tubes: [...], totals: {...}, cleared: true }
|
|
1434
|
+
|
|
1435
|
+
# Same validation - must be in configuration
|
|
1436
|
+
result = conn.clear_tubes!(['postburner.production.default'])
|
|
1437
|
+
end
|
|
1438
|
+
```
|
|
1439
|
+
|
|
1305
1440
|
## Web UI
|
|
1306
1441
|
|
|
1307
1442
|
Mount the inspection interface:
|
|
@@ -24,6 +24,10 @@ module Postburner
|
|
|
24
24
|
module Insertion
|
|
25
25
|
extend ActiveSupport::Concern
|
|
26
26
|
|
|
27
|
+
included do
|
|
28
|
+
after_save_commit :insert_if_queued!
|
|
29
|
+
end
|
|
30
|
+
|
|
27
31
|
# Enqueues the job to Beanstalkd for processing.
|
|
28
32
|
#
|
|
29
33
|
# Sets queued_at timestamp and optionally run_at for scheduled execution.
|
|
@@ -155,9 +159,8 @@ module Postburner
|
|
|
155
159
|
#debugger
|
|
156
160
|
|
|
157
161
|
# Response must be a hash with an :id key (value can be nil)
|
|
158
|
-
# Backburner returns symbol keys
|
|
159
162
|
unless response.is_a?(Hash) && response.key?(:id)
|
|
160
|
-
raise MalformedResponse, "Missing :id key in response: #{response.inspect}"
|
|
163
|
+
raise Postburner::Job::MalformedResponse, "Missing :id key in response: #{response.inspect}"
|
|
161
164
|
end
|
|
162
165
|
|
|
163
166
|
persist_metadata!(bkid: response[:id])
|
|
@@ -170,6 +170,20 @@ module Postburner
|
|
|
170
170
|
self.class.queue
|
|
171
171
|
end
|
|
172
172
|
|
|
173
|
+
# Returns the full tube name with environment prefix.
|
|
174
|
+
#
|
|
175
|
+
# Expands the queue name to include the environment prefix
|
|
176
|
+
# (e.g., 'critical' becomes 'postburner.development.critical').
|
|
177
|
+
#
|
|
178
|
+
# @return [String] Full tube name with environment prefix
|
|
179
|
+
#
|
|
180
|
+
# @example
|
|
181
|
+
# job.expanded_tube_name # => 'postburner.development.critical'
|
|
182
|
+
#
|
|
183
|
+
def expanded_tube_name
|
|
184
|
+
Postburner.configuration.expand_tube_name(queue_name)
|
|
185
|
+
end
|
|
186
|
+
|
|
173
187
|
# Returns the priority for this job instance.
|
|
174
188
|
#
|
|
175
189
|
# Checks instance-level override first, then falls back to class-level configuration.
|
|
@@ -184,6 +198,11 @@ module Postburner
|
|
|
184
198
|
@priority || self.class.priority
|
|
185
199
|
end
|
|
186
200
|
|
|
201
|
+
# Alias for {#priority}.
|
|
202
|
+
#
|
|
203
|
+
# @see #priority
|
|
204
|
+
alias_method :pri, :priority
|
|
205
|
+
|
|
187
206
|
# Returns the TTR (time-to-run) for this job instance.
|
|
188
207
|
#
|
|
189
208
|
# Checks instance-level override first, then falls back to class-level configuration.
|
|
@@ -198,18 +217,5 @@ module Postburner
|
|
|
198
217
|
@ttr || self.class.ttr
|
|
199
218
|
end
|
|
200
219
|
|
|
201
|
-
# Returns the full tube name with environment prefix.
|
|
202
|
-
#
|
|
203
|
-
# Expands the queue name to include the environment prefix
|
|
204
|
-
# (e.g., 'critical' becomes 'postburner.development.critical').
|
|
205
|
-
#
|
|
206
|
-
# @return [String] Full tube name with environment prefix
|
|
207
|
-
#
|
|
208
|
-
# @example
|
|
209
|
-
# job.expanded_tube_name # => 'postburner.development.critical'
|
|
210
|
-
#
|
|
211
|
-
def expanded_tube_name
|
|
212
|
-
Postburner.configuration.expand_tube_name(queue_name)
|
|
213
|
-
end
|
|
214
220
|
end
|
|
215
221
|
end
|
|
@@ -81,7 +81,6 @@ module Postburner
|
|
|
81
81
|
|
|
82
82
|
before_validation :ensure_sid!
|
|
83
83
|
before_destroy :delete!
|
|
84
|
-
after_save_commit :insert_if_queued!
|
|
85
84
|
|
|
86
85
|
validates :sid, presence: {strict: true}
|
|
87
86
|
|
|
@@ -237,6 +236,40 @@ module Postburner
|
|
|
237
236
|
#
|
|
238
237
|
class MalformedResponse < StandardError; end
|
|
239
238
|
|
|
239
|
+
# Exception raised when Beanstalkd rejects a job with BAD_FORMAT error.
|
|
240
|
+
#
|
|
241
|
+
# Beanstalkd returns BAD_FORMAT when job parameters are invalid, such as:
|
|
242
|
+
# - Negative delay values
|
|
243
|
+
# - Invalid priority (outside 0-4294967295 range)
|
|
244
|
+
# - Invalid TTR (time-to-run)
|
|
245
|
+
# - Malformed job data
|
|
246
|
+
#
|
|
247
|
+
# This exception wraps the original Beaneater::BadFormatError and includes
|
|
248
|
+
# detailed debugging information about the parameters that caused the error.
|
|
249
|
+
# The original exception is preserved via the `cause` chain.
|
|
250
|
+
#
|
|
251
|
+
# @example Common causes
|
|
252
|
+
# # Negative delay (now prevented by fix in queue.rb)
|
|
253
|
+
# job.queue!(at: 1.hour.ago) # Previously caused BAD_FORMAT
|
|
254
|
+
#
|
|
255
|
+
# # Invalid priority
|
|
256
|
+
# job.queue!(pri: -1) # Outside valid range
|
|
257
|
+
#
|
|
258
|
+
# @example Accessing detailed error information
|
|
259
|
+
# begin
|
|
260
|
+
# job.queue!(at: past_time)
|
|
261
|
+
# rescue Postburner::Job::BadFormat => e
|
|
262
|
+
# puts e.message # Includes tube, data, pri, delay, ttr
|
|
263
|
+
# puts e.cause # Original Beaneater::BadFormatError
|
|
264
|
+
# end
|
|
265
|
+
#
|
|
266
|
+
# @note The error message includes full job parameters for debugging
|
|
267
|
+
# @note Access original Beanstalkd error via exception.cause
|
|
268
|
+
#
|
|
269
|
+
# @see Postburner::Queue#insert
|
|
270
|
+
#
|
|
271
|
+
class BadFormat < StandardError; end
|
|
272
|
+
|
|
240
273
|
private
|
|
241
274
|
|
|
242
275
|
|
data/bin/postburner
CHANGED
|
@@ -55,37 +55,6 @@ require File.expand_path('config/environment', Dir.pwd)
|
|
|
55
55
|
|
|
56
56
|
# Postburner is loaded automatically via the Rails engine when the gem is in your Gemfile
|
|
57
57
|
|
|
58
|
-
#
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
begin
|
|
62
|
-
config = Postburner::Configuration.load_yaml(config_path, options[:env], options[:worker])
|
|
63
|
-
rescue ArgumentError => e
|
|
64
|
-
Rails.logger.error "[Postburner] ERROR: #{e.message}"
|
|
65
|
-
exit 1
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
# Filter queues if --queues option provided
|
|
69
|
-
if options[:queues]
|
|
70
|
-
# Validate that all specified queues exist in config
|
|
71
|
-
invalid_queues = options[:queues] - config.queue_names
|
|
72
|
-
unless invalid_queues.empty?
|
|
73
|
-
config.logger.error "[Postburner] ERROR: Unknown queue(s): #{invalid_queues.join(', ')}"
|
|
74
|
-
config.logger.error "[Postburner] Available queues: #{config.queue_names.join(', ')}"
|
|
75
|
-
exit 1
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
# Filter config to only include specified queues
|
|
79
|
-
filtered_queues = config.queues.select { |name, _| options[:queues].include?(name.to_s) }
|
|
80
|
-
config.queues = filtered_queues
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
config.logger.info "[Postburner] Configuration: #{config_path}"
|
|
84
|
-
config.logger.info "[Postburner] Environment: #{options[:env]}"
|
|
85
|
-
config.logger.info "[Postburner] Worker: #{options[:worker] || '(auto-selected)'}" if options[:worker] || options[:queues].nil?
|
|
86
|
-
config.logger.info "[Postburner] Queues: #{config.queue_names.join(', ')}"
|
|
87
|
-
config.logger.info "[Postburner] Defaults: forks=#{config.default_forks}, threads=#{config.default_threads}, gc_limit=#{config.default_gc_limit || 'none'}"
|
|
88
|
-
|
|
89
|
-
# Create and start worker
|
|
90
|
-
worker = Postburner::Workers::Worker.new(config)
|
|
91
|
-
worker.start
|
|
58
|
+
# Create and run worker
|
|
59
|
+
runner = Postburner::Runner.new(options)
|
|
60
|
+
runner.run
|
|
@@ -99,11 +99,18 @@ module Postburner
|
|
|
99
99
|
if workers.size == 1
|
|
100
100
|
worker_name = workers.keys.first
|
|
101
101
|
else
|
|
102
|
-
raise ArgumentError,
|
|
102
|
+
raise ArgumentError, <<~ERROR
|
|
103
|
+
Configuration has multiple workers, but --worker not specified
|
|
104
|
+
Available workers: #{workers.keys.join(', ')}
|
|
105
|
+
Usage: bin/postburner --worker <name>
|
|
106
|
+
ERROR
|
|
103
107
|
end
|
|
104
108
|
else
|
|
105
109
|
unless workers.key?(worker_name)
|
|
106
|
-
raise ArgumentError,
|
|
110
|
+
raise ArgumentError, <<~ERROR
|
|
111
|
+
Worker '#{worker_name}' not found in #{path}
|
|
112
|
+
Available workers: #{workers.keys.join(', ')}
|
|
113
|
+
ERROR
|
|
107
114
|
end
|
|
108
115
|
end
|
|
109
116
|
|
|
@@ -88,6 +88,68 @@ module Postburner
|
|
|
88
88
|
@pool = nil
|
|
89
89
|
end
|
|
90
90
|
|
|
91
|
+
# Clears jobs from specified tubes or collects stats for all tubes.
|
|
92
|
+
#
|
|
93
|
+
# Low-level method that returns data only (no output to stdout).
|
|
94
|
+
# Delegates to Postburner.stats for collecting statistics.
|
|
95
|
+
# For user-facing output, use Postburner.clear_jobs! instead.
|
|
96
|
+
#
|
|
97
|
+
# SAFETY: Only allows clearing tubes that are defined in the loaded
|
|
98
|
+
# configuration (watched_tube_names). This prevents accidentally clearing
|
|
99
|
+
# tubes from other applications or environments.
|
|
100
|
+
#
|
|
101
|
+
# @param tube_names [Array<String>, nil] Array of tube names to clear, or nil to only collect stats
|
|
102
|
+
#
|
|
103
|
+
# @return [Hash] Statistics and results with keys:
|
|
104
|
+
# - tubes: Array of hashes with per-tube stats
|
|
105
|
+
# - totals: Hash with aggregated counts across all tubes
|
|
106
|
+
# - cleared: Boolean indicating if tubes were actually cleared
|
|
107
|
+
#
|
|
108
|
+
# @raise [ArgumentError] if tube_names contains tubes not in watched_tube_names
|
|
109
|
+
#
|
|
110
|
+
# @example Collect stats only (no clearing)
|
|
111
|
+
# result = conn.clear_tubes!
|
|
112
|
+
# result[:totals][:total] # => 42
|
|
113
|
+
#
|
|
114
|
+
# @example Clear configured tubes only
|
|
115
|
+
# result = conn.clear_tubes!(Postburner.watched_tube_names)
|
|
116
|
+
# result[:cleared] # => true
|
|
117
|
+
#
|
|
118
|
+
# @example Invalid tube raises error
|
|
119
|
+
# conn.clear_tubes!(['random-tube'])
|
|
120
|
+
# # => ArgumentError: Cannot clear tubes not in configuration
|
|
121
|
+
#
|
|
122
|
+
def clear_tubes!(tube_names = nil)
|
|
123
|
+
ensure_connected!
|
|
124
|
+
|
|
125
|
+
# Validate that tubes to clear are in the loaded configuration
|
|
126
|
+
if tube_names&.any?
|
|
127
|
+
watched = Postburner.watched_tube_names
|
|
128
|
+
invalid_tubes = tube_names - watched
|
|
129
|
+
|
|
130
|
+
if invalid_tubes.any?
|
|
131
|
+
raise ArgumentError, <<~ERROR
|
|
132
|
+
Cannot clear tubes not in configuration.
|
|
133
|
+
Invalid tubes: #{invalid_tubes.join(', ')}
|
|
134
|
+
Configured tubes: #{watched.join(', ')}
|
|
135
|
+
ERROR
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Get stats using Postburner.stats
|
|
140
|
+
result = Postburner.stats(tube_names)
|
|
141
|
+
result[:cleared] = tube_names&.any? ? true : false
|
|
142
|
+
|
|
143
|
+
# Actually clear if tube names were provided and validated
|
|
144
|
+
if tube_names&.any?
|
|
145
|
+
tube_names.each do |tube_name|
|
|
146
|
+
tubes[tube_name].clear
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
result
|
|
151
|
+
end
|
|
152
|
+
|
|
91
153
|
private
|
|
92
154
|
|
|
93
155
|
# Establishes connection to Beanstalkd.
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Postburner
|
|
4
|
+
# Shared runner logic for bin/postburner and rake tasks.
|
|
5
|
+
#
|
|
6
|
+
# Provides a unified interface for starting Postburner workers from
|
|
7
|
+
# both the command-line executable and Rake tasks. Handles configuration
|
|
8
|
+
# loading, queue filtering, logging, and worker startup.
|
|
9
|
+
#
|
|
10
|
+
# @example Using from bin/postburner
|
|
11
|
+
# options = { config: 'config/postburner.yml', env: 'production', worker: 'default' }
|
|
12
|
+
# runner = Postburner::Runner.new(options)
|
|
13
|
+
# runner.run
|
|
14
|
+
#
|
|
15
|
+
# @example Using from Rake task
|
|
16
|
+
# options = { worker: ENV['WORKER'], queues: ENV['QUEUES']&.split(',') }
|
|
17
|
+
# runner = Postburner::Runner.new(options)
|
|
18
|
+
# runner.run
|
|
19
|
+
#
|
|
20
|
+
class Runner
|
|
21
|
+
attr_reader :options
|
|
22
|
+
|
|
23
|
+
# Initialize runner with options.
|
|
24
|
+
#
|
|
25
|
+
# @param options [Hash] Runner options
|
|
26
|
+
# @option options [String] :config Path to YAML configuration file (default: 'config/postburner.yml')
|
|
27
|
+
# @option options [String] :env Environment name (default: RAILS_ENV or 'development')
|
|
28
|
+
# @option options [String, nil] :worker Worker name from config (required if multiple workers defined)
|
|
29
|
+
# @option options [Array<String>, nil] :queues List of queue names to filter (overrides config)
|
|
30
|
+
def initialize(options = {})
|
|
31
|
+
@options = {
|
|
32
|
+
config: 'config/postburner.yml',
|
|
33
|
+
env: ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development',
|
|
34
|
+
worker: nil,
|
|
35
|
+
queues: nil
|
|
36
|
+
}.merge(options)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Load configuration, filter queues, and start worker.
|
|
40
|
+
#
|
|
41
|
+
# @return [void]
|
|
42
|
+
def run
|
|
43
|
+
config = load_configuration
|
|
44
|
+
filter_queues(config) if options[:queues]
|
|
45
|
+
log_configuration(config)
|
|
46
|
+
start_worker(config)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
# Load configuration from YAML file.
|
|
52
|
+
#
|
|
53
|
+
# @return [Postburner::Configuration]
|
|
54
|
+
def load_configuration
|
|
55
|
+
config_path = File.expand_path(options[:config], root_directory)
|
|
56
|
+
Postburner::Configuration.load_yaml(config_path, options[:env], options[:worker])
|
|
57
|
+
rescue ArgumentError => e
|
|
58
|
+
logger.error "[Postburner] ERROR: #{e.message}"
|
|
59
|
+
exit 1
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Filter configuration to only include specified queues.
|
|
63
|
+
#
|
|
64
|
+
# @param config [Postburner::Configuration]
|
|
65
|
+
# @return [void]
|
|
66
|
+
def filter_queues(config)
|
|
67
|
+
# Validate that all specified queues exist in config
|
|
68
|
+
invalid_queues = options[:queues] - config.queue_names
|
|
69
|
+
unless invalid_queues.empty?
|
|
70
|
+
config.logger.error "[Postburner] ERROR: Unknown queue(s): #{invalid_queues.join(', ')}"
|
|
71
|
+
config.logger.error "[Postburner] Available queues: #{config.queue_names.join(', ')}"
|
|
72
|
+
exit 1
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Filter config to only include specified queues
|
|
76
|
+
filtered_queues = config.queues.select { |name, _| options[:queues].include?(name.to_s) }
|
|
77
|
+
config.queues = filtered_queues
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Log configuration details.
|
|
81
|
+
#
|
|
82
|
+
# @param config [Postburner::Configuration]
|
|
83
|
+
# @return [void]
|
|
84
|
+
def log_configuration(config)
|
|
85
|
+
config_path = File.expand_path(options[:config], root_directory)
|
|
86
|
+
config.logger.info "[Postburner] Configuration: #{config_path}"
|
|
87
|
+
config.logger.info "[Postburner] Environment: #{options[:env]}"
|
|
88
|
+
config.logger.info "[Postburner] Worker: #{options[:worker] || '(auto-selected)'}" if options[:worker] || options[:queues].nil?
|
|
89
|
+
config.logger.info "[Postburner] Queues: #{config.queue_names.join(', ')}"
|
|
90
|
+
config.logger.info "[Postburner] Defaults: forks=#{config.default_forks}, threads=#{config.default_threads}, gc_limit=#{config.default_gc_limit || 'none'}"
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Returns root directory for config file resolution.
|
|
94
|
+
#
|
|
95
|
+
# Uses Rails.root when Rails is defined, otherwise falls back to current directory.
|
|
96
|
+
#
|
|
97
|
+
# @return [String] Root directory path
|
|
98
|
+
def root_directory
|
|
99
|
+
if defined?(Rails) && Rails.respond_to?(:root)
|
|
100
|
+
Rails.root.to_s
|
|
101
|
+
else
|
|
102
|
+
Dir.pwd
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Create and start worker.
|
|
107
|
+
#
|
|
108
|
+
# @param config [Postburner::Configuration]
|
|
109
|
+
# @return [void]
|
|
110
|
+
def start_worker(config)
|
|
111
|
+
worker = Postburner::Workers::Worker.new(config)
|
|
112
|
+
worker.start
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Returns logger for error reporting during initialization.
|
|
116
|
+
#
|
|
117
|
+
# @return [Logger]
|
|
118
|
+
def logger
|
|
119
|
+
if defined?(Rails)
|
|
120
|
+
Rails.logger
|
|
121
|
+
else
|
|
122
|
+
Logger.new($stderr)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -81,24 +81,21 @@ module Postburner
|
|
|
81
81
|
|
|
82
82
|
# Get priority, TTR from job instance (respects instance overrides) or options
|
|
83
83
|
pri = options[:pri] || job.priority || Postburner.configuration.default_priority
|
|
84
|
-
delay = options[:delay]
|
|
84
|
+
delay = options[:delay].present? ? [0, options[:delay].to_i].max : 0
|
|
85
85
|
ttr = options[:ttr] || job.ttr || Postburner.configuration.default_ttr
|
|
86
|
+
payload = JSON.generate(data)
|
|
86
87
|
|
|
87
88
|
begin
|
|
88
89
|
response = conn.tubes[tube_name].put(
|
|
89
|
-
|
|
90
|
+
payload,
|
|
90
91
|
pri: pri,
|
|
91
92
|
delay: delay,
|
|
92
93
|
ttr: ttr
|
|
93
94
|
)
|
|
94
95
|
rescue Beaneater::BadFormatError => e
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
Rails.logger.error " pri: #{pri.inspect} (#{pri.class})"
|
|
99
|
-
Rails.logger.error " delay: #{delay.inspect} (#{delay.class})"
|
|
100
|
-
Rails.logger.error " ttr: #{ttr.inspect} (#{ttr.class})"
|
|
101
|
-
raise
|
|
96
|
+
raise Postburner::Job::BadFormat.new(
|
|
97
|
+
"Beanstalkd BAD_FORMAT: tube=#{tube_name} payload=#{payload} pri=#{pri.inspect} delay=#{delay.inspect} ttr=#{ttr.inspect}"
|
|
98
|
+
), cause: e
|
|
102
99
|
end
|
|
103
100
|
|
|
104
101
|
response
|
|
@@ -64,8 +64,10 @@ module Postburner
|
|
|
64
64
|
#
|
|
65
65
|
def travel_to(time, &block)
|
|
66
66
|
unless defined?(ActiveSupport::Testing::TimeHelpers)
|
|
67
|
-
raise
|
|
68
|
-
|
|
67
|
+
raise <<~ERROR
|
|
68
|
+
ActiveSupport::Testing::TimeHelpers not available.
|
|
69
|
+
Postburner::TimeHelpers requires Rails testing helpers for time travel.
|
|
70
|
+
ERROR
|
|
69
71
|
end
|
|
70
72
|
|
|
71
73
|
helper = Object.new.extend(ActiveSupport::Testing::TimeHelpers)
|
data/lib/postburner/version.rb
CHANGED
data/lib/postburner.rb
CHANGED
|
@@ -8,6 +8,7 @@ require "postburner/active_job/execution"
|
|
|
8
8
|
require "postburner/active_job/adapter"
|
|
9
9
|
require "postburner/workers/base"
|
|
10
10
|
require "postburner/workers/worker"
|
|
11
|
+
require "postburner/runner"
|
|
11
12
|
require "postburner/engine"
|
|
12
13
|
require "postburner/strategies/queue"
|
|
13
14
|
require "postburner/strategies/nice_queue"
|
|
@@ -374,25 +375,50 @@ module Postburner
|
|
|
374
375
|
end
|
|
375
376
|
end
|
|
376
377
|
|
|
377
|
-
#
|
|
378
|
+
# Clears jobs from specified tubes or shows stats for all tubes.
|
|
378
379
|
#
|
|
379
|
-
#
|
|
380
|
-
#
|
|
380
|
+
# High-level method with formatted output. Delegates to Connection#clear_tubes!
|
|
381
|
+
# for the actual work, then pretty-prints the results.
|
|
381
382
|
#
|
|
382
|
-
#
|
|
383
|
+
# SAFETY: Only allows clearing tubes that are defined in the loaded
|
|
384
|
+
# configuration. This prevents accidentally clearing tubes from other
|
|
385
|
+
# applications or environments sharing the same Beanstalkd server.
|
|
383
386
|
#
|
|
384
|
-
# @
|
|
387
|
+
# @param tube_names [Array<String>, nil] Array of tube names to clear, or nil to only show stats
|
|
388
|
+
# @param silent [Boolean] If true, suppress output to stdout (default: false)
|
|
385
389
|
#
|
|
386
|
-
# @
|
|
387
|
-
#
|
|
390
|
+
# @return [Hash] Statistics and results (see Connection#clear_tubes!)
|
|
391
|
+
#
|
|
392
|
+
# @raise [ArgumentError] if tube_names contains tubes not in watched_tube_names
|
|
393
|
+
#
|
|
394
|
+
# @example Show stats only (no clearing) - SAFE
|
|
395
|
+
# Postburner.clear_jobs!
|
|
396
|
+
# # Shows stats for ALL tubes on Beanstalkd, but doesn't clear anything
|
|
397
|
+
#
|
|
398
|
+
# @example Clear watched tubes only - SAFE
|
|
399
|
+
# Postburner.clear_jobs!(Postburner.watched_tube_names)
|
|
400
|
+
# # Only clears tubes defined in your config
|
|
388
401
|
#
|
|
389
|
-
# @
|
|
390
|
-
#
|
|
402
|
+
# @example Trying to clear unconfigured tube - RAISES ERROR
|
|
403
|
+
# Postburner.clear_jobs!(['some-other-app-tube'])
|
|
404
|
+
# # => ArgumentError: Cannot clear tubes not in configuration
|
|
391
405
|
#
|
|
392
|
-
|
|
393
|
-
|
|
406
|
+
# @example Silent mode (programmatic use)
|
|
407
|
+
# result = Postburner.clear_jobs!(Postburner.watched_tube_names, silent: true)
|
|
408
|
+
# result[:totals][:total] # => 42
|
|
409
|
+
#
|
|
410
|
+
# @see Connection#clear_tubes!
|
|
411
|
+
#
|
|
412
|
+
def self.clear_jobs!(tube_names = nil, silent: false)
|
|
413
|
+
require 'json'
|
|
414
|
+
|
|
415
|
+
result = connection.clear_tubes!(tube_names)
|
|
416
|
+
|
|
417
|
+
unless silent
|
|
418
|
+
puts JSON.pretty_generate(result)
|
|
419
|
+
end
|
|
394
420
|
|
|
395
|
-
|
|
421
|
+
result
|
|
396
422
|
end
|
|
397
423
|
|
|
398
424
|
# Returns array of watched tube names with environment prefix.
|
|
@@ -422,39 +448,82 @@ module Postburner
|
|
|
422
448
|
@__watched_tubes ||= watched_tube_names.map { |tube_name| connection.tubes[tube_name] }
|
|
423
449
|
end
|
|
424
450
|
|
|
425
|
-
# Returns statistics
|
|
451
|
+
# Returns detailed statistics about Beanstalkd tubes.
|
|
426
452
|
#
|
|
427
|
-
#
|
|
428
|
-
#
|
|
453
|
+
# Collects job counts (ready, delayed, buried, reserved) for each tube
|
|
454
|
+
# and provides aggregate totals across all tubes.
|
|
429
455
|
#
|
|
430
|
-
#
|
|
431
|
-
# - tube.stats - Tube statistics hash (current-jobs-ready, current-jobs-buried, etc.)
|
|
432
|
-
# - tube.peek_ready - Next ready job
|
|
433
|
-
# - tube.peek_delayed - Next delayed job
|
|
434
|
-
# - tube.peek_buried - Next buried job
|
|
435
|
-
# - tube.kick(n) - Kick n buried jobs back to ready
|
|
436
|
-
# - tube.pause(delay) - Pause tube for delay seconds
|
|
437
|
-
# - tube.clear - Delete all jobs in tube
|
|
456
|
+
# @param tube_names [Array<String>, nil] Specific tube names to inspect, or nil for all tubes
|
|
438
457
|
#
|
|
439
|
-
# @return [Hash] Statistics hash with
|
|
440
|
-
# -
|
|
441
|
-
# -
|
|
458
|
+
# @return [Hash] Statistics hash with keys:
|
|
459
|
+
# - tubes: Array of hashes with per-tube stats (name, ready, delayed, buried, reserved, total)
|
|
460
|
+
# - totals: Hash with aggregated counts across all tubes
|
|
442
461
|
#
|
|
443
462
|
# @raise [Beaneater::NotConnected] if connection to Beanstalkd fails
|
|
444
463
|
#
|
|
445
|
-
# @example
|
|
464
|
+
# @example Get stats for all tubes
|
|
446
465
|
# stats = Postburner.stats
|
|
447
|
-
# stats[:
|
|
448
|
-
# stats[:tubes].first
|
|
466
|
+
# stats[:totals][:total] # => 42
|
|
467
|
+
# stats[:tubes].first[:name] # => "default"
|
|
468
|
+
#
|
|
469
|
+
# @example Get stats for specific tubes
|
|
470
|
+
# stats = Postburner.stats(Postburner.watched_tube_names)
|
|
471
|
+
# stats[:tubes].size # => 3
|
|
449
472
|
#
|
|
450
|
-
def self.stats
|
|
473
|
+
def self.stats(tube_names = nil)
|
|
451
474
|
connected do |conn|
|
|
475
|
+
# Get tubes to inspect
|
|
476
|
+
tubes_to_inspect = if tube_names&.any?
|
|
477
|
+
tube_names.map { |name| conn.tubes[name] }
|
|
478
|
+
else
|
|
479
|
+
conn.beanstalk.tubes.all
|
|
480
|
+
end
|
|
452
481
|
|
|
453
|
-
{
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
482
|
+
result = {
|
|
483
|
+
tubes: [],
|
|
484
|
+
totals: {
|
|
485
|
+
ready: 0,
|
|
486
|
+
delayed: 0,
|
|
487
|
+
buried: 0,
|
|
488
|
+
reserved: 0,
|
|
489
|
+
total: 0
|
|
490
|
+
}
|
|
457
491
|
}
|
|
492
|
+
|
|
493
|
+
# Collect stats from each tube
|
|
494
|
+
tubes_to_inspect.each do |tube|
|
|
495
|
+
begin
|
|
496
|
+
stats = tube.stats
|
|
497
|
+
# Beaneater returns a StatStruct; access the underlying hash
|
|
498
|
+
stats_hash = stats.instance_variable_get(:@hash) || {}
|
|
499
|
+
|
|
500
|
+
tube_data = {
|
|
501
|
+
name: tube.name,
|
|
502
|
+
ready: stats_hash['current-jobs-ready'] || 0,
|
|
503
|
+
delayed: stats_hash['current-jobs-delayed'] || 0,
|
|
504
|
+
buried: stats_hash['current-jobs-buried'] || 0,
|
|
505
|
+
reserved: stats_hash['current-jobs-reserved'] || 0,
|
|
506
|
+
total: (stats_hash['current-jobs-ready'] || 0) +
|
|
507
|
+
(stats_hash['current-jobs-delayed'] || 0) +
|
|
508
|
+
(stats_hash['current-jobs-buried'] || 0) +
|
|
509
|
+
(stats_hash['current-jobs-reserved'] || 0)
|
|
510
|
+
}
|
|
511
|
+
rescue Beaneater::NotFoundError
|
|
512
|
+
# Tube doesn't exist yet, skip it
|
|
513
|
+
next
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
result[:tubes] << tube_data
|
|
517
|
+
|
|
518
|
+
# Aggregate totals
|
|
519
|
+
result[:totals][:ready] += tube_data[:ready]
|
|
520
|
+
result[:totals][:delayed] += tube_data[:delayed]
|
|
521
|
+
result[:totals][:buried] += tube_data[:buried]
|
|
522
|
+
result[:totals][:reserved] += tube_data[:reserved]
|
|
523
|
+
result[:totals][:total] += tube_data[:total]
|
|
524
|
+
end
|
|
525
|
+
|
|
526
|
+
result
|
|
458
527
|
end
|
|
459
528
|
end
|
|
460
529
|
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
namespace :postburner do
|
|
4
|
+
desc "Start Postburner worker (ENV: WORKER=name, QUEUES=queue1,queue2, CONFIG=path)"
|
|
5
|
+
task work: :environment do
|
|
6
|
+
# Parse options from environment variables
|
|
7
|
+
options = {
|
|
8
|
+
config: ENV['CONFIG'] || 'config/postburner.yml',
|
|
9
|
+
env: Rails.env,
|
|
10
|
+
worker: ENV['WORKER'],
|
|
11
|
+
queues: ENV['QUEUES']&.split(',')&.map(&:strip)
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
# Create and run worker
|
|
15
|
+
runner = Postburner::Runner.new(options)
|
|
16
|
+
runner.run
|
|
17
|
+
end
|
|
18
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: postburner
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.0.pre.
|
|
4
|
+
version: 1.0.0.pre.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Matt Smith
|
|
@@ -104,7 +104,6 @@ extensions: []
|
|
|
104
104
|
extra_rdoc_files: []
|
|
105
105
|
files:
|
|
106
106
|
- CHANGELOG.md
|
|
107
|
-
- MIT-LICENSE
|
|
108
107
|
- README.md
|
|
109
108
|
- Rakefile
|
|
110
109
|
- app/assets/config/postburner_manifest.js
|
|
@@ -147,6 +146,7 @@ files:
|
|
|
147
146
|
- lib/postburner/configuration.rb
|
|
148
147
|
- lib/postburner/connection.rb
|
|
149
148
|
- lib/postburner/engine.rb
|
|
149
|
+
- lib/postburner/runner.rb
|
|
150
150
|
- lib/postburner/strategies/immediate_test_queue.rb
|
|
151
151
|
- lib/postburner/strategies/nice_queue.rb
|
|
152
152
|
- lib/postburner/strategies/null_queue.rb
|
|
@@ -157,6 +157,7 @@ files:
|
|
|
157
157
|
- lib/postburner/version.rb
|
|
158
158
|
- lib/postburner/workers/base.rb
|
|
159
159
|
- lib/postburner/workers/worker.rb
|
|
160
|
+
- lib/tasks/postburner.rake
|
|
160
161
|
- lib/tasks/postburner_tasks.rake
|
|
161
162
|
homepage: https://gitlab.nearapogee.com/opensource/postburner
|
|
162
163
|
licenses:
|
data/MIT-LICENSE
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
Copyright 2021 Matt Smith
|
|
2
|
-
|
|
3
|
-
Permission is hereby granted, free of charge, to any person obtaining
|
|
4
|
-
a copy of this software and associated documentation files (the
|
|
5
|
-
"Software"), to deal in the Software without restriction, including
|
|
6
|
-
without limitation the rights to use, copy, modify, merge, publish,
|
|
7
|
-
distribute, sublicense, and/or sell copies of the Software, and to
|
|
8
|
-
permit persons to whom the Software is furnished to do so, subject to
|
|
9
|
-
the following conditions:
|
|
10
|
-
|
|
11
|
-
The above copyright notice and this permission notice shall be
|
|
12
|
-
included in all copies or substantial portions of the Software.
|
|
13
|
-
|
|
14
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
15
|
-
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
16
|
-
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
17
|
-
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
18
|
-
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
19
|
-
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
20
|
-
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|