postburner 1.0.0.pre.2 → 1.0.0.pre.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +248 -167
- data/app/concerns/postburner/commands.rb +143 -0
- data/app/concerns/postburner/execution.rb +190 -0
- data/app/concerns/postburner/insertion.rb +170 -0
- data/app/concerns/postburner/logging.rb +181 -0
- data/{lib/postburner/queue_config.rb → app/concerns/postburner/properties.rb} +90 -26
- data/app/concerns/postburner/statistics.rb +125 -0
- data/app/models/postburner/job.rb +15 -757
- data/app/models/postburner/mailer.rb +2 -2
- data/app/views/postburner/jobs/show.html.haml +2 -2
- data/config/postburner.yml.example +32 -30
- data/lib/generators/postburner/install/templates/config/postburner.yml +32 -30
- data/lib/postburner/active_job/adapter.rb +9 -9
- data/lib/postburner/beanstalkd.rb +18 -39
- data/lib/postburner/configuration.rb +41 -14
- data/lib/postburner/strategies/queue.rb +19 -9
- data/lib/postburner/version.rb +1 -1
- data/lib/postburner/workers/worker.rb +27 -28
- data/lib/postburner.rb +0 -1
- metadata +7 -2
|
@@ -1,28 +1,30 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Postburner
|
|
4
|
-
# Queue
|
|
4
|
+
# Queue properties module for Postburner::Job classes.
|
|
5
5
|
#
|
|
6
6
|
# Provides DSL methods for configuring queue behavior (name, priority, TTR, retries).
|
|
7
|
-
#
|
|
8
|
-
# with ActiveSupport::Callbacks.
|
|
7
|
+
# Defines configurable properties for job queue management.
|
|
9
8
|
#
|
|
10
9
|
# @example Basic usage
|
|
11
10
|
# class ProcessPayment < Postburner::Job
|
|
12
11
|
# queue 'critical'
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
12
|
+
# priority 0
|
|
13
|
+
# ttr 300
|
|
14
|
+
# max_retries 3
|
|
16
15
|
#
|
|
17
16
|
# def perform(args)
|
|
18
17
|
# # ...
|
|
19
18
|
# end
|
|
20
19
|
# end
|
|
21
20
|
#
|
|
22
|
-
module
|
|
21
|
+
module Properties
|
|
23
22
|
extend ActiveSupport::Concern
|
|
24
23
|
|
|
25
24
|
included do
|
|
25
|
+
# Instance-level queue configuration (overrides class-level defaults)
|
|
26
|
+
attr_writer :priority, :ttr
|
|
27
|
+
|
|
26
28
|
class_attribute :postburner_queue_name, default: 'default'
|
|
27
29
|
class_attribute :postburner_priority, default: nil
|
|
28
30
|
class_attribute :postburner_ttr, default: nil
|
|
@@ -32,6 +34,9 @@ module Postburner
|
|
|
32
34
|
|
|
33
35
|
class_methods do
|
|
34
36
|
# Sets or returns the queue name.
|
|
37
|
+
#
|
|
38
|
+
# The queue name expands 'postburner.<environment>.<queue_name>' for the
|
|
39
|
+
# tube name in Beanstalkd.
|
|
35
40
|
#
|
|
36
41
|
# @param name [String, Symbol, nil] Queue name to set, or nil to get current value
|
|
37
42
|
#
|
|
@@ -52,21 +57,21 @@ module Postburner
|
|
|
52
57
|
end
|
|
53
58
|
end
|
|
54
59
|
|
|
55
|
-
# Sets or returns the
|
|
60
|
+
# Sets or returns the priority.
|
|
56
61
|
#
|
|
57
|
-
# Lower numbers = higher priority in Beanstalkd.
|
|
62
|
+
# Lower numbers = higher priority in Beanstalkd (0 is highest).
|
|
58
63
|
#
|
|
59
64
|
# @param pri [Integer, nil] Priority to set (0-4294967295), or nil to get current value
|
|
60
65
|
#
|
|
61
66
|
# @return [Integer, nil] Current priority when getting, nil when setting
|
|
62
67
|
#
|
|
63
68
|
# @example Set priority
|
|
64
|
-
#
|
|
69
|
+
# priority 0 # Highest priority
|
|
65
70
|
#
|
|
66
71
|
# @example Get priority
|
|
67
|
-
# ProcessPayment.
|
|
72
|
+
# ProcessPayment.priority # => 0
|
|
68
73
|
#
|
|
69
|
-
def
|
|
74
|
+
def priority(pri = nil)
|
|
70
75
|
if pri
|
|
71
76
|
self.postburner_priority = pri
|
|
72
77
|
nil
|
|
@@ -75,24 +80,24 @@ module Postburner
|
|
|
75
80
|
end
|
|
76
81
|
end
|
|
77
82
|
|
|
78
|
-
# Sets or returns the
|
|
83
|
+
# Sets or returns the TTR (time-to-run).
|
|
79
84
|
#
|
|
80
85
|
# Number of seconds Beanstalkd will wait for job completion before
|
|
81
86
|
# making it available again.
|
|
82
87
|
#
|
|
83
|
-
# @param
|
|
88
|
+
# @param seconds [Integer, nil] Timeout in seconds, or nil to get current value
|
|
84
89
|
#
|
|
85
90
|
# @return [Integer, nil] Current TTR when getting, nil when setting
|
|
86
91
|
#
|
|
87
92
|
# @example Set TTR
|
|
88
|
-
#
|
|
93
|
+
# ttr 300 # 5 minutes
|
|
89
94
|
#
|
|
90
95
|
# @example Get TTR
|
|
91
|
-
# ProcessPayment.
|
|
96
|
+
# ProcessPayment.ttr # => 300
|
|
92
97
|
#
|
|
93
|
-
def
|
|
94
|
-
if
|
|
95
|
-
self.postburner_ttr =
|
|
98
|
+
def ttr(seconds = nil)
|
|
99
|
+
if seconds
|
|
100
|
+
self.postburner_ttr = seconds
|
|
96
101
|
nil
|
|
97
102
|
else
|
|
98
103
|
postburner_ttr
|
|
@@ -106,12 +111,12 @@ module Postburner
|
|
|
106
111
|
# @return [Integer, nil] Current max retries when getting, nil when setting
|
|
107
112
|
#
|
|
108
113
|
# @example Set max retries
|
|
109
|
-
#
|
|
114
|
+
# max_retries 3
|
|
110
115
|
#
|
|
111
116
|
# @example Get max retries
|
|
112
|
-
# ProcessPayment.
|
|
117
|
+
# ProcessPayment.max_retries # => 3
|
|
113
118
|
#
|
|
114
|
-
def
|
|
119
|
+
def max_retries(retries = nil)
|
|
115
120
|
if retries
|
|
116
121
|
self.postburner_max_retries = retries
|
|
117
122
|
nil
|
|
@@ -130,15 +135,15 @@ module Postburner
|
|
|
130
135
|
# @return [Integer, Proc, nil] Current delay when getting, nil when setting
|
|
131
136
|
#
|
|
132
137
|
# @example Set fixed retry delay
|
|
133
|
-
#
|
|
138
|
+
# retry_delay 10
|
|
134
139
|
#
|
|
135
140
|
# @example Set exponential backoff with proc
|
|
136
|
-
#
|
|
141
|
+
# retry_delay ->(retries) { 2 ** retries }
|
|
137
142
|
#
|
|
138
143
|
# @example Get retry delay
|
|
139
|
-
# ProcessPayment.
|
|
144
|
+
# ProcessPayment.retry_delay # => 10 or #<Proc>
|
|
140
145
|
#
|
|
141
|
-
def
|
|
146
|
+
def retry_delay(delay = nil)
|
|
142
147
|
if delay
|
|
143
148
|
self.postburner_retry_delay = delay
|
|
144
149
|
nil
|
|
@@ -147,5 +152,64 @@ module Postburner
|
|
|
147
152
|
end
|
|
148
153
|
end
|
|
149
154
|
end
|
|
155
|
+
|
|
156
|
+
# Returns the queue name for this job instance.
|
|
157
|
+
#
|
|
158
|
+
# Checks instance-level override first, then falls back to class-level configuration.
|
|
159
|
+
#
|
|
160
|
+
# @return [String] Queue name
|
|
161
|
+
#
|
|
162
|
+
# @example Class-level configuration
|
|
163
|
+
# class MyJob < Postburner::Job
|
|
164
|
+
# queue 'critical'
|
|
165
|
+
# end
|
|
166
|
+
# job = MyJob.create!(args: {})
|
|
167
|
+
# job.queue_name # => 'critical'
|
|
168
|
+
#
|
|
169
|
+
def queue_name
|
|
170
|
+
self.class.queue
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# Returns the priority for this job instance.
|
|
174
|
+
#
|
|
175
|
+
# Checks instance-level override first, then falls back to class-level configuration.
|
|
176
|
+
#
|
|
177
|
+
# @return [Integer, nil] Priority (lower = higher priority)
|
|
178
|
+
#
|
|
179
|
+
# @example Instance-level override
|
|
180
|
+
# job = MyJob.create!(args: {}, priority: 1500)
|
|
181
|
+
# job.priority # => 1500
|
|
182
|
+
#
|
|
183
|
+
def priority
|
|
184
|
+
@priority || self.class.priority
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Returns the TTR (time-to-run) for this job instance.
|
|
188
|
+
#
|
|
189
|
+
# Checks instance-level override first, then falls back to class-level configuration.
|
|
190
|
+
#
|
|
191
|
+
# @return [Integer, nil] TTR in seconds
|
|
192
|
+
#
|
|
193
|
+
# @example Instance-level override
|
|
194
|
+
# job = MyJob.create!(args: {}, ttr: 600)
|
|
195
|
+
# job.ttr # => 600
|
|
196
|
+
#
|
|
197
|
+
def ttr
|
|
198
|
+
@ttr || self.class.ttr
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# Returns the full tube name with environment prefix.
|
|
202
|
+
#
|
|
203
|
+
# Expands the queue name to include the environment prefix
|
|
204
|
+
# (e.g., 'critical' becomes 'postburner.development.critical').
|
|
205
|
+
#
|
|
206
|
+
# @return [String] Full tube name with environment prefix
|
|
207
|
+
#
|
|
208
|
+
# @example
|
|
209
|
+
# job.expanded_tube_name # => 'postburner.development.critical'
|
|
210
|
+
#
|
|
211
|
+
def expanded_tube_name
|
|
212
|
+
Postburner.configuration.expand_tube_name(queue_name)
|
|
213
|
+
end
|
|
150
214
|
end
|
|
151
215
|
end
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Postburner
|
|
4
|
+
# Concern providing statistics and metrics methods for Postburner jobs.
|
|
5
|
+
#
|
|
6
|
+
# Provides methods for calculating timing metrics, retrieving job statistics,
|
|
7
|
+
# and determining intended execution times for scheduled jobs.
|
|
8
|
+
#
|
|
9
|
+
# @example Get job statistics
|
|
10
|
+
# job.stats
|
|
11
|
+
# # => { id: 5, bkid: 1, queue: "default", ... }
|
|
12
|
+
#
|
|
13
|
+
# @example Check elapsed time
|
|
14
|
+
# job.elapsed_ms # => 1234.567
|
|
15
|
+
#
|
|
16
|
+
module Statistics
|
|
17
|
+
extend ActiveSupport::Concern
|
|
18
|
+
|
|
19
|
+
# Returns job statistics including Beanstalkd job state.
|
|
20
|
+
#
|
|
21
|
+
# Fetches current job state from Beanstalkd and returns combined statistics
|
|
22
|
+
# about the job's PostgreSQL record and its current Beanstalkd status.
|
|
23
|
+
#
|
|
24
|
+
# @return [Hash] Statistics hash with the following keys:
|
|
25
|
+
# - id: PostgreSQL job ID
|
|
26
|
+
# - bkid: Beanstalkd job ID
|
|
27
|
+
# - queue: Queue name configured for this job class (e.g., 'sleep-jobs')
|
|
28
|
+
# - tube: Derived tube name with environment prefix (e.g., 'postburner.development.sleep-jobs')
|
|
29
|
+
# - watched: Boolean indicating if tube is in configured watch list
|
|
30
|
+
# - beanstalk: Hash of Beanstalkd job statistics:
|
|
31
|
+
# - id: Beanstalkd job ID
|
|
32
|
+
# - tube: Tube name where job resides
|
|
33
|
+
# - state: Job state (ready, reserved, delayed, buried)
|
|
34
|
+
# - pri: Priority (0 = highest, 4294967295 = lowest)
|
|
35
|
+
# - age: Seconds since job was created
|
|
36
|
+
# - delay: Seconds remaining before job becomes ready (0 if ready now)
|
|
37
|
+
# - ttr: Time-to-run in seconds (time allowed for job processing)
|
|
38
|
+
# - time_left: Seconds remaining before job times out (0 if not reserved)
|
|
39
|
+
# - file: Binlog file number containing job
|
|
40
|
+
# - reserves: Number of times job has been reserved
|
|
41
|
+
# - timeouts: Number of times job has timed out during processing
|
|
42
|
+
# - releases: Number of times job has been released back to ready
|
|
43
|
+
# - buries: Number of times job has been buried
|
|
44
|
+
# - kicks: Number of times job has been kicked from buried/delayed
|
|
45
|
+
#
|
|
46
|
+
# @raise [Beaneater::NotFoundError] if job no longer exists in Beanstalkd
|
|
47
|
+
#
|
|
48
|
+
# @example
|
|
49
|
+
# job.stats
|
|
50
|
+
# # => {
|
|
51
|
+
# # id: 5,
|
|
52
|
+
# # bkid: 1,
|
|
53
|
+
# # queue: "sleep-jobs",
|
|
54
|
+
# # tube: "postburner.development.sleep-jobs",
|
|
55
|
+
# # watched: false,
|
|
56
|
+
# # beanstalk: {
|
|
57
|
+
# # id: 1,
|
|
58
|
+
# # tube: "postburner.development.sleep-jobs",
|
|
59
|
+
# # state: "ready",
|
|
60
|
+
# # pri: 50,
|
|
61
|
+
# # age: 1391,
|
|
62
|
+
# # delay: 0,
|
|
63
|
+
# # ttr: 120,
|
|
64
|
+
# # time_left: 0,
|
|
65
|
+
# # file: 0,
|
|
66
|
+
# # reserves: 0,
|
|
67
|
+
# # timeouts: 0,
|
|
68
|
+
# # releases: 0,
|
|
69
|
+
# # buries: 0,
|
|
70
|
+
# # kicks: 0
|
|
71
|
+
# # }
|
|
72
|
+
# # }
|
|
73
|
+
#
|
|
74
|
+
def stats
|
|
75
|
+
# Get configured watched tubes (expanded with environment prefix)
|
|
76
|
+
watched = Postburner.watched_tube_names.include?(self.expanded_tube_name)
|
|
77
|
+
|
|
78
|
+
{
|
|
79
|
+
id: self.id,
|
|
80
|
+
bkid: self.bkid,
|
|
81
|
+
queue: queue_name,
|
|
82
|
+
tube: expanded_tube_name,
|
|
83
|
+
watched: watched,
|
|
84
|
+
beanstalk: self.bk.stats.to_h.symbolize_keys,
|
|
85
|
+
}
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Returns elapsed time in milliseconds since job execution started.
|
|
89
|
+
#
|
|
90
|
+
# Calculates milliseconds between attempting_at and current time.
|
|
91
|
+
# Used to track how long a job has been executing.
|
|
92
|
+
#
|
|
93
|
+
# @return [Float, nil] Milliseconds since attempting_at, or nil if not yet attempting
|
|
94
|
+
#
|
|
95
|
+
# @example
|
|
96
|
+
# elapsed_ms # => 1234.567 (job has been running for ~1.2 seconds)
|
|
97
|
+
#
|
|
98
|
+
# @see #log
|
|
99
|
+
#
|
|
100
|
+
def elapsed_ms
|
|
101
|
+
return unless self.attempting_at
|
|
102
|
+
(Time.zone.now - self.attempting_at) * 1000
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Returns the intended execution time for the job.
|
|
106
|
+
#
|
|
107
|
+
# Returns run_at if scheduled, otherwise returns queued_at.
|
|
108
|
+
# Used for calculating lag (delay between intended and actual execution).
|
|
109
|
+
#
|
|
110
|
+
# @return [Time] Intended execution timestamp
|
|
111
|
+
#
|
|
112
|
+
# @example
|
|
113
|
+
# job.queue!(delay: 1.hour)
|
|
114
|
+
# job.intended_at # => 1 hour from now (run_at)
|
|
115
|
+
#
|
|
116
|
+
# job.queue!
|
|
117
|
+
# job.intended_at # => now (queued_at)
|
|
118
|
+
#
|
|
119
|
+
# @see #lag
|
|
120
|
+
#
|
|
121
|
+
def intended_at
|
|
122
|
+
self.run_at ? self.run_at : self.queued_at
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|