postburner 1.0.0.pre.11 → 1.0.0.pre.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +961 -555
- data/app/concerns/postburner/commands.rb +1 -1
- data/app/concerns/postburner/execution.rb +11 -11
- data/app/concerns/postburner/insertion.rb +1 -1
- data/app/concerns/postburner/logging.rb +2 -2
- data/app/concerns/postburner/statistics.rb +1 -1
- data/app/models/postburner/job.rb +27 -4
- data/app/models/postburner/mailer.rb +1 -1
- data/app/models/postburner/schedule.rb +703 -0
- data/app/models/postburner/schedule_execution.rb +353 -0
- data/app/views/postburner/jobs/show.html.haml +3 -3
- data/lib/generators/postburner/install/install_generator.rb +1 -0
- data/lib/generators/postburner/install/templates/config/postburner.yml +15 -6
- data/lib/generators/postburner/install/templates/migrations/create_postburner_schedules.rb.erb +71 -0
- data/lib/postburner/active_job/adapter.rb +3 -3
- data/lib/postburner/active_job/payload.rb +5 -0
- data/lib/postburner/advisory_lock.rb +123 -0
- data/lib/postburner/configuration.rb +43 -7
- data/lib/postburner/connection.rb +7 -6
- data/lib/postburner/runner.rb +26 -3
- data/lib/postburner/scheduler.rb +427 -0
- data/lib/postburner/strategies/immediate_test_queue.rb +24 -7
- data/lib/postburner/strategies/nice_queue.rb +1 -1
- data/lib/postburner/strategies/null_queue.rb +2 -2
- data/lib/postburner/strategies/test_queue.rb +2 -2
- data/lib/postburner/time_helpers.rb +4 -2
- data/lib/postburner/tube.rb +9 -1
- data/lib/postburner/version.rb +1 -1
- data/lib/postburner/worker.rb +684 -0
- data/lib/postburner.rb +32 -13
- metadata +7 -3
- data/lib/postburner/workers/base.rb +0 -205
- data/lib/postburner/workers/worker.rb +0 -396
|
@@ -22,6 +22,7 @@ module Postburner
|
|
|
22
22
|
class Configuration
|
|
23
23
|
# Global settings
|
|
24
24
|
attr_accessor :beanstalk_url, :logger, :default_queue, :default_priority, :default_ttr
|
|
25
|
+
attr_accessor :default_scheduler_interval, :default_scheduler_priority
|
|
25
26
|
|
|
26
27
|
# Worker-specific settings (loaded for a single worker)
|
|
27
28
|
attr_accessor :worker_config
|
|
@@ -32,6 +33,8 @@ module Postburner
|
|
|
32
33
|
# @option options [String] :default_queue Default queue name (default: 'default')
|
|
33
34
|
# @option options [Integer] :default_priority Default job priority (default: 65536, lower = higher priority)
|
|
34
35
|
# @option options [Integer] :default_ttr Default time-to-run in seconds (default: 300)
|
|
36
|
+
# @option options [Integer] :default_scheduler_interval Scheduler check interval in seconds (default: 300)
|
|
37
|
+
# @option options [Integer] :default_scheduler_priority Scheduler job priority (default: 100)
|
|
35
38
|
# @option options [Hash] :worker_config Worker configuration hash with keys:
|
|
36
39
|
# - :name [String] Worker name
|
|
37
40
|
# - :queues [Array<String>] Queue/tube names to process
|
|
@@ -39,6 +42,7 @@ module Postburner
|
|
|
39
42
|
# - :threads [Integer] Number of threads per fork
|
|
40
43
|
# - :gc_limit [Integer, nil] Jobs to process before restart (nil = unlimited)
|
|
41
44
|
# - :timeout [Integer] Reserve command timeout in seconds (1-10, default: 3)
|
|
45
|
+
# - :shutdown_timeout [Integer] Seconds to wait for graceful shutdown (default: default_ttr)
|
|
42
46
|
#
|
|
43
47
|
def initialize(options = {})
|
|
44
48
|
@beanstalk_url = options[:beanstalk_url] || ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300'
|
|
@@ -46,13 +50,16 @@ module Postburner
|
|
|
46
50
|
@default_queue = options[:default_queue] || 'default'
|
|
47
51
|
@default_priority = options[:default_priority] || 65536
|
|
48
52
|
@default_ttr = options[:default_ttr] || 300
|
|
53
|
+
@default_scheduler_interval = options[:default_scheduler_interval] || 300
|
|
54
|
+
@default_scheduler_priority = options[:default_scheduler_priority] || 100
|
|
49
55
|
@worker_config = options[:worker_config] || {
|
|
50
56
|
name: 'default',
|
|
51
57
|
queues: ['default'],
|
|
52
58
|
forks: 0,
|
|
53
59
|
threads: 1,
|
|
54
60
|
gc_limit: nil,
|
|
55
|
-
timeout: 3
|
|
61
|
+
timeout: 3,
|
|
62
|
+
shutdown_timeout: @default_ttr
|
|
56
63
|
}
|
|
57
64
|
end
|
|
58
65
|
|
|
@@ -90,13 +97,15 @@ module Postburner
|
|
|
90
97
|
# default_threads: 10
|
|
91
98
|
# default_gc_limit: 5000
|
|
92
99
|
# default_ttr: 300
|
|
100
|
+
# default_shutdown_timeout: 300 # Defaults to default_ttr if not set
|
|
93
101
|
# workers: # <- worker configs
|
|
94
102
|
# imports: # <- worker name
|
|
95
|
-
# timeout: 3
|
|
96
|
-
#
|
|
97
|
-
# forks: 4
|
|
98
|
-
# threads: 1
|
|
99
|
-
# gc_limit: 500
|
|
103
|
+
# timeout: 3 # Reserve timeout in seconds (1-10, default: 3)
|
|
104
|
+
# # Lower values enable faster graceful shutdowns
|
|
105
|
+
# forks: 4 # Overrides default_forks
|
|
106
|
+
# threads: 1 # Overrides default_threads
|
|
107
|
+
# gc_limit: 500 # Overrides default_gc_limit
|
|
108
|
+
# shutdown_timeout: 600 # Wait 10 min for long imports to finish
|
|
100
109
|
# queues:
|
|
101
110
|
# - imports
|
|
102
111
|
# - data_processing
|
|
@@ -135,13 +144,15 @@ module Postburner
|
|
|
135
144
|
worker_yaml = workers[worker_name]
|
|
136
145
|
|
|
137
146
|
# Build worker_config hash - worker-level overrides env-level defaults
|
|
147
|
+
default_ttr = env_config['default_ttr'] || 300
|
|
138
148
|
worker_config = {
|
|
139
149
|
name: worker_name,
|
|
140
150
|
queues: worker_yaml['queues'] || ['default'],
|
|
141
151
|
forks: worker_yaml['forks'] || env_config['default_forks'] || 0,
|
|
142
152
|
threads: worker_yaml['threads'] || env_config['default_threads'] || 1,
|
|
143
153
|
gc_limit: worker_yaml['gc_limit'] || env_config['default_gc_limit'],
|
|
144
|
-
timeout: worker_yaml['timeout'] || 3
|
|
154
|
+
timeout: worker_yaml['timeout'] || 3,
|
|
155
|
+
shutdown_timeout: worker_yaml['shutdown_timeout'] || env_config['default_shutdown_timeout'] || default_ttr
|
|
145
156
|
}
|
|
146
157
|
|
|
147
158
|
options = {
|
|
@@ -149,6 +160,8 @@ module Postburner
|
|
|
149
160
|
default_queue: env_config['default_queue'],
|
|
150
161
|
default_priority: env_config['default_priority'],
|
|
151
162
|
default_ttr: env_config['default_ttr'],
|
|
163
|
+
default_scheduler_interval: env_config['default_scheduler_interval'],
|
|
164
|
+
default_scheduler_priority: env_config['default_scheduler_priority'],
|
|
152
165
|
worker_config: worker_config
|
|
153
166
|
}
|
|
154
167
|
|
|
@@ -232,6 +245,20 @@ module Postburner
|
|
|
232
245
|
def expanded_tube_names(env = nil)
|
|
233
246
|
queue_names.map { |q| expand_tube_name(q, env) }
|
|
234
247
|
end
|
|
248
|
+
|
|
249
|
+
# Returns the scheduler tube name with environment prefix.
|
|
250
|
+
#
|
|
251
|
+
# @param env [String, nil] Environment name (defaults to Rails.env)
|
|
252
|
+
#
|
|
253
|
+
# @return [String] Expanded scheduler tube name
|
|
254
|
+
#
|
|
255
|
+
# @example
|
|
256
|
+
# config.scheduler_tube_name
|
|
257
|
+
# # => 'postburner.production.scheduler'
|
|
258
|
+
#
|
|
259
|
+
def scheduler_tube_name(env = nil)
|
|
260
|
+
expand_tube_name(Postburner::Scheduler::SCHEDULER_TUBE_NAME, env)
|
|
261
|
+
end
|
|
235
262
|
end
|
|
236
263
|
|
|
237
264
|
# Returns the global configuration instance.
|
|
@@ -242,6 +269,15 @@ module Postburner
|
|
|
242
269
|
@configuration ||= Configuration.new
|
|
243
270
|
end
|
|
244
271
|
|
|
272
|
+
# Sets the global configuration instance.
|
|
273
|
+
#
|
|
274
|
+
# @param config [Configuration] Configuration to use globally
|
|
275
|
+
# @return [Configuration]
|
|
276
|
+
#
|
|
277
|
+
def self.configuration=(config)
|
|
278
|
+
@configuration = config
|
|
279
|
+
end
|
|
280
|
+
|
|
245
281
|
# Configures Postburner via block.
|
|
246
282
|
#
|
|
247
283
|
# @yield [config] Configuration instance
|
|
@@ -5,6 +5,7 @@ module Postburner
|
|
|
5
5
|
#
|
|
6
6
|
# Provides a simplified interface to Beanstalkd via Beaneater, with
|
|
7
7
|
# automatic connection management and reconnection on failures.
|
|
8
|
+
# Each worker thread creates its own connection instance for thread safety.
|
|
8
9
|
#
|
|
9
10
|
# @example Direct usage
|
|
10
11
|
# conn = Postburner::Connection.new
|
|
@@ -95,8 +96,8 @@ module Postburner
|
|
|
95
96
|
# For user-facing output, use Postburner.clear_jobs! instead.
|
|
96
97
|
#
|
|
97
98
|
# SAFETY: Only allows clearing tubes that are defined in the loaded
|
|
98
|
-
# configuration (watched_tube_names). This prevents
|
|
99
|
-
# tubes from other applications or environments.
|
|
99
|
+
# configuration (watched_tube_names) or the scheduler tube. This prevents
|
|
100
|
+
# accidentally clearing tubes from other applications or environments.
|
|
100
101
|
#
|
|
101
102
|
# @param tube_names [Array<String>, nil] Array of tube names to clear, or nil to only collect stats
|
|
102
103
|
#
|
|
@@ -122,16 +123,16 @@ module Postburner
|
|
|
122
123
|
def clear_tubes!(tube_names = nil)
|
|
123
124
|
ensure_connected!
|
|
124
125
|
|
|
125
|
-
# Validate that tubes to clear are in the loaded configuration
|
|
126
|
+
# Validate that tubes to clear are in the loaded configuration (or scheduler tube)
|
|
126
127
|
if tube_names&.any?
|
|
127
|
-
|
|
128
|
-
invalid_tubes = tube_names -
|
|
128
|
+
allowed = Postburner.watched_tube_names + [Postburner.scheduler_tube_name]
|
|
129
|
+
invalid_tubes = tube_names - allowed
|
|
129
130
|
|
|
130
131
|
if invalid_tubes.any?
|
|
131
132
|
raise ArgumentError, <<~ERROR
|
|
132
133
|
Cannot clear tubes not in configuration.
|
|
133
134
|
Invalid tubes: #{invalid_tubes.join(', ')}
|
|
134
|
-
Configured tubes: #{
|
|
135
|
+
Configured tubes: #{allowed.join(', ')}
|
|
135
136
|
ERROR
|
|
136
137
|
end
|
|
137
138
|
end
|
data/lib/postburner/runner.rb
CHANGED
|
@@ -17,7 +17,12 @@ module Postburner
|
|
|
17
17
|
# runner = Postburner::Runner.new(options)
|
|
18
18
|
# runner.run
|
|
19
19
|
#
|
|
20
|
+
# @see Postburner::Worker
|
|
21
|
+
# @see Postburner::Configuration
|
|
22
|
+
#
|
|
20
23
|
class Runner
|
|
24
|
+
# @!attribute [r] options
|
|
25
|
+
# @return [Hash] The runner options
|
|
21
26
|
attr_reader :options
|
|
22
27
|
|
|
23
28
|
# Initialize runner with options.
|
|
@@ -50,10 +55,15 @@ module Postburner
|
|
|
50
55
|
|
|
51
56
|
# Load configuration from YAML file.
|
|
52
57
|
#
|
|
58
|
+
# Sets the global {Postburner.configuration} after loading.
|
|
59
|
+
#
|
|
53
60
|
# @return [Postburner::Configuration]
|
|
61
|
+
# @api private
|
|
54
62
|
def load_configuration
|
|
55
63
|
config_path = File.expand_path(options[:config], root_directory)
|
|
56
|
-
Postburner::Configuration.load_yaml(config_path, options[:env], options[:worker])
|
|
64
|
+
config = Postburner::Configuration.load_yaml(config_path, options[:env], options[:worker])
|
|
65
|
+
Postburner.configuration = config
|
|
66
|
+
config
|
|
57
67
|
rescue ArgumentError => e
|
|
58
68
|
logger.error "[Postburner] ERROR: #{e.message}"
|
|
59
69
|
exit 1
|
|
@@ -61,8 +71,12 @@ module Postburner
|
|
|
61
71
|
|
|
62
72
|
# Filter configuration to only include specified queues.
|
|
63
73
|
#
|
|
74
|
+
# Validates that all specified queues exist in config and exits
|
|
75
|
+
# with error if unknown queues are specified.
|
|
76
|
+
#
|
|
64
77
|
# @param config [Postburner::Configuration]
|
|
65
78
|
# @return [void]
|
|
79
|
+
# @api private
|
|
66
80
|
def filter_queues(config)
|
|
67
81
|
# Validate that all specified queues exist in config
|
|
68
82
|
invalid_queues = options[:queues] - config.queue_names
|
|
@@ -81,11 +95,15 @@ module Postburner
|
|
|
81
95
|
#
|
|
82
96
|
# @param config [Postburner::Configuration]
|
|
83
97
|
# @return [void]
|
|
98
|
+
# @api private
|
|
84
99
|
def log_configuration(config)
|
|
85
100
|
config_path = File.expand_path(options[:config], root_directory)
|
|
86
101
|
config.logger.info "[Postburner] Configuration: #{config_path}"
|
|
87
102
|
config.logger.info "[Postburner] Environment: #{options[:env]}"
|
|
88
|
-
|
|
103
|
+
if options[:worker] || options[:queues].nil?
|
|
104
|
+
worker_label = options[:worker] || "(auto-selected: #{config.worker_config[:name]})"
|
|
105
|
+
config.logger.info "[Postburner] Worker: #{worker_label}"
|
|
106
|
+
end
|
|
89
107
|
config.logger.info "[Postburner] Queues: #{config.queue_names.join(', ')}"
|
|
90
108
|
wc = config.worker_config
|
|
91
109
|
config.logger.info "[Postburner] Worker config: forks=#{wc[:forks]}, threads=#{wc[:threads]}, gc_limit=#{wc[:gc_limit] || 'none'}, timeout=#{wc[:timeout]}s"
|
|
@@ -96,6 +114,7 @@ module Postburner
|
|
|
96
114
|
# Uses Rails.root when Rails is defined, otherwise falls back to current directory.
|
|
97
115
|
#
|
|
98
116
|
# @return [String] Root directory path
|
|
117
|
+
# @api private
|
|
99
118
|
def root_directory
|
|
100
119
|
if defined?(Rails) && Rails.respond_to?(:root)
|
|
101
120
|
Rails.root.to_s
|
|
@@ -108,14 +127,18 @@ module Postburner
|
|
|
108
127
|
#
|
|
109
128
|
# @param config [Postburner::Configuration]
|
|
110
129
|
# @return [void]
|
|
130
|
+
# @api private
|
|
111
131
|
def start_worker(config)
|
|
112
|
-
worker = Postburner::
|
|
132
|
+
worker = Postburner::Worker.new(config)
|
|
113
133
|
worker.start
|
|
114
134
|
end
|
|
115
135
|
|
|
116
136
|
# Returns logger for error reporting during initialization.
|
|
117
137
|
#
|
|
138
|
+
# Falls back to stderr when Rails is not available.
|
|
139
|
+
#
|
|
118
140
|
# @return [Logger]
|
|
141
|
+
# @api private
|
|
119
142
|
def logger
|
|
120
143
|
if defined?(Rails)
|
|
121
144
|
Rails.logger
|
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'postburner/advisory_lock'
|
|
4
|
+
|
|
5
|
+
module Postburner
|
|
6
|
+
# Lightweight scheduler that acts as a safety net for recurring job execution.
|
|
7
|
+
#
|
|
8
|
+
# Unlike traditional schedulers that poll for due jobs, Postburner uses **immediate
|
|
9
|
+
# enqueue** where executions are created and immediately queued to Beanstalkd's
|
|
10
|
+
# delayed queue. The scheduler acts as a **watchdog safety net** to ensure every
|
|
11
|
+
# schedule has a future execution queued.
|
|
12
|
+
#
|
|
13
|
+
# ## Architecture
|
|
14
|
+
#
|
|
15
|
+
# This class is NOT persisted to the database. It's instantiated on-the-fly by
|
|
16
|
+
# workers when they reserve a watchdog job from the scheduler tube. The watchdog
|
|
17
|
+
# is ephemeral data in Beanstalkd with this payload:
|
|
18
|
+
#
|
|
19
|
+
# { "scheduler": true, "interval": 300 }
|
|
20
|
+
#
|
|
21
|
+
# No dedicated scheduler process is needed - existing workers handle everything.
|
|
22
|
+
#
|
|
23
|
+
# ## Watchdog Pattern
|
|
24
|
+
#
|
|
25
|
+
# 1. Workers automatically watch the 'scheduler' tube
|
|
26
|
+
# 2. On reserve timeout, workers check if watchdog exists and create if missing
|
|
27
|
+
# 3. When worker reserves watchdog, it instantiates Postburner::Scheduler
|
|
28
|
+
# 4. Scheduler executes with PostgreSQL advisory lock for coordination
|
|
29
|
+
# 5. After completion, watchdog re-queues itself with delay for next interval
|
|
30
|
+
#
|
|
31
|
+
# ## Safety Net Functions
|
|
32
|
+
#
|
|
33
|
+
# The watchdog performs three safety net functions:
|
|
34
|
+
#
|
|
35
|
+
# 1. **Auto-bootstrap**: Creates first execution for schedules that haven't been started
|
|
36
|
+
# 2. **Future execution guarantee**: Ensures each schedule has a future execution queued
|
|
37
|
+
# 3. **Orphan cleanup**: Enqueues any pending executions that weren't properly queued
|
|
38
|
+
#
|
|
39
|
+
# Note: For Postburner::Job schedules, a before_attempt callback creates the next
|
|
40
|
+
# execution when the current job runs, providing immediate pickup without waiting
|
|
41
|
+
# for the watchdog. The watchdog is just the safety net.
|
|
42
|
+
#
|
|
43
|
+
# ## Configuration
|
|
44
|
+
#
|
|
45
|
+
# Add to config/postburner.yml:
|
|
46
|
+
#
|
|
47
|
+
# production:
|
|
48
|
+
# default_scheduler_interval: 300 # Check every 5 minutes (default)
|
|
49
|
+
# default_scheduler_priority: 100 # Watchdog priority (default)
|
|
50
|
+
#
|
|
51
|
+
# The interval primarily affects:
|
|
52
|
+
# - How quickly new schedules are auto-bootstrapped (if you don't call start!)
|
|
53
|
+
# - Recovery time if an execution somehow fails to enqueue
|
|
54
|
+
# - How often last_audit_at is updated for monitoring
|
|
55
|
+
#
|
|
56
|
+
# Since executions are enqueued immediately to Beanstalkd's delayed queue, the
|
|
57
|
+
# watchdog interval doesn't affect execution timing - only the safety net checks.
|
|
58
|
+
#
|
|
59
|
+
# ## Execution Flow
|
|
60
|
+
#
|
|
61
|
+
# 1. Worker reserves watchdog from scheduler tube
|
|
62
|
+
# 2. Instantiates Scheduler with interval from payload
|
|
63
|
+
# 3. Scheduler#perform acquires PostgreSQL advisory lock
|
|
64
|
+
# 4. Processes all enabled schedules:
|
|
65
|
+
# - Auto-bootstraps unstarted schedules (creates + enqueues first execution)
|
|
66
|
+
# - Ensures future execution exists (creates + enqueues if missing)
|
|
67
|
+
# - Enqueues any orphaned pending executions
|
|
68
|
+
# - Updates last_audit_at timestamp
|
|
69
|
+
# 5. Re-queues watchdog with delay for next interval
|
|
70
|
+
#
|
|
71
|
+
# ## Observability
|
|
72
|
+
#
|
|
73
|
+
# Monitor scheduler health via last_audit_at:
|
|
74
|
+
#
|
|
75
|
+
# stale = Postburner::Schedule.enabled
|
|
76
|
+
# .where('last_audit_at < ?', 15.minutes.ago)
|
|
77
|
+
# .or(Postburner::Schedule.where(last_audit_at: nil))
|
|
78
|
+
#
|
|
79
|
+
# @example Watchdog payload in Beanstalkd
|
|
80
|
+
# {
|
|
81
|
+
# "scheduler": true,
|
|
82
|
+
# "interval": 300
|
|
83
|
+
# }
|
|
84
|
+
#
|
|
85
|
+
# @example Manual watchdog creation (usually automatic)
|
|
86
|
+
# Postburner::Scheduler.enqueue_watchdog(interval: 300, priority: 100)
|
|
87
|
+
#
|
|
88
|
+
# @see Postburner::Schedule
|
|
89
|
+
# @see Postburner::ScheduleExecution
|
|
90
|
+
#
|
|
91
|
+
class Scheduler
|
|
92
|
+
SCHEDULER_TUBE_NAME = 'scheduler'
|
|
93
|
+
DEFAULT_INTERVAL = 300 # 5 minutes
|
|
94
|
+
DEFAULT_PRIORITY = 100 # Lower number = higher priority
|
|
95
|
+
|
|
96
|
+
attr_reader :interval, :logger
|
|
97
|
+
|
|
98
|
+
# Initialize a new scheduler instance.
|
|
99
|
+
#
|
|
100
|
+
# The scheduler is instantiated by workers when they reserve a watchdog job
|
|
101
|
+
# from the scheduler tube. It's not persisted - each run creates a new instance.
|
|
102
|
+
#
|
|
103
|
+
# @param interval [Integer] Seconds until next run (default: 300)
|
|
104
|
+
# @param logger [Logger, nil] Logger instance (default: Rails.logger or stdout)
|
|
105
|
+
#
|
|
106
|
+
# @example
|
|
107
|
+
# scheduler = Postburner::Scheduler.new(interval: 300)
|
|
108
|
+
# scheduler.perform
|
|
109
|
+
#
|
|
110
|
+
def initialize(interval: DEFAULT_INTERVAL, logger: nil)
|
|
111
|
+
@interval = interval
|
|
112
|
+
@logger = logger || (defined?(Rails) ? Rails.logger : Logger.new($stdout))
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Execute the scheduler.
|
|
116
|
+
#
|
|
117
|
+
# Processes all enabled schedules with PostgreSQL advisory lock coordination
|
|
118
|
+
# to prevent concurrent execution across multiple workers. After processing,
|
|
119
|
+
# automatically re-queues the watchdog for the next run.
|
|
120
|
+
#
|
|
121
|
+
# The scheduler performs two main functions:
|
|
122
|
+
# 1. Auto-bootstrap new schedules (create first execution if not started)
|
|
123
|
+
# 2. Ensure each schedule has a future execution queued
|
|
124
|
+
# 3. Enqueue any orphaned pending executions that weren't properly queued
|
|
125
|
+
#
|
|
126
|
+
# @return [void]
|
|
127
|
+
#
|
|
128
|
+
# @example Called by worker
|
|
129
|
+
# # Worker reserves watchdog job with payload:
|
|
130
|
+
# # { "scheduler" => true, "interval" => 300 }
|
|
131
|
+
# scheduler = Postburner::Scheduler.new(interval: 300)
|
|
132
|
+
# scheduler.perform
|
|
133
|
+
#
|
|
134
|
+
def perform
|
|
135
|
+
logger.info "[Postburner::Scheduler] Starting scheduler run"
|
|
136
|
+
|
|
137
|
+
# Use advisory lock to coordinate multiple workers
|
|
138
|
+
acquired = Postburner::AdvisoryLock.with_lock(AdvisoryLock::SCHEDULER_LOCK_KEY, blocking: false) do
|
|
139
|
+
process_all_schedules
|
|
140
|
+
true
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
if acquired
|
|
144
|
+
logger.info "[Postburner::Scheduler] Scheduler run complete"
|
|
145
|
+
else
|
|
146
|
+
logger.info "[Postburner::Scheduler] Could not acquire lock, skipping"
|
|
147
|
+
end
|
|
148
|
+
ensure
|
|
149
|
+
# Always re-queue watchdog for next run
|
|
150
|
+
requeue_watchdog
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Class method to enqueue watchdog to Beanstalkd
|
|
154
|
+
# Mutex for coordinating watchdog checks across threads
|
|
155
|
+
WATCHDOG_MUTEX = Mutex.new
|
|
156
|
+
@watchdog_last_checked_at = nil
|
|
157
|
+
@watchdog_check_failed_at = nil
|
|
158
|
+
|
|
159
|
+
class << self
|
|
160
|
+
attr_accessor :watchdog_last_checked_at, :watchdog_check_failed_at
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Ensure a scheduler watchdog exists in the queue.
|
|
164
|
+
#
|
|
165
|
+
# Uses process-level mutex coordination so only one thread checks at a time.
|
|
166
|
+
# Called by workers on reserve timeout to automatically recreate watchdog
|
|
167
|
+
# if it's missing (e.g., after Beanstalkd restart or watchdog expiration).
|
|
168
|
+
#
|
|
169
|
+
# Implements throttling:
|
|
170
|
+
# - Skips check if successful check within last 60 seconds
|
|
171
|
+
# - Skips check if failed check within last 60 seconds
|
|
172
|
+
# - Only one thread can check at a time (mutex)
|
|
173
|
+
#
|
|
174
|
+
# @param connection [Postburner::Connection] Existing beanstalkd connection
|
|
175
|
+
# @return [void]
|
|
176
|
+
#
|
|
177
|
+
# @example Called by worker on timeout
|
|
178
|
+
# Postburner.connected do |conn|
|
|
179
|
+
# Postburner::Scheduler.ensure_watchdog!(connection: conn)
|
|
180
|
+
# end
|
|
181
|
+
#
|
|
182
|
+
def self.ensure_watchdog!(connection:)
|
|
183
|
+
# Quick check without lock - skip if recently checked
|
|
184
|
+
return if watchdog_last_checked_at && Time.current - watchdog_last_checked_at < 60
|
|
185
|
+
return if watchdog_check_failed_at && Time.current - watchdog_check_failed_at < 60
|
|
186
|
+
|
|
187
|
+
# Try to acquire lock, skip if another thread is checking
|
|
188
|
+
return unless WATCHDOG_MUTEX.try_lock
|
|
189
|
+
|
|
190
|
+
begin
|
|
191
|
+
# Double-check after acquiring lock
|
|
192
|
+
return if watchdog_last_checked_at && Time.current - watchdog_last_checked_at < 60
|
|
193
|
+
|
|
194
|
+
if watchdog_exists?(connection: connection)
|
|
195
|
+
self.watchdog_last_checked_at = Time.current
|
|
196
|
+
return
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
enqueue_watchdog
|
|
200
|
+
self.watchdog_last_checked_at = Time.current
|
|
201
|
+
self.watchdog_check_failed_at = nil
|
|
202
|
+
rescue => e
|
|
203
|
+
self.watchdog_check_failed_at = Time.current
|
|
204
|
+
Rails.logger.error "[Postburner::Scheduler] Watchdog check failed, will retry in 60s: #{e.message}"
|
|
205
|
+
ensure
|
|
206
|
+
WATCHDOG_MUTEX.unlock
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Enqueue a new scheduler watchdog job.
|
|
211
|
+
#
|
|
212
|
+
# Creates a watchdog job in the scheduler tube with a delay equal to the interval.
|
|
213
|
+
# The watchdog will execute after the delay, process all schedules, and re-queue itself.
|
|
214
|
+
#
|
|
215
|
+
# Reads interval and priority from configuration if not provided:
|
|
216
|
+
# - default_scheduler_interval (default: 300 seconds)
|
|
217
|
+
# - default_scheduler_priority (default: 100)
|
|
218
|
+
#
|
|
219
|
+
# @param interval [Integer, nil] Seconds until next run (default: from config)
|
|
220
|
+
# @param priority [Integer, nil] Beanstalkd priority (default: from config)
|
|
221
|
+
# @return [Hash] Beanstalkd response with :status and :id keys
|
|
222
|
+
#
|
|
223
|
+
# @example Enqueue with defaults
|
|
224
|
+
# Postburner::Scheduler.enqueue_watchdog
|
|
225
|
+
# # => { status: "INSERTED", id: 12345 }
|
|
226
|
+
#
|
|
227
|
+
# @example Enqueue with custom interval
|
|
228
|
+
# Postburner::Scheduler.enqueue_watchdog(interval: 60, priority: 0)
|
|
229
|
+
#
|
|
230
|
+
def self.enqueue_watchdog(interval: nil, priority: nil)
|
|
231
|
+
interval ||= Postburner.configuration.default_scheduler_interval || DEFAULT_INTERVAL
|
|
232
|
+
priority ||= Postburner.configuration.default_scheduler_priority || DEFAULT_PRIORITY
|
|
233
|
+
|
|
234
|
+
payload = {
|
|
235
|
+
'scheduler' => true,
|
|
236
|
+
'interval' => interval
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
tube_name = Postburner.scheduler_tube_name
|
|
240
|
+
response = nil
|
|
241
|
+
|
|
242
|
+
Postburner.connected do |conn|
|
|
243
|
+
response = conn.tubes[tube_name].put(
|
|
244
|
+
JSON.generate(payload),
|
|
245
|
+
pri: priority,
|
|
246
|
+
delay: interval,
|
|
247
|
+
ttr: 120 # 2 minutes to complete
|
|
248
|
+
)
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
if response[:status] == "INSERTED"
|
|
252
|
+
runs_at = Time.current + interval
|
|
253
|
+
Rails.logger.info "[Postburner::Scheduler] Inserted watchdog: #{response[:id]} (#{Time.current.iso8601}, delay: #{interval}s (#{runs_at.iso8601}), tube: #{tube_name})"
|
|
254
|
+
else
|
|
255
|
+
Rails.logger.error "[Postburner::Scheduler] Failed to insert watchdog: #{response.inspect} (delay: #{interval}s, tube: #{tube_name})"
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
response
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# Check if scheduler watchdog exists in queue.
|
|
262
|
+
#
|
|
263
|
+
# Peeks both the delayed and ready queues in the scheduler tube.
|
|
264
|
+
# Returns true if watchdog exists in either state.
|
|
265
|
+
#
|
|
266
|
+
# @param connection [Postburner::Connection] Existing beanstalkd connection
|
|
267
|
+
# @return [Boolean] true if watchdog exists (delayed or ready), false otherwise
|
|
268
|
+
#
|
|
269
|
+
# @example Check for watchdog
|
|
270
|
+
# Postburner.connected do |conn|
|
|
271
|
+
# exists = Postburner::Scheduler.watchdog_exists?(connection: conn)
|
|
272
|
+
# puts "Watchdog present" if exists
|
|
273
|
+
# end
|
|
274
|
+
#
|
|
275
|
+
def self.watchdog_exists?(connection:)
|
|
276
|
+
tube_name = Postburner.scheduler_tube_name
|
|
277
|
+
tube = connection.beanstalk.tubes[tube_name]
|
|
278
|
+
|
|
279
|
+
# Peek is lighter than stats
|
|
280
|
+
delayed = tube.peek(:delayed) rescue nil
|
|
281
|
+
ready = tube.peek(:ready) rescue nil
|
|
282
|
+
|
|
283
|
+
delayed.present? || ready.present?
|
|
284
|
+
rescue Beaneater::NotFoundError
|
|
285
|
+
false
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
private
|
|
289
|
+
|
|
290
|
+
# Process all enabled schedules.
|
|
291
|
+
#
|
|
292
|
+
# Iterates through all enabled schedules and calls process_schedule for each.
|
|
293
|
+
# Logs count of successful and failed schedule processing.
|
|
294
|
+
#
|
|
295
|
+
# @return [void]
|
|
296
|
+
#
|
|
297
|
+
# @api private
|
|
298
|
+
#
|
|
299
|
+
def process_all_schedules
|
|
300
|
+
processed_count = 0
|
|
301
|
+
failed_count = 0
|
|
302
|
+
|
|
303
|
+
Postburner::Schedule.enabled.find_each do |schedule|
|
|
304
|
+
begin
|
|
305
|
+
process_schedule(schedule)
|
|
306
|
+
processed_count += 1
|
|
307
|
+
rescue => e
|
|
308
|
+
logger.error "[Postburner::Scheduler] Failed to process schedule '#{schedule.name}': #{e.class} - #{e.message}"
|
|
309
|
+
logger.error e.backtrace.join("\n")
|
|
310
|
+
failed_count += 1
|
|
311
|
+
end
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
logger.info "[Postburner::Scheduler] Processed #{processed_count} schedules, #{failed_count} failed"
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# Process a single schedule.
|
|
318
|
+
#
|
|
319
|
+
# The watchdog performs three safety net functions:
|
|
320
|
+
# 1. Auto-bootstrap: Create first execution if schedule hasn't been started
|
|
321
|
+
# 2. Ensure there's always a future execution (creates + enqueues if missing)
|
|
322
|
+
# 3. Enqueue any orphaned pending executions (that somehow weren't enqueued)
|
|
323
|
+
#
|
|
324
|
+
# Note: We check for future executions FIRST, then clean up any orphans.
|
|
325
|
+
# This ensures newly created executions are available for processing in the
|
|
326
|
+
# same run if they happen to be due.
|
|
327
|
+
#
|
|
328
|
+
# @param schedule [Postburner::Schedule] The schedule to process
|
|
329
|
+
# @return [void]
|
|
330
|
+
#
|
|
331
|
+
# @api private
|
|
332
|
+
#
|
|
333
|
+
def process_schedule(schedule)
|
|
334
|
+
# Auto-bootstrap: create first execution if schedule hasn't been started
|
|
335
|
+
# This will create the execution AND enqueue it to Beanstalkd
|
|
336
|
+
unless schedule.started?
|
|
337
|
+
logger.info "[Postburner::Scheduler] Bootstrapping schedule '#{schedule.name}'"
|
|
338
|
+
schedule.start!
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# Safety net 1: Ensure schedule has a future execution
|
|
342
|
+
# If missing, this will create AND enqueue it to Beanstalkd's delayed queue
|
|
343
|
+
ensure_future_execution!(schedule)
|
|
344
|
+
|
|
345
|
+
# Safety net 2: Find any orphaned pending executions and enqueue them
|
|
346
|
+
# This should rarely happen - only if enqueue! previously failed
|
|
347
|
+
execution_count = 0
|
|
348
|
+
schedule.executions.due.find_each do |execution|
|
|
349
|
+
begin
|
|
350
|
+
logger.warn "[Postburner::Scheduler] Found orphaned pending execution #{execution.id} for schedule '#{schedule.name}', enqueuing"
|
|
351
|
+
execution.enqueue!
|
|
352
|
+
execution_count += 1
|
|
353
|
+
rescue => e
|
|
354
|
+
logger.error "[Postburner::Scheduler] Failed to enqueue execution #{execution.id}: #{e.class} - #{e.message}"
|
|
355
|
+
raise
|
|
356
|
+
end
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
# Update last_audit_at
|
|
360
|
+
schedule.update_column(:last_audit_at, Time.current)
|
|
361
|
+
|
|
362
|
+
logger.debug "[Postburner::Scheduler] Schedule '#{schedule.name}': enqueued #{execution_count} orphaned executions" if execution_count > 0
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# Ensure schedule has a future scheduled or pending execution.
|
|
366
|
+
#
|
|
367
|
+
# Uses Schedule#create_next_execution! which is idempotent and only
|
|
368
|
+
# creates an execution if none exists in the future. If no executions
|
|
369
|
+
# exist at all, bootstraps the schedule first.
|
|
370
|
+
#
|
|
371
|
+
# This is the key safety net that ensures every schedule always has
|
|
372
|
+
# at least one future execution ready to run.
|
|
373
|
+
#
|
|
374
|
+
# @param schedule [Postburner::Schedule] The schedule to ensure future execution for
|
|
375
|
+
# @return [void]
|
|
376
|
+
#
|
|
377
|
+
# @api private
|
|
378
|
+
#
|
|
379
|
+
def ensure_future_execution!(schedule)
|
|
380
|
+
# Find the most recent execution to calculate next from
|
|
381
|
+
last_execution = schedule.executions.order(run_at: :desc).first
|
|
382
|
+
|
|
383
|
+
unless last_execution
|
|
384
|
+
# No executions at all - this shouldn't happen since we bootstrap first,
|
|
385
|
+
# but handle it as a safety net
|
|
386
|
+
logger.warn "[Postburner::Scheduler] Schedule '#{schedule.name}' has no executions, bootstrapping"
|
|
387
|
+
schedule.start!
|
|
388
|
+
return
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
# Delegate to Schedule - it knows whether a future execution is needed
|
|
392
|
+
execution = schedule.create_next_execution!(after: last_execution)
|
|
393
|
+
|
|
394
|
+
if execution
|
|
395
|
+
logger.info "[Postburner::Scheduler] Created future execution for '#{schedule.name}'"
|
|
396
|
+
end
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
# Re-queue the watchdog for the next run (if not already queued).
|
|
400
|
+
#
|
|
401
|
+
# Called in the ensure block of perform to guarantee the watchdog
|
|
402
|
+
# perpetuates itself. Always reads interval from config to pick up
|
|
403
|
+
# any runtime configuration changes.
|
|
404
|
+
#
|
|
405
|
+
# If re-queueing fails, logs error but doesn't raise - workers will
|
|
406
|
+
# recreate the watchdog on next timeout via ensure_watchdog!
|
|
407
|
+
#
|
|
408
|
+
# @return [void]
|
|
409
|
+
#
|
|
410
|
+
# @api private
|
|
411
|
+
#
|
|
412
|
+
def requeue_watchdog
|
|
413
|
+
Postburner.connected do |conn|
|
|
414
|
+
if self.class.watchdog_exists?(connection: conn)
|
|
415
|
+
logger.debug "[Postburner::Scheduler] Watchdog already exists, skipping re-queue"
|
|
416
|
+
return
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
self.class.enqueue_watchdog
|
|
420
|
+
end
|
|
421
|
+
rescue => e
|
|
422
|
+
logger.error "[Postburner::Scheduler] Failed to re-queue watchdog: #{e.class} - #{e.message}"
|
|
423
|
+
# This is critical - if watchdog isn't re-queued, scheduling stops
|
|
424
|
+
# Workers will recreate it on next timeout, but log loudly
|
|
425
|
+
end
|
|
426
|
+
end
|
|
427
|
+
end
|