postburner 1.0.0.pre.11 → 1.0.0.pre.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +961 -555
  3. data/app/concerns/postburner/commands.rb +1 -1
  4. data/app/concerns/postburner/execution.rb +11 -11
  5. data/app/concerns/postburner/insertion.rb +1 -1
  6. data/app/concerns/postburner/logging.rb +2 -2
  7. data/app/concerns/postburner/statistics.rb +1 -1
  8. data/app/models/postburner/job.rb +27 -4
  9. data/app/models/postburner/mailer.rb +1 -1
  10. data/app/models/postburner/schedule.rb +703 -0
  11. data/app/models/postburner/schedule_execution.rb +353 -0
  12. data/app/views/postburner/jobs/show.html.haml +3 -3
  13. data/lib/generators/postburner/install/install_generator.rb +1 -0
  14. data/lib/generators/postburner/install/templates/config/postburner.yml +15 -6
  15. data/lib/generators/postburner/install/templates/migrations/create_postburner_schedules.rb.erb +71 -0
  16. data/lib/postburner/active_job/adapter.rb +3 -3
  17. data/lib/postburner/active_job/payload.rb +5 -0
  18. data/lib/postburner/advisory_lock.rb +123 -0
  19. data/lib/postburner/configuration.rb +43 -7
  20. data/lib/postburner/connection.rb +7 -6
  21. data/lib/postburner/runner.rb +26 -3
  22. data/lib/postburner/scheduler.rb +427 -0
  23. data/lib/postburner/strategies/immediate_test_queue.rb +24 -7
  24. data/lib/postburner/strategies/nice_queue.rb +1 -1
  25. data/lib/postburner/strategies/null_queue.rb +2 -2
  26. data/lib/postburner/strategies/test_queue.rb +2 -2
  27. data/lib/postburner/time_helpers.rb +4 -2
  28. data/lib/postburner/tube.rb +9 -1
  29. data/lib/postburner/version.rb +1 -1
  30. data/lib/postburner/worker.rb +684 -0
  31. data/lib/postburner.rb +32 -13
  32. metadata +7 -3
  33. data/lib/postburner/workers/base.rb +0 -205
  34. data/lib/postburner/workers/worker.rb +0 -396
@@ -22,6 +22,7 @@ module Postburner
22
22
  class Configuration
23
23
  # Global settings
24
24
  attr_accessor :beanstalk_url, :logger, :default_queue, :default_priority, :default_ttr
25
+ attr_accessor :default_scheduler_interval, :default_scheduler_priority
25
26
 
26
27
  # Worker-specific settings (loaded for a single worker)
27
28
  attr_accessor :worker_config
@@ -32,6 +33,8 @@ module Postburner
32
33
  # @option options [String] :default_queue Default queue name (default: 'default')
33
34
  # @option options [Integer] :default_priority Default job priority (default: 65536, lower = higher priority)
34
35
  # @option options [Integer] :default_ttr Default time-to-run in seconds (default: 300)
36
+ # @option options [Integer] :default_scheduler_interval Scheduler check interval in seconds (default: 300)
37
+ # @option options [Integer] :default_scheduler_priority Scheduler job priority (default: 100)
35
38
  # @option options [Hash] :worker_config Worker configuration hash with keys:
36
39
  # - :name [String] Worker name
37
40
  # - :queues [Array<String>] Queue/tube names to process
@@ -39,6 +42,7 @@ module Postburner
39
42
  # - :threads [Integer] Number of threads per fork
40
43
  # - :gc_limit [Integer, nil] Jobs to process before restart (nil = unlimited)
41
44
  # - :timeout [Integer] Reserve command timeout in seconds (1-10, default: 3)
45
+ # - :shutdown_timeout [Integer] Seconds to wait for graceful shutdown (default: default_ttr)
42
46
  #
43
47
  def initialize(options = {})
44
48
  @beanstalk_url = options[:beanstalk_url] || ENV['BEANSTALK_URL'] || 'beanstalk://localhost:11300'
@@ -46,13 +50,16 @@ module Postburner
46
50
  @default_queue = options[:default_queue] || 'default'
47
51
  @default_priority = options[:default_priority] || 65536
48
52
  @default_ttr = options[:default_ttr] || 300
53
+ @default_scheduler_interval = options[:default_scheduler_interval] || 300
54
+ @default_scheduler_priority = options[:default_scheduler_priority] || 100
49
55
  @worker_config = options[:worker_config] || {
50
56
  name: 'default',
51
57
  queues: ['default'],
52
58
  forks: 0,
53
59
  threads: 1,
54
60
  gc_limit: nil,
55
- timeout: 3
61
+ timeout: 3,
62
+ shutdown_timeout: @default_ttr
56
63
  }
57
64
  end
58
65
 
@@ -90,13 +97,15 @@ module Postburner
90
97
  # default_threads: 10
91
98
  # default_gc_limit: 5000
92
99
  # default_ttr: 300
100
+ # default_shutdown_timeout: 300 # Defaults to default_ttr if not set
93
101
  # workers: # <- worker configs
94
102
  # imports: # <- worker name
95
- # timeout: 3 # Reserve timeout in seconds (1-10, default: 3)
96
- # # Lower values enable faster graceful shutdowns
97
- # forks: 4 # Overrides default_forks
98
- # threads: 1 # Overrides default_threads
99
- # gc_limit: 500 # Overrides default_gc_limit
103
+ # timeout: 3 # Reserve timeout in seconds (1-10, default: 3)
104
+ # # Lower values enable faster graceful shutdowns
105
+ # forks: 4 # Overrides default_forks
106
+ # threads: 1 # Overrides default_threads
107
+ # gc_limit: 500 # Overrides default_gc_limit
108
+ # shutdown_timeout: 600 # Wait 10 min for long imports to finish
100
109
  # queues:
101
110
  # - imports
102
111
  # - data_processing
@@ -135,13 +144,15 @@ module Postburner
135
144
  worker_yaml = workers[worker_name]
136
145
 
137
146
  # Build worker_config hash - worker-level overrides env-level defaults
147
+ default_ttr = env_config['default_ttr'] || 300
138
148
  worker_config = {
139
149
  name: worker_name,
140
150
  queues: worker_yaml['queues'] || ['default'],
141
151
  forks: worker_yaml['forks'] || env_config['default_forks'] || 0,
142
152
  threads: worker_yaml['threads'] || env_config['default_threads'] || 1,
143
153
  gc_limit: worker_yaml['gc_limit'] || env_config['default_gc_limit'],
144
- timeout: worker_yaml['timeout'] || 3
154
+ timeout: worker_yaml['timeout'] || 3,
155
+ shutdown_timeout: worker_yaml['shutdown_timeout'] || env_config['default_shutdown_timeout'] || default_ttr
145
156
  }
146
157
 
147
158
  options = {
@@ -149,6 +160,8 @@ module Postburner
149
160
  default_queue: env_config['default_queue'],
150
161
  default_priority: env_config['default_priority'],
151
162
  default_ttr: env_config['default_ttr'],
163
+ default_scheduler_interval: env_config['default_scheduler_interval'],
164
+ default_scheduler_priority: env_config['default_scheduler_priority'],
152
165
  worker_config: worker_config
153
166
  }
154
167
 
@@ -232,6 +245,20 @@ module Postburner
232
245
  def expanded_tube_names(env = nil)
233
246
  queue_names.map { |q| expand_tube_name(q, env) }
234
247
  end
248
+
249
+ # Returns the scheduler tube name with environment prefix.
250
+ #
251
+ # @param env [String, nil] Environment name (defaults to Rails.env)
252
+ #
253
+ # @return [String] Expanded scheduler tube name
254
+ #
255
+ # @example
256
+ # config.scheduler_tube_name
257
+ # # => 'postburner.production.scheduler'
258
+ #
259
+ def scheduler_tube_name(env = nil)
260
+ expand_tube_name(Postburner::Scheduler::SCHEDULER_TUBE_NAME, env)
261
+ end
235
262
  end
236
263
 
237
264
  # Returns the global configuration instance.
@@ -242,6 +269,15 @@ module Postburner
242
269
  @configuration ||= Configuration.new
243
270
  end
244
271
 
272
+ # Sets the global configuration instance.
273
+ #
274
+ # @param config [Configuration] Configuration to use globally
275
+ # @return [Configuration]
276
+ #
277
+ def self.configuration=(config)
278
+ @configuration = config
279
+ end
280
+
245
281
  # Configures Postburner via block.
246
282
  #
247
283
  # @yield [config] Configuration instance
@@ -5,6 +5,7 @@ module Postburner
5
5
  #
6
6
  # Provides a simplified interface to Beanstalkd via Beaneater, with
7
7
  # automatic connection management and reconnection on failures.
8
+ # Each worker thread creates its own connection instance for thread safety.
8
9
  #
9
10
  # @example Direct usage
10
11
  # conn = Postburner::Connection.new
@@ -95,8 +96,8 @@ module Postburner
95
96
  # For user-facing output, use Postburner.clear_jobs! instead.
96
97
  #
97
98
  # SAFETY: Only allows clearing tubes that are defined in the loaded
98
- # configuration (watched_tube_names). This prevents accidentally clearing
99
- # tubes from other applications or environments.
99
+ # configuration (watched_tube_names) or the scheduler tube. This prevents
100
+ # accidentally clearing tubes from other applications or environments.
100
101
  #
101
102
  # @param tube_names [Array<String>, nil] Array of tube names to clear, or nil to only collect stats
102
103
  #
@@ -122,16 +123,16 @@ module Postburner
122
123
  def clear_tubes!(tube_names = nil)
123
124
  ensure_connected!
124
125
 
125
- # Validate that tubes to clear are in the loaded configuration
126
+ # Validate that tubes to clear are in the loaded configuration (or scheduler tube)
126
127
  if tube_names&.any?
127
- watched = Postburner.watched_tube_names
128
- invalid_tubes = tube_names - watched
128
+ allowed = Postburner.watched_tube_names + [Postburner.scheduler_tube_name]
129
+ invalid_tubes = tube_names - allowed
129
130
 
130
131
  if invalid_tubes.any?
131
132
  raise ArgumentError, <<~ERROR
132
133
  Cannot clear tubes not in configuration.
133
134
  Invalid tubes: #{invalid_tubes.join(', ')}
134
- Configured tubes: #{watched.join(', ')}
135
+ Configured tubes: #{allowed.join(', ')}
135
136
  ERROR
136
137
  end
137
138
  end
@@ -17,7 +17,12 @@ module Postburner
17
17
  # runner = Postburner::Runner.new(options)
18
18
  # runner.run
19
19
  #
20
+ # @see Postburner::Worker
21
+ # @see Postburner::Configuration
22
+ #
20
23
  class Runner
24
+ # @!attribute [r] options
25
+ # @return [Hash] The runner options
21
26
  attr_reader :options
22
27
 
23
28
  # Initialize runner with options.
@@ -50,10 +55,15 @@ module Postburner
50
55
 
51
56
  # Load configuration from YAML file.
52
57
  #
58
+ # Sets the global {Postburner.configuration} after loading.
59
+ #
53
60
  # @return [Postburner::Configuration]
61
+ # @api private
54
62
  def load_configuration
55
63
  config_path = File.expand_path(options[:config], root_directory)
56
- Postburner::Configuration.load_yaml(config_path, options[:env], options[:worker])
64
+ config = Postburner::Configuration.load_yaml(config_path, options[:env], options[:worker])
65
+ Postburner.configuration = config
66
+ config
57
67
  rescue ArgumentError => e
58
68
  logger.error "[Postburner] ERROR: #{e.message}"
59
69
  exit 1
@@ -61,8 +71,12 @@ module Postburner
61
71
 
62
72
  # Filter configuration to only include specified queues.
63
73
  #
74
+ # Validates that all specified queues exist in config and exits
75
+ # with error if unknown queues are specified.
76
+ #
64
77
  # @param config [Postburner::Configuration]
65
78
  # @return [void]
79
+ # @api private
66
80
  def filter_queues(config)
67
81
  # Validate that all specified queues exist in config
68
82
  invalid_queues = options[:queues] - config.queue_names
@@ -81,11 +95,15 @@ module Postburner
81
95
  #
82
96
  # @param config [Postburner::Configuration]
83
97
  # @return [void]
98
+ # @api private
84
99
  def log_configuration(config)
85
100
  config_path = File.expand_path(options[:config], root_directory)
86
101
  config.logger.info "[Postburner] Configuration: #{config_path}"
87
102
  config.logger.info "[Postburner] Environment: #{options[:env]}"
88
- config.logger.info "[Postburner] Worker: #{options[:worker] || '(auto-selected)'}" if options[:worker] || options[:queues].nil?
103
+ if options[:worker] || options[:queues].nil?
104
+ worker_label = options[:worker] || "(auto-selected: #{config.worker_config[:name]})"
105
+ config.logger.info "[Postburner] Worker: #{worker_label}"
106
+ end
89
107
  config.logger.info "[Postburner] Queues: #{config.queue_names.join(', ')}"
90
108
  wc = config.worker_config
91
109
  config.logger.info "[Postburner] Worker config: forks=#{wc[:forks]}, threads=#{wc[:threads]}, gc_limit=#{wc[:gc_limit] || 'none'}, timeout=#{wc[:timeout]}s"
@@ -96,6 +114,7 @@ module Postburner
96
114
  # Uses Rails.root when Rails is defined, otherwise falls back to current directory.
97
115
  #
98
116
  # @return [String] Root directory path
117
+ # @api private
99
118
  def root_directory
100
119
  if defined?(Rails) && Rails.respond_to?(:root)
101
120
  Rails.root.to_s
@@ -108,14 +127,18 @@ module Postburner
108
127
  #
109
128
  # @param config [Postburner::Configuration]
110
129
  # @return [void]
130
+ # @api private
111
131
  def start_worker(config)
112
- worker = Postburner::Workers::Worker.new(config)
132
+ worker = Postburner::Worker.new(config)
113
133
  worker.start
114
134
  end
115
135
 
116
136
  # Returns logger for error reporting during initialization.
117
137
  #
138
+ # Falls back to stderr when Rails is not available.
139
+ #
118
140
  # @return [Logger]
141
+ # @api private
119
142
  def logger
120
143
  if defined?(Rails)
121
144
  Rails.logger
@@ -0,0 +1,427 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'postburner/advisory_lock'
4
+
5
+ module Postburner
6
+ # Lightweight scheduler that acts as a safety net for recurring job execution.
7
+ #
8
+ # Unlike traditional schedulers that poll for due jobs, Postburner uses **immediate
9
+ # enqueue** where executions are created and immediately queued to Beanstalkd's
10
+ # delayed queue. The scheduler acts as a **watchdog safety net** to ensure every
11
+ # schedule has a future execution queued.
12
+ #
13
+ # ## Architecture
14
+ #
15
+ # This class is NOT persisted to the database. It's instantiated on-the-fly by
16
+ # workers when they reserve a watchdog job from the scheduler tube. The watchdog
17
+ # is ephemeral data in Beanstalkd with this payload:
18
+ #
19
+ # { "scheduler": true, "interval": 300 }
20
+ #
21
+ # No dedicated scheduler process is needed - existing workers handle everything.
22
+ #
23
+ # ## Watchdog Pattern
24
+ #
25
+ # 1. Workers automatically watch the 'scheduler' tube
26
+ # 2. On reserve timeout, workers check if watchdog exists and create if missing
27
+ # 3. When worker reserves watchdog, it instantiates Postburner::Scheduler
28
+ # 4. Scheduler executes with PostgreSQL advisory lock for coordination
29
+ # 5. After completion, watchdog re-queues itself with delay for next interval
30
+ #
31
+ # ## Safety Net Functions
32
+ #
33
+ # The watchdog performs three safety net functions:
34
+ #
35
+ # 1. **Auto-bootstrap**: Creates first execution for schedules that haven't been started
36
+ # 2. **Future execution guarantee**: Ensures each schedule has a future execution queued
37
+ # 3. **Orphan cleanup**: Enqueues any pending executions that weren't properly queued
38
+ #
39
+ # Note: For Postburner::Job schedules, a before_attempt callback creates the next
40
+ # execution when the current job runs, providing immediate pickup without waiting
41
+ # for the watchdog. The watchdog is just the safety net.
42
+ #
43
+ # ## Configuration
44
+ #
45
+ # Add to config/postburner.yml:
46
+ #
47
+ # production:
48
+ # default_scheduler_interval: 300 # Check every 5 minutes (default)
49
+ # default_scheduler_priority: 100 # Watchdog priority (default)
50
+ #
51
+ # The interval primarily affects:
52
+ # - How quickly new schedules are auto-bootstrapped (if you don't call start!)
53
+ # - Recovery time if an execution somehow fails to enqueue
54
+ # - How often last_audit_at is updated for monitoring
55
+ #
56
+ # Since executions are enqueued immediately to Beanstalkd's delayed queue, the
57
+ # watchdog interval doesn't affect execution timing - only the safety net checks.
58
+ #
59
+ # ## Execution Flow
60
+ #
61
+ # 1. Worker reserves watchdog from scheduler tube
62
+ # 2. Instantiates Scheduler with interval from payload
63
+ # 3. Scheduler#perform acquires PostgreSQL advisory lock
64
+ # 4. Processes all enabled schedules:
65
+ # - Auto-bootstraps unstarted schedules (creates + enqueues first execution)
66
+ # - Ensures future execution exists (creates + enqueues if missing)
67
+ # - Enqueues any orphaned pending executions
68
+ # - Updates last_audit_at timestamp
69
+ # 5. Re-queues watchdog with delay for next interval
70
+ #
71
+ # ## Observability
72
+ #
73
+ # Monitor scheduler health via last_audit_at:
74
+ #
75
+ # stale = Postburner::Schedule.enabled
76
+ # .where('last_audit_at < ?', 15.minutes.ago)
77
+ # .or(Postburner::Schedule.where(last_audit_at: nil))
78
+ #
79
+ # @example Watchdog payload in Beanstalkd
80
+ # {
81
+ # "scheduler": true,
82
+ # "interval": 300
83
+ # }
84
+ #
85
+ # @example Manual watchdog creation (usually automatic)
86
+ # Postburner::Scheduler.enqueue_watchdog(interval: 300, priority: 100)
87
+ #
88
+ # @see Postburner::Schedule
89
+ # @see Postburner::ScheduleExecution
90
+ #
91
+ class Scheduler
92
+ SCHEDULER_TUBE_NAME = 'scheduler'
93
+ DEFAULT_INTERVAL = 300 # 5 minutes
94
+ DEFAULT_PRIORITY = 100 # Lower number = higher priority
95
+
96
+ attr_reader :interval, :logger
97
+
98
+ # Initialize a new scheduler instance.
99
+ #
100
+ # The scheduler is instantiated by workers when they reserve a watchdog job
101
+ # from the scheduler tube. It's not persisted - each run creates a new instance.
102
+ #
103
+ # @param interval [Integer] Seconds until next run (default: 300)
104
+ # @param logger [Logger, nil] Logger instance (default: Rails.logger or stdout)
105
+ #
106
+ # @example
107
+ # scheduler = Postburner::Scheduler.new(interval: 300)
108
+ # scheduler.perform
109
+ #
110
+ def initialize(interval: DEFAULT_INTERVAL, logger: nil)
111
+ @interval = interval
112
+ @logger = logger || (defined?(Rails) ? Rails.logger : Logger.new($stdout))
113
+ end
114
+
115
+ # Execute the scheduler.
116
+ #
117
+ # Processes all enabled schedules with PostgreSQL advisory lock coordination
118
+ # to prevent concurrent execution across multiple workers. After processing,
119
+ # automatically re-queues the watchdog for the next run.
120
+ #
121
+ # The scheduler performs two main functions:
122
+ # 1. Auto-bootstrap new schedules (create first execution if not started)
123
+ # 2. Ensure each schedule has a future execution queued
124
+ # 3. Enqueue any orphaned pending executions that weren't properly queued
125
+ #
126
+ # @return [void]
127
+ #
128
+ # @example Called by worker
129
+ # # Worker reserves watchdog job with payload:
130
+ # # { "scheduler" => true, "interval" => 300 }
131
+ # scheduler = Postburner::Scheduler.new(interval: 300)
132
+ # scheduler.perform
133
+ #
134
+ def perform
135
+ logger.info "[Postburner::Scheduler] Starting scheduler run"
136
+
137
+ # Use advisory lock to coordinate multiple workers
138
+ acquired = Postburner::AdvisoryLock.with_lock(AdvisoryLock::SCHEDULER_LOCK_KEY, blocking: false) do
139
+ process_all_schedules
140
+ true
141
+ end
142
+
143
+ if acquired
144
+ logger.info "[Postburner::Scheduler] Scheduler run complete"
145
+ else
146
+ logger.info "[Postburner::Scheduler] Could not acquire lock, skipping"
147
+ end
148
+ ensure
149
+ # Always re-queue watchdog for next run
150
+ requeue_watchdog
151
+ end
152
+
153
+ # Class method to enqueue watchdog to Beanstalkd
154
+ # Mutex for coordinating watchdog checks across threads
155
+ WATCHDOG_MUTEX = Mutex.new
156
+ @watchdog_last_checked_at = nil
157
+ @watchdog_check_failed_at = nil
158
+
159
+ class << self
160
+ attr_accessor :watchdog_last_checked_at, :watchdog_check_failed_at
161
+ end
162
+
163
+ # Ensure a scheduler watchdog exists in the queue.
164
+ #
165
+ # Uses process-level mutex coordination so only one thread checks at a time.
166
+ # Called by workers on reserve timeout to automatically recreate watchdog
167
+ # if it's missing (e.g., after Beanstalkd restart or watchdog expiration).
168
+ #
169
+ # Implements throttling:
170
+ # - Skips check if successful check within last 60 seconds
171
+ # - Skips check if failed check within last 60 seconds
172
+ # - Only one thread can check at a time (mutex)
173
+ #
174
+ # @param connection [Postburner::Connection] Existing beanstalkd connection
175
+ # @return [void]
176
+ #
177
+ # @example Called by worker on timeout
178
+ # Postburner.connected do |conn|
179
+ # Postburner::Scheduler.ensure_watchdog!(connection: conn)
180
+ # end
181
+ #
182
+ def self.ensure_watchdog!(connection:)
183
+ # Quick check without lock - skip if recently checked
184
+ return if watchdog_last_checked_at && Time.current - watchdog_last_checked_at < 60
185
+ return if watchdog_check_failed_at && Time.current - watchdog_check_failed_at < 60
186
+
187
+ # Try to acquire lock, skip if another thread is checking
188
+ return unless WATCHDOG_MUTEX.try_lock
189
+
190
+ begin
191
+ # Double-check after acquiring lock
192
+ return if watchdog_last_checked_at && Time.current - watchdog_last_checked_at < 60
193
+
194
+ if watchdog_exists?(connection: connection)
195
+ self.watchdog_last_checked_at = Time.current
196
+ return
197
+ end
198
+
199
+ enqueue_watchdog
200
+ self.watchdog_last_checked_at = Time.current
201
+ self.watchdog_check_failed_at = nil
202
+ rescue => e
203
+ self.watchdog_check_failed_at = Time.current
204
+ Rails.logger.error "[Postburner::Scheduler] Watchdog check failed, will retry in 60s: #{e.message}"
205
+ ensure
206
+ WATCHDOG_MUTEX.unlock
207
+ end
208
+ end
209
+
210
+ # Enqueue a new scheduler watchdog job.
211
+ #
212
+ # Creates a watchdog job in the scheduler tube with a delay equal to the interval.
213
+ # The watchdog will execute after the delay, process all schedules, and re-queue itself.
214
+ #
215
+ # Reads interval and priority from configuration if not provided:
216
+ # - default_scheduler_interval (default: 300 seconds)
217
+ # - default_scheduler_priority (default: 100)
218
+ #
219
+ # @param interval [Integer, nil] Seconds until next run (default: from config)
220
+ # @param priority [Integer, nil] Beanstalkd priority (default: from config)
221
+ # @return [Hash] Beanstalkd response with :status and :id keys
222
+ #
223
+ # @example Enqueue with defaults
224
+ # Postburner::Scheduler.enqueue_watchdog
225
+ # # => { status: "INSERTED", id: 12345 }
226
+ #
227
+ # @example Enqueue with custom interval
228
+ # Postburner::Scheduler.enqueue_watchdog(interval: 60, priority: 0)
229
+ #
230
+ def self.enqueue_watchdog(interval: nil, priority: nil)
231
+ interval ||= Postburner.configuration.default_scheduler_interval || DEFAULT_INTERVAL
232
+ priority ||= Postburner.configuration.default_scheduler_priority || DEFAULT_PRIORITY
233
+
234
+ payload = {
235
+ 'scheduler' => true,
236
+ 'interval' => interval
237
+ }
238
+
239
+ tube_name = Postburner.scheduler_tube_name
240
+ response = nil
241
+
242
+ Postburner.connected do |conn|
243
+ response = conn.tubes[tube_name].put(
244
+ JSON.generate(payload),
245
+ pri: priority,
246
+ delay: interval,
247
+ ttr: 120 # 2 minutes to complete
248
+ )
249
+ end
250
+
251
+ if response[:status] == "INSERTED"
252
+ runs_at = Time.current + interval
253
+ Rails.logger.info "[Postburner::Scheduler] Inserted watchdog: #{response[:id]} (#{Time.current.iso8601}, delay: #{interval}s (#{runs_at.iso8601}), tube: #{tube_name})"
254
+ else
255
+ Rails.logger.error "[Postburner::Scheduler] Failed to insert watchdog: #{response.inspect} (delay: #{interval}s, tube: #{tube_name})"
256
+ end
257
+
258
+ response
259
+ end
260
+
261
+ # Check if scheduler watchdog exists in queue.
262
+ #
263
+ # Peeks both the delayed and ready queues in the scheduler tube.
264
+ # Returns true if watchdog exists in either state.
265
+ #
266
+ # @param connection [Postburner::Connection] Existing beanstalkd connection
267
+ # @return [Boolean] true if watchdog exists (delayed or ready), false otherwise
268
+ #
269
+ # @example Check for watchdog
270
+ # Postburner.connected do |conn|
271
+ # exists = Postburner::Scheduler.watchdog_exists?(connection: conn)
272
+ # puts "Watchdog present" if exists
273
+ # end
274
+ #
275
+ def self.watchdog_exists?(connection:)
276
+ tube_name = Postburner.scheduler_tube_name
277
+ tube = connection.beanstalk.tubes[tube_name]
278
+
279
+ # Peek is lighter than stats
280
+ delayed = tube.peek(:delayed) rescue nil
281
+ ready = tube.peek(:ready) rescue nil
282
+
283
+ delayed.present? || ready.present?
284
+ rescue Beaneater::NotFoundError
285
+ false
286
+ end
287
+
288
+ private
289
+
290
+ # Process all enabled schedules.
291
+ #
292
+ # Iterates through all enabled schedules and calls process_schedule for each.
293
+ # Logs count of successful and failed schedule processing.
294
+ #
295
+ # @return [void]
296
+ #
297
+ # @api private
298
+ #
299
+ def process_all_schedules
300
+ processed_count = 0
301
+ failed_count = 0
302
+
303
+ Postburner::Schedule.enabled.find_each do |schedule|
304
+ begin
305
+ process_schedule(schedule)
306
+ processed_count += 1
307
+ rescue => e
308
+ logger.error "[Postburner::Scheduler] Failed to process schedule '#{schedule.name}': #{e.class} - #{e.message}"
309
+ logger.error e.backtrace.join("\n")
310
+ failed_count += 1
311
+ end
312
+ end
313
+
314
+ logger.info "[Postburner::Scheduler] Processed #{processed_count} schedules, #{failed_count} failed"
315
+ end
316
+
317
+ # Process a single schedule.
318
+ #
319
+ # The watchdog performs three safety net functions:
320
+ # 1. Auto-bootstrap: Create first execution if schedule hasn't been started
321
+ # 2. Ensure there's always a future execution (creates + enqueues if missing)
322
+ # 3. Enqueue any orphaned pending executions (that somehow weren't enqueued)
323
+ #
324
+ # Note: We check for future executions FIRST, then clean up any orphans.
325
+ # This ensures newly created executions are available for processing in the
326
+ # same run if they happen to be due.
327
+ #
328
+ # @param schedule [Postburner::Schedule] The schedule to process
329
+ # @return [void]
330
+ #
331
+ # @api private
332
+ #
333
+ def process_schedule(schedule)
334
+ # Auto-bootstrap: create first execution if schedule hasn't been started
335
+ # This will create the execution AND enqueue it to Beanstalkd
336
+ unless schedule.started?
337
+ logger.info "[Postburner::Scheduler] Bootstrapping schedule '#{schedule.name}'"
338
+ schedule.start!
339
+ end
340
+
341
+ # Safety net 1: Ensure schedule has a future execution
342
+ # If missing, this will create AND enqueue it to Beanstalkd's delayed queue
343
+ ensure_future_execution!(schedule)
344
+
345
+ # Safety net 2: Find any orphaned pending executions and enqueue them
346
+ # This should rarely happen - only if enqueue! previously failed
347
+ execution_count = 0
348
+ schedule.executions.due.find_each do |execution|
349
+ begin
350
+ logger.warn "[Postburner::Scheduler] Found orphaned pending execution #{execution.id} for schedule '#{schedule.name}', enqueuing"
351
+ execution.enqueue!
352
+ execution_count += 1
353
+ rescue => e
354
+ logger.error "[Postburner::Scheduler] Failed to enqueue execution #{execution.id}: #{e.class} - #{e.message}"
355
+ raise
356
+ end
357
+ end
358
+
359
+ # Update last_audit_at
360
+ schedule.update_column(:last_audit_at, Time.current)
361
+
362
+ logger.debug "[Postburner::Scheduler] Schedule '#{schedule.name}': enqueued #{execution_count} orphaned executions" if execution_count > 0
363
+ end
364
+
365
+ # Ensure schedule has a future scheduled or pending execution.
366
+ #
367
+ # Uses Schedule#create_next_execution! which is idempotent and only
368
+ # creates an execution if none exists in the future. If no executions
369
+ # exist at all, bootstraps the schedule first.
370
+ #
371
+ # This is the key safety net that ensures every schedule always has
372
+ # at least one future execution ready to run.
373
+ #
374
+ # @param schedule [Postburner::Schedule] The schedule to ensure future execution for
375
+ # @return [void]
376
+ #
377
+ # @api private
378
+ #
379
+ def ensure_future_execution!(schedule)
380
+ # Find the most recent execution to calculate next from
381
+ last_execution = schedule.executions.order(run_at: :desc).first
382
+
383
+ unless last_execution
384
+ # No executions at all - this shouldn't happen since we bootstrap first,
385
+ # but handle it as a safety net
386
+ logger.warn "[Postburner::Scheduler] Schedule '#{schedule.name}' has no executions, bootstrapping"
387
+ schedule.start!
388
+ return
389
+ end
390
+
391
+ # Delegate to Schedule - it knows whether a future execution is needed
392
+ execution = schedule.create_next_execution!(after: last_execution)
393
+
394
+ if execution
395
+ logger.info "[Postburner::Scheduler] Created future execution for '#{schedule.name}'"
396
+ end
397
+ end
398
+
399
+ # Re-queue the watchdog for the next run (if not already queued).
400
+ #
401
+ # Called in the ensure block of perform to guarantee the watchdog
402
+ # perpetuates itself. Always reads interval from config to pick up
403
+ # any runtime configuration changes.
404
+ #
405
+ # If re-queueing fails, logs error but doesn't raise - workers will
406
+ # recreate the watchdog on next timeout via ensure_watchdog!
407
+ #
408
+ # @return [void]
409
+ #
410
+ # @api private
411
+ #
412
+ def requeue_watchdog
413
+ Postburner.connected do |conn|
414
+ if self.class.watchdog_exists?(connection: conn)
415
+ logger.debug "[Postburner::Scheduler] Watchdog already exists, skipping re-queue"
416
+ return
417
+ end
418
+
419
+ self.class.enqueue_watchdog
420
+ end
421
+ rescue => e
422
+ logger.error "[Postburner::Scheduler] Failed to re-queue watchdog: #{e.class} - #{e.message}"
423
+ # This is critical - if watchdog isn't re-queued, scheduling stops
424
+ # Workers will recreate it on next timeout, but log loudly
425
+ end
426
+ end
427
+ end