exekutor 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/exe/exekutor +2 -2
  4. data/lib/active_job/queue_adapters/exekutor_adapter.rb +2 -1
  5. data/lib/exekutor/asynchronous.rb +143 -75
  6. data/lib/exekutor/cleanup.rb +27 -28
  7. data/lib/exekutor/configuration.rb +48 -25
  8. data/lib/exekutor/hook.rb +15 -11
  9. data/lib/exekutor/info/worker.rb +3 -3
  10. data/lib/exekutor/internal/base_record.rb +2 -1
  11. data/lib/exekutor/internal/callbacks.rb +55 -35
  12. data/lib/exekutor/internal/cli/app.rb +31 -23
  13. data/lib/exekutor/internal/cli/application_loader.rb +17 -6
  14. data/lib/exekutor/internal/cli/cleanup.rb +54 -40
  15. data/lib/exekutor/internal/cli/daemon.rb +9 -11
  16. data/lib/exekutor/internal/cli/default_option_value.rb +3 -1
  17. data/lib/exekutor/internal/cli/info.rb +117 -84
  18. data/lib/exekutor/internal/cli/manager.rb +190 -123
  19. data/lib/exekutor/internal/configuration_builder.rb +40 -27
  20. data/lib/exekutor/internal/database_connection.rb +6 -0
  21. data/lib/exekutor/internal/executable.rb +12 -7
  22. data/lib/exekutor/internal/executor.rb +50 -21
  23. data/lib/exekutor/internal/hooks.rb +11 -8
  24. data/lib/exekutor/internal/listener.rb +66 -39
  25. data/lib/exekutor/internal/logger.rb +28 -10
  26. data/lib/exekutor/internal/provider.rb +93 -74
  27. data/lib/exekutor/internal/reserver.rb +27 -12
  28. data/lib/exekutor/internal/status_server.rb +81 -49
  29. data/lib/exekutor/job.rb +1 -1
  30. data/lib/exekutor/job_error.rb +1 -1
  31. data/lib/exekutor/job_options.rb +22 -13
  32. data/lib/exekutor/plugins/appsignal.rb +7 -5
  33. data/lib/exekutor/plugins.rb +8 -4
  34. data/lib/exekutor/queue.rb +40 -22
  35. data/lib/exekutor/version.rb +1 -1
  36. data/lib/exekutor/worker.rb +88 -47
  37. data/lib/exekutor.rb +2 -2
  38. data/lib/generators/exekutor/configuration_generator.rb +9 -5
  39. data/lib/generators/exekutor/install_generator.rb +26 -15
  40. data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb +11 -10
  41. data.tar.gz.sig +0 -0
  42. metadata +63 -19
  43. metadata.gz.sig +0 -0
@@ -8,16 +8,18 @@ module Exekutor
8
8
  module Internal
9
9
  # Reserves jobs and provides them to an executor
10
10
  class Provider
11
- include Executable, Callbacks, Logger
11
+ include Logger
12
+ include Callbacks
13
+ include Executable
12
14
 
13
15
  define_callbacks :on_queue_empty, freeze: true
14
16
 
15
17
  # Represents an unknown value
16
18
  UNKNOWN = Object.new.freeze
17
- private_constant "UNKNOWN"
19
+ private_constant :UNKNOWN
18
20
 
19
21
  MAX_WAIT_TIMEOUT = 300
20
- private_constant "MAX_WAIT_TIMEOUT"
22
+ private_constant :MAX_WAIT_TIMEOUT
21
23
 
22
24
  # Creates a new provider
23
25
  # @param reserver [Reserver] the job reserver
@@ -48,14 +50,14 @@ module Exekutor
48
50
 
49
51
  # Always poll at startup to fill up threads, use small jitter so workers started at the same time dont hit
50
52
  # the db at the same time
51
- @next_poll_at.set (1 + 2 * Kernel.rand).second.from_now
53
+ @next_poll_at.set (1 + (2 * Kernel.rand)).seconds.from_now.to_f
52
54
  start_thread
53
55
  true
54
56
  end
55
57
 
56
58
  # Stops the provider
57
59
  def stop
58
- set_state :stopped
60
+ self.state = :stopped
59
61
  @event.set
60
62
  end
61
63
 
@@ -63,7 +65,7 @@ module Exekutor
63
65
  def poll
64
66
  raise Exekutor::Error, "Provider is not running" unless running?
65
67
 
66
- @next_poll_at.set Time.now
68
+ @next_poll_at.set Time.now.to_f
67
69
  @event.set
68
70
  end
69
71
 
@@ -71,51 +73,39 @@ module Exekutor
71
73
  # argument is given. Updates the timestamp for the earliest job is a timestamp is given and that timestamp is
72
74
  # before the known timestamp. Does nothing if a timestamp is given and the earliest job timestamp is not known.
73
75
  # @param scheduled_at [Time,Numeric] the time a job is scheduled at
74
- # @return [Time] the timestamp for the next job, or +nil+ if the timestamp is unknown or no jobs are pending
76
+ # @return [Float,nil] the timestamp for the next job, or +nil+ if the timestamp is unknown or no jobs are pending
75
77
  def update_earliest_scheduled_at(scheduled_at = UNKNOWN)
76
- overwrite_unknown = false
77
- case scheduled_at
78
- when UNKNOWN
79
- # If we fetch the value from the DB, we can safely overwrite the UNKNOWN value
80
- overwrite_unknown = true
81
- scheduled_at = @reserver.earliest_scheduled_at
82
- when Numeric
83
- scheduled_at = Time.at(scheduled_at)
84
- when Time
85
- # All good
86
- else
78
+ scheduled_at = scheduled_at.to_f if scheduled_at.is_a? Time
79
+ unless scheduled_at == UNKNOWN || scheduled_at.is_a?(Numeric)
87
80
  raise ArgumentError, "scheduled_at must be a Time or Numeric"
88
81
  end
89
82
 
90
- updated = false
91
- scheduled_at = @next_job_scheduled_at.update do |current|
92
- if current == UNKNOWN
93
- if overwrite_unknown || scheduled_at <= Time.now
94
- updated = true
95
- scheduled_at
96
- else
97
- current
98
- end
99
- elsif current.nil? || scheduled_at.nil? || current > scheduled_at
100
- updated = true
101
- scheduled_at
102
- else
103
- current
104
- end
83
+ changed = false
84
+ earliest_scheduled_at = @next_job_scheduled_at.update do |current|
85
+ earliest = ScheduledAtComparer.determine_earliest(current, scheduled_at, @reserver)
86
+ changed = earliest != current
87
+ earliest
105
88
  end
106
- if scheduled_at == UNKNOWN
89
+
90
+ if earliest_scheduled_at == UNKNOWN
107
91
  nil
108
92
  else
109
- @event.set if updated && scheduled_at.present?
110
- scheduled_at
93
+ @event.set if changed && earliest_scheduled_at.present?
94
+ earliest_scheduled_at
111
95
  end
112
96
  end
113
97
 
114
98
  private
115
99
 
116
100
  # Starts the provision thread
117
- def start_thread
118
- @pool.post(&method(:run)) if running?
101
+ def start_thread(delay: nil)
102
+ return unless running?
103
+
104
+ if delay
105
+ Concurrent::ScheduledTask.execute(delay, executor: @pool) { run }
106
+ else
107
+ @pool.post { run }
108
+ end
119
109
  end
120
110
 
121
111
  # Does the provisioning of jobs to the executor. Blocks until the provider is stopped.
@@ -134,19 +124,24 @@ module Exekutor
134
124
  consecutive_errors.value = 0
135
125
  end
136
126
  end
137
- rescue StandardError => err
138
- Exekutor.on_fatal_error err, "[Provider] Runtime error!"
139
- consecutive_errors.increment
140
- if running?
141
- delay = restart_delay
142
- logger.info "Restarting in %0.1f seconds…" % [delay]
143
- Concurrent::ScheduledTask.execute(delay, executor: @pool, &method(:run))
144
- end
127
+ rescue StandardError => e
128
+ on_thread_error(e)
145
129
  ensure
146
130
  BaseRecord.connection_pool.release_connection
147
131
  @thread_running.make_false
148
132
  end
149
133
 
134
+ # Called when an error is raised in #run
135
+ def on_thread_error(error)
136
+ Exekutor.on_fatal_error error, "[Provider] Runtime error!"
137
+ return unless running?
138
+
139
+ consecutive_errors.increment
140
+ delay = restart_delay
141
+ logger.info format("Restarting in %0.1f seconds…", delay)
142
+ start_thread delay: delay
143
+ end
144
+
150
145
  # Waits for any event to happen. An event could be:
151
146
  # - The listener was notified of a new job;
152
147
  # - The next job is scheduled for the current time;
@@ -157,8 +152,8 @@ module Exekutor
157
152
  return unless timeout.positive?
158
153
 
159
154
  @event.wait timeout
160
- rescue StandardError => err
161
- Exekutor.on_fatal_error err, "[Provider] An error occurred while waiting"
155
+ rescue StandardError => e
156
+ Exekutor.on_fatal_error e, "[Provider] An error occurred while waiting"
162
157
  sleep 0.1 if running?
163
158
  ensure
164
159
  throw :shutdown unless running?
@@ -171,21 +166,8 @@ module Exekutor
171
166
  return unless available_workers.positive?
172
167
 
173
168
  jobs = @reserver.reserve available_workers
174
- unless jobs.nil?
175
- begin
176
- logger.debug "Reserved #{jobs.size} job(s)"
177
- jobs.each(&@executor.method(:post))
178
- rescue Exception # rubocop:disable Lint/RescueException
179
- # Try to release all jobs before re-raising
180
- begin
181
- Exekutor::Job.where(id: jobs.collect { |job| job[:id] }, status: "e")
182
- .update_all(status: "p", worker_id: nil)
183
- rescue # rubocop:disable Lint/RescueStandardError
184
- # ignored
185
- end
186
- raise
187
- end
188
- end
169
+ execute_jobs jobs unless jobs.nil?
170
+
189
171
  if jobs.nil? || jobs.size.to_i < available_workers
190
172
  # If we ran out of work, update the earliest scheduled at
191
173
  update_earliest_scheduled_at
@@ -194,8 +176,24 @@ module Exekutor
194
176
 
195
177
  elsif @next_job_scheduled_at.get == UNKNOWN
196
178
  # If the next job timestamp is still unknown, set it to now to indicate there's still work to do
197
- @next_job_scheduled_at.set Time.now
179
+ @next_job_scheduled_at.set Time.now.to_f
180
+ end
181
+ end
182
+
183
+ # Posts the given jobs to the executor thread pool
184
+ # @param jobs [Array] the jobs to execute
185
+ def execute_jobs(jobs)
186
+ logger.debug "Reserved #{jobs.size} job(s)"
187
+ jobs.each { |job| @executor.post(job) }
188
+ rescue Exception # rubocop:disable Lint/RescueException
189
+ # Try to release all jobs before re-raising
190
+ begin
191
+ Exekutor::Job.where(id: jobs.pluck(:id), status: "e")
192
+ .update_all(status: "p", worker_id: nil)
193
+ rescue StandardError
194
+ # ignored
198
195
  end
196
+ raise
199
197
  end
200
198
 
201
199
  def perform_pending_job_updates
@@ -224,7 +222,7 @@ module Exekutor
224
222
  return if jobs&.size.to_i.zero?
225
223
 
226
224
  logger.info "Restarting #{jobs.size} abandoned job#{"s" if jobs.size > 1}"
227
- jobs.each(&@executor.method(:post))
225
+ jobs.each { |job| @executor.post(job) }
228
226
  end
229
227
 
230
228
  # @return [Boolean] Whether the polling is enabled. Ie. whether a polling interval is set.
@@ -232,7 +230,7 @@ module Exekutor
232
230
  @polling_interval.present?
233
231
  end
234
232
 
235
- # @return [Time,nil] the 'scheduled at' value for the next job, or nil if unknown or if there is no pending job
233
+ # @return [Float,nil] the 'scheduled at' value for the next job, or nil if unknown or if there is no pending job
236
234
  def next_job_scheduled_at
237
235
  at = @next_job_scheduled_at.get
238
236
  if at == UNKNOWN
@@ -243,10 +241,10 @@ module Exekutor
243
241
  end
244
242
  end
245
243
 
246
- # @return [Time,nil] When the next poll is scheduled, or nil if polling is disabled
244
+ # @return [Float,nil] When the next poll is scheduled, or nil if polling is disabled
247
245
  def next_poll_scheduled_at
248
246
  if polling_enabled?
249
- @next_poll_at.update { |planned_at| planned_at || Time.now + polling_interval }
247
+ @next_poll_at.update { |planned_at| planned_at || (Time.now.to_f + polling_interval) }
250
248
  else
251
249
  # noinspection RubyMismatchedReturnType
252
250
  @next_poll_at.get
@@ -262,9 +260,9 @@ module Exekutor
262
260
 
263
261
  timeout = [MAX_WAIT_TIMEOUT].tap do |timeouts|
264
262
  # noinspection RubyMismatchedArgumentType
265
- timeouts.append next_job_at - Time.now if next_job_at
263
+ timeouts.append next_job_at - Time.now.to_f if next_job_at
266
264
  # noinspection RubyMismatchedArgumentType
267
- timeouts.append next_poll_at - Time.now if next_poll_at
265
+ timeouts.append next_poll_at - Time.now.to_f if next_poll_at
268
266
  end.min
269
267
 
270
268
  if timeout <= 0.001
@@ -278,13 +276,13 @@ module Exekutor
278
276
  # @return [Boolean] Whether the `reserver` should be called.
279
277
  def reserve_jobs_now?
280
278
  next_poll_at = next_poll_scheduled_at
281
- if next_poll_at && next_poll_at - Time.now <= 0.001
282
- @next_poll_at.update { Time.now + polling_interval if polling_enabled? }
279
+ if next_poll_at && next_poll_at - Time.now.to_f <= 0.001
280
+ @next_poll_at.update { Time.now.to_f + polling_interval if polling_enabled? }
283
281
  return true
284
282
  end
285
283
 
286
284
  next_job_at = next_job_scheduled_at
287
- next_job_at && next_job_at <= Time.now
285
+ next_job_at && next_job_at <= Time.now.to_f
288
286
  end
289
287
 
290
288
  # @return [Float] Gets the polling interval jitter
@@ -295,7 +293,7 @@ module Exekutor
295
293
  # Get the polling interval. If a jitter is configured, the interval is reduced or increased by `0.5 * jitter`.
296
294
  # @return [Float] The amount of seconds before the next poll
297
295
  def polling_interval
298
- raise "Polling is disabled" unless @polling_interval.present?
296
+ raise "Polling is disabled" if @polling_interval.blank?
299
297
 
300
298
  @polling_interval + if polling_interval_jitter.zero?
301
299
  0
@@ -303,6 +301,27 @@ module Exekutor
303
301
  (Kernel.rand - 0.5) * polling_interval_jitter
304
302
  end
305
303
  end
304
+
305
+ # Abstraction to determine earliest out of 2 scheduled at timestamps
306
+ class ScheduledAtComparer
307
+ def self.determine_earliest(current_value, other_value, reserver)
308
+ if other_value == UNKNOWN
309
+ # Fetch the value from the db
310
+ reserver.earliest_scheduled_at&.to_f
311
+ elsif current_value == UNKNOWN
312
+ # We don't know the value yet, don't use the given value as earliest unless work is scheduled immediately
313
+ other_value > Time.now.to_f ? UNKNOWN : other_value
314
+ elsif current_value.nil? || current_value > other_value
315
+ # The given value is earlier than the known value
316
+ other_value
317
+ else
318
+ # No changes
319
+ current_value
320
+ end
321
+ end
322
+ end
323
+
324
+ private_constant :ScheduledAtComparer
306
325
  end
307
326
  end
308
327
  end
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Exekutor
3
4
  # @private
4
5
  module Internal
@@ -18,7 +19,7 @@ module Exekutor
18
19
 
19
20
  # Reserves pending jobs
20
21
  # @param limit [Integer] the number of jobs to reserve
21
- # @return [Array<Job>,nil] the reserved jobs, or nil if no jobs were reserved
22
+ # @return [Array<Hash>,nil] the reserved jobs, or nil if no jobs were reserved
22
23
  def reserve(limit)
23
24
  return unless limit.positive?
24
25
 
@@ -36,6 +37,9 @@ module Exekutor
36
37
  parse_jobs results
37
38
  end
38
39
 
40
+ # Gets the jobs that are assigned to this worker and have an id that is not included in +active_job_ids+
41
+ # @param active_job_ids [Array<String>] The ids of the jobs that should be excluded
42
+ # @return [Array<Hash>] the jobs
39
43
  def get_abandoned_jobs(active_job_ids)
40
44
  jobs = Exekutor::Job.executing.where(worker_id: @worker_id)
41
45
  jobs = jobs.where.not(id: active_job_ids) if active_job_ids.present?
@@ -59,7 +63,7 @@ module Exekutor
59
63
  { id: result["id"],
60
64
  payload: parse_json(result["payload"]),
61
65
  options: parse_json(result["options"]),
62
- scheduled_at: result['scheduled_at'] }
66
+ scheduled_at: result["scheduled_at"] }
63
67
  end
64
68
  end
65
69
 
@@ -71,25 +75,36 @@ module Exekutor
71
75
  # Builds SQL filter for the given queues
72
76
  def build_queue_filter_sql(queues)
73
77
  return nil if queues.nil? || (queues.is_a?(Array) && queues.empty?)
74
- unless queues.is_a?(String) || queues.is_a?(Symbol) || queues.is_a?(Array)
75
- raise ArgumentError, "queues must be nil, a String, Symbol, or an array of Strings or Symbols"
76
- end
77
78
 
78
79
  queues = queues.first if queues.is_a?(Array) && queues.one?
79
- if queues.is_a? Array
80
- unless queues.all? { |q| (q.is_a?(String) || q.is_a?(Symbol)) && !q.blank? }
81
- raise ArgumentError, "queues contains an invalid value"
82
- end
80
+ validate_queues! queues
83
81
 
82
+ if queues.is_a? Array
84
83
  Exekutor::Job.sanitize_sql_for_conditions(["AND queue IN (?)", queues])
85
84
  else
86
- raise ArgumentError, "queue name cannot be empty" if queues.blank?
87
-
88
85
  Exekutor::Job.sanitize_sql_for_conditions(["AND queue = ?", queues])
89
86
  end
90
87
  end
91
88
 
89
+ # Raises an error if the queues value is invalid
90
+ # @param queues [String,Symbol,Array<String,Symbol>] the queues to validate
91
+ # @raise [ArgumentError] if the queue is invalid or includes an invalid value
92
+ def validate_queues!(queues)
93
+ case queues
94
+ when Array
95
+ raise ArgumentError, "queues contains an invalid value" unless queues.all? { |queue| valid_queue_name? queue }
96
+ when String, Symbol
97
+ raise ArgumentError, "queue name cannot be empty" unless valid_queue_name? queues
98
+ else
99
+ raise ArgumentError, "queues must be nil, a String, Symbol, or an array of Strings or Symbols"
100
+ end
101
+ end
102
+
103
+ # @param queue [String,Symbol] the name of a queue
104
+ # @return [Boolean] whether the name is a valid queue name
105
+ def valid_queue_name?(queue)
106
+ (queue.is_a?(String) || queue.is_a?(Symbol)) && queue.present? && queue.length <= Queue::MAX_NAME_LENGTH
107
+ end
92
108
  end
93
109
  end
94
110
  end
95
-
@@ -1,7 +1,22 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Exekutor
3
4
  module Internal
4
- # Serves a simple health check app
5
+ # Serves a simple health check app. The app provides 4 endpoints:
6
+ # - +/+, which lists the other endpoints;
7
+ # - +/ready+, which indicates whether the worker is ready to start work;
8
+ # - +/live+, which indicates whether the worker is ready and whether the worker is still alive;
9
+ # - +/threads+, which indicated the thread usage of the worker.
10
+ #
11
+ # Please note that this server uses +webrick+ by default, which is no longer a default gem from ruby 3.0 onwards.
12
+ #
13
+ # === Example requests
14
+ # $ curl localhost:9000/ready
15
+ # [OK] ID: f1a2ee6a-cdac-459c-a4b8-de7c6a8bbae6; State: started
16
+ # $ curl localhost:9000/live
17
+ # [OK] ID: f1a2ee6a-cdac-459c-a4b8-de7c6a8bbae6; State: started; Heartbeat: 2023-04-05T16:27:00Z
18
+ # $ curl localhost:9000/threads
19
+ # {"minimum":1,"maximum":10,"available":4,"usage_percent":60.0}
5
20
  class StatusServer
6
21
  include Internal::Logger
7
22
  include Internal::Executable
@@ -19,24 +34,27 @@ module Exekutor
19
34
  @server = Concurrent::AtomicReference.new
20
35
  end
21
36
 
37
+ # Starts the web server
22
38
  def start
23
39
  return false unless compare_and_set_state :pending, :started
24
40
 
25
41
  start_thread
26
42
  end
27
43
 
44
+ # @return [Boolean] whether the web server is active
28
45
  def running?
29
46
  super && @thread_running.value
30
47
  end
31
48
 
49
+ # Stops the web server
32
50
  def stop
33
- set_state :stopped
51
+ self.state = :stopped
34
52
  return unless @thread_running.value
35
53
 
36
54
  server = @server.value
37
- if server&.respond_to? :shutdown
55
+ if server.respond_to? :shutdown
38
56
  server.shutdown
39
- elsif server&.respond_to? :stop
57
+ elsif server.respond_to? :stop
40
58
  server.stop
41
59
  elsif server
42
60
  Exekutor.say! "Cannot shutdown status server, #{server.class.name} does not respond to shutdown or stop"
@@ -45,6 +63,7 @@ module Exekutor
45
63
 
46
64
  protected
47
65
 
66
+ # Runs the web server, should be called from a separate thread
48
67
  def run(worker, port)
49
68
  return unless state == :started && @thread_running.make_true
50
69
 
@@ -53,11 +72,11 @@ module Exekutor
53
72
  Logger: ::Logger.new(File.open(File::NULL, "w")), AccessLog: []) do |server|
54
73
  @server.set server
55
74
  end
56
- rescue StandardError => err
57
- Exekutor.on_fatal_error err, "[HealthServer] Runtime error!"
75
+ rescue StandardError => e
76
+ Exekutor.on_fatal_error e, "[Status server] Runtime error!"
58
77
  if running?
59
78
  logger.info "Restarting in 10 seconds…"
60
- Concurrent::ScheduledTask.execute(10.0, executor: @pool, &method(:start_thread))
79
+ Concurrent::ScheduledTask.execute(10.0, executor: @pool) { start_thread }
61
80
  end
62
81
  ensure
63
82
  @thread_running.make_false
@@ -65,68 +84,81 @@ module Exekutor
65
84
 
66
85
  # The Rack-app for the health-check server
67
86
  class App
68
-
69
87
  def initialize(worker, heartbeat_timeout)
70
88
  @worker = worker
71
89
  @heartbeat_timeout = heartbeat_timeout
72
90
  end
73
91
 
74
- def flatlined?
75
- last_heartbeat = @worker.last_heartbeat
76
- last_heartbeat.nil? || last_heartbeat < @heartbeat_timeout.minutes.ago
77
- end
78
-
79
92
  def call(env)
80
93
  case Rack::Request.new(env).path
81
94
  when "/"
82
- [200, {}, [
83
- <<~RESPONSE
84
- [Exekutor]
85
- - Use GET /ready to check whether the worker is running and connected to the DB
86
- - Use GET /live to check whether the worker is running and is not hanging
87
- - Use GET /threads to check thread usage
88
- RESPONSE
89
- ]]
95
+ render_root
90
96
  when "/ready"
91
- running = @worker.running?
92
- if running
93
- Exekutor::Job.connection_pool.with_connection do |connection|
94
- running = connection.active?
95
- end
96
- end
97
- running = false if running && flatlined?
98
- [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
99
- "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}"
100
- ]]
97
+ render_ready
101
98
  when "/live"
102
- running = @worker.running?
103
- last_heartbeat = if running
104
- @worker.last_heartbeat
105
- end
106
- if running && (last_heartbeat.nil? || last_heartbeat < @heartbeat_timeout.minutes.ago)
107
- running = false
108
- end
109
- [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
110
- "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}; Heartbeat: #{last_heartbeat&.iso8601 || "null"}"
111
- ]]
99
+ render_live
112
100
  when "/threads"
113
- if @worker.running?
114
- info = @worker.thread_stats
115
- [(info ? 200 : 503), { "Content-Type" => "application/json" }, [info.to_json]]
116
- else
117
- [503, {"Content-Type" => "application/json"}, [{ error: "Worker not running" }.to_json]]
118
- end
101
+ render_threads
119
102
  else
120
103
  [404, {}, ["Not found"]]
121
104
  end
122
105
  end
106
+
107
+ private
108
+
109
+ def flatlined?(last_heartbeat = @worker.last_heartbeat)
110
+ last_heartbeat.nil? || last_heartbeat < @heartbeat_timeout.minutes.ago
111
+ end
112
+
113
+ def render_threads
114
+ if @worker.running?
115
+ info = @worker.thread_stats
116
+ [(info ? 200 : 503), { "Content-Type" => "application/json" }, [info.to_json]]
117
+ else
118
+ [503, { "Content-Type" => "application/json" }, [{ error: "Worker not running" }.to_json]]
119
+ end
120
+ end
121
+
122
+ def render_live
123
+ running = @worker.running?
124
+ last_heartbeat = (@worker.last_heartbeat if running)
125
+ running = false if flatlined?(last_heartbeat)
126
+ [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
127
+ "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}; " \
128
+ "Heartbeat: #{last_heartbeat&.iso8601 || "null"}"
129
+ ]]
130
+ end
131
+
132
+ def render_ready
133
+ running = @worker.running?
134
+ if running
135
+ Exekutor::Job.connection_pool.with_connection do |connection|
136
+ running = connection.active?
137
+ end
138
+ end
139
+ running = false if running && flatlined?
140
+ [(running ? 200 : 503), { "Content-Type" => "text/plain" }, [
141
+ "#{running ? "[OK]" : "[Service unavailable]"} ID: #{@worker.id}; State: #{@worker.state}"
142
+ ]]
143
+ end
144
+
145
+ def render_root
146
+ [200, {}, [
147
+ <<~RESPONSE
148
+ [Exekutor]
149
+ - Use GET /ready to check whether the worker is running and connected to the DB
150
+ - Use GET /live to check whether the worker is running and is not hanging
151
+ - Use GET /threads to check thread usage
152
+ RESPONSE
153
+ ]]
154
+ end
123
155
  end
124
156
 
125
157
  private
126
158
 
127
159
  def start_thread
128
- @pool.post(@worker, @port, &method(:run)) if state == :started
160
+ @pool.post(@worker, @port) { |*args| run(*args) } if state == :started
129
161
  end
130
162
  end
131
163
  end
132
- end
164
+ end
data/lib/exekutor/job.rb CHANGED
@@ -28,4 +28,4 @@ module Exekutor
28
28
  update! status: "d"
29
29
  end
30
30
  end
31
- end
31
+ end
@@ -8,4 +8,4 @@ module Exekutor
8
8
  self.implicit_order_column = :created_at
9
9
  belongs_to :job
10
10
  end
11
- end
11
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Exekutor
2
4
  # Mixin which defines custom job options for Exekutor. This module should be included in your job class.
3
5
  # You can define the following options after including this module:
@@ -32,7 +34,7 @@ module Exekutor
32
34
 
33
35
  # @private
34
36
  VALID_EXEKUTOR_OPTIONS = %i[queue_timeout execution_timeout].freeze
35
- private_constant "VALID_EXEKUTOR_OPTIONS"
37
+ private_constant :VALID_EXEKUTOR_OPTIONS
36
38
 
37
39
  # @return [Hash<Symbol, Object>] the exekutor options for this job
38
40
  attr_reader :exekutor_options
@@ -67,28 +69,35 @@ module Exekutor
67
69
  end
68
70
 
69
71
  # Validates the exekutor job options passed to {#exekutor_options} and +#set+
70
- # @param options [Hash<Symbol, Object>] the options to validate
72
+ # @param options [Hash<Symbol, Any>] the options to validate
71
73
  # @raise [InvalidOption] if any of the options are invalid
72
74
  # @private
73
75
  # @return [void]
74
76
  def validate_exekutor_options!(options)
75
- return unless options.present?
77
+ return true if options.blank?
76
78
 
77
- invalid_options = options.keys - VALID_EXEKUTOR_OPTIONS
78
- if invalid_options.present?
79
+ if (invalid_options = options.keys - VALID_EXEKUTOR_OPTIONS).present?
79
80
  raise InvalidOption, "Invalid option#{"s" if invalid_options.many?}: " \
80
- "#{invalid_options.map(&:inspect).join(", ")}. " \
81
- "Valid options are: #{VALID_EXEKUTOR_OPTIONS.map(&:inspect).join(", ")}"
82
- end
83
- if options[:queue_timeout]
84
- raise InvalidOption, ":queue_timeout must be an interval" unless options[:queue_timeout].is_a? ActiveSupport::Duration
85
- end
86
- if options[:execution_timeout]
87
- raise InvalidOption, ":execution_timeout must be an interval" unless options[:execution_timeout].is_a? ActiveSupport::Duration
81
+ "#{invalid_options.map(&:inspect).join(", ")}. " \
82
+ "Valid options are: #{VALID_EXEKUTOR_OPTIONS.map(&:inspect).join(", ")}"
88
83
  end
84
+ JobOptions.validate_option_type! options, :queue_timeout, ::ActiveSupport::Duration
85
+ JobOptions.validate_option_type! options, :execution_timeout, ::ActiveSupport::Duration
86
+ true
89
87
  end
90
88
  end
91
89
 
90
+ # Validates the type of an option
91
+ # @param options [Hash<Symbol, Any>] the options
92
+ # @param name [Symbol] the name of the option to validate
93
+ # @param valid_type [Class] the valid type for the value
94
+ # @raise [InvalidOption] if the configured value is not an instance of +valid_type+
95
+ def self.validate_option_type!(options, name, valid_type)
96
+ return if options[name].nil? || options[name].is_a?(valid_type)
97
+
98
+ raise InvalidOption, ":#{name} must be an instance of #{valid_type.name} (given: #{options[name].class.name})"
99
+ end
100
+
92
101
  # Raised when invalid options are given
93
102
  class InvalidOption < ::Exekutor::Error; end
94
103
  end