good_job 4.22.1 → 4.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +89 -0
- data/README.md +20 -2
- data/app/charts/good_job/base_chart.rb +1 -0
- data/app/charts/good_job/latency_metric/base.rb +2 -1
- data/app/charts/good_job/latency_metric/execution.rb +1 -0
- data/app/charts/good_job/latency_metric/queue.rb +1 -0
- data/app/charts/good_job/latency_metric/total.rb +1 -0
- data/app/charts/good_job/latency_metric.rb +1 -0
- data/app/charts/good_job/performance_index_chart.rb +2 -1
- data/app/charts/good_job/performance_range.rb +5 -3
- data/app/charts/good_job/performance_show_chart.rb +1 -0
- data/app/charts/good_job/scheduled_by_queue_chart.rb +2 -1
- data/app/controllers/good_job/application_controller.rb +7 -4
- data/app/controllers/good_job/batches_controller.rb +1 -0
- data/app/controllers/good_job/cleaner_controller.rb +1 -0
- data/app/controllers/good_job/cron_entries_controller.rb +1 -0
- data/app/controllers/good_job/executions_controller.rb +12 -0
- data/app/controllers/good_job/frontends_controller.rb +9 -4
- data/app/controllers/good_job/jobs_controller.rb +1 -0
- data/app/controllers/good_job/metrics_controller.rb +4 -1
- data/app/controllers/good_job/pauses_controller.rb +1 -0
- data/app/controllers/good_job/performance_controller.rb +1 -0
- data/app/controllers/good_job/processes_controller.rb +1 -0
- data/app/filters/good_job/base_filter.rb +10 -1
- data/app/filters/good_job/batches_filter.rb +1 -0
- data/app/filters/good_job/jobs_filter.rb +4 -1
- data/app/helpers/good_job/application_helper.rb +1 -0
- data/app/helpers/good_job/icons_helper.rb +1 -0
- data/app/models/concerns/good_job/advisory_lockable.rb +23 -5
- data/app/models/concerns/good_job/error_events.rb +1 -0
- data/app/models/concerns/good_job/filterable.rb +1 -0
- data/app/models/concerns/good_job/filtered_arguments.rb +28 -0
- data/app/models/concerns/good_job/reportable.rb +1 -0
- data/app/models/good_job/active_record_parent_class.rb +1 -0
- data/app/models/good_job/base_record.rb +1 -0
- data/app/models/good_job/batch.rb +17 -16
- data/app/models/good_job/batch_record.rb +10 -8
- data/app/models/good_job/concurrency_claim.rb +3 -0
- data/app/models/good_job/cron_entry.rb +10 -11
- data/app/models/good_job/discrete_execution.rb +1 -0
- data/app/models/good_job/execution.rb +8 -3
- data/app/models/good_job/execution_result.rb +1 -0
- data/app/models/good_job/i18n_config.rb +1 -0
- data/app/models/good_job/job/lockable.rb +68 -42
- data/app/models/good_job/job.rb +22 -16
- data/app/models/good_job/process.rb +5 -5
- data/app/models/good_job/setting.rb +5 -2
- data/app/views/good_job/executions/show.html.erb +11 -0
- data/app/views/good_job/jobs/_executions.erb +2 -6
- data/app/views/good_job/jobs/show.html.erb +1 -1
- data/config/brakeman.ignore +9 -9
- data/config/locales/pt-BR.yml +25 -23
- data/config/routes.rb +3 -1
- data/lib/active_job/queue_adapters/good_job_adapter.rb +1 -0
- data/lib/generators/good_job/install_generator.rb +1 -0
- data/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb +1 -0
- data/lib/generators/good_job/templates/update/migrations/01_create_good_jobs.rb.erb +1 -0
- data/lib/generators/good_job/templates/update/migrations/22_add_index_good_job_executions_on_created_at.rb.erb +12 -0
- data/lib/generators/good_job/update_generator.rb +1 -0
- data/lib/good_job/active_job_extensions/batches.rb +1 -0
- data/lib/good_job/active_job_extensions/concurrency.rb +1 -0
- data/lib/good_job/active_job_extensions/interrupt_errors.rb +1 -0
- data/lib/good_job/active_job_extensions/labels.rb +1 -0
- data/lib/good_job/active_job_extensions/notify_options.rb +1 -0
- data/lib/good_job/adapter/inline_buffer.rb +1 -0
- data/lib/good_job/adapter.rb +52 -59
- data/lib/good_job/bulk.rb +1 -0
- data/lib/good_job/callable.rb +1 -0
- data/lib/good_job/capsule.rb +12 -8
- data/lib/good_job/capsule_tracker.rb +35 -17
- data/lib/good_job/cleanup_tracker.rb +1 -0
- data/lib/good_job/cli.rb +5 -0
- data/lib/good_job/configuration/validator.rb +1 -0
- data/lib/good_job/configuration.rb +13 -0
- data/lib/good_job/cron_manager.rb +6 -6
- data/lib/good_job/current_thread.rb +2 -1
- data/lib/good_job/daemon.rb +1 -0
- data/lib/good_job/dependencies.rb +1 -0
- data/lib/good_job/engine.rb +3 -1
- data/lib/good_job/instance_registry.rb +48 -0
- data/lib/good_job/interrupt_error.rb +1 -0
- data/lib/good_job/interrupted_error.rb +1 -0
- data/lib/good_job/job_performer/metrics.rb +1 -0
- data/lib/good_job/job_performer.rb +7 -5
- data/lib/good_job/lifecycle_hooks.rb +1 -0
- data/lib/good_job/log_subscriber.rb +1 -0
- data/lib/good_job/multi_scheduler.rb +1 -0
- data/lib/good_job/notifier/process_heartbeat.rb +1 -0
- data/lib/good_job/notifier.rb +6 -6
- data/lib/good_job/poller.rb +5 -6
- data/lib/good_job/probe_server/cluster_healthcheck_middleware.rb +1 -0
- data/lib/good_job/probe_server/healthcheck_middleware.rb +1 -0
- data/lib/good_job/probe_server/not_found_app.rb +1 -0
- data/lib/good_job/probe_server/simple_handler.rb +51 -11
- data/lib/good_job/probe_server/webrick_handler.rb +3 -0
- data/lib/good_job/probe_server.rb +3 -0
- data/lib/good_job/ractors.rb +3 -0
- data/lib/good_job/safe_state.rb +1 -0
- data/lib/good_job/scheduler.rb +4 -6
- data/lib/good_job/sd_notify.rb +1 -0
- data/lib/good_job/shared_executor.rb +5 -6
- data/lib/good_job/subprocess.rb +1 -0
- data/lib/good_job/supervisor.rb +22 -15
- data/lib/good_job/systemd_service.rb +2 -0
- data/lib/good_job/thread_status.rb +1 -0
- data/lib/good_job/version.rb +2 -1
- data/lib/good_job.rb +60 -23
- metadata +7 -2
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# typed: true
|
|
1
2
|
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
module GoodJob
|
|
@@ -109,22 +110,22 @@ module GoodJob
|
|
|
109
110
|
|
|
110
111
|
begin
|
|
111
112
|
if build_result[:bulkable].any?
|
|
112
|
-
Job.transaction(requires_new: true, joinable: false) do
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
end
|
|
123
|
-
case lock_strategy
|
|
124
|
-
when :advisory, :hybrid
|
|
125
|
-
inline_jobs.each(&:advisory_lock!)
|
|
126
|
-
end
|
|
113
|
+
persisted_jobs, inline_jobs = Job.transaction(requires_new: true, joinable: false) do
|
|
114
|
+
inserted_jobs = _insert_jobs(build_result[:bulkable], build_result[:active_jobs_by_job_id])
|
|
115
|
+
next [inserted_jobs, []] unless execute_inline
|
|
116
|
+
|
|
117
|
+
jobs_to_inline = inserted_jobs.select { |job| job.scheduled_at.nil? || job.scheduled_at <= current_time }
|
|
118
|
+
if lock_strategy != :advisory && lock_id && jobs_to_inline.any?
|
|
119
|
+
Job.where(id: jobs_to_inline.map(&:id)).update_all( # rubocop:disable Rails/SkipsModelValidations
|
|
120
|
+
locked_by_id: lock_id, locked_at: current_time, lock_type: Job.lock_types[lock_strategy.to_s]
|
|
121
|
+
)
|
|
122
|
+
jobs_to_inline.each { |j| j.assign_attributes(locked_by_id: lock_id, locked_at: current_time, lock_type: lock_strategy) }
|
|
127
123
|
end
|
|
124
|
+
case lock_strategy
|
|
125
|
+
when :advisory, :hybrid
|
|
126
|
+
jobs_to_inline.each(&:advisory_lock!)
|
|
127
|
+
end
|
|
128
|
+
[inserted_jobs, jobs_to_inline]
|
|
128
129
|
end
|
|
129
130
|
end
|
|
130
131
|
|
|
@@ -444,7 +445,7 @@ module GoodJob
|
|
|
444
445
|
state = { queue_name: queue_name, count: grouped_jobs.size }
|
|
445
446
|
state[:scheduled_at] = scheduled_at if scheduled_at
|
|
446
447
|
|
|
447
|
-
executed_locally = adapter.respond_to?(:execute_async?) && adapter.execute_async? && GoodJob.capsule
|
|
448
|
+
executed_locally = adapter.respond_to?(:execute_async?) && adapter.execute_async? && GoodJob.capsule.create_thread(state)
|
|
448
449
|
unless executed_locally
|
|
449
450
|
state[:count] = grouped_jobs.count { |job| _send_notify?(active_jobs_by_job_id[job.active_job_id]) }
|
|
450
451
|
Notifier.notify(state) unless state[:count].zero?
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# typed: true
|
|
1
2
|
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
require 'active_job/arguments'
|
|
@@ -63,8 +64,8 @@ module GoodJob
|
|
|
63
64
|
if job_discarded && !discarded_at
|
|
64
65
|
update(discarded_at: Time.current)
|
|
65
66
|
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
discard_job_class = on_discard.presence&.constantize
|
|
68
|
+
if discard_job_class
|
|
68
69
|
Job.defer_after_commit_maybe(discard_job_class) do
|
|
69
70
|
Batch.within_thread(batch_id: nil, batch_callback_id: id) do
|
|
70
71
|
discard_job_class.set(priority: callback_priority, queue: callback_queue_name).perform_later(to_batch, { event: :discard })
|
|
@@ -76,8 +77,8 @@ module GoodJob
|
|
|
76
77
|
if enqueued_at && !(self.class.jobs_finished_at_migrated? ? jobs_finished_at : finished_at) && jobs.where(finished_at: nil).none?
|
|
77
78
|
self.class.jobs_finished_at_migrated? ? update(jobs_finished_at: Time.current) : update(finished_at: Time.current)
|
|
78
79
|
|
|
79
|
-
|
|
80
|
-
|
|
80
|
+
success_job_class = on_success.presence&.constantize unless discarded_at
|
|
81
|
+
if success_job_class
|
|
81
82
|
Job.defer_after_commit_maybe(success_job_class) do
|
|
82
83
|
Batch.within_thread(batch_id: nil, batch_callback_id: id) do
|
|
83
84
|
success_job_class.set(priority: callback_priority, queue: callback_queue_name).perform_later(to_batch, { event: :success })
|
|
@@ -85,11 +86,11 @@ module GoodJob
|
|
|
85
86
|
end
|
|
86
87
|
end
|
|
87
88
|
|
|
88
|
-
|
|
89
|
-
|
|
89
|
+
finish_job_class = on_finish.presence&.constantize
|
|
90
|
+
if finish_job_class
|
|
90
91
|
Job.defer_after_commit_maybe(finish_job_class) do
|
|
91
92
|
Batch.within_thread(batch_id: nil, batch_callback_id: id) do
|
|
92
|
-
|
|
93
|
+
finish_job_class.set(priority: callback_priority, queue: callback_queue_name).perform_later(to_batch, { event: :finish })
|
|
93
94
|
end
|
|
94
95
|
end
|
|
95
96
|
end
|
|
@@ -113,7 +114,8 @@ module GoodJob
|
|
|
113
114
|
end
|
|
114
115
|
|
|
115
116
|
if Rails.gem_version < Gem::Version.new('7.1.0.alpha')
|
|
116
|
-
|
|
117
|
+
# Positional coder argument; Sorbet only knows the Rails 7.1+ signature
|
|
118
|
+
public_send(:serialize, :serialized_properties, PropertySerializer, default: -> { {} }) # rubocop:disable Style/SendWithLiteralMethodName
|
|
117
119
|
else
|
|
118
120
|
serialize :serialized_properties, coder: PropertySerializer, default: -> { {} }
|
|
119
121
|
end
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# typed: true
|
|
1
2
|
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
module GoodJob
|
|
@@ -172,6 +173,8 @@ module GoodJob
|
|
|
172
173
|
end
|
|
173
174
|
|
|
174
175
|
def notify(job_id)
|
|
176
|
+
return unless GoodJob.configuration.enable_listen_notify
|
|
177
|
+
|
|
175
178
|
queue_name = GoodJob::Job.where(id: job_id).pick(:queue_name)
|
|
176
179
|
GoodJob::Notifier.notify({ queue_name: queue_name }) if queue_name
|
|
177
180
|
end
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# typed: true
|
|
1
2
|
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
require "concurrent/hash"
|
|
@@ -139,8 +140,10 @@ module GoodJob # :nodoc:
|
|
|
139
140
|
job_klass = resolve_job_class
|
|
140
141
|
next unless job_klass.is_a?(Class)
|
|
141
142
|
|
|
142
|
-
configured_job = job_klass.set(
|
|
143
|
-
|
|
143
|
+
configured_job = job_klass.set(resolve_set)
|
|
144
|
+
job_args = resolve_args
|
|
145
|
+
job_kwargs = resolve_kwargs
|
|
146
|
+
job_kwargs.present? ? configured_job.perform_later(*job_args, **job_kwargs) : configured_job.perform_later(*job_args)
|
|
144
147
|
end
|
|
145
148
|
end
|
|
146
149
|
end
|
|
@@ -213,7 +216,8 @@ module GoodJob # :nodoc:
|
|
|
213
216
|
end
|
|
214
217
|
|
|
215
218
|
def resolve_job_class
|
|
216
|
-
value =
|
|
219
|
+
value = job_class
|
|
220
|
+
value = value.call if value.respond_to?(:call)
|
|
217
221
|
value.is_a?(String) ? value.constantize : value
|
|
218
222
|
end
|
|
219
223
|
|
|
@@ -236,22 +240,17 @@ module GoodJob # :nodoc:
|
|
|
236
240
|
schedule if schedule.instance_of?(Fugit::Cron)
|
|
237
241
|
end
|
|
238
242
|
|
|
239
|
-
def
|
|
240
|
-
value = job_class || nil
|
|
241
|
-
value.respond_to?(:call) ? value.call : value
|
|
242
|
-
end
|
|
243
|
-
|
|
244
|
-
def set_value
|
|
243
|
+
def resolve_set
|
|
245
244
|
value = set || {}
|
|
246
245
|
value.respond_to?(:call) ? value.call : value
|
|
247
246
|
end
|
|
248
247
|
|
|
249
|
-
def
|
|
248
|
+
def resolve_args
|
|
250
249
|
value = args || []
|
|
251
250
|
value.respond_to?(:call) ? value.call : value
|
|
252
251
|
end
|
|
253
252
|
|
|
254
|
-
def
|
|
253
|
+
def resolve_kwargs
|
|
255
254
|
value = kwargs || nil
|
|
256
255
|
value.respond_to?(:call) ? value.call : value
|
|
257
256
|
end
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
# typed: true
|
|
1
2
|
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
module GoodJob # :nodoc:
|
|
4
5
|
class Execution < BaseRecord
|
|
5
6
|
include ErrorEvents
|
|
7
|
+
include FilteredArguments
|
|
6
8
|
|
|
7
9
|
self.table_name = 'good_job_executions'
|
|
8
10
|
self.implicit_order_column = 'created_at'
|
|
@@ -43,7 +45,7 @@ module GoodJob # :nodoc:
|
|
|
43
45
|
|
|
44
46
|
def status
|
|
45
47
|
if finished_at.present?
|
|
46
|
-
if error.present? && job
|
|
48
|
+
if error.present? && job&.finished_at.present?
|
|
47
49
|
:discarded
|
|
48
50
|
elsif error.present?
|
|
49
51
|
:retried
|
|
@@ -56,13 +58,16 @@ module GoodJob # :nodoc:
|
|
|
56
58
|
end
|
|
57
59
|
|
|
58
60
|
def interrupted_duration
|
|
59
|
-
|
|
61
|
+
interrupted_at = finished_at
|
|
62
|
+
started_at = performed_at
|
|
63
|
+
return unless error_event == "interrupted" && interrupted_at && started_at
|
|
60
64
|
|
|
61
|
-
|
|
65
|
+
interrupted_at - started_at
|
|
62
66
|
end
|
|
63
67
|
|
|
64
68
|
def display_serialized_params
|
|
65
69
|
serialized_params.merge({
|
|
70
|
+
"arguments" => display_arguments,
|
|
66
71
|
_good_job_execution: attributes.except('serialized_params'),
|
|
67
72
|
})
|
|
68
73
|
end
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# typed: false
|
|
1
2
|
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
module GoodJob
|
|
@@ -8,6 +9,11 @@ module GoodJob
|
|
|
8
9
|
module Lockable
|
|
9
10
|
extend ActiveSupport::Concern
|
|
10
11
|
|
|
12
|
+
# Seconds to wait between attempts to release a claim after a connection failure.
|
|
13
|
+
RELEASE_CLAIM_RETRY_INTERVAL = 5
|
|
14
|
+
# Log a warning on the first failed release and then every this many attempts.
|
|
15
|
+
RELEASE_CLAIM_RETRY_WARNING_EVERY = 10
|
|
16
|
+
|
|
11
17
|
class_methods do
|
|
12
18
|
# Claims a job by acquiring a session-level advisory lock via the CTE
|
|
13
19
|
# query, and records the lock acquisition for phantom re-entrancy cleanup.
|
|
@@ -20,13 +26,17 @@ module GoodJob
|
|
|
20
26
|
def with_advisory_lock_claim(select_limit: nil, &block)
|
|
21
27
|
raise ArgumentError, "Must provide a block" unless block
|
|
22
28
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
29
|
+
record = nil
|
|
30
|
+
begin
|
|
31
|
+
advisory_lock(select_limit: select_limit) do |records|
|
|
32
|
+
record = records.first
|
|
26
33
|
yield(record)
|
|
27
|
-
ensure
|
|
28
|
-
record&.run_callbacks(:perform_unlocked)
|
|
29
34
|
end
|
|
35
|
+
ensure
|
|
36
|
+
# Runs after the advisory lock is released. A failed completion leaves the
|
|
37
|
+
# row lock in the database and may leave unsaved completion fields on the
|
|
38
|
+
# record, which batch callbacks would act on.
|
|
39
|
+
record.run_callbacks(:perform_unlocked) if record && (record.destroyed? || record.locked_by_id_in_database.nil?)
|
|
30
40
|
end
|
|
31
41
|
end
|
|
32
42
|
|
|
@@ -40,30 +50,7 @@ module GoodJob
|
|
|
40
50
|
# @yield [GoodJob::Job, nil] The claimed record, or nil if none found
|
|
41
51
|
# @return [Object] The return value of the block
|
|
42
52
|
def with_skip_locked_claim(locked_by_id:, locked_at:, lock_type:, &block)
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
candidate_sql = select(:id).lock("FOR NO KEY UPDATE SKIP LOCKED").to_sql
|
|
46
|
-
materialized = supports_cte_materialization_specifiers? ? "MATERIALIZED " : ""
|
|
47
|
-
|
|
48
|
-
sql = <<~SQL.squish
|
|
49
|
-
WITH candidate AS #{materialized}(#{candidate_sql})
|
|
50
|
-
UPDATE #{quoted_table_name}
|
|
51
|
-
SET locked_by_id = ?,
|
|
52
|
-
locked_at = ?,
|
|
53
|
-
lock_type = ?
|
|
54
|
-
FROM candidate
|
|
55
|
-
WHERE #{quoted_table_name}.id = candidate.id
|
|
56
|
-
RETURNING #{quoted_table_name}.*
|
|
57
|
-
SQL
|
|
58
|
-
|
|
59
|
-
record = unscoped.find_by_sql([sql, locked_by_id, locked_at, lock_types[lock_type.to_s]]).first
|
|
60
|
-
|
|
61
|
-
begin
|
|
62
|
-
unscoped { yield(record) }
|
|
63
|
-
ensure
|
|
64
|
-
record.update(locked_by_id: nil, locked_at: nil, lock_type: nil) if record && !record.destroyed? && record.locked_by_id.present?
|
|
65
|
-
record&.run_callbacks(:perform_unlocked)
|
|
66
|
-
end
|
|
53
|
+
with_row_lock_claim(locked_by_id: locked_by_id, locked_at: locked_at, lock_type: lock_type, &block)
|
|
67
54
|
end
|
|
68
55
|
|
|
69
56
|
# Atomically claims a job using SELECT FOR NO KEY UPDATE SKIP LOCKED with an
|
|
@@ -77,13 +64,21 @@ module GoodJob
|
|
|
77
64
|
# @yield [GoodJob::Job, nil] The claimed record, or nil if none found
|
|
78
65
|
# @return [Object] The return value of the block
|
|
79
66
|
def with_hybrid_lock_claim(locked_by_id:, locked_at:, lock_type:, &block)
|
|
80
|
-
|
|
67
|
+
with_row_lock_claim(locked_by_id: locked_by_id, locked_at: locked_at, lock_type: lock_type, advisory: true, &block)
|
|
68
|
+
end
|
|
81
69
|
|
|
82
|
-
|
|
70
|
+
private
|
|
83
71
|
|
|
72
|
+
def with_row_lock_claim(locked_by_id:, locked_at:, lock_type:, advisory: false, &block)
|
|
73
|
+
raise ArgumentError, "Must provide a block" unless block
|
|
74
|
+
|
|
75
|
+
connection = lease_connection if advisory # retain the session holding the advisory lock
|
|
84
76
|
candidate_sql = select(:id).lock("FOR NO KEY UPDATE SKIP LOCKED").to_sql
|
|
85
|
-
advisory_lock_expr = "('x' || substr(md5(#{_quoted_table_name_string} || '-' || id::text), 1, 16))::bit(64)::bigint"
|
|
86
77
|
materialized = supports_cte_materialization_specifiers? ? "MATERIALIZED " : ""
|
|
78
|
+
candidate_filter = if advisory
|
|
79
|
+
advisory_key = advisory_lock_bigint_sql("#{_quoted_table_name_string} || '-' || id::text")
|
|
80
|
+
"WHERE pg_try_advisory_lock(#{advisory_key})"
|
|
81
|
+
end
|
|
87
82
|
|
|
88
83
|
sql = <<~SQL.squish
|
|
89
84
|
WITH candidate AS #{materialized}(#{candidate_sql})
|
|
@@ -91,26 +86,57 @@ module GoodJob
|
|
|
91
86
|
SET locked_by_id = ?,
|
|
92
87
|
locked_at = ?,
|
|
93
88
|
lock_type = ?
|
|
94
|
-
FROM (
|
|
95
|
-
SELECT id FROM candidate
|
|
96
|
-
WHERE pg_try_advisory_lock(#{advisory_lock_expr})
|
|
97
|
-
) AS locked
|
|
89
|
+
FROM (SELECT id FROM candidate #{candidate_filter}) AS locked
|
|
98
90
|
WHERE #{quoted_table_name}.id = locked.id
|
|
99
91
|
RETURNING #{quoted_table_name}.*
|
|
100
92
|
SQL
|
|
101
93
|
|
|
102
94
|
record = unscoped.find_by_sql([sql, locked_by_id, locked_at, lock_types[lock_type.to_s]]).first
|
|
103
|
-
record_advisory_lock(
|
|
95
|
+
record_advisory_lock(connection, record.lockable_column_key, cte: true) if advisory && record
|
|
104
96
|
|
|
97
|
+
completed = false
|
|
105
98
|
begin
|
|
106
|
-
unscoped { yield(record) }
|
|
99
|
+
result = unscoped { yield(record) }
|
|
100
|
+
completed = true
|
|
101
|
+
result
|
|
107
102
|
ensure
|
|
108
|
-
|
|
109
|
-
record.advisory_unlock
|
|
110
|
-
|
|
103
|
+
begin
|
|
104
|
+
record.advisory_unlock(connection: connection) if advisory && record
|
|
105
|
+
ensure
|
|
106
|
+
# A failed completion may have cleared the attributes in memory only.
|
|
107
|
+
if record && (!completed || (!record.destroyed? && record.locked_by_id_in_database.present?))
|
|
108
|
+
released = release_lock_claim(record, locked_by_id: locked_by_id, locked_at: locked_at)
|
|
109
|
+
completed &&= released
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
record&.run_callbacks(:perform_unlocked) if completed
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def release_lock_claim(record, locked_by_id:, locked_at:)
|
|
117
|
+
transaction do
|
|
118
|
+
# Match the original claim, including its time, so a newer execution is untouched.
|
|
119
|
+
released = unscoped.where(id: record.id, locked_by_id: locked_by_id, locked_at: locked_at)
|
|
120
|
+
.update_all(locked_by_id: nil, locked_at: nil, lock_type: nil) # rubocop:disable Rails/SkipsModelValidations
|
|
121
|
+
if released.positive?
|
|
122
|
+
GoodJob::ConcurrencyClaim.release_job(record.id) if GoodJob::ConcurrencyClaim.table_exists?
|
|
123
|
+
record.reload
|
|
111
124
|
end
|
|
112
|
-
|
|
125
|
+
released.positive?
|
|
113
126
|
end
|
|
127
|
+
rescue StandardError => e
|
|
128
|
+
connection_errors = %w[ActiveRecord::ConnectionNotEstablished ActiveRecord::ConnectionFailed PG::ConnectionBad PG::UnableToSend].filter_map(&:safe_constantize)
|
|
129
|
+
cause = e.is_a?(ActiveRecord::StatementInvalid) ? e.cause : nil # older Rails wraps PG connection errors
|
|
130
|
+
raise unless connection_errors.any? { |klass| e.is_a?(klass) || cause.is_a?(klass) }
|
|
131
|
+
# Once the process is exiting, its stale process record lets cleanup release the claim.
|
|
132
|
+
return false if Thread.current.status == "aborting" || GoodJob.current_thread_shutting_down?
|
|
133
|
+
|
|
134
|
+
# Keep the worker registered until its claim can be released. Otherwise a
|
|
135
|
+
# live process can keep the abandoned job locked indefinitely.
|
|
136
|
+
attempts = (attempts || 0) + 1
|
|
137
|
+
GoodJob.logger.warn("GoodJob could not release the claim on job #{record.id} (attempt #{attempts}): #{e.class}: #{e.message}. Retrying every #{RELEASE_CLAIM_RETRY_INTERVAL} seconds.") if attempts == 1 || (attempts % RELEASE_CLAIM_RETRY_WARNING_EVERY).zero?
|
|
138
|
+
sleep RELEASE_CLAIM_RETRY_INTERVAL
|
|
139
|
+
retry
|
|
114
140
|
end
|
|
115
141
|
end
|
|
116
142
|
end
|
data/app/models/good_job/job.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# typed: true
|
|
1
2
|
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
module GoodJob
|
|
@@ -6,6 +7,7 @@ module GoodJob
|
|
|
6
7
|
include AdvisoryLockable
|
|
7
8
|
include Job::Lockable
|
|
8
9
|
include ErrorEvents
|
|
10
|
+
include FilteredArguments
|
|
9
11
|
include Filterable
|
|
10
12
|
include Reportable
|
|
11
13
|
|
|
@@ -71,7 +73,7 @@ module GoodJob
|
|
|
71
73
|
has_many :executions, class_name: 'GoodJob::Execution', foreign_key: 'active_job_id', primary_key: "id", inverse_of: :job, dependent: :delete_all
|
|
72
74
|
has_many :concurrency_claims, class_name: 'GoodJob::ConcurrencyClaim', primary_key: :id, inverse_of: false, dependent: nil
|
|
73
75
|
|
|
74
|
-
before_create -> {
|
|
76
|
+
before_create ->(job) { job.id = job.active_job_id }, if: -> { active_job_id.present? }
|
|
75
77
|
|
|
76
78
|
# Get Jobs finished before the given timestamp.
|
|
77
79
|
# @!method finished_before(timestamp)
|
|
@@ -581,7 +583,7 @@ module GoodJob
|
|
|
581
583
|
when String
|
|
582
584
|
klass.constantize
|
|
583
585
|
when Job
|
|
584
|
-
klass.job_class
|
|
586
|
+
klass.job_class&.constantize
|
|
585
587
|
else
|
|
586
588
|
klass
|
|
587
589
|
end
|
|
@@ -622,6 +624,7 @@ module GoodJob
|
|
|
622
624
|
# @return [Hash]
|
|
623
625
|
def display_serialized_params
|
|
624
626
|
serialized_params.merge({
|
|
627
|
+
"arguments" => display_arguments,
|
|
625
628
|
_good_job: attributes.except('serialized_params', 'locktype', 'owns_advisory_lock'),
|
|
626
629
|
})
|
|
627
630
|
end
|
|
@@ -683,13 +686,14 @@ module GoodJob
|
|
|
683
686
|
active_job.executions = (active_job.executions || 0) + 1
|
|
684
687
|
|
|
685
688
|
begin
|
|
686
|
-
error_class, error_message = error.split(ERROR_MESSAGE_SEPARATOR).map(&:strip)
|
|
687
|
-
error = error_class.constantize.new(error_message)
|
|
689
|
+
error_class, error_message = error.to_s.split(ERROR_MESSAGE_SEPARATOR).map(&:strip)
|
|
690
|
+
error = error_class ? error_class.constantize.new(error_message) : StandardError.new(error)
|
|
688
691
|
rescue StandardError
|
|
689
692
|
error = StandardError.new(error)
|
|
690
693
|
end
|
|
691
694
|
|
|
692
|
-
|
|
695
|
+
# Populated by a block that may be deferred until after the transaction commits
|
|
696
|
+
new_active_jobs = []
|
|
693
697
|
|
|
694
698
|
transaction do
|
|
695
699
|
Job.defer_after_commit_maybe(active_job.class) do
|
|
@@ -699,7 +703,7 @@ module GoodJob
|
|
|
699
703
|
|
|
700
704
|
# NOTE: I18n.with_locale necessary until fixed in rails https://github.com/rails/rails/pull/52121
|
|
701
705
|
I18n.with_locale(active_job.locale) do
|
|
702
|
-
|
|
706
|
+
new_active_jobs << active_job.retry_job(wait: 0, error: error)
|
|
703
707
|
end
|
|
704
708
|
end
|
|
705
709
|
end
|
|
@@ -707,7 +711,7 @@ module GoodJob
|
|
|
707
711
|
save!
|
|
708
712
|
end
|
|
709
713
|
|
|
710
|
-
|
|
714
|
+
new_active_jobs.first
|
|
711
715
|
end
|
|
712
716
|
end
|
|
713
717
|
end
|
|
@@ -778,8 +782,7 @@ module GoodJob
|
|
|
778
782
|
|
|
779
783
|
job_performed_at = Time.current
|
|
780
784
|
monotonic_start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
781
|
-
execution =
|
|
782
|
-
result = GoodJob::CurrentThread.within do |current_thread|
|
|
785
|
+
execution, result = GoodJob::CurrentThread.within do |current_thread|
|
|
783
786
|
current_thread.reset
|
|
784
787
|
current_thread.job = self
|
|
785
788
|
|
|
@@ -792,16 +795,16 @@ module GoodJob
|
|
|
792
795
|
self.error_event = ErrorEvents::INTERRUPTED
|
|
793
796
|
monotonic_duration = (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - monotonic_start).seconds
|
|
794
797
|
|
|
795
|
-
|
|
798
|
+
interrupted_execution_attrs = {
|
|
796
799
|
error: interrupt_error_string,
|
|
797
800
|
finished_at: job_performed_at,
|
|
798
801
|
error_event: error_event,
|
|
799
802
|
duration: monotonic_duration,
|
|
800
803
|
}
|
|
801
|
-
executions.where(finished_at: nil).where.not(performed_at: nil).update_all(
|
|
804
|
+
executions.where(finished_at: nil).where.not(performed_at: nil).update_all(interrupted_execution_attrs) # rubocop:disable Rails/SkipsModelValidations
|
|
802
805
|
end
|
|
803
806
|
|
|
804
|
-
transaction do
|
|
807
|
+
new_execution = transaction do
|
|
805
808
|
execution_attrs = {
|
|
806
809
|
job_class: job_class,
|
|
807
810
|
queue_name: queue_name,
|
|
@@ -820,11 +823,12 @@ module GoodJob
|
|
|
820
823
|
job_attrs[:lock_type] = :advisory if self.class.lock_type_column_exists?
|
|
821
824
|
end
|
|
822
825
|
|
|
823
|
-
|
|
826
|
+
created_execution = executions.create!(execution_attrs)
|
|
824
827
|
update!(job_attrs)
|
|
828
|
+
created_execution
|
|
825
829
|
end
|
|
826
830
|
|
|
827
|
-
ActiveSupport::Notifications.instrument("perform_job.good_job", { job: self, execution:
|
|
831
|
+
execution_result = ActiveSupport::Notifications.instrument("perform_job.good_job", { job: self, execution: new_execution, process_id: current_thread.process_id, thread_name: current_thread.thread_name }) do |instrument_payload|
|
|
828
832
|
value = ActiveJob::Base.execute(active_job_data)
|
|
829
833
|
active_job = current_thread.active_job
|
|
830
834
|
|
|
@@ -870,6 +874,8 @@ module GoodJob
|
|
|
870
874
|
)
|
|
871
875
|
ExecutionResult.new(value: nil, active_job: active_job, unhandled_error: e, error_event: error_event)
|
|
872
876
|
end
|
|
877
|
+
|
|
878
|
+
[new_execution, execution_result]
|
|
873
879
|
end
|
|
874
880
|
|
|
875
881
|
job_attributes = { locked_by_id: nil, locked_at: nil }
|
|
@@ -1017,8 +1023,8 @@ module GoodJob
|
|
|
1017
1023
|
end
|
|
1018
1024
|
|
|
1019
1025
|
def continue_discard_or_finish_batch
|
|
1020
|
-
batch
|
|
1021
|
-
callback_batch
|
|
1026
|
+
batch&._continue_discard_or_finish(self)
|
|
1027
|
+
callback_batch&._continue_discard_or_finish
|
|
1022
1028
|
end
|
|
1023
1029
|
|
|
1024
1030
|
def active_job_data
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# typed: true
|
|
1
2
|
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
require 'socket'
|
|
@@ -28,7 +29,7 @@ module GoodJob # :nodoc:
|
|
|
28
29
|
end
|
|
29
30
|
when /darwin|bsd/
|
|
30
31
|
lambda do |pid|
|
|
31
|
-
`ps -o pid,rss -p #{pid.to_i}`.lines.last
|
|
32
|
+
`ps -o pid,rss -p #{pid.to_i}`.lines.last&.split&.last.to_i
|
|
32
33
|
end
|
|
33
34
|
else
|
|
34
35
|
->(_pid) { 0 }
|
|
@@ -92,12 +93,11 @@ module GoodJob # :nodoc:
|
|
|
92
93
|
}
|
|
93
94
|
attributes[:lock_type] = :advisory if with_advisory_lock
|
|
94
95
|
|
|
95
|
-
record = nil
|
|
96
96
|
transaction do
|
|
97
97
|
record = create!(attributes)
|
|
98
98
|
record.advisory_lock!(connection: advisory_lock_connection) if with_advisory_lock
|
|
99
|
+
record
|
|
99
100
|
end
|
|
100
|
-
record
|
|
101
101
|
rescue ActiveRecord::RecordNotUnique
|
|
102
102
|
find_by(id: id).tap do |existing_record|
|
|
103
103
|
next unless existing_record
|
|
@@ -119,7 +119,7 @@ module GoodJob # :nodoc:
|
|
|
119
119
|
proctitle: $PROGRAM_NAME,
|
|
120
120
|
preserve_job_records: GoodJob.preserve_job_records,
|
|
121
121
|
retry_on_unhandled_error: GoodJob.retry_on_unhandled_error,
|
|
122
|
-
schedulers: GoodJob::Scheduler.instances.map(&:stats),
|
|
122
|
+
schedulers: GoodJob::Scheduler.instances.reject(&:shutdown?).map(&:stats),
|
|
123
123
|
cron_enabled: GoodJob.configuration.enable_cron?,
|
|
124
124
|
total_succeeded_executions_count: GoodJob::Scheduler.instances.sum { |scheduler| scheduler.stats.fetch(:succeeded_executions_count) },
|
|
125
125
|
total_errored_executions_count: GoodJob::Scheduler.instances.sum { |scheduler| scheduler.stats.fetch(:errored_executions_count) },
|
|
@@ -136,7 +136,7 @@ module GoodJob # :nodoc:
|
|
|
136
136
|
update(state: state, updated_at: Time.current)
|
|
137
137
|
rescue ActiveRecord::RecordNotFound
|
|
138
138
|
@new_record = true
|
|
139
|
-
|
|
139
|
+
assign_attributes(created_at: nil, updated_at: nil)
|
|
140
140
|
id_will_change!
|
|
141
141
|
state_will_change!
|
|
142
142
|
save
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# typed: true
|
|
1
2
|
# frozen_string_literal: true
|
|
2
3
|
|
|
3
4
|
module GoodJob
|
|
@@ -38,7 +39,8 @@ module GoodJob
|
|
|
38
39
|
enabled_setting.save!
|
|
39
40
|
|
|
40
41
|
disabled_setting = GoodJob::Setting.find_by(key: CRON_KEYS_DISABLED)
|
|
41
|
-
return unless disabled_setting
|
|
42
|
+
return unless disabled_setting
|
|
43
|
+
return unless disabled_setting.value&.include?(key_string)
|
|
42
44
|
|
|
43
45
|
disabled_setting.value.delete(key_string)
|
|
44
46
|
disabled_setting.save!
|
|
@@ -47,7 +49,8 @@ module GoodJob
|
|
|
47
49
|
def self.cron_key_disable(key)
|
|
48
50
|
enabled_setting = GoodJob::Setting.find_by(key: CRON_KEYS_ENABLED)
|
|
49
51
|
key_string = key.to_s
|
|
50
|
-
|
|
52
|
+
enabled_keys = enabled_setting&.value
|
|
53
|
+
if enabled_setting && enabled_keys&.include?(key_string)
|
|
51
54
|
enabled_setting.value.delete(key_string)
|
|
52
55
|
enabled_setting.save!
|
|
53
56
|
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
<%= content_tag "turbo-frame", id: dom_id(@execution, :full_trace) do %>
|
|
2
|
+
<div class="small">
|
|
3
|
+
<code class="text-wrap text-break m-0 text-secondary-emphasis">
|
|
4
|
+
<%= safe_join(@execution.error_backtrace || [], tag.br) %>
|
|
5
|
+
</code>
|
|
6
|
+
</div>
|
|
7
|
+
<% end %>
|
|
8
|
+
|
|
9
|
+
<%= content_tag "turbo-frame", id: dom_id(@execution, :serialized_params) do %>
|
|
10
|
+
<%= tag.pre JSON.pretty_generate(truncate_display_value(@execution.display_serialized_params)) %>
|
|
11
|
+
<% end %>
|
|
@@ -58,11 +58,7 @@
|
|
|
58
58
|
</div>
|
|
59
59
|
<% end %>
|
|
60
60
|
<%= tag.div class: "tab-pane fade", id: dom_id(execution, :full_pane), role: "tabpane", aria: { labelledby: dom_id(execution, :full) }, tabindex: 0 do %>
|
|
61
|
-
|
|
62
|
-
<code class="text-wrap text-break m-0 text-secondary-emphasis">
|
|
63
|
-
<%= safe_join(execution.error_backtrace, tag.br) %>
|
|
64
|
-
</code>
|
|
65
|
-
</div>
|
|
61
|
+
<%= content_tag "turbo-frame", "", id: dom_id(execution, :full_trace), src: job_execution_path(@job.id, execution.id), loading: "lazy" %>
|
|
66
62
|
<% end %>
|
|
67
63
|
<% end %>
|
|
68
64
|
<% end %>
|
|
@@ -70,7 +66,7 @@
|
|
|
70
66
|
<% end %>
|
|
71
67
|
<%= render 'good_job/custom_execution_details', execution: execution, job: @job %>
|
|
72
68
|
<%= tag.div id: dom_id(execution, "params"), class: "list-group-item collapse small bg-dark text-light" do %>
|
|
73
|
-
<%=
|
|
69
|
+
<%= content_tag "turbo-frame", "", id: dom_id(execution, :serialized_params), src: job_execution_path(@job.id, execution.id), loading: "lazy" %>
|
|
74
70
|
<% end %>
|
|
75
71
|
<% end %>
|
|
76
72
|
</div>
|
|
@@ -85,7 +85,7 @@
|
|
|
85
85
|
<% end %>
|
|
86
86
|
</h5>
|
|
87
87
|
</div>
|
|
88
|
-
<%= tag.pre @job.
|
|
88
|
+
<%= tag.pre @job.display_arguments.map { |argument| truncate_display_value(argument).inspect }.join(', '), class: 'text-wrap text-break' %>
|
|
89
89
|
|
|
90
90
|
<%= tag.div id: dom_id(@job, "params"), class: "list-group-item collapse small bg-dark text-light" do %>
|
|
91
91
|
<%= tag.pre JSON.pretty_generate(truncate_display_value(@job.display_serialized_params)) %>
|