good_job 1.9.2 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +98 -1
- data/README.md +17 -0
- data/engine/app/assets/vendor/bootstrap/bootstrap.bundle.min.js +7 -0
- data/engine/app/assets/vendor/bootstrap/bootstrap.min.css +7 -0
- data/engine/app/controllers/good_job/assets_controller.rb +29 -0
- data/engine/app/controllers/good_job/dashboards_controller.rb +8 -6
- data/engine/app/controllers/good_job/jobs_controller.rb +9 -0
- data/engine/app/views/good_job/dashboards/index.html.erb +1 -1
- data/engine/app/views/layouts/good_job/base.html.erb +21 -12
- data/engine/app/views/shared/_chart.erb +3 -2
- data/engine/app/views/shared/_jobs_table.erb +13 -1
- data/engine/app/views/shared/icons/_check.html.erb +4 -0
- data/engine/app/views/shared/icons/_exclamation.html.erb +4 -0
- data/engine/app/views/shared/icons/_trash.html.erb +5 -0
- data/engine/config/routes.rb +10 -1
- data/lib/generators/good_job/install_generator.rb +5 -15
- data/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb +27 -0
- data/lib/generators/good_job/templates/{migration.rb.erb → update/migrations/01_create_good_jobs.rb} +3 -3
- data/lib/generators/good_job/templates/update/migrations/02_add_active_job_id_concurrency_key_cron_key_to_good_jobs.rb +15 -0
- data/lib/generators/good_job/templates/update/migrations/03_add_active_job_id_index_and_concurrency_key_index_to_good_jobs.rb +32 -0
- data/lib/generators/good_job/update_generator.rb +29 -0
- data/lib/good_job.rb +12 -8
- data/lib/good_job/adapter.rb +15 -11
- data/lib/good_job/configuration.rb +3 -3
- data/lib/good_job/current_execution.rb +0 -1
- data/lib/good_job/daemon.rb +6 -0
- data/lib/good_job/job.rb +38 -7
- data/lib/good_job/job_performer.rb +2 -2
- data/lib/good_job/lockable.rb +109 -56
- data/lib/good_job/log_subscriber.rb +15 -14
- data/lib/good_job/multi_scheduler.rb +9 -0
- data/lib/good_job/notifier.rb +4 -2
- data/lib/good_job/poller.rb +16 -7
- data/lib/good_job/scheduler.rb +12 -6
- data/lib/good_job/version.rb +1 -1
- metadata +28 -5
- data/engine/app/assets/vendor/bootstrap/bootstrap-native.js +0 -1662
- data/engine/app/assets/vendor/bootstrap/bootstrap.css +0 -10258
data/lib/generators/good_job/templates/{migration.rb.erb → update/migrations/01_create_good_jobs.rb}
RENAMED
@@ -1,4 +1,4 @@
|
|
1
|
-
class CreateGoodJobs < ActiveRecord::Migration
|
1
|
+
class CreateGoodJobs < ActiveRecord::Migration[5.2]
|
2
2
|
def change
|
3
3
|
enable_extension 'pgcrypto'
|
4
4
|
|
@@ -14,7 +14,7 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
|
|
14
14
|
t.timestamps
|
15
15
|
end
|
16
16
|
|
17
|
-
add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)"
|
18
|
-
add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)"
|
17
|
+
add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: "index_good_jobs_on_scheduled_at"
|
18
|
+
add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at
|
19
19
|
end
|
20
20
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class AddActiveJobIdConcurrencyKeyCronKeyToGoodJobs < ActiveRecord::Migration[5.2]
|
2
|
+
def change
|
3
|
+
reversible do |dir|
|
4
|
+
dir.up do
|
5
|
+
# Ensure this incremental update migration is idempotent
|
6
|
+
# with monolithic install migration.
|
7
|
+
return if connection.column_exists?(:good_jobs, :active_job_id)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
add_column :good_jobs, :active_job_id, :uuid
|
12
|
+
add_column :good_jobs, :concurrency_key, :text
|
13
|
+
add_column :good_jobs, :cron_key, :text
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
class AddActiveJobIdIndexAndConcurrencyKeyIndexToGoodJobs < ActiveRecord::Migration[5.2]
|
2
|
+
disable_ddl_transaction!
|
3
|
+
|
4
|
+
UPDATE_BATCH_SIZE = 1_000
|
5
|
+
|
6
|
+
class GoodJobJobs < ActiveRecord::Base
|
7
|
+
self.table_name = "good_jobs"
|
8
|
+
end
|
9
|
+
|
10
|
+
def change
|
11
|
+
reversible do |dir|
|
12
|
+
dir.up do
|
13
|
+
# Ensure this incremental update migration is idempotent
|
14
|
+
# with monolithic install migration.
|
15
|
+
return if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_active_job_id_and_created_at)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
add_index :good_jobs, [:active_job_id, :created_at], algorithm: :concurrently, name: :index_good_jobs_on_active_job_id_and_created_at
|
20
|
+
add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", algorithm: :concurrently, name: :index_good_jobs_on_concurrency_key_when_unfinished
|
21
|
+
add_index :good_jobs, [:cron_key, :created_at], algorithm: :concurrently, name: :index_good_jobs_on_cron_key_and_created_at
|
22
|
+
|
23
|
+
reversible do |dir|
|
24
|
+
dir.up do
|
25
|
+
start_time = Time.current
|
26
|
+
loop do
|
27
|
+
break if GoodJobJobs.where(active_job_id: nil, finished_at: nil).where("created_at < ?", start_time).limit(UPDATE_BATCH_SIZE).update_all("active_job_id = (serialized_params->>'job_id')::uuid").zero?
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'rails/generators'
|
2
|
+
require 'rails/generators/active_record'
|
3
|
+
|
4
|
+
module GoodJob
|
5
|
+
#
|
6
|
+
# Rails generator used for updating GoodJob in a Rails application.
|
7
|
+
# Run it with +bin/rails g good_job:update+ in your console.
|
8
|
+
#
|
9
|
+
class UpdateGenerator < Rails::Generators::Base
|
10
|
+
include Rails::Generators::Migration
|
11
|
+
|
12
|
+
class << self
|
13
|
+
delegate :next_migration_number, to: ActiveRecord::Generators::Base
|
14
|
+
end
|
15
|
+
|
16
|
+
TEMPLATES = File.join(File.dirname(__FILE__), "templates/update")
|
17
|
+
source_paths << TEMPLATES
|
18
|
+
|
19
|
+
# Generates incremental migration files unless they already exist.
|
20
|
+
# All migrations should be idempotent e.g. +add_index+ is guarded with +if_index_exists?+
|
21
|
+
def update_migration_files
|
22
|
+
migration_templates = Dir.children(File.join(TEMPLATES, 'migrations')).sort
|
23
|
+
migration_templates.each do |template_file|
|
24
|
+
destination_file = template_file.match(/^\d*_(.*\.rb)/)[1] # 01_create_good_jobs.rb.erb => create_good_jobs.rb
|
25
|
+
migration_template "migrations/#{template_file}", "db/migrate/#{destination_file}", skip: true
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/good_job.rb
CHANGED
@@ -30,7 +30,7 @@ module GoodJob
|
|
30
30
|
# @!scope class
|
31
31
|
# The logger used by GoodJob (default: +Rails.logger+).
|
32
32
|
# Use this to redirect logs to a special location or file.
|
33
|
-
# @return [Logger]
|
33
|
+
# @return [Logger, nil]
|
34
34
|
# @example Output GoodJob logs to a file:
|
35
35
|
# GoodJob.logger = ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new("log/my_logs.log"))
|
36
36
|
mattr_accessor :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout))
|
@@ -42,7 +42,7 @@ module GoodJob
|
|
42
42
|
# If you want to preserve jobs for latter inspection, set this to +true+.
|
43
43
|
# If you want to preserve only jobs that finished with error for latter inspection, set this to +:on_unhandled_error+.
|
44
44
|
# If +true+, you will need to clean out jobs using the +good_job cleanup_preserved_jobs+ CLI command.
|
45
|
-
# @return [Boolean]
|
45
|
+
# @return [Boolean, nil]
|
46
46
|
mattr_accessor :preserve_job_records, default: false
|
47
47
|
|
48
48
|
# @!attribute [rw] retry_on_unhandled_error
|
@@ -51,10 +51,11 @@ module GoodJob
|
|
51
51
|
# If +true+, causes jobs to be re-queued and retried if they raise an instance of +StandardError+.
|
52
52
|
# If +false+, jobs will be discarded or marked as finished if they raise an instance of +StandardError+.
|
53
53
|
# Instances of +Exception+, like +SIGINT+, will *always* be retried, regardless of this attribute's value.
|
54
|
-
# @return [Boolean]
|
54
|
+
# @return [Boolean, nil]
|
55
55
|
mattr_accessor :retry_on_unhandled_error, default: true
|
56
56
|
|
57
57
|
# @deprecated Use {GoodJob#retry_on_unhandled_error} instead.
|
58
|
+
# @return [Boolean, nil]
|
58
59
|
def self.reperform_jobs_on_standard_error
|
59
60
|
ActiveSupport::Deprecation.warn(
|
60
61
|
"Calling 'GoodJob.reperform_jobs_on_standard_error' is deprecated. Please use 'retry_on_unhandled_error'"
|
@@ -63,6 +64,8 @@ module GoodJob
|
|
63
64
|
end
|
64
65
|
|
65
66
|
# @deprecated Use {GoodJob#retry_on_unhandled_error=} instead.
|
67
|
+
# @param value [Boolean]
|
68
|
+
# @return [Boolean]
|
66
69
|
def self.reperform_jobs_on_standard_error=(value)
|
67
70
|
ActiveSupport::Deprecation.warn(
|
68
71
|
"Setting 'GoodJob.reperform_jobs_on_standard_error=' is deprecated. Please use 'retry_on_unhandled_error='"
|
@@ -77,7 +80,7 @@ module GoodJob
|
|
77
80
|
# @example Send errors to Sentry
|
78
81
|
# # config/initializers/good_job.rb
|
79
82
|
# GoodJob.on_thread_error = -> (exception) { Raven.capture_exception(exception) }
|
80
|
-
# @return [
|
83
|
+
# @return [Proc, nil]
|
81
84
|
mattr_accessor :on_thread_error, default: nil
|
82
85
|
|
83
86
|
# Stop executing jobs.
|
@@ -93,13 +96,13 @@ module GoodJob
|
|
93
96
|
# @param wait [Boolean] whether to wait for shutdown
|
94
97
|
# @return [void]
|
95
98
|
def self.shutdown(timeout: -1, wait: nil)
|
96
|
-
timeout = if wait.
|
99
|
+
timeout = if wait.nil?
|
100
|
+
timeout
|
101
|
+
else
|
97
102
|
ActiveSupport::Deprecation.warn(
|
98
103
|
"Using `GoodJob.shutdown` with `wait:` kwarg is deprecated; use `timeout:` kwarg instead e.g. GoodJob.shutdown(timeout: #{wait ? '-1' : 'nil'})"
|
99
104
|
)
|
100
105
|
wait ? -1 : nil
|
101
|
-
else
|
102
|
-
timeout
|
103
106
|
end
|
104
107
|
|
105
108
|
executables = Array(Notifier.instances) + Array(Poller.instances) + Array(Scheduler.instances)
|
@@ -119,6 +122,7 @@ module GoodJob
|
|
119
122
|
# When forking processes you should shut down these background threads before forking, and restart them after forking.
|
120
123
|
# For example, you should use +shutdown+ and +restart+ when using async execution mode with Puma.
|
121
124
|
# See the {file:README.md#executing-jobs-async--in-process} for more explanation and examples.
|
125
|
+
# @param timeout [Numeric, nil] Seconds to wait for active threads to finish.
|
122
126
|
# @return [void]
|
123
127
|
def self.restart(timeout: -1)
|
124
128
|
executables = Array(Notifier.instances) + Array(Poller.instances) + Array(Scheduler.instances)
|
@@ -126,7 +130,7 @@ module GoodJob
|
|
126
130
|
end
|
127
131
|
|
128
132
|
# Sends +#shutdown+ or +#restart+ to executable objects ({GoodJob::Notifier}, {GoodJob::Poller}, {GoodJob::Scheduler})
|
129
|
-
# @param executables [Array<
|
133
|
+
# @param executables [Array<Notifier, Poller, Scheduler, MultiScheduler>] Objects to shut down.
|
130
134
|
# @param method_name [:symbol] Method to call, e.g. +:shutdown+ or +:restart+.
|
131
135
|
# @param timeout [nil,Numeric]
|
132
136
|
# @return [void]
|
data/lib/good_job/adapter.rb
CHANGED
@@ -6,7 +6,7 @@ module GoodJob
|
|
6
6
|
# Valid execution modes.
|
7
7
|
EXECUTION_MODES = [:async, :async_server, :external, :inline].freeze
|
8
8
|
|
9
|
-
# @param execution_mode [
|
9
|
+
# @param execution_mode [Symbol, nil] specifies how and where jobs should be executed. You can also set this with the environment variable +GOOD_JOB_EXECUTION_MODE+.
|
10
10
|
#
|
11
11
|
# - +:inline+ executes jobs immediately in whatever process queued them (usually the web server process). This should only be used in test and development environments.
|
12
12
|
# - +:external+ causes the adapter to enqueue jobs, but not execute them. When using this option (the default for production environments), you'll need to use the command-line tool to actually execute your jobs.
|
@@ -19,9 +19,9 @@ module GoodJob
|
|
19
19
|
# - +development+ and +test+: +:inline+
|
20
20
|
# - +production+ and all other environments: +:external+
|
21
21
|
#
|
22
|
-
# @param max_threads [
|
23
|
-
# @param queues [
|
24
|
-
# @param poll_interval [
|
22
|
+
# @param max_threads [Integer, nil] sets the number of threads per scheduler to use when +execution_mode+ is set to +:async+. The +queues+ parameter can specify a number of threads for each group of queues which will override this value. You can also set this with the environment variable +GOOD_JOB_MAX_THREADS+. Defaults to +5+.
|
23
|
+
# @param queues [String, nil] determines which queues to execute jobs from when +execution_mode+ is set to +:async+. See {file:README.md#optimize-queues-threads-and-processes} for more details on the format of this string. You can also set this with the environment variable +GOOD_JOB_QUEUES+. Defaults to +"*"+.
|
24
|
+
# @param poll_interval [Integer, nil] sets the number of seconds between polls for jobs when +execution_mode+ is set to +:async+. You can also set this with the environment variable +GOOD_JOB_POLL_INTERVAL+. Defaults to +1+.
|
25
25
|
def initialize(execution_mode: nil, queues: nil, max_threads: nil, poll_interval: nil)
|
26
26
|
if caller[0..4].find { |c| c.include?("/config/application.rb") || c.include?("/config/environments/") }
|
27
27
|
ActiveSupport::Deprecation.warn(<<~DEPRECATION)
|
@@ -70,7 +70,7 @@ module GoodJob
|
|
70
70
|
# Enqueues an ActiveJob job to be run at a specific time.
|
71
71
|
# For use by Rails; you should generally not call this directly.
|
72
72
|
# @param active_job [ActiveJob::Base] the job to be enqueued from +#perform_later+
|
73
|
-
# @param timestamp [Integer] the epoch time to perform the job
|
73
|
+
# @param timestamp [Integer, nil] the epoch time to perform the job
|
74
74
|
# @return [GoodJob::Job]
|
75
75
|
def enqueue_at(active_job, timestamp)
|
76
76
|
good_job = GoodJob::Job.enqueue(
|
@@ -97,22 +97,21 @@ module GoodJob
|
|
97
97
|
end
|
98
98
|
|
99
99
|
# Shut down the thread pool executors.
|
100
|
-
# @param timeout [nil, Numeric] Seconds to wait for active threads.
|
101
|
-
#
|
100
|
+
# @param timeout [nil, Numeric, Symbol] Seconds to wait for active threads.
|
102
101
|
# * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
|
103
102
|
# * +-1+, the scheduler will wait until the shutdown is complete.
|
104
103
|
# * +0+, the scheduler will immediately shutdown and stop any threads.
|
105
104
|
# * A positive number will wait that many seconds before stopping any remaining active threads.
|
106
|
-
# @param wait [Boolean] Deprecated. Use +timeout:+ instead.
|
105
|
+
# @param wait [Boolean, nil] Deprecated. Use +timeout:+ instead.
|
107
106
|
# @return [void]
|
108
107
|
def shutdown(timeout: :default, wait: nil)
|
109
|
-
timeout = if wait.
|
108
|
+
timeout = if wait.nil?
|
109
|
+
timeout
|
110
|
+
else
|
110
111
|
ActiveSupport::Deprecation.warn(
|
111
112
|
"Using `GoodJob::Adapter.shutdown` with `wait:` kwarg is deprecated; use `timeout:` kwarg instead e.g. GoodJob::Adapter.shutdown(timeout: #{wait ? '-1' : 'nil'})"
|
112
113
|
)
|
113
114
|
wait ? -1 : nil
|
114
|
-
else
|
115
|
-
timeout
|
116
115
|
end
|
117
116
|
|
118
117
|
timeout = if timeout == :default
|
@@ -126,18 +125,21 @@ module GoodJob
|
|
126
125
|
end
|
127
126
|
|
128
127
|
# Whether in +:async+ execution mode.
|
128
|
+
# @return [Boolean]
|
129
129
|
def execute_async?
|
130
130
|
@configuration.execution_mode == :async ||
|
131
131
|
@configuration.execution_mode == :async_server && in_server_process?
|
132
132
|
end
|
133
133
|
|
134
134
|
# Whether in +:external+ execution mode.
|
135
|
+
# @return [Boolean]
|
135
136
|
def execute_externally?
|
136
137
|
@configuration.execution_mode == :external ||
|
137
138
|
@configuration.execution_mode == :async_server && !in_server_process?
|
138
139
|
end
|
139
140
|
|
140
141
|
# Whether in +:inline+ execution mode.
|
142
|
+
# @return [Boolean]
|
141
143
|
def execute_inline?
|
142
144
|
@configuration.execution_mode == :inline
|
143
145
|
end
|
@@ -145,11 +147,13 @@ module GoodJob
|
|
145
147
|
private
|
146
148
|
|
147
149
|
# Whether running in a web server process.
|
150
|
+
# @return [Boolean, nil]
|
148
151
|
def in_server_process?
|
149
152
|
return @_in_server_process if defined? @_in_server_process
|
150
153
|
|
151
154
|
@_in_server_process = Rails.const_defined?('Server') ||
|
152
155
|
caller.grep(%r{config.ru}).any? || # EXAMPLE: config.ru:3:in `block in <main>' OR config.ru:3:in `new_from_string'
|
156
|
+
caller.grep(%{/rack/handler/}).any? || # EXAMPLE: iodine-0.7.44/lib/rack/handler/iodine.rb:13:in `start'
|
153
157
|
(Concurrent.on_jruby? && caller.grep(%r{jruby/rack/rails_booter}).any?) # EXAMPLE: uri:classloader:/jruby/rack/rails_booter.rb:83:in `load_environment'
|
154
158
|
end
|
155
159
|
end
|
@@ -84,9 +84,9 @@ module GoodJob
|
|
84
84
|
# on the format of this string.
|
85
85
|
# @return [String]
|
86
86
|
def queue_string
|
87
|
-
options[:queues] ||
|
88
|
-
rails_config[:queues] ||
|
89
|
-
env['GOOD_JOB_QUEUES'] ||
|
87
|
+
options[:queues].presence ||
|
88
|
+
rails_config[:queues].presence ||
|
89
|
+
env['GOOD_JOB_QUEUES'].presence ||
|
90
90
|
'*'
|
91
91
|
end
|
92
92
|
|
@@ -3,7 +3,6 @@ require 'active_support/core_ext/module/attribute_accessors_per_thread'
|
|
3
3
|
module GoodJob
|
4
4
|
# Thread-local attributes for passing values from Instrumentation.
|
5
5
|
# (Cannot use ActiveSupport::CurrentAttributes because ActiveJob resets it)
|
6
|
-
|
7
6
|
module CurrentExecution
|
8
7
|
# @!attribute [rw] error_on_retry
|
9
8
|
# @!scope class
|
data/lib/good_job/daemon.rb
CHANGED
@@ -13,6 +13,7 @@ module GoodJob
|
|
13
13
|
end
|
14
14
|
|
15
15
|
# Daemonizes the current process and writes out a pidfile.
|
16
|
+
# @return [void]
|
16
17
|
def daemonize
|
17
18
|
check_pid
|
18
19
|
Process.daemon
|
@@ -21,6 +22,7 @@ module GoodJob
|
|
21
22
|
|
22
23
|
private
|
23
24
|
|
25
|
+
# @return [void]
|
24
26
|
def write_pid
|
25
27
|
File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |f| f.write(Process.pid.to_s) }
|
26
28
|
at_exit { File.delete(pidfile) if File.exist?(pidfile) }
|
@@ -29,10 +31,12 @@ module GoodJob
|
|
29
31
|
retry
|
30
32
|
end
|
31
33
|
|
34
|
+
# @return [void]
|
32
35
|
def delete_pid
|
33
36
|
File.delete(pidfile) if File.exist?(pidfile)
|
34
37
|
end
|
35
38
|
|
39
|
+
# @return [void]
|
36
40
|
def check_pid
|
37
41
|
case pid_status(pidfile)
|
38
42
|
when :running, :not_owned
|
@@ -42,6 +46,8 @@ module GoodJob
|
|
42
46
|
end
|
43
47
|
end
|
44
48
|
|
49
|
+
# @param pidfile [Pathname, String]
|
50
|
+
# @return [Symbol]
|
45
51
|
def pid_status(pidfile)
|
46
52
|
return :exited unless File.exist?(pidfile)
|
47
53
|
|
data/lib/good_job/job.rb
CHANGED
@@ -50,6 +50,14 @@ module GoodJob
|
|
50
50
|
end
|
51
51
|
end
|
52
52
|
|
53
|
+
# Get Jobs with given class name
|
54
|
+
# @!method with_job_class
|
55
|
+
# @!scope class
|
56
|
+
# @param string [String]
|
57
|
+
# Job class name
|
58
|
+
# @return [ActiveRecord::Relation]
|
59
|
+
scope :with_job_class, ->(job_class) { where("serialized_params->>'job_class' = ?", job_class) }
|
60
|
+
|
53
61
|
# Get Jobs that have not yet been completed.
|
54
62
|
# @!method unfinished
|
55
63
|
# @!scope class
|
@@ -94,6 +102,12 @@ module GoodJob
|
|
94
102
|
# @return [ActiveRecord::Relation]
|
95
103
|
scope :finished, ->(timestamp = nil) { timestamp ? where(arel_table['finished_at'].lteq(timestamp)) : where.not(finished_at: nil) }
|
96
104
|
|
105
|
+
# Get Jobs that started but not finished yet.
|
106
|
+
# @!method running
|
107
|
+
# @!scope class
|
108
|
+
# @return [ActiveRecord::Relation]
|
109
|
+
scope :running, -> { where.not(performed_at: nil).where(finished_at: nil) }
|
110
|
+
|
97
111
|
# Get Jobs on queues that match the given queue string.
|
98
112
|
# @!method queue_string(string)
|
99
113
|
# @!scope class
|
@@ -142,10 +156,10 @@ module GoodJob
|
|
142
156
|
# raised, if any (if the job raised, then the second array entry will be
|
143
157
|
# +nil+). If there were no jobs to execute, returns +nil+.
|
144
158
|
def self.perform_with_advisory_lock
|
145
|
-
unfinished.priority_ordered.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
|
159
|
+
unfinished.priority_ordered.only_scheduled.limit(1).with_advisory_lock(unlock_session: true) do |good_jobs|
|
146
160
|
good_job = good_jobs.first
|
147
|
-
|
148
|
-
break unless good_job&.executable?
|
161
|
+
break if good_job.blank?
|
162
|
+
break :unlocked unless good_job&.executable?
|
149
163
|
|
150
164
|
good_job.perform
|
151
165
|
end
|
@@ -182,13 +196,30 @@ module GoodJob
|
|
182
196
|
# The new {Job} instance representing the queued ActiveJob job.
|
183
197
|
def self.enqueue(active_job, scheduled_at: nil, create_with_advisory_lock: false)
|
184
198
|
ActiveSupport::Notifications.instrument("enqueue_job.good_job", { active_job: active_job, scheduled_at: scheduled_at, create_with_advisory_lock: create_with_advisory_lock }) do |instrument_payload|
|
185
|
-
|
199
|
+
good_job_args = {
|
186
200
|
queue_name: active_job.queue_name.presence || DEFAULT_QUEUE_NAME,
|
187
201
|
priority: active_job.priority || DEFAULT_PRIORITY,
|
188
202
|
serialized_params: active_job.serialize,
|
189
203
|
scheduled_at: scheduled_at,
|
190
|
-
create_with_advisory_lock: create_with_advisory_lock
|
191
|
-
|
204
|
+
create_with_advisory_lock: create_with_advisory_lock,
|
205
|
+
}
|
206
|
+
|
207
|
+
if column_names.include?('active_job_id')
|
208
|
+
good_job_args[:active_job_id] = active_job.job_id
|
209
|
+
else
|
210
|
+
ActiveSupport::Deprecation.warn(<<~DEPRECATION)
|
211
|
+
GoodJob has pending database migrations. To create the migration files, run:
|
212
|
+
|
213
|
+
rails generate good_job:update
|
214
|
+
|
215
|
+
To apply the migration files, run:
|
216
|
+
|
217
|
+
rails db:migrate
|
218
|
+
|
219
|
+
DEPRECATION
|
220
|
+
end
|
221
|
+
|
222
|
+
good_job = GoodJob::Job.new(**good_job_args)
|
192
223
|
|
193
224
|
instrument_payload[:good_job] = good_job
|
194
225
|
|
@@ -235,7 +266,7 @@ module GoodJob
|
|
235
266
|
|
236
267
|
private
|
237
268
|
|
238
|
-
# @return [
|
269
|
+
# @return [ExecutionResult]
|
239
270
|
def execute
|
240
271
|
params = serialized_params.merge(
|
241
272
|
"provider_job_id" => id
|
@@ -24,7 +24,7 @@ module GoodJob
|
|
24
24
|
end
|
25
25
|
|
26
26
|
# Perform the next eligible job
|
27
|
-
# @return [
|
27
|
+
# @return [Object, nil] Returns job result or +nil+ if no job was found
|
28
28
|
def next
|
29
29
|
job_query.perform_with_advisory_lock
|
30
30
|
end
|
@@ -54,7 +54,7 @@ module GoodJob
|
|
54
54
|
# @param after [DateTime, Time, nil] future jobs scheduled after this time
|
55
55
|
# @param limit [Integer] number of future timestamps to return
|
56
56
|
# @param now_limit [Integer] number of past timestamps to return
|
57
|
-
# @return [Array<
|
57
|
+
# @return [Array<DateTime, Time>, nil]
|
58
58
|
def next_at(after: nil, limit: nil, now_limit: nil)
|
59
59
|
job_query.next_scheduled_at(after: after, limit: limit, now_limit: now_limit)
|
60
60
|
end
|
data/lib/good_job/lockable.rb
CHANGED
@@ -22,17 +22,25 @@ module GoodJob
|
|
22
22
|
RecordAlreadyAdvisoryLockedError = Class.new(StandardError)
|
23
23
|
|
24
24
|
included do
|
25
|
+
# Default column to be used when creating Advisory Locks
|
26
|
+
cattr_accessor(:advisory_lockable_column, instance_accessor: false) { primary_key }
|
27
|
+
|
28
|
+
# Default Postgres function to be used for Advisory Locks
|
29
|
+
cattr_accessor(:advisory_lockable_function) { "pg_try_advisory_lock" }
|
30
|
+
|
25
31
|
# Attempt to acquire an advisory lock on the selected records and
|
26
32
|
# return only those records for which a lock could be acquired.
|
27
|
-
# @!method advisory_lock
|
33
|
+
# @!method advisory_lock(column: advisory_lockable_column, function: advisory_lockable_function)
|
28
34
|
# @!scope class
|
35
|
+
# @param column [String, Symbol] column values to Advisory Lock against
|
36
|
+
# @param function [String, Symbol] Postgres Advisory Lock function name to use
|
29
37
|
# @return [ActiveRecord::Relation]
|
30
38
|
# A relation selecting only the records that were locked.
|
31
|
-
scope :advisory_lock, (lambda do
|
39
|
+
scope :advisory_lock, (lambda do |column: advisory_lockable_column, function: advisory_lockable_function|
|
32
40
|
original_query = self
|
33
41
|
|
34
42
|
cte_table = Arel::Table.new(:rows)
|
35
|
-
cte_query = original_query.select(primary_key).except(:limit)
|
43
|
+
cte_query = original_query.select(primary_key, column).except(:limit)
|
36
44
|
cte_type = if supports_cte_materialization_specifiers?
|
37
45
|
'MATERIALIZED'
|
38
46
|
else
|
@@ -41,9 +49,14 @@ module GoodJob
|
|
41
49
|
|
42
50
|
composed_cte = Arel::Nodes::As.new(cte_table, Arel::Nodes::SqlLiteral.new([cte_type, "(", cte_query.to_sql, ")"].join(' ')))
|
43
51
|
|
52
|
+
# In addition to an advisory lock, there is also a FOR UPDATE SKIP LOCKED
|
53
|
+
# because this causes the query to skip jobs that were completed (and deleted)
|
54
|
+
# by another session in the time since the table snapshot was taken.
|
55
|
+
# In rare cases under high concurrency levels, leaving this out can result in double executions.
|
44
56
|
query = cte_table.project(cte_table[:id])
|
45
57
|
.with(composed_cte)
|
46
|
-
.where(Arel.sql(sanitize_sql_for_conditions(["
|
58
|
+
.where(Arel.sql(sanitize_sql_for_conditions(["#{function}(('x' || substr(md5(:table_name || #{connection.quote_table_name(cte_table.name)}.#{connection.quote_column_name(column)}::text), 1, 16))::bit(64)::bigint)", { table_name: table_name }])))
|
59
|
+
.lock(Arel.sql("FOR UPDATE SKIP LOCKED"))
|
47
60
|
|
48
61
|
limit = original_query.arel.ast.limit
|
49
62
|
query.limit = limit.value if limit.present?
|
@@ -57,40 +70,44 @@ module GoodJob
|
|
57
70
|
#
|
58
71
|
# For details on +pg_locks+, see
|
59
72
|
# {https://www.postgresql.org/docs/current/view-pg-locks.html}.
|
60
|
-
# @!method joins_advisory_locks
|
73
|
+
# @!method joins_advisory_locks(column: advisory_lockable_column)
|
61
74
|
# @!scope class
|
75
|
+
# @param column [String, Symbol] column values to Advisory Lock against
|
62
76
|
# @return [ActiveRecord::Relation]
|
63
77
|
# @example Get the records that have a session awaiting a lock:
|
64
78
|
# MyLockableRecord.joins_advisory_locks.where("pg_locks.granted = ?", false)
|
65
|
-
scope :joins_advisory_locks, (lambda do
|
79
|
+
scope :joins_advisory_locks, (lambda do |column: advisory_lockable_column|
|
66
80
|
join_sql = <<~SQL.squish
|
67
81
|
LEFT JOIN pg_locks ON pg_locks.locktype = 'advisory'
|
68
82
|
AND pg_locks.objsubid = 1
|
69
|
-
AND pg_locks.classid = ('x' || substr(md5(:table_name || #{quoted_table_name}.#{
|
70
|
-
AND pg_locks.objid = (('x' || substr(md5(:table_name || #{quoted_table_name}.#{
|
83
|
+
AND pg_locks.classid = ('x' || substr(md5(:table_name || #{quoted_table_name}.#{connection.quote_column_name(column)}::text), 1, 16))::bit(32)::int
|
84
|
+
AND pg_locks.objid = (('x' || substr(md5(:table_name || #{quoted_table_name}.#{connection.quote_column_name(column)}::text), 1, 16))::bit(64) << 32)::bit(32)::int
|
71
85
|
SQL
|
72
86
|
|
73
87
|
joins(sanitize_sql_for_conditions([join_sql, { table_name: table_name }]))
|
74
88
|
end)
|
75
89
|
|
76
90
|
# Find records that do not have an advisory lock on them.
|
77
|
-
# @!method advisory_unlocked
|
91
|
+
# @!method advisory_unlocked(column: advisory_lockable_column)
|
78
92
|
# @!scope class
|
93
|
+
# @param column [String, Symbol] column values to Advisory Lock against
|
79
94
|
# @return [ActiveRecord::Relation]
|
80
|
-
scope :advisory_unlocked, -> { joins_advisory_locks.where(pg_locks: { locktype: nil }) }
|
95
|
+
scope :advisory_unlocked, ->(column: advisory_lockable_column) { joins_advisory_locks(column: column).where(pg_locks: { locktype: nil }) }
|
81
96
|
|
82
97
|
# Find records that have an advisory lock on them.
|
83
|
-
# @!method advisory_locked
|
98
|
+
# @!method advisory_locked(column: advisory_lockable_column)
|
84
99
|
# @!scope class
|
100
|
+
# @param column [String, Symbol] column values to Advisory Lock against
|
85
101
|
# @return [ActiveRecord::Relation]
|
86
|
-
scope :advisory_locked, -> { joins_advisory_locks.where.not(pg_locks: { locktype: nil }) }
|
102
|
+
scope :advisory_locked, ->(column: advisory_lockable_column) { joins_advisory_locks(column: column).where.not(pg_locks: { locktype: nil }) }
|
87
103
|
|
88
104
|
# Find records with advisory locks owned by the current Postgres
|
89
105
|
# session/connection.
|
90
|
-
# @!method advisory_locked
|
106
|
+
# @!method advisory_locked(column: advisory_lockable_column)
|
91
107
|
# @!scope class
|
108
|
+
# @param column [String, Symbol] column values to Advisory Lock against
|
92
109
|
# @return [ActiveRecord::Relation]
|
93
|
-
scope :owns_advisory_locked, -> { joins_advisory_locks.where('"pg_locks"."pid" = pg_backend_pid()') }
|
110
|
+
scope :owns_advisory_locked, ->(column: advisory_lockable_column) { joins_advisory_locks(column: column).where('"pg_locks"."pid" = pg_backend_pid()') }
|
94
111
|
|
95
112
|
# Whether an advisory lock should be acquired in the same transaction
|
96
113
|
# that created the record.
|
@@ -122,6 +139,9 @@ module GoodJob
|
|
122
139
|
# can (as in {Lockable.advisory_lock}) and only pass those that could be
|
123
140
|
# locked to the block.
|
124
141
|
#
|
142
|
+
# @param column [String, Symbol] name of advisory lock or unlock function
|
143
|
+
# @param function [String, Symbol] Postgres Advisory Lock function name to use
|
144
|
+
# @param unlock_session [Boolean] Whether to unlock all advisory locks in the session afterwards
|
125
145
|
# @yield [Array<Lockable>] the records that were successfully locked.
|
126
146
|
# @return [Object] the result of the block.
|
127
147
|
#
|
@@ -129,14 +149,21 @@ module GoodJob
|
|
129
149
|
# MyLockableRecord.order(created_at: :asc).limit(2).with_advisory_lock do |record|
|
130
150
|
# do_something_with record
|
131
151
|
# end
|
132
|
-
def with_advisory_lock
|
152
|
+
def with_advisory_lock(column: advisory_lockable_column, function: advisory_lockable_function, unlock_session: false)
|
133
153
|
raise ArgumentError, "Must provide a block" unless block_given?
|
134
154
|
|
135
|
-
records = advisory_lock.to_a
|
155
|
+
records = advisory_lock(column: column, function: function).to_a
|
136
156
|
begin
|
137
157
|
yield(records)
|
138
158
|
ensure
|
139
|
-
|
159
|
+
if unlock_session
|
160
|
+
advisory_unlock_session
|
161
|
+
else
|
162
|
+
records.each do |record|
|
163
|
+
key = [table_name, record[advisory_lockable_column]].join
|
164
|
+
record.advisory_unlock(key: key, function: advisory_unlockable_function(function))
|
165
|
+
end
|
166
|
+
end
|
140
167
|
end
|
141
168
|
end
|
142
169
|
|
@@ -145,49 +172,79 @@ module GoodJob
|
|
145
172
|
|
146
173
|
@_supports_cte_materialization_specifiers = connection.postgresql_version >= 120000
|
147
174
|
end
|
175
|
+
|
176
|
+
# Postgres advisory unlocking function for the class
|
177
|
+
# @param function [String, Symbol] name of advisory lock or unlock function
|
178
|
+
# @return [Boolean]
|
179
|
+
def advisory_unlockable_function(function = advisory_lockable_function)
|
180
|
+
function.to_s.sub("_lock", "_unlock").sub("_try_", "_")
|
181
|
+
end
|
182
|
+
|
183
|
+
# Unlocks all advisory locks active in the current database session/connection
|
184
|
+
# @return [void]
|
185
|
+
def advisory_unlock_session
|
186
|
+
connection.exec_query("SELECT pg_advisory_unlock_all()::text AS unlocked", 'GoodJob::Lockable Unlock Session').first[:unlocked]
|
187
|
+
end
|
188
|
+
|
189
|
+
# Converts SQL query strings between PG-compatible and JDBC-compatible syntax
|
190
|
+
# @param query [String]
|
191
|
+
# @return [Boolean]
|
192
|
+
def pg_or_jdbc_query(query)
|
193
|
+
if Concurrent.on_jruby?
|
194
|
+
# Replace $1 bind parameters with ?
|
195
|
+
query.gsub(/\$\d*/, '?')
|
196
|
+
else
|
197
|
+
query
|
198
|
+
end
|
199
|
+
end
|
148
200
|
end
|
149
201
|
|
150
202
|
# Acquires an advisory lock on this record if it is not already locked by
|
151
203
|
# another database session. Be careful to ensure you release the lock when
|
152
204
|
# you are done with {#advisory_unlock} (or {#advisory_unlock!} to release
|
153
205
|
# all remaining locks).
|
206
|
+
# @param key [String, Symbol] Key to Advisory Lock against
|
207
|
+
# @param function [String, Symbol] Postgres Advisory Lock function name to use
|
154
208
|
# @return [Boolean] whether the lock was acquired.
|
155
|
-
def advisory_lock
|
209
|
+
def advisory_lock(key: lockable_key, function: advisory_lockable_function)
|
156
210
|
query = <<~SQL.squish
|
157
|
-
SELECT 1 AS
|
158
|
-
WHERE pg_try_advisory_lock(('x'||substr(md5($1 || $2::text), 1, 16))::bit(64)::bigint)
|
211
|
+
SELECT #{function}(('x'||substr(md5($1::text), 1, 16))::bit(64)::bigint) AS locked
|
159
212
|
SQL
|
160
|
-
binds = [[nil,
|
161
|
-
self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Lock', binds).
|
213
|
+
binds = [[nil, key]]
|
214
|
+
self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Lock', binds).first['locked']
|
162
215
|
end
|
163
216
|
|
164
217
|
# Releases an advisory lock on this record if it is locked by this database
|
165
218
|
# session. Note that advisory locks stack, so you must call
|
166
219
|
# {#advisory_unlock} and {#advisory_lock} the same number of times.
|
220
|
+
# @param key [String, Symbol] Key to lock against
|
221
|
+
# @param function [String, Symbol] Postgres Advisory Lock function name to use
|
167
222
|
# @return [Boolean] whether the lock was released.
|
168
|
-
def advisory_unlock
|
223
|
+
def advisory_unlock(key: lockable_key, function: self.class.advisory_unlockable_function(advisory_lockable_function))
|
169
224
|
query = <<~SQL.squish
|
170
|
-
SELECT 1 AS
|
171
|
-
WHERE pg_advisory_unlock(('x'||substr(md5($1 || $2::text), 1, 16))::bit(64)::bigint)
|
225
|
+
SELECT #{function}(('x'||substr(md5($1::text), 1, 16))::bit(64)::bigint) AS unlocked
|
172
226
|
SQL
|
173
|
-
binds = [[nil,
|
174
|
-
self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Unlock', binds).
|
227
|
+
binds = [[nil, key]]
|
228
|
+
self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Unlock', binds).first['unlocked']
|
175
229
|
end
|
176
230
|
|
177
231
|
# Acquires an advisory lock on this record or raises
|
178
232
|
# {RecordAlreadyAdvisoryLockedError} if it is already locked by another
|
179
233
|
# database session.
|
234
|
+
# @param key [String, Symbol] Key to lock against
|
235
|
+
# @param function [String, Symbol] Postgres Advisory Lock function name to use
|
180
236
|
# @raise [RecordAlreadyAdvisoryLockedError]
|
181
237
|
# @return [Boolean] +true+
|
182
|
-
def advisory_lock!
|
183
|
-
result = advisory_lock
|
238
|
+
def advisory_lock!(key: lockable_key, function: advisory_lockable_function)
|
239
|
+
result = advisory_lock(key: key, function: function)
|
184
240
|
result || raise(RecordAlreadyAdvisoryLockedError)
|
185
241
|
end
|
186
242
|
|
187
243
|
# Acquires an advisory lock on this record and safely releases it after the
|
188
244
|
# passed block is completed. If the record is locked by another database
|
189
245
|
# session, this raises {RecordAlreadyAdvisoryLockedError}.
|
190
|
-
#
|
246
|
+
# @param key [String, Symbol] Key to lock against
|
247
|
+
# @param function [String, Symbol] Postgres Advisory Lock function name to use
|
191
248
|
# @yield Nothing
|
192
249
|
# @return [Object] The result of the block.
|
193
250
|
#
|
@@ -196,67 +253,63 @@ module GoodJob
|
|
196
253
|
# record.with_advisory_lock do
|
197
254
|
# do_something_with record
|
198
255
|
# end
|
199
|
-
def with_advisory_lock
|
256
|
+
def with_advisory_lock(key: lockable_key, function: advisory_lockable_function)
|
200
257
|
raise ArgumentError, "Must provide a block" unless block_given?
|
201
258
|
|
202
|
-
advisory_lock!
|
259
|
+
advisory_lock!(key: key, function: function)
|
203
260
|
yield
|
204
261
|
ensure
|
205
|
-
advisory_unlock unless $ERROR_INFO.is_a? RecordAlreadyAdvisoryLockedError
|
262
|
+
advisory_unlock(key: key, function: self.class.advisory_unlockable_function(function)) unless $ERROR_INFO.is_a? RecordAlreadyAdvisoryLockedError
|
206
263
|
end
|
207
264
|
|
208
265
|
# Tests whether this record has an advisory lock on it.
|
266
|
+
# @param key [String, Symbol] Key to test lock against
|
209
267
|
# @return [Boolean]
|
210
|
-
def advisory_locked?
|
268
|
+
def advisory_locked?(key: lockable_key)
|
211
269
|
query = <<~SQL.squish
|
212
270
|
SELECT 1 AS one
|
213
271
|
FROM pg_locks
|
214
272
|
WHERE pg_locks.locktype = 'advisory'
|
215
273
|
AND pg_locks.objsubid = 1
|
216
|
-
AND pg_locks.classid = ('x' || substr(md5($1
|
217
|
-
AND pg_locks.objid = (('x' || substr(md5($
|
274
|
+
AND pg_locks.classid = ('x' || substr(md5($1::text), 1, 16))::bit(32)::int
|
275
|
+
AND pg_locks.objid = (('x' || substr(md5($2::text), 1, 16))::bit(64) << 32)::bit(32)::int
|
218
276
|
SQL
|
219
|
-
binds = [[nil,
|
277
|
+
binds = [[nil, key], [nil, key]]
|
220
278
|
self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Locked?', binds).any?
|
221
279
|
end
|
222
280
|
|
223
281
|
# Tests whether this record is locked by the current database session.
|
282
|
+
# @param key [String, Symbol] Key to test lock against
|
224
283
|
# @return [Boolean]
|
225
|
-
def owns_advisory_lock?
|
284
|
+
def owns_advisory_lock?(key: lockable_key)
|
226
285
|
query = <<~SQL.squish
|
227
286
|
SELECT 1 AS one
|
228
287
|
FROM pg_locks
|
229
288
|
WHERE pg_locks.locktype = 'advisory'
|
230
289
|
AND pg_locks.objsubid = 1
|
231
|
-
AND pg_locks.classid = ('x' || substr(md5($1
|
232
|
-
AND pg_locks.objid = (('x' || substr(md5($
|
290
|
+
AND pg_locks.classid = ('x' || substr(md5($1::text), 1, 16))::bit(32)::int
|
291
|
+
AND pg_locks.objid = (('x' || substr(md5($2::text), 1, 16))::bit(64) << 32)::bit(32)::int
|
233
292
|
AND pg_locks.pid = pg_backend_pid()
|
234
293
|
SQL
|
235
|
-
binds = [[nil,
|
294
|
+
binds = [[nil, key], [nil, key]]
|
236
295
|
self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Owns Advisory Lock?', binds).any?
|
237
296
|
end
|
238
297
|
|
239
298
|
# Releases all advisory locks on the record that are held by the current
|
240
299
|
# database session.
|
300
|
+
# @param key [String, Symbol] Key to lock against
|
301
|
+
# @param function [String, Symbol] Postgres Advisory Lock function name to use
|
241
302
|
# @return [void]
|
242
|
-
def advisory_unlock!
|
243
|
-
advisory_unlock while advisory_locked?
|
303
|
+
def advisory_unlock!(key: lockable_key, function: self.class.advisory_unlockable_function(advisory_lockable_function))
|
304
|
+
advisory_unlock(key: key, function: function) while advisory_locked?
|
244
305
|
end
|
245
306
|
|
246
|
-
|
247
|
-
|
248
|
-
def
|
249
|
-
|
250
|
-
self.class.send(:sanitize_sql_for_conditions, *args)
|
307
|
+
# Default Advisory Lock key
|
308
|
+
# @return [String]
|
309
|
+
def lockable_key
|
310
|
+
[self.class.table_name, self[self.class.advisory_lockable_column]].join
|
251
311
|
end
|
252
312
|
|
253
|
-
|
254
|
-
if Concurrent.on_jruby?
|
255
|
-
# Replace $1 bind parameters with ?
|
256
|
-
query.gsub(/\$\d*/, '?')
|
257
|
-
else
|
258
|
-
query
|
259
|
-
end
|
260
|
-
end
|
313
|
+
delegate :pg_or_jdbc_query, to: :class
|
261
314
|
end
|
262
315
|
end
|