good_job 1.9.2 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +98 -1
  3. data/README.md +17 -0
  4. data/engine/app/assets/vendor/bootstrap/bootstrap.bundle.min.js +7 -0
  5. data/engine/app/assets/vendor/bootstrap/bootstrap.min.css +7 -0
  6. data/engine/app/controllers/good_job/assets_controller.rb +29 -0
  7. data/engine/app/controllers/good_job/dashboards_controller.rb +8 -6
  8. data/engine/app/controllers/good_job/jobs_controller.rb +9 -0
  9. data/engine/app/views/good_job/dashboards/index.html.erb +1 -1
  10. data/engine/app/views/layouts/good_job/base.html.erb +21 -12
  11. data/engine/app/views/shared/_chart.erb +3 -2
  12. data/engine/app/views/shared/_jobs_table.erb +13 -1
  13. data/engine/app/views/shared/icons/_check.html.erb +4 -0
  14. data/engine/app/views/shared/icons/_exclamation.html.erb +4 -0
  15. data/engine/app/views/shared/icons/_trash.html.erb +5 -0
  16. data/engine/config/routes.rb +10 -1
  17. data/lib/generators/good_job/install_generator.rb +5 -15
  18. data/lib/generators/good_job/templates/install/migrations/create_good_jobs.rb.erb +27 -0
  19. data/lib/generators/good_job/templates/{migration.rb.erb → update/migrations/01_create_good_jobs.rb} +3 -3
  20. data/lib/generators/good_job/templates/update/migrations/02_add_active_job_id_concurrency_key_cron_key_to_good_jobs.rb +15 -0
  21. data/lib/generators/good_job/templates/update/migrations/03_add_active_job_id_index_and_concurrency_key_index_to_good_jobs.rb +32 -0
  22. data/lib/generators/good_job/update_generator.rb +29 -0
  23. data/lib/good_job.rb +12 -8
  24. data/lib/good_job/adapter.rb +15 -11
  25. data/lib/good_job/configuration.rb +3 -3
  26. data/lib/good_job/current_execution.rb +0 -1
  27. data/lib/good_job/daemon.rb +6 -0
  28. data/lib/good_job/job.rb +38 -7
  29. data/lib/good_job/job_performer.rb +2 -2
  30. data/lib/good_job/lockable.rb +109 -56
  31. data/lib/good_job/log_subscriber.rb +15 -14
  32. data/lib/good_job/multi_scheduler.rb +9 -0
  33. data/lib/good_job/notifier.rb +4 -2
  34. data/lib/good_job/poller.rb +16 -7
  35. data/lib/good_job/scheduler.rb +12 -6
  36. data/lib/good_job/version.rb +1 -1
  37. metadata +28 -5
  38. data/engine/app/assets/vendor/bootstrap/bootstrap-native.js +0 -1662
  39. data/engine/app/assets/vendor/bootstrap/bootstrap.css +0 -10258
@@ -1,4 +1,4 @@
1
- class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
1
+ class CreateGoodJobs < ActiveRecord::Migration[5.2]
2
2
  def change
3
3
  enable_extension 'pgcrypto'
4
4
 
@@ -14,7 +14,7 @@ class CreateGoodJobs < ActiveRecord::Migration<%= migration_version %>
14
14
  t.timestamps
15
15
  end
16
16
 
17
- add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)"
18
- add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)"
17
+ add_index :good_jobs, :scheduled_at, where: "(finished_at IS NULL)", name: "index_good_jobs_on_scheduled_at"
18
+ add_index :good_jobs, [:queue_name, :scheduled_at], where: "(finished_at IS NULL)", name: :index_good_jobs_on_queue_name_and_scheduled_at
19
19
  end
20
20
  end
@@ -0,0 +1,15 @@
1
+ class AddActiveJobIdConcurrencyKeyCronKeyToGoodJobs < ActiveRecord::Migration[5.2]
2
+ def change
3
+ reversible do |dir|
4
+ dir.up do
5
+ # Ensure this incremental update migration is idempotent
6
+ # with monolithic install migration.
7
+ return if connection.column_exists?(:good_jobs, :active_job_id)
8
+ end
9
+ end
10
+
11
+ add_column :good_jobs, :active_job_id, :uuid
12
+ add_column :good_jobs, :concurrency_key, :text
13
+ add_column :good_jobs, :cron_key, :text
14
+ end
15
+ end
@@ -0,0 +1,32 @@
1
+ class AddActiveJobIdIndexAndConcurrencyKeyIndexToGoodJobs < ActiveRecord::Migration[5.2]
2
+ disable_ddl_transaction!
3
+
4
+ UPDATE_BATCH_SIZE = 1_000
5
+
6
+ class GoodJobJobs < ActiveRecord::Base
7
+ self.table_name = "good_jobs"
8
+ end
9
+
10
+ def change
11
+ reversible do |dir|
12
+ dir.up do
13
+ # Ensure this incremental update migration is idempotent
14
+ # with monolithic install migration.
15
+ return if connection.index_name_exists?(:good_jobs, :index_good_jobs_on_active_job_id_and_created_at)
16
+ end
17
+ end
18
+
19
+ add_index :good_jobs, [:active_job_id, :created_at], algorithm: :concurrently, name: :index_good_jobs_on_active_job_id_and_created_at
20
+ add_index :good_jobs, :concurrency_key, where: "(finished_at IS NULL)", algorithm: :concurrently, name: :index_good_jobs_on_concurrency_key_when_unfinished
21
+ add_index :good_jobs, [:cron_key, :created_at], algorithm: :concurrently, name: :index_good_jobs_on_cron_key_and_created_at
22
+
23
+ reversible do |dir|
24
+ dir.up do
25
+ start_time = Time.current
26
+ loop do
27
+ break if GoodJobJobs.where(active_job_id: nil, finished_at: nil).where("created_at < ?", start_time).limit(UPDATE_BATCH_SIZE).update_all("active_job_id = (serialized_params->>'job_id')::uuid").zero?
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,29 @@
1
+ require 'rails/generators'
2
+ require 'rails/generators/active_record'
3
+
4
+ module GoodJob
5
+ #
6
+ # Rails generator used for updating GoodJob in a Rails application.
7
+ # Run it with +bin/rails g good_job:update+ in your console.
8
+ #
9
+ class UpdateGenerator < Rails::Generators::Base
10
+ include Rails::Generators::Migration
11
+
12
+ class << self
13
+ delegate :next_migration_number, to: ActiveRecord::Generators::Base
14
+ end
15
+
16
+ TEMPLATES = File.join(File.dirname(__FILE__), "templates/update")
17
+ source_paths << TEMPLATES
18
+
19
+ # Generates incremental migration files unless they already exist.
20
+ # All migrations should be idempotent e.g. +add_index+ is guarded with +if_index_exists?+
21
+ def update_migration_files
22
+ migration_templates = Dir.children(File.join(TEMPLATES, 'migrations')).sort
23
+ migration_templates.each do |template_file|
24
+ destination_file = template_file.match(/^\d*_(.*\.rb)/)[1] # 01_create_good_jobs.rb.erb => create_good_jobs.rb
25
+ migration_template "migrations/#{template_file}", "db/migrate/#{destination_file}", skip: true
26
+ end
27
+ end
28
+ end
29
+ end
data/lib/good_job.rb CHANGED
@@ -30,7 +30,7 @@ module GoodJob
30
30
  # @!scope class
31
31
  # The logger used by GoodJob (default: +Rails.logger+).
32
32
  # Use this to redirect logs to a special location or file.
33
- # @return [Logger]
33
+ # @return [Logger, nil]
34
34
  # @example Output GoodJob logs to a file:
35
35
  # GoodJob.logger = ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new("log/my_logs.log"))
36
36
  mattr_accessor :logger, default: ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new($stdout))
@@ -42,7 +42,7 @@ module GoodJob
42
42
  # If you want to preserve jobs for latter inspection, set this to +true+.
43
43
  # If you want to preserve only jobs that finished with error for latter inspection, set this to +:on_unhandled_error+.
44
44
  # If +true+, you will need to clean out jobs using the +good_job cleanup_preserved_jobs+ CLI command.
45
- # @return [Boolean]
45
+ # @return [Boolean, nil]
46
46
  mattr_accessor :preserve_job_records, default: false
47
47
 
48
48
  # @!attribute [rw] retry_on_unhandled_error
@@ -51,10 +51,11 @@ module GoodJob
51
51
  # If +true+, causes jobs to be re-queued and retried if they raise an instance of +StandardError+.
52
52
  # If +false+, jobs will be discarded or marked as finished if they raise an instance of +StandardError+.
53
53
  # Instances of +Exception+, like +SIGINT+, will *always* be retried, regardless of this attribute's value.
54
- # @return [Boolean]
54
+ # @return [Boolean, nil]
55
55
  mattr_accessor :retry_on_unhandled_error, default: true
56
56
 
57
57
  # @deprecated Use {GoodJob#retry_on_unhandled_error} instead.
58
+ # @return [Boolean, nil]
58
59
  def self.reperform_jobs_on_standard_error
59
60
  ActiveSupport::Deprecation.warn(
60
61
  "Calling 'GoodJob.reperform_jobs_on_standard_error' is deprecated. Please use 'retry_on_unhandled_error'"
@@ -63,6 +64,8 @@ module GoodJob
63
64
  end
64
65
 
65
66
  # @deprecated Use {GoodJob#retry_on_unhandled_error=} instead.
67
+ # @param value [Boolean]
68
+ # @return [Boolean]
66
69
  def self.reperform_jobs_on_standard_error=(value)
67
70
  ActiveSupport::Deprecation.warn(
68
71
  "Setting 'GoodJob.reperform_jobs_on_standard_error=' is deprecated. Please use 'retry_on_unhandled_error='"
@@ -77,7 +80,7 @@ module GoodJob
77
80
  # @example Send errors to Sentry
78
81
  # # config/initializers/good_job.rb
79
82
  # GoodJob.on_thread_error = -> (exception) { Raven.capture_exception(exception) }
80
- # @return [#call, nil]
83
+ # @return [Proc, nil]
81
84
  mattr_accessor :on_thread_error, default: nil
82
85
 
83
86
  # Stop executing jobs.
@@ -93,13 +96,13 @@ module GoodJob
93
96
  # @param wait [Boolean] whether to wait for shutdown
94
97
  # @return [void]
95
98
  def self.shutdown(timeout: -1, wait: nil)
96
- timeout = if wait.present?
99
+ timeout = if wait.nil?
100
+ timeout
101
+ else
97
102
  ActiveSupport::Deprecation.warn(
98
103
  "Using `GoodJob.shutdown` with `wait:` kwarg is deprecated; use `timeout:` kwarg instead e.g. GoodJob.shutdown(timeout: #{wait ? '-1' : 'nil'})"
99
104
  )
100
105
  wait ? -1 : nil
101
- else
102
- timeout
103
106
  end
104
107
 
105
108
  executables = Array(Notifier.instances) + Array(Poller.instances) + Array(Scheduler.instances)
@@ -119,6 +122,7 @@ module GoodJob
119
122
  # When forking processes you should shut down these background threads before forking, and restart them after forking.
120
123
  # For example, you should use +shutdown+ and +restart+ when using async execution mode with Puma.
121
124
  # See the {file:README.md#executing-jobs-async--in-process} for more explanation and examples.
125
+ # @param timeout [Numeric, nil] Seconds to wait for active threads to finish.
122
126
  # @return [void]
123
127
  def self.restart(timeout: -1)
124
128
  executables = Array(Notifier.instances) + Array(Poller.instances) + Array(Scheduler.instances)
@@ -126,7 +130,7 @@ module GoodJob
126
130
  end
127
131
 
128
132
  # Sends +#shutdown+ or +#restart+ to executable objects ({GoodJob::Notifier}, {GoodJob::Poller}, {GoodJob::Scheduler})
129
- # @param executables [Array<(Notifier, Poller, Scheduler)>] Objects to shut down.
133
+ # @param executables [Array<Notifier, Poller, Scheduler, MultiScheduler>] Objects to shut down.
130
134
  # @param method_name [:symbol] Method to call, e.g. +:shutdown+ or +:restart+.
131
135
  # @param timeout [nil,Numeric]
132
136
  # @return [void]
@@ -6,7 +6,7 @@ module GoodJob
6
6
  # Valid execution modes.
7
7
  EXECUTION_MODES = [:async, :async_server, :external, :inline].freeze
8
8
 
9
- # @param execution_mode [nil, Symbol] specifies how and where jobs should be executed. You can also set this with the environment variable +GOOD_JOB_EXECUTION_MODE+.
9
+ # @param execution_mode [Symbol, nil] specifies how and where jobs should be executed. You can also set this with the environment variable +GOOD_JOB_EXECUTION_MODE+.
10
10
  #
11
11
  # - +:inline+ executes jobs immediately in whatever process queued them (usually the web server process). This should only be used in test and development environments.
12
12
  # - +:external+ causes the adapter to enqueue jobs, but not execute them. When using this option (the default for production environments), you'll need to use the command-line tool to actually execute your jobs.
@@ -19,9 +19,9 @@ module GoodJob
19
19
  # - +development+ and +test+: +:inline+
20
20
  # - +production+ and all other environments: +:external+
21
21
  #
22
- # @param max_threads [nil, Integer] sets the number of threads per scheduler to use when +execution_mode+ is set to +:async+. The +queues+ parameter can specify a number of threads for each group of queues which will override this value. You can also set this with the environment variable +GOOD_JOB_MAX_THREADS+. Defaults to +5+.
23
- # @param queues [nil, String] determines which queues to execute jobs from when +execution_mode+ is set to +:async+. See {file:README.md#optimize-queues-threads-and-processes} for more details on the format of this string. You can also set this with the environment variable +GOOD_JOB_QUEUES+. Defaults to +"*"+.
24
- # @param poll_interval [nil, Integer] sets the number of seconds between polls for jobs when +execution_mode+ is set to +:async+. You can also set this with the environment variable +GOOD_JOB_POLL_INTERVAL+. Defaults to +1+.
22
+ # @param max_threads [Integer, nil] sets the number of threads per scheduler to use when +execution_mode+ is set to +:async+. The +queues+ parameter can specify a number of threads for each group of queues which will override this value. You can also set this with the environment variable +GOOD_JOB_MAX_THREADS+. Defaults to +5+.
23
+ # @param queues [String, nil] determines which queues to execute jobs from when +execution_mode+ is set to +:async+. See {file:README.md#optimize-queues-threads-and-processes} for more details on the format of this string. You can also set this with the environment variable +GOOD_JOB_QUEUES+. Defaults to +"*"+.
24
+ # @param poll_interval [Integer, nil] sets the number of seconds between polls for jobs when +execution_mode+ is set to +:async+. You can also set this with the environment variable +GOOD_JOB_POLL_INTERVAL+. Defaults to +1+.
25
25
  def initialize(execution_mode: nil, queues: nil, max_threads: nil, poll_interval: nil)
26
26
  if caller[0..4].find { |c| c.include?("/config/application.rb") || c.include?("/config/environments/") }
27
27
  ActiveSupport::Deprecation.warn(<<~DEPRECATION)
@@ -70,7 +70,7 @@ module GoodJob
70
70
  # Enqueues an ActiveJob job to be run at a specific time.
71
71
  # For use by Rails; you should generally not call this directly.
72
72
  # @param active_job [ActiveJob::Base] the job to be enqueued from +#perform_later+
73
- # @param timestamp [Integer] the epoch time to perform the job
73
+ # @param timestamp [Integer, nil] the epoch time to perform the job
74
74
  # @return [GoodJob::Job]
75
75
  def enqueue_at(active_job, timestamp)
76
76
  good_job = GoodJob::Job.enqueue(
@@ -97,22 +97,21 @@ module GoodJob
97
97
  end
98
98
 
99
99
  # Shut down the thread pool executors.
100
- # @param timeout [nil, Numeric] Seconds to wait for active threads.
101
- #
100
+ # @param timeout [nil, Numeric, Symbol] Seconds to wait for active threads.
102
101
  # * +nil+, the scheduler will trigger a shutdown but not wait for it to complete.
103
102
  # * +-1+, the scheduler will wait until the shutdown is complete.
104
103
  # * +0+, the scheduler will immediately shutdown and stop any threads.
105
104
  # * A positive number will wait that many seconds before stopping any remaining active threads.
106
- # @param wait [Boolean] Deprecated. Use +timeout:+ instead.
105
+ # @param wait [Boolean, nil] Deprecated. Use +timeout:+ instead.
107
106
  # @return [void]
108
107
  def shutdown(timeout: :default, wait: nil)
109
- timeout = if wait.present?
108
+ timeout = if wait.nil?
109
+ timeout
110
+ else
110
111
  ActiveSupport::Deprecation.warn(
111
112
  "Using `GoodJob::Adapter.shutdown` with `wait:` kwarg is deprecated; use `timeout:` kwarg instead e.g. GoodJob::Adapter.shutdown(timeout: #{wait ? '-1' : 'nil'})"
112
113
  )
113
114
  wait ? -1 : nil
114
- else
115
- timeout
116
115
  end
117
116
 
118
117
  timeout = if timeout == :default
@@ -126,18 +125,21 @@ module GoodJob
126
125
  end
127
126
 
128
127
  # Whether in +:async+ execution mode.
128
+ # @return [Boolean]
129
129
  def execute_async?
130
130
  @configuration.execution_mode == :async ||
131
131
  @configuration.execution_mode == :async_server && in_server_process?
132
132
  end
133
133
 
134
134
  # Whether in +:external+ execution mode.
135
+ # @return [Boolean]
135
136
  def execute_externally?
136
137
  @configuration.execution_mode == :external ||
137
138
  @configuration.execution_mode == :async_server && !in_server_process?
138
139
  end
139
140
 
140
141
  # Whether in +:inline+ execution mode.
142
+ # @return [Boolean]
141
143
  def execute_inline?
142
144
  @configuration.execution_mode == :inline
143
145
  end
@@ -145,11 +147,13 @@ module GoodJob
145
147
  private
146
148
 
147
149
  # Whether running in a web server process.
150
+ # @return [Boolean, nil]
148
151
  def in_server_process?
149
152
  return @_in_server_process if defined? @_in_server_process
150
153
 
151
154
  @_in_server_process = Rails.const_defined?('Server') ||
152
155
  caller.grep(%r{config.ru}).any? || # EXAMPLE: config.ru:3:in `block in <main>' OR config.ru:3:in `new_from_string'
156
+ caller.grep(%{/rack/handler/}).any? || # EXAMPLE: iodine-0.7.44/lib/rack/handler/iodine.rb:13:in `start'
153
157
  (Concurrent.on_jruby? && caller.grep(%r{jruby/rack/rails_booter}).any?) # EXAMPLE: uri:classloader:/jruby/rack/rails_booter.rb:83:in `load_environment'
154
158
  end
155
159
  end
@@ -84,9 +84,9 @@ module GoodJob
84
84
  # on the format of this string.
85
85
  # @return [String]
86
86
  def queue_string
87
- options[:queues] ||
88
- rails_config[:queues] ||
89
- env['GOOD_JOB_QUEUES'] ||
87
+ options[:queues].presence ||
88
+ rails_config[:queues].presence ||
89
+ env['GOOD_JOB_QUEUES'].presence ||
90
90
  '*'
91
91
  end
92
92
 
@@ -3,7 +3,6 @@ require 'active_support/core_ext/module/attribute_accessors_per_thread'
3
3
  module GoodJob
4
4
  # Thread-local attributes for passing values from Instrumentation.
5
5
  # (Cannot use ActiveSupport::CurrentAttributes because ActiveJob resets it)
6
-
7
6
  module CurrentExecution
8
7
  # @!attribute [rw] error_on_retry
9
8
  # @!scope class
@@ -13,6 +13,7 @@ module GoodJob
13
13
  end
14
14
 
15
15
  # Daemonizes the current process and writes out a pidfile.
16
+ # @return [void]
16
17
  def daemonize
17
18
  check_pid
18
19
  Process.daemon
@@ -21,6 +22,7 @@ module GoodJob
21
22
 
22
23
  private
23
24
 
25
+ # @return [void]
24
26
  def write_pid
25
27
  File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |f| f.write(Process.pid.to_s) }
26
28
  at_exit { File.delete(pidfile) if File.exist?(pidfile) }
@@ -29,10 +31,12 @@ module GoodJob
29
31
  retry
30
32
  end
31
33
 
34
+ # @return [void]
32
35
  def delete_pid
33
36
  File.delete(pidfile) if File.exist?(pidfile)
34
37
  end
35
38
 
39
+ # @return [void]
36
40
  def check_pid
37
41
  case pid_status(pidfile)
38
42
  when :running, :not_owned
@@ -42,6 +46,8 @@ module GoodJob
42
46
  end
43
47
  end
44
48
 
49
+ # @param pidfile [Pathname, String]
50
+ # @return [Symbol]
45
51
  def pid_status(pidfile)
46
52
  return :exited unless File.exist?(pidfile)
47
53
 
data/lib/good_job/job.rb CHANGED
@@ -50,6 +50,14 @@ module GoodJob
50
50
  end
51
51
  end
52
52
 
53
+ # Get Jobs with given class name
54
+ # @!method with_job_class
55
+ # @!scope class
56
+ # @param string [String]
57
+ # Job class name
58
+ # @return [ActiveRecord::Relation]
59
+ scope :with_job_class, ->(job_class) { where("serialized_params->>'job_class' = ?", job_class) }
60
+
53
61
  # Get Jobs that have not yet been completed.
54
62
  # @!method unfinished
55
63
  # @!scope class
@@ -94,6 +102,12 @@ module GoodJob
94
102
  # @return [ActiveRecord::Relation]
95
103
  scope :finished, ->(timestamp = nil) { timestamp ? where(arel_table['finished_at'].lteq(timestamp)) : where.not(finished_at: nil) }
96
104
 
105
+ # Get Jobs that started but not finished yet.
106
+ # @!method running
107
+ # @!scope class
108
+ # @return [ActiveRecord::Relation]
109
+ scope :running, -> { where.not(performed_at: nil).where(finished_at: nil) }
110
+
97
111
  # Get Jobs on queues that match the given queue string.
98
112
  # @!method queue_string(string)
99
113
  # @!scope class
@@ -142,10 +156,10 @@ module GoodJob
142
156
  # raised, if any (if the job raised, then the second array entry will be
143
157
  # +nil+). If there were no jobs to execute, returns +nil+.
144
158
  def self.perform_with_advisory_lock
145
- unfinished.priority_ordered.only_scheduled.limit(1).with_advisory_lock do |good_jobs|
159
+ unfinished.priority_ordered.only_scheduled.limit(1).with_advisory_lock(unlock_session: true) do |good_jobs|
146
160
  good_job = good_jobs.first
147
- # TODO: Determine why some records are fetched without an advisory lock at all
148
- break unless good_job&.executable?
161
+ break if good_job.blank?
162
+ break :unlocked unless good_job&.executable?
149
163
 
150
164
  good_job.perform
151
165
  end
@@ -182,13 +196,30 @@ module GoodJob
182
196
  # The new {Job} instance representing the queued ActiveJob job.
183
197
  def self.enqueue(active_job, scheduled_at: nil, create_with_advisory_lock: false)
184
198
  ActiveSupport::Notifications.instrument("enqueue_job.good_job", { active_job: active_job, scheduled_at: scheduled_at, create_with_advisory_lock: create_with_advisory_lock }) do |instrument_payload|
185
- good_job = GoodJob::Job.new(
199
+ good_job_args = {
186
200
  queue_name: active_job.queue_name.presence || DEFAULT_QUEUE_NAME,
187
201
  priority: active_job.priority || DEFAULT_PRIORITY,
188
202
  serialized_params: active_job.serialize,
189
203
  scheduled_at: scheduled_at,
190
- create_with_advisory_lock: create_with_advisory_lock
191
- )
204
+ create_with_advisory_lock: create_with_advisory_lock,
205
+ }
206
+
207
+ if column_names.include?('active_job_id')
208
+ good_job_args[:active_job_id] = active_job.job_id
209
+ else
210
+ ActiveSupport::Deprecation.warn(<<~DEPRECATION)
211
+ GoodJob has pending database migrations. To create the migration files, run:
212
+
213
+ rails generate good_job:update
214
+
215
+ To apply the migration files, run:
216
+
217
+ rails db:migrate
218
+
219
+ DEPRECATION
220
+ end
221
+
222
+ good_job = GoodJob::Job.new(**good_job_args)
192
223
 
193
224
  instrument_payload[:good_job] = good_job
194
225
 
@@ -235,7 +266,7 @@ module GoodJob
235
266
 
236
267
  private
237
268
 
238
- # @return [GoodJob::ExecutionResult]
269
+ # @return [ExecutionResult]
239
270
  def execute
240
271
  params = serialized_params.merge(
241
272
  "provider_job_id" => id
@@ -24,7 +24,7 @@ module GoodJob
24
24
  end
25
25
 
26
26
  # Perform the next eligible job
27
- # @return [nil, Object] Returns job result or +nil+ if no job was found
27
+ # @return [Object, nil] Returns job result or +nil+ if no job was found
28
28
  def next
29
29
  job_query.perform_with_advisory_lock
30
30
  end
@@ -54,7 +54,7 @@ module GoodJob
54
54
  # @param after [DateTime, Time, nil] future jobs scheduled after this time
55
55
  # @param limit [Integer] number of future timestamps to return
56
56
  # @param now_limit [Integer] number of past timestamps to return
57
- # @return [Array<(Time, DateTime)>, nil]
57
+ # @return [Array<DateTime, Time>, nil]
58
58
  def next_at(after: nil, limit: nil, now_limit: nil)
59
59
  job_query.next_scheduled_at(after: after, limit: limit, now_limit: now_limit)
60
60
  end
@@ -22,17 +22,25 @@ module GoodJob
22
22
  RecordAlreadyAdvisoryLockedError = Class.new(StandardError)
23
23
 
24
24
  included do
25
+ # Default column to be used when creating Advisory Locks
26
+ cattr_accessor(:advisory_lockable_column, instance_accessor: false) { primary_key }
27
+
28
+ # Default Postgres function to be used for Advisory Locks
29
+ cattr_accessor(:advisory_lockable_function) { "pg_try_advisory_lock" }
30
+
25
31
  # Attempt to acquire an advisory lock on the selected records and
26
32
  # return only those records for which a lock could be acquired.
27
- # @!method advisory_lock
33
+ # @!method advisory_lock(column: advisory_lockable_column, function: advisory_lockable_function)
28
34
  # @!scope class
35
+ # @param column [String, Symbol] column values to Advisory Lock against
36
+ # @param function [String, Symbol] Postgres Advisory Lock function name to use
29
37
  # @return [ActiveRecord::Relation]
30
38
  # A relation selecting only the records that were locked.
31
- scope :advisory_lock, (lambda do
39
+ scope :advisory_lock, (lambda do |column: advisory_lockable_column, function: advisory_lockable_function|
32
40
  original_query = self
33
41
 
34
42
  cte_table = Arel::Table.new(:rows)
35
- cte_query = original_query.select(primary_key).except(:limit)
43
+ cte_query = original_query.select(primary_key, column).except(:limit)
36
44
  cte_type = if supports_cte_materialization_specifiers?
37
45
  'MATERIALIZED'
38
46
  else
@@ -41,9 +49,14 @@ module GoodJob
41
49
 
42
50
  composed_cte = Arel::Nodes::As.new(cte_table, Arel::Nodes::SqlLiteral.new([cte_type, "(", cte_query.to_sql, ")"].join(' ')))
43
51
 
52
+ # In addition to an advisory lock, there is also a FOR UPDATE SKIP LOCKED
53
+ # because this causes the query to skip jobs that were completed (and deleted)
54
+ # by another session in the time since the table snapshot was taken.
55
+ # In rare cases under high concurrency levels, leaving this out can result in double executions.
44
56
  query = cte_table.project(cte_table[:id])
45
57
  .with(composed_cte)
46
- .where(Arel.sql(sanitize_sql_for_conditions(["pg_try_advisory_lock(('x' || substr(md5(:table_name || #{connection.quote_table_name(cte_table.name)}.#{quoted_primary_key}::text), 1, 16))::bit(64)::bigint)", { table_name: table_name }])))
58
+ .where(Arel.sql(sanitize_sql_for_conditions(["#{function}(('x' || substr(md5(:table_name || #{connection.quote_table_name(cte_table.name)}.#{connection.quote_column_name(column)}::text), 1, 16))::bit(64)::bigint)", { table_name: table_name }])))
59
+ .lock(Arel.sql("FOR UPDATE SKIP LOCKED"))
47
60
 
48
61
  limit = original_query.arel.ast.limit
49
62
  query.limit = limit.value if limit.present?
@@ -57,40 +70,44 @@ module GoodJob
57
70
  #
58
71
  # For details on +pg_locks+, see
59
72
  # {https://www.postgresql.org/docs/current/view-pg-locks.html}.
60
- # @!method joins_advisory_locks
73
+ # @!method joins_advisory_locks(column: advisory_lockable_column)
61
74
  # @!scope class
75
+ # @param column [String, Symbol] column values to Advisory Lock against
62
76
  # @return [ActiveRecord::Relation]
63
77
  # @example Get the records that have a session awaiting a lock:
64
78
  # MyLockableRecord.joins_advisory_locks.where("pg_locks.granted = ?", false)
65
- scope :joins_advisory_locks, (lambda do
79
+ scope :joins_advisory_locks, (lambda do |column: advisory_lockable_column|
66
80
  join_sql = <<~SQL.squish
67
81
  LEFT JOIN pg_locks ON pg_locks.locktype = 'advisory'
68
82
  AND pg_locks.objsubid = 1
69
- AND pg_locks.classid = ('x' || substr(md5(:table_name || #{quoted_table_name}.#{quoted_primary_key}::text), 1, 16))::bit(32)::int
70
- AND pg_locks.objid = (('x' || substr(md5(:table_name || #{quoted_table_name}.#{quoted_primary_key}::text), 1, 16))::bit(64) << 32)::bit(32)::int
83
+ AND pg_locks.classid = ('x' || substr(md5(:table_name || #{quoted_table_name}.#{connection.quote_column_name(column)}::text), 1, 16))::bit(32)::int
84
+ AND pg_locks.objid = (('x' || substr(md5(:table_name || #{quoted_table_name}.#{connection.quote_column_name(column)}::text), 1, 16))::bit(64) << 32)::bit(32)::int
71
85
  SQL
72
86
 
73
87
  joins(sanitize_sql_for_conditions([join_sql, { table_name: table_name }]))
74
88
  end)
75
89
 
76
90
  # Find records that do not have an advisory lock on them.
77
- # @!method advisory_unlocked
91
+ # @!method advisory_unlocked(column: advisory_lockable_column)
78
92
  # @!scope class
93
+ # @param column [String, Symbol] column values to Advisory Lock against
79
94
  # @return [ActiveRecord::Relation]
80
- scope :advisory_unlocked, -> { joins_advisory_locks.where(pg_locks: { locktype: nil }) }
95
+ scope :advisory_unlocked, ->(column: advisory_lockable_column) { joins_advisory_locks(column: column).where(pg_locks: { locktype: nil }) }
81
96
 
82
97
  # Find records that have an advisory lock on them.
83
- # @!method advisory_locked
98
+ # @!method advisory_locked(column: advisory_lockable_column)
84
99
  # @!scope class
100
+ # @param column [String, Symbol] column values to Advisory Lock against
85
101
  # @return [ActiveRecord::Relation]
86
- scope :advisory_locked, -> { joins_advisory_locks.where.not(pg_locks: { locktype: nil }) }
102
+ scope :advisory_locked, ->(column: advisory_lockable_column) { joins_advisory_locks(column: column).where.not(pg_locks: { locktype: nil }) }
87
103
 
88
104
  # Find records with advisory locks owned by the current Postgres
89
105
  # session/connection.
90
- # @!method advisory_locked
106
+ # @!method advisory_locked(column: advisory_lockable_column)
91
107
  # @!scope class
108
+ # @param column [String, Symbol] column values to Advisory Lock against
92
109
  # @return [ActiveRecord::Relation]
93
- scope :owns_advisory_locked, -> { joins_advisory_locks.where('"pg_locks"."pid" = pg_backend_pid()') }
110
+ scope :owns_advisory_locked, ->(column: advisory_lockable_column) { joins_advisory_locks(column: column).where('"pg_locks"."pid" = pg_backend_pid()') }
94
111
 
95
112
  # Whether an advisory lock should be acquired in the same transaction
96
113
  # that created the record.
@@ -122,6 +139,9 @@ module GoodJob
122
139
  # can (as in {Lockable.advisory_lock}) and only pass those that could be
123
140
  # locked to the block.
124
141
  #
142
+ # @param column [String, Symbol] name of advisory lock or unlock function
143
+ # @param function [String, Symbol] Postgres Advisory Lock function name to use
144
+ # @param unlock_session [Boolean] Whether to unlock all advisory locks in the session afterwards
125
145
  # @yield [Array<Lockable>] the records that were successfully locked.
126
146
  # @return [Object] the result of the block.
127
147
  #
@@ -129,14 +149,21 @@ module GoodJob
129
149
  # MyLockableRecord.order(created_at: :asc).limit(2).with_advisory_lock do |record|
130
150
  # do_something_with record
131
151
  # end
132
- def with_advisory_lock
152
+ def with_advisory_lock(column: advisory_lockable_column, function: advisory_lockable_function, unlock_session: false)
133
153
  raise ArgumentError, "Must provide a block" unless block_given?
134
154
 
135
- records = advisory_lock.to_a
155
+ records = advisory_lock(column: column, function: function).to_a
136
156
  begin
137
157
  yield(records)
138
158
  ensure
139
- records.each(&:advisory_unlock)
159
+ if unlock_session
160
+ advisory_unlock_session
161
+ else
162
+ records.each do |record|
163
+ key = [table_name, record[advisory_lockable_column]].join
164
+ record.advisory_unlock(key: key, function: advisory_unlockable_function(function))
165
+ end
166
+ end
140
167
  end
141
168
  end
142
169
 
@@ -145,49 +172,79 @@ module GoodJob
145
172
 
146
173
  @_supports_cte_materialization_specifiers = connection.postgresql_version >= 120000
147
174
  end
175
+
176
+ # Postgres advisory unlocking function for the class
177
+ # @param function [String, Symbol] name of advisory lock or unlock function
178
+ # @return [Boolean]
179
+ def advisory_unlockable_function(function = advisory_lockable_function)
180
+ function.to_s.sub("_lock", "_unlock").sub("_try_", "_")
181
+ end
182
+
183
+ # Unlocks all advisory locks active in the current database session/connection
184
+ # @return [void]
185
+ def advisory_unlock_session
186
+ connection.exec_query("SELECT pg_advisory_unlock_all()::text AS unlocked", 'GoodJob::Lockable Unlock Session').first[:unlocked]
187
+ end
188
+
189
+ # Converts SQL query strings between PG-compatible and JDBC-compatible syntax
190
+ # @param query [String]
191
+ # @return [Boolean]
192
+ def pg_or_jdbc_query(query)
193
+ if Concurrent.on_jruby?
194
+ # Replace $1 bind parameters with ?
195
+ query.gsub(/\$\d*/, '?')
196
+ else
197
+ query
198
+ end
199
+ end
148
200
  end
149
201
 
150
202
  # Acquires an advisory lock on this record if it is not already locked by
151
203
  # another database session. Be careful to ensure you release the lock when
152
204
  # you are done with {#advisory_unlock} (or {#advisory_unlock!} to release
153
205
  # all remaining locks).
206
+ # @param key [String, Symbol] Key to Advisory Lock against
207
+ # @param function [String, Symbol] Postgres Advisory Lock function name to use
154
208
  # @return [Boolean] whether the lock was acquired.
155
- def advisory_lock
209
+ def advisory_lock(key: lockable_key, function: advisory_lockable_function)
156
210
  query = <<~SQL.squish
157
- SELECT 1 AS one
158
- WHERE pg_try_advisory_lock(('x'||substr(md5($1 || $2::text), 1, 16))::bit(64)::bigint)
211
+ SELECT #{function}(('x'||substr(md5($1::text), 1, 16))::bit(64)::bigint) AS locked
159
212
  SQL
160
- binds = [[nil, self.class.table_name], [nil, send(self.class.primary_key)]]
161
- self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Lock', binds).any?
213
+ binds = [[nil, key]]
214
+ self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Lock', binds).first['locked']
162
215
  end
163
216
 
164
217
  # Releases an advisory lock on this record if it is locked by this database
165
218
  # session. Note that advisory locks stack, so you must call
166
219
  # {#advisory_unlock} and {#advisory_lock} the same number of times.
220
+ # @param key [String, Symbol] Key to lock against
221
+ # @param function [String, Symbol] Postgres Advisory Lock function name to use
167
222
  # @return [Boolean] whether the lock was released.
168
- def advisory_unlock
223
+ def advisory_unlock(key: lockable_key, function: self.class.advisory_unlockable_function(advisory_lockable_function))
169
224
  query = <<~SQL.squish
170
- SELECT 1 AS one
171
- WHERE pg_advisory_unlock(('x'||substr(md5($1 || $2::text), 1, 16))::bit(64)::bigint)
225
+ SELECT #{function}(('x'||substr(md5($1::text), 1, 16))::bit(64)::bigint) AS unlocked
172
226
  SQL
173
- binds = [[nil, self.class.table_name], [nil, send(self.class.primary_key)]]
174
- self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Unlock', binds).any?
227
+ binds = [[nil, key]]
228
+ self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Unlock', binds).first['unlocked']
175
229
  end
176
230
 
177
231
  # Acquires an advisory lock on this record or raises
178
232
  # {RecordAlreadyAdvisoryLockedError} if it is already locked by another
179
233
  # database session.
234
+ # @param key [String, Symbol] Key to lock against
235
+ # @param function [String, Symbol] Postgres Advisory Lock function name to use
180
236
  # @raise [RecordAlreadyAdvisoryLockedError]
181
237
  # @return [Boolean] +true+
182
- def advisory_lock!
183
- result = advisory_lock
238
+ def advisory_lock!(key: lockable_key, function: advisory_lockable_function)
239
+ result = advisory_lock(key: key, function: function)
184
240
  result || raise(RecordAlreadyAdvisoryLockedError)
185
241
  end
186
242
 
187
243
  # Acquires an advisory lock on this record and safely releases it after the
188
244
  # passed block is completed. If the record is locked by another database
189
245
  # session, this raises {RecordAlreadyAdvisoryLockedError}.
190
- #
246
+ # @param key [String, Symbol] Key to lock against
247
+ # @param function [String, Symbol] Postgres Advisory Lock function name to use
191
248
  # @yield Nothing
192
249
  # @return [Object] The result of the block.
193
250
  #
@@ -196,67 +253,63 @@ module GoodJob
196
253
  # record.with_advisory_lock do
197
254
  # do_something_with record
198
255
  # end
199
- def with_advisory_lock
256
+ def with_advisory_lock(key: lockable_key, function: advisory_lockable_function)
200
257
  raise ArgumentError, "Must provide a block" unless block_given?
201
258
 
202
- advisory_lock!
259
+ advisory_lock!(key: key, function: function)
203
260
  yield
204
261
  ensure
205
- advisory_unlock unless $ERROR_INFO.is_a? RecordAlreadyAdvisoryLockedError
262
+ advisory_unlock(key: key, function: self.class.advisory_unlockable_function(function)) unless $ERROR_INFO.is_a? RecordAlreadyAdvisoryLockedError
206
263
  end
207
264
 
208
265
  # Tests whether this record has an advisory lock on it.
266
+ # @param key [String, Symbol] Key to test lock against
209
267
  # @return [Boolean]
210
- def advisory_locked?
268
+ def advisory_locked?(key: lockable_key)
211
269
  query = <<~SQL.squish
212
270
  SELECT 1 AS one
213
271
  FROM pg_locks
214
272
  WHERE pg_locks.locktype = 'advisory'
215
273
  AND pg_locks.objsubid = 1
216
- AND pg_locks.classid = ('x' || substr(md5($1 || $2::text), 1, 16))::bit(32)::int
217
- AND pg_locks.objid = (('x' || substr(md5($3 || $4::text), 1, 16))::bit(64) << 32)::bit(32)::int
274
+ AND pg_locks.classid = ('x' || substr(md5($1::text), 1, 16))::bit(32)::int
275
+ AND pg_locks.objid = (('x' || substr(md5($2::text), 1, 16))::bit(64) << 32)::bit(32)::int
218
276
  SQL
219
- binds = [[nil, self.class.table_name], [nil, send(self.class.primary_key)], [nil, self.class.table_name], [nil, send(self.class.primary_key)]]
277
+ binds = [[nil, key], [nil, key]]
220
278
  self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Advisory Locked?', binds).any?
221
279
  end
222
280
 
223
281
  # Tests whether this record is locked by the current database session.
282
+ # @param key [String, Symbol] Key to test lock against
224
283
  # @return [Boolean]
225
- def owns_advisory_lock?
284
+ def owns_advisory_lock?(key: lockable_key)
226
285
  query = <<~SQL.squish
227
286
  SELECT 1 AS one
228
287
  FROM pg_locks
229
288
  WHERE pg_locks.locktype = 'advisory'
230
289
  AND pg_locks.objsubid = 1
231
- AND pg_locks.classid = ('x' || substr(md5($1 || $2::text), 1, 16))::bit(32)::int
232
- AND pg_locks.objid = (('x' || substr(md5($3 || $4::text), 1, 16))::bit(64) << 32)::bit(32)::int
290
+ AND pg_locks.classid = ('x' || substr(md5($1::text), 1, 16))::bit(32)::int
291
+ AND pg_locks.objid = (('x' || substr(md5($2::text), 1, 16))::bit(64) << 32)::bit(32)::int
233
292
  AND pg_locks.pid = pg_backend_pid()
234
293
  SQL
235
- binds = [[nil, self.class.table_name], [nil, send(self.class.primary_key)], [nil, self.class.table_name], [nil, send(self.class.primary_key)]]
294
+ binds = [[nil, key], [nil, key]]
236
295
  self.class.connection.exec_query(pg_or_jdbc_query(query), 'GoodJob::Lockable Owns Advisory Lock?', binds).any?
237
296
  end
238
297
 
239
298
  # Releases all advisory locks on the record that are held by the current
240
299
  # database session.
300
+ # @param key [String, Symbol] Key to lock against
301
+ # @param function [String, Symbol] Postgres Advisory Lock function name to use
241
302
  # @return [void]
242
- def advisory_unlock!
243
- advisory_unlock while advisory_locked?
303
+ def advisory_unlock!(key: lockable_key, function: self.class.advisory_unlockable_function(advisory_lockable_function))
304
+ advisory_unlock(key: key, function: function) while advisory_locked?
244
305
  end
245
306
 
246
- private
247
-
248
- def sanitize_sql_for_conditions(*args)
249
- # Made public in Rails 5.2
250
- self.class.send(:sanitize_sql_for_conditions, *args)
307
+ # Default Advisory Lock key
308
+ # @return [String]
309
+ def lockable_key
310
+ [self.class.table_name, self[self.class.advisory_lockable_column]].join
251
311
  end
252
312
 
253
- def pg_or_jdbc_query(query)
254
- if Concurrent.on_jruby?
255
- # Replace $1 bind parameters with ?
256
- query.gsub(/\$\d*/, '?')
257
- else
258
- query
259
- end
260
- end
313
+ delegate :pg_or_jdbc_query, to: :class
261
314
  end
262
315
  end