inst-jobs 2.4.4 → 2.4.9

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 90e78e46299d111959e7e48f25fe4e05a7939c6fb54fcd423deff4b960e79e6c
4
- data.tar.gz: 20963910cd2cb1856f73cabbdc1547131a0d5096e26f7cc444f0a6e7d26b0ded
3
+ metadata.gz: 8e17453e7b2cfd39920dfb36c615bcd813ecf3416985e405e26d5062716ccaa7
4
+ data.tar.gz: 17ad240a2304ca6ee648308199ac8d83d53498ee93cd130bf3cbd86dde729673
5
5
  SHA512:
6
- metadata.gz: 22155dba3fd9f1201e67930be8a3cc488da21b02b2066b59fe9a0f28f5670393534ddbb87a937979df356ba602c23a27e97f7650463fa2eb7c53e76647df5f02
7
- data.tar.gz: 41a6bec7f263657aac33b62eeb8169db1898dfdcdbfadc32d91a985f54cb037a5691d75f02dff80b9c80ea57387a746fa70e6710545fe6b9c12724dde9f8f720
6
+ metadata.gz: 29ed31003455122f52a8f45ac22751acf3f71213e4f93fae280425d90f5a35919ef881220660c453469904cec107383d0764afa0d87458da3b4fdded272c5c78
7
+ data.tar.gz: a9cb6ff603ad8c683b326cebfdb24acef31a450f3e6fab88af3760adf8f726e4c4dc28797bbcb7bb5e38b0f2fef0af97ee3c145a8031db723e5c67bb9c94d385
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ class FixSingletonConditionInBeforeInsert < ActiveRecord::Migration[5.2]
4
+ def change
5
+ reversible do |direction|
6
+ direction.up do
7
+ execute(<<~SQL)
8
+ CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
9
+ BEGIN
10
+ IF NEW.strand IS NOT NULL THEN
11
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
12
+ IF (SELECT COUNT(*) FROM (
13
+ SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
14
+ ) s) = NEW.max_concurrent THEN
15
+ NEW.next_in_strand := false;
16
+ END IF;
17
+ END IF;
18
+ IF NEW.singleton IS NOT NULL THEN
19
+ -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
20
+ -- rather than doing a seq scan
21
+ PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
22
+ IF FOUND THEN
23
+ NEW.next_in_strand := false;
24
+ END IF;
25
+ END IF;
26
+ RETURN NEW;
27
+ END;
28
+ $$ LANGUAGE plpgsql;
29
+ SQL
30
+ end
31
+ direction.down do
32
+ execute(<<~SQL)
33
+ CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
34
+ BEGIN
35
+ IF NEW.strand IS NOT NULL THEN
36
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
37
+ IF (SELECT COUNT(*) FROM (
38
+ SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
39
+ ) s) = NEW.max_concurrent THEN
40
+ NEW.next_in_strand := false;
41
+ END IF;
42
+ END IF;
43
+ IF NEW.singleton IS NOT NULL THEN
44
+ PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
45
+ IF FOUND THEN
46
+ NEW.next_in_strand := false;
47
+ END IF;
48
+ END IF;
49
+ RETURN NEW;
50
+ END;
51
+ $$ LANGUAGE plpgsql;
52
+ SQL
53
+ end
54
+ end
55
+ end
56
+ end
@@ -38,6 +38,16 @@ module Delayed
38
38
  job = new(attributes, &block)
39
39
  job.single_step_create(on_conflict: on_conflict)
40
40
  end
41
+
42
+ def attempt_advisory_lock(lock_name)
43
+ fn_name = connection.quote_table_name("half_md5_as_bigint")
44
+ connection.select_value("SELECT pg_try_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
45
+ end
46
+
47
+ def advisory_lock(lock_name)
48
+ fn_name = connection.quote_table_name("half_md5_as_bigint")
49
+ connection.execute("SELECT pg_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
50
+ end
41
51
  end
42
52
 
43
53
  def single_step_create(on_conflict: nil)
@@ -458,9 +468,19 @@ module Delayed
458
468
  where("locked_by LIKE ?", "#{name}:%").pluck(:locked_by).map { |locked_by| locked_by.split(":").last.to_i }
459
469
  end
460
470
 
471
+ def self.prefetch_jobs_lock_name
472
+ "Delayed::Job.unlock_orphaned_prefetched_jobs"
473
+ end
474
+
461
475
  def self.unlock_orphaned_prefetched_jobs
462
- horizon = db_time_now - (Settings.parent_process[:prefetched_jobs_timeout] * 4)
463
- where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
476
+ transaction do
477
+ # for db performance reasons, we only need one process doing this at a time
478
+ # so if we can't get an advisory lock, just abort. we'll try again soon
479
+ return unless attempt_advisory_lock(prefetch_jobs_lock_name)
480
+
481
+ horizon = db_time_now - (Settings.parent_process[:prefetched_jobs_timeout] * 4)
482
+ where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
483
+ end
464
484
  end
465
485
 
466
486
  def self.unlock(jobs)
@@ -166,17 +166,36 @@ module Delayed
166
166
  pid_regex = pid || '(\d+)'
167
167
  regex = Regexp.new("^#{Regexp.escape(name)}:#{pid_regex}$")
168
168
  unlocked_jobs = 0
169
+ escaped_name = name.gsub("\\", "\\\\")
170
+ .gsub("%", "\\%")
171
+ .gsub("_", "\\_")
172
+ locked_by_like = "#{escaped_name}:%"
169
173
  running = false if pid
170
- running_jobs.each do |job|
171
- next unless job.locked_by =~ regex
172
-
173
- unless pid
174
- job_pid = $1.to_i
175
- running = Process.kill(0, job_pid) rescue false
176
- end
177
- unless running
178
- unlocked_jobs += 1
179
- job.reschedule("process died")
174
+ jobs = running_jobs.limit(100)
175
+ jobs = pid ? jobs.where(locked_by: "#{name}:#{pid}") : jobs.where("locked_by LIKE ?", locked_by_like)
176
+ ignores = []
177
+ loop do
178
+ batch_scope = ignores.empty? ? jobs : jobs.where.not(id: ignores)
179
+ batch = batch_scope.to_a
180
+ break if batch.empty?
181
+
182
+ batch.each do |job|
183
+ unless job.locked_by =~ regex
184
+ ignores << job.id
185
+ next
186
+ end
187
+
188
+ unless pid
189
+ job_pid = $1.to_i
190
+ running = Process.kill(0, job_pid) rescue false
191
+ end
192
+
193
+ if running
194
+ ignores << job.id
195
+ else
196
+ unlocked_jobs += 1
197
+ job.reschedule("process died")
198
+ end
180
199
  end
181
200
  end
182
201
  unlocked_jobs
@@ -33,7 +33,13 @@ module Delayed
33
33
  # we used to queue up a job in a strand here, and perform the audit inside that job
34
34
  # however, now that we're using singletons for scheduling periodic jobs,
35
35
  # it's fine to just do the audit in-line here without risk of creating duplicates
36
- perform_audit!
36
+ Delayed::Job.transaction do
37
+ # for db performance reasons, we only need one process doing this at a time
38
+ # so if we can't get an advisory lock, just abort. we'll try again soon
39
+ return unless Delayed::Job.attempt_advisory_lock("Delayed::Periodic#audit_queue")
40
+
41
+ perform_audit!
42
+ end
37
43
  end
38
44
 
39
45
  # make sure all periodic jobs are scheduled for their next run in the job queue
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Delayed
4
- VERSION = "2.4.4"
4
+ VERSION = "2.4.9"
5
5
  end
@@ -195,11 +195,13 @@ module Delayed
195
195
  def unlock_timed_out_prefetched_jobs
196
196
  @prefetched_jobs.each do |(worker_config, jobs)|
197
197
  next if jobs.empty?
198
+ next unless jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
198
199
 
199
- if jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
200
+ Delayed::Job.transaction do
201
+ Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
200
202
  Delayed::Job.unlock(jobs)
201
- @prefetched_jobs[worker_config] = []
202
203
  end
204
+ @prefetched_jobs[worker_config] = []
203
205
  end
204
206
  end
205
207
 
@@ -207,7 +209,10 @@ module Delayed
207
209
  @prefetched_jobs.each do |(_worker_config, jobs)|
208
210
  next if jobs.empty?
209
211
 
210
- Delayed::Job.unlock(jobs)
212
+ Delayed::Job.transaction do
213
+ Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
214
+ Delayed::Job.unlock(jobs)
215
+ end
211
216
  end
212
217
  @prefetched_jobs = {}
213
218
  end
@@ -28,14 +28,16 @@ module Delayed
28
28
  Delayed::Job.transaction do
29
29
  # this action is a special case, and SHOULD NOT be a periodic job
30
30
  # because if it gets wiped out suddenly during execution
31
- # it can't go clean up it's abandoned self. Therefore,
31
+ # it can't go clean up its abandoned self. Therefore,
32
32
  # we expect it to get run from it's own process forked from the job pool
33
33
  # and we try to get an advisory lock when it runs. If we succeed,
34
34
  # no other worker is trying to do this right now (and if we abandon the
35
35
  # operation, the transaction will end, releasing the advisory lock).
36
- result = attempt_advisory_lock
36
+ result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
37
37
  return unless result
38
38
 
39
+ horizon = 5.minutes.ago
40
+
39
41
  checker = Worker::HealthCheck.build(
40
42
  type: Settings.worker_health_check_type,
41
43
  config: Settings.worker_health_check_config,
@@ -43,13 +45,16 @@ module Delayed
43
45
  )
44
46
  live_workers = checker.live_workers
45
47
 
46
- Delayed::Job.running_jobs.each do |job|
47
- # prefetched jobs have their own way of automatically unlocking themselves
48
- next if job.locked_by.start_with?("prefetch:")
49
-
50
- next if live_workers.include?(job.locked_by)
48
+ loop do
49
+ batch = Delayed::Job.running_jobs
50
+ .where("locked_at<?", horizon)
51
+ .where.not("locked_by LIKE 'prefetch:%'")
52
+ .where.not(locked_by: live_workers)
53
+ .limit(100)
54
+ .to_a
55
+ break if batch.empty?
51
56
 
52
- begin
57
+ batch.each do |job|
53
58
  Delayed::Job.transaction do
54
59
  # double check that the job is still there. locked_by will immediately be reset
55
60
  # to nil in this transaction by Job#reschedule
@@ -59,19 +64,12 @@ module Delayed
59
64
 
60
65
  job.reschedule
61
66
  end
62
- rescue
63
- ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
64
67
  end
68
+ rescue
69
+ ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
65
70
  end
66
71
  end
67
72
  end
68
-
69
- def attempt_advisory_lock
70
- lock_name = "Delayed::Worker::HealthCheck#reschedule_abandoned_jobs"
71
- conn = ActiveRecord::Base.connection
72
- fn_name = conn.quote_table_name("half_md5_as_bigint")
73
- conn.select_value("SELECT pg_try_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
74
- end
75
73
  end
76
74
 
77
75
  attr_accessor :config, :worker_name
@@ -31,8 +31,8 @@ describe "Delayed::Backed::ActiveRecord::Job" do
31
31
  expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
32
32
  end
33
33
 
34
- it "doesn't allow a second worker to get exclusive access if failed to be processed by worker1 and
35
- run_at time is now in future (due to backing off behaviour)" do
34
+ it "doesn't allow a second worker to get exclusive access if failed to be " \
35
+ "processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
36
36
  @job.update(attempts: 1, run_at: 1.day.from_now)
37
37
  expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
38
38
  end
@@ -3,9 +3,10 @@
3
3
  require "spec_helper"
4
4
 
5
5
  RSpec.describe Delayed::Daemon do
6
+ subject { described_class.new(pid_folder) }
7
+
6
8
  let(:pid_folder) { "/test/pid/folder" }
7
9
  let(:pid) { 9999 }
8
- let(:subject) { described_class.new(pid_folder) }
9
10
 
10
11
  before do
11
12
  allow(subject).to receive(:pid).and_return(pid)
@@ -11,7 +11,6 @@ RSpec.describe Delayed::WorkQueue::InProcess do
11
11
  Delayed::Worker.lifecycle.reset!
12
12
  end
13
13
 
14
- let(:subject) { described_class.new }
15
14
  let(:worker_config) { { queue: "test", min_priority: 1, max_priority: 2 } }
16
15
  let(:args) { ["worker_name", worker_config] }
17
16
 
@@ -3,7 +3,8 @@
3
3
  require "spec_helper"
4
4
 
5
5
  RSpec.describe Delayed::WorkQueue::ParentProcess::Client do
6
- let(:subject) { described_class.new(addrinfo).tap(&:init) }
6
+ subject { described_class.new(addrinfo).tap(&:init) }
7
+
7
8
  let(:addrinfo) { double("Addrinfo") }
8
9
  let(:connection) { double("Socket") }
9
10
  let(:job) { Delayed::Job.new(locked_by: "worker_name") }
@@ -15,8 +15,9 @@ class JobClass
15
15
  end
16
16
 
17
17
  RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
18
+ subject { described_class.new(listen_socket) }
19
+
18
20
  let(:parent) { Delayed::WorkQueue::ParentProcess.new }
19
- let(:subject) { described_class.new(listen_socket) }
20
21
  let(:listen_socket) { Socket.unix_server_socket(parent.server_address) }
21
22
  let(:job) { JobClass.new }
22
23
  let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }
@@ -21,8 +21,6 @@ RSpec.describe Delayed::WorkQueue::ParentProcess do
21
21
  Delayed::Worker.lifecycle.reset!
22
22
  end
23
23
 
24
- let(:subject) { described_class.new }
25
-
26
24
  describe "#initalize(config = Settings.parent_process)" do
27
25
  it "must expand a relative path to be within the Rails root" do
28
26
  queue = described_class.new("server_address" => "tmp/foo.sock")
@@ -51,7 +51,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
51
51
  end
52
52
  end
53
53
 
54
- let(:initial_run_at) { Time.zone.now }
54
+ let(:initial_run_at) { 10.minutes.ago }
55
55
 
56
56
  before do
57
57
  klass.live_workers = %w[alive]
@@ -96,7 +96,13 @@ RSpec.describe Delayed::Worker::HealthCheck do
96
96
 
97
97
  it "ignores jobs that are re-locked after fetching from db" do
98
98
  Delayed::Job.where(id: @dead_job).update_all(locked_by: "someone_else")
99
- allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
99
+ # we need to return @dead_job itself, which doesn't match the database
100
+ jobs_scope = double
101
+ allow(jobs_scope).to receive(:where).and_return(jobs_scope)
102
+ allow(jobs_scope).to receive(:not).and_return(jobs_scope)
103
+ allow(jobs_scope).to receive(:limit).and_return(jobs_scope)
104
+ allow(jobs_scope).to receive(:to_a).and_return([@dead_job], [])
105
+ allow(Delayed::Job).to receive(:running_jobs).and_return(jobs_scope)
100
106
  described_class.reschedule_abandoned_jobs
101
107
  @dead_job.reload
102
108
  expect(@dead_job.locked_by).to eq "someone_else"
@@ -104,14 +110,14 @@ RSpec.describe Delayed::Worker::HealthCheck do
104
110
 
105
111
  it "ignores jobs that are prefetched" do
106
112
  Delayed::Job.where(id: @dead_job).update_all(locked_by: "prefetch:some_node")
107
- allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
113
+ allow(Delayed::Job).to receive(:running_jobs).and_return(Delayed::Job.where(id: @dead_job.id))
108
114
  described_class.reschedule_abandoned_jobs
109
115
  @dead_job.reload
110
116
  expect(@dead_job.locked_by).to eq "prefetch:some_node"
111
117
  end
112
118
 
113
119
  it "bails immediately if advisory lock already taken" do
114
- allow(described_class).to receive(:attempt_advisory_lock).and_return(false)
120
+ allow(Delayed::Job).to receive(:attempt_advisory_lock).and_return(false)
115
121
  described_class.reschedule_abandoned_jobs
116
122
  @dead_job.reload
117
123
  expect(@dead_job.run_at.to_i).to eq(initial_run_at.to_i)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: inst-jobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.4.4
4
+ version: 2.4.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tobias Luetke
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: exe
11
11
  cert_chain: []
12
- date: 2021-09-18 00:00:00.000000000 Z
12
+ date: 2021-09-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activerecord
@@ -432,6 +432,7 @@ files:
432
432
  - db/migrate/20210809145804_add_n_strand_index.rb
433
433
  - db/migrate/20210812210128_add_singleton_column.rb
434
434
  - db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
435
+ - db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
435
436
  - exe/inst_jobs
436
437
  - lib/delayed/backend/active_record.rb
437
438
  - lib/delayed/backend/base.rb