inst-jobs 2.4.7 → 2.4.9

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 48186ffba65e38c8e10702a68c93d9b4bb7f8ea13f98a7175e6755a1e693595f
4
- data.tar.gz: 8a6e51516ffeaaa31d8b3bcb07a25e859259b9e734c5a8e14af019d842a01ed7
3
+ metadata.gz: 8e17453e7b2cfd39920dfb36c615bcd813ecf3416985e405e26d5062716ccaa7
4
+ data.tar.gz: 17ad240a2304ca6ee648308199ac8d83d53498ee93cd130bf3cbd86dde729673
5
5
  SHA512:
6
- metadata.gz: 385dcba329c82516cc6ac3b17aa7254cb427fce7d964c40c20ddd64c73a04af0cc687e4626e0dc2af890bf10ef1ec534f355e0dc8203416120c36ec453a9eb71
7
- data.tar.gz: 42c9316bb0ab5ad237fb70719ddbfa9ba9dc7f4b8dfd75b13880e7be228f9a03b566eb94c0771bdd6672f92919b5c79db4dafd041b1194a3cd4a829a96c006c4
6
+ metadata.gz: 29ed31003455122f52a8f45ac22751acf3f71213e4f93fae280425d90f5a35919ef881220660c453469904cec107383d0764afa0d87458da3b4fdded272c5c78
7
+ data.tar.gz: a9cb6ff603ad8c683b326cebfdb24acef31a450f3e6fab88af3760adf8f726e4c4dc28797bbcb7bb5e38b0f2fef0af97ee3c145a8031db723e5c67bb9c94d385
@@ -0,0 +1,56 @@
1
+ # frozen_string_literal: true
2
+
3
+ class FixSingletonConditionInBeforeInsert < ActiveRecord::Migration[5.2]
4
+ def change
5
+ reversible do |direction|
6
+ direction.up do
7
+ execute(<<~SQL)
8
+ CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
9
+ BEGIN
10
+ IF NEW.strand IS NOT NULL THEN
11
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
12
+ IF (SELECT COUNT(*) FROM (
13
+ SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
14
+ ) s) = NEW.max_concurrent THEN
15
+ NEW.next_in_strand := false;
16
+ END IF;
17
+ END IF;
18
+ IF NEW.singleton IS NOT NULL THEN
19
+ -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
20
+ -- rather than doing a seq scan
21
+ PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
22
+ IF FOUND THEN
23
+ NEW.next_in_strand := false;
24
+ END IF;
25
+ END IF;
26
+ RETURN NEW;
27
+ END;
28
+ $$ LANGUAGE plpgsql;
29
+ SQL
30
+ end
31
+ direction.down do
32
+ execute(<<~SQL)
33
+ CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
34
+ BEGIN
35
+ IF NEW.strand IS NOT NULL THEN
36
+ PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
37
+ IF (SELECT COUNT(*) FROM (
38
+ SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
39
+ ) s) = NEW.max_concurrent THEN
40
+ NEW.next_in_strand := false;
41
+ END IF;
42
+ END IF;
43
+ IF NEW.singleton IS NOT NULL THEN
44
+ PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
45
+ IF FOUND THEN
46
+ NEW.next_in_strand := false;
47
+ END IF;
48
+ END IF;
49
+ RETURN NEW;
50
+ END;
51
+ $$ LANGUAGE plpgsql;
52
+ SQL
53
+ end
54
+ end
55
+ end
56
+ end
@@ -166,17 +166,36 @@ module Delayed
166
166
  pid_regex = pid || '(\d+)'
167
167
  regex = Regexp.new("^#{Regexp.escape(name)}:#{pid_regex}$")
168
168
  unlocked_jobs = 0
169
+ escaped_name = name.gsub("\\", "\\\\")
170
+ .gsub("%", "\\%")
171
+ .gsub("_", "\\_")
172
+ locked_by_like = "#{escaped_name}:%"
169
173
  running = false if pid
170
- running_jobs.each do |job|
171
- next unless job.locked_by =~ regex
172
-
173
- unless pid
174
- job_pid = $1.to_i
175
- running = Process.kill(0, job_pid) rescue false
176
- end
177
- unless running
178
- unlocked_jobs += 1
179
- job.reschedule("process died")
174
+ jobs = running_jobs.limit(100)
175
+ jobs = pid ? jobs.where(locked_by: "#{name}:#{pid}") : jobs.where("locked_by LIKE ?", locked_by_like)
176
+ ignores = []
177
+ loop do
178
+ batch_scope = ignores.empty? ? jobs : jobs.where.not(id: ignores)
179
+ batch = batch_scope.to_a
180
+ break if batch.empty?
181
+
182
+ batch.each do |job|
183
+ unless job.locked_by =~ regex
184
+ ignores << job.id
185
+ next
186
+ end
187
+
188
+ unless pid
189
+ job_pid = $1.to_i
190
+ running = Process.kill(0, job_pid) rescue false
191
+ end
192
+
193
+ if running
194
+ ignores << job.id
195
+ else
196
+ unlocked_jobs += 1
197
+ job.reschedule("process died")
198
+ end
180
199
  end
181
200
  end
182
201
  unlocked_jobs
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Delayed
4
- VERSION = "2.4.7"
4
+ VERSION = "2.4.9"
5
5
  end
@@ -28,7 +28,7 @@ module Delayed
28
28
  Delayed::Job.transaction do
29
29
  # this action is a special case, and SHOULD NOT be a periodic job
30
30
  # because if it gets wiped out suddenly during execution
31
- # it can't go clean up it's abandoned self. Therefore,
31
+ # it can't go clean up its abandoned self. Therefore,
32
32
  # we expect it to get run from it's own process forked from the job pool
33
33
  # and we try to get an advisory lock when it runs. If we succeed,
34
34
  # no other worker is trying to do this right now (and if we abandon the
@@ -36,6 +36,8 @@ module Delayed
36
36
  result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
37
37
  return unless result
38
38
 
39
+ horizon = 5.minutes.ago
40
+
39
41
  checker = Worker::HealthCheck.build(
40
42
  type: Settings.worker_health_check_type,
41
43
  config: Settings.worker_health_check_config,
@@ -43,13 +45,16 @@ module Delayed
43
45
  )
44
46
  live_workers = checker.live_workers
45
47
 
46
- Delayed::Job.running_jobs.each do |job|
47
- # prefetched jobs have their own way of automatically unlocking themselves
48
- next if job.locked_by.start_with?("prefetch:")
49
-
50
- next if live_workers.include?(job.locked_by)
48
+ loop do
49
+ batch = Delayed::Job.running_jobs
50
+ .where("locked_at<?", horizon)
51
+ .where.not("locked_by LIKE 'prefetch:%'")
52
+ .where.not(locked_by: live_workers)
53
+ .limit(100)
54
+ .to_a
55
+ break if batch.empty?
51
56
 
52
- begin
57
+ batch.each do |job|
53
58
  Delayed::Job.transaction do
54
59
  # double check that the job is still there. locked_by will immediately be reset
55
60
  # to nil in this transaction by Job#reschedule
@@ -59,9 +64,9 @@ module Delayed
59
64
 
60
65
  job.reschedule
61
66
  end
62
- rescue
63
- ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
64
67
  end
68
+ rescue
69
+ ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
65
70
  end
66
71
  end
67
72
  end
@@ -31,8 +31,8 @@ describe "Delayed::Backed::ActiveRecord::Job" do
31
31
  expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
32
32
  end
33
33
 
34
- it "doesn't allow a second worker to get exclusive access if failed to be processed by worker1 and
35
- run_at time is now in future (due to backing off behaviour)" do
34
+ it "doesn't allow a second worker to get exclusive access if failed to be " \
35
+ "processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
36
36
  @job.update(attempts: 1, run_at: 1.day.from_now)
37
37
  expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
38
38
  end
@@ -3,9 +3,10 @@
3
3
  require "spec_helper"
4
4
 
5
5
  RSpec.describe Delayed::Daemon do
6
+ subject { described_class.new(pid_folder) }
7
+
6
8
  let(:pid_folder) { "/test/pid/folder" }
7
9
  let(:pid) { 9999 }
8
- let(:subject) { described_class.new(pid_folder) }
9
10
 
10
11
  before do
11
12
  allow(subject).to receive(:pid).and_return(pid)
@@ -11,7 +11,6 @@ RSpec.describe Delayed::WorkQueue::InProcess do
11
11
  Delayed::Worker.lifecycle.reset!
12
12
  end
13
13
 
14
- let(:subject) { described_class.new }
15
14
  let(:worker_config) { { queue: "test", min_priority: 1, max_priority: 2 } }
16
15
  let(:args) { ["worker_name", worker_config] }
17
16
 
@@ -3,7 +3,8 @@
3
3
  require "spec_helper"
4
4
 
5
5
  RSpec.describe Delayed::WorkQueue::ParentProcess::Client do
6
- let(:subject) { described_class.new(addrinfo).tap(&:init) }
6
+ subject { described_class.new(addrinfo).tap(&:init) }
7
+
7
8
  let(:addrinfo) { double("Addrinfo") }
8
9
  let(:connection) { double("Socket") }
9
10
  let(:job) { Delayed::Job.new(locked_by: "worker_name") }
@@ -15,8 +15,9 @@ class JobClass
15
15
  end
16
16
 
17
17
  RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
18
+ subject { described_class.new(listen_socket) }
19
+
18
20
  let(:parent) { Delayed::WorkQueue::ParentProcess.new }
19
- let(:subject) { described_class.new(listen_socket) }
20
21
  let(:listen_socket) { Socket.unix_server_socket(parent.server_address) }
21
22
  let(:job) { JobClass.new }
22
23
  let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }
@@ -21,8 +21,6 @@ RSpec.describe Delayed::WorkQueue::ParentProcess do
21
21
  Delayed::Worker.lifecycle.reset!
22
22
  end
23
23
 
24
- let(:subject) { described_class.new }
25
-
26
24
  describe "#initalize(config = Settings.parent_process)" do
27
25
  it "must expand a relative path to be within the Rails root" do
28
26
  queue = described_class.new("server_address" => "tmp/foo.sock")
@@ -51,7 +51,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
51
51
  end
52
52
  end
53
53
 
54
- let(:initial_run_at) { Time.zone.now }
54
+ let(:initial_run_at) { 10.minutes.ago }
55
55
 
56
56
  before do
57
57
  klass.live_workers = %w[alive]
@@ -96,7 +96,13 @@ RSpec.describe Delayed::Worker::HealthCheck do
96
96
 
97
97
  it "ignores jobs that are re-locked after fetching from db" do
98
98
  Delayed::Job.where(id: @dead_job).update_all(locked_by: "someone_else")
99
- allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
99
+ # we need to return @dead_job itself, which doesn't match the database
100
+ jobs_scope = double
101
+ allow(jobs_scope).to receive(:where).and_return(jobs_scope)
102
+ allow(jobs_scope).to receive(:not).and_return(jobs_scope)
103
+ allow(jobs_scope).to receive(:limit).and_return(jobs_scope)
104
+ allow(jobs_scope).to receive(:to_a).and_return([@dead_job], [])
105
+ allow(Delayed::Job).to receive(:running_jobs).and_return(jobs_scope)
100
106
  described_class.reschedule_abandoned_jobs
101
107
  @dead_job.reload
102
108
  expect(@dead_job.locked_by).to eq "someone_else"
@@ -104,7 +110,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
104
110
 
105
111
  it "ignores jobs that are prefetched" do
106
112
  Delayed::Job.where(id: @dead_job).update_all(locked_by: "prefetch:some_node")
107
- allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
113
+ allow(Delayed::Job).to receive(:running_jobs).and_return(Delayed::Job.where(id: @dead_job.id))
108
114
  described_class.reschedule_abandoned_jobs
109
115
  @dead_job.reload
110
116
  expect(@dead_job.locked_by).to eq "prefetch:some_node"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: inst-jobs
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.4.7
4
+ version: 2.4.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tobias Luetke
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: exe
11
11
  cert_chain: []
12
- date: 2021-09-21 00:00:00.000000000 Z
12
+ date: 2021-09-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activerecord
@@ -432,6 +432,7 @@ files:
432
432
  - db/migrate/20210809145804_add_n_strand_index.rb
433
433
  - db/migrate/20210812210128_add_singleton_column.rb
434
434
  - db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
435
+ - db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
435
436
  - exe/inst_jobs
436
437
  - lib/delayed/backend/active_record.rb
437
438
  - lib/delayed/backend/base.rb