inst-jobs 2.4.7 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb +56 -0
- data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb +27 -0
- data/lib/delayed/backend/active_record.rb +3 -8
- data/lib/delayed/backend/base.rb +29 -10
- data/lib/delayed/version.rb +1 -1
- data/lib/delayed/work_queue/parent_process/server.rb +40 -13
- data/lib/delayed/worker/health_check.rb +14 -9
- data/spec/active_record_job_spec.rb +2 -2
- data/spec/delayed/daemon_spec.rb +2 -1
- data/spec/delayed/work_queue/in_process_spec.rb +0 -1
- data/spec/delayed/work_queue/parent_process/client_spec.rb +2 -1
- data/spec/delayed/work_queue/parent_process/server_spec.rb +74 -1
- data/spec/delayed/work_queue/parent_process_spec.rb +0 -2
- data/spec/delayed/worker/health_check_spec.rb +9 -3
- data/spec/spec_helper.rb +6 -5
- metadata +24 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8a9b37f7e43ec441a00056cbacce6f749e93de4496f4cbdff2c765015056ac71
|
4
|
+
data.tar.gz: ccbb0ed96932e436e109db2ae5c3076ce64245bc98a04424f5f9858f045d7a81
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 560e60386a4ea4e2851b3c6304ab38982134be7141c6d84671259f744c1d1fbb2dbb7af099ecbe2606fe35364b0d20dd54a6a6dccc6cea95ed02dc56c840d8d7
|
7
|
+
data.tar.gz: 9aceb6dc65083dc71a7f360d6a7e3fafbb8992f17b9a080e53e190d5a7772b417ef7caecd05f3c6f9fe8462e7ea9f67f788e9423d45dd251979547e3329924a0
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class FixSingletonConditionInBeforeInsert < ActiveRecord::Migration[5.2]
|
4
|
+
def change
|
5
|
+
reversible do |direction|
|
6
|
+
direction.up do
|
7
|
+
execute(<<~SQL)
|
8
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
|
9
|
+
BEGIN
|
10
|
+
IF NEW.strand IS NOT NULL THEN
|
11
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
12
|
+
IF (SELECT COUNT(*) FROM (
|
13
|
+
SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
|
14
|
+
) s) = NEW.max_concurrent THEN
|
15
|
+
NEW.next_in_strand := false;
|
16
|
+
END IF;
|
17
|
+
END IF;
|
18
|
+
IF NEW.singleton IS NOT NULL THEN
|
19
|
+
-- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
|
20
|
+
-- rather than doing a seq scan
|
21
|
+
PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
|
22
|
+
IF FOUND THEN
|
23
|
+
NEW.next_in_strand := false;
|
24
|
+
END IF;
|
25
|
+
END IF;
|
26
|
+
RETURN NEW;
|
27
|
+
END;
|
28
|
+
$$ LANGUAGE plpgsql;
|
29
|
+
SQL
|
30
|
+
end
|
31
|
+
direction.down do
|
32
|
+
execute(<<~SQL)
|
33
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
|
34
|
+
BEGIN
|
35
|
+
IF NEW.strand IS NOT NULL THEN
|
36
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
37
|
+
IF (SELECT COUNT(*) FROM (
|
38
|
+
SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
|
39
|
+
) s) = NEW.max_concurrent THEN
|
40
|
+
NEW.next_in_strand := false;
|
41
|
+
END IF;
|
42
|
+
END IF;
|
43
|
+
IF NEW.singleton IS NOT NULL THEN
|
44
|
+
PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
|
45
|
+
IF FOUND THEN
|
46
|
+
NEW.next_in_strand := false;
|
47
|
+
END IF;
|
48
|
+
END IF;
|
49
|
+
RETURN NEW;
|
50
|
+
END;
|
51
|
+
$$ LANGUAGE plpgsql;
|
52
|
+
SQL
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class UpdateConflictingSingletonFunctionToUseIndex < ActiveRecord::Migration[5.2]
|
4
|
+
def up
|
5
|
+
execute(<<~SQL)
|
6
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn () RETURNS trigger AS $$
|
7
|
+
BEGIN
|
8
|
+
DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton AND locked_by IS NULL;
|
9
|
+
RETURN NEW;
|
10
|
+
END;
|
11
|
+
$$ LANGUAGE plpgsql;
|
12
|
+
SQL
|
13
|
+
end
|
14
|
+
|
15
|
+
def down
|
16
|
+
execute(<<~SQL)
|
17
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn () RETURNS trigger AS $$
|
18
|
+
BEGIN
|
19
|
+
IF EXISTS (SELECT 1 FROM delayed_jobs j2 WHERE j2.singleton=OLD.singleton) THEN
|
20
|
+
DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton;
|
21
|
+
END IF;
|
22
|
+
RETURN NEW;
|
23
|
+
END;
|
24
|
+
$$ LANGUAGE plpgsql;
|
25
|
+
SQL
|
26
|
+
end
|
27
|
+
end
|
@@ -62,14 +62,9 @@ module Delayed
|
|
62
62
|
_write_attribute(column, current_time) unless attribute_present?(column)
|
63
63
|
end
|
64
64
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
values = attributes_with_values(attribute_names)
|
69
|
-
else
|
70
|
-
attribute_names = partial_writes? ? keys_for_partial_write : self.attribute_names
|
71
|
-
values = attributes_with_values_for_create(attribute_names)
|
72
|
-
end
|
65
|
+
attribute_names = attribute_names_for_partial_writes
|
66
|
+
attribute_names = attributes_for_create(attribute_names)
|
67
|
+
values = attributes_with_values(attribute_names)
|
73
68
|
|
74
69
|
im = self.class.arel_table.compile_insert(self.class.send(:_substitute_values, values))
|
75
70
|
|
data/lib/delayed/backend/base.rb
CHANGED
@@ -166,17 +166,36 @@ module Delayed
|
|
166
166
|
pid_regex = pid || '(\d+)'
|
167
167
|
regex = Regexp.new("^#{Regexp.escape(name)}:#{pid_regex}$")
|
168
168
|
unlocked_jobs = 0
|
169
|
+
escaped_name = name.gsub("\\", "\\\\")
|
170
|
+
.gsub("%", "\\%")
|
171
|
+
.gsub("_", "\\_")
|
172
|
+
locked_by_like = "#{escaped_name}:%"
|
169
173
|
running = false if pid
|
170
|
-
running_jobs.
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
job.
|
174
|
+
jobs = running_jobs.limit(100)
|
175
|
+
jobs = pid ? jobs.where(locked_by: "#{name}:#{pid}") : jobs.where("locked_by LIKE ?", locked_by_like)
|
176
|
+
ignores = []
|
177
|
+
loop do
|
178
|
+
batch_scope = ignores.empty? ? jobs : jobs.where.not(id: ignores)
|
179
|
+
batch = batch_scope.to_a
|
180
|
+
break if batch.empty?
|
181
|
+
|
182
|
+
batch.each do |job|
|
183
|
+
unless job.locked_by =~ regex
|
184
|
+
ignores << job.id
|
185
|
+
next
|
186
|
+
end
|
187
|
+
|
188
|
+
unless pid
|
189
|
+
job_pid = $1.to_i
|
190
|
+
running = Process.kill(0, job_pid) rescue false
|
191
|
+
end
|
192
|
+
|
193
|
+
if running
|
194
|
+
ignores << job.id
|
195
|
+
else
|
196
|
+
unlocked_jobs += 1
|
197
|
+
job.reschedule("process died")
|
198
|
+
end
|
180
199
|
end
|
181
200
|
end
|
182
201
|
unlocked_jobs
|
data/lib/delayed/version.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "activerecord-pg-extensions"
|
4
|
+
|
3
5
|
module Delayed
|
4
6
|
module WorkQueue
|
5
7
|
class ParentProcess
|
@@ -179,7 +181,7 @@ module Delayed
|
|
179
181
|
end
|
180
182
|
|
181
183
|
jobs_to_send.each do |(recipient, job_to_send)|
|
182
|
-
@waiting_clients[worker_config].delete(
|
184
|
+
@waiting_clients[worker_config].delete(recipient)
|
183
185
|
begin
|
184
186
|
logger.debug("Sending job #{job_to_send.id} to #{recipient.name}")
|
185
187
|
client_timeout { Marshal.dump(job_to_send, recipient.socket) }
|
@@ -192,29 +194,54 @@ module Delayed
|
|
192
194
|
end
|
193
195
|
end
|
194
196
|
|
195
|
-
def
|
197
|
+
def unlock_prefetched_jobs
|
196
198
|
@prefetched_jobs.each do |(worker_config, jobs)|
|
197
199
|
next if jobs.empty?
|
198
|
-
next
|
200
|
+
next if block_given? && !yield(jobs)
|
199
201
|
|
200
|
-
Delayed::Job.
|
202
|
+
connection = Delayed::Job.connection
|
203
|
+
connection.transaction do
|
204
|
+
# make absolutely sure we don't get hung up and leave things
|
205
|
+
# locked in the database
|
206
|
+
if connection.postgresql_version >= 9_06_00 # rubocop:disable Style/NumericLiterals
|
207
|
+
connection.idle_in_transaction_session_timeout = 5
|
208
|
+
end
|
209
|
+
# relatively short timeout for acquiring the lock
|
210
|
+
connection.statement_timeout = Settings.sleep_delay
|
201
211
|
Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
|
212
|
+
|
213
|
+
# this query might take longer, and we really want to get it
|
214
|
+
# done if we got the lock, but still don't want an inadvertent
|
215
|
+
# hang
|
216
|
+
connection.statement_timeout = 30
|
202
217
|
Delayed::Job.unlock(jobs)
|
218
|
+
@prefetched_jobs[worker_config] = []
|
203
219
|
end
|
204
|
-
|
220
|
+
rescue ActiveRecord::QueryCanceled
|
221
|
+
# ignore; we'll retry anyway
|
222
|
+
logger.warn("unable to unlock prefetched jobs; skipping for now")
|
223
|
+
rescue ActiveRecord::StatementInvalid
|
224
|
+
# see if we dropped the connection
|
225
|
+
raise if connection.active?
|
226
|
+
|
227
|
+
# otherwise just reconnect and let it retry
|
228
|
+
logger.warn("failed to unlock prefetched jobs - connection terminated; skipping for now")
|
229
|
+
Delayed::Job.clear_all_connections!
|
205
230
|
end
|
206
231
|
end
|
207
232
|
|
208
|
-
def
|
209
|
-
|
210
|
-
|
233
|
+
def unlock_timed_out_prefetched_jobs
|
234
|
+
unlock_prefetched_jobs do |jobs|
|
235
|
+
jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
|
236
|
+
end
|
237
|
+
end
|
211
238
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
239
|
+
def unlock_all_prefetched_jobs
|
240
|
+
# we try really hard; it may not have done any work if it timed out
|
241
|
+
10.times do
|
242
|
+
unlock_prefetched_jobs
|
243
|
+
break if @prefetched_jobs.each_value.all?(&:empty?)
|
216
244
|
end
|
217
|
-
@prefetched_jobs = {}
|
218
245
|
end
|
219
246
|
|
220
247
|
def drop_socket(socket)
|
@@ -28,7 +28,7 @@ module Delayed
|
|
28
28
|
Delayed::Job.transaction do
|
29
29
|
# this action is a special case, and SHOULD NOT be a periodic job
|
30
30
|
# because if it gets wiped out suddenly during execution
|
31
|
-
# it can't go clean up
|
31
|
+
# it can't go clean up its abandoned self. Therefore,
|
32
32
|
# we expect it to get run from it's own process forked from the job pool
|
33
33
|
# and we try to get an advisory lock when it runs. If we succeed,
|
34
34
|
# no other worker is trying to do this right now (and if we abandon the
|
@@ -36,6 +36,8 @@ module Delayed
|
|
36
36
|
result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
|
37
37
|
return unless result
|
38
38
|
|
39
|
+
horizon = 5.minutes.ago
|
40
|
+
|
39
41
|
checker = Worker::HealthCheck.build(
|
40
42
|
type: Settings.worker_health_check_type,
|
41
43
|
config: Settings.worker_health_check_config,
|
@@ -43,13 +45,16 @@ module Delayed
|
|
43
45
|
)
|
44
46
|
live_workers = checker.live_workers
|
45
47
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
48
|
+
loop do
|
49
|
+
batch = Delayed::Job.running_jobs
|
50
|
+
.where("locked_at<?", horizon)
|
51
|
+
.where.not("locked_by LIKE 'prefetch:%'")
|
52
|
+
.where.not(locked_by: live_workers)
|
53
|
+
.limit(100)
|
54
|
+
.to_a
|
55
|
+
break if batch.empty?
|
51
56
|
|
52
|
-
|
57
|
+
batch.each do |job|
|
53
58
|
Delayed::Job.transaction do
|
54
59
|
# double check that the job is still there. locked_by will immediately be reset
|
55
60
|
# to nil in this transaction by Job#reschedule
|
@@ -59,9 +64,9 @@ module Delayed
|
|
59
64
|
|
60
65
|
job.reschedule
|
61
66
|
end
|
62
|
-
rescue
|
63
|
-
::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
|
64
67
|
end
|
68
|
+
rescue
|
69
|
+
::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
|
65
70
|
end
|
66
71
|
end
|
67
72
|
end
|
@@ -31,8 +31,8 @@ describe "Delayed::Backed::ActiveRecord::Job" do
|
|
31
31
|
expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
|
32
32
|
end
|
33
33
|
|
34
|
-
it "doesn't
|
35
|
-
|
34
|
+
it "doesn't allow a second worker to get exclusive access if failed to be " \
|
35
|
+
"processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
|
36
36
|
@job.update(attempts: 1, run_at: 1.day.from_now)
|
37
37
|
expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
|
38
38
|
end
|
data/spec/delayed/daemon_spec.rb
CHANGED
@@ -3,9 +3,10 @@
|
|
3
3
|
require "spec_helper"
|
4
4
|
|
5
5
|
RSpec.describe Delayed::Daemon do
|
6
|
+
subject { described_class.new(pid_folder) }
|
7
|
+
|
6
8
|
let(:pid_folder) { "/test/pid/folder" }
|
7
9
|
let(:pid) { 9999 }
|
8
|
-
let(:subject) { described_class.new(pid_folder) }
|
9
10
|
|
10
11
|
before do
|
11
12
|
allow(subject).to receive(:pid).and_return(pid)
|
@@ -11,7 +11,6 @@ RSpec.describe Delayed::WorkQueue::InProcess do
|
|
11
11
|
Delayed::Worker.lifecycle.reset!
|
12
12
|
end
|
13
13
|
|
14
|
-
let(:subject) { described_class.new }
|
15
14
|
let(:worker_config) { { queue: "test", min_priority: 1, max_priority: 2 } }
|
16
15
|
let(:args) { ["worker_name", worker_config] }
|
17
16
|
|
@@ -3,7 +3,8 @@
|
|
3
3
|
require "spec_helper"
|
4
4
|
|
5
5
|
RSpec.describe Delayed::WorkQueue::ParentProcess::Client do
|
6
|
-
|
6
|
+
subject { described_class.new(addrinfo).tap(&:init) }
|
7
|
+
|
7
8
|
let(:addrinfo) { double("Addrinfo") }
|
8
9
|
let(:connection) { double("Socket") }
|
9
10
|
let(:job) { Delayed::Job.new(locked_by: "worker_name") }
|
@@ -15,14 +15,19 @@ class JobClass
|
|
15
15
|
end
|
16
16
|
|
17
17
|
RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
|
18
|
+
subject { described_class.new(listen_socket) }
|
19
|
+
|
18
20
|
let(:parent) { Delayed::WorkQueue::ParentProcess.new }
|
19
|
-
let(:subject) { described_class.new(listen_socket) }
|
20
21
|
let(:listen_socket) { Socket.unix_server_socket(parent.server_address) }
|
21
22
|
let(:job) { JobClass.new }
|
22
23
|
let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }
|
23
24
|
let(:args) { ["worker_name", worker_config] }
|
24
25
|
let(:job_args) { [["worker_name"], "queue_name", 1, 2, hash_including(prefetch: 4)] }
|
25
26
|
|
27
|
+
before do
|
28
|
+
Delayed::Worker.lifecycle.reset!
|
29
|
+
end
|
30
|
+
|
26
31
|
before :all do
|
27
32
|
Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
|
28
33
|
Delayed::Settings.parent_process = {
|
@@ -36,6 +41,7 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
|
|
36
41
|
|
37
42
|
after do
|
38
43
|
File.unlink("/tmp/inst-jobs-test.sock") if File.exist?("/tmp/inst-jobs-test.sock")
|
44
|
+
Delayed::Worker.lifecycle.reset!
|
39
45
|
end
|
40
46
|
|
41
47
|
it "accepts new clients" do
|
@@ -98,6 +104,53 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
|
|
98
104
|
expect(Marshal.load(client)).to eq(job2)
|
99
105
|
end
|
100
106
|
|
107
|
+
context "prefetched job unlocking" do
|
108
|
+
let(:job_args) do
|
109
|
+
[["worker_name1"], "queue_name", 1, 2,
|
110
|
+
{ prefetch: 4, prefetch_owner: "prefetch:work_queue:X", forced_latency: 6.0 }]
|
111
|
+
end
|
112
|
+
let(:job2) { Delayed::Job.new(tag: "tag").tap { |j| j.create_and_lock!("prefetch:work_queue:X") } }
|
113
|
+
let(:job3) { Delayed::Job.new(tag: "tag").tap { |j| j.create_and_lock!("prefetch:work_queue:X") } }
|
114
|
+
|
115
|
+
before do
|
116
|
+
client = Socket.unix(subject.listen_socket.local_address.unix_path)
|
117
|
+
subject.run_once
|
118
|
+
|
119
|
+
jobs = { "worker_name1" => job, "prefetch:work_queue:X" => [job2, job3] }
|
120
|
+
allow(subject).to receive(:prefetch_owner).and_return("prefetch:work_queue:X")
|
121
|
+
allow(Delayed::Job).to receive(:get_and_lock_next_available).once.with(*job_args).and_return(jobs)
|
122
|
+
Marshal.dump(["worker_name1", worker_config], client)
|
123
|
+
subject.run_once
|
124
|
+
end
|
125
|
+
|
126
|
+
it "doesn't unlock anything if nothing is timed out" do
|
127
|
+
expect(Delayed::Job).not_to receive(:advisory_lock)
|
128
|
+
expect(Delayed::Job).not_to receive(:unlock)
|
129
|
+
subject.unlock_timed_out_prefetched_jobs
|
130
|
+
end
|
131
|
+
|
132
|
+
it "unlocks timed out prefetched jobs" do
|
133
|
+
allow(Delayed::Settings).to receive(:parent_process).and_return(prefetched_jobs_timeout: -1)
|
134
|
+
expect(Delayed::Job).to receive(:unlock).with([job2, job3])
|
135
|
+
subject.unlock_timed_out_prefetched_jobs
|
136
|
+
expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 0
|
137
|
+
end
|
138
|
+
|
139
|
+
it "fails gracefully if the lock times out" do
|
140
|
+
allow(Delayed::Settings).to receive(:parent_process).and_return(prefetched_jobs_timeout: -1)
|
141
|
+
expect(Delayed::Job).not_to receive(:unlock)
|
142
|
+
expect(Delayed::Job).to receive(:advisory_lock).and_raise(ActiveRecord::QueryCanceled)
|
143
|
+
subject.unlock_timed_out_prefetched_jobs
|
144
|
+
expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 2
|
145
|
+
end
|
146
|
+
|
147
|
+
it "unlocks all jobs" do
|
148
|
+
expect(Delayed::Job).to receive(:unlock).with([job2, job3])
|
149
|
+
subject.unlock_all_prefetched_jobs
|
150
|
+
expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 0
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
101
154
|
it "doesn't respond immediately if there are no jobs available" do
|
102
155
|
client = Socket.unix(subject.listen_socket.local_address.unix_path)
|
103
156
|
subject.run_once
|
@@ -204,4 +257,24 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
|
|
204
257
|
expect(Marshal.load(client)).to eq(job)
|
205
258
|
expect(called).to eq(true)
|
206
259
|
end
|
260
|
+
|
261
|
+
it "deletes the correct worker when transferring jobs" do
|
262
|
+
client1 = Socket.unix(subject.listen_socket.local_address.unix_path)
|
263
|
+
client2 = Socket.unix(subject.listen_socket.local_address.unix_path)
|
264
|
+
subject.run_once
|
265
|
+
subject.run_once
|
266
|
+
|
267
|
+
Marshal.dump(args, client1)
|
268
|
+
Marshal.dump(["worker_name2", worker_config], client2)
|
269
|
+
subject.run_once
|
270
|
+
subject.run_once
|
271
|
+
|
272
|
+
waiting_clients = subject.instance_variable_get(:@waiting_clients)
|
273
|
+
expect(waiting_clients.first.last.length).to eq 2
|
274
|
+
|
275
|
+
expect(Delayed::Job).to receive(:get_and_lock_next_available).and_return("worker_name" => job,
|
276
|
+
"worker_name2" => job)
|
277
|
+
subject.run_once
|
278
|
+
expect(waiting_clients.first.last).to be_empty
|
279
|
+
end
|
207
280
|
end
|
@@ -21,8 +21,6 @@ RSpec.describe Delayed::WorkQueue::ParentProcess do
|
|
21
21
|
Delayed::Worker.lifecycle.reset!
|
22
22
|
end
|
23
23
|
|
24
|
-
let(:subject) { described_class.new }
|
25
|
-
|
26
24
|
describe "#initalize(config = Settings.parent_process)" do
|
27
25
|
it "must expand a relative path to be within the Rails root" do
|
28
26
|
queue = described_class.new("server_address" => "tmp/foo.sock")
|
@@ -51,7 +51,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
|
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
-
let(:initial_run_at) {
|
54
|
+
let(:initial_run_at) { 10.minutes.ago }
|
55
55
|
|
56
56
|
before do
|
57
57
|
klass.live_workers = %w[alive]
|
@@ -96,7 +96,13 @@ RSpec.describe Delayed::Worker::HealthCheck do
|
|
96
96
|
|
97
97
|
it "ignores jobs that are re-locked after fetching from db" do
|
98
98
|
Delayed::Job.where(id: @dead_job).update_all(locked_by: "someone_else")
|
99
|
-
|
99
|
+
# we need to return @dead_job itself, which doesn't match the database
|
100
|
+
jobs_scope = double
|
101
|
+
allow(jobs_scope).to receive(:where).and_return(jobs_scope)
|
102
|
+
allow(jobs_scope).to receive(:not).and_return(jobs_scope)
|
103
|
+
allow(jobs_scope).to receive(:limit).and_return(jobs_scope)
|
104
|
+
allow(jobs_scope).to receive(:to_a).and_return([@dead_job], [])
|
105
|
+
allow(Delayed::Job).to receive(:running_jobs).and_return(jobs_scope)
|
100
106
|
described_class.reschedule_abandoned_jobs
|
101
107
|
@dead_job.reload
|
102
108
|
expect(@dead_job.locked_by).to eq "someone_else"
|
@@ -104,7 +110,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
|
|
104
110
|
|
105
111
|
it "ignores jobs that are prefetched" do
|
106
112
|
Delayed::Job.where(id: @dead_job).update_all(locked_by: "prefetch:some_node")
|
107
|
-
allow(Delayed::Job).to receive(:running_jobs).and_return(
|
113
|
+
allow(Delayed::Job).to receive(:running_jobs).and_return(Delayed::Job.where(id: @dead_job.id))
|
108
114
|
described_class.reschedule_abandoned_jobs
|
109
115
|
@dead_job.reload
|
110
116
|
expect(@dead_job.locked_by).to eq "prefetch:some_node"
|
data/spec/spec_helper.rb
CHANGED
@@ -58,11 +58,7 @@ connection_config = {
|
|
58
58
|
}
|
59
59
|
|
60
60
|
def migrate(file)
|
61
|
-
|
62
|
-
ActiveRecord::MigrationContext.new(file, ActiveRecord::SchemaMigration).migrate
|
63
|
-
else
|
64
|
-
ActiveRecord::MigrationContext.new(file).migrate
|
65
|
-
end
|
61
|
+
ActiveRecord::MigrationContext.new(file, ActiveRecord::SchemaMigration).migrate
|
66
62
|
end
|
67
63
|
|
68
64
|
# create the test db if it does not exist, to help out wwtd
|
@@ -73,6 +69,11 @@ rescue ActiveRecord::StatementInvalid
|
|
73
69
|
nil
|
74
70
|
end
|
75
71
|
ActiveRecord::Base.establish_connection(connection_config)
|
72
|
+
|
73
|
+
# we need to ensure this callback is called for activerecord-pg-extensions,
|
74
|
+
# which isn't running because we're not using Rails to setup the database
|
75
|
+
ActiveRecord::PGExtensions::Railtie.run_initializers
|
76
|
+
|
76
77
|
# TODO: reset db and migrate again, to test migrations
|
77
78
|
|
78
79
|
migrate("db/migrate")
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: inst-jobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tobias Luetke
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-09-
|
12
|
+
date: 2021-09-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activerecord
|
@@ -17,28 +17,42 @@ dependencies:
|
|
17
17
|
requirements:
|
18
18
|
- - ">="
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: '
|
20
|
+
version: '6.0'
|
21
21
|
type: :runtime
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - ">="
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: '
|
27
|
+
version: '6.0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: activerecord-pg-extensions
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0.4'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0.4'
|
28
42
|
- !ruby/object:Gem::Dependency
|
29
43
|
name: activesupport
|
30
44
|
requirement: !ruby/object:Gem::Requirement
|
31
45
|
requirements:
|
32
46
|
- - ">="
|
33
47
|
- !ruby/object:Gem::Version
|
34
|
-
version: '
|
48
|
+
version: '6.0'
|
35
49
|
type: :runtime
|
36
50
|
prerelease: false
|
37
51
|
version_requirements: !ruby/object:Gem::Requirement
|
38
52
|
requirements:
|
39
53
|
- - ">="
|
40
54
|
- !ruby/object:Gem::Version
|
41
|
-
version: '
|
55
|
+
version: '6.0'
|
42
56
|
- !ruby/object:Gem::Dependency
|
43
57
|
name: after_transaction_commit
|
44
58
|
requirement: !ruby/object:Gem::Requirement
|
@@ -93,14 +107,14 @@ dependencies:
|
|
93
107
|
requirements:
|
94
108
|
- - ">="
|
95
109
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
110
|
+
version: '6.0'
|
97
111
|
type: :runtime
|
98
112
|
prerelease: false
|
99
113
|
version_requirements: !ruby/object:Gem::Requirement
|
100
114
|
requirements:
|
101
115
|
- - ">="
|
102
116
|
- !ruby/object:Gem::Version
|
103
|
-
version: '
|
117
|
+
version: '6.0'
|
104
118
|
- !ruby/object:Gem::Dependency
|
105
119
|
name: appraisal
|
106
120
|
requirement: !ruby/object:Gem::Requirement
|
@@ -432,6 +446,8 @@ files:
|
|
432
446
|
- db/migrate/20210809145804_add_n_strand_index.rb
|
433
447
|
- db/migrate/20210812210128_add_singleton_column.rb
|
434
448
|
- db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
|
449
|
+
- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
|
450
|
+
- db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
|
435
451
|
- exe/inst_jobs
|
436
452
|
- lib/delayed/backend/active_record.rb
|
437
453
|
- lib/delayed/backend/base.rb
|