inst-jobs 2.4.7 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb +56 -0
- data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb +27 -0
- data/lib/delayed/backend/active_record.rb +3 -8
- data/lib/delayed/backend/base.rb +29 -10
- data/lib/delayed/version.rb +1 -1
- data/lib/delayed/work_queue/parent_process/server.rb +40 -13
- data/lib/delayed/worker/health_check.rb +14 -9
- data/spec/active_record_job_spec.rb +2 -2
- data/spec/delayed/daemon_spec.rb +2 -1
- data/spec/delayed/work_queue/in_process_spec.rb +0 -1
- data/spec/delayed/work_queue/parent_process/client_spec.rb +2 -1
- data/spec/delayed/work_queue/parent_process/server_spec.rb +74 -1
- data/spec/delayed/work_queue/parent_process_spec.rb +0 -2
- data/spec/delayed/worker/health_check_spec.rb +9 -3
- data/spec/spec_helper.rb +6 -5
- metadata +24 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8a9b37f7e43ec441a00056cbacce6f749e93de4496f4cbdff2c765015056ac71
|
4
|
+
data.tar.gz: ccbb0ed96932e436e109db2ae5c3076ce64245bc98a04424f5f9858f045d7a81
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 560e60386a4ea4e2851b3c6304ab38982134be7141c6d84671259f744c1d1fbb2dbb7af099ecbe2606fe35364b0d20dd54a6a6dccc6cea95ed02dc56c840d8d7
|
7
|
+
data.tar.gz: 9aceb6dc65083dc71a7f360d6a7e3fafbb8992f17b9a080e53e190d5a7772b417ef7caecd05f3c6f9fe8462e7ea9f67f788e9423d45dd251979547e3329924a0
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class FixSingletonConditionInBeforeInsert < ActiveRecord::Migration[5.2]
|
4
|
+
def change
|
5
|
+
reversible do |direction|
|
6
|
+
direction.up do
|
7
|
+
execute(<<~SQL)
|
8
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
|
9
|
+
BEGIN
|
10
|
+
IF NEW.strand IS NOT NULL THEN
|
11
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
12
|
+
IF (SELECT COUNT(*) FROM (
|
13
|
+
SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
|
14
|
+
) s) = NEW.max_concurrent THEN
|
15
|
+
NEW.next_in_strand := false;
|
16
|
+
END IF;
|
17
|
+
END IF;
|
18
|
+
IF NEW.singleton IS NOT NULL THEN
|
19
|
+
-- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
|
20
|
+
-- rather than doing a seq scan
|
21
|
+
PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
|
22
|
+
IF FOUND THEN
|
23
|
+
NEW.next_in_strand := false;
|
24
|
+
END IF;
|
25
|
+
END IF;
|
26
|
+
RETURN NEW;
|
27
|
+
END;
|
28
|
+
$$ LANGUAGE plpgsql;
|
29
|
+
SQL
|
30
|
+
end
|
31
|
+
direction.down do
|
32
|
+
execute(<<~SQL)
|
33
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
|
34
|
+
BEGIN
|
35
|
+
IF NEW.strand IS NOT NULL THEN
|
36
|
+
PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
|
37
|
+
IF (SELECT COUNT(*) FROM (
|
38
|
+
SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
|
39
|
+
) s) = NEW.max_concurrent THEN
|
40
|
+
NEW.next_in_strand := false;
|
41
|
+
END IF;
|
42
|
+
END IF;
|
43
|
+
IF NEW.singleton IS NOT NULL THEN
|
44
|
+
PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
|
45
|
+
IF FOUND THEN
|
46
|
+
NEW.next_in_strand := false;
|
47
|
+
END IF;
|
48
|
+
END IF;
|
49
|
+
RETURN NEW;
|
50
|
+
END;
|
51
|
+
$$ LANGUAGE plpgsql;
|
52
|
+
SQL
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class UpdateConflictingSingletonFunctionToUseIndex < ActiveRecord::Migration[5.2]
|
4
|
+
def up
|
5
|
+
execute(<<~SQL)
|
6
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn () RETURNS trigger AS $$
|
7
|
+
BEGIN
|
8
|
+
DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton AND locked_by IS NULL;
|
9
|
+
RETURN NEW;
|
10
|
+
END;
|
11
|
+
$$ LANGUAGE plpgsql;
|
12
|
+
SQL
|
13
|
+
end
|
14
|
+
|
15
|
+
def down
|
16
|
+
execute(<<~SQL)
|
17
|
+
CREATE OR REPLACE FUNCTION delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn () RETURNS trigger AS $$
|
18
|
+
BEGIN
|
19
|
+
IF EXISTS (SELECT 1 FROM delayed_jobs j2 WHERE j2.singleton=OLD.singleton) THEN
|
20
|
+
DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton;
|
21
|
+
END IF;
|
22
|
+
RETURN NEW;
|
23
|
+
END;
|
24
|
+
$$ LANGUAGE plpgsql;
|
25
|
+
SQL
|
26
|
+
end
|
27
|
+
end
|
@@ -62,14 +62,9 @@ module Delayed
|
|
62
62
|
_write_attribute(column, current_time) unless attribute_present?(column)
|
63
63
|
end
|
64
64
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
values = attributes_with_values(attribute_names)
|
69
|
-
else
|
70
|
-
attribute_names = partial_writes? ? keys_for_partial_write : self.attribute_names
|
71
|
-
values = attributes_with_values_for_create(attribute_names)
|
72
|
-
end
|
65
|
+
attribute_names = attribute_names_for_partial_writes
|
66
|
+
attribute_names = attributes_for_create(attribute_names)
|
67
|
+
values = attributes_with_values(attribute_names)
|
73
68
|
|
74
69
|
im = self.class.arel_table.compile_insert(self.class.send(:_substitute_values, values))
|
75
70
|
|
data/lib/delayed/backend/base.rb
CHANGED
@@ -166,17 +166,36 @@ module Delayed
|
|
166
166
|
pid_regex = pid || '(\d+)'
|
167
167
|
regex = Regexp.new("^#{Regexp.escape(name)}:#{pid_regex}$")
|
168
168
|
unlocked_jobs = 0
|
169
|
+
escaped_name = name.gsub("\\", "\\\\")
|
170
|
+
.gsub("%", "\\%")
|
171
|
+
.gsub("_", "\\_")
|
172
|
+
locked_by_like = "#{escaped_name}:%"
|
169
173
|
running = false if pid
|
170
|
-
running_jobs.
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
job.
|
174
|
+
jobs = running_jobs.limit(100)
|
175
|
+
jobs = pid ? jobs.where(locked_by: "#{name}:#{pid}") : jobs.where("locked_by LIKE ?", locked_by_like)
|
176
|
+
ignores = []
|
177
|
+
loop do
|
178
|
+
batch_scope = ignores.empty? ? jobs : jobs.where.not(id: ignores)
|
179
|
+
batch = batch_scope.to_a
|
180
|
+
break if batch.empty?
|
181
|
+
|
182
|
+
batch.each do |job|
|
183
|
+
unless job.locked_by =~ regex
|
184
|
+
ignores << job.id
|
185
|
+
next
|
186
|
+
end
|
187
|
+
|
188
|
+
unless pid
|
189
|
+
job_pid = $1.to_i
|
190
|
+
running = Process.kill(0, job_pid) rescue false
|
191
|
+
end
|
192
|
+
|
193
|
+
if running
|
194
|
+
ignores << job.id
|
195
|
+
else
|
196
|
+
unlocked_jobs += 1
|
197
|
+
job.reschedule("process died")
|
198
|
+
end
|
180
199
|
end
|
181
200
|
end
|
182
201
|
unlocked_jobs
|
data/lib/delayed/version.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "activerecord-pg-extensions"
|
4
|
+
|
3
5
|
module Delayed
|
4
6
|
module WorkQueue
|
5
7
|
class ParentProcess
|
@@ -179,7 +181,7 @@ module Delayed
|
|
179
181
|
end
|
180
182
|
|
181
183
|
jobs_to_send.each do |(recipient, job_to_send)|
|
182
|
-
@waiting_clients[worker_config].delete(
|
184
|
+
@waiting_clients[worker_config].delete(recipient)
|
183
185
|
begin
|
184
186
|
logger.debug("Sending job #{job_to_send.id} to #{recipient.name}")
|
185
187
|
client_timeout { Marshal.dump(job_to_send, recipient.socket) }
|
@@ -192,29 +194,54 @@ module Delayed
|
|
192
194
|
end
|
193
195
|
end
|
194
196
|
|
195
|
-
def
|
197
|
+
def unlock_prefetched_jobs
|
196
198
|
@prefetched_jobs.each do |(worker_config, jobs)|
|
197
199
|
next if jobs.empty?
|
198
|
-
next
|
200
|
+
next if block_given? && !yield(jobs)
|
199
201
|
|
200
|
-
Delayed::Job.
|
202
|
+
connection = Delayed::Job.connection
|
203
|
+
connection.transaction do
|
204
|
+
# make absolutely sure we don't get hung up and leave things
|
205
|
+
# locked in the database
|
206
|
+
if connection.postgresql_version >= 9_06_00 # rubocop:disable Style/NumericLiterals
|
207
|
+
connection.idle_in_transaction_session_timeout = 5
|
208
|
+
end
|
209
|
+
# relatively short timeout for acquiring the lock
|
210
|
+
connection.statement_timeout = Settings.sleep_delay
|
201
211
|
Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
|
212
|
+
|
213
|
+
# this query might take longer, and we really want to get it
|
214
|
+
# done if we got the lock, but still don't want an inadvertent
|
215
|
+
# hang
|
216
|
+
connection.statement_timeout = 30
|
202
217
|
Delayed::Job.unlock(jobs)
|
218
|
+
@prefetched_jobs[worker_config] = []
|
203
219
|
end
|
204
|
-
|
220
|
+
rescue ActiveRecord::QueryCanceled
|
221
|
+
# ignore; we'll retry anyway
|
222
|
+
logger.warn("unable to unlock prefetched jobs; skipping for now")
|
223
|
+
rescue ActiveRecord::StatementInvalid
|
224
|
+
# see if we dropped the connection
|
225
|
+
raise if connection.active?
|
226
|
+
|
227
|
+
# otherwise just reconnect and let it retry
|
228
|
+
logger.warn("failed to unlock prefetched jobs - connection terminated; skipping for now")
|
229
|
+
Delayed::Job.clear_all_connections!
|
205
230
|
end
|
206
231
|
end
|
207
232
|
|
208
|
-
def
|
209
|
-
|
210
|
-
|
233
|
+
def unlock_timed_out_prefetched_jobs
|
234
|
+
unlock_prefetched_jobs do |jobs|
|
235
|
+
jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
|
236
|
+
end
|
237
|
+
end
|
211
238
|
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
239
|
+
def unlock_all_prefetched_jobs
|
240
|
+
# we try really hard; it may not have done any work if it timed out
|
241
|
+
10.times do
|
242
|
+
unlock_prefetched_jobs
|
243
|
+
break if @prefetched_jobs.each_value.all?(&:empty?)
|
216
244
|
end
|
217
|
-
@prefetched_jobs = {}
|
218
245
|
end
|
219
246
|
|
220
247
|
def drop_socket(socket)
|
@@ -28,7 +28,7 @@ module Delayed
|
|
28
28
|
Delayed::Job.transaction do
|
29
29
|
# this action is a special case, and SHOULD NOT be a periodic job
|
30
30
|
# because if it gets wiped out suddenly during execution
|
31
|
-
# it can't go clean up
|
31
|
+
# it can't go clean up its abandoned self. Therefore,
|
32
32
|
# we expect it to get run from it's own process forked from the job pool
|
33
33
|
# and we try to get an advisory lock when it runs. If we succeed,
|
34
34
|
# no other worker is trying to do this right now (and if we abandon the
|
@@ -36,6 +36,8 @@ module Delayed
|
|
36
36
|
result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
|
37
37
|
return unless result
|
38
38
|
|
39
|
+
horizon = 5.minutes.ago
|
40
|
+
|
39
41
|
checker = Worker::HealthCheck.build(
|
40
42
|
type: Settings.worker_health_check_type,
|
41
43
|
config: Settings.worker_health_check_config,
|
@@ -43,13 +45,16 @@ module Delayed
|
|
43
45
|
)
|
44
46
|
live_workers = checker.live_workers
|
45
47
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
48
|
+
loop do
|
49
|
+
batch = Delayed::Job.running_jobs
|
50
|
+
.where("locked_at<?", horizon)
|
51
|
+
.where.not("locked_by LIKE 'prefetch:%'")
|
52
|
+
.where.not(locked_by: live_workers)
|
53
|
+
.limit(100)
|
54
|
+
.to_a
|
55
|
+
break if batch.empty?
|
51
56
|
|
52
|
-
|
57
|
+
batch.each do |job|
|
53
58
|
Delayed::Job.transaction do
|
54
59
|
# double check that the job is still there. locked_by will immediately be reset
|
55
60
|
# to nil in this transaction by Job#reschedule
|
@@ -59,9 +64,9 @@ module Delayed
|
|
59
64
|
|
60
65
|
job.reschedule
|
61
66
|
end
|
62
|
-
rescue
|
63
|
-
::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
|
64
67
|
end
|
68
|
+
rescue
|
69
|
+
::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
|
65
70
|
end
|
66
71
|
end
|
67
72
|
end
|
@@ -31,8 +31,8 @@ describe "Delayed::Backed::ActiveRecord::Job" do
|
|
31
31
|
expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
|
32
32
|
end
|
33
33
|
|
34
|
-
it "doesn't
|
35
|
-
|
34
|
+
it "doesn't allow a second worker to get exclusive access if failed to be " \
|
35
|
+
"processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
|
36
36
|
@job.update(attempts: 1, run_at: 1.day.from_now)
|
37
37
|
expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
|
38
38
|
end
|
data/spec/delayed/daemon_spec.rb
CHANGED
@@ -3,9 +3,10 @@
|
|
3
3
|
require "spec_helper"
|
4
4
|
|
5
5
|
RSpec.describe Delayed::Daemon do
|
6
|
+
subject { described_class.new(pid_folder) }
|
7
|
+
|
6
8
|
let(:pid_folder) { "/test/pid/folder" }
|
7
9
|
let(:pid) { 9999 }
|
8
|
-
let(:subject) { described_class.new(pid_folder) }
|
9
10
|
|
10
11
|
before do
|
11
12
|
allow(subject).to receive(:pid).and_return(pid)
|
@@ -11,7 +11,6 @@ RSpec.describe Delayed::WorkQueue::InProcess do
|
|
11
11
|
Delayed::Worker.lifecycle.reset!
|
12
12
|
end
|
13
13
|
|
14
|
-
let(:subject) { described_class.new }
|
15
14
|
let(:worker_config) { { queue: "test", min_priority: 1, max_priority: 2 } }
|
16
15
|
let(:args) { ["worker_name", worker_config] }
|
17
16
|
|
@@ -3,7 +3,8 @@
|
|
3
3
|
require "spec_helper"
|
4
4
|
|
5
5
|
RSpec.describe Delayed::WorkQueue::ParentProcess::Client do
|
6
|
-
|
6
|
+
subject { described_class.new(addrinfo).tap(&:init) }
|
7
|
+
|
7
8
|
let(:addrinfo) { double("Addrinfo") }
|
8
9
|
let(:connection) { double("Socket") }
|
9
10
|
let(:job) { Delayed::Job.new(locked_by: "worker_name") }
|
@@ -15,14 +15,19 @@ class JobClass
|
|
15
15
|
end
|
16
16
|
|
17
17
|
RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
|
18
|
+
subject { described_class.new(listen_socket) }
|
19
|
+
|
18
20
|
let(:parent) { Delayed::WorkQueue::ParentProcess.new }
|
19
|
-
let(:subject) { described_class.new(listen_socket) }
|
20
21
|
let(:listen_socket) { Socket.unix_server_socket(parent.server_address) }
|
21
22
|
let(:job) { JobClass.new }
|
22
23
|
let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }
|
23
24
|
let(:args) { ["worker_name", worker_config] }
|
24
25
|
let(:job_args) { [["worker_name"], "queue_name", 1, 2, hash_including(prefetch: 4)] }
|
25
26
|
|
27
|
+
before do
|
28
|
+
Delayed::Worker.lifecycle.reset!
|
29
|
+
end
|
30
|
+
|
26
31
|
before :all do
|
27
32
|
Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
|
28
33
|
Delayed::Settings.parent_process = {
|
@@ -36,6 +41,7 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
|
|
36
41
|
|
37
42
|
after do
|
38
43
|
File.unlink("/tmp/inst-jobs-test.sock") if File.exist?("/tmp/inst-jobs-test.sock")
|
44
|
+
Delayed::Worker.lifecycle.reset!
|
39
45
|
end
|
40
46
|
|
41
47
|
it "accepts new clients" do
|
@@ -98,6 +104,53 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
|
|
98
104
|
expect(Marshal.load(client)).to eq(job2)
|
99
105
|
end
|
100
106
|
|
107
|
+
context "prefetched job unlocking" do
|
108
|
+
let(:job_args) do
|
109
|
+
[["worker_name1"], "queue_name", 1, 2,
|
110
|
+
{ prefetch: 4, prefetch_owner: "prefetch:work_queue:X", forced_latency: 6.0 }]
|
111
|
+
end
|
112
|
+
let(:job2) { Delayed::Job.new(tag: "tag").tap { |j| j.create_and_lock!("prefetch:work_queue:X") } }
|
113
|
+
let(:job3) { Delayed::Job.new(tag: "tag").tap { |j| j.create_and_lock!("prefetch:work_queue:X") } }
|
114
|
+
|
115
|
+
before do
|
116
|
+
client = Socket.unix(subject.listen_socket.local_address.unix_path)
|
117
|
+
subject.run_once
|
118
|
+
|
119
|
+
jobs = { "worker_name1" => job, "prefetch:work_queue:X" => [job2, job3] }
|
120
|
+
allow(subject).to receive(:prefetch_owner).and_return("prefetch:work_queue:X")
|
121
|
+
allow(Delayed::Job).to receive(:get_and_lock_next_available).once.with(*job_args).and_return(jobs)
|
122
|
+
Marshal.dump(["worker_name1", worker_config], client)
|
123
|
+
subject.run_once
|
124
|
+
end
|
125
|
+
|
126
|
+
it "doesn't unlock anything if nothing is timed out" do
|
127
|
+
expect(Delayed::Job).not_to receive(:advisory_lock)
|
128
|
+
expect(Delayed::Job).not_to receive(:unlock)
|
129
|
+
subject.unlock_timed_out_prefetched_jobs
|
130
|
+
end
|
131
|
+
|
132
|
+
it "unlocks timed out prefetched jobs" do
|
133
|
+
allow(Delayed::Settings).to receive(:parent_process).and_return(prefetched_jobs_timeout: -1)
|
134
|
+
expect(Delayed::Job).to receive(:unlock).with([job2, job3])
|
135
|
+
subject.unlock_timed_out_prefetched_jobs
|
136
|
+
expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 0
|
137
|
+
end
|
138
|
+
|
139
|
+
it "fails gracefully if the lock times out" do
|
140
|
+
allow(Delayed::Settings).to receive(:parent_process).and_return(prefetched_jobs_timeout: -1)
|
141
|
+
expect(Delayed::Job).not_to receive(:unlock)
|
142
|
+
expect(Delayed::Job).to receive(:advisory_lock).and_raise(ActiveRecord::QueryCanceled)
|
143
|
+
subject.unlock_timed_out_prefetched_jobs
|
144
|
+
expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 2
|
145
|
+
end
|
146
|
+
|
147
|
+
it "unlocks all jobs" do
|
148
|
+
expect(Delayed::Job).to receive(:unlock).with([job2, job3])
|
149
|
+
subject.unlock_all_prefetched_jobs
|
150
|
+
expect(subject.instance_variable_get(:@prefetched_jobs).values.sum(&:length)).to eq 0
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
101
154
|
it "doesn't respond immediately if there are no jobs available" do
|
102
155
|
client = Socket.unix(subject.listen_socket.local_address.unix_path)
|
103
156
|
subject.run_once
|
@@ -204,4 +257,24 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
|
|
204
257
|
expect(Marshal.load(client)).to eq(job)
|
205
258
|
expect(called).to eq(true)
|
206
259
|
end
|
260
|
+
|
261
|
+
it "deletes the correct worker when transferring jobs" do
|
262
|
+
client1 = Socket.unix(subject.listen_socket.local_address.unix_path)
|
263
|
+
client2 = Socket.unix(subject.listen_socket.local_address.unix_path)
|
264
|
+
subject.run_once
|
265
|
+
subject.run_once
|
266
|
+
|
267
|
+
Marshal.dump(args, client1)
|
268
|
+
Marshal.dump(["worker_name2", worker_config], client2)
|
269
|
+
subject.run_once
|
270
|
+
subject.run_once
|
271
|
+
|
272
|
+
waiting_clients = subject.instance_variable_get(:@waiting_clients)
|
273
|
+
expect(waiting_clients.first.last.length).to eq 2
|
274
|
+
|
275
|
+
expect(Delayed::Job).to receive(:get_and_lock_next_available).and_return("worker_name" => job,
|
276
|
+
"worker_name2" => job)
|
277
|
+
subject.run_once
|
278
|
+
expect(waiting_clients.first.last).to be_empty
|
279
|
+
end
|
207
280
|
end
|
@@ -21,8 +21,6 @@ RSpec.describe Delayed::WorkQueue::ParentProcess do
|
|
21
21
|
Delayed::Worker.lifecycle.reset!
|
22
22
|
end
|
23
23
|
|
24
|
-
let(:subject) { described_class.new }
|
25
|
-
|
26
24
|
describe "#initalize(config = Settings.parent_process)" do
|
27
25
|
it "must expand a relative path to be within the Rails root" do
|
28
26
|
queue = described_class.new("server_address" => "tmp/foo.sock")
|
@@ -51,7 +51,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
|
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
-
let(:initial_run_at) {
|
54
|
+
let(:initial_run_at) { 10.minutes.ago }
|
55
55
|
|
56
56
|
before do
|
57
57
|
klass.live_workers = %w[alive]
|
@@ -96,7 +96,13 @@ RSpec.describe Delayed::Worker::HealthCheck do
|
|
96
96
|
|
97
97
|
it "ignores jobs that are re-locked after fetching from db" do
|
98
98
|
Delayed::Job.where(id: @dead_job).update_all(locked_by: "someone_else")
|
99
|
-
|
99
|
+
# we need to return @dead_job itself, which doesn't match the database
|
100
|
+
jobs_scope = double
|
101
|
+
allow(jobs_scope).to receive(:where).and_return(jobs_scope)
|
102
|
+
allow(jobs_scope).to receive(:not).and_return(jobs_scope)
|
103
|
+
allow(jobs_scope).to receive(:limit).and_return(jobs_scope)
|
104
|
+
allow(jobs_scope).to receive(:to_a).and_return([@dead_job], [])
|
105
|
+
allow(Delayed::Job).to receive(:running_jobs).and_return(jobs_scope)
|
100
106
|
described_class.reschedule_abandoned_jobs
|
101
107
|
@dead_job.reload
|
102
108
|
expect(@dead_job.locked_by).to eq "someone_else"
|
@@ -104,7 +110,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
|
|
104
110
|
|
105
111
|
it "ignores jobs that are prefetched" do
|
106
112
|
Delayed::Job.where(id: @dead_job).update_all(locked_by: "prefetch:some_node")
|
107
|
-
allow(Delayed::Job).to receive(:running_jobs).and_return(
|
113
|
+
allow(Delayed::Job).to receive(:running_jobs).and_return(Delayed::Job.where(id: @dead_job.id))
|
108
114
|
described_class.reschedule_abandoned_jobs
|
109
115
|
@dead_job.reload
|
110
116
|
expect(@dead_job.locked_by).to eq "prefetch:some_node"
|
data/spec/spec_helper.rb
CHANGED
@@ -58,11 +58,7 @@ connection_config = {
|
|
58
58
|
}
|
59
59
|
|
60
60
|
def migrate(file)
|
61
|
-
|
62
|
-
ActiveRecord::MigrationContext.new(file, ActiveRecord::SchemaMigration).migrate
|
63
|
-
else
|
64
|
-
ActiveRecord::MigrationContext.new(file).migrate
|
65
|
-
end
|
61
|
+
ActiveRecord::MigrationContext.new(file, ActiveRecord::SchemaMigration).migrate
|
66
62
|
end
|
67
63
|
|
68
64
|
# create the test db if it does not exist, to help out wwtd
|
@@ -73,6 +69,11 @@ rescue ActiveRecord::StatementInvalid
|
|
73
69
|
nil
|
74
70
|
end
|
75
71
|
ActiveRecord::Base.establish_connection(connection_config)
|
72
|
+
|
73
|
+
# we need to ensure this callback is called for activerecord-pg-extensions,
|
74
|
+
# which isn't running because we're not using Rails to setup the database
|
75
|
+
ActiveRecord::PGExtensions::Railtie.run_initializers
|
76
|
+
|
76
77
|
# TODO: reset db and migrate again, to test migrations
|
77
78
|
|
78
79
|
migrate("db/migrate")
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: inst-jobs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tobias Luetke
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-09-
|
12
|
+
date: 2021-09-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activerecord
|
@@ -17,28 +17,42 @@ dependencies:
|
|
17
17
|
requirements:
|
18
18
|
- - ">="
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: '
|
20
|
+
version: '6.0'
|
21
21
|
type: :runtime
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - ">="
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: '
|
27
|
+
version: '6.0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: activerecord-pg-extensions
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '0.4'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0.4'
|
28
42
|
- !ruby/object:Gem::Dependency
|
29
43
|
name: activesupport
|
30
44
|
requirement: !ruby/object:Gem::Requirement
|
31
45
|
requirements:
|
32
46
|
- - ">="
|
33
47
|
- !ruby/object:Gem::Version
|
34
|
-
version: '
|
48
|
+
version: '6.0'
|
35
49
|
type: :runtime
|
36
50
|
prerelease: false
|
37
51
|
version_requirements: !ruby/object:Gem::Requirement
|
38
52
|
requirements:
|
39
53
|
- - ">="
|
40
54
|
- !ruby/object:Gem::Version
|
41
|
-
version: '
|
55
|
+
version: '6.0'
|
42
56
|
- !ruby/object:Gem::Dependency
|
43
57
|
name: after_transaction_commit
|
44
58
|
requirement: !ruby/object:Gem::Requirement
|
@@ -93,14 +107,14 @@ dependencies:
|
|
93
107
|
requirements:
|
94
108
|
- - ">="
|
95
109
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
110
|
+
version: '6.0'
|
97
111
|
type: :runtime
|
98
112
|
prerelease: false
|
99
113
|
version_requirements: !ruby/object:Gem::Requirement
|
100
114
|
requirements:
|
101
115
|
- - ">="
|
102
116
|
- !ruby/object:Gem::Version
|
103
|
-
version: '
|
117
|
+
version: '6.0'
|
104
118
|
- !ruby/object:Gem::Dependency
|
105
119
|
name: appraisal
|
106
120
|
requirement: !ruby/object:Gem::Requirement
|
@@ -432,6 +446,8 @@ files:
|
|
432
446
|
- db/migrate/20210809145804_add_n_strand_index.rb
|
433
447
|
- db/migrate/20210812210128_add_singleton_column.rb
|
434
448
|
- db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
|
449
|
+
- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
|
450
|
+
- db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
|
435
451
|
- exe/inst_jobs
|
436
452
|
- lib/delayed/backend/active_record.rb
|
437
453
|
- lib/delayed/backend/base.rb
|