solid_queue 0.6.0 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/models/solid_queue/claimed_execution.rb +5 -4
- data/app/models/solid_queue/process/executor.rb +7 -1
- data/app/models/solid_queue/process/prunable.rb +1 -7
- data/app/models/solid_queue/process.rb +10 -1
- data/db/migrate/20240813160053_make_name_not_null.rb +1 -1
- data/lib/solid_queue/configuration.rb +16 -6
- data/lib/solid_queue/processes/poller.rb +6 -4
- data/lib/solid_queue/processes/process_exit_error.rb +17 -0
- data/lib/solid_queue/processes/process_missing_error.rb +9 -0
- data/lib/solid_queue/processes/process_pruned_error.rb +11 -0
- data/lib/solid_queue/supervisor/fork_supervisor.rb +1 -13
- data/lib/solid_queue/supervisor/maintenance.rb +1 -7
- data/lib/solid_queue/supervisor.rb +6 -2
- data/lib/solid_queue/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c2734f6cbcc795345207dc0cf221bd2744f2f98449da8b0edfc8175f3bbd7869
|
4
|
+
data.tar.gz: c4b4c2eca7dcb93e86a2f69d220ba1b9f8817c1d52a7cd1ff137ed5ec84e50f8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9d09b58e43c4bc19ad2ab89d10072b36b53a449e9602f5c5a8fc7b637e91d8472eb70f417148657e6b146e5cd1034025afdbe6d39fa2b9fc82ce6bba3997e57f
|
7
|
+
data.tar.gz: 27168fd2216fbca4e9b19677a7885b23663012d8aed7bf54c25be8335bd528a733b55790e662a30815b9ba24d824390372eff24ed0b9f72947b6ff4feeec17c5
|
@@ -29,8 +29,9 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
|
|
29
29
|
def release_all
|
30
30
|
SolidQueue.instrument(:release_many_claimed) do |payload|
|
31
31
|
includes(:job).tap do |executions|
|
32
|
-
payload[:size] = executions.size
|
33
32
|
executions.each(&:release)
|
33
|
+
|
34
|
+
payload[:size] = executions.size
|
34
35
|
end
|
35
36
|
end
|
36
37
|
end
|
@@ -38,11 +39,11 @@ class SolidQueue::ClaimedExecution < SolidQueue::Execution
|
|
38
39
|
def fail_all_with(error)
|
39
40
|
SolidQueue.instrument(:fail_many_claimed) do |payload|
|
40
41
|
includes(:job).tap do |executions|
|
41
|
-
|
42
|
+
executions.each { |execution| execution.failed_with(error) }
|
43
|
+
|
42
44
|
payload[:process_ids] = executions.map(&:process_id).uniq
|
43
45
|
payload[:job_ids] = executions.map(&:job_id).uniq
|
44
|
-
|
45
|
-
executions.each { |execution| execution.failed_with(error) }
|
46
|
+
payload[:size] = executions.size
|
46
47
|
end
|
47
48
|
end
|
48
49
|
end
|
@@ -8,7 +8,7 @@ module SolidQueue
|
|
8
8
|
included do
|
9
9
|
has_many :claimed_executions
|
10
10
|
|
11
|
-
after_destroy
|
11
|
+
after_destroy :release_all_claimed_executions
|
12
12
|
end
|
13
13
|
|
14
14
|
def fail_all_claimed_executions_with(error)
|
@@ -17,6 +17,12 @@ module SolidQueue
|
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
|
+
def release_all_claimed_executions
|
21
|
+
if claims_executions?
|
22
|
+
claimed_executions.release_all
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
20
26
|
private
|
21
27
|
def claims_executions?
|
22
28
|
kind == "Worker"
|
@@ -1,12 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module SolidQueue
|
4
|
-
class ProcessPrunedError < RuntimeError
|
5
|
-
def initialize(last_heartbeat_at)
|
6
|
-
super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at}")
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
4
|
class Process
|
11
5
|
module Prunable
|
12
6
|
extend ActiveSupport::Concern
|
@@ -28,7 +22,7 @@ module SolidQueue
|
|
28
22
|
end
|
29
23
|
|
30
24
|
def prune
|
31
|
-
error = ProcessPrunedError.new(last_heartbeat_at)
|
25
|
+
error = Processes::ProcessPrunedError.new(last_heartbeat_at)
|
32
26
|
fail_all_claimed_executions_with(error)
|
33
27
|
|
34
28
|
deregister(pruned: true)
|
@@ -4,7 +4,7 @@ class SolidQueue::Process < SolidQueue::Record
|
|
4
4
|
include Executor, Prunable
|
5
5
|
|
6
6
|
belongs_to :supervisor, class_name: "SolidQueue::Process", optional: true, inverse_of: :supervisees
|
7
|
-
has_many :supervisees, class_name: "SolidQueue::Process", inverse_of: :supervisor, foreign_key: :supervisor_id
|
7
|
+
has_many :supervisees, class_name: "SolidQueue::Process", inverse_of: :supervisor, foreign_key: :supervisor_id
|
8
8
|
|
9
9
|
store :metadata, coder: JSON
|
10
10
|
|
@@ -26,9 +26,18 @@ class SolidQueue::Process < SolidQueue::Record
|
|
26
26
|
def deregister(pruned: false)
|
27
27
|
SolidQueue.instrument :deregister_process, process: self, pruned: pruned do |payload|
|
28
28
|
destroy!
|
29
|
+
|
30
|
+
unless supervised? || pruned
|
31
|
+
supervisees.each(&:deregister)
|
32
|
+
end
|
29
33
|
rescue Exception => error
|
30
34
|
payload[:error] = error
|
31
35
|
raise
|
32
36
|
end
|
33
37
|
end
|
38
|
+
|
39
|
+
private
|
40
|
+
def supervised?
|
41
|
+
supervisor_id.present?
|
42
|
+
end
|
34
43
|
end
|
@@ -11,6 +11,6 @@ class MakeNameNotNull < ActiveRecord::Migration[7.1]
|
|
11
11
|
|
12
12
|
def down
|
13
13
|
remove_index :solid_queue_processes, [ :name, :supervisor_id ]
|
14
|
-
change_column :solid_queue_processes, :name, :string, null:
|
14
|
+
change_column :solid_queue_processes, :name, :string, null: true
|
15
15
|
end
|
16
16
|
end
|
@@ -23,6 +23,11 @@ module SolidQueue
|
|
23
23
|
recurring_tasks: []
|
24
24
|
}
|
25
25
|
|
26
|
+
DEFAULT_CONFIG = {
|
27
|
+
workers: [ WORKER_DEFAULTS ],
|
28
|
+
dispatchers: [ DISPATCHER_DEFAULTS ]
|
29
|
+
}
|
30
|
+
|
26
31
|
def initialize(mode: :fork, load_from: nil)
|
27
32
|
@mode = mode.to_s.inquiry
|
28
33
|
@raw_config = config_from(load_from)
|
@@ -61,22 +66,27 @@ module SolidQueue
|
|
61
66
|
end
|
62
67
|
|
63
68
|
def config_from(file_or_hash, env: Rails.env)
|
64
|
-
|
65
|
-
|
69
|
+
load_config_from(file_or_hash).then do |config|
|
70
|
+
config = config[env.to_sym] ? config[env.to_sym] : config
|
71
|
+
if (config.keys & DEFAULT_CONFIG.keys).any? then config
|
72
|
+
else
|
73
|
+
DEFAULT_CONFIG
|
74
|
+
end
|
75
|
+
end
|
66
76
|
end
|
67
77
|
|
68
78
|
def workers_options
|
69
|
-
@workers_options ||= options_from_raw_config(:workers
|
79
|
+
@workers_options ||= options_from_raw_config(:workers)
|
70
80
|
.map { |options| options.dup.symbolize_keys }
|
71
81
|
end
|
72
82
|
|
73
83
|
def dispatchers_options
|
74
|
-
@dispatchers_options ||= options_from_raw_config(:dispatchers
|
84
|
+
@dispatchers_options ||= options_from_raw_config(:dispatchers)
|
75
85
|
.map { |options| options.dup.symbolize_keys }
|
76
86
|
end
|
77
87
|
|
78
|
-
def options_from_raw_config(key
|
79
|
-
|
88
|
+
def options_from_raw_config(key)
|
89
|
+
Array(raw_config[key])
|
80
90
|
end
|
81
91
|
|
82
92
|
def parse_recurring_tasks(tasks)
|
@@ -45,10 +45,12 @@ module SolidQueue::Processes
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def with_polling_volume
|
48
|
-
|
49
|
-
ActiveRecord::Base.logger
|
50
|
-
|
51
|
-
|
48
|
+
SolidQueue.instrument(:polling) do
|
49
|
+
if SolidQueue.silence_polling? && ActiveRecord::Base.logger
|
50
|
+
ActiveRecord::Base.logger.silence { yield }
|
51
|
+
else
|
52
|
+
yield
|
53
|
+
end
|
52
54
|
end
|
53
55
|
end
|
54
56
|
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue
|
4
|
+
module Processes
|
5
|
+
class ProcessExitError < RuntimeError
|
6
|
+
def initialize(status)
|
7
|
+
message = case
|
8
|
+
when status.exitstatus.present? then "Process pid=#{status.pid} exited with status #{status. exitstatus}"
|
9
|
+
when status.signaled? then "Process pid=#{status.pid} received unhandled signal #{status. termsig}"
|
10
|
+
else "Process pid=#{status.pid} exited unexpectedly"
|
11
|
+
end
|
12
|
+
|
13
|
+
super(message)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module SolidQueue
|
4
|
+
module Processes
|
5
|
+
class ProcessPrunedError < RuntimeError
|
6
|
+
def initialize(last_heartbeat_at)
|
7
|
+
super("Process was found dead and pruned (last heartbeat at: #{last_heartbeat_at}")
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -1,18 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module SolidQueue
|
4
|
-
class ProcessExitError < RuntimeError
|
5
|
-
def initialize(status)
|
6
|
-
message = case
|
7
|
-
when status.exitstatus.present? then "Process pid=#{status.pid} exited with status #{status.exitstatus}"
|
8
|
-
when status.signaled? then "Process pid=#{status.pid} received unhandled signal #{status.termsig}"
|
9
|
-
else "Process pid=#{status.pid} exited unexpectedly"
|
10
|
-
end
|
11
|
-
|
12
|
-
super(message)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
4
|
class Supervisor::ForkSupervisor < Supervisor
|
17
5
|
include Signals, Pidfiled
|
18
6
|
|
@@ -126,7 +114,7 @@ module SolidQueue
|
|
126
114
|
|
127
115
|
def handle_claimed_jobs_by(terminated_fork, status)
|
128
116
|
if registered_process = process.supervisees.find_by(name: terminated_fork.name)
|
129
|
-
error = ProcessExitError.new(status)
|
117
|
+
error = Processes::ProcessExitError.new(status)
|
130
118
|
registered_process.fail_all_claimed_executions_with(error)
|
131
119
|
end
|
132
120
|
end
|
@@ -1,10 +1,4 @@
|
|
1
1
|
module SolidQueue
|
2
|
-
class ProcessMissingError < RuntimeError
|
3
|
-
def initialize
|
4
|
-
super("The process that was running this job no longer exists")
|
5
|
-
end
|
6
|
-
end
|
7
|
-
|
8
2
|
module Supervisor::Maintenance
|
9
3
|
extend ActiveSupport::Concern
|
10
4
|
|
@@ -35,7 +29,7 @@ module SolidQueue
|
|
35
29
|
|
36
30
|
def fail_orphaned_executions
|
37
31
|
wrap_in_app_executor do
|
38
|
-
|
32
|
+
ClaimedExecution.orphaned.fail_all_with(Processes::ProcessMissingError.new)
|
39
33
|
end
|
40
34
|
end
|
41
35
|
end
|
@@ -9,8 +9,12 @@ module SolidQueue
|
|
9
9
|
SolidQueue.supervisor = true
|
10
10
|
configuration = Configuration.new(mode: mode, load_from: load_configuration_from)
|
11
11
|
|
12
|
-
|
13
|
-
|
12
|
+
if configuration.configured_processes.any?
|
13
|
+
klass = mode == :fork ? ForkSupervisor : AsyncSupervisor
|
14
|
+
klass.new(configuration).tap(&:start)
|
15
|
+
else
|
16
|
+
abort "No workers or processed configured. Exiting..."
|
17
|
+
end
|
14
18
|
end
|
15
19
|
end
|
16
20
|
|
data/lib/solid_queue/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: solid_queue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rosa Gutierrez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-08-
|
11
|
+
date: 2024-08-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -242,6 +242,9 @@ files:
|
|
242
242
|
- lib/solid_queue/processes/callbacks.rb
|
243
243
|
- lib/solid_queue/processes/interruptible.rb
|
244
244
|
- lib/solid_queue/processes/poller.rb
|
245
|
+
- lib/solid_queue/processes/process_exit_error.rb
|
246
|
+
- lib/solid_queue/processes/process_missing_error.rb
|
247
|
+
- lib/solid_queue/processes/process_pruned_error.rb
|
245
248
|
- lib/solid_queue/processes/procline.rb
|
246
249
|
- lib/solid_queue/processes/registrable.rb
|
247
250
|
- lib/solid_queue/processes/runnable.rb
|